处理后台新需求
Build Backend JAR / build (push) Failing after 16m58s

This commit is contained in:
supernijia
2026-08-09 20:08:39 +08:00
parent f7e26ee266
commit 38216a1885
42 changed files with 3425 additions and 424 deletions
@@ -12,7 +12,7 @@ public class ModuleCleanupProperties {
private boolean enabled = true;
private String cron = "0 0 0 * * *";
private long retentionDays = 7;
// SHOP_DATA_CRAWL is governed by per-shop latest-three retention in its
// task service and must not be removed by the age-based sweep.
// SHOP_DATA_CRAWL keeps one per-shop daily workbook in its task service
// and must not be removed by the age-based sweep.
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
}
@@ -805,6 +805,7 @@ public class CollectDataService {
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
entity.setDataValue(row.getAsin());
entity.setBrand(brand);
entity.setRecordSource("AUTO");
try {
invalidAsinDataMapper.insert(entity);
} catch (DuplicateKeyException ignored) {
@@ -1,6 +1,8 @@
package com.nanri.aiimage.modules.dedupe.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
@@ -10,6 +12,8 @@ import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
@@ -17,6 +21,8 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.beans.factory.annotation.Value;
import lombok.RequiredArgsConstructor;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.http.HttpHeaders;
@@ -36,6 +42,10 @@ import org.springframework.web.multipart.MultipartFile;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
@RestController
@RequiredArgsConstructor
@@ -43,9 +53,20 @@ import java.time.format.DateTimeFormatter;
@Tag(name = "数据去重总数据", description = "维护数据去重模块的总数据列表,支持增删改查。")
public class DedupeTotalDataController {
private static final String DEDUPE_TOTAL_DATA_COLUMN_KEY = "admin_dedupe_total_data";
private static final String DEDUPE_TOTAL_DATA_ROUTE_PATH = "dedupe-total-data";
private static final DateTimeFormatter EXPORT_FILENAME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
private final DedupeTotalDataService dedupeTotalDataService;
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
@Value("${aiimage.security.internal-token:}")
private String internalToken;
@Value("${aiimage.security.internal-token-file:}")
private String internalTokenFile;
@GetMapping
@Operation(summary = "分页查询总数据", description = "分页查询数据去重总数据,支持按值模糊搜索。")
@@ -61,9 +82,11 @@ public class DedupeTotalDataController {
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
@Parameter(description = "结束日期(包含)")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success(dedupeTotalDataService.page(
page, pageSize, keyword, username, startDate, endDate, operatorId));
page, pageSize, keyword, username, startDate, endDate, groupId, operator.id()));
}
@GetMapping("/export")
@@ -74,8 +97,10 @@ public class DedupeTotalDataController {
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
@Parameter(description = "结束日期(包含)")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
byte[] bytes = dedupeTotalDataService.export(username, startDate, endDate, operatorId);
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
byte[] bytes = dedupeTotalDataService.export(username, startDate, endDate, groupId, operator.id());
String filename = "dedupe-total-data-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
@@ -91,9 +116,10 @@ public class DedupeTotalDataController {
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复")
})
public ApiResponse<DedupeTotalDataItemVo> create(
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId,
HttpServletRequest httpRequest,
@Valid @RequestBody DedupeTotalDataCreateRequest request) {
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operatorId));
RequestOperator operator = requireDedupeTotalDataAccess(httpRequest);
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operator.id()));
}
@PostMapping("/import")
@@ -104,16 +130,19 @@ public class DedupeTotalDataController {
})
public ApiResponse<DedupeTotalDataImportStartVo> importExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, operatorId));
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, groupId, operator.id()));
}
@GetMapping("/import/{importId}")
@Operation(summary = "查询导入进度", description = "根据导入任务 ID 查询当前进度。")
public ApiResponse<DedupeTotalDataImportProgressVo> importProgress(
@PathVariable String importId,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
return ApiResponse.success(dedupeTotalDataService.getImportProgress(importId, operatorId));
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success(dedupeTotalDataService.getImportProgress(importId, operator.id()));
}
@PostMapping("/delete-import")
@@ -124,16 +153,19 @@ public class DedupeTotalDataController {
})
public ApiResponse<DedupeTotalDataImportStartVo> deleteImportExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, operatorId));
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, groupId, operator.id()));
}
@GetMapping("/delete-import/{importId}")
@Operation(summary = "查询删除导入进度", description = "根据删除导入任务 ID 查询当前进度。")
public ApiResponse<DedupeTotalDataImportProgressVo> deleteImportProgress(
@PathVariable String importId,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
return ApiResponse.success(dedupeTotalDataService.getDeleteImportProgress(importId, operatorId));
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success(dedupeTotalDataService.getDeleteImportProgress(importId, operator.id()));
}
@PutMapping("/{id}")
@@ -145,9 +177,10 @@ public class DedupeTotalDataController {
})
public ApiResponse<DedupeTotalDataItemVo> update(
@Parameter(description = "主键ID", required = true) @PathVariable Long id,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId,
HttpServletRequest httpRequest,
@Valid @RequestBody DedupeTotalDataUpdateRequest request) {
return ApiResponse.success("更新成功", dedupeTotalDataService.update(id, request, operatorId));
RequestOperator operator = requireDedupeTotalDataAccess(httpRequest);
return ApiResponse.success("更新成功", dedupeTotalDataService.update(id, request, operator.id()));
}
@DeleteMapping("/{id}")
@@ -158,8 +191,105 @@ public class DedupeTotalDataController {
})
public ApiResponse<Void> delete(
@Parameter(description = "主键ID", required = true) @PathVariable Long id,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
dedupeTotalDataService.delete(id, operatorId);
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
dedupeTotalDataService.delete(id, operator.id());
return ApiResponse.success("删除成功", null);
}
private RequestOperator requireDedupeTotalDataAccess(HttpServletRequest request) {
AdminUserEntity operator = resolveOperator(request);
if (operator == null || operator.getId() == null || operator.getId() <= 0) {
throw new BusinessException(403, "无权访问数据去重总数据");
}
boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(operator));
if (!superAdmin && !hasDedupeTotalDataPermission(operator)) {
throw new BusinessException(403, "无权访问数据去重总数据");
}
return new RequestOperator(operator.getId(), superAdmin);
}
private AdminUserEntity resolveOperator(HttpServletRequest request) {
try {
return adminAuthSupport.requireUser(request);
} catch (BusinessException authFailure) {
AdminUserEntity internalOperator = resolveInternalOperator(request);
if (internalOperator != null) {
return internalOperator;
}
throw authFailure;
}
}
private boolean hasDedupeTotalDataPermission(AdminUserEntity operator) {
return permissionMenuService.getUserColumnPermissions(operator.getId(), "admin")
.stream()
.anyMatch(item -> DEDUPE_TOTAL_DATA_COLUMN_KEY.equals(item.getColumnKey())
|| DEDUPE_TOTAL_DATA_ROUTE_PATH.equals(item.getRoutePath()));
}
private AdminUserEntity resolveInternalOperator(HttpServletRequest request) {
if (!isTrustedInternalRequest(request.getHeader("X-Internal-Token"))) {
return null;
}
String rawOperatorId = request.getParameter("operatorId");
if (rawOperatorId == null || rawOperatorId.isBlank()) {
rawOperatorId = request.getParameter("operator_id");
}
if (rawOperatorId == null || rawOperatorId.isBlank()) {
return null;
}
try {
return permissionMenuService.requireUserOperator(Long.parseLong(rawOperatorId.trim()));
} catch (NumberFormatException | BusinessException ex) {
return null;
}
}
private boolean isTrustedInternalRequest(String suppliedToken) {
String expectedToken = resolveExpectedInternalToken();
return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank()
&& MessageDigest.isEqual(
expectedToken.getBytes(StandardCharsets.UTF_8),
suppliedToken.trim().getBytes(StandardCharsets.UTF_8));
}
private String resolveExpectedInternalToken() {
if (internalToken != null && !internalToken.isBlank()) {
return internalToken.trim();
}
Path path = resolveInternalTokenFile();
if (path == null || !Files.isRegularFile(path)) {
return "";
}
try {
return Files.readString(path, StandardCharsets.UTF_8).trim();
} catch (Exception ignored) {
return "";
}
}
private Path resolveInternalTokenFile() {
String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim();
if (!configuredPath.isEmpty()) {
if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) {
String userHome = System.getProperty("user.home", "").trim();
if (userHome.isEmpty()) {
return null;
}
configuredPath = configuredPath.length() == 1
? userHome
: Path.of(userHome, configuredPath.substring(2)).toString();
}
Path configuredTokenPath = Path.of(configuredPath);
return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null;
}
String userHome = System.getProperty("user.home", "").trim();
return userHome.isEmpty()
? null
: Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize();
}
private record RequestOperator(Long id, boolean superAdmin) {
}
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.dedupe.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
@@ -13,4 +14,8 @@ public class DedupeTotalDataCreateRequest {
@Size(max = 128, message = "数据长度不能超过128个字符")
@Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED)
private String dataValue;
@NotNull(message = "请选择分组")
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
private Long groupId;
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.dedupe.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
@@ -13,4 +14,8 @@ public class DedupeTotalDataUpdateRequest {
@Size(max = 128, message = "数据长度不能超过128个字符")
@Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED)
private String dataValue;
@NotNull(message = "请选择分组")
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
private Long groupId;
}
@@ -14,6 +14,7 @@ public class DedupeTotalDataEntity {
@TableId(type = IdType.AUTO)
private Long id;
private String dataValue;
private Long groupId;
private Long uploaderUserId;
private String uploaderUsername;
private LocalDateTime createdAt;
@@ -15,6 +15,12 @@ public class DedupeTotalDataItemVo {
@Schema(description = "总数据值")
private String dataValue;
@Schema(description = "分组 ID")
private Long groupId;
@Schema(description = "分组名称")
private String groupName;
@Schema(description = "上传用户ID")
private Long uploaderUserId;
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
import lombok.RequiredArgsConstructor;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
@@ -42,6 +43,7 @@ import java.time.format.DateTimeFormatter;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
@@ -65,6 +67,8 @@ public class DedupeTotalDataService {
private final Map<String, DedupeTotalDataImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
private final Map<String, Long> importOwnerMap = new ConcurrentHashMap<>();
private final Map<String, Long> deleteImportOwnerMap = new ConcurrentHashMap<>();
private final Map<String, Long> importGroupMap = new ConcurrentHashMap<>();
private final Map<String, Long> deleteImportGroupMap = new ConcurrentHashMap<>();
private final Map<String, Long> importCompletedAtMap = new ConcurrentHashMap<>();
private final Map<String, Long> deleteImportCompletedAtMap = new ConcurrentHashMap<>();
@@ -76,6 +80,11 @@ public class DedupeTotalDataService {
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
LocalDate startDate, LocalDate endDate, Long operatorId) {
return page(page, pageSize, keyword, username, startDate, endDate, null, operatorId);
}
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
LocalDate startDate, LocalDate endDate, Long groupId, Long operatorId) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("开始日期不能晚于结束日期");
}
@@ -86,17 +95,21 @@ public class DedupeTotalDataService {
AccessScope scope = resolveAccessScope(operatorId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
.in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds())
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
.orderByDesc(DedupeTotalDataEntity::getId);
applyGroupScope(query, scope, groupId);
Long total = dedupeTotalDataMapper.selectCount(query);
List<DedupeTotalDataItemVo> items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
.stream()
.map(this::toItemVo)
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(
query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize));
Map<Long, String> groupNames = loadGroupNames(rows);
List<DedupeTotalDataItemVo> items = rows.stream()
.map(row -> toItemVo(row, row.getGroupId() == null
? ""
: groupNames.getOrDefault(row.getGroupId(), "")))
.toList();
DedupeTotalDataPageVo vo = new DedupeTotalDataPageVo();
vo.setItems(items);
@@ -107,24 +120,29 @@ public class DedupeTotalDataService {
}
public byte[] export(String username, LocalDate startDate, LocalDate endDate, Long operatorId) {
return export(username, startDate, endDate, null, operatorId);
}
public byte[] export(String username, LocalDate startDate, LocalDate endDate,
Long groupId, Long operatorId) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("开始日期不能晚于结束日期");
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds())
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
.orderByDesc(DedupeTotalDataEntity::getId);
applyGroupScope(query, scope, groupId);
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(query);
return buildExportWorkbook(rows);
return buildExportWorkbook(rows, loadGroupNames(rows));
}
private byte[] buildExportWorkbook(List<DedupeTotalDataEntity> rows) {
private byte[] buildExportWorkbook(List<DedupeTotalDataEntity> rows, Map<Long, String> groupNames) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
Sheet sheet = workbook.createSheet("DedupeTotalData");
@@ -132,19 +150,24 @@ public class DedupeTotalDataService {
header.createCell(0).setCellValue("ID");
header.createCell(1).setCellValue("ASIN值");
header.createCell(2).setCellValue("用户名");
header.createCell(3).setCellValue("创建时间");
header.createCell(3).setCellValue("分组");
header.createCell(4).setCellValue("创建时间");
for (int index = 0; index < rows.size(); index++) {
DedupeTotalDataEntity entity = rows.get(index);
Row row = sheet.createRow(index + 1);
row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId()));
row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue());
row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
row.createCell(3).setCellValue(formatExportTime(entity.getCreatedAt()));
row.createCell(3).setCellValue(entity.getGroupId() == null
? ""
: groupNames.getOrDefault(entity.getGroupId(), ""));
row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt()));
}
sheet.setColumnWidth(0, 3600);
sheet.setColumnWidth(1, 5200);
sheet.setColumnWidth(2, 5200);
sheet.setColumnWidth(3, 5600);
sheet.setColumnWidth(3, 5200);
sheet.setColumnWidth(4, 5600);
workbook.write(outputStream);
workbook.dispose();
return outputStream.toByteArray();
@@ -160,14 +183,16 @@ public class DedupeTotalDataService {
@Transactional
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long operatorId) {
AdminUserEntity uploader = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), uploader);
String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, null);
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(dataValue);
entity.setGroupId(group.getId());
entity.setUploaderUserId(uploader.getId());
entity.setUploaderUsername(uploader.getUsername());
dedupeTotalDataMapper.insert(entity);
return toItemVo(getById(entity.getId()));
return toItemVo(getById(entity.getId()), group.getGroupName());
}
public Set<String> listComparableValues() {
@@ -204,11 +229,16 @@ public class DedupeTotalDataService {
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long operatorId) {
return startImport(file, null, operatorId);
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long operatorId) {
cleanupExpiredProgress();
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity uploader = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
String importId = IdUtil.fastSimpleUUID();
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("pending");
@@ -219,15 +249,17 @@ public class DedupeTotalDataService {
progress.setSkippedCount(0);
importProgressMap.put(importId, progress);
importOwnerMap.put(importId, uploader.getId());
importGroupMap.put(importId, group.getId());
try {
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runImportTask(
importId, tempFile, filename, uploader.getId(), uploader.getUsername()));
importId, tempFile, filename, uploader.getId(), uploader.getUsername(), group.getId()));
} catch (Exception e) {
importProgressMap.remove(importId);
importOwnerMap.remove(importId);
importGroupMap.remove(importId);
throw new BusinessException("读取上传文件失败");
}
@@ -240,19 +272,25 @@ public class DedupeTotalDataService {
cleanupExpiredProgress();
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
Long ownerId = importOwnerMap.get(importId);
if (progress == null || ownerId == null) {
Long groupId = importGroupMap.get(importId);
if (progress == null || ownerId == null || groupId == null) {
throw new BusinessException("导入任务不存在");
}
ensureUploaderAccess(ownerId, resolveAccessScope(operatorId));
ensureImportTaskAccess(groupId, resolveAccessScope(operatorId));
return progress;
}
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long operatorId) {
return startDeleteImport(file, null, operatorId);
}
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long operatorId) {
cleanupExpiredProgress();
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity operator = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
String importId = IdUtil.fastSimpleUUID();
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("pending");
@@ -263,14 +301,17 @@ public class DedupeTotalDataService {
progress.setSkippedCount(0);
deleteImportProgressMap.put(importId, progress);
deleteImportOwnerMap.put(importId, operator.getId());
deleteImportGroupMap.put(importId, group.getId());
try {
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, tempFile, filename, operator.getId()));
Thread.ofVirtual().start(() -> runDeleteImportTask(
importId, tempFile, filename, operator.getId(), group.getId()));
} catch (Exception e) {
deleteImportProgressMap.remove(importId);
deleteImportOwnerMap.remove(importId);
deleteImportGroupMap.remove(importId);
throw new BusinessException("读取上传文件失败");
}
@@ -283,14 +324,16 @@ public class DedupeTotalDataService {
cleanupExpiredProgress();
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
Long ownerId = deleteImportOwnerMap.get(importId);
if (progress == null || ownerId == null) {
Long groupId = deleteImportGroupMap.get(importId);
if (progress == null || ownerId == null || groupId == null) {
throw new BusinessException("删除任务不存在");
}
ensureUploaderAccess(ownerId, resolveAccessScope(operatorId));
ensureImportTaskAccess(groupId, resolveAccessScope(operatorId));
return progress;
}
private void runDeleteImportTask(String importId, File tempFile, String filename, Long operatorId) {
private void runDeleteImportTask(String importId, File tempFile, String filename,
Long operatorId, Long groupId) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
return;
@@ -298,7 +341,8 @@ public class DedupeTotalDataService {
progress.setStatus("running");
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
AccessScope scope = resolveAccessScope(operatorId);
DedupeTotalDataImportVo result = deleteFromExcelInternal(inputStream, filename, progress, scope);
DedupeTotalDataImportVo result = deleteFromExcelInternal(
inputStream, filename, progress, scope, groupId);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
@@ -315,7 +359,7 @@ public class DedupeTotalDataService {
}
private void runImportTask(String importId, File tempFile, String filename,
Long uploaderUserId, String uploaderUsername) {
Long uploaderUserId, String uploaderUsername, Long groupId) {
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
if (progress == null) {
return;
@@ -323,7 +367,7 @@ public class DedupeTotalDataService {
progress.setStatus("running");
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
DedupeTotalDataImportVo result = importFromExcelInternal(
inputStream, filename, progress, uploaderUserId, uploaderUsername);
inputStream, filename, progress, uploaderUserId, uploaderUsername, groupId);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
@@ -364,13 +408,19 @@ public class DedupeTotalDataService {
}
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long operatorId) {
return importFromExcel(file, null, operatorId);
}
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long operatorId) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity uploader = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
try (InputStream inputStream = file.getInputStream()) {
return importFromExcelInternal(
inputStream, file.getOriginalFilename(), null, uploader.getId(), uploader.getUsername());
inputStream, file.getOriginalFilename(), null, uploader.getId(), uploader.getUsername(),
group.getId());
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
@@ -379,12 +429,19 @@ public class DedupeTotalDataService {
}
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long operatorId) {
return deleteFromExcel(file, null, operatorId);
}
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long groupId, Long operatorId) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity operator = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
AccessScope scope = resolveAccessScope(operatorId);
try (InputStream inputStream = file.getInputStream()) {
return deleteFromExcelInternal(inputStream, file.getOriginalFilename(), null, scope);
return deleteFromExcelInternal(
inputStream, file.getOriginalFilename(), null, scope, group.getId());
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
@@ -394,7 +451,8 @@ public class DedupeTotalDataService {
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename,
DedupeTotalDataImportProgressVo progress,
Long uploaderUserId, String uploaderUsername) {
Long uploaderUserId, String uploaderUsername,
Long groupId) {
String lowerFilename = filename == null ? "" : filename.toLowerCase();
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
@@ -490,6 +548,7 @@ public class DedupeTotalDataService {
}
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(pendingDataValue);
entity.setGroupId(groupId);
entity.setUploaderUserId(uploaderUserId);
entity.setUploaderUsername(uploaderUsername);
dedupeTotalDataMapper.insert(entity);
@@ -526,7 +585,7 @@ public class DedupeTotalDataService {
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
DedupeTotalDataImportProgressVo progress,
AccessScope scope) {
AccessScope scope, Long groupId) {
String lowerFilename = filename == null ? "" : filename.toLowerCase();
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
@@ -604,7 +663,8 @@ public class DedupeTotalDataService {
continue;
}
int deletedThisRow = newRequiresNewTemplate().execute(status -> deleteByDataValue(dataValue, scope));
int deletedThisRow = newRequiresNewTemplate().execute(
status -> deleteByDataValue(dataValue, scope, groupId));
if (deletedThisRow > 0) {
deletedCount += deletedThisRow;
} else {
@@ -634,11 +694,14 @@ public class DedupeTotalDataService {
@Transactional
public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request, Long operatorId) {
DedupeTotalDataEntity entity = getAccessibleById(id, operatorId);
AdminUserEntity operator = getOperator(operatorId);
String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, id);
entity.setDataValue(dataValue);
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), operator);
entity.setGroupId(group.getId());
dedupeTotalDataMapper.updateById(entity);
return toItemVo(getById(id));
return toItemVo(getById(id), group.getGroupName());
}
@Transactional
@@ -649,7 +712,7 @@ public class DedupeTotalDataService {
private DedupeTotalDataEntity getAccessibleById(Long id, Long operatorId) {
DedupeTotalDataEntity entity = getById(id);
ensureUploaderAccess(entity.getUploaderUserId(), resolveAccessScope(operatorId));
ensureRecordAccess(entity, resolveAccessScope(operatorId));
return entity;
}
@@ -690,26 +753,78 @@ public class DedupeTotalDataService {
return dedupeTotalDataMapper.selectOne(query) != null;
}
private int deleteByDataValue(String dataValue, AccessScope scope) {
return dedupeTotalDataMapper.delete(new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
.in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds()));
private int deleteByDataValue(String dataValue, AccessScope scope, Long groupId) {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getDataValue, dataValue);
applyGroupScope(query, scope, groupId);
return dedupeTotalDataMapper.delete(query);
}
private AccessScope resolveAccessScope(Long operatorId) {
AdminUserEntity operator = getOperator(operatorId);
if (isSuperAdmin(operator)) {
return new AccessScope(true, Set.of());
return new AccessScope(true, Set.of(), Set.of());
}
List<Long> accessibleGroupIds = shopManageGroupMapper.selectAccessibleGroupIds(operator.getId());
LinkedHashSet<Long> visibleUserIds = new LinkedHashSet<>();
visibleUserIds.add(operator.getId());
if (accessibleGroupIds != null && !accessibleGroupIds.isEmpty()) {
List<Long> groupUserIds = shopManageGroupMapper.selectUserIdsByGroupIds(accessibleGroupIds);
if (groupUserIds != null) {
groupUserIds.stream()
.filter(id -> id != null && id > 0)
.forEach(visibleUserIds::add);
}
}
List<Long> memberUserIds = shopManageGroupMapper.selectManagedMemberUserIds(operator.getId());
if (memberUserIds != null) {
memberUserIds.stream()
.filter(id -> id != null && id > 0)
.forEach(visibleUserIds::add);
}
return new AccessScope(false, Set.copyOf(visibleUserIds));
return new AccessScope(
false,
Set.copyOf(accessibleGroupIds == null ? List.of() : accessibleGroupIds),
Set.copyOf(visibleUserIds));
}
private void applyGroupScope(LambdaQueryWrapper<DedupeTotalDataEntity> query,
AccessScope scope, Long groupId) {
if (groupId != null && groupId > 0) {
resolveAccessibleGroup(groupId, scope);
query.eq(DedupeTotalDataEntity::getGroupId, groupId);
return;
}
if (scope.allUsers()) {
return;
}
query.and(wrapper -> {
if (!scope.groupIds().isEmpty()) {
wrapper.in(DedupeTotalDataEntity::getGroupId, scope.groupIds())
.or();
}
wrapper.isNull(DedupeTotalDataEntity::getGroupId)
.in(DedupeTotalDataEntity::getUploaderUserId, scope.userIds());
});
}
private ShopManageGroupEntity resolveWritableGroup(Long groupId, AdminUserEntity operator) {
if (groupId == null || groupId <= 0) {
throw new BusinessException("请选择分组");
}
AccessScope scope = resolveAccessScope(operator.getId());
return resolveAccessibleGroup(groupId, scope);
}
private ShopManageGroupEntity resolveAccessibleGroup(Long groupId, AccessScope scope) {
ShopManageGroupEntity group = shopManageGroupMapper.selectById(groupId);
if (group == null) {
throw new BusinessException("分组不存在");
}
if (!scope.allUsers() && !scope.groupIds().contains(groupId)) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该分组数据");
}
return group;
}
private AdminUserEntity getOperator(Long operatorId) {
@@ -729,27 +844,46 @@ public class DedupeTotalDataService {
|| (role.isEmpty() && Integer.valueOf(1).equals(user.getIsAdmin()) && user.getCreatedById() == null);
}
private void ensureUploaderAccess(Long uploaderUserId, AccessScope scope) {
if (!scope.allUsers() && (uploaderUserId == null || !scope.userIds().contains(uploaderUserId))) {
private void ensureRecordAccess(DedupeTotalDataEntity entity, AccessScope scope) {
if (scope.allUsers()) {
return;
}
if (entity.getGroupId() != null && scope.groupIds().contains(entity.getGroupId())) {
return;
}
if (entity.getGroupId() == null && entity.getUploaderUserId() != null
&& scope.userIds().contains(entity.getUploaderUserId())) {
return;
}
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该总数据");
}
private void ensureImportTaskAccess(Long groupId, AccessScope scope) {
if (!scope.allUsers() && !scope.groupIds().contains(groupId)) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权查看该导入任务");
}
}
private void cleanupExpiredProgress() {
long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS;
cleanupExpiredProgressEntries(importCompletedAtMap, importProgressMap, importOwnerMap, cutoff);
cleanupExpiredProgressEntries(deleteImportCompletedAtMap, deleteImportProgressMap, deleteImportOwnerMap, cutoff);
cleanupExpiredProgressEntries(
importCompletedAtMap, importProgressMap, importOwnerMap, importGroupMap, cutoff);
cleanupExpiredProgressEntries(
deleteImportCompletedAtMap, deleteImportProgressMap, deleteImportOwnerMap,
deleteImportGroupMap, cutoff);
}
private void cleanupExpiredProgressEntries(
Map<String, Long> completedAtMap,
Map<String, DedupeTotalDataImportProgressVo> progressMap,
Map<String, Long> ownerMap,
Map<String, Long> groupMap,
long cutoff) {
completedAtMap.forEach((id, completedAt) -> {
if (completedAt != null && completedAt < cutoff && completedAtMap.remove(id, completedAt)) {
progressMap.remove(id);
ownerMap.remove(id);
groupMap.remove(id);
}
});
}
@@ -768,16 +902,36 @@ public class DedupeTotalDataService {
.replaceAll("\\s+", " ");
}
private DedupeTotalDataItemVo toItemVo(DedupeTotalDataEntity entity) {
private Map<Long, String> loadGroupNames(List<DedupeTotalDataEntity> rows) {
List<Long> groupIds = rows.stream()
.map(DedupeTotalDataEntity::getGroupId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (groupIds.isEmpty()) {
return Map.of();
}
return shopManageGroupMapper.selectBatchIds(groupIds).stream()
.filter(group -> group.getId() != null)
.collect(java.util.stream.Collectors.toMap(
ShopManageGroupEntity::getId,
group -> group.getGroupName() == null ? "" : group.getGroupName(),
(left, right) -> left,
LinkedHashMap::new));
}
private DedupeTotalDataItemVo toItemVo(DedupeTotalDataEntity entity, String groupName) {
DedupeTotalDataItemVo vo = new DedupeTotalDataItemVo();
vo.setId(entity.getId());
vo.setDataValue(entity.getDataValue());
vo.setGroupId(entity.getGroupId());
vo.setGroupName(groupName == null ? "" : groupName);
vo.setUploaderUserId(entity.getUploaderUserId());
vo.setUsername(entity.getUploaderUsername());
vo.setCreatedAt(entity.getCreatedAt());
return vo;
}
private record AccessScope(boolean allUsers, Set<Long> userIds) {
private record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds) {
}
}
@@ -147,6 +147,18 @@ public class OssStorageService {
}
}
/**
* Reads a managed result object from either its object key or stored public URL.
* This overload keeps callers from depending on the configured bucket name.
*/
public byte[] readObjectBytes(String value) {
StorageLocation location = resolveStorageLocation(value);
if (location == null) {
throw new IllegalArgumentException("object value must not be blank");
}
return readObjectBytes(location.bucket(), location.objectKey());
}
public boolean objectExists(String bucket, String objectKey) {
String normalizedBucket = requireStorageName(bucket, "bucket");
String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
@@ -1,19 +1,25 @@
package com.nanri.aiimage.modules.invalidasin.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataCreateRequest;
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequest;
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo;
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo;
import com.nanri.aiimage.modules.invalidasin.service.InvalidAsinDataService;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -24,13 +30,29 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/admin/invalid-asin-data")
@Tag(name = "不符合ASIN数据", description = "维护不符合ASIN数据列表,支持增删改查。")
public class InvalidAsinDataController {
private static final String INVALID_ASIN_DATA_COLUMN_KEY = "admin_invalid_asin_data";
private static final String INVALID_ASIN_DATA_ROUTE_PATH = "invalid-asin-data";
@Value("${aiimage.security.internal-token:}")
private String internalToken;
@Value("${aiimage.security.internal-token-file:}")
private String internalTokenFile;
private final InvalidAsinDataService invalidAsinDataService;
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
@GetMapping
@Operation(summary = "分页查询不符合ASIN数据", description = "分页查询不符合ASIN数据,支持按 ASIN/品牌 模糊搜索。")
@@ -40,8 +62,12 @@ public class InvalidAsinDataController {
public ApiResponse<InvalidAsinDataPageVo> page(
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword) {
return ApiResponse.success(invalidAsinDataService.page(page, pageSize, keyword));
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword,
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
HttpServletRequest request) {
RequestOperator operator = requireInvalidAsinDataAccess(request);
return ApiResponse.success(invalidAsinDataService.page(
page, pageSize, keyword, groupId, operator.id(), operator.superAdmin()));
}
@PostMapping
@@ -50,8 +76,12 @@ public class InvalidAsinDataController {
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功", content = @Content(schema = @Schema(implementation = InvalidAsinDataItemVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复")
})
public ApiResponse<InvalidAsinDataItemVo> create(@Valid @RequestBody InvalidAsinDataCreateRequest request) {
return ApiResponse.success("创建成功", invalidAsinDataService.create(request));
public ApiResponse<InvalidAsinDataItemVo> create(
HttpServletRequest httpRequest,
@Valid @RequestBody InvalidAsinDataCreateRequest request) {
RequestOperator operator = requireInvalidAsinDataAccess(httpRequest);
return ApiResponse.success("创建成功", invalidAsinDataService.create(
request, operator.id(), operator.superAdmin()));
}
@PutMapping("/{id}")
@@ -63,8 +93,11 @@ public class InvalidAsinDataController {
})
public ApiResponse<InvalidAsinDataItemVo> update(
@Parameter(description = "主键ID", required = true) @PathVariable Long id,
HttpServletRequest httpRequest,
@Valid @RequestBody InvalidAsinDataUpdateRequest request) {
return ApiResponse.success("更新成功", invalidAsinDataService.update(id, request));
RequestOperator operator = requireInvalidAsinDataAccess(httpRequest);
return ApiResponse.success("更新成功", invalidAsinDataService.update(
id, request, operator.id(), operator.superAdmin()));
}
@DeleteMapping("/{id}")
@@ -73,8 +106,107 @@ public class InvalidAsinDataController {
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在")
})
public ApiResponse<Void> delete(@Parameter(description = "主键ID", required = true) @PathVariable Long id) {
invalidAsinDataService.delete(id);
public ApiResponse<Void> delete(
@Parameter(description = "主键ID", required = true) @PathVariable Long id,
HttpServletRequest request) {
RequestOperator operator = requireInvalidAsinDataAccess(request);
invalidAsinDataService.delete(id, operator.id(), operator.superAdmin());
return ApiResponse.success("删除成功", null);
}
private RequestOperator requireInvalidAsinDataAccess(HttpServletRequest request) {
AdminUserEntity operator = resolveOperator(request);
if (operator == null || operator.getId() == null || operator.getId() <= 0) {
throw new BusinessException(403, "无权访问品牌数据库");
}
boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(operator));
if (!superAdmin && !hasInvalidAsinDataPermission(operator)) {
throw new BusinessException(403, "无权访问品牌数据库");
}
return new RequestOperator(operator.getId(), superAdmin);
}
private AdminUserEntity resolveOperator(HttpServletRequest request) {
try {
return adminAuthSupport.requireUser(request);
} catch (BusinessException authFailure) {
AdminUserEntity internalOperator = resolveInternalOperator(request);
if (internalOperator != null) {
return internalOperator;
}
throw authFailure;
}
}
private boolean hasInvalidAsinDataPermission(AdminUserEntity operator) {
return permissionMenuService.getUserColumnPermissions(operator.getId(), "admin")
.stream()
.anyMatch(item -> INVALID_ASIN_DATA_COLUMN_KEY.equals(item.getColumnKey())
|| INVALID_ASIN_DATA_ROUTE_PATH.equals(item.getRoutePath()));
}
private AdminUserEntity resolveInternalOperator(HttpServletRequest request) {
if (!isTrustedInternalRequest(request.getHeader("X-Internal-Token"))) {
return null;
}
String rawOperatorId = request.getParameter("operatorId");
if (rawOperatorId == null || rawOperatorId.isBlank()) {
rawOperatorId = request.getParameter("operator_id");
}
if (rawOperatorId == null || rawOperatorId.isBlank()) {
return null;
}
try {
return permissionMenuService.requireUserOperator(Long.parseLong(rawOperatorId.trim()));
} catch (NumberFormatException | BusinessException ex) {
return null;
}
}
private boolean isTrustedInternalRequest(String suppliedToken) {
String expectedToken = resolveExpectedInternalToken();
return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank()
&& MessageDigest.isEqual(
expectedToken.getBytes(StandardCharsets.UTF_8),
suppliedToken.trim().getBytes(StandardCharsets.UTF_8));
}
private String resolveExpectedInternalToken() {
if (internalToken != null && !internalToken.isBlank()) {
return internalToken.trim();
}
Path path = resolveInternalTokenFile();
if (path == null || !Files.isRegularFile(path)) {
return "";
}
try {
return Files.readString(path, StandardCharsets.UTF_8).trim();
} catch (Exception ignored) {
return "";
}
}
private Path resolveInternalTokenFile() {
String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim();
if (!configuredPath.isEmpty()) {
if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) {
String userHome = System.getProperty("user.home", "").trim();
if (userHome.isEmpty()) {
return null;
}
configuredPath = configuredPath.length() == 1
? userHome
: Path.of(userHome, configuredPath.substring(2)).toString();
}
Path configuredTokenPath = Path.of(configuredPath);
return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null;
}
String userHome = System.getProperty("user.home", "").trim();
return userHome.isEmpty()
? null
: Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize();
}
private record RequestOperator(Long id, boolean superAdmin) {
}
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.invalidasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
@@ -17,4 +18,8 @@ public class InvalidAsinDataCreateRequest {
@Size(max = 128, message = "品牌长度不能超过128个字符")
@Schema(description = "品牌名称")
private String brand;
@NotNull(message = "请选择分组")
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
private Long groupId;
}
@@ -17,4 +17,7 @@ public class InvalidAsinDataUpdateRequest {
@Size(max = 128, message = "品牌长度不能超过128个字符")
@Schema(description = "品牌名称")
private String brand;
@Schema(description = "分组 ID;自动导入数据无需填写")
private Long groupId;
}
@@ -15,6 +15,8 @@ public class InvalidAsinDataEntity {
private Long id;
private String dataValue;
private String brand;
private Long groupId;
private String recordSource;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -18,6 +18,15 @@ public class InvalidAsinDataItemVo {
@Schema(description = "品牌名称")
private String brand;
@Schema(description = "分组 ID")
private Long groupId;
@Schema(description = "分组名称")
private String groupName;
@Schema(description = "记录来源:MANUAL/AUTO")
private String recordSource;
@Schema(description = "创建时间")
private LocalDateTime createdAt;
}
@@ -8,20 +8,30 @@ import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequ
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo;
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
import com.nanri.aiimage.modules.shopkey.service.ShopManageGroupService;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class InvalidAsinDataService {
private final InvalidAsinDataMapper invalidAsinDataMapper;
public static final String RECORD_SOURCE_MANUAL = "MANUAL";
public static final String RECORD_SOURCE_AUTO = "AUTO";
public InvalidAsinDataPageVo page(long page, long pageSize, String keyword) {
private final InvalidAsinDataMapper invalidAsinDataMapper;
private final ShopManageGroupService shopManageGroupService;
public InvalidAsinDataPageVo page(long page, long pageSize, String keyword, Long groupId, Long operatorId, boolean superAdmin) {
long safePage = Math.max(page, 1);
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
String safeKeyword = keyword == null ? "" : keyword.trim();
@@ -31,11 +41,28 @@ public class InvalidAsinDataService {
.or()
.like(InvalidAsinDataEntity::getBrand, safeKeyword))
.orderByDesc(InvalidAsinDataEntity::getId);
if (!superAdmin) {
Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId);
if (fixedGroupId == null) {
query.eq(InvalidAsinDataEntity::getId, -1L);
} else {
query.eq(InvalidAsinDataEntity::getRecordSource, RECORD_SOURCE_MANUAL)
.eq(InvalidAsinDataEntity::getGroupId, fixedGroupId);
}
} else if (groupId != null && groupId > 0) {
query.eq(InvalidAsinDataEntity::getGroupId, groupId);
}
Long total = invalidAsinDataMapper.selectCount(query);
List<InvalidAsinDataItemVo> items = invalidAsinDataMapper
.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
.stream()
.map(this::toItemVo)
List<InvalidAsinDataEntity> rows = invalidAsinDataMapper
.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize));
Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream()
.map(InvalidAsinDataEntity::getGroupId)
.filter(rowGroupId -> rowGroupId != null && rowGroupId > 0)
.distinct()
.toList());
List<InvalidAsinDataItemVo> items = rows.stream()
.map(entity -> toItemVo(entity,
entity.getGroupId() == null ? "" : groupNameById.getOrDefault(entity.getGroupId(), "")))
.toList();
InvalidAsinDataPageVo vo = new InvalidAsinDataPageVo();
vo.setItems(items);
@@ -46,35 +73,63 @@ public class InvalidAsinDataService {
}
@Transactional
public InvalidAsinDataItemVo create(InvalidAsinDataCreateRequest request) {
public InvalidAsinDataItemVo create(InvalidAsinDataCreateRequest request, Long operatorId, boolean superAdmin) {
ShopManageGroupEntity group = resolveManualWriteGroup(request.getGroupId(), operatorId, superAdmin);
String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空");
String brand = normalizeRequired(request.getBrand(), "品牌不能为空").toLowerCase(Locale.ROOT);
ensureUnique(dataValue, null);
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
entity.setDataValue(dataValue);
entity.setBrand(brand);
entity.setGroupId(group.getId());
entity.setRecordSource(RECORD_SOURCE_MANUAL);
invalidAsinDataMapper.insert(entity);
return toItemVo(getById(entity.getId()));
return toItemVo(getById(entity.getId()), group.getGroupName());
}
@Transactional
public InvalidAsinDataItemVo update(Long id, InvalidAsinDataUpdateRequest request) {
public InvalidAsinDataItemVo update(Long id, InvalidAsinDataUpdateRequest request, Long operatorId, boolean superAdmin) {
InvalidAsinDataEntity entity = getById(id);
ShopManageGroupEntity fixedGroup = ensureRecordAccess(entity, operatorId, superAdmin);
ShopManageGroupEntity writeGroup = null;
if (isManualRecord(entity)) {
Long requestedGroupId = requireGroupId(request.getGroupId());
writeGroup = superAdmin
? shopManageGroupService.getAccessibleById(requestedGroupId, operatorId, true)
: resolveLockedGroup(requestedGroupId, fixedGroup);
}
String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空");
String brand = normalizeRequired(request.getBrand(), "品牌不能为空").toLowerCase(Locale.ROOT);
ensureUnique(dataValue, id);
entity.setDataValue(dataValue);
entity.setBrand(brand);
String groupName = "";
if (writeGroup != null) {
entity.setGroupId(writeGroup.getId());
groupName = writeGroup.getGroupName();
}
invalidAsinDataMapper.updateById(entity);
return toItemVo(getById(id));
return toItemVo(getById(id), groupName);
}
@Transactional
public void delete(Long id) {
public void delete(Long id, Long operatorId, boolean superAdmin) {
InvalidAsinDataEntity entity = getById(id);
ensureRecordAccess(entity, operatorId, superAdmin);
invalidAsinDataMapper.deleteById(entity.getId());
}
private ShopManageGroupEntity resolveManualWriteGroup(Long requestedGroupId, Long operatorId, boolean superAdmin) {
Long normalizedRequestedGroupId = requireGroupId(requestedGroupId);
if (superAdmin) {
return shopManageGroupService.getAccessibleById(
normalizedRequestedGroupId, operatorId, true);
}
Long fixedGroupId = requireFixedAccessibleGroupId(operatorId);
ensureRequestedLockedGroup(normalizedRequestedGroupId, fixedGroupId);
return shopManageGroupService.getAccessibleById(fixedGroupId, operatorId, false);
}
private InvalidAsinDataEntity getById(Long id) {
InvalidAsinDataEntity entity = invalidAsinDataMapper.selectById(id);
if (entity == null) {
@@ -115,11 +170,76 @@ public class InvalidAsinDataService {
.replaceAll("\\s+", " ");
}
private InvalidAsinDataItemVo toItemVo(InvalidAsinDataEntity entity) {
private ShopManageGroupEntity ensureRecordAccess(InvalidAsinDataEntity entity, Long operatorId, boolean superAdmin) {
if (superAdmin) {
return null;
}
if (!isManualRecord(entity) || entity.getGroupId() == null || entity.getGroupId() <= 0) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该品牌数据");
}
Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId);
if (fixedGroupId == null || !fixedGroupId.equals(entity.getGroupId())) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作非固定分组数据");
}
try {
return shopManageGroupService.getAccessibleById(fixedGroupId, operatorId, false);
} catch (BusinessException exception) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该品牌数据");
}
}
private Long requireFixedAccessibleGroupId(Long operatorId) {
Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId);
if (fixedGroupId == null) {
throw new BusinessException("当前用户没有可访问的分组");
}
return fixedGroupId;
}
private Long resolveFixedAccessibleGroupId(Long operatorId) {
Set<Long> accessibleGroupIds = shopManageGroupService.listAccessibleGroupIds(operatorId, false);
if (accessibleGroupIds == null) {
return null;
}
return accessibleGroupIds.stream()
.filter(groupId -> groupId != null && groupId > 0)
.min(Long::compareTo)
.orElse(null);
}
private ShopManageGroupEntity resolveLockedGroup(Long requestedGroupId, ShopManageGroupEntity fixedGroup) {
if (fixedGroup == null || fixedGroup.getId() == null) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作非固定分组数据");
}
ensureRequestedLockedGroup(requestedGroupId, fixedGroup.getId());
return fixedGroup;
}
private void ensureRequestedLockedGroup(Long requestedGroupId, Long fixedGroupId) {
if (fixedGroupId == null || !fixedGroupId.equals(requestedGroupId)) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作非固定分组数据");
}
}
private Long requireGroupId(Long groupId) {
if (groupId == null || groupId <= 0) {
throw new BusinessException("请选择分组");
}
return groupId;
}
private boolean isManualRecord(InvalidAsinDataEntity entity) {
return entity != null && RECORD_SOURCE_MANUAL.equalsIgnoreCase(normalizeText(entity.getRecordSource()));
}
private InvalidAsinDataItemVo toItemVo(InvalidAsinDataEntity entity, String groupName) {
InvalidAsinDataItemVo vo = new InvalidAsinDataItemVo();
vo.setId(entity.getId());
vo.setDataValue(entity.getDataValue());
vo.setBrand(entity.getBrand());
vo.setGroupId(entity.getGroupId());
vo.setGroupName(groupName == null ? "" : groupName);
vo.setRecordSource(isManualRecord(entity) ? RECORD_SOURCE_MANUAL : RECORD_SOURCE_AUTO);
vo.setCreatedAt(entity.getCreatedAt());
return vo;
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.shopdatacrawl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ShopDataCrawlDailyFileMapper extends BaseMapper<ShopDataCrawlDailyFileEntity> {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.shopdatacrawl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ShopDataCrawlDailyMemberMapper extends BaseMapper<ShopDataCrawlDailyMemberEntity> {
}
@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDate;
import java.time.LocalDateTime;
@Data
@TableName("biz_shop_data_crawl_daily_file")
public class ShopDataCrawlDailyFileEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private String shopKeyHash;
private String shopKey;
private LocalDate businessDate;
private Long latestTaskId;
private Long latestResultId;
private String resultFilename;
private String resultFileUrl;
private Long resultFileSize;
private String resultContentType;
private Integer rowCount;
private Long version;
private LocalDateTime lastSuccessAt;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_shop_data_crawl_daily_member")
public class ShopDataCrawlDailyMemberEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long dailyFileId;
private Long taskId;
private Long resultId;
private LocalDateTime createdAt;
}
@@ -0,0 +1,218 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlDailyFileMapper;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlDailyMemberMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.HexFormat;
import java.util.List;
import java.util.Set;
/**
* Persistence and locking primitives for the shop-data-crawl daily workbook.
* Workbook assembly remains in ShopDataCrawlTaskService so task state changes
* and the daily pointer are committed together.
*/
@Service
@RequiredArgsConstructor
public class ShopDataCrawlDailyFileService {
public static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
private static final Duration DAILY_LOCK_TTL = Duration.ofMinutes(30);
private static final long DAILY_LOCK_WAIT_MILLIS = 15000L;
private static final String DAILY_LOCK_MODULE_PREFIX = "SHOP_DATA_CRAWL_DAILY_";
private final ShopDataCrawlDailyFileMapper dailyFileMapper;
private final ShopDataCrawlDailyMemberMapper dailyMemberMapper;
private final TaskDistributedLockService taskDistributedLockService;
public LocalDate currentBusinessDate() {
return LocalDate.now(BUSINESS_ZONE);
}
public LocalDateTime currentBusinessDateTime() {
return LocalDateTime.now(BUSINESS_ZONE);
}
public String shopKey(FileResultEntity row) {
if (row == null) {
return null;
}
String shopId = trimToNull(row.getSourceFileUrl());
if (shopId != null) {
return "shop-id:" + shopId;
}
String shopName = trimToNull(row.getSourceFilename());
return shopName == null ? null : "shop-name:" + shopName;
}
public String shopKeyHash(String shopKey) {
if (shopKey == null || shopKey.isBlank()) {
return null;
}
byte[] digest;
try {
digest = MessageDigest.getInstance("SHA-256")
.digest(shopKey.getBytes(StandardCharsets.UTF_8));
} catch (Exception ex) {
throw new IllegalStateException("SHA-256 is unavailable", ex);
}
return HexFormat.of().formatHex(digest);
}
public TaskDistributedLockService.LockHandle acquireLock(Long userId, String shopKey) {
if (userId == null || userId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
// The lock protects the shop's whole daily-file lifecycle. Including the
// date would allow yesterday and today to update the same shop together.
String identity = userId + "|" + shopKey;
String lockModule = DAILY_LOCK_MODULE_PREFIX + shopKeyHash(identity);
return taskDistributedLockService.acquire(lockModule, 1L, DAILY_LOCK_TTL, DAILY_LOCK_WAIT_MILLIS);
}
public ShopDataCrawlDailyFileEntity findForUpdate(Long userId, String shopKeyHash, LocalDate businessDate) {
if (userId == null || shopKeyHash == null || businessDate == null) {
return null;
}
return dailyFileMapper.selectOne(new LambdaQueryWrapper<ShopDataCrawlDailyFileEntity>()
.eq(ShopDataCrawlDailyFileEntity::getUserId, userId)
.eq(ShopDataCrawlDailyFileEntity::getShopKeyHash, shopKeyHash)
.eq(ShopDataCrawlDailyFileEntity::getBusinessDate, businessDate)
.last("FOR UPDATE"));
}
public List<ShopDataCrawlDailyFileEntity> findOlder(Long userId, String shopKeyHash, LocalDate businessDate) {
if (userId == null || shopKeyHash == null || businessDate == null) {
return List.of();
}
return dailyFileMapper.selectList(new LambdaQueryWrapper<ShopDataCrawlDailyFileEntity>()
.eq(ShopDataCrawlDailyFileEntity::getUserId, userId)
.eq(ShopDataCrawlDailyFileEntity::getShopKeyHash, shopKeyHash)
.lt(ShopDataCrawlDailyFileEntity::getBusinessDate, businessDate)
.orderByDesc(ShopDataCrawlDailyFileEntity::getBusinessDate)
.orderByDesc(ShopDataCrawlDailyFileEntity::getId)
.last("FOR UPDATE"));
}
public List<ShopDataCrawlDailyFileEntity> findByLatestResultId(Long resultId) {
if (resultId == null || resultId <= 0) {
return List.of();
}
return dailyFileMapper.selectList(new LambdaQueryWrapper<ShopDataCrawlDailyFileEntity>()
.eq(ShopDataCrawlDailyFileEntity::getLatestResultId, resultId)
.last("FOR UPDATE"));
}
public ShopDataCrawlDailyFileEntity findById(Long dailyFileId) {
return dailyFileId == null || dailyFileId <= 0 ? null : dailyFileMapper.selectById(dailyFileId);
}
public List<ShopDataCrawlDailyMemberEntity> findMembersByResultId(Long resultId) {
if (resultId == null || resultId <= 0) {
return List.of();
}
return dailyMemberMapper.selectList(new LambdaQueryWrapper<ShopDataCrawlDailyMemberEntity>()
.eq(ShopDataCrawlDailyMemberEntity::getResultId, resultId));
}
public boolean containsResult(Long dailyFileId, Long resultId) {
if (dailyFileId == null || dailyFileId <= 0 || resultId == null || resultId <= 0) {
return false;
}
return dailyMemberMapper.selectCount(new LambdaQueryWrapper<ShopDataCrawlDailyMemberEntity>()
.eq(ShopDataCrawlDailyMemberEntity::getDailyFileId, dailyFileId)
.eq(ShopDataCrawlDailyMemberEntity::getResultId, resultId)) > 0;
}
public boolean addMember(Long dailyFileId, Long taskId, Long resultId) {
if (dailyFileId == null || dailyFileId <= 0 || taskId == null || taskId <= 0
|| resultId == null || resultId <= 0) {
return false;
}
ShopDataCrawlDailyMemberEntity member = new ShopDataCrawlDailyMemberEntity();
member.setDailyFileId(dailyFileId);
member.setTaskId(taskId);
member.setResultId(resultId);
member.setCreatedAt(currentBusinessDateTime());
try {
dailyMemberMapper.insert(member);
return true;
} catch (DuplicateKeyException ignored) {
return false;
}
}
public List<ShopDataCrawlDailyMemberEntity> listMembers(Long dailyFileId) {
if (dailyFileId == null || dailyFileId <= 0) {
return List.of();
}
return dailyMemberMapper.selectList(new LambdaQueryWrapper<ShopDataCrawlDailyMemberEntity>()
.eq(ShopDataCrawlDailyMemberEntity::getDailyFileId, dailyFileId)
.orderByDesc(ShopDataCrawlDailyMemberEntity::getCreatedAt)
.orderByDesc(ShopDataCrawlDailyMemberEntity::getId)
.last("FOR UPDATE"));
}
public void deleteMembers(Long dailyFileId) {
if (dailyFileId == null || dailyFileId <= 0) {
return;
}
dailyMemberMapper.delete(new LambdaQueryWrapper<ShopDataCrawlDailyMemberEntity>()
.eq(ShopDataCrawlDailyMemberEntity::getDailyFileId, dailyFileId));
}
public void deleteMembersForResults(Set<Long> resultIds) {
if (resultIds == null || resultIds.isEmpty()) {
return;
}
dailyMemberMapper.delete(new LambdaQueryWrapper<ShopDataCrawlDailyMemberEntity>()
.in(ShopDataCrawlDailyMemberEntity::getResultId, resultIds));
}
public long countObjectReferences(String objectKey) {
if (objectKey == null || objectKey.isBlank()) {
return 0L;
}
Long count = dailyFileMapper.selectCount(new LambdaQueryWrapper<ShopDataCrawlDailyFileEntity>()
.eq(ShopDataCrawlDailyFileEntity::getResultFileUrl, objectKey));
return count == null ? 0L : count;
}
public void deleteDailyFile(Long dailyFileId) {
if (dailyFileId == null || dailyFileId <= 0) {
return;
}
deleteMembers(dailyFileId);
dailyFileMapper.deleteById(dailyFileId);
}
public void insert(ShopDataCrawlDailyFileEntity entity) {
dailyFileMapper.insert(entity);
}
public void update(ShopDataCrawlDailyFileEntity entity) {
dailyFileMapper.updateById(entity);
}
private String trimToNull(String value) {
if (value == null) {
return null;
}
String normalized = value.trim();
return normalized.isEmpty() ? null : normalized;
}
}
@@ -19,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
@@ -64,6 +65,36 @@ public class ShopDataCrawlExcelAssemblyService {
}
}
/**
* Appends the supplied task rows to an already assembled daily workbook.
* Existing rows, styles and drawings are deliberately left untouched.
*/
public void appendWorkbook(File baseXlsx, File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
if (baseXlsx == null || !baseXlsx.isFile()) {
throw new BusinessException("当天累计文件不存在");
}
try (InputStream input = new FileInputStream(baseXlsx);
XSSFWorkbook workbook = new XSSFWorkbook(input);
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache = new ConcurrentHashMap<>();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
List<ShopDataCrawlRowDto> rows = rowsByCountry.get(COUNTRIES.get(i));
if (rows != null && !rows.isEmpty()) {
appendSheet(workbook, workbook.getSheetAt(i), rows, imageCache, pictureIndexes);
}
}
workbook.write(output);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("店铺数据抓取累计 Excel 追加失败: " + ex.getMessage());
}
}
public int countRows(List<ShopDataCrawlResultItemVo> items) {
return rowsByCountry(items).values().stream().mapToInt(List::size).sum();
}
@@ -141,6 +172,50 @@ public class ShopDataCrawlExcelAssemblyService {
}
}
private void appendSheet(XSSFWorkbook workbook,
Sheet sheet,
List<ShopDataCrawlRowDto> rows,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1);
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
boolean templateHasBrand = header != null && "品牌".equals(cellText(header, BRAND_COLUMN));
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
styles[column] = cell == null ? null : cell.getCellStyle();
}
writeHeaders(sheet, currentTemplate, templateHasBrand);
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
int rowIndex = Math.max(1, sheet.getLastRowNum() + 1);
for (ShopDataCrawlRowDto value : rows) {
Row row = sheet.createRow(rowIndex++);
writeRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes);
}
}
private void writeRow(XSSFWorkbook workbook,
Sheet sheet,
Row row,
ShopDataCrawlRowDto value,
CellStyle[] styles,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
if (!blank(value.getCommodityImage())) {
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
}
}
private void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
Row header = sheet.getRow(0);
if (header == null) header = sheet.createRow(0);
@@ -17,6 +17,8 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTask
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
@@ -45,16 +47,24 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.io.File;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.time.Duration;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
@Service
@RequiredArgsConstructor
@@ -66,7 +76,6 @@ public class ShopDataCrawlTaskService {
private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0;
private static final int RESULT_SUCCESS = 1;
private static final int SHOP_HISTORY_RETENTION_LIMIT = 3;
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断,已保留已回传的部分数据";
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
@@ -88,6 +97,7 @@ public class ShopDataCrawlTaskService {
private final TaskScopeStateMapper taskScopeStateMapper;
private final TransientPayloadStorageService transientPayloadStorageService;
private final InstanceMetadata instanceMetadata;
private final ShopDataCrawlDailyFileService dailyFileService;
@Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
@@ -601,8 +611,18 @@ public class ShopDataCrawlTaskService {
}
ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task");
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
List<String> resultFileUrls = listTaskRows(taskId).stream()
.map(FileResultEntity::getResultFileUrl).filter(url -> !blank(url)).distinct().toList();
List<FileResultEntity> taskRows = listTaskRows(taskId);
try (DailyLockSet dailyLocks = acquireDailyLocks(task.getUserId(), taskRows)) {
Set<Long> removedResultIds = taskRows.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.collect(java.util.stream.Collectors.toSet());
DailyDeletionResult dailyResult = prepareDailyForDeletion(removedResultIds);
registerUploadedObjectRollback(dailyResult.uploadedObjectKeys());
List<String> resultFileUrls = new ArrayList<>(dailyResult.obsoleteObjectKeys());
resultFileUrls.addAll(taskRows.stream()
.map(FileResultEntity::getResultFileUrl).filter(url -> !blank(url)).distinct().toList());
resultFileUrls = resultFileUrls.stream().filter(url -> !blank(url)).distinct().toList();
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
@@ -615,6 +635,7 @@ public class ShopDataCrawlTaskService {
resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced);
}
}
}
@Transactional
public void deleteHistory(Long resultId, Long userId) {
@@ -632,9 +653,11 @@ public class ShopDataCrawlTaskService {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("记录不存在");
}
try (DailyLockSet dailyLocks = acquireDailyLocks(userId, List.of(latestEntity))) {
deleteResultHistoryRow(latestEntity);
}
}
}
/**
* Deletes an administrative result without accepting a caller-controlled owner ID.
@@ -661,6 +684,8 @@ public class ShopDataCrawlTaskService {
Long taskId = entity.getTaskId();
Long resultId = entity.getId();
String resultFileUrl = entity.getResultFileUrl();
DailyDeletionResult dailyResult = prepareDailyForDeletion(Set.of(resultId));
registerUploadedObjectRollback(dailyResult.uploadedObjectKeys());
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
fileResultMapper.deleteById(resultId);
@@ -673,7 +698,161 @@ public class ShopDataCrawlTaskService {
taskId, resultId, safeMessage(ex));
}
// Run this after the row delete so shared object references are counted correctly.
deleteResultObjectIfUnreferenced(resultFileUrl);
Set<String> obsoleteObjectKeys = new HashSet<>(dailyResult.obsoleteObjectKeys());
collectObjectKey(obsoleteObjectKeys, resultFileUrl);
obsoleteObjectKeys.forEach(this::deleteResultObjectIfUnreferenced);
}
private DailyDeletionResult prepareDailyForDeletion(Set<Long> removedResultIds) {
if (removedResultIds == null || removedResultIds.isEmpty()) {
return new DailyDeletionResult(List.of(), List.of());
}
Set<Long> handledDailyIds = new HashSet<>();
Set<String> obsoleteObjectKeys = new HashSet<>();
List<String> uploadedObjectKeys = new ArrayList<>();
try {
for (Long resultId : removedResultIds) {
List<ShopDataCrawlDailyFileEntity> latestFiles = dailyFileService.findByLatestResultId(resultId);
List<ShopDataCrawlDailyFileEntity> affectedFiles = new ArrayList<>(
latestFiles == null ? List.of() : latestFiles);
List<ShopDataCrawlDailyMemberEntity> memberRows = dailyFileService.findMembersByResultId(resultId);
for (ShopDataCrawlDailyMemberEntity member : memberRows == null ? List.<ShopDataCrawlDailyMemberEntity>of() : memberRows) {
if (member == null) {
continue;
}
ShopDataCrawlDailyFileEntity memberFile = dailyFileService.findById(member.getDailyFileId());
if (memberFile != null) {
affectedFiles.add(memberFile);
}
}
for (ShopDataCrawlDailyFileEntity dailyFile : affectedFiles) {
if (dailyFile == null || dailyFile.getId() == null || !handledDailyIds.add(dailyFile.getId())) {
continue;
}
ShopDataCrawlDailyFileEntity lockedDailyFile = dailyFileService.findForUpdate(
dailyFile.getUserId(), dailyFile.getShopKeyHash(), dailyFile.getBusinessDate());
if (lockedDailyFile != null) {
dailyFile = lockedDailyFile;
}
List<DailyMemberData> survivors = loadDailyMemberData(dailyFile, removedResultIds);
if (survivors.isEmpty()) {
collectObjectKey(obsoleteObjectKeys, dailyFile.getResultFileUrl());
dailyFileService.deleteDailyFile(dailyFile.getId());
continue;
}
rebuildDailyFileAfterDeletion(dailyFile, survivors, uploadedObjectKeys, obsoleteObjectKeys);
}
}
dailyFileService.deleteMembersForResults(removedResultIds);
return new DailyDeletionResult(new ArrayList<>(obsoleteObjectKeys), uploadedObjectKeys);
} catch (RuntimeException ex) {
registerRollbackObjectCleanup(uploadedObjectKeys);
throw ex;
}
}
private List<DailyMemberData> loadDailyMemberData(ShopDataCrawlDailyFileEntity dailyFile,
Set<Long> removedResultIds) {
List<ShopDataCrawlDailyMemberEntity> memberRows = dailyFileService.listMembers(dailyFile.getId());
List<ShopDataCrawlDailyMemberEntity> members = new ArrayList<>(
memberRows == null ? List.of() : memberRows);
members.removeIf(Objects::isNull);
members.sort(Comparator
.comparing(ShopDataCrawlDailyMemberEntity::getCreatedAt,
Comparator.nullsLast(Comparator.naturalOrder()))
.thenComparing(ShopDataCrawlDailyMemberEntity::getId,
Comparator.nullsLast(Comparator.naturalOrder())));
List<DailyMemberData> survivors = new ArrayList<>();
for (ShopDataCrawlDailyMemberEntity member : members) {
if (removedResultIds.contains(member.getResultId())) {
continue;
}
FileResultEntity result = fileResultMapper.selectById(member.getResultId());
if (result == null || !Integer.valueOf(RESULT_SUCCESS).equals(result.getSuccess())) {
continue;
}
ShopDataCrawlResultItemVo snapshot = loadSnapshotForDailyMember(result);
if (snapshot == null || !Boolean.TRUE.equals(snapshot.getSuccess())) {
throw new BusinessException("无法读取累计文件中的剩余结果");
}
survivors.add(new DailyMemberData(member, result, snapshot));
}
return survivors;
}
private ShopDataCrawlResultItemVo loadSnapshotForDailyMember(FileResultEntity result) {
ShopDataCrawlResultItemVo snapshot = taskResultItemService.getResultSnapshot(
result.getTaskId(), MODULE_TYPE, result.getId(), ShopDataCrawlResultItemVo.class);
if (snapshot != null) {
return snapshot;
}
FileTaskEntity task = fileTaskMapper.selectById(result.getTaskId());
if (task == null) {
return null;
}
return indexSnapshotByResultId(buildSnapshotFromDb(task, List.of(result))).get(result.getId());
}
private void rebuildDailyFileAfterDeletion(ShopDataCrawlDailyFileEntity dailyFile,
List<DailyMemberData> survivors,
List<String> uploadedObjectKeys,
Set<String> obsoleteObjectKeys) {
DailyMemberData latest = survivors.get(survivors.size() - 1);
FileTaskEntity latestTask = fileTaskMapper.selectById(latest.result().getTaskId());
String filename = blank(dailyFile.getResultFilename())
? buildTaskWorkbookFilename(latestTask)
: dailyFile.getResultFilename();
File workRoot = FileUtil.mkdir(FileUtil.file(
System.getProperty("java.io.tmpdir"),
"shop-data-crawl-result",
"daily-delete-" + UUID.randomUUID()));
File outputXlsx = FileUtil.file(workRoot, filename);
String oldObjectKey = dailyFile.getResultFileUrl();
List<ShopDataCrawlResultItemVo> snapshots = survivors.stream()
.map(DailyMemberData::snapshot)
.toList();
try {
excelAssemblyService.writeWorkbook(outputXlsx, snapshots);
String newObjectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
if (blank(newObjectKey)) {
throw new BusinessException("累计文件上传结果为空");
}
uploadedObjectKeys.add(newObjectKey);
int rowCount = excelAssemblyService.countRows(snapshots);
for (DailyMemberData survivor : survivors) {
FileResultEntity result = survivor.result();
if (!Objects.equals(result.getId(), latest.result().getId()) && !blank(result.getResultFileUrl())) {
result.setResultFileUrl(null);
result.setResultFileSize(null);
result.setResultContentType(null);
fileResultMapper.updateById(result);
}
}
latest.result().setResultFilename(filename);
latest.result().setResultFileUrl(newObjectKey);
latest.result().setResultFileSize(outputXlsx.length());
latest.result().setResultContentType(CONTENT_TYPE_XLSX);
latest.result().setRowCount(rowCount);
fileResultMapper.updateById(latest.result());
LocalDateTime now = dailyFileService.currentBusinessDateTime();
dailyFile.setLatestTaskId(latest.result().getTaskId());
dailyFile.setLatestResultId(latest.result().getId());
dailyFile.setResultFilename(filename);
dailyFile.setResultFileUrl(newObjectKey);
dailyFile.setResultFileSize(latest.result().getResultFileSize());
dailyFile.setResultContentType(CONTENT_TYPE_XLSX);
dailyFile.setRowCount(rowCount);
dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L);
dailyFile.setLastSuccessAt(now);
dailyFile.setUpdatedAt(now);
dailyFileService.update(dailyFile);
collectObjectKey(obsoleteObjectKeys, oldObjectKey);
obsoleteObjectKeys.remove(newObjectKey);
} finally {
FileUtil.del(outputXlsx);
FileUtil.del(workRoot);
}
}
private void reconcileTaskAfterResultRemoval(Long taskId) {
@@ -700,131 +879,6 @@ public class ShopDataCrawlTaskService {
taskCacheService.saveTaskCache(task);
}
private void pruneCompletedHistoryQuietly(FileTaskEntity currentTask, List<FileResultEntity> currentRows) {
if (currentRows == null || currentRows.isEmpty()) {
return;
}
for (FileResultEntity row : currentRows) {
if (!isRetentionCandidate(row)) {
continue;
}
Long userId = row.getUserId() != null
? row.getUserId()
: currentTask == null ? null : currentTask.getUserId();
String shopKey = retentionShopKey(row);
if (userId == null || shopKey == null) {
log.warn("[shop-data-crawl] skip history retention because ownership key is incomplete taskId={} resultId={}",
currentTask == null ? null : currentTask.getId(), row.getId());
continue;
}
try {
pruneCompletedHistoryForShop(userId, shopKey);
} catch (Exception ex) {
// Retention is best effort. A cleanup failure must not fail the newly assembled workbook job.
log.warn("[shop-data-crawl] history retention failed taskId={} resultId={} msg={}",
currentTask == null ? null : currentTask.getId(), row.getId(), safeMessage(ex));
}
}
}
void pruneCompletedHistoryForShop(Long userId, String shopKey) {
if (userId == null || shopKey == null) {
return;
}
List<FileResultEntity> candidates = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId)
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
.isNotNull(FileResultEntity::getResultFileUrl)
.ne(FileResultEntity::getResultFileUrl, "")
.orderByDesc(FileResultEntity::getCreatedAt)
.orderByDesc(FileResultEntity::getId));
if (candidates == null || candidates.isEmpty()) {
return;
}
List<FileResultEntity> shopResults = new ArrayList<>();
for (FileResultEntity candidate : candidates) {
if (!isRetentionCandidate(candidate)
|| !Objects.equals(userId, candidate.getUserId())) {
continue;
}
if (!Objects.equals(shopKey, retentionShopKey(candidate))) {
continue;
}
shopResults.add(candidate);
}
Comparator<FileResultEntity> newestFirst = Comparator
.comparing(FileResultEntity::getCreatedAt, Comparator.nullsLast(Comparator.reverseOrder()))
.thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder()));
shopResults.sort(newestFirst);
for (int index = SHOP_HISTORY_RETENTION_LIMIT; index < shopResults.size(); index++) {
deleteRetentionResultQuietly(shopResults.get(index), userId, shopKey);
}
}
private void deleteRetentionResultQuietly(FileResultEntity candidate, Long userId, String shopKey) {
if (candidate == null || candidate.getId() == null || candidate.getId() <= 0
|| candidate.getTaskId() == null || candidate.getTaskId() <= 0) {
return;
}
try {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(candidate.getTaskId());
if (lockHandle == null) {
log.info("[shop-data-crawl] skip retained-history deletion because task lock is busy taskId={} resultId={}",
candidate.getTaskId(), candidate.getId());
return;
}
try (lockHandle) {
FileResultEntity latest = fileResultMapper.selectById(candidate.getId());
if (!isRetentionCandidate(latest)
|| !Objects.equals(userId, latest.getUserId())
|| !Objects.equals(shopKey, retentionShopKey(latest))) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(candidate.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())
|| !isTerminalTaskStatus(task.getStatus())) {
log.info("[shop-data-crawl] skip retained-history deletion because task is not terminal taskId={} resultId={}",
candidate.getTaskId(), candidate.getId());
return;
}
deleteResultHistoryRow(latest);
}
} catch (Exception ex) {
log.warn("[shop-data-crawl] retained-history deletion failed taskId={} resultId={} msg={}",
candidate.getTaskId(), candidate.getId(), safeMessage(ex));
}
}
private boolean isRetentionCandidate(FileResultEntity row) {
return row != null
&& Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())
&& !blank(row.getResultFileUrl());
}
private String retentionShopKey(FileResultEntity row) {
if (row == null) {
return null;
}
String shopId = trimToNull(row.getSourceFileUrl());
if (shopId != null) {
return "shop-id:" + shopId;
}
String shopName = trimToNull(row.getSourceFilename());
return shopName == null ? null : "shop-name:" + shopName;
}
private String trimToNull(String value) {
if (value == null) {
return null;
@@ -1519,6 +1573,7 @@ public class ShopDataCrawlTaskService {
}
}
@Transactional
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
@@ -1539,31 +1594,434 @@ public class ShopDataCrawlTaskService {
if (successItems.isEmpty()) {
throw new BusinessException("没有可生成的店铺数据抓取结果");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "shop-data-crawl-result", String.valueOf(task.getId())));
String filename = buildTaskWorkbookFilename(task);
File xlsx = FileUtil.file(workRoot, filename);
Map<Long, ShopDataCrawlResultItemVo> snapshotsByResultId = indexSnapshotByResultId(successItems);
LocalDate businessDate = dailyFileService.currentBusinessDate();
List<String> uploadedObjectKeys = new ArrayList<>();
List<String> obsoleteObjectKeys = new ArrayList<>();
try {
excelAssemblyService.writeWorkbook(xlsx, successItems);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (!Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
continue;
}
ShopDataCrawlResultItemVo snapshot = snapshotsByResultId.get(row.getId());
if (snapshot == null) {
snapshot = successItems.size() == 1 ? successItems.get(0) : null;
}
if (snapshot == null) {
throw new BusinessException("店铺结果快照不存在,无法生成累计文件");
}
DailyAggregationResult result = aggregateDailyResult(
task, row, snapshot, businessDate, uploadedObjectKeys);
obsoleteObjectKeys.addAll(result.obsoleteObjectKeys());
}
updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task);
} finally {
FileUtil.del(xlsx);
} catch (RuntimeException ex) {
registerRollbackObjectCleanup(uploadedObjectKeys);
throw ex;
}
pruneCompletedHistoryQuietly(task, rows);
registerDailyObjectLifecycle(uploadedObjectKeys, obsoleteObjectKeys);
}
private DailyAggregationResult aggregateDailyResult(FileTaskEntity task,
FileResultEntity row,
ShopDataCrawlResultItemVo snapshot,
LocalDate businessDate,
List<String> uploadedObjectKeys) {
Long userId = row.getUserId() != null ? row.getUserId() : task.getUserId();
String shopKey = dailyFileService.shopKey(row);
String shopKeyHash = dailyFileService.shopKeyHash(shopKey);
if (userId == null || shopKeyHash == null) {
throw new BusinessException("店铺累计文件归属信息不完整");
}
TaskDistributedLockService.LockHandle lock = dailyFileService.acquireLock(userId, shopKey);
if (lock == null) {
throw new BusinessException("店铺当天累计文件正在处理中,请稍后重试");
}
try {
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(userId, shopKeyHash, businessDate);
ShopDataCrawlDailyFileEntity existingMembership = findExistingDailyMembership(row.getId());
if (existingMembership != null) {
boolean currentFileOwnsMembership = dailyFile != null
&& Objects.equals(dailyFile.getId(), existingMembership.getId());
if (Objects.equals(existingMembership.getLatestResultId(), row.getId())
&& (dailyFile == null || currentFileOwnsMembership)) {
attachCanonicalResult(row, existingMembership);
} else if (!blank(row.getResultFileUrl())) {
row.setResultFileUrl(null);
row.setResultFileSize(null);
row.setResultContentType(null);
fileResultMapper.updateById(row);
}
return new DailyAggregationResult(List.of());
}
if (dailyFile != null && dailyFileService.containsResult(dailyFile.getId(), row.getId())) {
if (Objects.equals(dailyFile.getLatestResultId(), row.getId())) {
attachCanonicalResult(row, dailyFile);
} else if (!blank(row.getResultFileUrl())) {
row.setResultFileUrl(null);
row.setResultFileSize(null);
row.setResultContentType(null);
fileResultMapper.updateById(row);
}
return new DailyAggregationResult(List.of());
}
List<ShopDataCrawlDailyFileEntity> olderFiles = dailyFileService.findOlder(userId, shopKeyHash, businessDate);
Set<String> obsoleteObjectKeys = new HashSet<>();
collectObjectKey(obsoleteObjectKeys, dailyFile == null ? null : dailyFile.getResultFileUrl());
for (ShopDataCrawlDailyFileEntity older : olderFiles) {
collectObjectKey(obsoleteObjectKeys, older.getResultFileUrl());
}
int addedRowCount = excelAssemblyService.countRows(List.of(snapshot));
String filename = dailyFile != null && !blank(dailyFile.getResultFilename())
? dailyFile.getResultFilename()
: buildTaskWorkbookFilename(task);
File workRoot = FileUtil.mkdir(FileUtil.file(
System.getProperty("java.io.tmpdir"),
"shop-data-crawl-result",
String.valueOf(task.getId()),
"daily-" + UUID.randomUUID()));
File baseXlsx = FileUtil.file(workRoot, "base.xlsx");
File outputXlsx = FileUtil.file(workRoot, filename);
String objectKey = null;
try {
if (dailyFile != null && !blank(dailyFile.getResultFileUrl())) {
try {
Files.write(baseXlsx.toPath(), ossStorageService.readObjectBytes(dailyFile.getResultFileUrl()));
} catch (Exception ex) {
throw new BusinessException("读取当天累计文件失败: " + safeMessage(ex));
}
if (addedRowCount > 0) {
excelAssemblyService.appendWorkbook(baseXlsx, outputXlsx, List.of(snapshot));
} else {
try {
Files.copy(baseXlsx.toPath(), outputXlsx.toPath());
} catch (Exception ex) {
throw new BusinessException("复制当天累计文件失败: " + safeMessage(ex));
}
}
} else {
excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot));
}
if (dailyFile != null && addedRowCount == 0) {
objectKey = dailyFile.getResultFileUrl();
} else {
objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
uploadedObjectKeys.add(objectKey);
}
List<FileResultEntity> shopRows = findShopResultRows(userId, row);
clearShopResultPointers(shopRows, row.getId());
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(outputXlsx.length());
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(dailyFile == null
? addedRowCount
: Math.max(0, Objects.requireNonNullElse(dailyFile.getRowCount(), 0)) + addedRowCount);
fileResultMapper.updateById(row);
LocalDateTime now = dailyFileService.currentBusinessDateTime();
if (dailyFile == null) {
dailyFile = new ShopDataCrawlDailyFileEntity();
dailyFile.setUserId(userId);
dailyFile.setShopKeyHash(shopKeyHash);
dailyFile.setShopKey(shopKey);
dailyFile.setBusinessDate(businessDate);
dailyFile.setVersion(1L);
dailyFile.setCreatedAt(now);
} else {
dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L);
}
dailyFile.setLatestTaskId(row.getTaskId());
dailyFile.setLatestResultId(row.getId());
dailyFile.setResultFilename(filename);
dailyFile.setResultFileUrl(objectKey);
dailyFile.setResultFileSize(row.getResultFileSize());
dailyFile.setResultContentType(CONTENT_TYPE_XLSX);
dailyFile.setRowCount(row.getRowCount());
dailyFile.setLastSuccessAt(now);
dailyFile.setUpdatedAt(now);
if (dailyFile.getId() == null) {
dailyFileService.insert(dailyFile);
} else {
dailyFileService.update(dailyFile);
}
if (!dailyFileService.addMember(dailyFile.getId(), row.getTaskId(), row.getId())) {
throw new BusinessException("结果已归档,请重试文件任务");
}
for (ShopDataCrawlDailyFileEntity older : olderFiles) {
dailyFileService.deleteDailyFile(older.getId());
}
obsoleteObjectKeys.remove(objectKey);
return new DailyAggregationResult(new ArrayList<>(obsoleteObjectKeys));
} finally {
FileUtil.del(baseXlsx);
FileUtil.del(outputXlsx);
FileUtil.del(workRoot);
}
} finally {
releaseDailyLockAfterTransaction(lock);
}
}
private ShopDataCrawlDailyFileEntity findExistingDailyMembership(Long resultId) {
List<ShopDataCrawlDailyMemberEntity> members = dailyFileService.findMembersByResultId(resultId);
for (ShopDataCrawlDailyMemberEntity member : members == null ? List.<ShopDataCrawlDailyMemberEntity>of() : members) {
if (member == null) {
continue;
}
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findById(member.getDailyFileId());
if (dailyFile != null) {
return dailyFile;
}
}
return null;
}
private void attachCanonicalResult(FileResultEntity row, ShopDataCrawlDailyFileEntity dailyFile) {
if (row == null || dailyFile == null) {
return;
}
row.setResultFilename(dailyFile.getResultFilename());
row.setResultFileUrl(dailyFile.getResultFileUrl());
row.setResultFileSize(dailyFile.getResultFileSize());
row.setResultContentType(dailyFile.getResultContentType());
row.setRowCount(dailyFile.getRowCount());
fileResultMapper.updateById(row);
}
private List<FileResultEntity> findShopResultRows(Long userId, FileResultEntity sourceRow) {
if (userId == null || sourceRow == null) {
return List.of();
}
String shopId = trimToNull(sourceRow.getSourceFileUrl());
String shopName = trimToNull(sourceRow.getSourceFilename());
LambdaQueryWrapper<FileResultEntity> wrapper = new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.and(owner -> owner.eq(FileResultEntity::getUserId, userId)
.or().isNull(FileResultEntity::getUserId));
if (shopId != null) {
wrapper.eq(FileResultEntity::getSourceFileUrl, shopId);
} else if (shopName != null) {
wrapper.eq(FileResultEntity::getSourceFilename, shopName)
.apply("TRIM(COALESCE(source_file_url, '')) = ''");
} else {
return List.of();
}
List<FileResultEntity> candidates = fileResultMapper.selectList(wrapper);
if (candidates == null || candidates.isEmpty()) {
return List.of();
}
Map<Long, FileTaskEntity> legacyTaskOwners = loadTaskMapByIds(candidates.stream()
.filter(candidate -> candidate != null && candidate.getUserId() == null)
.map(FileResultEntity::getTaskId)
.filter(Objects::nonNull)
.distinct()
.toList());
return candidates.stream()
.filter(Objects::nonNull)
.filter(candidate -> Objects.equals(userId, candidate.getUserId())
|| Objects.equals(userId,
legacyTaskOwners.get(candidate.getTaskId()) == null
? null
: legacyTaskOwners.get(candidate.getTaskId()).getUserId()))
.toList();
}
private void clearShopResultPointers(List<FileResultEntity> rows, Long keepResultId) {
if (rows == null) {
return;
}
for (FileResultEntity candidate : rows) {
if (candidate == null || Objects.equals(candidate.getId(), keepResultId)
|| blank(candidate.getResultFileUrl())) {
continue;
}
candidate.setResultFileUrl(null);
candidate.setResultFileSize(null);
candidate.setResultContentType(null);
fileResultMapper.updateById(candidate);
}
}
private void collectObjectKey(Set<String> target, String value) {
if (target == null || blank(value)) {
return;
}
target.add(value);
}
private void registerDailyObjectLifecycle(List<String> uploadedObjectKeys, List<String> obsoleteObjectKeys) {
Set<String> uploaded = uploadedObjectKeys == null ? Set.of() : new HashSet<>(uploadedObjectKeys);
Set<String> obsolete = obsoleteObjectKeys == null ? new HashSet<>() : new HashSet<>(obsoleteObjectKeys);
obsolete.removeAll(uploaded);
if (!TransactionSynchronizationManager.isSynchronizationActive()) {
obsolete.forEach(this::deleteObjectQuietly);
return;
}
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
obsolete.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly);
}
@Override
public void afterCompletion(int status) {
if (status != STATUS_COMMITTED) {
uploaded.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly);
}
}
});
}
private void registerRollbackObjectCleanup(List<String> uploadedObjectKeys) {
if (uploadedObjectKeys == null || uploadedObjectKeys.isEmpty()) {
return;
}
Set<String> uploaded = new HashSet<>(uploadedObjectKeys);
if (!TransactionSynchronizationManager.isSynchronizationActive()) {
uploaded.forEach(this::deleteObjectQuietly);
return;
}
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCompletion(int status) {
if (status != STATUS_COMMITTED) {
uploaded.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly);
}
}
});
}
private void registerUploadedObjectRollback(List<String> uploadedObjectKeys) {
if (uploadedObjectKeys == null || uploadedObjectKeys.isEmpty()
|| !TransactionSynchronizationManager.isSynchronizationActive()) {
return;
}
Set<String> uploaded = new HashSet<>(uploadedObjectKeys);
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCompletion(int status) {
if (status != STATUS_COMMITTED) {
uploaded.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly);
}
}
});
}
private void deleteObjectQuietly(String objectKey) {
if (blank(objectKey)) {
return;
}
try {
deleteResultObjectNowIfUnreferenced(objectKey);
} catch (Exception ex) {
log.warn("[shop-data-crawl] daily object cleanup failed object={} msg={}", objectKey, safeMessage(ex));
}
}
private DailyLockSet acquireDailyLocks(Long fallbackUserId, List<FileResultEntity> rows) {
Map<String, DailyLockRequest> requests = new TreeMap<>();
if (rows != null) {
for (FileResultEntity row : rows) {
if (row == null) {
continue;
}
Long userId = row.getUserId() != null ? row.getUserId() : fallbackUserId;
String shopKey = dailyFileService.shopKey(row);
if (userId == null || userId <= 0 || blank(shopKey)) {
continue;
}
requests.putIfAbsent(userId + "|" + shopKey, new DailyLockRequest(userId, shopKey));
}
}
if (requests.isEmpty()) {
return new DailyLockSet(List.of());
}
List<TaskDistributedLockService.LockHandle> handles = new ArrayList<>();
try {
for (DailyLockRequest request : requests.values()) {
TaskDistributedLockService.LockHandle handle = dailyFileService.acquireLock(
request.userId(), request.shopKey());
if (handle == null) {
throw new BusinessException("店铺累计文件正在处理中,请稍后重试");
}
handles.add(handle);
}
return new DailyLockSet(handles);
} catch (RuntimeException ex) {
handles.forEach(TaskDistributedLockService.LockHandle::close);
throw ex;
}
}
private void releaseDailyLockAfterTransaction(TaskDistributedLockService.LockHandle lock) {
if (lock == null) {
return;
}
if (!TransactionSynchronizationManager.isSynchronizationActive()) {
lock.close();
return;
}
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCompletion(int status) {
lock.close();
}
});
}
private record DailyLockRequest(Long userId, String shopKey) {
}
private record DailyMemberData(ShopDataCrawlDailyMemberEntity member,
FileResultEntity result,
ShopDataCrawlResultItemVo snapshot) {
}
private record DailyDeletionResult(List<String> obsoleteObjectKeys,
List<String> uploadedObjectKeys) {
}
private static final class DailyLockSet implements AutoCloseable {
private final List<TaskDistributedLockService.LockHandle> handles;
private boolean closed;
private DailyLockSet(List<TaskDistributedLockService.LockHandle> handles) {
this.handles = handles == null ? List.of() : handles;
}
@Override
public void close() {
if (closed) {
return;
}
closed = true;
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCompletion(int status) {
closeNow();
}
});
} else {
closeNow();
}
}
private void closeNow() {
for (int index = handles.size() - 1; index >= 0; index--) {
handles.get(index).close();
}
}
}
private record DailyAggregationResult(List<String> obsoleteObjectKeys) {
}
public void cleanupResultFileJob(TaskFileJobEntity job) {
@@ -1813,9 +2271,29 @@ public class ShopDataCrawlTaskService {
void deleteResultObjectIfUnreferenced(String resultFileUrl) {
if (blank(resultFileUrl)) return;
if (!isResultObjectUnreferenced(resultFileUrl)) return;
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deleteObjectQuietly(resultFileUrl);
}
});
} else {
ossStorageService.deleteObject(resultFileUrl);
}
}
private void deleteResultObjectNowIfUnreferenced(String resultFileUrl) {
if (blank(resultFileUrl) || !isResultObjectUnreferenced(resultFileUrl)) return;
ossStorageService.deleteObject(resultFileUrl);
}
private boolean isResultObjectUnreferenced(String resultFileUrl) {
Long references = fileResultMapper.selectCount(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getResultFileUrl, resultFileUrl));
if (references == null || references == 0L) ossStorageService.deleteObject(resultFileUrl);
long dailyReferences = dailyFileService.countObjectReferences(resultFileUrl);
return (references == null || references == 0L) && dailyReferences == 0L;
}
public void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) {
@@ -78,4 +78,28 @@ public interface ShopManageGroupMapper extends BaseMapper<ShopManageGroupEntity>
ORDER BY gm.user_id ASC
""")
List<Long> selectManagedMemberUserIds(@Param("operatorId") Long operatorId);
@Select("""
<script>
SELECT DISTINCT visible_user_id
FROM (
SELECT COALESCE(g.created_by_id, g.user_id) AS visible_user_id
FROM biz_shop_manage_group g
WHERE g.id IN
<foreach collection='groupIds' item='groupId' open='(' separator=',' close=')'>
#{groupId}
</foreach>
UNION
SELECT gm.user_id AS visible_user_id
FROM biz_shop_manage_group_member gm
WHERE gm.group_id IN
<foreach collection='groupIds' item='groupId' open='(' separator=',' close=')'>
#{groupId}
</foreach>
) visible_users
WHERE visible_user_id IS NOT NULL
ORDER BY visible_user_id ASC
</script>
""")
List<Long> selectUserIdsByGroupIds(@Param("groupIds") List<Long> groupIds);
}
@@ -9,6 +9,8 @@ import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper;
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMemberMapper;
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper;
import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.shopkey.model.dto.ShopManageGroupCreateRequest;
import com.nanri.aiimage.modules.shopkey.model.dto.ShopManageGroupUpdateRequest;
import com.nanri.aiimage.modules.shopkey.model.entity.QueryAsinEntity;
@@ -16,6 +18,8 @@ import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageEntity;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupMemberEntity;
import com.nanri.aiimage.modules.shopkey.model.entity.SkipPriceAsinEntity;
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import com.nanri.aiimage.modules.shopkey.model.vo.ShopManageGroupItemVo;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@@ -39,6 +43,8 @@ public class ShopManageGroupService {
private final ShopManageMapper shopManageMapper;
private final QueryAsinMapper queryAsinMapper;
private final SkipPriceAsinMapper skipPriceAsinMapper;
private final InvalidAsinDataMapper invalidAsinDataMapper;
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final AdminUserMapper adminUserMapper;
public List<ShopManageGroupItemVo> list() {
@@ -160,6 +166,16 @@ public class ShopManageGroupService {
if (queryAsinCount != null && queryAsinCount > 0) {
throw new BusinessException("该分组下存在查询 ASIN,无法删除");
}
Long invalidAsinDataCount = invalidAsinDataMapper.selectCount(new LambdaQueryWrapper<InvalidAsinDataEntity>()
.eq(InvalidAsinDataEntity::getGroupId, entity.getId()));
if (invalidAsinDataCount != null && invalidAsinDataCount > 0) {
throw new BusinessException("该分组下存在品牌数据库数据,无法删除");
}
Long dedupeTotalDataCount = dedupeTotalDataMapper.selectCount(new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getGroupId, entity.getId()));
if (dedupeTotalDataCount != null && dedupeTotalDataCount > 0) {
throw new BusinessException("该分组下存在数据去重总数据,无法删除");
}
groupMemberMapper.delete(new LambdaQueryWrapper<ShopManageGroupMemberEntity>()
.eq(ShopManageGroupMemberEntity::getGroupId, entity.getId()));
groupMapper.deleteById(entity.getId());
@@ -0,0 +1,33 @@
CREATE TABLE IF NOT EXISTS biz_shop_data_crawl_daily_file (
id BIGINT NOT NULL AUTO_INCREMENT,
user_id BIGINT NOT NULL,
shop_key_hash CHAR(64) NOT NULL,
shop_key VARCHAR(1000) NOT NULL,
business_date DATE NOT NULL,
latest_task_id BIGINT NULL,
latest_result_id BIGINT NULL,
result_filename VARCHAR(255) NULL,
result_file_url VARCHAR(1000) NULL,
result_file_size BIGINT NULL,
result_content_type VARCHAR(128) NULL,
row_count INT NULL,
version BIGINT NOT NULL DEFAULT 0,
last_success_at DATETIME NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uk_shop_data_crawl_daily_file (user_id, shop_key_hash, business_date),
KEY idx_shop_data_crawl_daily_result (latest_result_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS biz_shop_data_crawl_daily_member (
id BIGINT NOT NULL AUTO_INCREMENT,
daily_file_id BIGINT NOT NULL,
task_id BIGINT NOT NULL,
result_id BIGINT NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uk_shop_data_crawl_daily_member_result (result_id),
KEY idx_shop_data_crawl_daily_member_file_result (daily_file_id, result_id),
KEY idx_shop_data_crawl_daily_member_task (task_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
@@ -0,0 +1,74 @@
SET @invalid_asin_group_id_col_exists := (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND COLUMN_NAME = 'group_id'
);
SET @sql_add_invalid_asin_group_id := IF(
@invalid_asin_group_id_col_exists = 0,
'ALTER TABLE biz_invalid_asin_data ADD COLUMN group_id BIGINT NULL AFTER brand',
'SELECT 1'
);
PREPARE stmt_add_invalid_asin_group_id FROM @sql_add_invalid_asin_group_id;
EXECUTE stmt_add_invalid_asin_group_id;
DEALLOCATE PREPARE stmt_add_invalid_asin_group_id;
SET @invalid_asin_record_source_col_exists := (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND COLUMN_NAME = 'record_source'
);
SET @sql_add_invalid_asin_record_source := IF(
@invalid_asin_record_source_col_exists = 0,
'ALTER TABLE biz_invalid_asin_data ADD COLUMN record_source VARCHAR(16) NOT NULL DEFAULT ''AUTO'' AFTER group_id',
'SELECT 1'
);
PREPARE stmt_add_invalid_asin_record_source FROM @sql_add_invalid_asin_record_source;
EXECUTE stmt_add_invalid_asin_record_source;
DEALLOCATE PREPARE stmt_add_invalid_asin_record_source;
-- Historical rows have no reliable origin marker. Treat them as automatic rows.
UPDATE biz_invalid_asin_data
SET record_source = 'AUTO'
WHERE record_source IS NULL
OR TRIM(record_source) = ''
OR UPPER(TRIM(record_source)) NOT IN ('AUTO', 'MANUAL');
SET @invalid_asin_source_group_index_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND INDEX_NAME = 'idx_invalid_asin_source_group_id'
);
SET @sql_add_invalid_asin_source_group_index := IF(
@invalid_asin_source_group_index_exists = 0,
'ALTER TABLE biz_invalid_asin_data ADD INDEX idx_invalid_asin_source_group_id (record_source, group_id, id)',
'SELECT 1'
);
PREPARE stmt_add_invalid_asin_source_group_index FROM @sql_add_invalid_asin_source_group_index;
EXECUTE stmt_add_invalid_asin_source_group_index;
DEALLOCATE PREPARE stmt_add_invalid_asin_source_group_index;
SET @invalid_asin_group_index_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND INDEX_NAME = 'idx_invalid_asin_group_id'
);
SET @sql_add_invalid_asin_group_index := IF(
@invalid_asin_group_index_exists = 0,
'ALTER TABLE biz_invalid_asin_data ADD INDEX idx_invalid_asin_group_id (group_id)',
'SELECT 1'
);
PREPARE stmt_add_invalid_asin_group_index FROM @sql_add_invalid_asin_group_index;
EXECUTE stmt_add_invalid_asin_group_index;
DEALLOCATE PREPARE stmt_add_invalid_asin_group_index;
@@ -0,0 +1,53 @@
SET @dedupe_group_id_col_exists := (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_dedupe_total_data'
AND COLUMN_NAME = 'group_id'
);
SET @sql_add_dedupe_group_id := IF(
@dedupe_group_id_col_exists = 0,
'ALTER TABLE biz_dedupe_total_data ADD COLUMN group_id BIGINT NULL AFTER data_value',
'SELECT 1'
);
PREPARE stmt_add_dedupe_group_id FROM @sql_add_dedupe_group_id;
EXECUTE stmt_add_dedupe_group_id;
DEALLOCATE PREPARE stmt_add_dedupe_group_id;
UPDATE biz_dedupe_total_data d
JOIN (
SELECT matched.id, MIN(matched.group_id) AS group_id
FROM (
SELECT d0.id, g.id AS group_id
FROM biz_dedupe_total_data d0
INNER JOIN biz_shop_manage_group g
ON COALESCE(g.created_by_id, g.user_id) = d0.uploader_user_id
UNION
SELECT d1.id, gm.group_id
FROM biz_dedupe_total_data d1
INNER JOIN biz_shop_manage_group_member gm
ON gm.user_id = d1.uploader_user_id
) matched
GROUP BY matched.id
HAVING COUNT(DISTINCT matched.group_id) = 1
) resolved ON resolved.id = d.id
SET d.group_id = resolved.group_id
WHERE d.group_id IS NULL;
SET @dedupe_group_idx_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_dedupe_total_data'
AND INDEX_NAME = 'idx_dedupe_total_data_group_id'
);
SET @sql_add_dedupe_group_idx := IF(
@dedupe_group_idx_exists = 0,
'ALTER TABLE biz_dedupe_total_data ADD INDEX idx_dedupe_total_data_group_id (group_id, id)',
'SELECT 1'
);
PREPARE stmt_add_dedupe_group_idx FROM @sql_add_dedupe_group_idx;
EXECUTE stmt_add_dedupe_group_idx;
DEALLOCATE PREPARE stmt_add_dedupe_group_idx;
@@ -13,6 +13,7 @@ import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
@@ -65,6 +66,7 @@ class DedupeTotalDataServiceTest {
void createRecordsUploaderIdentity() {
AdminUserEntity uploader = user(23L, "normal", "member-a");
when(adminUserMapper.selectById(23L)).thenReturn(uploader);
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))).thenAnswer(invocation -> {
DedupeTotalDataEntity entity = invocation.getArgument(0);
@@ -75,6 +77,7 @@ class DedupeTotalDataServiceTest {
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setId(91L);
entity.setDataValue("B012345678");
entity.setGroupId(7L);
entity.setUploaderUserId(23L);
entity.setUploaderUsername("member-a");
return entity;
@@ -82,6 +85,7 @@ class DedupeTotalDataServiceTest {
DedupeTotalDataCreateRequest request = new DedupeTotalDataCreateRequest();
request.setDataValue(" b012345678 ");
request.setGroupId(7L);
DedupeTotalDataItemVo item = service.create(request, 23L);
ArgumentCaptor<DedupeTotalDataEntity> captor = ArgumentCaptor.forClass(DedupeTotalDataEntity.class);
@@ -89,7 +93,10 @@ class DedupeTotalDataServiceTest {
assertEquals(23L, captor.getValue().getUploaderUserId());
assertEquals("member-a", captor.getValue().getUploaderUsername());
assertEquals("B012345678", captor.getValue().getDataValue());
assertEquals(7L, captor.getValue().getGroupId());
assertEquals("member-a", item.getUsername());
assertEquals(7L, item.getGroupId());
assertEquals("group-7", item.getGroupName());
}
@Test
@@ -117,6 +124,39 @@ class DedupeTotalDataServiceTest {
verify(dedupeTotalDataMapper, never()).deleteById(any(Long.class));
}
@Test
void memberCanDeleteDataInAccessibleGroup() {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L));
when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L));
when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of());
DedupeTotalDataEntity entity = data(91L, 99L);
entity.setGroupId(7L);
when(dedupeTotalDataMapper.selectById(91L)).thenReturn(entity);
service.delete(91L, 23L);
verify(dedupeTotalDataMapper).deleteById(91L);
}
@Test
void memberCannotDeleteDataInAnotherGroup() {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L));
when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L));
when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of());
DedupeTotalDataEntity entity = data(91L, 23L);
entity.setGroupId(8L);
when(dedupeTotalDataMapper.selectById(91L)).thenReturn(entity);
ResponseStatusException exception = assertThrows(
ResponseStatusException.class,
() -> service.delete(91L, 23L));
assertEquals(403, exception.getStatusCode().value());
verify(dedupeTotalDataMapper, never()).deleteById(any(Long.class));
}
@Test
void superAdminCanDeleteAnyUsersData() {
when(adminUserMapper.selectById(1L)).thenReturn(user(1L, "super_admin", "root"));
@@ -131,10 +171,11 @@ class DedupeTotalDataServiceTest {
@Test
void duplicateExcelValueKeepsOriginalUploader() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(data(91L, 99L));
MockMultipartFile file = asinWorkbook("B012345678");
var result = service.importFromExcel(file, 23L);
var result = service.importFromExcel(file, 7L, 23L);
assertEquals(0, result.getInsertedCount());
assertEquals(1, result.getSkippedCount());
@@ -144,12 +185,13 @@ class DedupeTotalDataServiceTest {
@Test
void concurrentDuplicateDuringImportIsSkipped() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class)))
.thenThrow(new DuplicateKeyException("duplicate"));
var result = service.importFromExcel(asinWorkbook("b012345678"), 23L);
var result = service.importFromExcel(asinWorkbook("b012345678"), 7L, 23L);
assertEquals(0, result.getInsertedCount());
assertEquals(1, result.getSkippedCount());
@@ -172,6 +214,18 @@ class DedupeTotalDataServiceTest {
verify(dedupeTotalDataMapper).selectList(any());
}
@Test
void pageSupportsUngroupedHistoricalRows() {
when(adminUserMapper.selectById(1L)).thenReturn(user(1L, "super_admin", "root"));
when(dedupeTotalDataMapper.selectCount(any())).thenReturn(1L);
when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of(data(91L, 23L)));
DedupeTotalDataPageVo page = service.page(1, 15, "", "", null, null, 1L);
assertEquals(1L, page.getTotal());
assertEquals("", page.getItems().getFirst().getGroupName());
}
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void pageUsesInclusiveDateRange() {
@@ -245,7 +299,8 @@ class DedupeTotalDataServiceTest {
assertEquals("ASIN值", sheet.getRow(0).getCell(1).getStringCellValue());
assertEquals("B012345678", sheet.getRow(1).getCell(1).getStringCellValue());
assertEquals("member-a", sheet.getRow(1).getCell(2).getStringCellValue());
assertEquals("2026-07-20 12:30:00", sheet.getRow(1).getCell(3).getStringCellValue());
assertEquals("", sheet.getRow(1).getCell(3).getStringCellValue());
assertEquals("2026-07-20 12:30:00", sheet.getRow(1).getCell(4).getStringCellValue());
}
verify(shopManageGroupMapper).selectManagedMemberUserIds(10L);
@@ -262,27 +317,70 @@ class DedupeTotalDataServiceTest {
verify(dedupeTotalDataMapper, never()).selectList(any());
}
@Test
void importProgressAllowsAccessibleGroup() {
DedupeTotalDataImportProgressVo progress = registerImportProgress("same-group", 24L, 7L);
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L));
when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L, 24L));
when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of());
assertEquals(progress, service.getImportProgress("same-group", 23L));
}
@Test
void importProgressRejectsVisibleOwnerFromAnotherGroup() {
registerImportProgress("other-group", 24L, 8L);
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L));
when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L, 24L));
when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of());
ResponseStatusException exception = assertThrows(
ResponseStatusException.class,
() -> service.getImportProgress("other-group", 23L));
assertEquals(403, exception.getStatusCode().value());
}
@Test
@SuppressWarnings("unchecked")
void expiredCompletedProgressIsRemovedOnNextLookup() {
Map<String, DedupeTotalDataImportProgressVo> progressMap =
(Map<String, DedupeTotalDataImportProgressVo>) ReflectionTestUtils.getField(service, "importProgressMap");
Map<String, Long> ownerMap = (Map<String, Long>) ReflectionTestUtils.getField(service, "importOwnerMap");
Map<String, Long> groupMap = (Map<String, Long>) ReflectionTestUtils.getField(service, "importGroupMap");
Map<String, Long> completedAtMap =
(Map<String, Long>) ReflectionTestUtils.getField(service, "importCompletedAtMap");
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("success");
progressMap.put("expired", progress);
ownerMap.put("expired", 23L);
groupMap.put("expired", 7L);
completedAtMap.put("expired", System.currentTimeMillis() - (2 * 60 * 60 * 1000L));
assertThrows(BusinessException.class, () -> service.getImportProgress("expired", 23L));
assertFalse(progressMap.containsKey("expired"));
assertFalse(ownerMap.containsKey("expired"));
assertFalse(groupMap.containsKey("expired"));
assertFalse(completedAtMap.containsKey("expired"));
}
@SuppressWarnings("unchecked")
private DedupeTotalDataImportProgressVo registerImportProgress(String importId, Long ownerId, Long groupId) {
Map<String, DedupeTotalDataImportProgressVo> progressMap =
(Map<String, DedupeTotalDataImportProgressVo>) ReflectionTestUtils.getField(service, "importProgressMap");
Map<String, Long> ownerMap = (Map<String, Long>) ReflectionTestUtils.getField(service, "importOwnerMap");
Map<String, Long> groupMap = (Map<String, Long>) ReflectionTestUtils.getField(service, "importGroupMap");
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("running");
progressMap.put(importId, progress);
ownerMap.put(importId, ownerId);
groupMap.put(importId, groupId);
return progress;
}
private AdminUserEntity user(Long id, String role, String username) {
AdminUserEntity user = new AdminUserEntity();
user.setId(id);
@@ -301,6 +399,21 @@ class DedupeTotalDataServiceTest {
return entity;
}
private void stubWritableGroup(Long operatorId, Long groupId) {
when(shopManageGroupMapper.selectAccessibleGroupIds(operatorId)).thenReturn(List.of(groupId));
when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(groupId))).thenReturn(List.of(operatorId));
when(shopManageGroupMapper.selectManagedMemberUserIds(operatorId)).thenReturn(List.of());
when(shopManageGroupMapper.selectById(groupId)).thenReturn(group(groupId));
}
private ShopManageGroupEntity group(Long id) {
ShopManageGroupEntity group = new ShopManageGroupEntity();
group.setId(id);
group.setGroupName("group-" + id);
group.setCreatedById(23L);
return group;
}
private MockMultipartFile asinWorkbook(String asin) throws Exception {
try (XSSFWorkbook workbook = new XSSFWorkbook();
ByteArrayOutputStream output = new ByteArrayOutputStream()) {
@@ -0,0 +1,108 @@
package com.nanri.aiimage.modules.invalidasin.controller;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo;
import com.nanri.aiimage.modules.invalidasin.service.InvalidAsinDataService;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class InvalidAsinDataControllerTest {
@Mock
private InvalidAsinDataService invalidAsinDataService;
@Mock
private AdminAuthSupport adminAuthSupport;
@Mock
private PermissionMenuService permissionMenuService;
@InjectMocks
private InvalidAsinDataController controller;
@BeforeEach
void setUp() {
ReflectionTestUtils.setField(controller, "internalToken", "test-internal-token");
}
@Test
void trustedFlaskProxyUsesTheDatabaseUserInsteadOfRequestRole() {
MockHttpServletRequest request = new MockHttpServletRequest();
request.addHeader("X-Internal-Token", "test-internal-token");
request.addParameter("operatorId", "7");
request.addParameter("superAdmin", "true");
AdminUserEntity operator = user(7L, "normal");
when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录"));
when(permissionMenuService.requireUserOperator(7L)).thenReturn(operator);
when(adminAuthSupport.currentRole(operator)).thenReturn(null);
when(permissionMenuService.getUserColumnPermissions(7L, "admin"))
.thenReturn(List.of(invalidAsinDataPermission()));
when(invalidAsinDataService.page(1L, 15L, "", 3L, 7L, false))
.thenReturn(new InvalidAsinDataPageVo());
controller.page(1L, 15L, "", 3L, request);
verify(invalidAsinDataService).page(1L, 15L, "", 3L, 7L, false);
}
@Test
void directRequestCannotForgeSuperAdminWithoutTrustedAuthentication() {
MockHttpServletRequest request = new MockHttpServletRequest();
request.addParameter("operatorId", "1");
request.addParameter("superAdmin", "true");
when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录"));
assertThrows(BusinessException.class, () -> controller.page(1L, 15L, "", 3L, request));
verifyNoInteractions(invalidAsinDataService);
verify(permissionMenuService, never()).requireUserOperator(any());
}
@Test
void authenticatedSuperAdminRoleComesFromTheServerUser() {
MockHttpServletRequest request = new MockHttpServletRequest();
request.addParameter("superAdmin", "false");
AdminUserEntity operator = user(1L, "super_admin");
when(adminAuthSupport.requireUser(request)).thenReturn(operator);
when(adminAuthSupport.currentRole(operator)).thenReturn("super_admin");
when(invalidAsinDataService.page(1L, 15L, "", 3L, 1L, true))
.thenReturn(new InvalidAsinDataPageVo());
controller.page(1L, 15L, "", 3L, request);
verify(invalidAsinDataService).page(1L, 15L, "", 3L, 1L, true);
verify(permissionMenuService, never()).getUserColumnPermissions(eq(1L), eq("admin"));
}
private AdminUserEntity user(Long id, String role) {
AdminUserEntity user = new AdminUserEntity();
user.setId(id);
user.setRole(role);
return user;
}
private PermissionMenuItemVo invalidAsinDataPermission() {
PermissionMenuItemVo permission = new PermissionMenuItemVo();
permission.setColumnKey("admin_invalid_asin_data");
permission.setRoutePath("invalid-asin-data");
return permission;
}
}
@@ -0,0 +1,250 @@
package com.nanri.aiimage.modules.invalidasin.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataCreateRequest;
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequest;
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo;
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
import com.nanri.aiimage.modules.shopkey.service.ShopManageGroupService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.web.server.ResponseStatusException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class InvalidAsinDataServiceTest {
@Mock
private InvalidAsinDataMapper invalidAsinDataMapper;
@Mock
private ShopManageGroupService shopManageGroupService;
@InjectMocks
private InvalidAsinDataService service;
@Test
void createManualRecordUsesAccessibleGroup() {
InvalidAsinDataCreateRequest request = new InvalidAsinDataCreateRequest();
request.setDataValue("B012345678");
request.setBrand("Acme");
request.setGroupId(9L);
ShopManageGroupEntity group = group(9L, "group-a");
InvalidAsinDataEntity saved = data(91L, "MANUAL", 9L);
when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(12L, 9L));
when(shopManageGroupService.getAccessibleById(9L, 7L, false)).thenReturn(group);
when(invalidAsinDataMapper.selectOne(any())).thenReturn(null);
when(invalidAsinDataMapper.insert(any(InvalidAsinDataEntity.class))).thenAnswer(invocation -> {
invocation.getArgument(0, InvalidAsinDataEntity.class).setId(91L);
return 1;
});
when(invalidAsinDataMapper.selectById(91L)).thenReturn(saved);
InvalidAsinDataItemVo item = service.create(request, 7L, false);
ArgumentCaptor<InvalidAsinDataEntity> captor = ArgumentCaptor.forClass(InvalidAsinDataEntity.class);
verify(invalidAsinDataMapper).insert(captor.capture());
assertEquals("B012345678", captor.getValue().getDataValue());
assertEquals("acme", captor.getValue().getBrand());
assertEquals(9L, captor.getValue().getGroupId());
assertEquals("MANUAL", captor.getValue().getRecordSource());
assertEquals("group-a", item.getGroupName());
assertEquals("MANUAL", item.getRecordSource());
}
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void normalUserPageOnlyQueriesManualRowsInAccessibleGroups() {
InvalidAsinDataEntity manual = data(91L, "MANUAL", 9L);
when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L));
when(invalidAsinDataMapper.selectCount(any())).thenReturn(1L);
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(manual));
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of(9L, "group-a"));
InvalidAsinDataPageVo page = service.page(1, 15, "", 10L, 7L, false);
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
verify(invalidAsinDataMapper).selectCount(captor.capture());
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
InvalidAsinDataEntity.class);
String sql = captor.getValue().getSqlSegment();
assertTrue(sql.contains("record_source"));
assertTrue(sql.contains("group_id"));
assertTrue(captor.getValue().getParamNameValuePairs().containsValue("MANUAL"));
assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L));
assertFalse(captor.getValue().getParamNameValuePairs().containsValue(10L));
assertEquals("group-a", page.getItems().getFirst().getGroupName());
}
@Test
void superAdminPageHandlesRowsWithoutResolvableGroup() {
InvalidAsinDataEntity automatic = data(92L, "AUTO", null);
InvalidAsinDataEntity orphan = data(93L, "MANUAL", 999L);
when(invalidAsinDataMapper.selectCount(any())).thenReturn(1L);
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(automatic, orphan));
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of());
InvalidAsinDataPageVo page = service.page(1, 15, "", null, 1L, true);
assertEquals(2, page.getItems().size());
assertEquals("AUTO", page.getItems().getFirst().getRecordSource());
assertEquals("", page.getItems().getFirst().getGroupName());
assertEquals("", page.getItems().get(1).getGroupName());
}
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void superAdminPageFiltersByRequestedGroup() {
when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L);
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of());
service.page(1, 15, "", 9L, 1L, true);
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
verify(invalidAsinDataMapper).selectCount(captor.capture());
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
InvalidAsinDataEntity.class);
assertTrue(captor.getValue().getSqlSegment().contains("group_id"));
assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L));
}
@Test
void normalUserCannotDeleteAutoRecord() {
when(invalidAsinDataMapper.selectById(91L)).thenReturn(data(91L, "AUTO", null));
ResponseStatusException exception = assertThrows(
ResponseStatusException.class,
() -> service.delete(91L, 7L, false));
assertEquals(403, exception.getStatusCode().value());
verify(invalidAsinDataMapper, never()).deleteById(any(Long.class));
verifyNoInteractions(shopManageGroupService);
}
@Test
void normalUserCannotUpdateIntoAnotherAccessibleGroup() {
InvalidAsinDataEntity record = data(91L, "MANUAL", 9L);
InvalidAsinDataUpdateRequest request = new InvalidAsinDataUpdateRequest();
request.setDataValue("B012345678");
request.setBrand("New Brand");
request.setGroupId(10L);
when(invalidAsinDataMapper.selectById(91L)).thenReturn(record);
when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L));
when(shopManageGroupService.getAccessibleById(9L, 7L, false)).thenReturn(group(9L, "group-a"));
ResponseStatusException exception = assertThrows(
ResponseStatusException.class,
() -> service.update(91L, request, 7L, false));
assertEquals(403, exception.getStatusCode().value());
assertEquals(9L, record.getGroupId());
assertEquals("acme", record.getBrand());
verify(invalidAsinDataMapper, never()).updateById(any(InvalidAsinDataEntity.class));
verify(shopManageGroupService, never()).getAccessibleById(10L, 7L, false);
}
@Test
void normalUserCannotCreateInAnotherAccessibleGroup() {
InvalidAsinDataCreateRequest request = new InvalidAsinDataCreateRequest();
request.setDataValue("B012345678");
request.setBrand("Acme");
request.setGroupId(10L);
when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L));
ResponseStatusException exception = assertThrows(
ResponseStatusException.class,
() -> service.create(request, 7L, false));
assertEquals(403, exception.getStatusCode().value());
verify(invalidAsinDataMapper, never()).insert(any(InvalidAsinDataEntity.class));
}
@Test
void normalUserCannotDeleteRecordOutsideFixedGroup() {
InvalidAsinDataEntity record = data(91L, "MANUAL", 10L);
when(invalidAsinDataMapper.selectById(91L)).thenReturn(record);
when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L));
ResponseStatusException exception = assertThrows(
ResponseStatusException.class,
() -> service.delete(91L, 7L, false));
assertEquals(403, exception.getStatusCode().value());
verify(invalidAsinDataMapper, never()).deleteById(any(Long.class));
verify(shopManageGroupService, never()).getAccessibleById(any(Long.class), any(Long.class), org.mockito.ArgumentMatchers.eq(false));
}
@Test
void normalUserCreateRequiresAtLeastOneAccessibleGroup() {
InvalidAsinDataCreateRequest request = new InvalidAsinDataCreateRequest();
request.setDataValue("B012345678");
request.setBrand("Acme");
request.setGroupId(9L);
when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of());
assertThrows(BusinessException.class, () -> service.create(request, 7L, false));
verify(invalidAsinDataMapper, never()).insert(any(InvalidAsinDataEntity.class));
verify(shopManageGroupService, never()).getAccessibleById(any(Long.class), any(Long.class), org.mockito.ArgumentMatchers.eq(false));
}
@Test
void superAdminCanUpdateAutoRecordWithoutGroup() {
InvalidAsinDataEntity record = data(91L, "AUTO", null);
InvalidAsinDataUpdateRequest request = new InvalidAsinDataUpdateRequest();
request.setDataValue("B012345678");
request.setBrand("New Brand");
when(invalidAsinDataMapper.selectById(91L)).thenReturn(record);
when(invalidAsinDataMapper.selectOne(any())).thenReturn(null);
InvalidAsinDataItemVo item = service.update(91L, request, 1L, true);
assertEquals("AUTO", item.getRecordSource());
assertNull(record.getGroupId());
verify(invalidAsinDataMapper).updateById(record);
verifyNoInteractions(shopManageGroupService);
}
private ShopManageGroupEntity group(Long id, String groupName) {
ShopManageGroupEntity group = new ShopManageGroupEntity();
group.setId(id);
group.setGroupName(groupName);
return group;
}
private InvalidAsinDataEntity data(Long id, String source, Long groupId) {
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
entity.setId(id);
entity.setDataValue("B012345678");
entity.setBrand("acme");
entity.setGroupId(groupId);
entity.setRecordSource(source);
return entity;
}
}
@@ -72,6 +72,55 @@ class ShopDataCrawlExcelAssemblyServiceTest {
}
}
@Test
void appendsOnlyTheSelectedCountryAndPreservesExistingPictures() throws Exception {
ShopDataCrawlRowDto firstRow = row("2026-07-25", "B012345678");
firstRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
ShopDataCrawlRowDto secondRow = row("2026-07-26", "B099999999");
secondRow.setCommodityImage(null);
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
when(imageEmbedder.fetchAndResizeForCache(firstRow.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
File base = tempDir.resolve("base.xlsx").toFile();
File output = tempDir.resolve("daily.xlsx").toFile();
service.writeWorkbook(base, List.of(item("UK", firstRow)));
service.appendWorkbook(base, output, List.of(item("DE", secondRow)));
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue());
assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
assertEquals(1, workbook.getAllPictures().size());
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
assertEquals(0, workbook.getSheetAt(2).getLastRowNum());
}
}
private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
country.setCountry(countryCode);
country.setItems(List.of(row));
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
item.setSuccess(true);
item.setCountryResults(List.of(country));
return item;
}
private ShopDataCrawlRowDto row(String date, String asin) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setBrand("Example Brand");
row.setInventorySales("11");
row.setSalesRank("22");
row.setPageViews("33");
row.setUnitsSold("44");
row.setPrice("12.50");
row.setRecommendedOffer("12.00");
return row;
}
private byte[] jpegBytes() throws Exception {
BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB);
ByteArrayOutputStream output = new ByteArrayOutputStream();
@@ -92,6 +92,7 @@ class ShopDataCrawlTaskServiceChunkTest {
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private ShopDataCrawlDailyFileService dailyFileService;
@Spy private ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
private ShopDataCrawlTaskService service;
@@ -122,7 +123,8 @@ class ShopDataCrawlTaskServiceChunkTest {
taskChunkMapper,
taskScopeStateMapper,
transientPayloadStorageService,
instanceMetadata);
instanceMetadata,
dailyFileService);
storedChunks.clear();
storedScopes.clear();
@@ -39,6 +39,7 @@ class ShopDataCrawlTaskServiceOwnerTest {
@Mock com.nanri.aiimage.modules.task.mapper.TaskChunkMapper taskChunkMapper;
@Mock com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper taskScopeStateMapper;
@Mock com.nanri.aiimage.modules.task.service.TransientPayloadStorageService transientPayloadStorageService;
@Mock ShopDataCrawlDailyFileService dailyFileService;
@Spy private final ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks ShopDataCrawlTaskService service;
@@ -5,12 +5,18 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
@@ -20,34 +26,55 @@ import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
class ShopDataCrawlTaskServiceRetentionTest {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
private static final Long USER_ID = 7L;
private static final Long TASK_ID = 101L;
private static final Long RESULT_ID = 201L;
private static final LocalDate BUSINESS_DATE = LocalDate.of(2026, 8, 6);
private static final LocalDateTime BUSINESS_TIME = LocalDateTime.of(2026, 8, 6, 12, 0);
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), "");
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@Mock private FileTaskMapper fileTaskMapper;
@@ -66,106 +93,302 @@ class ShopDataCrawlTaskServiceRetentionTest {
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private InstanceMetadata instanceMetadata;
@Spy private final ObjectMapper objectMapper = new ObjectMapper();
@Mock private ShopDataCrawlDailyFileService dailyFileService;
@Spy private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
@InjectMocks private ShopDataCrawlTaskService service;
@Test
void keepsNewestThreePerStableShopAndFallsBackToShopName() {
LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0);
FileResultEntity shopIdOldest = result(11L, 111L, USER_ID, "shop-1", "Renamed Shop",
"result/shop-id-oldest.xlsx", 1, now.minusDays(2));
FileResultEntity shopNameOldest = result(21L, 121L, USER_ID, " ", "Fallback Shop",
"result/shop-name-oldest.xlsx", 1, now.minusDays(4));
private FileTaskEntity task;
private FileResultEntity currentRow;
private ShopDataCrawlResultItemVo snapshot;
private TaskFileJobEntity job;
List<FileResultEntity> rows = List.of(
result(13L, 113L, USER_ID, "shop-1", "Current Name", "result/13.xlsx", 1, now.minusDays(1)),
result(23L, 123L, USER_ID, null, "Fallback Shop", "result/23.xlsx", 1, now.minusDays(2)),
shopIdOldest,
result(14L, 114L, USER_ID, "shop-1", "Current Name", "result/14.xlsx", 1, now),
result(24L, 124L, USER_ID, null, "Fallback Shop", "result/24.xlsx", 1, now),
result(12L, 112L, USER_ID, "shop-1", "Old Name", "result/12.xlsx", 1, now.minusDays(2)),
shopNameOldest,
result(22L, 122L, USER_ID, null, "Fallback Shop", "result/22.xlsx", 1, now.minusDays(3)),
result(1L, 101L, USER_ID, "shop-1", "Current Name", "result/failed.xlsx", 0, now.minusDays(9)),
result(2L, 102L, USER_ID, "shop-1", "Current Name", null, 1, now.minusDays(9)),
result(3L, 103L, 99L, "shop-1", "Current Name", "result/other-user.xlsx", 1, now.minusDays(9)),
result(4L, 104L, USER_ID, "shop-2", "Current Name", "result/other-shop.xlsx", 1, now.minusDays(9)));
@BeforeEach
void setUp() {
task = task();
currentRow = result(RESULT_ID, TASK_ID, null);
snapshot = snapshot(RESULT_ID, TASK_ID);
job = new TaskFileJobEntity();
job.setTaskId(TASK_ID);
job.setResultId(RESULT_ID);
when(fileResultMapper.selectList(any())).thenReturn(rows);
when(fileResultMapper.selectById(11L)).thenReturn(shopIdOldest);
when(fileResultMapper.selectById(21L)).thenReturn(shopNameOldest);
when(fileTaskMapper.selectById(111L)).thenReturn(terminalTask(111L));
when(fileTaskMapper.selectById(121L)).thenReturn(terminalTask(121L));
when(taskDistributedLockService.acquire(MODULE_TYPE, 111L))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(taskDistributedLockService.acquire(MODULE_TYPE, 121L))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(cacheService.getTaskCacheBatch(any())).thenReturn(Map.of());
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
when(fileResultMapper.updateById(any(FileResultEntity.class))).thenReturn(1);
when(fileResultMapper.selectCount(any())).thenReturn(0L);
service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-1");
service.pruneCompletedHistoryForShop(USER_ID, "shop-name:Fallback Shop");
verify(fileResultMapper).deleteById(11L);
verify(fileResultMapper).deleteById(21L);
verify(taskFileJobService).deleteResultJobs(111L, MODULE_TYPE, 11L);
verify(taskFileJobService).deleteResultJobs(121L, MODULE_TYPE, 21L);
verify(taskResultItemService).deleteResultItem(111L, MODULE_TYPE, 11L);
verify(taskResultItemService).deleteResultItem(121L, MODULE_TYPE, 21L);
verify(ossStorageService).deleteObject("result/shop-id-oldest.xlsx");
verify(ossStorageService).deleteObject("result/shop-name-oldest.xlsx");
verify(fileResultMapper, never()).deleteById(1L);
verify(fileResultMapper, never()).deleteById(2L);
verify(fileResultMapper, never()).deleteById(3L);
verify(fileResultMapper, never()).deleteById(4L);
when(taskResultItemService.listResultSnapshots(TASK_ID, MODULE_TYPE, ShopDataCrawlResultItemVo.class))
.thenReturn(List.of(snapshot));
when(excelAssemblyService.countRows(any())).thenReturn(1);
when(dailyFileService.currentBusinessDate()).thenReturn(BUSINESS_DATE);
when(dailyFileService.currentBusinessDateTime()).thenReturn(BUSINESS_TIME);
when(dailyFileService.shopKey(any())).thenReturn("shop-id:shop-1");
when(dailyFileService.shopKeyHash(anyString())).thenReturn("hash-1");
when(dailyFileService.acquireLock(eq(USER_ID), eq("shop-id:shop-1")))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(dailyFileService.findMembersByResultId(RESULT_ID)).thenReturn(List.of());
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of());
when(dailyFileService.countObjectReferences(anyString())).thenReturn(0L);
when(dailyFileService.addMember(anyLong(), anyLong(), anyLong())).thenReturn(true);
doAnswer(invocation -> {
ShopDataCrawlDailyFileEntity entity = invocation.getArgument(0);
entity.setId(301L);
return null;
}).when(dailyFileService).insert(any());
}
@Test
void doesNotDeleteOldFileWhileOwningTaskIsStillRunning() {
LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0);
FileResultEntity oldest = result(31L, 131L, USER_ID, "shop-running", "Running Shop",
"result/running-oldest.xlsx", 1, now.minusDays(3));
when(fileResultMapper.selectList(any())).thenReturn(List.of(
result(34L, 134L, USER_ID, "shop-running", "Running Shop", "result/34.xlsx", 1, now),
result(33L, 133L, USER_ID, "shop-running", "Running Shop", "result/33.xlsx", 1, now.minusDays(1)),
result(32L, 132L, USER_ID, "shop-running", "Running Shop", "result/32.xlsx", 1, now.minusDays(2)),
oldest));
when(fileResultMapper.selectById(31L)).thenReturn(oldest);
when(fileTaskMapper.selectById(131L)).thenReturn(task(131L, "RUNNING"));
when(taskDistributedLockService.acquire(MODULE_TYPE, 131L))
void firstSuccessCreatesDailyWorkbookAndMembership() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
service.processResultFileJob(job);
verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot)));
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
assertEquals(1, currentRow.getRowCount());
ArgumentCaptor<ShopDataCrawlDailyFileEntity> captor = ArgumentCaptor.forClass(ShopDataCrawlDailyFileEntity.class);
verify(dailyFileService).insert(captor.capture());
assertEquals(BUSINESS_DATE, captor.getValue().getBusinessDate());
assertEquals(1L, captor.getValue().getVersion());
}
@Test
void sameDaySuccessAppendsAndReplacesPreviousPointer() {
FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
previous.setUserId(null);
FileTaskEntity previousTask = task();
previousTask.setId(100L);
ShopDataCrawlDailyFileEntity daily = daily("result/old.xlsx", 2);
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
when(fileTaskMapper.selectBatchIds(List.of(100L))).thenReturn(List.of(previousTask));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily);
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
TransactionSynchronizationManager.initSynchronization();
try {
service.processResultFileJob(job);
verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot)));
verify(dailyFileService).update(daily);
verify(ossStorageService, never()).deleteObject("result/old.xlsx");
assertNull(previous.getResultFileUrl());
assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
assertEquals(3, currentRow.getRowCount());
List<TransactionSynchronization> synchronizations = TransactionSynchronizationManager.getSynchronizations();
synchronizations.forEach(TransactionSynchronization::afterCommit);
verify(ossStorageService).deleteObject("result/old.xlsx");
synchronizations.forEach(synchronization ->
synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED));
} finally {
TransactionSynchronizationManager.clearSynchronization();
}
}
@Test
void repeatedResultDoesNotAppendOrUploadAgain() {
ShopDataCrawlDailyFileEntity daily = daily("result/current.xlsx", 3);
daily.setLatestResultId(RESULT_ID);
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily);
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(true);
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
assertEquals(3, currentRow.getRowCount());
}
@Test
void sameDaySuccessKeepsLegacyPointerOwnedByAnotherUser() {
FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
previous.setUserId(null);
FileTaskEntity previousTask = task();
previousTask.setId(100L);
previousTask.setUserId(8L);
ShopDataCrawlDailyFileEntity daily = daily("result/old.xlsx", 2);
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
when(fileTaskMapper.selectBatchIds(List.of(100L))).thenReturn(List.of(previousTask));
when(fileResultMapper.selectCount(any())).thenReturn(1L);
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily);
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
service.processResultFileJob(job);
assertEquals("result/old.xlsx", previous.getResultFileUrl());
assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
}
@Test
void previousDayRetryDoesNotAppendToCurrentDailyFile() {
currentRow.setResultFileUrl("result/yesterday.xlsx");
ShopDataCrawlDailyFileEntity today = daily("result/today.xlsx", 1);
today.setId(302L);
ShopDataCrawlDailyFileEntity yesterday = daily("result/yesterday.xlsx", 3);
yesterday.setBusinessDate(BUSINESS_DATE.minusDays(1));
yesterday.setLatestResultId(RESULT_ID);
ShopDataCrawlDailyMemberEntity member = member(301L, TASK_ID, RESULT_ID, BUSINESS_TIME.minusDays(1));
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(today);
when(dailyFileService.findMembersByResultId(RESULT_ID)).thenReturn(List.of(member));
when(dailyFileService.findById(301L)).thenReturn(yesterday);
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertNull(currentRow.getResultFileUrl());
}
@Test
void newDaySuccessDeletesOlderDailyFileAfterNewUpload() {
FileResultEntity previous = result(200L, 100L, "result/yesterday.xlsx");
ShopDataCrawlDailyFileEntity yesterday = daily("result/yesterday.xlsx", 4);
yesterday.setId(300L);
yesterday.setBusinessDate(BUSINESS_DATE.minusDays(1));
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx");
service.processResultFileJob(job);
verify(dailyFileService).deleteDailyFile(300L);
verify(ossStorageService).deleteObject("result/yesterday.xlsx");
assertEquals("result/today.xlsx", currentRow.getResultFileUrl());
assertNull(previous.getResultFileUrl());
}
@Test
void failedNewDayUploadKeepsOlderDailyFile() {
ShopDataCrawlDailyFileEntity yesterday = daily("result/yesterday.xlsx", 4);
yesterday.setId(300L);
yesterday.setBusinessDate(BUSINESS_DATE.minusDays(1));
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
doThrow(new IllegalStateException("upload failed"))
.when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE));
assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job));
verify(dailyFileService, never()).deleteDailyFile(anyLong());
verify(ossStorageService, never()).deleteObject("result/yesterday.xlsx");
}
@Test
void deletingOneMemberRebuildsDailyWorkbookFromRemainingResults() {
task.setStatus("SUCCESS");
currentRow.setResultFileUrl("result/old.xlsx");
FileResultEntity previous = result(200L, 100L, null);
ShopDataCrawlResultItemVo previousSnapshot = snapshot(200L, 100L);
ShopDataCrawlDailyFileEntity daily = daily("result/old.xlsx", 2);
daily.setLatestTaskId(TASK_ID);
daily.setLatestResultId(RESULT_ID);
ShopDataCrawlDailyMemberEntity previousMember = member(
301L, 100L, 200L, BUSINESS_TIME.minusMinutes(10));
ShopDataCrawlDailyMemberEntity removedMember = member(
301L, TASK_ID, RESULT_ID, BUSINESS_TIME.minusMinutes(5));
when(taskDistributedLockService.acquire(MODULE_TYPE, TASK_ID))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(currentRow);
when(fileResultMapper.selectById(200L)).thenReturn(previous);
when(fileResultMapper.selectList(any())).thenReturn(List.of());
when(dailyFileService.findByLatestResultId(RESULT_ID)).thenReturn(List.of(daily));
when(dailyFileService.findMembersByResultId(RESULT_ID)).thenReturn(List.of(removedMember));
when(dailyFileService.findById(301L)).thenReturn(daily);
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily);
when(dailyFileService.listMembers(301L)).thenReturn(List.of(removedMember, previousMember));
when(taskResultItemService.getResultSnapshot(
100L, MODULE_TYPE, 200L, ShopDataCrawlResultItemVo.class)).thenReturn(previousSnapshot);
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/rebuilt.xlsx");
service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-running");
service.deleteHistory(RESULT_ID, USER_ID);
verify(fileResultMapper, never()).deleteById(31L);
verify(taskFileJobService, never()).deleteResultJobs(131L, MODULE_TYPE, 31L);
verify(ossStorageService, never()).deleteObject("result/running-oldest.xlsx");
verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(previousSnapshot)));
verify(dailyFileService).deleteMembersForResults(Set.of(RESULT_ID));
verify(dailyFileService).update(daily);
verify(ossStorageService).deleteObject("result/old.xlsx");
assertEquals(200L, daily.getLatestResultId());
assertEquals("result/rebuilt.xlsx", daily.getResultFileUrl());
assertEquals("result/rebuilt.xlsx", previous.getResultFileUrl());
assertEquals(1, previous.getRowCount());
}
private FileResultEntity result(Long id, Long taskId, Long userId, String shopId, String shopName,
String resultFileUrl, int success, LocalDateTime createdAt) {
FileResultEntity row = new FileResultEntity();
row.setId(id);
row.setTaskId(taskId);
row.setModuleType(MODULE_TYPE);
row.setUserId(userId);
row.setSourceFileUrl(shopId);
row.setSourceFilename(shopName);
row.setResultFileUrl(resultFileUrl);
row.setSuccess(success);
row.setCreatedAt(createdAt);
return row;
private FileTaskEntity task() {
FileTaskEntity entity = new FileTaskEntity();
entity.setId(TASK_ID);
entity.setModuleType(MODULE_TYPE);
entity.setUserId(USER_ID);
entity.setStatus("RUNNING");
entity.setTaskNo("task-101");
return entity;
}
private FileTaskEntity terminalTask(Long id) {
return task(id, "SUCCESS");
private FileResultEntity result(Long id, Long taskId, String objectKey) {
FileResultEntity entity = new FileResultEntity();
entity.setId(id);
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setUserId(USER_ID);
entity.setSourceFilename("Demo Shop");
entity.setSourceFileUrl("shop-1");
entity.setSuccess(1);
entity.setResultFileUrl(objectKey);
entity.setCreatedAt(BUSINESS_TIME.minusMinutes(5));
return entity;
}
private FileTaskEntity task(Long id, String status) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType(MODULE_TYPE);
task.setStatus(status);
return task;
private ShopDataCrawlResultItemVo snapshot(Long resultId, Long taskId) {
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
item.setResultId(resultId);
item.setTaskId(taskId);
item.setShopName("Demo Shop");
item.setShopId("shop-1");
item.setSuccess(true);
item.setCountryResults(List.of());
return item;
}
private ShopDataCrawlDailyFileEntity daily(String objectKey, int rowCount) {
ShopDataCrawlDailyFileEntity entity = new ShopDataCrawlDailyFileEntity();
entity.setId(301L);
entity.setUserId(USER_ID);
entity.setShopKey("shop-id:shop-1");
entity.setShopKeyHash("hash-1");
entity.setBusinessDate(BUSINESS_DATE);
entity.setLatestTaskId(100L);
entity.setLatestResultId(200L);
entity.setResultFilename("daily.xlsx");
entity.setResultFileUrl(objectKey);
entity.setResultFileSize(10L);
entity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
entity.setRowCount(rowCount);
entity.setVersion(1L);
return entity;
}
private ShopDataCrawlDailyMemberEntity member(Long dailyFileId,
Long taskId,
Long resultId,
LocalDateTime createdAt) {
ShopDataCrawlDailyMemberEntity entity = new ShopDataCrawlDailyMemberEntity();
entity.setId(resultId);
entity.setDailyFileId(dailyFileId);
entity.setTaskId(taskId);
entity.setResultId(resultId);
entity.setCreatedAt(createdAt);
return entity;
}
}
+135 -66
View File
@@ -334,7 +334,8 @@ class _PermissionProxyError(Exception):
self.status = status
def _proxy_permission_java(method, path, *, params=None, json_data=None, current_row=None):
def _proxy_permission_java(
method, path, *, params=None, json_data=None, files=None, data=None, current_row=None):
"""Call Java permission APIs using either forwarded JWT or trusted Flask identity."""
proxy_params = {}
request_row = getattr(g, '_current_user_row', None) if has_request_context() else None
@@ -353,6 +354,8 @@ def _proxy_permission_java(method, path, *, params=None, json_data=None, current
path,
params=proxy_params or None,
json_data=json_data,
files=files,
data=data,
headers=headers,
)
return result, error_response, status
@@ -905,7 +908,7 @@ def list_users():
return jsonify({'success': False, 'error': '需要登录'}), 403
page = max(1, int(request.args.get('page', 1)))
_, _, denied = _ensure_backend_menu_access(
'users', 'history', 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'users', 'history', 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
@@ -1335,7 +1338,12 @@ def _parse_admin_datetime_arg(name):
raise ValueError(f'{name} 时间格式无效') from exc
_SHOP_DATA_CRAWL_ADMIN_COLUMNS = """
_SHOP_DATA_CRAWL_LATEST_TIME_SQL = (
'COALESCE(df.last_success_at, df.updated_at, t.finished_at, t.updated_at, t.created_at)'
)
_SHOP_DATA_CRAWL_ADMIN_COLUMNS = f"""
r.id AS result_id, r.task_id, r.user_id, r.source_filename AS shop_name,
r.source_file_url AS shop_id, r.result_filename, r.result_file_url,
r.result_file_size, r.result_content_type, r.row_count,
@@ -1343,6 +1351,7 @@ _SHOP_DATA_CRAWL_ADMIN_COLUMNS = """
r.created_at AS result_created_at,
t.task_no, t.status AS task_status, t.request_json, t.result_json,
t.error_message AS task_error, t.created_at, t.updated_at, t.finished_at,
{_SHOP_DATA_CRAWL_LATEST_TIME_SQL} AS latest_file_updated_at,
u.username,
(SELECT j.id FROM biz_task_file_job j
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
@@ -1446,16 +1455,18 @@ def _shop_data_crawl_admin_item(row, group_names=None):
def _shop_data_crawl_group_item(group_row, result_rows, group_names):
"""Build one shop group and cap its children to the newest three results."""
"""Build one shop group and expose its current daily workbook only."""
raw_shop_name = group_row.get('shop_name') or ''
display_shop_name = raw_shop_name or '未命名'
group_key = _shop_data_crawl_shop_key(raw_shop_name)
children = result_rows.get(group_key)
if children is None:
children = result_rows.get(str(raw_shop_name).strip(), [])
children = children[:3]
children = children[:1]
result_items = [_shop_data_crawl_admin_item(row, group_names) for row in children]
latest_created_at = group_row.get('latest_created_at')
if latest_created_at is None and children:
latest_created_at = children[0].get('latest_file_updated_at')
if latest_created_at is None and result_items:
latest_created_at = result_items[0].get('created_at')
return {
@@ -1541,6 +1552,7 @@ def list_shop_data_crawl_tasks():
grouped_from_sql = (
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql
)
@@ -1554,7 +1566,8 @@ def list_shop_data_crawl_tasks():
total = int((cur.fetchone() or {}).get('total') or 0)
cur.execute(
'SELECT ' + shop_key_sql + ' AS shop_name, MAX(t.created_at) AS latest_created_at'
'SELECT ' + shop_key_sql + ' AS shop_name, MAX('
+ _SHOP_DATA_CRAWL_LATEST_TIME_SQL + ') AS latest_created_at'
+ grouped_from_sql +
' GROUP BY ' + shop_key_sql +
' ORDER BY latest_created_at DESC, shop_name ASC LIMIT %s OFFSET %s',
@@ -1570,14 +1583,16 @@ def list_shop_data_crawl_tasks():
cur.execute(
'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS +
', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql +
' ORDER BY t.created_at DESC, r.id DESC) AS shop_row_number '
' ORDER BY ' + _SHOP_DATA_CRAWL_LATEST_TIME_SQL
+ ' DESC, r.id DESC) AS shop_row_number '
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql +
f' AND {shop_key_sql} IN ({placeholders})' +
') ranked WHERE ranked.shop_row_number <= 3 '
'ORDER BY ranked.created_at DESC, ranked.result_id DESC',
') ranked WHERE ranked.shop_row_number <= 1 '
'ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC',
tuple(params + selected_shop_names),
)
for row in cur.fetchall():
@@ -2863,6 +2878,26 @@ def delete_shop_key(item_id):
# ---------- 数据去重总数据 ----------
def _parse_positive_group_id(raw_value):
try:
group_id = int(str(raw_value or '').strip())
except (TypeError, ValueError):
return None
return group_id if group_id > 0 else None
def _format_dedupe_total_data_item(item):
return {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'group_id': item.get('groupId'),
'group_name': item.get('groupName') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
}
@admin_api.route('/dedupe-total-data')
@login_required
def list_dedupe_total_data():
@@ -2873,6 +2908,7 @@ def list_dedupe_total_data():
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
keyword = (request.args.get('keyword') or '').strip()
username = (request.args.get('username') or '').strip()
group_id = _parse_positive_group_id(request.args.get('group_id') or request.args.get('groupId'))
start_date = (request.args.get('start_date') or request.args.get('startDate') or '').strip()
end_date = (request.args.get('end_date') or request.args.get('endDate') or '').strip()
params = {
@@ -2886,22 +2922,19 @@ def list_dedupe_total_data():
params['startDate'] = start_date
if end_date:
params['endDate'] = end_date
data, error_response, status = _proxy_backend_java(
if group_id is not None:
params['groupId'] = group_id
data, error_response, status = _proxy_permission_java(
'GET',
'/api/admin/dedupe-total-data',
params=params,
current_row=current_row,
)
if error_response is not None:
return error_response, status
payload = data.get('data') or {}
items = [
{
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
}
_format_dedupe_total_data_item(item)
for item in (payload.get('items') or [])
]
return jsonify({
@@ -2929,10 +2962,18 @@ def export_dedupe_total_data():
params['startDate'] = start_date
if end_date:
params['endDate'] = end_date
group_id = _parse_positive_group_id(request.args.get('group_id') or request.args.get('groupId'))
if group_id is not None:
params['groupId'] = group_id
url = f"{backend_java_base_url}/api/admin/dedupe-total-data/export"
try:
resp = _get_backend_java_session().get(url, params=params, timeout=60)
resp = _get_backend_java_session().get(
url,
params=params,
headers={'X-Internal-Token': _resolve_internal_token()},
timeout=60,
)
except requests.RequestException:
return jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502
if resp.status_code >= 400:
@@ -2964,10 +3005,10 @@ def dedupe_total_data_import_progress(import_id):
_, current_row, denied = _ensure_dedupe_total_data_access()
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'GET',
f'/api/admin/dedupe-total-data/import/{import_id}',
params={'operatorId': current_row.get('id')},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -2995,14 +3036,18 @@ def import_dedupe_total_data():
file_storage = request.files.get('file')
if not file_storage or file_storage.filename == '':
return jsonify({'success': False, 'error': '请选择 Excel 文件'})
group_id = _parse_positive_group_id(request.form.get('group_id') or request.form.get('groupId'))
if group_id is None:
return jsonify({'success': False, 'error': '请选择分组'})
files = {
'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream')
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/dedupe-total-data/import',
files=files,
data={'operatorId': current_row.get('id')},
data={'groupId': group_id},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3020,10 +3065,10 @@ def dedupe_total_data_delete_import_progress(import_id):
_, current_row, denied = _ensure_dedupe_total_data_access()
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'GET',
f'/api/admin/dedupe-total-data/delete-import/{import_id}',
params={'operatorId': current_row.get('id')},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3051,14 +3096,18 @@ def delete_import_dedupe_total_data():
file_storage = request.files.get('file')
if not file_storage or file_storage.filename == '':
return jsonify({'success': False, 'error': '请选择 Excel 文件'})
group_id = _parse_positive_group_id(request.form.get('group_id') or request.form.get('groupId'))
if group_id is None:
return jsonify({'success': False, 'error': '请选择分组'})
files = {
'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream')
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/dedupe-total-data/delete-import',
files=files,
data={'operatorId': current_row.get('id')},
data={'groupId': group_id},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3077,12 +3126,15 @@ def create_dedupe_total_data():
if denied:
return denied
data = request.get_json() or {}
payload = {'dataValue': (data.get('data_value') or '').strip()}
result, error_response, status = _proxy_backend_java(
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'groupId': _parse_positive_group_id(data.get('group_id') or data.get('groupId')),
}
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/dedupe-total-data',
params={'operatorId': current_row.get('id')},
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3090,13 +3142,7 @@ def create_dedupe_total_data():
return jsonify({
'success': True,
'msg': result.get('message') or '创建成功',
'item': {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
},
'item': _format_dedupe_total_data_item(item),
})
@@ -3107,12 +3153,15 @@ def update_dedupe_total_data(item_id):
if denied:
return denied
data = request.get_json() or {}
payload = {'dataValue': (data.get('data_value') or '').strip()}
result, error_response, status = _proxy_backend_java(
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'groupId': _parse_positive_group_id(data.get('group_id') or data.get('groupId')),
}
result, error_response, status = _proxy_permission_java(
'PUT',
f'/api/admin/dedupe-total-data/{item_id}',
params={'operatorId': current_row.get('id')},
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3120,13 +3169,7 @@ def update_dedupe_total_data(item_id):
return jsonify({
'success': True,
'msg': result.get('message') or '更新成功',
'item': {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
},
'item': _format_dedupe_total_data_item(item),
})
@@ -3136,10 +3179,10 @@ def delete_dedupe_total_data(item_id):
_, current_row, denied = _ensure_dedupe_total_data_access()
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'DELETE',
f'/api/admin/dedupe-total-data/{item_id}',
params={'operatorId': current_row.get('id')},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3157,23 +3200,36 @@ def _format_invalid_asin_data_item(item):
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'brand': item.get('brand') or '',
'group_id': item.get('groupId'),
'group_name': item.get('groupName') or '',
'record_source': item.get('recordSource') or 'AUTO',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
}
@admin_api.route('/invalid-asin-data')
@admin_required
@login_required
def list_invalid_asin_data():
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
page = max(1, int(request.args.get('page', 1)))
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
keyword = (request.args.get('keyword') or '').strip()
data, error_response, status = _proxy_backend_java(
group_id_raw = (request.args.get('group_id') or '').strip()
group_id = int(group_id_raw) if group_id_raw.isdigit() and int(group_id_raw) > 0 else None
params = {
'page': page,
'pageSize': page_size,
'keyword': keyword,
}
if group_id is not None:
params['groupId'] = group_id
data, error_response, status = _proxy_permission_java(
'GET',
'/api/admin/invalid-asin-data',
params={'page': page, 'pageSize': page_size, 'keyword': keyword},
params=params,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3189,20 +3245,22 @@ def list_invalid_asin_data():
@admin_api.route('/invalid-asin-data', methods=['POST'])
@admin_required
@login_required
def create_invalid_asin_data():
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
data = request.get_json() or {}
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'brand': (data.get('brand') or '').strip(),
'groupId': data.get('group_id'),
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/invalid-asin-data',
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3215,20 +3273,22 @@ def create_invalid_asin_data():
@admin_api.route('/invalid-asin-data/<int:item_id>', methods=['PUT'])
@admin_required
@login_required
def update_invalid_asin_data(item_id):
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
data = request.get_json() or {}
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'brand': (data.get('brand') or '').strip(),
'groupId': data.get('group_id'),
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'PUT',
f'/api/admin/invalid-asin-data/{item_id}',
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3241,14 +3301,15 @@ def update_invalid_asin_data(item_id):
@admin_api.route('/invalid-asin-data/<int:item_id>', methods=['DELETE'])
@admin_required
@login_required
def delete_invalid_asin_data(item_id):
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'DELETE',
f'/api/admin/invalid-asin-data/{item_id}',
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3515,21 +3576,29 @@ def delete_shop_manage(item_id):
@login_required
def list_shop_manage_groups():
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
groups, error_response, status = _load_expanded_shop_manage_groups(role, current_row)
if error_response is not None:
return error_response, status
return jsonify({'success': True, 'items': [_format_shop_manage_group_item(item) for item in (groups or [])]})
items = [_format_shop_manage_group_item(item) for item in (groups or [])]
locked_group_id = None
if role != 'super_admin':
locked_group_id = next((item.get('id') for item in items if item.get('id')), None)
return jsonify({
'success': True,
'items': items,
'locked_group_id': locked_group_id,
})
@admin_api.route('/shop-manage-group', methods=['POST'])
@login_required
def create_shop_manage_group():
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
@@ -3576,7 +3645,7 @@ def create_shop_manage_group():
@login_required
def update_shop_manage_group(item_id):
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
@@ -3823,7 +3892,7 @@ def delete_skip_price_asin_country(item_id, country):
@login_required
def delete_shop_manage_group(item_id):
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
+2
View File
@@ -3,6 +3,8 @@ annotated-types==0.7.0
anyio==4.12.1
Authlib==1.6.8
blinker==1.9.0
boto3==1.43.65
botocore==1.43.65
bottle==0.13.4
certifi==2026.1.4
cffi==2.0.0
+173 -18
View File
@@ -120,7 +120,10 @@
loadDedupeTotalData(1);
loadDedupeGroupSummary();
}
else if (tabName === 'invalid-asin-data') loadInvalidAsinData(1);
else if (tabName === 'invalid-asin-data') {
loadShopManageGroups();
loadInvalidAsinData(1);
}
else if (tabName === 'shop-keys') loadShopKeys(1);
else if (tabName === 'shop-manage') loadShopManage(1);
else if (tabName === 'skip-price-asin') loadSkipPriceAsin(1);
@@ -1526,7 +1529,7 @@
});
groups.forEach(function (group) {
group.results.sort(shopDataResultSort);
group.results = group.results.slice(0, 3);
group.results = group.results.slice(0, 1);
if (!group.latest_created_at && group.results.length) {
group.latest_created_at = group.results[0].created_at || group.results[0].finished_at || '';
}
@@ -1579,7 +1582,7 @@
'<div class="image-video-card-body">' +
'<div class="image-video-card-head shop-data-group-head">' +
'<div class="shop-data-task-title"><span title="' + escapeHtml(group.shop_name || '-') + '">' + escapeHtml(group.shop_name || '-') + '</span></div>' +
'<span class="shop-data-group-meta">' + results.length + '/3结果</span>' +
'<span class="shop-data-group-meta">' + results.length + '/1当日累计文件</span>' +
'</div>' +
'<div class="image-video-card-info">' +
'<div class="image-video-info-row"><label>分组</label><span>' + escapeHtml(group.group_name || '-') + '</span></div>' +
@@ -1641,7 +1644,7 @@
var total = payload.total != null ? Number(payload.total) : shopDataTaskGroups.length;
var responsePage = payload.page || page;
var responsePageSize = payload.page_size || shopDataTaskPageSize;
document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留最新 3 份任务结果';
document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留 1 份当日累计文件';
renderShopDataTasks();
renderPagination('shopDataTaskPagination', total, responsePage, responsePageSize, loadShopDataCrawlTasks);
})
@@ -2176,9 +2179,11 @@
var q = 'page=' + (page || 1) + '&page_size=' + dedupeTotalDataPageSize;
var keyword = (document.getElementById('searchDedupeTotalData').value || '').trim();
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
var groupId = (document.getElementById('dedupeTotalDataGroupFilterId').value || '').trim();
var dateRange = getDedupeTotalDataDateRange();
if (keyword) q += '&keyword=' + encodeURIComponent(keyword);
if (username) q += '&username=' + encodeURIComponent(username);
if (groupId) q += '&group_id=' + encodeURIComponent(groupId);
if (dateRange.startDate) q += '&start_date=' + encodeURIComponent(dateRange.startDate);
if (dateRange.endDate) q += '&end_date=' + encodeURIComponent(dateRange.endDate);
return q;
@@ -2191,16 +2196,16 @@
.then(function (res) {
var tbody = document.getElementById('dedupeTotalDataListBody');
if (!res.success) {
tbody.innerHTML = '<tr><td colspan="5" class="empty-tip">加载失败: ' + escapeHtml(res.error || '') + '</td></tr>';
tbody.innerHTML = '<tr><td colspan="6" class="empty-tip">加载失败: ' + escapeHtml(res.error || '') + '</td></tr>';
return;
}
var items = res.items || [];
if (items.length === 0) {
tbody.innerHTML = '<tr><td colspan="5" class="empty-tip">暂无总数据</td></tr>';
tbody.innerHTML = '<tr><td colspan="6" class="empty-tip">暂无总数据</td></tr>';
} else {
tbody.innerHTML = items.map(function (item) {
return '<tr><td>' + escapeHtml(item.id) + '</td><td>' + escapeHtml(item.data_value || '') + '</td><td>' + escapeHtml(item.username || '') + '</td><td>' + escapeHtml(item.created_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-dedupe-total-edit="' + escapeHtml(item.id) + '" data-value="' + escapeHtml(item.data_value || '') + '">编辑</button> ' +
return '<tr><td>' + escapeHtml(item.id) + '</td><td>' + escapeHtml(item.data_value || '') + '</td><td>' + escapeHtml(item.username || '') + '</td><td>' + escapeHtml(item.group_name || '未分组') + '</td><td>' + escapeHtml(item.created_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-dedupe-total-edit="' + escapeHtml(item.id) + '" data-value="' + escapeHtml(item.data_value || '') + '" data-group-id="' + escapeHtml(item.group_id || '') + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-dedupe-total-delete="' + escapeHtml(item.id) + '" data-value="' + escapeHtml(item.data_value || '') + '">删除</button>' +
'</td></tr>';
}).join('');
@@ -2209,7 +2214,7 @@
bindDedupeTotalDataActions();
})
.catch(function () {
document.getElementById('dedupeTotalDataListBody').innerHTML = '<tr><td colspan="5" class="empty-tip">请求失败</td></tr>';
document.getElementById('dedupeTotalDataListBody').innerHTML = '<tr><td colspan="6" class="empty-tip">请求失败</td></tr>';
});
}
function bindDedupeTotalDataActions() {
@@ -2219,7 +2224,11 @@
document.getElementById('editDedupeTotalDataValue').value = (btn.dataset.value || '').replace(/&quot;/g, '"');
document.getElementById('msgEditDedupeTotalData').textContent = '';
document.getElementById('msgEditDedupeTotalData').className = 'msg';
var groupId = btn.dataset.groupId || '';
loadShopManageGroups().then(function () {
document.getElementById('editDedupeTotalDataGroupId').value = groupId;
document.getElementById('editDedupeTotalDataModal').classList.add('show');
});
};
});
document.querySelectorAll('[data-dedupe-total-delete]').forEach(function (btn) {
@@ -2238,10 +2247,12 @@
document.getElementById('btnSearchDedupeTotalData').onclick = function () { loadDedupeTotalData(1); };
document.getElementById('btnExportDedupeTotalData').onclick = function () {
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
var groupId = (document.getElementById('dedupeTotalDataGroupFilterId').value || '').trim();
var dateRange = getDedupeTotalDataDateRange();
if (!validateDedupeTotalDataDateRange()) return;
var params = [];
if (username) params.push('username=' + encodeURIComponent(username));
if (groupId) params.push('group_id=' + encodeURIComponent(groupId));
if (dateRange.startDate) params.push('start_date=' + encodeURIComponent(dateRange.startDate));
if (dateRange.endDate) params.push('end_date=' + encodeURIComponent(dateRange.endDate));
fetch('/api/admin/dedupe-total-data/export' + (params.length ? ('?' + params.join('&')) : ''))
@@ -2386,6 +2397,12 @@
msgEl.classList.add('err');
return;
}
var groupId = (document.getElementById('dedupeTotalDataImportGroupId').value || '').trim();
if (!groupId) {
msgEl.textContent = '请选择分组';
msgEl.classList.add('err');
return;
}
var file = fileInput.files[0];
var lowerName = (file.name || '').toLowerCase();
if (!(lowerName.endsWith('.xlsx') || lowerName.endsWith('.xls'))) {
@@ -2395,6 +2412,7 @@
}
var formData = new FormData();
formData.append('file', file);
formData.append('group_id', groupId);
var xhr = new XMLHttpRequest();
xhr.open('POST', '/api/admin/dedupe-total-data/import', true);
xhr.upload.onprogress = function (event) {
@@ -2449,8 +2467,15 @@
if (!confirm('确定按 Excel 中的 ASIN 批量删除匹配的总数据吗?')) {
return;
}
var groupId = (document.getElementById('dedupeTotalDataDeleteGroupId').value || '').trim();
if (!groupId) {
msgEl.textContent = '请选择分组';
msgEl.classList.add('err');
return;
}
var formData = new FormData();
formData.append('file', file);
formData.append('group_id', groupId);
var xhr = new XMLHttpRequest();
xhr.open('POST', '/api/admin/dedupe-total-data/delete-import', true);
xhr.upload.onprogress = function (event) {
@@ -2486,6 +2511,7 @@
document.getElementById('btnSaveDedupeTotalData').onclick = function () {
var itemId = document.getElementById('editDedupeTotalDataId').value;
var value = (document.getElementById('editDedupeTotalDataValue').value || '').trim();
var groupId = (document.getElementById('editDedupeTotalDataGroupId').value || '').trim();
var msgEl = document.getElementById('msgEditDedupeTotalData');
msgEl.textContent = '';
msgEl.className = 'msg';
@@ -2494,10 +2520,15 @@
msgEl.classList.add('err');
return;
}
if (!groupId) {
msgEl.textContent = '请选择分组';
msgEl.classList.add('err');
return;
}
fetch('/api/admin/dedupe-total-data/' + itemId, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ data_value: value })
body: JSON.stringify({ data_value: value, group_id: Number(groupId) })
})
.then(function (r) { return r.json(); })
.then(function (res) {
@@ -2519,7 +2550,9 @@
function buildInvalidAsinDataQuery(page) {
var q = 'page=' + (page || 1) + '&page_size=' + invalidAsinDataPageSize;
var keyword = (document.getElementById('searchInvalidAsinData').value || '').trim();
var groupId = (document.getElementById('invalidAsinDataFilterGroupId').value || '').trim();
if (keyword) q += '&keyword=' + encodeURIComponent(keyword);
if (groupId) q += '&group_id=' + encodeURIComponent(groupId);
return q;
}
function loadInvalidAsinData(page) {
@@ -2529,18 +2562,21 @@
.then(function (res) {
var tbody = document.getElementById('invalidAsinDataListBody');
if (!res.success) {
tbody.innerHTML = '<tr><td colspan="5" class="empty-tip">加载失败: ' + (res.error || '') + '</td></tr>';
tbody.innerHTML = '<tr><td colspan="7" class="empty-tip">加载失败: ' + (res.error || '') + '</td></tr>';
return;
}
var items = res.items || [];
if (items.length === 0) {
tbody.innerHTML = '<tr><td colspan="5" class="empty-tip">暂无数据</td></tr>';
tbody.innerHTML = '<tr><td colspan="7" class="empty-tip">暂无数据</td></tr>';
} else {
tbody.innerHTML = items.map(function (item) {
var dataValueAttr = (item.data_value || '').replace(/"/g, '&quot;');
var brandAttr = (item.brand || '').replace(/"/g, '&quot;');
return '<tr><td>' + item.id + '</td><td>' + (item.data_value || '') + '</td><td>' + (item.brand || '') + '</td><td>' + (item.created_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-invalid-asin-edit="' + item.id + '" data-value="' + dataValueAttr + '" data-brand="' + brandAttr + '">编辑</button> ' +
var groupId = item.group_id == null ? '' : String(item.group_id);
var source = String(item.record_source || 'AUTO').toUpperCase();
var sourceLabel = source === 'MANUAL' ? '\u624b\u52a8\u65b0\u589e' : '\u81ea\u52a8\u5bfc\u5165';
return '<tr><td>' + escapeHtml(item.id || '') + '</td><td>' + escapeHtml(item.data_value || '') + '</td><td>' + escapeHtml(item.brand || '') + '</td><td>' + escapeHtml(item.group_name || '') + '</td><td>' + sourceLabel + '</td><td>' + escapeHtml(item.created_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-invalid-asin-edit="' + item.id + '" data-value="' + dataValueAttr + '" data-brand="' + brandAttr + '" data-group-id="' + groupId + '" data-source="' + source + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-invalid-asin-delete="' + item.id + '" data-value="' + dataValueAttr + '">删除</button>' +
'</td></tr>';
}).join('');
@@ -2549,7 +2585,7 @@
bindInvalidAsinDataActions();
})
.catch(function () {
document.getElementById('invalidAsinDataListBody').innerHTML = '<tr><td colspan="5" class="empty-tip">请求失败</td></tr>';
document.getElementById('invalidAsinDataListBody').innerHTML = '<tr><td colspan="7" class="empty-tip">请求失败</td></tr>';
});
}
function bindInvalidAsinDataActions() {
@@ -2558,6 +2594,20 @@
document.getElementById('editInvalidAsinDataId').value = btn.dataset.invalidAsinEdit || '';
document.getElementById('editInvalidAsinDataValue').value = (btn.dataset.value || '').replace(/&quot;/g, '"');
document.getElementById('editInvalidAsinDataBrand').value = (btn.dataset.brand || '').replace(/&quot;/g, '"');
var source = String(btn.dataset.source || 'AUTO').toUpperCase();
var isManual = source === 'MANUAL';
document.getElementById('editInvalidAsinDataRecordSource').value = source;
document.getElementById('editInvalidAsinDataSourceLabel').value = isManual
? '\u624b\u52a8\u65b0\u589e'
: '\u81ea\u52a8\u5bfc\u5165';
document.getElementById('editInvalidAsinDataGroupForm').style.display = isManual ? '' : 'none';
if (isManual) {
loadShopManageGroups().then(function () {
document.getElementById('editInvalidAsinDataGroupSelect').value = currentUserRole === 'super_admin'
? (btn.dataset.groupId || '')
: getInvalidAsinDataLockedGroupId();
});
}
document.getElementById('msgEditInvalidAsinData').textContent = '';
document.getElementById('msgEditInvalidAsinData').className = 'msg';
document.getElementById('editInvalidAsinDataModal').classList.add('show');
@@ -2580,6 +2630,7 @@
document.getElementById('btnAddInvalidAsinData').onclick = function () {
var dataValue = (document.getElementById('invalidAsinDataValue').value || '').trim();
var brand = (document.getElementById('invalidAsinDataBrand').value || '').trim();
var groupId = (document.getElementById('invalidAsinDataGroupSelect').value || '').trim();
var msgEl = document.getElementById('msgInvalidAsinData');
msgEl.textContent = '';
msgEl.className = 'msg';
@@ -2588,10 +2639,15 @@
msgEl.classList.add('err');
return;
}
if (!groupId) {
msgEl.textContent = '请选择分组';
msgEl.classList.add('err');
return;
}
fetch('/api/admin/invalid-asin-data', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ data_value: dataValue, brand: brand })
body: JSON.stringify({ data_value: dataValue, brand: brand, group_id: Number(groupId) })
})
.then(function (r) { return r.json(); })
.then(function (res) {
@@ -2600,6 +2656,9 @@
msgEl.classList.add('ok');
document.getElementById('invalidAsinDataValue').value = '';
document.getElementById('invalidAsinDataBrand').value = '';
document.getElementById('invalidAsinDataGroupSelect').value = currentUserRole === 'super_admin'
? ''
: getInvalidAsinDataLockedGroupId();
loadInvalidAsinData(1);
} else {
msgEl.textContent = res.error || '创建失败';
@@ -2615,6 +2674,8 @@
var itemId = document.getElementById('editInvalidAsinDataId').value;
var dataValue = (document.getElementById('editInvalidAsinDataValue').value || '').trim();
var brand = (document.getElementById('editInvalidAsinDataBrand').value || '').trim();
var source = String(document.getElementById('editInvalidAsinDataRecordSource').value || 'AUTO').toUpperCase();
var groupId = (document.getElementById('editInvalidAsinDataGroupSelect').value || '').trim();
var msgEl = document.getElementById('msgEditInvalidAsinData');
msgEl.textContent = '';
msgEl.className = 'msg';
@@ -2623,10 +2684,19 @@
msgEl.classList.add('err');
return;
}
if (source === 'MANUAL' && !groupId) {
msgEl.textContent = '请选择分组';
msgEl.classList.add('err');
return;
}
fetch('/api/admin/invalid-asin-data/' + itemId, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ data_value: dataValue, brand: brand })
body: JSON.stringify({
data_value: dataValue,
brand: brand,
group_id: source === 'MANUAL' ? Number(groupId) : null
})
})
.then(function (r) { return r.json(); })
.then(function (res) {
@@ -2803,6 +2873,7 @@
var shopManagePage = 1, shopManagePageSize = 15;
var shopManageGroups = [];
var shopManageGroupsLoadedAt = 0;
var shopManageLockedGroupId = null;
var currentShopManageGroupGrantRoutes = [];
function buildShopManageQuery(page) {
@@ -2926,9 +2997,22 @@
});
}
function getInvalidAsinDataLockedGroupId() {
if (currentUserRole === 'super_admin') return '';
if (shopManageLockedGroupId) return String(shopManageLockedGroupId);
return shopManageGroups.length ? String(shopManageGroups[0].id || '') : '';
}
function refreshShopGroupSelects(selectedCreateId, selectedEditId) {
var createSel = document.getElementById('shopManageGroupSelect');
var editSel = document.getElementById('editShopManageGroupSelect');
var invalidCreateSel = document.getElementById('invalidAsinDataGroupSelect');
var invalidEditSel = document.getElementById('editInvalidAsinDataGroupSelect');
var invalidFilterSel = document.getElementById('invalidAsinDataFilterGroupId');
var dedupeImportSel = document.getElementById('dedupeTotalDataImportGroupId');
var dedupeDeleteSel = document.getElementById('dedupeTotalDataDeleteGroupId');
var dedupeFilterSel = document.getElementById('dedupeTotalDataGroupFilterId');
var dedupeEditSel = document.getElementById('editDedupeTotalDataGroupId');
var filterSel = document.getElementById('shopManageFilterGroupId');
var skipCreateSel = document.getElementById('skipPriceAsinGroupSelect');
var skipFilterSel = document.getElementById('skipPriceAsinFilterGroupId');
@@ -2936,6 +3020,13 @@
var queryCreateSel = document.getElementById('queryAsinGroupSelect');
var queryFilterSel = document.getElementById('queryAsinFilterGroupId');
var chooseQueryShopSel = document.getElementById('chooseQueryAsinShopGroupId');
var selectedInvalidCreateId = invalidCreateSel ? invalidCreateSel.value : '';
var selectedInvalidEditId = invalidEditSel ? invalidEditSel.value : '';
var selectedInvalidFilterId = invalidFilterSel ? invalidFilterSel.value : '';
var selectedDedupeImportId = dedupeImportSel ? dedupeImportSel.value : '';
var selectedDedupeDeleteId = dedupeDeleteSel ? dedupeDeleteSel.value : '';
var selectedDedupeFilterId = dedupeFilterSel ? dedupeFilterSel.value : '';
var selectedDedupeEditId = dedupeEditSel ? dedupeEditSel.value : '';
var selectedFilterId = filterSel ? filterSel.value : '';
var selectedSkipCreateId = skipCreateSel ? skipCreateSel.value : '';
var selectedSkipFilterId = skipFilterSel ? skipFilterSel.value : '';
@@ -2952,6 +3043,37 @@
});
createSel.innerHTML = createOpts.join('');
editSel.innerHTML = createOpts.join('');
var lockedInvalidGroupId = getInvalidAsinDataLockedGroupId();
var lockedGroup = shopManageGroups.find(function (group) {
return String(group.id || '') === lockedInvalidGroupId;
});
var invalidGroupOpts = createOpts;
if (currentUserRole !== 'super_admin') {
invalidGroupOpts = lockedGroup
? ['<option value="' + lockedGroup.id + '">' + (lockedGroup.group_name || '') + '</option>']
: [createOpts[0]];
}
var invalidFilterOpts = currentUserRole === 'super_admin'
? filterOpts
: (lockedGroup
? ['<option value="' + lockedGroup.id + '">' + (lockedGroup.group_name || '') + '</option>']
: [createOpts[0]]);
if (invalidCreateSel) {
invalidCreateSel.innerHTML = invalidGroupOpts.join('');
invalidCreateSel.disabled = currentUserRole !== 'super_admin';
}
if (invalidEditSel) {
invalidEditSel.innerHTML = invalidGroupOpts.join('');
invalidEditSel.disabled = currentUserRole !== 'super_admin';
}
if (invalidFilterSel) {
invalidFilterSel.innerHTML = invalidFilterOpts.join('');
invalidFilterSel.disabled = currentUserRole !== 'super_admin';
}
if (dedupeImportSel) dedupeImportSel.innerHTML = createOpts.join('');
if (dedupeDeleteSel) dedupeDeleteSel.innerHTML = createOpts.join('');
if (dedupeFilterSel) dedupeFilterSel.innerHTML = filterOpts.join('');
if (dedupeEditSel) dedupeEditSel.innerHTML = createOpts.join('');
if (skipCreateSel) skipCreateSel.innerHTML = createOpts.join('');
if (queryCreateSel) queryCreateSel.innerHTML = createOpts.join('');
if (filterSel) filterSel.innerHTML = filterOpts.join('');
@@ -2961,6 +3083,25 @@
if (chooseQueryShopSel) chooseQueryShopSel.innerHTML = filterOpts.join('');
if (selectedCreateId != null) createSel.value = String(selectedCreateId);
if (selectedEditId != null) editSel.value = String(selectedEditId);
if (invalidCreateSel) {
invalidCreateSel.value = currentUserRole === 'super_admin'
? selectedInvalidCreateId
: lockedInvalidGroupId;
}
if (invalidEditSel) {
invalidEditSel.value = currentUserRole === 'super_admin'
? selectedInvalidEditId
: lockedInvalidGroupId;
}
if (invalidFilterSel) {
invalidFilterSel.value = currentUserRole === 'super_admin'
? selectedInvalidFilterId
: lockedInvalidGroupId;
}
if (dedupeImportSel && selectedDedupeImportId) dedupeImportSel.value = selectedDedupeImportId;
if (dedupeDeleteSel && selectedDedupeDeleteId) dedupeDeleteSel.value = selectedDedupeDeleteId;
if (dedupeFilterSel && selectedDedupeFilterId) dedupeFilterSel.value = selectedDedupeFilterId;
if (dedupeEditSel && selectedDedupeEditId) dedupeEditSel.value = selectedDedupeEditId;
if (filterSel && selectedFilterId) filterSel.value = selectedFilterId;
if (skipCreateSel && selectedSkipCreateId) skipCreateSel.value = selectedSkipCreateId;
if (skipFilterSel && selectedSkipFilterId) skipFilterSel.value = selectedSkipFilterId;
@@ -2980,12 +3121,14 @@
.then(function (res) {
if (!res.success) throw new Error(res.error || '加载分组失败');
shopManageGroups = res.items || [];
shopManageLockedGroupId = res.locked_group_id || null;
shopManageGroupsLoadedAt = Date.now();
refreshShopGroupSelects(selectedCreateId, selectedEditId);
return shopManageGroups;
})
.catch(function () {
shopManageGroups = [];
shopManageLockedGroupId = null;
shopManageGroupsLoadedAt = 0;
refreshShopGroupSelects();
return [];
@@ -3040,7 +3183,7 @@
function updateShopManageGroupButtonsAccess() {
var canManageGroups = !!currentUserId;
['btnManageShopGroups', 'btnManageShopGroupsFromEdit', 'btnManageSkipPriceAsinGroups', 'btnManageQueryAsinGroups', 'btnManageDedupeGroups'].forEach(function (id) {
['btnManageShopGroups', 'btnManageShopGroupsFromEdit', 'btnManageSkipPriceAsinGroups', 'btnManageQueryAsinGroups', 'btnManageDedupeGroups', 'btnManageInvalidAsinDataGroups', 'btnManageInvalidAsinDataGroupsFromEdit'].forEach(function (id) {
var btn = document.getElementById(id);
if (btn) btn.style.display = canManageGroups ? '' : 'none';
});
@@ -3123,6 +3266,8 @@
renderDedupeGroupSummary();
loadShopManage(shopManagePage);
loadSkipPriceAsin(skipPriceAsinPage);
if (getActiveAdminTabName() === 'dedupe-total-data') loadDedupeTotalData(dedupeTotalDataPage);
if (getActiveAdminTabName() === 'invalid-asin-data') loadInvalidAsinData(invalidAsinDataPage);
});
});
};
@@ -3149,6 +3294,14 @@
setShopManageGroupGrantRoutes(['dedupe-total-data']);
openShopManageGroupModal();
};
document.getElementById('btnManageInvalidAsinDataGroups').onclick = function () {
setShopManageGroupGrantRoutes(['invalid-asin-data']);
openShopManageGroupModal();
};
document.getElementById('btnManageInvalidAsinDataGroupsFromEdit').onclick = function () {
setShopManageGroupGrantRoutes(['invalid-asin-data']);
openShopManageGroupModal();
};
document.getElementById('btnSearchShopManage').onclick = function () {
loadShopManage(1);
};
@@ -3200,6 +3353,8 @@
renderDedupeGroupSummary();
loadShopManage(shopManagePage);
loadSkipPriceAsin(skipPriceAsinPage);
if (getActiveAdminTabName() === 'dedupe-total-data') loadDedupeTotalData(dedupeTotalDataPage);
if (getActiveAdminTabName() === 'invalid-asin-data') loadInvalidAsinData(invalidAsinDataPage);
});
})
.catch(function () {
@@ -0,0 +1,161 @@
import io
import sys
import unittest
from pathlib import Path
from unittest.mock import patch
from flask import Flask
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from blueprints import admin_api
class _FakeExportResponse:
status_code = 200
content = b'xlsx'
headers = {
'Content-Disposition': 'attachment; filename=dedupe.xlsx',
'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
}
class AdminDedupeTotalDataTest(unittest.TestCase):
def setUp(self):
self.app = Flask(__name__)
def _menu_access(self):
return patch.object(
admin_api,
'_ensure_backend_menu_access',
return_value=('admin', {'id': 7}, None),
)
def test_list_forwards_group_and_formats_item(self):
java_response = {
'data': {
'items': [{
'id': 9,
'dataValue': 'B012345678',
'groupId': 3,
'groupName': 'group-a',
'uploaderUserId': 7,
'username': 'operator',
'createdAt': '2026-08-08T12:30:00',
}],
'total': 1,
'page': 1,
'pageSize': 15,
},
}
with self.app.test_request_context(
'/api/admin/dedupe-total-data?page=1&page_size=15&group_id=3'):
with self._menu_access(), patch.object(
admin_api,
'_proxy_permission_java',
return_value=(java_response, None, 200),
) as proxy:
response = admin_api.list_dedupe_total_data.__wrapped__()
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get_json()['items'][0]['group_name'], 'group-a')
self.assertEqual(proxy.call_args.kwargs['params'], {
'page': 1,
'pageSize': 15,
'keyword': '',
'username': '',
'operatorId': 7,
'groupId': 3,
})
def test_create_and_update_forward_group(self):
java_response = {
'message': 'ok',
'data': {'id': 9, 'dataValue': 'B012345678', 'groupId': 3, 'groupName': 'group-a'},
}
with self.app.test_request_context(
'/api/admin/dedupe-total-data', method='POST',
json={'data_value': ' B012345678 ', 'group_id': 3}):
with self._menu_access(), patch.object(
admin_api,
'_proxy_permission_java',
return_value=(java_response, None, 200),
) as proxy:
response = admin_api.create_dedupe_total_data.__wrapped__()
self.assertEqual(response.status_code, 200)
self.assertEqual(proxy.call_args.kwargs['json_data'], {
'dataValue': 'B012345678',
'groupId': 3,
})
with self.app.test_request_context(
'/api/admin/dedupe-total-data/9', method='PUT',
json={'data_value': ' C012345678 ', 'group_id': 8}):
with self._menu_access(), patch.object(
admin_api,
'_proxy_permission_java',
return_value=(java_response, None, 200),
) as proxy:
response = admin_api.update_dedupe_total_data.__wrapped__(9)
self.assertEqual(response.status_code, 200)
self.assertEqual(proxy.call_args.kwargs['json_data'], {
'dataValue': 'C012345678',
'groupId': 8,
})
def test_import_forwards_multipart_group(self):
with self.app.test_request_context(
'/api/admin/dedupe-total-data/import', method='POST',
data={'file': (io.BytesIO(b'xlsx'), 'data.xlsx'), 'group_id': '3'}):
with self._menu_access(), patch.object(
admin_api,
'_proxy_permission_java',
return_value=({'message': 'started', 'data': {'importId': 'task-1'}}, None, 200),
) as proxy:
response = admin_api.import_dedupe_total_data.__wrapped__()
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get_json()['import_id'], 'task-1')
self.assertEqual(proxy.call_args.kwargs['data'], {'groupId': 3})
self.assertEqual(set(proxy.call_args.kwargs['files']), {'file'})
def test_export_forwards_group_filter_and_internal_token(self):
class _Session:
def get(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
return _FakeExportResponse()
session = _Session()
with self.app.test_request_context(
'/api/admin/dedupe-total-data/export?group_id=3&username=operator'):
with self._menu_access(), patch.object(
admin_api,
'_get_backend_java_session',
return_value=session,
), patch.object(admin_api, '_resolve_internal_token', return_value='token'):
response = admin_api.export_dedupe_total_data.__wrapped__()
self.assertEqual(response.status_code, 200)
self.assertEqual(session.kwargs['params'], {
'operatorId': 7,
'username': 'operator',
'groupId': 3,
})
self.assertEqual(session.kwargs['headers'], {'X-Internal-Token': 'token'})
def test_import_requires_group(self):
with self.app.test_request_context(
'/api/admin/dedupe-total-data/import', method='POST',
data={'file': (io.BytesIO(b'xlsx'), 'data.xlsx')}):
with self._menu_access(), patch.object(admin_api, '_proxy_permission_java') as proxy:
response = admin_api.import_dedupe_total_data.__wrapped__()
self.assertEqual(response.status_code, 200)
self.assertFalse(response.get_json()['success'])
proxy.assert_not_called()
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,141 @@
import sys
import unittest
from pathlib import Path
from unittest.mock import patch
from flask import Flask
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from blueprints import admin_api
class AdminInvalidAsinDataTest(unittest.TestCase):
def setUp(self):
self.app = Flask(__name__)
def test_list_forwards_operator_and_formats_group_fields(self):
java_response = {
'data': {
'items': [{
'id': 9,
'dataValue': 'B012345678',
'brand': 'acme',
'groupId': 3,
'groupName': 'group-a',
'recordSource': 'MANUAL',
'createdAt': '2026-08-08T12:30:00',
}],
'total': 1,
'page': 1,
'pageSize': 15,
},
}
with self.app.test_request_context('/api/admin/invalid-asin-data?page=1&page_size=15&group_id=3'):
with patch.object(
admin_api,
'_ensure_backend_menu_access',
return_value=('admin', {'id': 7}, None),
), patch.object(
admin_api,
'_proxy_permission_java',
return_value=(java_response, None, 200),
) as proxy:
response = admin_api.list_invalid_asin_data.__wrapped__()
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get_json()['items'], [{
'id': 9,
'data_value': 'B012345678',
'brand': 'acme',
'group_id': 3,
'group_name': 'group-a',
'record_source': 'MANUAL',
'created_at': '2026-08-08 12:30',
}])
self.assertEqual(
proxy.call_args.kwargs['params'],
{
'page': 1,
'pageSize': 15,
'keyword': '',
'groupId': 3,
},
)
self.assertEqual(proxy.call_args.kwargs['current_row'], {'id': 7})
def test_create_forwards_group_with_trusted_current_user(self):
java_response = {
'message': 'created',
'data': {
'id': 9,
'dataValue': 'B012345678',
'brand': 'acme',
'groupId': 3,
'groupName': 'group-a',
'recordSource': 'MANUAL',
},
}
with self.app.test_request_context(
'/api/admin/invalid-asin-data',
method='POST',
json={'data_value': ' B012345678 ', 'brand': 'Acme', 'group_id': 3},
):
with patch.object(
admin_api,
'_ensure_backend_menu_access',
return_value=('super_admin', {'id': 1}, None),
), patch.object(
admin_api,
'_proxy_permission_java',
return_value=(java_response, None, 200),
) as proxy:
response = admin_api.create_invalid_asin_data.__wrapped__()
self.assertEqual(response.status_code, 200)
self.assertNotIn('params', proxy.call_args.kwargs)
self.assertEqual(proxy.call_args.kwargs['current_row'], {'id': 1})
self.assertEqual(proxy.call_args.kwargs['json_data'], {
'dataValue': 'B012345678',
'brand': 'Acme',
'groupId': 3,
})
def test_group_list_marks_first_group_as_locked_for_normal_account(self):
groups = [
{'id': 3, 'groupName': 'group-a'},
{'id': 8, 'groupName': 'group-b'},
]
with self.app.test_request_context('/api/admin/shop-manage-groups'):
with patch.object(
admin_api,
'_ensure_backend_menu_access',
return_value=('admin', {'id': 7}, None),
), patch.object(
admin_api,
'_load_expanded_shop_manage_groups',
return_value=(groups, None, 200),
):
response = admin_api.list_shop_manage_groups.__wrapped__()
body = response.get_json()
self.assertEqual(body['locked_group_id'], 3)
self.assertEqual([item['id'] for item in body['items']], [3, 8])
def test_group_list_does_not_lock_super_admin(self):
with self.app.test_request_context('/api/admin/shop-manage-groups'):
with patch.object(
admin_api,
'_ensure_backend_menu_access',
return_value=('super_admin', {'id': 1}, None),
), patch.object(
admin_api,
'_load_expanded_shop_manage_groups',
return_value=([{'id': 3, 'groupName': 'group-a'}], None, 200),
):
response = admin_api.list_shop_manage_groups.__wrapped__()
self.assertIsNone(response.get_json()['locked_group_id'])
if __name__ == '__main__':
unittest.main()
+36 -9
View File
@@ -32,7 +32,7 @@ class _FakeCursor:
self.calls.append((sql, tuple(params)))
if 'COUNT(*) AS total' in sql:
self.kind = 'count'
elif 'MAX(t.created_at)' in sql:
elif 'AS latest_created_at' in sql and 'GROUP BY' in sql:
self.kind = 'groups'
self.group_limit = int(params[-2])
self.group_offset = int(params[-1])
@@ -59,14 +59,14 @@ class _FakeCursor:
limited = []
for row in rows:
shop_key = row['shop_name'].strip().casefold()
if counts.get(shop_key, 0) >= 3:
if counts.get(shop_key, 0) >= 1:
continue
counts[shop_key] = counts.get(shop_key, 0) + 1
limited.append(row)
return limited
if self.current_shop is not None:
rows = [row for row in rows if row['shop_name'].strip() == self.current_shop]
return rows[:3]
return rows[:1]
return []
@@ -85,7 +85,7 @@ class AdminShopDataGroupTest(unittest.TestCase):
def setUp(self):
self.app = Flask(__name__)
self.group_rows = [
{'shop_name': 'Shop A', 'latest_created_at': datetime(2026, 8, 5, 12, 0)},
{'shop_name': 'Shop A', 'latest_created_at': datetime(2026, 8, 7, 5, 21, 45)},
{'shop_name': '', 'latest_created_at': datetime(2026, 8, 4, 12, 0)},
]
self.result_rows = [
@@ -98,7 +98,8 @@ class AdminShopDataGroupTest(unittest.TestCase):
]
@staticmethod
def _result_row(result_id, shop_name, created_at, result_file_url=None):
def _result_row(
result_id, shop_name, created_at, result_file_url=None, latest_file_updated_at=None):
return {
'result_id': result_id,
'task_id': result_id + 100,
@@ -119,6 +120,7 @@ class AdminShopDataGroupTest(unittest.TestCase):
'created_at': created_at,
'updated_at': created_at,
'finished_at': created_at,
'latest_file_updated_at': latest_file_updated_at or created_at,
'file_job_id': None,
'file_status': 'SUCCESS',
'username': 'operator',
@@ -133,9 +135,25 @@ class AdminShopDataGroupTest(unittest.TestCase):
self.assertEqual(group['shop_name'], 'Shop A')
self.assertEqual(group['group_name'], 'Group 1')
self.assertEqual([item['result_id'] for item in group['results']], [5, 4, 3])
self.assertEqual([item['result_id'] for item in group['results']], [5])
self.assertEqual(group['results'][0]['result_file_url'], 'object-5')
def test_group_item_falls_back_to_daily_file_update_time(self):
row = self._result_row(
21427,
'Shop A',
'2026-08-06T15:20:43',
latest_file_updated_at=datetime(2026, 8, 7, 5, 21, 45),
)
group = admin_api._shop_data_crawl_group_item(
{'shop_name': 'Shop A', 'latest_created_at': None},
{'shop a': [row]},
{},
)
self.assertEqual(group['latest_created_at'], '2026-08-07 05:21:45')
def test_list_paginates_groups_and_ignores_removed_user_status_filters(self):
cursor = _FakeCursor(
self.group_rows,
@@ -160,10 +178,11 @@ class AdminShopDataGroupTest(unittest.TestCase):
self.assertEqual(payload['total'], 2)
self.assertEqual(payload['page'], 1)
self.assertEqual(payload['items'][0]['shop_name'], 'Shop A')
self.assertEqual(len(payload['items'][0]['results']), 3)
self.assertEqual(payload['items'][0]['latest_created_at'], '2026-08-07 05:21:45')
self.assertEqual(len(payload['items'][0]['results']), 1)
self.assertEqual(
[item['result_id'] for item in payload['items'][0]['results']],
[5, 4, 3],
[5],
)
self.assertEqual(payload['items'][1]['shop_name'], '未命名')
self.assertEqual(len(payload['items'][1]['results']), 1)
@@ -173,7 +192,15 @@ class AdminShopDataGroupTest(unittest.TestCase):
self.assertNotIn('FAILED', params)
self.assertTrue(any('GROUP BY TRIM(COALESCE(r.source_filename, ' in sql for sql, _ in cursor.calls))
self.assertTrue(any('ROW_NUMBER() OVER' in sql for sql, _ in cursor.calls))
self.assertTrue(any('shop_row_number <= 3' in sql for sql, _ in cursor.calls))
self.assertTrue(any('shop_row_number <= 1' in sql for sql, _ in cursor.calls))
self.assertTrue(any(
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id' in sql
for sql, _ in cursor.calls
))
self.assertTrue(any(
'MAX(COALESCE(df.last_success_at, df.updated_at, t.finished_at, t.updated_at, t.created_at))'
in sql for sql, _ in cursor.calls
))
self.assertTrue(any('TRIM(COALESCE(sm.shop_name' in sql for sql, _ in cursor.calls))
self.assertTrue(any("TRIM(COALESCE(r.result_file_url, '')) <> ''" in sql for sql, _ in cursor.calls))
+57 -1
View File
@@ -1975,6 +1975,12 @@
<label>上传 Excel 文件(读取 ASIN 列)</label>
<input type="file" id="dedupeTotalDataFile" accept=".xlsx,.xls">
</div>
<div class="form-group" style="min-width:220px;">
<label>分组</label>
<select id="dedupeTotalDataImportGroupId">
<option value="">请选择分组</option>
</select>
</div>
<button class="btn" id="btnAddDedupeTotalData">上传并导入</button>
</div>
<p class="msg" id="msgDedupeTotalData"></p>
@@ -1992,6 +1998,12 @@
<label>上传 Excel 文件(读取 ASIN 列)</label>
<input type="file" id="dedupeTotalDataDeleteFile" accept=".xlsx,.xls">
</div>
<div class="form-group" style="min-width:220px;">
<label>分组</label>
<select id="dedupeTotalDataDeleteGroupId">
<option value="">请选择分组</option>
</select>
</div>
<button class="btn btn-danger" id="btnDeleteImportDedupeTotalData">上传并删除</button>
</div>
<p class="msg" id="msgDeleteDedupeTotalData"></p>
@@ -2015,6 +2027,12 @@
<label>用户名(模糊搜索)</label>
<input type="text" id="searchDedupeTotalDataUsername" placeholder="输入用户名">
</div>
<div class="form-group">
<label>分组</label>
<select id="dedupeTotalDataGroupFilterId">
<option value="">全部分组</option>
</select>
</div>
<div class="form-group">
<label>开始日期</label>
<input type="date" id="exportDedupeTotalDataStartDate">
@@ -2034,6 +2052,7 @@
<th>ID</th>
<th>ASIN值</th>
<th>用户名</th>
<th>分组</th>
<th>创建时间</th>
<th>操作</th>
</tr>
@@ -2057,6 +2076,15 @@
<label>品牌</label>
<input type="text" id="invalidAsinDataBrand" placeholder="请输入品牌(可选)">
</div>
<div class="form-group" style="min-width:220px;">
<label>分组</label>
<div style="display:flex;gap:8px;align-items:center;">
<select id="invalidAsinDataGroupSelect" style="min-width:150px;">
<option value="">请选择分组</option>
</select>
<button class="btn btn-secondary" id="btnManageInvalidAsinDataGroups" type="button">管理分组</button>
</div>
</div>
<button class="btn" id="btnAddInvalidAsinData">新增</button>
</div>
<p class="msg" id="msgInvalidAsinData"></p>
@@ -2068,6 +2096,12 @@
<label>ASIN / 品牌(模糊搜索)</label>
<input type="text" id="searchInvalidAsinData" placeholder="输入关键字">
</div>
<div class="form-group" style="min-width:180px;">
<label>分组</label>
<select id="invalidAsinDataFilterGroupId">
<option value="">全部分组</option>
</select>
</div>
<button class=" btn" id="btnSearchInvalidAsinData">查询</button>
</div>
<table>
@@ -2076,6 +2110,8 @@
<th>ID</th>
<th>ASIN</th>
<th>品牌</th>
<th>分组</th>
<th>来源</th>
<th>创建时间</th>
<th>操作</th>
</tr>
@@ -2662,6 +2698,12 @@
<label>ASIN值</label>
<input type="text" id="editDedupeTotalDataValue" placeholder="请输入总数据值">
</div>
<div class="form-group">
<label>分组</label>
<select id="editDedupeTotalDataGroupId">
<option value="">请选择分组</option>
</select>
</div>
<p class=" msg" id="msgEditDedupeTotalData"></p>
<div style="margin-top:16px;display:flex;gap:8px;">
<button class="btn" id="btnSaveDedupeTotalData">保存</button>
@@ -2675,6 +2717,7 @@
<div class="modal">
<h3>编辑不符合ASIN数据</h3>
<input type="hidden" id="editInvalidAsinDataId">
<input type="hidden" id="editInvalidAsinDataRecordSource">
<div class="form-group">
<label>ASIN</label>
<input type="text" id="editInvalidAsinDataValue" placeholder="请输入 ASIN">
@@ -2683,6 +2726,19 @@
<label>品牌</label>
<input type="text" id="editInvalidAsinDataBrand" placeholder="请输入品牌(可选)">
</div>
<div class="form-group">
<label>来源</label>
<input type="text" id="editInvalidAsinDataSourceLabel" readonly>
</div>
<div class="form-group" id="editInvalidAsinDataGroupForm">
<label>分组</label>
<div style="display:flex;gap:8px;align-items:center;">
<select id="editInvalidAsinDataGroupSelect" style="min-width:200px;">
<option value="">请选择分组</option>
</select>
<button class="btn btn-secondary" id="btnManageInvalidAsinDataGroupsFromEdit" type="button">管理分组</button>
</div>
</div>
<p class=" msg" id="msgEditInvalidAsinData"></p>
<div style="margin-top:16px;display:flex;gap:8px;">
<button class="btn" id="btnSaveInvalidAsinData">保存</button>
@@ -3052,7 +3108,7 @@
</div>
</div>
</div>
<script src="/static/admin.js?v=shop-data-task-admin-1"></script>
<script src="/static/admin.js?v=shop-data-task-admin-2"></script>
</body>
</html>
@@ -161,7 +161,10 @@
</div>
</div>
<div class="task-right">
<span class="status running">{{ statusText(item) }}</span>
<span class="status running">
{{ statusText(item) }}
<span v-if="taskFilterSummary(item)" class="status-filter-summary">{{ taskFilterSummary(item) }}</span>
</span>
<button type="button" class="btn-delete" @click="deleteTaskRecord(item)">删除</button>
</div>
</li>
@@ -190,7 +193,10 @@
</div>
</div>
<div class="task-right">
<span class="status" :class="statusClass(item)">{{ statusText(item) }}</span>
<span class="status" :class="statusClass(item)">
{{ statusText(item) }}
<span v-if="taskFilterSummary(item)" class="status-filter-summary">{{ taskFilterSummary(item) }}</span>
</span>
<button v-if="canDownload(item)" type="button" class="download" @click="downloadResult(item)">下载</button>
<button v-if="item.resultId" type="button" class="btn-delete" @click="deleteTaskRecord(item)">删除</button>
</div>
@@ -565,6 +571,17 @@ function statusClass(item: CollectDataHistoryItem) {
return 'pending'
}
function taskFilterSummary(item: CollectDataHistoryItem) {
const status = normalizeTaskStatus(item)
if (status !== 'SUCCESS' && !item.success) return ''
const dedupeFiltered = Number(item.dedupeFilteredCount) || 0
const invalidFiltered = Number(item.invalidFilteredCount) || 0
const brandRejected = Number(item.brandRejectedCount) || 0
const finalRowCount = Number(item.finalRowCount ?? item.rowCount) || 0
return `去重 ${dedupeFiltered} · 无效品牌 ${invalidFiltered} · 品牌拒绝 ${brandRejected} · 保留 ${finalRowCount}`
}
function taskProgressPercent(item: CollectDataHistoryItem) {
const value = Number(item.progressPercent)
if (Number.isFinite(value)) return Math.max(0, Math.min(100, Math.round(value)))
@@ -794,6 +811,7 @@ onBeforeUnmount(() => {
.status.failed { background: rgba(231, 76, 60, .18); color: #ff6b6b; }
.status.running { background: rgba(52, 152, 219, .18); color: #3498db; }
.status.pending { background: rgba(149, 165, 166, .18); color: #bdc3c7; }
.status-filter-summary { font-weight: 400; }
.download { padding: 6px 10px; color: #d6ecff; background: rgba(52, 152, 219, .18); }
.btn-delete { padding: 6px 10px; color: #ff8f8f; background: rgba(231, 76, 60, .12); }
.task-progress { margin-top: 8px; max-width: 520px; }