diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java index d98b8f5..91907ee 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java @@ -11,5 +11,6 @@ import java.util.List; public class ModuleCleanupProperties { private boolean enabled = true; private String cron = "0 0 0 * * *"; - private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN")); + private long retentionDays = 7; + private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA")); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java index 49c179f..02ca381 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java @@ -20,9 +20,11 @@ import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackLoopRunVo; import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackMatchShopsVo; import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackPendingDeleteVo; import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackTaskBatchVo; +import com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo; import com.nanri.aiimage.modules.pricetrack.service.PriceTrackLoopRunService; import com.nanri.aiimage.modules.pricetrack.service.PriceTrackService; import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService; +import com.nanri.aiimage.modules.shopkey.service.SkipPriceAsinService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; @@ -58,6 +60,7 @@ public class PriceTrackController { private final PriceTrackService priceTrackService; private final PriceTrackTaskService priceTrackTaskService; private final PriceTrackLoopRunService priceTrackLoopRunService; + private final SkipPriceAsinService skipPriceAsinService; @GetMapping("/candidates") @Operation(summary = "查询备选店铺列表", description = "返回当前用户在跟价模块中保存的备选店铺。") @@ -104,6 +107,24 @@ public class PriceTrackController { return ApiResponse.success(priceTrackService.matchShops(request)); } + @GetMapping("/tasks/{taskId}/skip-asins/paginated") + @Operation( + summary = "分页查询任务的跳过 ASIN 数据", + description = "根据任务 ID 分页拉取该任务关联的 ASIN 数据。" + + "全量模式：从 biz_skip_price_asin 表查询；" + + "文件模式：从任务 requestJson 中存储的上传文件数据查询。" + + "国家代码从任务的 requestJson.countryCodes 中获取。" + ) + public ApiResponse getTaskSkipAsinsPaginated( + @Parameter(description = "任务 ID", required = true, example = "200") + @PathVariable Long taskId, + @Parameter(description = "页码，从 1 开始", example = "1") + @RequestParam(value = "page", defaultValue = "1") Integer page, + @Parameter(description = "每页条数，默认 1000，最大 2000", example = "1000") + @RequestParam(value = "page_size", defaultValue = "1000") Integer pageSize) { + return ApiResponse.success(priceTrackTaskService.getTaskSkipAsinsPaginated(taskId, page, pageSize)); + } + @GetMapping("/dashboard") @Operation(summary = "统计看板", description = "返回备选店铺数、已结束任务数、成功任务数和失败任务数。") public ApiResponse dashboard( diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/SkipPriceAsinPageVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/SkipPriceAsinPageVo.java new file mode 100644 index 0000000..a72fd90 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/SkipPriceAsinPageVo.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.pricetrack.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 跟价跳过 ASIN 分页返回 + */ +@Data +@Schema(description = "跟价跳过 ASIN 分页返回") +public class SkipPriceAsinPageVo { + + @Schema(description = "当前页码，从 1 开始") + private Integer page; + + @Schema(description = "每页条数") + private Integer pageSize; + + @Schema(description = "总记录数") + private Long total; + + @Schema(description = "总页数") + private Integer totalPages; + + @Schema(description = "按国家分组的 ASIN 列表（简化版，仅 ASIN）") + private Map> skipAsinsByCountry = new LinkedHashMap<>(); + + @Schema(description = "按国家分组的 ASIN 详情列表（包含 asin 和 minimumPrice）") + private Map>> skipAsinDetailsByCountry = new LinkedHashMap<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java index d9210b6..f2f0955 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java @@ -145,24 +145,16 @@ public class PriceTrackService { } boolean asinMode = request.getAsinFiles() != null && !request.getAsinFiles().isEmpty(); - Map> skipAsinsByCountry = asinMode - ? new LinkedHashMap<>() - : skipPriceAsinService.listAllSkipAsinsByCountry(); - Map>> skipAsinDetailsByCountry = asinMode - ? new LinkedHashMap<>() - : skipPriceAsinService.listAllSkipAsinDetailsByCountry(); - Map>> asinRowsByCountry = - !asinMode - ? new LinkedHashMap<>() - : priceTrackTaskService.parseMatchAsinRowsByCountry( - request.getAsinFiles(), - request.getCountryCodes()); + // 不再在 matchShops 时加载全量数据，改由 Python 端分页拉取 + Map> skipAsinsByCountry = new LinkedHashMap<>(); + Map>> skipAsinDetailsByCountry = new LinkedHashMap<>(); + Map>> asinRowsByCountry = new LinkedHashMap<>(); PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo(); vo.setSkipAsinsByCountry(skipAsinsByCountry); vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry); vo.setAsinRowsByCountry(asinRowsByCountry); - vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry)); + vo.setMinimumPriceByCountryAndAsin(new LinkedHashMap<>()); for (String shopName : ordered) { vo.getItems().add(matchOneShop(shopName, skipAsinsByCountry)); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java index fd60a86..f0bf512 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java @@ -428,10 +428,15 @@ public class PriceTrackTaskService { ctx.put("statusMode", request.isStatusMode()); ctx.put("asinMode", request.isAsinMode()); ctx.put("countryCodes", request.getCountryCodes()); - ctx.put("skipAsinsByCountry", skipAsinsByCountry); - ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry); - ctx.put("asinRowsByCountry", asinRowsByCountry); - ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin); + // 不再保存全量 ASIN 数据到 requestJson + // ctx.put("skipAsinsByCountry", skipAsinsByCountry); + // ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry); + // ctx.put("asinRowsByCountry", asinRowsByCountry); + // ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin); + // 文件模式：保存文件路径，供后续分页读取 + if (request.isAsinMode()) { + ctx.put("asinFiles", request.getAsinFiles()); + } ctx.put("items", uniqueItems); ctx.put("loopRunId", request.getLoopRunId()); ctx.put("roundIndex", request.getRoundIndex()); @@ -764,6 +769,144 @@ public class PriceTrackTaskService { return buildMinimumPriceByCountryAndAsin(asinRowsByCountry); } + /** + * 分页查询任务关联的跳过 ASIN 数据 + * + * @param taskId 任务 ID + * @param page 页码，从 1 开始 + * @param pageSize 每页条数 + * @return 分页结果 + */ + public com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo getTaskSkipAsinsPaginated( + Long taskId, int page, int pageSize) { + if (taskId == null || taskId <= 0) { + throw new BusinessException("taskId 不合法"); + } + + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + + // 解析任务请求参数 + PriceTrackCreateTaskRequest request; + try { + if (task.getRequestJson() == null || task.getRequestJson().isBlank()) { + throw new BusinessException("任务请求参数为空"); + } + request = objectMapper.readValue(task.getRequestJson(), PriceTrackCreateTaskRequest.class); + } catch (Exception e) { + throw new BusinessException("任务请求参数解析失败: " + e.getMessage()); + } + + // 从任务中获取国家代码 + List countryCodes = request.getCountryCodes(); + + // 判断是全量模式还是文件模式 + boolean isAsinMode = request.isAsinMode(); + + if (isAsinMode) { + // 文件模式：从任务 requestJson 中的上传文件解析数据并分页返回 + return getTaskAsinFilePaginated(request, countryCodes, page, pageSize); + } else { + // 全量模式：从 biz_skip_price_asin 表查询 + return skipPriceAsinService.listSkipAsinsPaginated(null, countryCodes, page, pageSize); + } + } + + /** + * 文件模式：从任务上传的文件中分页返回 ASIN 数据 + */ + private com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo getTaskAsinFilePaginated( + PriceTrackCreateTaskRequest request, List countryCodes, int page, int pageSize) { + if (page < 1) { + page = 1; + } + if (pageSize < 1 || pageSize > 2000) { + pageSize = 1000; + } + + // 解析上传文件中的所有 ASIN 数据 + Map>> allAsinRows = parseAsinRowsByCountry( + request.getAsinFiles(), + request.getCountryCodes() + ); + + // 按国家分组并分页 + List targetCountries = (countryCodes == null || countryCodes.isEmpty()) + ? new ArrayList<>(allAsinRows.keySet()) + : countryCodes; + + // 计算总记录数（所有国家的 ASIN 行数总和） + long totalRows = 0; + for (String country : targetCountries) { + List> rows = allAsinRows.getOrDefault(country, List.of()); + totalRows += rows.size(); + } + + // 计算分页 + int totalPages = (int) ((totalRows + pageSize - 1) / pageSize); + int startIndex = (page - 1) * pageSize; + int endIndex = Math.min(startIndex + pageSize, (int) totalRows); + + // 提取当前页的数据（跨国家顺序提取） + Map> skipAsinsByCountry = new LinkedHashMap<>(); + Map>> skipDetailsByCountry = new LinkedHashMap<>(); + + // 初始化所有目标国家 + for (String country : targetCountries) { + skipAsinsByCountry.put(country, new ArrayList<>()); + skipDetailsByCountry.put(country, new ArrayList<>()); + } + + int currentIndex = 0; + boolean shouldBreak = false; + + for (String country : targetCountries) { + if (shouldBreak) { + break; + } + + List> countryRows = allAsinRows.getOrDefault(country, List.of()); + for (Map row : countryRows) { + // 判断当前行是否在分页范围内 + if (currentIndex >= startIndex && currentIndex < endIndex) { + String asin = row.get("asin"); + String minimumPrice = row.get("minimumPrice"); + + if (asin != null && !asin.isBlank()) { + skipAsinsByCountry.get(country).add(asin); + + Map detail = new LinkedHashMap<>(); + detail.put("asin", asin); + detail.put("minimumPrice", minimumPrice == null ? "" : minimumPrice); + skipDetailsByCountry.get(country).add(detail); + } + } + + currentIndex++; + + // 已达到本页结束位置 + if (currentIndex >= endIndex) { + shouldBreak = true; + break; + } + } + } + + // 构造返回 VO + com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo vo = + new com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo(); + vo.setPage(page); + vo.setPageSize(pageSize); + vo.setTotal(totalRows); + vo.setTotalPages(totalPages); + vo.setSkipAsinsByCountry(skipAsinsByCountry); + vo.setSkipAsinDetailsByCountry(skipDetailsByCountry); + + return vo; + } + private Map>> parseAsinRowsByCountry(List asinFiles, List countryCodes) { Map>> merged = new LinkedHashMap<>(); if (asinFiles == null || asinFiles.isEmpty()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java index 3beb2f6..1aea1cb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java @@ -51,7 +51,7 @@ public class ProductRiskExcelAssemblyService { createTextCell(row, 0, firstNonBlank(dto == null ? null : dto.getShopName(), shopDisplayName)); createTextCell(row, 1, dto == null ? null : dto.getProductAsinSku()); createTextCell(row, 2, dto == null ? null : dto.getStatus()); - createTextCell(row, 3, dto == null || dto.getDone() == null ? null : (dto.getDone() ? "是" : "否")); + createTextCell(row, 3, hasBusinessData(dto) ? "是" : "否"); createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin()); createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus()); } @@ -111,6 +111,18 @@ public class ProductRiskExcelAssemblyService { return fallback == null ? "" : fallback; } + private static boolean hasBusinessData(ProductRiskRowDto dto) { + return dto != null + && (hasText(dto.getProductAsinSku()) + || hasText(dto.getStatus()) + || hasText(dto.getRemoveAsin()) + || hasText(dto.getRemoveStatus())); + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } + private void applyDefaultColumnWidths(Sheet sheet) { int[] widths = {20, 22, 18, 10, 18, 18}; for (int i = 0; i < widths.length; i++) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java index ff3429c..83162f3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.shopkey.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import cn.hutool.core.util.IdUtil; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper; @@ -35,6 +36,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -1055,6 +1057,104 @@ public class SkipPriceAsinService { return grouped; } + /** + * 分页查询跳过 ASIN 数据（按商品状态全量模式使用） + * 按 ASIN 个数精确分页（而非表记录数） + * + * @param shopName 店铺名称（可选，为空则查询所有） + * @param countryCodes 国家代码列表 + * @param page 页码，从 1 开始 + * @param pageSize 每页条数 + * @return 分页结果 + */ + public com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo listSkipAsinsPaginated( + String shopName, List countryCodes, int page, int pageSize) { + if (page < 1) { + page = 1; + } + if (pageSize < 1 || pageSize > 2000) { + pageSize = 1000; + } + + // 构建查询条件 + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper() + .orderByDesc(SkipPriceAsinEntity::getId); + + // 如果指定了店铺名，按店铺过滤 + if (shopName != null && !shopName.trim().isEmpty()) { + String normalizedShopName = normalizeBlank(shopName); + queryWrapper.eq(SkipPriceAsinEntity::getShopName, normalizedShopName); + } + + // 查询所有记录（用于内存分页） + List allRecords = skipPriceAsinMapper.selectList(queryWrapper); + + List targetCountries = (countryCodes == null || countryCodes.isEmpty()) + ? List.of("DE", "UK", "FR", "IT", "ES") + : countryCodes; + + // 展开所有 ASIN 为扁平列表 + List flattenedAsins = new ArrayList<>(); + for (SkipPriceAsinEntity row : allRecords) { + for (String country : targetCountries) { + String asin = getCountryAsin(row, country); + BigDecimal minimumPrice = getCountryMinimumPrice(row, country); + + if (asin != null && !asin.isBlank()) { + AsinItem item = new AsinItem(); + item.country = country; + item.asin = asin; + item.minimumPrice = minimumPrice; + flattenedAsins.add(item); + } + } + } + + // 计算总数和分页 + long totalAsins = flattenedAsins.size(); + int totalPages = (int) ((totalAsins + pageSize - 1) / pageSize); + int startIndex = (page - 1) * pageSize; + int endIndex = Math.min(startIndex + pageSize, flattenedAsins.size()); + + // 提取当前页的 ASIN + Map> skipAsinsByCountry = new LinkedHashMap<>(); + Map>> skipDetailsByCountry = new LinkedHashMap<>(); + + for (String country : targetCountries) { + skipAsinsByCountry.put(country, new ArrayList<>()); + skipDetailsByCountry.put(country, new ArrayList<>()); + } + + for (int i = startIndex; i < endIndex; i++) { + AsinItem item = flattenedAsins.get(i); + skipAsinsByCountry.get(item.country).add(item.asin); + + Map detail = new LinkedHashMap<>(); + detail.put("asin", item.asin); + detail.put("minimumPrice", item.minimumPrice == null ? "" : item.minimumPrice.toPlainString()); + skipDetailsByCountry.get(item.country).add(detail); + } + + // 构造返回 VO + com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo vo = + new com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo(); + vo.setPage(page); + vo.setPageSize(pageSize); + vo.setTotal(totalAsins); + vo.setTotalPages(totalPages); + vo.setSkipAsinsByCountry(skipAsinsByCountry); + vo.setSkipAsinDetailsByCountry(skipDetailsByCountry); + + return vo; + } + + // 内部辅助类 + private static class AsinItem { + String country; + String asin; + BigDecimal minimumPrice; + } + @Transactional public boolean removeByShopCountryAndAsin(String shopName, String country, String asin) { String normalizedShopName = normalizeBlank(shopName); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java index 69513d4..0e8572d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java @@ -4,10 +4,24 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.config.ModuleCleanupProperties; +import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper; +import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper; +import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; @@ -15,6 +29,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.time.Duration; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -27,10 +42,18 @@ public class ModuleHistoryCleanupService { private static final Set TERMINAL_STATUSES = Set.of("SUCCESS", "FAILED", "CANCELLED", "CANCELED"); private static final Duration CLEANUP_LOCK_TTL = Duration.ofHours(2); + private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA"; private final ModuleCleanupProperties moduleCleanupProperties; private final FileTaskMapper fileTaskMapper; private final FileResultMapper fileResultMapper; + private final TaskFileJobMapper taskFileJobMapper; + private final TaskResultItemMapper taskResultItemMapper; + private final TaskProgressSnapshotMapper taskProgressSnapshotMapper; + private final TaskResultPayloadMapper taskResultPayloadMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TaskChunkMapper taskChunkMapper; + private final CollectDataItemMapper collectDataItemMapper; private final DistributedJobLockService distributedJobLockService; @Transactional @@ -50,30 +73,71 @@ public class ModuleHistoryCleanupService { if (moduleTypes == null || moduleTypes.isEmpty()) { return; } + LocalDateTime cutoff = LocalDateTime.now().minusDays(Math.max(0, moduleCleanupProperties.getRetentionDays())); List moduleTasks = fileTaskMapper.selectList(new LambdaQueryWrapper() .in(FileTaskEntity::getModuleType, moduleTypes) - .select(FileTaskEntity::getId, FileTaskEntity::getModuleType, FileTaskEntity::getStatus)); + .select(FileTaskEntity::getId, FileTaskEntity::getModuleType, FileTaskEntity::getStatus, + FileTaskEntity::getUpdatedAt, FileTaskEntity::getFinishedAt)); List cleanupTaskIds = new ArrayList<>(); + List cleanupCollectDataTaskIds = new ArrayList<>(); List skippedActiveTaskIds = new ArrayList<>(); + List skippedRetainedTaskIds = new ArrayList<>(); for (FileTaskEntity task : moduleTasks) { if (task == null || task.getId() == null) { continue; } - if (isTerminalStatus(task.getStatus())) { - cleanupTaskIds.add(task.getId()); - } else { + if (!isTerminalStatus(task.getStatus())) { skippedActiveTaskIds.add(task.getId()); + continue; + } + if (isExpired(task, cutoff)) { + cleanupTaskIds.add(task.getId()); + if (COLLECT_DATA_MODULE_TYPE.equals(task.getModuleType())) { + cleanupCollectDataTaskIds.add(task.getId()); + } + } else { + skippedRetainedTaskIds.add(task.getId()); } } if (cleanupTaskIds.isEmpty()) { - log.info("模块历史清理跳过: moduleTypes={}, activeTaskIds={}, reason=no-terminal-tasks", - moduleTypes, skippedActiveTaskIds); + log.info("[module-cleanup] skipped: moduleTypes={}, retentionDays={}, cutoff={}, activeTaskIds={}, retainedTaskIds={}, reason=no-expired-terminal-tasks", + moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff, skippedActiveTaskIds, skippedRetainedTaskIds); return; } + int deletedFileJobs = taskFileJobMapper.delete(new LambdaQueryWrapper() + .in(TaskFileJobEntity::getModuleType, moduleTypes) + .in(TaskFileJobEntity::getTaskId, cleanupTaskIds)); + + int deletedResultItems = taskResultItemMapper.delete(new LambdaQueryWrapper() + .in(TaskResultItemEntity::getModuleType, moduleTypes) + .in(TaskResultItemEntity::getTaskId, cleanupTaskIds)); + + int deletedProgressSnapshots = taskProgressSnapshotMapper.delete(new LambdaQueryWrapper() + .in(TaskProgressSnapshotEntity::getModuleType, moduleTypes) + .in(TaskProgressSnapshotEntity::getTaskId, cleanupTaskIds)); + + int deletedResultPayloads = taskResultPayloadMapper.delete(new LambdaQueryWrapper() + .in(TaskResultPayloadEntity::getModuleType, moduleTypes) + .in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds)); + + int deletedScopeStates = taskScopeStateMapper.delete(new LambdaQueryWrapper() + .in(TaskScopeStateEntity::getModuleType, moduleTypes) + .in(TaskScopeStateEntity::getTaskId, cleanupTaskIds)); + + int deletedChunks = taskChunkMapper.delete(new LambdaQueryWrapper() + .in(TaskChunkEntity::getModuleType, moduleTypes) + .in(TaskChunkEntity::getTaskId, cleanupTaskIds)); + + int deletedCollectDataItems = 0; + if (!cleanupCollectDataTaskIds.isEmpty()) { + deletedCollectDataItems = collectDataItemMapper.delete(new LambdaQueryWrapper() + .in(CollectDataItemEntity::getTaskId, cleanupCollectDataTaskIds)); + } + int deletedResults = fileResultMapper.delete(new LambdaQueryWrapper() .in(FileResultEntity::getModuleType, moduleTypes) .in(FileResultEntity::getTaskId, cleanupTaskIds)); @@ -87,11 +151,19 @@ public class ModuleHistoryCleanupService { int deletedTasks = fileTaskMapper.delete(new LambdaQueryWrapper() .in(FileTaskEntity::getId, cleanupTaskIds)); - log.info("模块历史清理完成: moduleTypes={}, deletedResults={}, resetTasks={}, deletedTasks={}, skippedActiveTaskIds={}", - moduleTypes, deletedResults, resetTasks, deletedTasks, skippedActiveTaskIds); + log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, deletedFileJobs={}, deletedResultItems={}, deletedProgressSnapshots={}, deletedResultPayloads={}, deletedScopeStates={}, deletedChunks={}, deletedCollectDataItems={}, deletedResults={}, resetTasks={}, deletedTasks={}, skippedActiveTaskIds={}, retainedTaskIds={}", + moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff, + deletedFileJobs, deletedResultItems, deletedProgressSnapshots, deletedResultPayloads, + deletedScopeStates, deletedChunks, deletedCollectDataItems, deletedResults, resetTasks, deletedTasks, + skippedActiveTaskIds, skippedRetainedTaskIds); } } + private boolean isExpired(FileTaskEntity task, LocalDateTime cutoff) { + LocalDateTime completedAt = task.getFinishedAt() != null ? task.getFinishedAt() : task.getUpdatedAt(); + return completedAt != null && !completedAt.isAfter(cutoff); + } + private boolean isTerminalStatus(String status) { if (status == null || status.isBlank()) { return false; diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index f829df1..df99564 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -148,7 +148,8 @@ aiimage: module-cleanup: enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true} cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *} - module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE} + retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7} + module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA} permission-schema-init: enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false} task-pressure: diff --git a/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue b/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue index 57fbf3e..8a6d5b2 100644 --- a/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue @@ -799,7 +799,9 @@ async function runMatch() { countryCodes: resolveCountryCodesForRequest(), }) const batch = res.items || [] - matchAsinRowsByCountry.value = res.asinRowsByCountry || {} + // 移除：不再使用后端返回的 ASIN 数据 + // matchAsinRowsByCountry.value = res.asinRowsByCountry || {} + matchAsinRowsByCountry.value = {} const nextByShop = new Map( matchedItems.value.map((item) => [((item.shopName || '').trim()), item] as const), ) @@ -989,9 +991,6 @@ async function pushToPythonQueueLegacy() { function buildQueuePayload(taskVo: PriceTrackCreateTaskVo, row: PriceTrackShopQueueItem) { const taskItem = taskVo.items?.[0] const shopName = (row.shopName || '').trim() - const skipAsinsByCountry = row.skipAsins || taskVo.skipAsinsByCountry || {} - const skipAsinDetailsByCountry = taskVo.skipAsinDetailsByCountry || {} - const minimumPriceByCountryAndAsin = taskVo.minimumPriceByCountryAndAsin || {} const skipAsinDeletePolicy = buildSkipAsinDeletePolicy() const resultId = taskItem?.resultId ?? null const loopRunId = taskItem?.loopRunId ?? null @@ -1001,6 +1000,7 @@ function buildQueuePayload(taskVo: PriceTrackCreateTaskVo, row: PriceTrackShopQu const error = taskItem?.error || null const outputFilename = taskItem?.outputFilename || null const downloadUrl = taskItem?.downloadUrl || null + return { type: 'price-track-run', ts: Date.now(), @@ -1024,16 +1024,14 @@ function buildQueuePayload(taskVo: PriceTrackCreateTaskVo, row: PriceTrackShopQu round_index: roundIndex, country_codes: resolveCountryCodesForRequest(), mode: statusModeEnabled.value ? 'status' : 'asin', - skip_asins: skipAsinsByCountry, - skip_asins_by_country: skipAsinsByCountry, - skip_asin_details_by_country: skipAsinDetailsByCountry, - minimum_price_by_country_and_asin: minimumPriceByCountryAndAsin, + // 分页拉取配置（Python 端统一处理） + use_paginated_skip_asins: true, + skip_asin_page_size: 1000, + // 移除：不再传递任何 ASIN 数据，Python 端统一分页拉取 skip_asin_delete_policy: skipAsinDeletePolicy, delete_skip_asin_when_price_below_minimum: skipAsinDeletePolicy.deleteWhenPriceBelowMinimum, deleteSkipAsinWhenPriceBelowMinimum: skipAsinDeletePolicy.deleteWhenPriceBelowMinimum, - asin_rows_by_country: Object.keys(matchAsinRowsByCountry.value).length - ? matchAsinRowsByCountry.value - : (taskVo.asinRowsByCountry || {}), + // 兼容字段 taskId: taskVo.taskId, resultId: resultId, shopName, diff --git a/frontend-vue/src/pages/brand/components/BrandProductRiskTab.vue b/frontend-vue/src/pages/brand/components/BrandProductRiskTab.vue index e503e5f..73c588c 100644 --- a/frontend-vue/src/pages/brand/components/BrandProductRiskTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandProductRiskTab.vue @@ -59,7 +59,7 @@

与「推送到 Python 队列」一并下发，供 Python 区分处理场景。

-
@@ -235,6 +235,12 @@ const autoQueueEnabled = ref(false) const queuePushResult = ref('') const queuePayloadText = ref('') +const PRODUCT_RISK_LISTING_FILTER_OPTIONS = [ + ...LISTING_FILTER_OPTIONS, + { value: 'AccountStatus', label: '账户状态处理' }, +] as const +type ProductRiskListingFilterValue = ListingFilterValue | 'AccountStatus' + /** 与后端 ProductRiskCountryCode / 默认顺序 DE→UK→FR→IT→ES 一致 */ const COUNTRY_OPTIONS = [ { code: 'DE', label: '德国' }, @@ -245,7 +251,7 @@ const COUNTRY_OPTIONS = [ ] as const const orderedCountryCodes = ref(['DE', 'UK', 'FR', 'IT', 'ES']) -const productRiskListingFilter = ref('SearchSuppressed') +const productRiskListingFilter = ref('SearchSuppressed') const dragCountryIndex = ref(null) const countryPrefSaving = ref(false) /** 用户已改过顺序/勾选后，忽略晚到的 GET，避免把界面打回全选 */ @@ -527,8 +533,8 @@ async function removeMatchedRow(row: ProductRiskShopQueueItem) { /** 历史列表中已无该 taskId 时，停止轮询并清本地快照 */ function syncPollingIdsWithHistory() { for (const taskId of [...pollingTaskIds.value]) { - const any = historyItems.value.some((r) => r.taskId === taskId) - if (!any) { + const rows = historyItems.value.filter((r) => r.taskId === taskId) + if (!rows.length || rows.some((row) => isHistoryItemTerminal(row))) { removePollingTask(taskId) } } @@ -640,9 +646,22 @@ function taskStatusOf(taskId?: number) { return taskDetails.value[taskId] || '' } +function isTerminalStatus(status?: string) { + return status === 'SUCCESS' || status === 'FAILED' +} + function isTaskTerminalById(taskId?: number) { const s = taskStatusOf(taskId) - return s === 'SUCCESS' || s === 'FAILED' + return isTerminalStatus(s) +} + +function isHistoryItemTerminal(item: ProductRiskHistoryItem) { + if (isTerminalStatus(item.taskStatus)) return true + if (item.success === true) return true + if (item.fileStatus === 'SUCCESS' && item.fileReady) return true + if (item.fileStatus === 'FAILED') return true + if (item.success === false && !!item.error) return true + return false } const currentSectionItems = computed(() => { @@ -650,7 +669,8 @@ const currentSectionItems = computed(() => { (row) => row.taskId && pollingTaskIds.value.includes(row.taskId) && - !isTaskTerminalById(row.taskId), + !isTaskTerminalById(row.taskId) && + !isHistoryItemTerminal(row), ) const existingTaskIds = new Set(out.map((row) => row.taskId).filter((id): id is number => typeof id === 'number')) @@ -669,7 +689,8 @@ const historySectionItems = computed(() => (row) => !row.taskId || !pollingTaskIds.value.includes(row.taskId) || - isTaskTerminalById(row.taskId), + isTaskTerminalById(row.taskId) || + isHistoryItemTerminal(row), ), ) @@ -953,6 +974,9 @@ async function runMatch() { } function resolvedTaskStatus(item: ProductRiskHistoryItem) { + if (isTerminalStatus(item.taskStatus)) return item.taskStatus || '' + if (item.success === true || (item.fileStatus === 'SUCCESS' && item.fileReady)) return 'SUCCESS' + if (item.fileStatus === 'FAILED' || (item.success === false && !!item.error)) return 'FAILED' const tid = item.taskId if (!tid) return item.taskStatus || '' return taskStatusOf(tid) || item.taskStatus || '' diff --git a/frontend-vue/src/pages/image-video/ImageVideoPage.vue b/frontend-vue/src/pages/image-video/ImageVideoPage.vue index efe707e..4cb70e8 100644 --- a/frontend-vue/src/pages/image-video/ImageVideoPage.vue +++ b/frontend-vue/src/pages/image-video/ImageVideoPage.vue @@ -25,6 +25,19 @@
{{ statusText }}
+ +
+
+ {{ launchProgress.message || '正在准备数字人程序...' }} + {{ launchProgress.percent }}% +
+
+
+
+
+ {{ formatBytes(launchProgress.downloaded) }} / {{ formatBytes(launchProgress.total) }} +
+
@@ -50,7 +63,7 @@