diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 50c98502..d769584e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; import com.nanri.aiimage.common.service.DistributedJobLockService; -import com.nanri.aiimage.common.util.GroupResultPropagator; import com.nanri.aiimage.common.util.FailedStatusRowFilter; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.SimilarAsinProperties; @@ -27,7 +26,6 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRe import com.nanri.aiimage.modules.similarasin.model.entity.SimilarAsinFilterConditionEntity; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinFilterConditionVo; -import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; @@ -37,11 +35,19 @@ import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinChunkMergeSupport; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinChunkPayloadSupport; import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser; import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter; import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler; -import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache; -import com.nanri.aiimage.modules.similarasin.util.WpsCellImageWriter; +import com.nanri.aiimage.modules.similarasin.service.support.LlmBatchContext; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinLimits; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPayloadSupport; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPoisonTracker; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultTextSupport; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultWorkbookAssembler; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinTaskProgressSupport; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinTaskOwnershipSupport; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinLogSupport; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; @@ -65,12 +71,6 @@ import com.nanri.aiimage.modules.usersecret.support.SecretUsageContext; import com.nanri.aiimage.modules.usersecret.support.UserSecretModule; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellStyle; -import org.apache.poi.ss.usermodel.Font; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DuplicateKeyException; @@ -81,16 +81,9 @@ import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.support.TransactionTemplate; -import jakarta.annotation.PreDestroy; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; -import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.nio.charset.StandardCharsets; @@ -103,22 +96,11 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; @Service @RequiredArgsConstructor @@ -134,89 +116,13 @@ public class SimilarAsinTaskService { private static final String LLM_STATUS_RUNNING = "RUNNING"; private static final String LLM_STATUS_DONE = "DONE"; private static final String LLM_STATUS_FAILED = "FAILED"; - /** orphan-llm-rows: 跨 chunk merge 时未匹配到任何 chunk 的回流行兜底落库前缀。 */ - private static final String ORPHAN_SCOPE_KEY_PREFIX = "orphan-llm-rows:"; - /** orphan 行兜底状态，写入时直接置为 DONE，避免被 stale-recovery 扫描到。 */ - private static final String ORPHAN_LLM_ROW_ERROR = "orphan-llm-row(no submitted chunk)"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_ZIP = "application/zip"; - private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; private static final int MAX_LIGHT_TASK_IDS = 200; /** Task 19：Python 回传逐行日志采样频率（每 N 行记一行）。 */ private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L; - private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5); private static final long TASK_LOCK_WAIT_MILLIS = 10000L; - private static final int PARSE_RESPONSE_PREVIEW_LIMIT = 100; - /** 预览行/预览组配置上限：超过该值 clamp，避免响应体无界增长。 */ - private static final int PARSE_RESPONSE_PREVIEW_LIMIT_MAX = 1000; - /** - * 解析接口响应预览上限：优先读取配置 parseResponsePreviewLimit。 - * 0/负值回退默认 100；超过 1000 clamp 到 1000，防止误配导致响应体膨胀。 - */ - private int resolvePreviewLimit() { - Integer configured = properties.getParseResponsePreviewLimit(); - if (configured == null || configured <= 0) { - return PARSE_RESPONSE_PREVIEW_LIMIT; - } - return Math.min(configured, PARSE_RESPONSE_PREVIEW_LIMIT_MAX); - } - - /** - * Task 7：单文件大小上限。0/负值回退默认 50MB，防止误配导致解析无界增长。 - */ - private long resolveMaxSourceFileBytes() { - Long configured = properties.getMaxSourceFileBytes(); - if (configured == null || configured <= 0) { - return 50L * 1024L * 1024L; - } - return configured; - } - - /** - * Task 7：单次解析最大有效行数。0/负值回退默认 50000，防止任务无界增长。 - */ - private int resolveMaxParseRows() { - Integer configured = properties.getMaxParseRows(); - if (configured == null || configured <= 0) { - return 50000; - } - return configured; - } - - /** - * Task 8：单字段最大长度（字符）。0/负值回退默认 2000。 - */ - private int resolveMaxFieldLength() { - Integer configured = properties.getMaxFieldLength(); - if (configured == null || configured <= 0) { - return 2000; - } - return configured; - } - - /** - * Task 8：xlsx(zip) 最大条目数。0/负值回退默认 20000。 - */ - private int resolveMaxWorkbookZipEntries() { - Integer configured = properties.getMaxWorkbookZipEntries(); - if (configured == null || configured <= 0) { - return 20000; - } - return configured; - } - - /** - * Task 8：xlsx(zip) 解压后总字节数上限。0/负值回退默认 512MB。 - */ - private long resolveMaxWorkbookUncompressedBytes() { - Long configured = properties.getMaxWorkbookUncompressedBytes(); - if (configured == null || configured <= 0) { - return 512L * 1024L * 1024L; - } - return configured; - } - /** * Task 8：WorkbookFactory 打开前的受控读取。xlsx 本质是 zip， * 先用 ZipFile 中央目录探测条目数与解压总字节（不读取压缩内容）， @@ -224,8 +130,8 @@ public class SimilarAsinTaskService { * 非 zip 文件（或损坏文件）由调用方 catch 转业务异常。 */ private void probeWorkbookZipBounds(File input, String sourceName) throws Exception { - int maxEntries = resolveMaxWorkbookZipEntries(); - long maxBytes = resolveMaxWorkbookUncompressedBytes(); + int maxEntries = SimilarAsinLimits.resolveMaxWorkbookZipEntries(properties); + long maxBytes = SimilarAsinLimits.resolveMaxWorkbookUncompressedBytes(properties); try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(input)) { if (zipFile.size() > maxEntries) { throw new BusinessException("Excel 条目数超过上限: " + zipFile.size() + " entries > " + maxEntries + " entries"); @@ -246,55 +152,11 @@ public class SimilarAsinTaskService { } /** - * P1-1：per-task 最近 10 次提交结果（true=720712008 命中）。 - * 容量 10，环形覆盖；与 recentPoisonCursorByTask 配合实现 O(1) record。 - * 当近 5+ 次提交中命中率 ≥ 40% 触发 batch 强制降到 1（隔离毒行）。 + * P1-1：LLM 毒行（720712008 类）滑窗熔断状态。 + * 进程内 per-task 环形滑窗、命中率阈值判定与毒行识别见 SimilarAsinPoisonTracker； * finalizeTask / deleteTask 时清理，避免内存泄漏。 */ - private static final int POISON_WINDOW_SIZE = 10; - /** - * 触发降档所需最小样本数：5 → 3。 - * 5000 行任务里 batchSize=10 时 1 次 720712008 会让 1 整批 row markFailed， - * 等不到 5 个样本就已经损失大量行。降到 3 让滑窗更早响应。 - */ - private static final int POISON_MIN_SAMPLES = 3; - /** - * 命中率触发阈值：40% → 33%。 - * 配合 POISON_MIN_SAMPLES=3，最快 1/3 命中率即触发，避免滑窗误判可由 batchSize=1 - * 后的连续成功样本快速恢复（1 次成功就把命中率打到 0/3）。 - */ - private static final int POISON_HIT_RATIO_PERCENT = 33; - private final ConcurrentHashMap> recentPoisonByTask = new ConcurrentHashMap<>(); - private final ConcurrentHashMap recentPoisonCursorByTask = new ConcurrentHashMap<>(); - - /** - * 仅命中确定性输入失败： - * - "fields cannot be extracted from null values"（工作流空输入兜底） - * - 错误码 720712000（显式输入校验失败） - * 注意：曾经裸串匹配 "null values" 会把 "got null values during merge" 等 - * 非确定性场景误判成 deterministic，导致行被直接 markFailed 而不进 retry。 - */ - private static final Pattern DETERMINISTIC_INPUT_FAILURE_PATTERN = Pattern.compile( - "fields\\s+cannot\\s+be\\s+extracted\\s+from\\s+null\\s+values" - + "|\\b720712000\\b"); - - private static final List RESULT_HEADERS = List.of( - "id", - "asin", - "国家", - "价格", - "卖家名称", - "品牌", - "是否有货", - "相似度", - "是否符合类目", - "不符合理由", - "产品类目", - "状态", - "主图", - "阿里巴巴图片1", - "阿里巴巴图片2" - ); + private final SimilarAsinPoisonTracker poisonTracker = new SimilarAsinPoisonTracker(); private static final Set CATEGORY_RESULT_HEADER_ALIASES = Set.of( FailedStatusRowFilter.canonicalizeHeader("是否符合类目"), FailedStatusRowFilter.canonicalizeHeader("is_conform"), @@ -313,10 +175,6 @@ public class SimilarAsinTaskService { FailedStatusRowFilter.canonicalizeHeader("category") ); - private static final int IMG_COL_MAIN = 12; - private static final int IMG_COL_PUZZLE1 = 13; - private static final int IMG_COL_PUZZLE2 = 14; - private final LocalFileStorageService localFileStorageService; private final OssStorageService ossStorageService; private final StorageProperties storageProperties; @@ -357,6 +215,53 @@ public class SimilarAsinTaskService { taskScopeStateMapper, taskChunkMapper, fileTaskMapper, taskProgressSnapshotService, ossStorageService, transientPayloadStorageService, objectMapper); } + + /** + * spec 05 续（拆分第 2 轮）：解析载荷编解码与读取门面。 + * 纯函数（行恢复/hydrate）与 payload 读取、结果索引 JSON 构造均委托此组件。 + */ + private SimilarAsinPayloadSupport payloadSupport() { + return new SimilarAsinPayloadSupport( + objectMapper, transientPayloadStorageService, userApiSecretService, instanceMetadata); + } + + /** + * spec 05 续（拆分第 5b 轮）：chunk 载荷读取、读失败诊断与 orphan 行兜底。 + */ + private SimilarAsinChunkPayloadSupport chunkPayloadSupport() { + return new SimilarAsinChunkPayloadSupport( + objectMapper, transientPayloadStorageService, taskScopeStateMapper); + } + + /** + * spec 05 续（拆分第 6 轮）：文件构建进度、任务视图映射与计数。 + */ + private SimilarAsinTaskProgressSupport progressSupport() { + return new SimilarAsinTaskProgressSupport( + properties, objectMapper, fileTaskMapper, taskChunkMapper, taskProgressSnapshotService); + } + + /** + * spec 05 续（拆分第 6 轮）：实例归属判定与 per-task 分布式锁。 + */ + private SimilarAsinTaskOwnershipSupport ownershipSupport() { + return new SimilarAsinTaskOwnershipSupport( + fileTaskMapper, taskDistributedLockService, taskScopeStateMapper, + objectMapper, instanceMetadata, chunkPayloadSupport()); + } + + /** + * spec 05 续（拆分第 5c 轮）：结果文件（xlsx/zip）装配。 + * assembleExecutor 仍由本服务持有（shutdownAssembleExecutor 语义不变），装配器只借用。 + */ + private SimilarAsinResultWorkbookAssembler resultAssembler() { + return new SimilarAsinResultWorkbookAssembler( + properties, storageProperties, ossStorageService, objectMapper, + transientPayloadStorageService, imageEmbedder, imagePrefetchService, + taskChunkMapper, taskScopeStateMapper, assembleExecutor, + payloadSupport(), chunkPayloadSupport()); + } + @Autowired @Qualifier("taskQueueExecutor") private TaskExecutor taskQueueExecutor; @@ -481,7 +386,7 @@ public class SimilarAsinTaskService { if (input == null || !input.exists()) { throw new BusinessException("源文件不存在"); } - long maxBytes = resolveMaxSourceFileBytes(); + long maxBytes = SimilarAsinLimits.resolveMaxSourceFileBytes(properties); if (input.length() > maxBytes) { throw new BusinessException("源文件超过大小限制: " + source.getOriginalFilename() + " (" + input.length() + " bytes > " + maxBytes + " bytes)"); @@ -497,7 +402,7 @@ public class SimilarAsinTaskService { if (allRows.isEmpty()) { throw new BusinessException("未解析到有效 ASIN 数据"); } - int maxParseRows = resolveMaxParseRows(); + int maxParseRows = SimilarAsinLimits.resolveMaxParseRows(properties); if (allRows.size() > maxParseRows) { throw new BusinessException("解析行数超过上限: " + allRows.size() + " rows > " + maxParseRows + " rows"); } @@ -536,9 +441,9 @@ public class SimilarAsinTaskService { String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey); String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, mergedHeaders, groups, allRows); long payloadBuiltAt = System.nanoTime(); - String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); + String parsedPayloadPointer = payloadSupport().storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); long payloadStoredAt = System.nanoTime(); - task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, parsedPayloadPointer)); + task.setResultJson(payloadSupport().buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, parsedPayloadPointer)); task.setUpdatedAt(LocalDateTime.now()); fileTaskMapper.updateById(task); @@ -587,25 +492,25 @@ public class SimilarAsinTaskService { sourceFiles.size(), allRows.size(), groups.size(), - elapsedMs(startedAt, finishedAt), - elapsedMs(parseStartedAt, parsedAt), - elapsedMs(parsedAt, groupedAt), - elapsedMs(groupedAt, taskInsertedAt), - elapsedMs(taskInsertedAt, payloadBuiltAt), - elapsedMs(payloadBuiltAt, payloadStoredAt), - elapsedMs(payloadStoredAt, persistedAt), - elapsedMs(persistedAt, finishedAt)); + SimilarAsinPayloadSupport.elapsedMs(startedAt, finishedAt), + SimilarAsinPayloadSupport.elapsedMs(parseStartedAt, parsedAt), + SimilarAsinPayloadSupport.elapsedMs(parsedAt, groupedAt), + SimilarAsinPayloadSupport.elapsedMs(groupedAt, taskInsertedAt), + SimilarAsinPayloadSupport.elapsedMs(taskInsertedAt, payloadBuiltAt), + SimilarAsinPayloadSupport.elapsedMs(payloadBuiltAt, payloadStoredAt), + SimilarAsinPayloadSupport.elapsedMs(payloadStoredAt, persistedAt), + SimilarAsinPayloadSupport.elapsedMs(persistedAt, finishedAt)); return vo; } @Transactional public void activateTask(Long taskId, Long userId) { - try (TaskDistributedLockService.LockHandle ignored = requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { + try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { FileTaskEntity task = fileTaskMapper.selectById(taskId); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) { throw new BusinessException("任务不存在"); } - ensureTaskOwnedByCurrentInstance(task, "activate"); + ownershipSupport().ensureTaskOwnedByCurrentInstance(task, "activate"); if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) { throw new BusinessException("任务已结束"); } @@ -618,12 +523,12 @@ public class SimilarAsinTaskService { public SimilarAsinDashboardVo dashboard(Long userId) { SimilarAsinDashboardVo vo = new SimilarAsinDashboardVo(); - long completedWithFile = countCompletedTasksWithResultFile(userId); - long rawRunning = countTask(userId, STATUS_RUNNING); - long rawSuccess = countTask(userId, STATUS_SUCCESS); + long completedWithFile = progressSupport().countCompletedTasksWithResultFile(userId); + long rawRunning = progressSupport().countTask(userId, STATUS_RUNNING); + long rawSuccess = progressSupport().countTask(userId, STATUS_SUCCESS); vo.setPendingTaskCount(Math.max(0L, rawRunning - completedWithFile)); vo.setSuccessTaskCount(rawSuccess + completedWithFile); - vo.setFailedTaskCount(countTask(userId, STATUS_FAILED)); + vo.setFailedTaskCount(progressSupport().countTask(userId, STATUS_FAILED)); vo.setProcessedTaskCount(vo.getSuccessTaskCount() + vo.getFailedTaskCount()); return vo; } @@ -694,7 +599,7 @@ public class SimilarAsinTaskService { } SimilarAsinTaskLightVo item = new SimilarAsinTaskLightVo(); item.setTaskId(task.getId()); - item.setStatus(resolveDisplayTaskStatus(task)); + item.setStatus(progressSupport().resolveDisplayTaskStatus(task)); TaskFileJobEntity job = jobMap.get(taskId); boolean fileReady = job != null && job.getResultFileUrl() != null @@ -762,7 +667,7 @@ public class SimilarAsinTaskService { continue; } SimilarAsinTaskDetailVo detail = new SimilarAsinTaskDetailVo(); - detail.setTask(toTaskItem(task)); + detail.setTask(progressSupport().toTaskItem(task)); FileResultEntity resultRow = resultByTaskId.get(taskId); if (resultRow != null) { detail.getItems().addAll(historyAssembler().buildHistoryItems( @@ -774,7 +679,7 @@ public class SimilarAsinTaskService { } public void submitResult(Long taskId, SimilarAsinSubmitResultRequest request) { - try (TaskDistributedLockService.LockHandle ignored = requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { + try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { submitResultLocked(taskId, request); } } @@ -804,14 +709,14 @@ public class SimilarAsinTaskService { public void deleteTask(Long taskId, Long userId) { List payloads; - try (TaskDistributedLockService.LockHandle ignored = requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { + try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { payloads = transactionManager == null ? deleteTaskRecords(taskId, userId) : inNewTransaction(() -> deleteTaskRecords(taskId, userId)); } deleteTransientPayloads(payloads, taskId); taskCacheService.deleteTaskCache(taskId); - clearPoisonWindow(taskId); + poisonTracker.clear(taskId); } public void deleteHistory(Long resultId, Long userId) { @@ -887,27 +792,6 @@ public class SimilarAsinTaskService { return scopeCount != null && scopeCount > 0L; } - public String resolveResultDownloadUrl(Long resultId, Long userId) { - FileResultEntity row = fileResultMapper.selectById(resultId); - if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { - throw new BusinessException("记录不存在"); - } - if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { - throw new BusinessException("暂无可下载文件"); - } - return ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl()); - } - - public String resolveResultDownloadFilename(Long resultId, Long userId) { - FileResultEntity row = fileResultMapper.selectById(resultId); - if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { - throw new BusinessException("记录不存在"); - } - return row.getResultFilename() == null || row.getResultFilename().isBlank() - ? "similar-asin-" + resultId + ".xlsx" - : row.getResultFilename(); - } - public ResultDownloadInfo resolveResultDownloadInfo(Long resultId, Long userId) { FileResultEntity row = fileResultMapper.selectById(resultId); if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { @@ -920,7 +804,7 @@ public class SimilarAsinTaskService { ? "similar-asin-" + resultId + ".xlsx" : row.getResultFilename(); String contentType = row.getResultContentType() == null || row.getResultContentType().isBlank() - ? guessDownloadContentType(filename) + ? progressSupport().guessDownloadContentType(filename) : row.getResultContentType(); return new ResultDownloadInfo( ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl()), @@ -930,14 +814,6 @@ public class SimilarAsinTaskService { ); } - private String guessDownloadContentType(String filename) { - String normalized = filename == null ? "" : filename.trim().toLowerCase(Locale.ROOT); - if (normalized.endsWith(".zip")) { - return CONTENT_TYPE_ZIP; - } - return CONTENT_TYPE_XLSX; - } - public void finalizeStaleTasks() { // P1-7：stale 判定改用 Redis heartbeat 作为主信号。 // 历史实现先用 file_task.updated_at 过滤，再二次校验 Redis 心跳。 @@ -964,12 +840,12 @@ public class SimilarAsinTaskService { if (!isHeartbeatStale(task, thresholdMillis)) { continue; } - String owner = ownerFromTask(task); - if (!isOwnerCurrent(owner)) { + String owner = ownershipSupport().ownerFromTask(task); + if (!ownershipSupport().isOwnerCurrent(owner)) { log.warn("[similar-asin] 心跳超时任务归其他实例所有，由本实例（job 锁持有者）接管判死/恢复 taskId={} owner={} current={}", - task.getId(), owner, currentInstanceId()); + task.getId(), owner, ownershipSupport().currentInstanceId()); } - TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(task.getId(), 0L); + TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(task.getId(), 0L); if (taskLockHandle == null) { // 锁被 owner 实例/文件任务 worker 占用说明其仍活跃，静默跳过等下一轮（2 分钟后再判） continue; @@ -984,7 +860,7 @@ public class SimilarAsinTaskService { if (taskId == null || taskId <= 0) { return; } - TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(taskId, 0L); + TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(taskId, 0L); if (taskLockHandle == null) { return; } @@ -1034,7 +910,7 @@ public class SimilarAsinTaskService { throw new BusinessException("任务不是运行中状态"); } - ensureTaskOwnedByCurrentInstance(task, "submit result"); + ownershipSupport().ensureTaskOwnedByCurrentInstance(task, "submit result"); int chunkIndex = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics .chunkIndex(request.getChunkIndex()); int chunkTotal = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics @@ -1065,86 +941,11 @@ public class SimilarAsinTaskService { } private SubmittedTaskMetadata readSubmittedTaskMetadata(FileTaskEntity task) { - SimilarAsinParsedPayloadDto payload = readParsedPayload(task); + SimilarAsinParsedPayloadDto payload = payloadSupport().readParsedPayload(task); List sourceFiles = payload.getSourceFiles() == null ? List.of() : List.copyOf(payload.getSourceFiles()); - return new SubmittedTaskMetadata(rowCount(payload), sourceFiles); - } - - private int rowCount(SimilarAsinParsedPayloadDto payload) { - if (payload == null) { - return 0; - } - return resolveAllRows(payload).size(); - } - - /** - * 从解析载荷统一恢复全量行：优先 items（新规范结构），其次 allItems（旧结构）， - * 最后 groups 展开（最旧结构）。新旧结构均可恢复完整行集合，供 rowCount、 - * LLM 候选加载与结果文件组装复用。 - */ - static List resolveAllRows(SimilarAsinParsedPayloadDto payload) { - if (payload == null) { - return List.of(); - } - if (containsGroupRefs(payload)) { - // 新格式：分组携带索引引用，信任引用展开结果（越界/非法区间安全跳过） - return expandGroupRefs(payload); - } - List rows = payload.getItems(); - if (rows == null || rows.isEmpty()) { - rows = payload.getAllItems(); - } - if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { - rows = payload.getGroups().stream() - .filter(Objects::nonNull) - .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) - .toList(); - } - return rows == null ? List.of() : rows; - } - - private static boolean containsGroupRefs(SimilarAsinParsedPayloadDto payload) { - if (payload.getGroups() == null) { - return false; - } - for (SimilarAsinParsedGroupVo group : payload.getGroups()) { - if (group != null && (group.getStartIndex() != null || group.getEndIndex() != null)) { - return true; - } - } - return false; - } - - /** - * Task 6：按分组引用 [startIndex, endIndex) 从 items 展开组内行。 - * 引用越界或区间非法时安全跳过，不抛异常；展开不修改 payload 内部状态。 - */ - static List expandGroupRefs(SimilarAsinParsedPayloadDto payload) { - if (payload == null || payload.getGroups() == null || payload.getGroups().isEmpty()) { - return List.of(); - } - List rows = payload.getItems(); - if (rows == null || rows.isEmpty()) { - rows = payload.getAllItems(); - } - if (rows == null || rows.isEmpty()) { - return List.of(); - } - List expanded = new ArrayList<>(); - for (SimilarAsinParsedGroupVo group : payload.getGroups()) { - if (group == null) { - continue; - } - int start = group.getStartIndex() == null ? 0 : group.getStartIndex(); - int end = group.getEndIndex() == null ? 0 : group.getEndIndex(); - if (start < 0 || end <= start || end > rows.size()) { - continue; - } - expanded.addAll(rows.subList(start, end)); - } - return expanded; + return new SubmittedTaskMetadata(SimilarAsinPayloadSupport.rowCount(payload), sourceFiles); } private PersistSubmittedChunkResult persistSubmittedChunk(PreparedSubmittedChunk prepared) { @@ -1156,7 +957,7 @@ public class SimilarAsinTaskService { if (!STATUS_RUNNING.equals(task.getStatus())) { throw new BusinessException("任务不是运行中状态"); } - ensureTaskOwnedByCurrentInstance(task, "submit result"); + ownershipSupport().ensureTaskOwnedByCurrentInstance(task, "submit result"); boolean payloadPersisted = prepared.storedPayload() == null; TaskChunkEntity existing = findSubmittedChunk(taskId, prepared.scopeHash(), prepared.chunkIndex()); @@ -1190,7 +991,7 @@ public class SimilarAsinTaskService { upsertScopeState(taskId, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkTotal(), prepared.error(), prepared.done(), false); if (prepared.localFallback() && payloadPersisted) { - bindTaskToCurrentOwnerForLocalFallback( + ownershipSupport().bindTaskToCurrentOwnerForLocalFallback( task, prepared.scopeHash(), prepared.chunkIndex()); } SubmitContext context = new SubmitContext(task, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkIndex(), @@ -1267,17 +1068,17 @@ public class SimilarAsinTaskService { } chunks = chunks == null ? List.of() : chunks.stream() .filter(Objects::nonNull) - .filter(chunk -> !readChunkRows(chunk).isEmpty()) + .filter(chunk -> !chunkPayloadSupport().readChunkRows(chunk).isEmpty()) .toList(); if (chunks.isEmpty()) { return; } - FileResultEntity result = findOrCreateResultRecordForAssembly(task, allRowCount(task)); + FileResultEntity result = findOrCreateResultRecordForAssembly(task, payloadSupport().allRowCount(task)); if (result == null) { return; } TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( - task.getId(), MODULE_TYPE, result.getId(), buildTaskOwnerScopeKey(task)); + task.getId(), MODULE_TYPE, result.getId(), ownershipSupport().buildTaskOwnerScopeKey(task)); if (job == null || "SUCCESS".equals(job.getStatus())) { return; } @@ -1288,7 +1089,7 @@ public class SimilarAsinTaskService { touchJavaSideTaskActivity(task.getId()); } else if (isResultSubmissionComplete(task.getId())) { maybeFinalizeLlmJobLocked(task.getId(), new LlmBatchContext( - job.getId(), result.getId(), context.scopeHash(), context.chunkIndex(), 1, 1, currentInstanceId(), 0, null, null)); + job.getId(), result.getId(), context.scopeHash(), context.chunkIndex(), 1, 1, ownershipSupport().currentInstanceId(), 0, null, null)); } } @@ -1307,12 +1108,12 @@ public class SimilarAsinTaskService { if (chunks.isEmpty()) { return; } - FileResultEntity result = findOrCreateResultRecordForAssembly(task, allRowCount(task)); + FileResultEntity result = findOrCreateResultRecordForAssembly(task, payloadSupport().allRowCount(task)); if (result == null) { return; } TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( - task.getId(), MODULE_TYPE, result.getId(), buildTaskOwnerScopeKey(task)); + task.getId(), MODULE_TYPE, result.getId(), ownershipSupport().buildTaskOwnerScopeKey(task)); if (job == null || "SUCCESS".equals(job.getStatus())) { return; } @@ -1343,7 +1144,7 @@ public class SimilarAsinTaskService { boolean uploadComplete = isResultSubmissionComplete(taskId); long pendingLlmStates = countPendingLlmStates(taskId); long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE); - if (!hasPersistedResultRows(taskId)) { + if (!resultAssembler().hasPersistedResultRows(taskId)) { return false; } if (!uploadComplete) { @@ -1431,7 +1232,7 @@ public class SimilarAsinTaskService { if (chunkTotal != null) { scope.setChunkTotal(chunkTotal); } - scope.setReceivedChunkCount(countChunks(taskId, scopeHash)); + scope.setReceivedChunkCount(progressSupport().countChunks(taskId, scopeHash)); scope.setLastChunkAt(now); scope.setLastError(error); scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0); @@ -1458,7 +1259,7 @@ public class SimilarAsinTaskService { if (chunkTotal != null) { scope.setChunkTotal(chunkTotal); } - scope.setReceivedChunkCount(resolveReceivedChunkProgress(taskId, scopeHash, scope.getChunkTotal())); + scope.setReceivedChunkCount(progressSupport().resolveReceivedChunkProgress(taskId, scopeHash, scope.getChunkTotal())); scope.setLastChunkAt(now); scope.setLastError(error); scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0); @@ -1487,11 +1288,11 @@ public class SimilarAsinTaskService { if (items == null || items.isEmpty()) { return List.of(); } - String prompt = readAiPrompt(task); - String apiKey = readApiKey(task); - boolean imgSwitch = readImgSwitch(task); + String prompt = payloadSupport().readAiPrompt(task); + String apiKey = payloadSupport().readApiKey(task); + boolean imgSwitch = payloadSupport().readImgSwitch(task); boolean categorySwitch = readCategorySwitch(task); - int batchSize = resolveLlmBatchSize(imgSwitch); + int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, imgSwitch); List result = new ArrayList<>(); for (int i = 0; i < items.size(); i += batchSize) { List batch = items.subList(i, Math.min(i + batchSize, items.size())); @@ -1505,164 +1306,6 @@ public class SimilarAsinTaskService { return result; } - /** - * Task 9：按 id keyset 批量分页拉取 chunk，保持低内存读取。 - * 每轮只取 pageSize 条（id > lastId 升序），全部取回后按 chunkIndex 升序合并。 - * taskId 为 null 安全返回空；pageSize <= 0 回退默认 500； - * 中途查询失败抛项目约定异常，不返回半截结果。 - * 使用 QueryWrapper（列名直写）避免对 lambda 元数据缓存的依赖。 - */ - static List loadChunksKeyset(TaskChunkMapper mapper, Long taskId, String moduleType, int pageSize) { - if (taskId == null || mapper == null) { - return List.of(); - } - int batch = pageSize > 0 ? pageSize : 500; - List all = new ArrayList<>(); - long lastId = 0L; - while (true) { - List page; - try { - page = mapper.selectList(new com.baomidou.mybatisplus.core.conditions.query.QueryWrapper() - .eq("task_id", taskId) - .eq("module_type", moduleType) - .gt("id", lastId) - .orderByAsc("id") - .last("limit " + batch)); - } catch (Exception ex) { - throw new BusinessException("chunk keyset 分页查询失败: " + ex.getMessage(), ex); - } - if (page == null || page.isEmpty()) { - break; - } - all.addAll(page); - lastId = page.getLast().getId(); - if (page.size() < batch) { - break; - } - } - all.sort(java.util.Comparator.comparing(TaskChunkEntity::getChunkIndex, - java.util.Comparator.nullsLast(java.util.Comparator.naturalOrder()))); - return all; - } - - /** - * Task 10：把 chunk 结果行建立成 rowKey → chunkKey 的批量索引。 - * 供 assignLlmRowsToChunks 使用，把跨 chunk 线性扫描降为 O(1) 查找。 - * 同一 rowKey 出现在多个 chunk 时保留第一个（putIfAbsent），行为确定。 - */ - Map indexRowsByChunkKey(Map> rowsByChunk) { - Map index = new java.util.HashMap<>(); - if (rowsByChunk == null || rowsByChunk.isEmpty()) { - return index; - } - for (Map.Entry> entry : rowsByChunk.entrySet()) { - if (entry.getValue() == null) { - continue; - } - for (String rowKey : entry.getValue().keySet()) { - if (rowKey == null || rowKey.isBlank()) { - continue; - } - index.putIfAbsent(rowKey, entry.getKey()); - } - } - return index; - } - - /** - * Task 11：LLM 结果合并前按稳定 rowKey 一次性去重。 - * 优先 rowToken，缺失时回退归一化的 legacy key（id::ASIN::country）， - * 保留首次出现顺序。合并路径此前对每个重复行都重复 expand/分配/写回，是 O(n²) 热点。 - */ - List dedupeRowsByRowKey(List llmRows) { - List deduped = new ArrayList<>(); - if (llmRows == null || llmRows.isEmpty()) { - return deduped; - } - Set seenKeys = new java.util.HashSet<>(); - for (SimilarAsinResultRowDto row : llmRows) { - if (row == null) { - continue; - } - String key = rowKey(row); - if (key.isBlank() || !seenKeys.add(key)) { - continue; - } - deduped.add(row); - } - return deduped; - } - - /** - * Task 10：基于 rowKey 索引为 llm 回传行分配归属 chunk。 - * 命中索引 → 归属该 chunk；未命中且有有效 fallback（chunkScopeHash + chunkIndex） - * 且 fallback chunk 存在 → 归属 fallback；否则进 orphan 列表。 - * 与原实现逐 chunk 线性扫描语义完全一致，但每个行查找降为 O(1)。 - */ - Map> assignLlmRowsToChunks( - Map> rowsByChunk, - List llmRows, - Map rowKeyIndex, - String chunkScopeHash, - Integer chunkIndex, - List orphans) { - Map> mergeRowsByChunk = new LinkedHashMap<>(); - if (llmRows == null || llmRows.isEmpty() || orphans == null) { - return mergeRowsByChunk; - } - String fallbackKey = chunkScopeHash != null && !chunkScopeHash.isBlank() && chunkIndex != null - ? chunkStorageKey(chunkScopeHash, chunkIndex) : null; - boolean fallbackValid = fallbackKey != null && rowsByChunk != null && rowsByChunk.containsKey(fallbackKey); - for (SimilarAsinResultRowDto expandedRow : llmRows) { - if (expandedRow == null) { - continue; - } - String rowKey = rowKey(expandedRow); - if (rowKey.isBlank()) { - continue; - } - String chunkKey = rowKeyIndex == null ? null : rowKeyIndex.get(rowKey); - if (chunkKey != null) { - mergeRowsByChunk.computeIfAbsent(chunkKey, ignored -> new LinkedHashMap<>()) - .put(rowKey, expandedRow); - } else if (fallbackValid) { - mergeRowsByChunk.computeIfAbsent(fallbackKey, ignored -> new LinkedHashMap<>()) - .put(rowKey, expandedRow); - } else { - orphans.add(expandedRow); - log.error("[similar-asin] llm row has no submitted chunk rowKey={} asin={} country={}", - rowKey, expandedRow.getAsin(), expandedRow.getCountry()); - } - } - return mergeRowsByChunk; - } - - private void applyLlmToPersistedChunks(FileTaskEntity task, Runnable progressHook) { - if (task == null || task.getId() == null) { - return; - } - Map> allRowsByBaseId = loadAllRowsByBaseId(task); - List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, task.getId()) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getChunkIndex)); - log.info("[similar-asin] async llm start taskId={} chunks={}", task.getId(), chunks.size()); - for (TaskChunkEntity chunk : chunks) { - Map persistedRows = readChunkRows(chunk); - if (persistedRows.isEmpty()) { - continue; - } - List unresolvedRows = collectPendingLlmRows(persistedRows.values()); - if (unresolvedRows.isEmpty()) { - continue; - } - List llmRows = applyLlmInBatches(unresolvedRows, task, progressHook); - mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), llmRows); - log.info("[similar-asin] async llm chunk merged taskId={} chunk={} unresolved={} merged={}", - task.getId(), chunk.getChunkIndex(), unresolvedRows.size(), llmRows.size()); - } - } - private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List rows) { if (rows == null || rows.isEmpty()) { return; @@ -1678,31 +1321,33 @@ public class SimilarAsinTaskService { if (chunk == null) { return; } - Map persistedRows = readChunkRows(chunk); + Map persistedRows = chunkPayloadSupport().readChunkRows(chunk); int mergedRowCount = persistedRows.size() + rows.size(); // Task 13：单次合并后总行数超过上限时，从最旧行（存量优先）开始降级到 orphan 兜底， // chunk 保持在上限内不无界增长；assemble 阶段 putIfAbsent 合并回结果不丢数据。 - if (mergedRowCount > getChunkMergeMaxRows()) { - splitChunkMergeOverflow(taskId, persistedRows, rows, chunk, mergedRowCount - getChunkMergeMaxRows()); + if (mergedRowCount > SimilarAsinLimits.getChunkMergeMaxRows(properties)) { + splitChunkMergeOverflow(taskId, persistedRows, rows, chunk, + mergedRowCount - SimilarAsinLimits.getChunkMergeMaxRows(properties)); if (rows.isEmpty()) { return; } } for (SimilarAsinResultRowDto row : rows) { - persistedRows.put(rowKey(row), row); + persistedRows.put(SimilarAsinChunkMergeSupport.rowKey(row), row); } String payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败"); // Task 13：合并后 payload 字节超过上限时，从最旧行开始降级到 orphan 兜底； // 降到只剩一行仍超上限时抛异常拒绝合并，防止无界 payload。 long payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length; - if (payloadBytes > getChunkMergePayloadMaxBytes()) { - demoteRowsToOrphan(taskId, persistedRows, payloadBytes - getChunkMergePayloadMaxBytes()); + if (payloadBytes > SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)) { + demoteRowsToOrphan(taskId, persistedRows, + payloadBytes - SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)); payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败"); payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length; - if (payloadBytes > getChunkMergePayloadMaxBytes() && persistedRows.size() <= 1) { + if (payloadBytes > SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties) && persistedRows.size() <= 1) { throw new BusinessException("相似ASIN分片载荷超字节上限 taskId=" + taskId + " scopeHash=" + scopeHash + " chunk=" + chunkIndex - + " bytes=" + payloadBytes + " limit=" + getChunkMergePayloadMaxBytes()); + + " bytes=" + payloadBytes + " limit=" + SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)); } } String oldPayload = chunk.getPayloadJson(); @@ -1736,16 +1381,6 @@ public class SimilarAsinTaskService { throw new IllegalStateException("相似ASIN分片载荷更新失败"); } - private int getChunkMergeMaxRows() { - int limit = properties.getChunkMergeMaxRows(); - return limit > 0 ? limit : 50000; - } - - private long getChunkMergePayloadMaxBytes() { - long limit = properties.getChunkMergePayloadMaxBytes(); - return limit > 0 ? limit : 16L * 1024L * 1024L; - } - /** * Task 13：按行数上限从最旧行开始降级到 orphan 兜底，保证合并后 chunk 行数不超过上限。 * 最旧优先：先降级存量行（LinkedHashMap 表头），不够再从新增行表头补齐； @@ -1772,9 +1407,10 @@ public class SimilarAsinTaskService { it.remove(); } } - persistOrphanLlmRows(taskId, demoted); + chunkPayloadSupport().persistOrphanLlmRows(taskId, demoted); log.warn("[similar-asin] chunk merge row-limit exceeded taskId={} chunk={} mergedRows={} limit={} demoted={}", - taskId, chunk.getChunkIndex(), mergedRowCountOf(persistedRows, rows), getChunkMergeMaxRows(), demoted.size()); + taskId, chunk.getChunkIndex(), SimilarAsinChunkMergeSupport.mergedRowCountOf(persistedRows, rows), + SimilarAsinLimits.getChunkMergeMaxRows(properties), demoted.size()); } /** @@ -1793,16 +1429,12 @@ public class SimilarAsinTaskService { } } if (!demoted.isEmpty()) { - persistOrphanLlmRows(taskId, demoted); + chunkPayloadSupport().persistOrphanLlmRows(taskId, demoted); log.warn("[similar-asin] chunk merge byte-limit exceeded taskId={} rows={} demoted={} releasedBytes={}", taskId, demoted.size(), demoted.size(), releasedBytes); } } - private static int mergedRowCountOf(Map persistedRows, List rows) { - return persistedRows.size() + rows.size(); - } - private long estimateRowBytes(String rowKey, SimilarAsinResultRowDto row) { try { String json = objectMapper.writeValueAsString(row); @@ -1812,45 +1444,16 @@ public class SimilarAsinTaskService { } } - private List expandRows(List rows, - Map> allRowsByBaseId) { - if (rows == null || rows.isEmpty()) { - return List.of(); - } - List result = new ArrayList<>(); - Set emittedKeys = new LinkedHashSet<>(); - for (SimilarAsinResultRowDto row : rows) { - String key = firstNonBlank(normalize(row.getRowToken()), rowKey(row)); - if (emittedKeys.add(key)) { - result.add(row); - } - } - return result; - } - private Map> loadAllRowsByBaseId(FileTaskEntity task) { try { - SimilarAsinParsedPayloadDto payload = readParsedPayload(task); - return SimilarAsinGroupingConverter.groupRowsByBaseId(resolveAllRows(payload)); + SimilarAsinParsedPayloadDto payload = payloadSupport().readParsedPayload(task); + return SimilarAsinGroupingConverter.groupRowsByBaseId(SimilarAsinPayloadSupport.resolveAllRows(payload)); } catch (Exception ex) { log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage()); return new LinkedHashMap<>(); } } - private List collectPendingLlmRows(java.util.Collection rows) { - if (rows == null) { - return List.of(); - } - List pendingRows = new ArrayList<>(); - for (SimilarAsinResultRowDto row : rows) { - if (row != null && !hasResolvedLlmFields(row) && shouldInspectRow(row)) { - pendingRows.add(row); - } - } - return pendingRows; - } - private List collectPendingLlmCandidates(List chunks, Map> allRowsByBaseId) { if (chunks == null || chunks.isEmpty()) { @@ -1858,12 +1461,12 @@ public class SimilarAsinTaskService { } List candidates = new ArrayList<>(); for (TaskChunkEntity chunk : chunks) { - Map persistedRows = readChunkRows(chunk); + Map persistedRows = chunkPayloadSupport().readChunkRows(chunk); if (persistedRows.isEmpty()) { continue; } List unresolvedRows = - enrichRowsForLlm(collectPendingLlmRows(persistedRows.values()), allRowsByBaseId); + enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()), allRowsByBaseId); for (SimilarAsinResultRowDto row : unresolvedRows) { candidates.add(new LlmCandidate(chunk.getScopeHash(), chunk.getChunkIndex(), row)); } @@ -1875,7 +1478,7 @@ public class SimilarAsinTaskService { if (groups == null || groups.isEmpty()) { return List.of(); } - int limit = Math.min(resolvePreviewLimit(), groups.size()); + int limit = Math.min(SimilarAsinLimits.resolvePreviewLimit(properties), groups.size()); List preview = new ArrayList<>(limit); for (int i = 0; i < limit; i++) { SimilarAsinParsedGroupVo group = groups.get(i); @@ -1910,7 +1513,7 @@ public class SimilarAsinTaskService { if (rows == null || rows.isEmpty()) { return List.of(); } - int limit = Math.min(resolvePreviewLimit(), rows.size()); + int limit = Math.min(SimilarAsinLimits.resolvePreviewLimit(properties), rows.size()); List preview = new ArrayList<>(limit); for (int i = 0; i < limit; i++) { preview.add(copyPreviewRow(rows.get(i))); @@ -2021,56 +1624,6 @@ public class SimilarAsinTaskService { return trimmed.substring(0, maxLength) + "..."; } - private int allRowCount(FileTaskEntity task) { - try { - return rowCount(readParsedPayload(task)); - } catch (Exception ex) { - log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage()); - return 0; - } - } - - private String readAiPrompt(FileTaskEntity task) { - try { - return readParsedPayload(task).getAiPrompt(); - } catch (Exception ignored) { - return ""; - } - } - - /** 密钥读取：任务 payload 优先（兼容老任务与显式覆盖），为空时按任务归属用户从服务端密钥表兜底。 */ - private String readApiKey(FileTaskEntity task) { - try { - String fromPayload = normalize(readParsedPayload(task).getApiKey()); - if (!fromPayload.isEmpty()) { - return fromPayload; - } - } catch (Exception ex) { - log.warn("[similar-asin] 读取任务 payload 密钥失败，尝试用户密钥兜底 taskId={} err={}", - task.getId(), ex.getMessage()); - } - String fromUserSecret = userApiSecretService.findPlainValue( - task.getUserId(), UserSecretModule.SIMILAR_ASIN.key()); - if (fromUserSecret.isEmpty()) { - log.warn("[similar-asin] 任务未携带密钥且用户未配置密钥，LLM 检测将保留原始行 taskId={} userId={}", - task.getId(), task.getUserId()); - } - return fromUserSecret; - } - - private boolean readImgSwitch(FileTaskEntity task) { - try { - return Boolean.TRUE.equals(readParsedPayload(task).getImgSwitch()); - } catch (Exception ignored) { - return false; - } - } - - private int resolveLlmBatchSize(boolean imgSwitch) { - int configured = imgSwitch ? properties.getLlmBatchSize() : properties.getLlmTextOnlyBatchSize(); - return Math.max(1, configured); - } - private FinalizeTaskResult finalizeTask(FileTaskEntity task, String error, SubmittedTaskMetadata taskMetadata, @@ -2078,7 +1631,7 @@ public class SimilarAsinTaskService { String finalError = error; int rowCount = taskMetadata == null ? 0 : taskMetadata.rowCount(); FileResultEntity result = null; - boolean hasPersistedResultRows = hasPersistedResultRows(task.getId()); + boolean hasPersistedResultRows = resultAssembler().hasPersistedResultRows(task.getId()); List rows = fileResultMapper.selectList(new LambdaQueryWrapper() .eq(FileResultEntity::getTaskId, task.getId()) .eq(FileResultEntity::getModuleType, MODULE_TYPE) @@ -2110,7 +1663,7 @@ public class SimilarAsinTaskService { result.setRowCount(rowCount > 0 ? rowCount : result.getRowCount()); boolean shouldAssembleResult = assembleWorkbook && (!failed || hasPersistedResultRows); if (shouldAssembleResult) { - result.setResultFilename(safeFileStem(result.getSourceFilename()) + "-result.xlsx"); + result.setResultFilename(SimilarAsinResultTextSupport.safeFileStem(result.getSourceFilename()) + "-result.xlsx"); result.setResultFileUrl(null); result.setResultFileSize(0L); result.setResultContentType(CONTENT_TYPE_XLSX); @@ -2131,7 +1684,7 @@ public class SimilarAsinTaskService { } taskCacheService.deleteTaskCache(result.taskId()); if (result.terminal()) { - clearPoisonWindow(result.taskId()); + poisonTracker.clear(result.taskId()); } } @@ -2154,7 +1707,7 @@ public class SimilarAsinTaskService { return null; } TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( - task.getId(), MODULE_TYPE, result.getId(), buildTaskOwnerScopeKey(task)); + task.getId(), MODULE_TYPE, result.getId(), ownershipSupport().buildTaskOwnerScopeKey(task)); if (dispatchWhenIdle && job != null && "RUNNING".equals(job.getStatus()) @@ -2165,7 +1718,7 @@ public class SimilarAsinTaskService { } private FileResultEntity createResultRecordForAssembly(FileTaskEntity task, int rowCount) { - SimilarAsinParsedPayloadDto payload = readParsedPayload(task); + SimilarAsinParsedPayloadDto payload = payloadSupport().readParsedPayload(task); return createResultRecordForAssembly(task, rowCount, payload.getSourceFiles() == null ? List.of() : payload.getSourceFiles()); } @@ -2198,11 +1751,11 @@ public class SimilarAsinTaskService { if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { throw new BusinessException("任务不存在"); } - ensureTaskOwnedByCurrentInstance(task, "assemble result file"); - job.setScopeKey(buildTaskOwnerScopeKey(task)); - if (!isJobOwnedByCurrentInstance(job)) { + ownershipSupport().ensureTaskOwnedByCurrentInstance(task, "assemble result file"); + job.setScopeKey(ownershipSupport().buildTaskOwnerScopeKey(task)); + if (!ownershipSupport().isJobOwnedByCurrentInstance(job)) { log.info("[相似ASIN] 跳过结果文件任务，因为归属实例不是当前实例 文件任务ID={} 任务ID={} 归属实例={} 当前实例={}", - job.getId(), job.getTaskId(), ownerFromScopeKey(job.getScopeKey()), currentInstanceId()); + job.getId(), job.getTaskId(), ownershipSupport().ownerFromScopeKey(job.getScopeKey()), ownershipSupport().currentInstanceId()); return false; } FileResultEntity result = fileResultMapper.selectById(job.getResultId()); @@ -2214,8 +1767,8 @@ public class SimilarAsinTaskService { .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) .orderByAsc(TaskChunkEntity::getChunkIndex)); Map> allRowsByBaseId = loadAllRowsByBaseId(task); - int batchSize = resolveLlmBatchSize(readImgSwitch(task)); - int llmWorkUnits = countLlmWorkUnits(chunks, batchSize); + int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, payloadSupport().readImgSwitch(task)); + int llmWorkUnits = ownershipSupport().countLlmWorkUnits(chunks, batchSize); int totalProgressUnits = Math.max(3, llmWorkUnits + 3); if (countPendingLlmStates(task.getId()) > 0) { finalizeTimedOutLlmStatesForTask(task.getId()); @@ -2223,7 +1776,7 @@ public class SimilarAsinTaskService { if (countPendingLlmStates(task.getId()) > 0) { taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); touchJavaSideTaskActivity(task.getId()); - saveFileBuildProgress(task, job, totalProgressUnits, 1, "LLM 已提交，等待结果"); + progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, 1, "LLM 已提交，等待结果"); return false; } // P0-3：stale-recovery / 异常路径兜底 —— maybeFinalizeLlmJob 可能未能成功 flush @@ -2240,18 +1793,18 @@ public class SimilarAsinTaskService { throw new BusinessException("刷新缓冲区 LLM 结果失败：" + firstNonBlank(flushEx.getMessage(), "未知错误")); } - saveFileBuildProgress(task, job, totalProgressUnits, 0, "正在提交 LLM"); + progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, 0, "正在提交 LLM"); boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId); if (pendingLlm) { taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); touchJavaSideTaskActivity(task.getId()); - saveFileBuildProgress(task, job, totalProgressUnits, 1, "LLM 已提交，等待结果"); + progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, 1, "LLM 已提交，等待结果"); return false; } if (STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(task.getId())) { taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); touchJavaSideTaskActivity(task.getId()); - saveFileBuildProgress(task, job, totalProgressUnits, Math.max(1, llmWorkUnits), + progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, Math.max(1, llmWorkUnits), "等待 Python 上传，每 " + batchSize + " 行提交一次 LLM"); return false; } @@ -2266,7 +1819,7 @@ public class SimilarAsinTaskService { if (savedSwitch != null && !savedSwitch.isNull()) { return savedSwitch.asBoolean(false); } - return Boolean.TRUE.equals(readParsedPayload(task).getCategorySwitch()); + return Boolean.TRUE.equals(payloadSupport().readParsedPayload(task).getCategorySwitch()); } catch (Exception ex) { throw new BusinessException("读取相似ASIN类目检测开关失败", ex); } @@ -2278,13 +1831,13 @@ public class SimilarAsinTaskService { return; } try (TaskDistributedLockService.LockHandle ignored = - requireTaskLock(job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)) { + ownershipSupport().requireTaskLock(job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)) { Long finalizedTaskId = transactionManager == null ? persistResultFileJobFailure(job, message) : inNewTransaction(() -> persistResultFileJobFailure(job, message)); if (finalizedTaskId != null) { taskCacheService.deleteTaskCache(finalizedTaskId); - clearPoisonWindow(finalizedTaskId); + poisonTracker.clear(finalizedTaskId); } } } @@ -2294,10 +1847,10 @@ public class SimilarAsinTaskService { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { return null; } - String owner = ownerFromTask(task); - if (!isOwnerCurrent(owner)) { + String owner = ownershipSupport().ownerFromTask(task); + if (!ownershipSupport().isOwnerCurrent(owner)) { log.warn("[similar-asin] finalize exhausted result job from another instance taskId={} jobId={} owner={} current={}", - task.getId(), job.getId(), owner, currentInstanceId()); + task.getId(), job.getId(), owner, ownershipSupport().currentInstanceId()); } String failureMessage = firstNonBlank(message, "相似ASIN结果文件生成失败"); LocalDateTime now = LocalDateTime.now(); @@ -2317,7 +1870,7 @@ public class SimilarAsinTaskService { result.setResultFileSize(0L); fileResultMapper.updateById(result); } - saveFileBuildProgress(task, job, 1, 1, failureMessage); + progressSupport().saveFileBuildProgress(task, job, 1, 1, failureMessage); return task.getId(); } @@ -2337,14 +1890,14 @@ public class SimilarAsinTaskService { // 直连模式：轮询器退化为"兜底调度器"，把还挂着 PENDING 的任务 // 重新调度一次批量提交（submitLlmBatch 同步直连），新任务本就走直连。 schedulePendingLlmBatches(); - List states = listOwnedPendingLlmStates(); + List states = ownershipSupport().listOwnedPendingLlmStates(); if (states == null || states.isEmpty()) { return; } log.info("[similar-asin] llm poll fallback picked pending states count={}", states.size()); Map> stateIdsByTaskId = new LinkedHashMap<>(); for (TaskScopeStateEntity state : states) { - if (state == null || state.getId() == null || state.getTaskId() == null || !isLlmStateOwnedByCurrentInstance(state)) { + if (state == null || state.getId() == null || state.getTaskId() == null || !ownershipSupport().isLlmStateOwnedByCurrentInstance(state)) { continue; } stateIdsByTaskId.computeIfAbsent(state.getTaskId(), key -> new ArrayList<>()).add(state.getId()); @@ -2362,7 +1915,7 @@ public class SimilarAsinTaskService { * 批量提交，新批次走 submitLlmBatch 同步直连，由提交路径落 DONE 缓冲/merge。 */ private void schedulePendingLlmBatches() { - List states = listOwnedPendingLlmStates(); + List states = ownershipSupport().listOwnedPendingLlmStates(); if (states == null || states.isEmpty()) { return; } @@ -2376,7 +1929,7 @@ public class SimilarAsinTaskService { taskIds.size()); for (Long taskId : taskIds) { taskQueueExecutor.execute(() -> { - TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(taskId, 0L); + TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(taskId, 0L); if (taskLockHandle == null) { return; } @@ -2385,9 +1938,9 @@ public class SimilarAsinTaskService { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { return; } - FileResultEntity result = findOrCreateResultRecordForAssembly(task, allRowCount(task)); + FileResultEntity result = findOrCreateResultRecordForAssembly(task, payloadSupport().allRowCount(task)); TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( - task.getId(), MODULE_TYPE, result.getId(), buildTaskOwnerScopeKey(task)); + task.getId(), MODULE_TYPE, result.getId(), ownershipSupport().buildTaskOwnerScopeKey(task)); if (job == null || "SUCCESS".equals(job.getStatus())) { return; } @@ -2405,7 +1958,7 @@ public class SimilarAsinTaskService { if (taskId == null || stateIds == null || stateIds.isEmpty()) { return; } - TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(taskId, 0L); + TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(taskId, 0L); if (taskLockHandle == null) { return; } @@ -2435,14 +1988,14 @@ public class SimilarAsinTaskService { if (chunks == null || chunks.isEmpty()) { return countPendingLlmStates(task.getId()) > 0; } - String prompt = readAiPrompt(task); - String apiKey = readApiKey(task); - boolean imgSwitch = readImgSwitch(task); + String prompt = payloadSupport().readAiPrompt(task); + String apiKey = payloadSupport().readApiKey(task); + boolean imgSwitch = payloadSupport().readImgSwitch(task); boolean categorySwitch = readCategorySwitch(task); - int batchSize = resolveLlmBatchSize(imgSwitch); + int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, imgSwitch); // P1-1：检测到 720712008 风暴时强制把 batch 降到 1，隔离毒行； // 持续 5+ 次提交命中率 ≥ 40% 才会触发，正常波动不影响吞吐。 - if (isPoisonStormActive(task.getId())) { + if (poisonTracker.isStormActive(task.getId())) { log.warn("[similar-asin] poison-storm detected, force batchSize=1 taskId={} originalBatchSize={}", task.getId(), batchSize); batchSize = 1; @@ -2481,7 +2034,7 @@ public class SimilarAsinTaskService { // P1-6: 防止 Python 端长时间慢回传时零头永久挂着：job.updatedAt 距今 ≥ llmFlushPendingMinutes 分钟则强制 flush。 if (!flushRemainder && readyCandidates.size() > 0) { LocalDateTime jobUpdatedAt = job.getUpdatedAt(); - long pendingFlushMillis = llmFlushPendingMillis(); + long pendingFlushMillis = SimilarAsinLimits.llmFlushPendingMillis(properties); if (jobUpdatedAt != null && Duration.between(jobUpdatedAt, LocalDateTime.now()).toMillis() >= pendingFlushMillis) { flushRemainder = true; @@ -2584,7 +2137,7 @@ public class SimilarAsinTaskService { return null; } String rowToken = normalize(row.getRowToken()); - String resultKey = rowKey(row); + String resultKey = SimilarAsinChunkMergeSupport.rowKey(row); for (SimilarAsinParsedRowVo candidate : candidates) { if (candidate == null) { continue; @@ -2592,7 +2145,7 @@ public class SimilarAsinTaskService { if (!rowToken.isBlank() && rowToken.equals(normalize(candidate.getRowToken()))) { return candidate; } - if (!resultKey.isBlank() && resultKey.equals(rowKey(candidate))) { + if (!resultKey.isBlank() && resultKey.equals(SimilarAsinChunkMergeSupport.rowKey(candidate))) { return candidate; } } @@ -2675,7 +2228,7 @@ public class SimilarAsinTaskService { String message = firstNonBlank(ex.getMessage(), "LLM submit failed"); log.warn("[similar-asin] llm submit failed taskId={} jobId={} rows={} batch={}/{} err={}", task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message); - recordLlmSubmitOutcome(task.getId(), LlmFailureClassifier.isPoisonRow(message)); + poisonTracker.recordSubmitOutcome(task.getId(), SimilarAsinPoisonTracker.isPoisonRow(message)); mergeLlmRowsIntoChunk(task, null, null, @@ -2705,11 +2258,11 @@ public class SimilarAsinTaskService { } // 落一条 DONE state 承载缓冲 pointer（对齐直连同步 immediate 路径）； // 缓冲失败/关闭时回退立即 merge，结果不丢失。 - if (isLlmResultBufferEnabled()) { + if (SimilarAsinLimits.isLlmResultBufferEnabled(properties)) { TaskScopeStateEntity doneState = persistImmediateLlmDoneState(task, result, job, batchRows, batchScopeKey, batchScopeHash, batchIndex, batchTotal, "llm-direct"); if (doneState != null) { - bufferLlmRowsOrMerge(doneState, readLlmBatchContext(doneState), llmRows, task, allRowsByBaseId); + bufferLlmRowsOrMerge(doneState, SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, doneState), llmRows, task, allRowsByBaseId); return false; } } @@ -2733,13 +2286,13 @@ public class SimilarAsinTaskService { List batchRows = readLlmBatchRows(state); if (batchRows == null || batchRows.isEmpty()) { markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 批次载荷缺失"); - maybeFinalizeLlmJob(state.getTaskId(), readLlmBatchContext(state)); + maybeFinalizeLlmJob(state.getTaskId(), SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state)); return true; } - LlmBatchContext context = readLlmBatchContext(state); - String prompt = readAiPrompt(task); - String apiKey = readApiKey(task); - boolean imgSwitch = readImgSwitch(task); + LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); + String prompt = payloadSupport().readAiPrompt(task); + String apiKey = payloadSupport().readApiKey(task); + boolean imgSwitch = payloadSupport().readImgSwitch(task); boolean categorySwitch = readCategorySwitch(task); boolean submitted = submitLlmBatch(task, null, taskFileJobService.findById( context == null ? null : context.jobId()), @@ -2775,7 +2328,7 @@ public class SimilarAsinTaskService { if (state == null || state.getId() == null || !isLlmStateTimedOut(state)) { continue; } - LlmBatchContext context = readLlmBatchContext(state); + LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); if (context == null || context.resultId() == null) { markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 批次上下文缺失"); continue; @@ -2797,89 +2350,6 @@ public class SimilarAsinTaskService { } } - /** - * P1-1：记录一次 LLM 提交结果是否触发了 720712008 类毒行。 - * 滑窗大小 POISON_WINDOW_SIZE=10，环形覆盖：满了之后从 0 重新开始。 - */ - private void recordLlmSubmitOutcome(Long taskId, boolean poisonHit) { - if (taskId == null) { - return; - } - AtomicReferenceArray ring = recentPoisonByTask.computeIfAbsent(taskId, - k -> new AtomicReferenceArray<>(POISON_WINDOW_SIZE)); - AtomicInteger cursor = recentPoisonCursorByTask.computeIfAbsent(taskId, k -> new AtomicInteger()); - int idx = Math.floorMod(cursor.getAndIncrement(), POISON_WINDOW_SIZE); - ring.set(idx, poisonHit); - } - - /** - * P1-1：判断当前 task 是否处于 720712008 风暴中。 - * 阈值：近 POISON_MIN_SAMPLES=3+ 次提交、命中率 ≥ POISON_HIT_RATIO_PERCENT=33% → 触发降档。 - */ - private boolean isPoisonStormActive(Long taskId) { - if (taskId == null) { - return false; - } - AtomicReferenceArray ring = recentPoisonByTask.get(taskId); - if (ring == null) { - return false; - } - int hits = 0; - int total = 0; - for (int i = 0; i < POISON_WINDOW_SIZE; i++) { - Boolean v = ring.get(i); - if (v != null) { - total++; - if (Boolean.TRUE.equals(v)) { - hits++; - } - } - } - return total >= POISON_MIN_SAMPLES && hits * 100 / total >= POISON_HIT_RATIO_PERCENT; - } - - /** - * P1-1：任务终结 / 删除时调用，清理 per-task 滑窗记录避免内存泄漏。 - */ - private void clearPoisonWindow(Long taskId) { - if (taskId == null) { - return; - } - recentPoisonByTask.remove(taskId); - recentPoisonCursorByTask.remove(taskId); - } - - /** - * P2-10：LLM 失败分类。poll/retry 链已移除，仅保留直连提交路径仍在使用的判定。 - */ - static final class LlmFailureClassifier { - private LlmFailureClassifier() {} - - private static String norm(String msg) { - if (msg == null) { - return ""; - } - String trimmed = msg.trim(); - if (trimmed.isEmpty()) { - return ""; - } - return trimmed.toLowerCase(Locale.ROOT); - } - - /** - * P1-5：720712008 / "node executed out of limit" / "工作流节点执行超限" - * 这一类强烈暗示“单点死循环”，命中即隔离降档（batchSize=1）， - * 避免毒行把整批拖垮。 - */ - static boolean isPoisonRow(String msg) { - String n = norm(msg); - return n.contains("720712008") - || n.contains("node executed out of limit") - || n.contains("execution limit") - || n.contains("工作流节点执行超限"); - } - } - /** * P0-4 / P2-9：pending 重跑链路直接查 DB 取 task（poll 链移除后无同线程复用上下文）。 */ @@ -2900,19 +2370,14 @@ public class SimilarAsinTaskService { return loadAllRowsByBaseId(task); } - private long llmFlushPendingMillis() { - int minutes = properties.getLlmFlushPendingMinutes(); - return Math.max(1, minutes) * 60_000L; - } - private String emptyLlmResultMessage(List llmRows, int expectedRows) { if (llmRows == null || llmRows.isEmpty()) { return expectedRows > 0 ? "LLM async workflow returned empty result rows" : ""; } long unresolved = llmRows.stream() .filter(row -> row != null - && !hasResolvedLlmFields(row) - && !isTechnicalLlmFailure(row.getError())) + && !SimilarAsinResultTextSupport.hasResolvedLlmFields(row) + && !SimilarAsinResultTextSupport.isTechnicalLlmFailure(row.getError())) .count(); if (unresolved <= 0) { return ""; @@ -2937,10 +2402,10 @@ public class SimilarAsinTaskService { if (taskId == null || context == null || countPendingLlmStates(taskId) > 0) { return; } - if (!isOwnerCurrent(context.ownerInstanceId())) { + if (!ownershipSupport().isOwnerCurrent(context.ownerInstanceId())) { return; } - TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L); + TaskDistributedLockService.LockHandle lockHandle = ownershipSupport().acquireTaskLock(taskId, 0L); if (lockHandle == null) { return; } @@ -3007,22 +2472,6 @@ public class SimilarAsinTaskService { } } - private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) { - TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_TTL, waitMillis); - if (lockHandle == null) { - log.info("[similar-asin] skip task operation because task lock is busy taskId={}", taskId); - } - return lockHandle; - } - - private TaskDistributedLockService.LockHandle requireTaskLock(Long taskId, long waitMillis) { - TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, waitMillis); - if (lockHandle == null) { - throw new BusinessException(40902, "Task is busy, please retry later"); - } - return lockHandle; - } - private void completeLlmFileJob(FileTaskEntity task, FileResultEntity result, TaskFileJobEntity job, @@ -3032,9 +2481,9 @@ public class SimilarAsinTaskService { throw new BusinessException("LLM result is still processing, cannot generate result file yet"); } int assembleProgress = Math.max(1, Math.min(totalProgressUnits - 2, llmWorkUnits)); - saveFileBuildProgress(task, job, totalProgressUnits, assembleProgress, "Assembling xlsx"); - assembleResultWorkbook(task, result); - saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "Uploading result file"); + progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, assembleProgress, "Assembling xlsx"); + resultAssembler().assembleResultWorkbook(task, result); + progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "Uploading result file"); fileResultMapper.updateById(result); boolean taskAlreadyFailed = STATUS_FAILED.equals(task.getStatus()) || (task.getErrorMessage() != null && !task.getErrorMessage().isBlank()); @@ -3047,9 +2496,9 @@ public class SimilarAsinTaskService { task.setFinishedAt(LocalDateTime.now()); fileTaskMapper.updateById(task); taskCacheService.deleteTaskCache(task.getId()); - saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "Result file generated"); + progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "Result file generated"); // P1-1：异步 assemble 完成 → task 进入 SUCCESS/FAILED 终态，清理滑窗记录避免内存泄漏。 - clearPoisonWindow(task.getId()); + poisonTracker.clear(task.getId()); } private void mergeLlmRowsIntoChunk(FileTaskEntity task, @@ -3072,9 +2521,9 @@ public class SimilarAsinTaskService { if (llmRow == null) { continue; } - addNonBlank(prefetchUrls, llmRow.getMainUrl()); - addNonBlank(prefetchUrls, llmRow.getPuzzleImg1()); - addNonBlank(prefetchUrls, llmRow.getPuzzleImg2()); + SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getMainUrl()); + SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getPuzzleImg1()); + SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getPuzzleImg2()); } imagePrefetchService.enqueue(task.getId(), prefetchUrls); } catch (Exception ex) { @@ -3084,23 +2533,23 @@ public class SimilarAsinTaskService { Map> rowsByChunk = new LinkedHashMap<>(); Map chunkByKey = new LinkedHashMap<>(); for (TaskChunkEntity chunk : chunks) { - String chunkKey = chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()); - rowsByChunk.put(chunkKey, readChunkRows(chunk)); + String chunkKey = SimilarAsinChunkMergeSupport.chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()); + rowsByChunk.put(chunkKey, chunkPayloadSupport().readChunkRows(chunk)); chunkByKey.put(chunkKey, chunk); } // Task 11：合并前按稳定 rowKey 去重，消除重复行逐行 expand/分配/写回 的 O(n²) 热点。 - List uniqueRows = dedupeRowsByRowKey(llmRows); + List uniqueRows = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); List expandedAll = new ArrayList<>(); for (SimilarAsinResultRowDto resultRow : uniqueRows) { - expandedAll.addAll(expandRows(List.of(resultRow), allRowsByBaseId)); + expandedAll.addAll(SimilarAsinChunkMergeSupport.expandRows(List.of(resultRow), allRowsByBaseId)); } // Task 10：先建 rowKey → chunkKey 批量索引，把逐 chunk 线性扫描降为 O(1) 查找。 - Map rowKeyIndex = indexRowsByChunkKey(rowsByChunk); + Map rowKeyIndex = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); List orphanRows = new ArrayList<>(); Map> mergeRowsByChunk = - assignLlmRowsToChunks(rowsByChunk, expandedAll, rowKeyIndex, chunkScopeHash, chunkIndex, orphanRows); + SimilarAsinChunkMergeSupport.assignLlmRowsToChunks(rowsByChunk, expandedAll, rowKeyIndex, chunkScopeHash, chunkIndex, orphanRows); if (!orphanRows.isEmpty()) { - persistOrphanLlmRows(task.getId(), orphanRows); + chunkPayloadSupport().persistOrphanLlmRows(task.getId(), orphanRows); } for (Map.Entry> entry : mergeRowsByChunk.entrySet()) { TaskChunkEntity chunk = chunkByKey.get(entry.getKey()); @@ -3124,14 +2573,10 @@ public class SimilarAsinTaskService { } return chunks.stream() .filter(Objects::nonNull) - .filter(chunk -> !readChunkRows(chunk).isEmpty()) + .filter(chunk -> !chunkPayloadSupport().readChunkRows(chunk).isEmpty()) .toList(); } - private String chunkStorageKey(String scopeHash, Integer chunkIndex) { - return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex); - } - private List markRowsFailed(List rows, String failureMessage) { if (rows == null || rows.isEmpty()) { return List.of(); @@ -3324,10 +2769,6 @@ public class SimilarAsinTaskService { } } - private boolean isLlmResultBufferEnabled() { - return properties.isLlmResultBufferEnabled(); - } - /** * Task 12：统一 LLM DONE 结果落库入口。 * 缓冲开关开启时把 llmRows 写入 transient storage（pointer 存进 state.stateJson）， @@ -3343,7 +2784,7 @@ public class SimilarAsinTaskService { if (state == null || context == null || llmRows == null || llmRows.isEmpty()) { return; } - if (!isLlmResultBufferEnabled()) { + if (!SimilarAsinLimits.isLlmResultBufferEnabled(properties)) { mergeLlmRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), llmRows, allRowsByBaseId); return; } @@ -3378,7 +2819,7 @@ public class SimilarAsinTaskService { null, batchIndex, batchTotal, - currentInstanceId(), + ownershipSupport().currentInstanceId(), 0, credentialName, null @@ -3433,7 +2874,7 @@ public class SimilarAsinTaskService { } List entries = new ArrayList<>(); for (TaskScopeStateEntity state : states) { - LlmBatchContext context = readLlmBatchContext(state); + LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); if (context == null) { continue; } @@ -3512,18 +2953,6 @@ public class SimilarAsinTaskService { List rows) { } - private LlmBatchContext readLlmBatchContext(TaskScopeStateEntity state) { - if (state == null || state.getStateJson() == null || state.getStateJson().isBlank()) { - return null; - } - try { - return objectMapper.readValue(state.getStateJson(), LlmBatchContext.class); - } catch (Exception ex) { - log.warn("[similar-asin] read llm batch context failed taskId={} stateId={} err={}", - state.getTaskId(), state.getId(), ex.getMessage()); - return null; - } - } private LlmBatchContext withResultPayloadPointer(LlmBatchContext context, String resultPayloadPointer) { return new LlmBatchContext( @@ -3563,17 +2992,6 @@ public class SimilarAsinTaskService { return count == null ? 0 : count.intValue(); } - private int countCompletedLlmStates(Long taskId) { - if (taskId == null || taskId <= 0) { - return 0; - } - Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_DONE, LLM_STATUS_FAILED))); - return count == null ? 0 : count.intValue(); - } - private boolean isResultSubmissionComplete(Long taskId) { if (taskId == null || taskId <= 0) { return false; @@ -3620,7 +3038,7 @@ public class SimilarAsinTaskService { StringBuilder rowKeys = new StringBuilder(); if (batchRows != null) { for (SimilarAsinResultRowDto row : batchRows) { - String key = rowKey(row); + String key = SimilarAsinChunkMergeSupport.rowKey(row); if (!key.isBlank()) { if (!rowKeys.isEmpty()) { rowKeys.append('|'); @@ -3640,268 +3058,6 @@ public class SimilarAsinTaskService { private record PythonUploadProgress(int current, int total, String unit) { } - private String buildTaskOwnerScopeKey(FileTaskEntity task) { - Long taskId = task == null ? null : task.getId(); - return "task:" + taskId + ":owner:" + firstNonBlank(ownerFromTask(task), currentInstanceId()); - } - - private String currentInstanceId() { - String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId(); - return firstNonBlank(instanceId, "unknown-instance"); - } - - private boolean isJobOwnedByCurrentInstance(TaskFileJobEntity job) { - return job == null || isOwnerCurrent(ownerFromScopeKey(job.getScopeKey())); - } - - private void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) { - String owner = ownerFromTask(task); - if (isOwnerCurrent(owner)) { - return; - } - log.warn("[similar-asin] reject task operation because owner is another instance taskId={} operation={} owner={} current={}", - task == null ? null : task.getId(), operation, owner, currentInstanceId()); - throw new TaskOwnerMismatchException( - task == null ? null : task.getId(), - operation, - owner, - currentInstanceId()); - } - - private boolean isLlmStateOwnedByCurrentInstance(TaskScopeStateEntity state) { - return state == null || isOwnerCurrent(ownerFromLlmState(state)); - } - - private boolean isOwnerCurrent(String owner) { - return owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId()); - } - - private String ownerFromLlmState(TaskScopeStateEntity state) { - LlmBatchContext context = readLlmBatchContext(state); - return context == null ? null : context.ownerInstanceId(); - } - - private String ownerFromTask(FileTaskEntity task) { - if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) { - return null; - } - try { - String owner = objectMapper.readTree(task.getResultJson()).path("ownerInstanceId").asText(""); - return owner.isBlank() ? null : owner; - } catch (Exception ex) { - log.warn("[similar-asin] read task owner failed taskId={} err={}", task.getId(), ex.getMessage()); - return null; - } - } - - /** - * P0-3：当 chunk 因 RustFS 失败落到本地时，把 task 锁定到当前实例。 - * 这样后续 assembleResultWorkbook 调度只会路由到这台机器（已有 - * isJobOwnedByCurrentInstance / ownerFromTask 兼容这条 ownerInstanceId）。 - * 同时把 fallback 信号写到 task_scope_state.state_json，便于排查。 - */ - private void bindTaskToCurrentOwnerForLocalFallback(FileTaskEntity task, String scopeHash, Integer chunkIndex) { - if (task == null || task.getId() == null) { - throw new IllegalStateException("Cannot bind local fallback without a task ID"); - } - String currentInstance = currentInstanceId(); - try { - String existingJson = task.getResultJson(); - ObjectNode payload; - if (existingJson == null || existingJson.isBlank()) { - payload = objectMapper.createObjectNode(); - } else { - JsonNode tree = objectMapper.readTree(existingJson); - payload = tree.isObject() ? (ObjectNode) tree : objectMapper.createObjectNode(); - } - String existingOwner = payload.path("ownerInstanceId").asText(""); - if (existingOwner.isBlank()) { - payload.put("ownerInstanceId", currentInstance); - payload.put("ownerInstanceReason", "rustfs-fallback-local"); - String updatedJson = objectMapper.writeValueAsString(payload); - task.setResultJson(updatedJson); - FileTaskEntity update = new FileTaskEntity(); - update.setId(task.getId()); - update.setResultJson(updatedJson); - update.setUpdatedAt(LocalDateTime.now()); - if (fileTaskMapper.updateById(update) != 1) { - throw new IllegalStateException("Task owner update affected no row"); - } - log.warn("[similar-asin] task bound to current instance due to rustfs fallback taskId={} instanceId={} chunk={}", - task.getId(), currentInstance, chunkIndex); - } else if (!Objects.equals(existingOwner, currentInstance)) { - throw new IllegalStateException("Local fallback task is owned by instance=" + existingOwner - + " current=" + currentInstance); - } - } catch (Exception ex) { - throw new IllegalStateException("Failed to bind local fallback task owner taskId=" + task.getId(), ex); - } - // 把 fallback 信号写到 task_scope_state.state_json，方便排查 / 后续告警钩子。 - if (scopeHash == null || scopeHash.isBlank()) { - return; - } - try { - TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, task.getId()) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .eq(TaskScopeStateEntity::getScopeHash, scopeHash) - .last("limit 1")); - if (scope == null) { - throw new IllegalStateException("Local fallback scope state does not exist"); - } - ObjectNode stateNode; - String stateJson = scope.getStateJson(); - if (stateJson == null || stateJson.isBlank()) { - stateNode = objectMapper.createObjectNode(); - stateNode.put("phase", "RECEIVED"); - stateNode.put("llm", "PENDING"); - } else { - JsonNode parsed = objectMapper.readTree(stateJson); - stateNode = parsed.isObject() ? (ObjectNode) parsed : objectMapper.createObjectNode(); - } - ArrayNode fallbackArr; - JsonNode existingArr = stateNode.path("localFallback"); - if (existingArr.isArray()) { - fallbackArr = (ArrayNode) existingArr; - } else { - fallbackArr = stateNode.putArray("localFallback"); - } - String tag = "chunk-" + chunkIndex + "@" + currentInstance; - boolean exists = false; - for (JsonNode node : fallbackArr) { - if (tag.equals(node.asText(""))) { - exists = true; - break; - } - } - if (!exists) { - fallbackArr.add(tag); - scope.setStateJson(objectMapper.writeValueAsString(stateNode)); - scope.setUpdatedAt(LocalDateTime.now()); - if (taskScopeStateMapper.updateById(scope) != 1) { - throw new IllegalStateException("Local fallback scope update affected no row"); - } - } - } catch (Exception ex) { - throw new IllegalStateException("Failed to record local fallback state taskId=" - + task.getId() + " chunk=" + chunkIndex, ex); - } - } - - private String ownerFromScopeKey(String scopeKey) { - if (scopeKey == null || scopeKey.isBlank()) { - return null; - } - String marker = ":owner:"; - int index = scopeKey.lastIndexOf(marker); - if (index < 0) { - return null; - } - String owner = scopeKey.substring(index + marker.length()).trim(); - return owner.isBlank() ? null : owner; - } - - private int countLlmWorkUnits(List chunks, int batchSize) { - if (chunks == null || chunks.isEmpty()) { - return 0; - } - int total = 0; - for (TaskChunkEntity chunk : chunks) { - Map persistedRows = readChunkRows(chunk); - if (persistedRows.isEmpty()) { - continue; - } - total += collectPendingLlmRows(persistedRows.values()).size(); - } - return total <= 0 ? 0 : Math.max(1, (total + batchSize - 1) / batchSize); - } - - private void saveFileBuildProgress(FileTaskEntity task, - TaskFileJobEntity job, - int total, - int completed, - String message) { - if (task == null || task.getId() == null || job == null || job.getId() == null) { - return; - } - int safeTotal = Math.max(1, total); - int safeCompleted = Math.max(0, Math.min(completed, safeTotal)); - TaskProgressSnapshotEntity existing = taskProgressSnapshotService.find(task.getId(), MODULE_TYPE); - int displayPercent = calculateDisplayProgressPercent(safeCompleted, safeTotal, job, LocalDateTime.now()); - boolean terminalProgress = isTerminalFileBuildProgress(task.getStatus(), safeCompleted, safeTotal); - if (terminalProgress) { - displayPercent = 100; - } - int previousDisplayPercent = calculateSnapshotDisplayPercent(existing, job); - if (previousDisplayPercent > displayPercent && safeCompleted < safeTotal) { - displayPercent = previousDisplayPercent; - } - taskProgressSnapshotService.save( - task.getId(), - MODULE_TYPE, - terminalProgress ? task.getStatus() : STATUS_RUNNING, - safeTotal, - safeCompleted, - 0, - job.getScopeKey(), - message, - Map.of("phase", "RESULT_FILE", "jobId", job.getId(), "displayPercent", displayPercent) - ); - } - - static boolean isTerminalFileBuildProgress(String taskStatus, int completed, int total) { - return total > 0 - && completed >= total - && (STATUS_SUCCESS.equals(taskStatus) || STATUS_FAILED.equals(taskStatus)); - } - - private int calculateDisplayProgressPercent(int current, - int total, - TaskFileJobEntity job, - LocalDateTime baseTime) { - if (total <= 0) { - return 0; - } - current = Math.max(0, Math.min(current, total)); - int percent = Math.max(1, Math.min(99, (int) Math.floor(current * 100.0 / total))); - if (job != null && STATUS_RUNNING.equals(job.getStatus())) { - long elapsedSeconds = baseTime == null ? 0 : Math.max(0, Duration.between(baseTime, LocalDateTime.now()).getSeconds()); - if (current <= 0) { - int firstRealProgressPercent = Math.max(1, Math.min(99, (int) Math.floor(100.0 / total))); - int waitingCap = Math.max(8, Math.min(35, firstRealProgressPercent - 1)); - percent = Math.max(percent, Math.min(waitingCap, 8 + (int) (elapsedSeconds / 6))); - } else if (current < total) { - percent = Math.max(percent, Math.min(92, percent + (int) (elapsedSeconds / 10))); - } - } - return percent; - } - - private int calculateSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot, TaskFileJobEntity job) { - if (snapshot == null) { - return 0; - } - int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount(); - int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount(); - int percent = total <= 0 ? 0 : calculateDisplayProgressPercent(current, total, job, snapshot.getUpdatedAt()); - return Math.max(percent, extractSnapshotDisplayPercent(snapshot)); - } - - private int extractSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot) { - if (snapshot == null || snapshot.getSnapshotJson() == null || snapshot.getSnapshotJson().isBlank()) { - return 0; - } - try { - JsonNode displayPercent = objectMapper.readTree(snapshot.getSnapshotJson()).path("displayPercent"); - if (!displayPercent.isNumber()) { - return 0; - } - return Math.max(0, Math.min(100, displayPercent.asInt())); - } catch (Exception ignored) { - return 0; - } - } - public void cleanupResultFileJob(TaskFileJobEntity job) { if (job == null || job.getTaskId() == null) { return; @@ -3915,861 +3071,6 @@ public class SimilarAsinTaskService { .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); } - /** - * Bounded assembly path. Chunk payloads are selected and decoded one at a - * time for the current source file; the result map is released when that - * source workbook finishes. This prevents a multi-source/large-task - * assemble from retaining every chunk JSON and every result DTO together. - */ - private void assembleResultWorkbookBounded(FileTaskEntity task, FileResultEntity result) { - SimilarAsinParsedPayloadDto parsed = readParsedPayload(task); - List parsedRows = resolveAllRows(parsed); - List sourceRows = splitRowsBySourceFile( - parsed, parsedRows, result.getSourceFilename()); - if (sourceRows.isEmpty()) { - throw new BusinessException("Similar ASIN result rows are empty"); - } - - File outputDir = new File(storageProperties.getLocalTempDir(), "similar-asin-result"); - if (!outputDir.exists() && !outputDir.mkdirs()) { - throw new BusinessException("failed to create result directory"); - } - List workbooks = new ArrayList<>(); - List workbookFiles = new ArrayList<>(sourceRows.size()); - File zip = null; - long imageCacheMaxBytes = properties.getImageCacheMaxBytes() > 0 - ? properties.getImageCacheMaxBytes() - : BoundedImageCache.DEFAULT_MAX_BYTES; - BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes); - try { - int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes()); - long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes); - long assembleStart = System.currentTimeMillis(); - List> futures = new ArrayList<>(sourceRows.size()); - for (SourceRows item : sourceRows) { - final SourceRows captured = item; - final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx"; - final String tempFilename = safeFileStem(captured.sourceFilename()) - + "-" + task.getId() - + "-" + result.getId() - + "-" + UUID.randomUUID() - + "-result.xlsx"; - final File xlsx = new File(outputDir, tempFilename); - workbookFiles.add(xlsx); - futures.add(assembleExecutor.submit(() -> { - ensureResultAssemblyNotInterrupted(); - Map sourceResultMap = - loadPersistedResultRowsForRowsWithRetry(task.getId(), captured.rows()); - int persistedRows = sourceResultMap.size(); - sourceResultMap.entrySet().removeIf(entry -> !isExportableResultRow(entry.getValue())); - long resolvedRows = captured.rows().stream() - .filter(row -> findResultRow(row, sourceResultMap) != null) - .count(); - int conformPropagated = GroupResultPropagator.propagateByGroup( - captured.rows(), - SimilarAsinParsedRowVo::getDisplayId, - row -> findResultRow(row, sourceResultMap), - SimilarAsinResultRowDto::getIsConform, - SimilarAsinResultRowDto::setIsConform, - List.of("不符合"), - "[similar-asin] taskId=" + task.getId() - ); - writeResultWorkbook(xlsx, captured.rows(), sourceResultMap, taskImageCache); - ensureResultAssemblyNotInterrupted(); - return new SourceResultWorkbook( - xlsx, filename, captured.rows().size(), persistedRows, - sourceResultMap.size(), resolvedRows, conformPropagated); - })); - } - try { - for (Future future : futures) { - long remainingNanos = deadlineNanos - System.nanoTime(); - if (remainingNanos <= 0L) { - throw new TimeoutException("result file deadline reached"); - } - workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS)); - } - } catch (InterruptedException ex) { - cancelResultAssemblyFutures(futures); - Thread.currentThread().interrupt(); - throw new BusinessException("result workbook assembly interrupted", ex); - } catch (TimeoutException ex) { - cancelResultAssemblyFutures(futures); - throw new BusinessException("result workbook assembly timed out", ex); - } catch (CancellationException ex) { - cancelResultAssemblyFutures(futures); - throw new BusinessException("result workbook assembly cancelled", ex); - } catch (ExecutionException ex) { - cancelResultAssemblyFutures(futures); - Throwable cause = ex.getCause(); - if (cause instanceof BusinessException businessException) { - throw businessException; - } - throw new BusinessException("result workbook assembly failed", cause == null ? ex : cause); - } - - int persistedResultRows = workbooks.stream() - .mapToInt(SourceResultWorkbook::persistedRows) - .sum(); - int resultRows = workbooks.stream() - .mapToInt(SourceResultWorkbook::resultRows) - .sum(); - long resolvedRows = workbooks.stream() - .mapToLong(SourceResultWorkbook::resolvedRows) - .sum(); - int conformPropagated = workbooks.stream() - .mapToInt(SourceResultWorkbook::conformPropagated) - .sum(); - log.info("[similar-asin] bounded assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={} propagated={} sources={} costMs={}", - task.getId(), parsedRows.size(), persistedResultRows, resultRows, - resolvedRows, conformPropagated, sourceRows.size(), - System.currentTimeMillis() - assembleStart); - if (resultRows == 0) { - String failureSummary = collectChunkReadFailureSummary(task.getId()); - throw new BusinessException(failureSummary.isBlank() - ? "Similar ASIN result rows are empty; please retry result generation" - : "Similar ASIN result rows are empty (" + failureSummary + ")"); - } - - taskImageCache.clear(); - File uploadFile; - String filename; - String contentType; - if (workbooks.size() == 1) { - SourceResultWorkbook workbook = workbooks.get(0); - uploadFile = workbook.file(); - filename = workbook.filename(); - contentType = CONTENT_TYPE_XLSX; - } else { - filename = safeFileStem(result.getSourceFilename()) + "-result.zip"; - String tempFilename = safeFileStem(result.getSourceFilename()) - + "-" + task.getId() - + "-" + result.getId() - + "-" + UUID.randomUUID() - + "-result.zip"; - zip = new File(outputDir, tempFilename); - packageResultWorkbooksAsZip(zip, workbooks); - uploadFile = zip; - contentType = CONTENT_TYPE_ZIP; - } - String objectKey = ossStorageService.uploadResultFile(uploadFile, MODULE_TYPE); - result.setResultFilename(filename); - result.setResultFileUrl(objectKey); - result.setResultFileSize(uploadFile.length()); - result.setResultContentType(contentType); - result.setRowCount(parsedRows.size()); - } finally { - taskImageCache.clear(); - for (File workbookFile : workbookFiles) { - if (workbookFile.exists() && !workbookFile.delete()) { - log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile); - } - } - if (zip != null && zip.exists() && !zip.delete()) { - log.warn("[similar-asin] delete temp zip failed file={}", zip); - } - } - } - - private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) { - if (properties.isBoundedResultAssemblyEnabled()) { - assembleResultWorkbookBounded(task, result); - return; - } - SimilarAsinParsedPayloadDto parsed = readParsedPayload(task); - List parsedRows = resolveAllRows(parsed); - Map resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsedRows.size()); - int persistedResultRows = resultMap.size(); - resultMap.entrySet().removeIf(entry -> !isExportableResultRow(entry.getValue())); - long resolvedRows = parsedRows.stream() - .filter(row -> findResultRow(row, resultMap) != null) - .count(); - log.info("[similar-asin] assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={}", - task.getId(), parsedRows.size(), persistedResultRows, resultMap.size(), resolvedRows); - if (!parsedRows.isEmpty() && resultMap.isEmpty()) { - // P1-4：检查是否有 chunk-read-failed 标记，把 chunk index 列表附在错误信息里。 - String chunkReadFailureSummary = collectChunkReadFailureSummary(task.getId()); - if (!chunkReadFailureSummary.isBlank()) { - throw new BusinessException("相似ASIN检测结果为空（" + chunkReadFailureSummary + "），请稍后重试生成结果文件"); - } - throw new BusinessException("相似ASIN检测结果为空，请稍后重试生成结果文件"); - } - // 公共分组处理：同 baseId 连续行视为一组（如 1、1_1、1_2），组内任一行的"是否符合类目" - // 命中"不符合"（包含匹配），则组内所有行的"是否符合类目"统一为标准值"不符合"。 - // 单行组跳过。仅修改 isConform 列，不影响其他列。 - int conformPropagated = GroupResultPropagator.propagateByGroup( - parsedRows, - SimilarAsinParsedRowVo::getDisplayId, - row -> findResultRow(row, resultMap), - SimilarAsinResultRowDto::getIsConform, - SimilarAsinResultRowDto::setIsConform, - List.of("不符合"), - "[similar-asin] taskId=" + task.getId() - ); - if (conformPropagated > 0) { - log.info("[similar-asin] group propagate isConform taskId={} updatedRows={}", - task.getId(), conformPropagated); - } - File outputDir = new File(storageProperties.getLocalTempDir(), "similar-asin-result"); - if (!outputDir.exists() && !outputDir.mkdirs()) { - throw new BusinessException("创建结果目录失败"); - } - List sourceRows = splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename()); - List workbooks = new ArrayList<>(); - List workbookFiles = new ArrayList<>(sourceRows.size()); - File zip = null; - // 跨 workbook 共享 taskImageCache：多源场景下相同 URL 仅下载一次。 - // 配合 embed() 写完即 remove()，cache 仅承载 in-flight 图片； - // 5000 行 × 200 实例规模下用 BoundedImageCache 按字节做 LRU 淘汰，硬上限 256MB - // （配置 aiimage.similar-asin.image-cache-max-bytes 可调），避免堆爆。 - long imageCacheMaxBytes = properties.getImageCacheMaxBytes() > 0 - ? properties.getImageCacheMaxBytes() - : BoundedImageCache.DEFAULT_MAX_BYTES; - BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes); - try { - // 单源和多源统一提交真实 Future，确保超时取消能中断实际 workbook 线程。 - // 所有源文件共享同一个绝对 deadline，避免逐个 Future 各等待一轮完整超时。 - int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes()); - long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes); - long assembleStart = System.currentTimeMillis(); - List> futures = new ArrayList<>(sourceRows.size()); - for (SourceRows item : sourceRows) { - final SourceRows captured = item; - final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx"; - final String tempFilename = safeFileStem(captured.sourceFilename()) - + "-" + task.getId() - + "-" + result.getId() - + "-" + UUID.randomUUID() - + "-result.xlsx"; - final File xlsx = new File(outputDir, tempFilename); - workbookFiles.add(xlsx); - futures.add(assembleExecutor.submit(() -> { - ensureResultAssemblyNotInterrupted(); - writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache); - ensureResultAssemblyNotInterrupted(); - return new SourceResultWorkbook(xlsx, filename, captured.rows().size()); - })); - } - try { - for (Future future : futures) { - long remainingNanos = deadlineNanos - System.nanoTime(); - if (remainingNanos <= 0L) { - throw new TimeoutException("result file deadline reached"); - } - workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS)); - } - } catch (InterruptedException ex) { - cancelResultAssemblyFutures(futures); - Thread.currentThread().interrupt(); - throw new BusinessException("生成相似ASIN检测结果中断", ex); - } catch (TimeoutException ex) { - cancelResultAssemblyFutures(futures); - throw new BusinessException("生成相似ASIN检测结果超时（" + timeoutMinutes + " 分钟）", ex); - } catch (CancellationException ex) { - cancelResultAssemblyFutures(futures); - throw new BusinessException("生成相似ASIN检测结果已取消", ex); - } catch (ExecutionException ex) { - cancelResultAssemblyFutures(futures); - Throwable cause = ex.getCause(); - if (cause instanceof BusinessException businessException) { - throw businessException; - } - throw new BusinessException("生成相似ASIN检测结果失败", cause != null ? cause : ex); - } - log.info("[similar-asin] assemble workbooks finished taskId={} sources={} timeoutMinutes={} costMs={}", - task.getId(), sourceRows.size(), timeoutMinutes, System.currentTimeMillis() - assembleStart); - log.info("[similar-asin] assemble image-cache stats taskId={} sources={} cacheSize={} currentBytes={} evictedCount={} evictedBytes={} maxBytes={}", - task.getId(), sourceRows.size(), taskImageCache.size(), - taskImageCache.currentBytes(), taskImageCache.evictedCount(), - taskImageCache.evictedBytes(), taskImageCache.maxBytes()); - // 主动清空，让 GC 尽早回收图片字节，避免 zip 阶段还占着堆。 - taskImageCache.clear(); - if (workbooks.isEmpty()) { - throw new BusinessException("相似ASIN检测结果为空，请稍后重试生成结果文件"); - } - - File uploadFile; - String filename; - String contentType; - if (workbooks.size() == 1) { - SourceResultWorkbook workbook = workbooks.get(0); - uploadFile = workbook.file(); - filename = workbook.filename(); - contentType = CONTENT_TYPE_XLSX; - } else { - filename = safeFileStem(result.getSourceFilename()) + "-result.zip"; - String tempFilename = safeFileStem(result.getSourceFilename()) - + "-" + task.getId() - + "-" + result.getId() - + "-" + UUID.randomUUID() - + "-result.zip"; - zip = new File(outputDir, tempFilename); - packageResultWorkbooksAsZip(zip, workbooks); - uploadFile = zip; - contentType = CONTENT_TYPE_ZIP; - } - - String objectKey = ossStorageService.uploadResultFile(uploadFile, MODULE_TYPE); - result.setResultFilename(filename); - result.setResultFileUrl(objectKey); - result.setResultFileSize(uploadFile.length()); - result.setResultContentType(contentType); - result.setRowCount(resolveAllRows(parsed).size()); - } finally { - for (File workbookFile : workbookFiles) { - if (workbookFile.exists() && !workbookFile.delete()) { - log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile); - } - } - if (zip != null && zip.exists() && !zip.delete()) { - log.warn("[similar-asin] delete temp zip failed file={}", zip); - } - } - } - - private List listOwnedPendingLlmStates() { - Map merged = new LinkedHashMap<>(); - for (TaskScopeStateEntity state : queryOwnedPendingLlmStates(false, 50)) { - if (state != null && state.getId() != null) { - merged.put(state.getId(), state); - } - } - for (TaskScopeStateEntity state : queryOwnedPendingLlmStates(true, 50)) { - if (state != null && state.getId() != null) { - merged.putIfAbsent(state.getId(), state); - } - } - return new ArrayList<>(merged.values()); - } - - private List queryOwnedPendingLlmStates(boolean oldestFirst, int limit) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)) - .and(query -> query - .apply("JSON_UNQUOTE(JSON_EXTRACT(state_json, '$.ownerInstanceId')) IS NULL") - .or() - .apply("JSON_UNQUOTE(JSON_EXTRACT(state_json, '$.ownerInstanceId')) = ''") - .or() - .apply("JSON_UNQUOTE(JSON_EXTRACT(state_json, '$.ownerInstanceId')) = {0}", currentInstanceId())) - .last("limit " + Math.max(1, Math.min(limit, 100))); - if (oldestFirst) { - wrapper.orderByAsc(TaskScopeStateEntity::getUpdatedAt); - } else { - wrapper.orderByDesc(TaskScopeStateEntity::getUpdatedAt); - } - List states = taskScopeStateMapper.selectList(wrapper); - return states == null ? List.of() : states; - } - - private List splitRowsBySourceFile(SimilarAsinParsedPayloadDto parsed, - List rows, - String fallbackFilename) { - String defaultFilename = firstNonBlank(fallbackFilename, "similar-asin"); - Map builders = new LinkedHashMap<>(); - if (parsed != null && parsed.getSourceFiles() != null) { - for (SimilarAsinSourceFileDto sourceFile : parsed.getSourceFiles()) { - if (sourceFile == null) { - continue; - } - String sourceFileKey = normalize(sourceFile.getFileKey()); - if (sourceFileKey.isBlank()) { - continue; - } - builders.putIfAbsent(sourceFileKey, - new SourceRowsBuilder(sourceFileKey, firstNonBlank(sourceFile.getOriginalFilename(), defaultFilename))); - } - } - - for (SimilarAsinParsedRowVo row : rows == null ? List.of() : rows) { - if (row == null) { - continue; - } - String sourceFileKey = normalize(row.getSourceFileKey()); - if (sourceFileKey.isBlank()) { - sourceFileKey = normalize(row.getSourceFilename()); - } - if (sourceFileKey.isBlank()) { - sourceFileKey = "__default__"; - } - SourceRowsBuilder builder = builders.computeIfAbsent(sourceFileKey, - key -> new SourceRowsBuilder(key, firstNonBlank(row.getSourceFilename(), defaultFilename))); - if (builder.sourceFilename().isBlank() || Objects.equals(builder.sourceFilename(), defaultFilename)) { - builder.setSourceFilename(firstNonBlank(row.getSourceFilename(), defaultFilename)); - } - builder.rows().add(row); - } - - return builders.values().stream() - .filter(builder -> !builder.rows().isEmpty()) - .map(builder -> new SourceRows(builder.sourceFileKey(), firstNonBlank(builder.sourceFilename(), defaultFilename), builder.rows())) - .toList(); - } - - private void packageResultWorkbooksAsZip(File zip, List workbooks) { - Set entryNames = new LinkedHashSet<>(); - try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zip))) { - byte[] buffer = new byte[8192]; - for (SourceResultWorkbook workbook : workbooks) { - writeResultZipEntry(zos, buffer, workbook.file(), uniqueZipEntryName(workbook.filename(), entryNames)); - } - } catch (IOException ex) { - throw new BusinessException("生成相似ASIN检测结果压缩包失败"); - } - } - - private void writeResultZipEntry(ZipOutputStream zos, - byte[] buffer, - File file, - String entryName) throws IOException { - zos.putNextEntry(new ZipEntry(entryName)); - try (FileInputStream inputStream = new FileInputStream(file)) { - int len; - while ((len = inputStream.read(buffer)) > 0) { - zos.write(buffer, 0, len); - } - } - zos.closeEntry(); - } - - private String uniqueZipEntryName(String filename, Set entryNames) { - String safeFilename = normalizeZipFilename(filename); - if (entryNames.add(safeFilename)) { - return safeFilename; - } - int dotIndex = safeFilename.lastIndexOf('.'); - String stem = dotIndex > 0 ? safeFilename.substring(0, dotIndex) : safeFilename; - String extension = dotIndex > 0 ? safeFilename.substring(dotIndex) : ""; - int index = 2; - while (true) { - String candidate = stem + "(" + index + ")" + extension; - if (entryNames.add(candidate)) { - return candidate; - } - index++; - } - } - - private String normalizeZipFilename(String filename) { - String normalized = firstNonBlank(filename, "similar-asin-result.xlsx").replace('\\', '/'); - while (normalized.startsWith("/")) { - normalized = normalized.substring(1); - } - return normalized.isBlank() ? "similar-asin-result.xlsx" : normalized; - } - - private Map loadPersistedResultRowsForRowsWithRetry( - Long taskId, - List requestedRows) { - Map result = - loadPersistedResultRowsForRows(taskId, requestedRows); - if (requestedRows == null || requestedRows.isEmpty() || !result.isEmpty()) { - return result; - } - for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) { - sleepBeforeResultRowsRetry(attempt); - result = loadPersistedResultRowsForRows(taskId, requestedRows); - if (!result.isEmpty()) { - log.info("[similar-asin] source result rows recovered after retry taskId={} attempt={} rows={}", - taskId, attempt, result.size()); - } - } - return result; - } - - /** - * Select and decode one chunk payload at a time. Only rows needed by the - * current source workbook are retained in the returned map. - */ - private Map loadPersistedResultRowsForRows( - Long taskId, - List requestedRows) { - Map result = new LinkedHashMap<>(); - Set requestedKeys = resultLookupKeys(requestedRows); - if (taskId == null || requestedKeys.isEmpty()) { - return result; - } - - Long lastChunkId = null; - while (true) { - LambdaQueryWrapper query = new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getId) - .last("LIMIT 1"); - if (lastChunkId != null) { - query.gt(TaskChunkEntity::getId, lastChunkId); - } - List page = taskChunkMapper.selectList(query); - if (page == null || page.isEmpty()) { - break; - } - TaskChunkEntity chunk = page.getFirst(); - appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys, false); - lastChunkId = chunk.getId(); - if (lastChunkId == null) { - break; - } - } - appendRequestedOrphanRows(taskId, requestedKeys, result); - return result; - } - - private Set resultLookupKeys(List rows) { - Set keys = new LinkedHashSet<>(); - for (SimilarAsinParsedRowVo row : rows == null ? List.of() : rows) { - String primary = rowKey(row); - if (!primary.isBlank()) { - keys.add(primary); - } - String legacy = legacyRowKey(row); - if (!legacy.isBlank()) { - keys.add(legacy); - } - } - return keys; - } - - private void appendRequestedResultRows(Map sink, - Iterable rows, - Set requestedKeys, - boolean onlyIfAbsent) { - if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) { - return; - } - for (SimilarAsinResultRowDto row : rows) { - if (row == null) { - continue; - } - String primary = rowKey(row); - String legacy = legacyRowKey(row); - String matchedKey = requestedKeys.contains(primary) - ? primary - : requestedKeys.contains(legacy) ? legacy : ""; - if (matchedKey.isBlank()) { - continue; - } - if (onlyIfAbsent) { - sink.putIfAbsent(matchedKey, row); - } else { - sink.put(matchedKey, row); - } - } - } - - private void appendRequestedOrphanRows(Long taskId, - Set requestedKeys, - Map sink) { - Long lastStateId = null; - while (true) { - LambdaQueryWrapper query = new LambdaQueryWrapper() - .select(TaskScopeStateEntity::getId, - TaskScopeStateEntity::getParsedPayloadJson) - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .likeRight(TaskScopeStateEntity::getScopeKey, ORPHAN_SCOPE_KEY_PREFIX) - .orderByAsc(TaskScopeStateEntity::getId) - .last("LIMIT 1"); - if (lastStateId != null) { - query.gt(TaskScopeStateEntity::getId, lastStateId); - } - List page = taskScopeStateMapper.selectList(query); - if (page == null || page.isEmpty()) { - return; - } - TaskScopeStateEntity state = page.getFirst(); - lastStateId = state.getId(); - if (lastStateId == null) { - return; - } - if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { - continue; - } - try { - String payloadJson = transientPayloadStorageService.resolvePayload( - state.getParsedPayloadJson(), "read similar ASIN orphan rows failed"); - JsonNode array = objectMapper.readTree(payloadJson); - if (!array.isArray()) { - continue; - } - List matchingRows = new ArrayList<>(); - for (JsonNode node : array) { - matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); - } - appendRequestedResultRows(sink, matchingRows, requestedKeys, true); - } catch (Exception ex) { - log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}", - taskId, state.getId(), ex.getMessage()); - } - } - } - - private Map loadPersistedResultRows(Long taskId) { - Map result = new LinkedHashMap<>(); - List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getChunkIndex)); - for (TaskChunkEntity chunk : chunks) { - result.putAll(readChunkRows(chunk)); - } - // 合并 orphan 行（跨 chunk merge 兜底），仅当 chunk 中没有同 rowKey 时才补上，避免覆盖正常结果。 - Map orphans = loadOrphanLlmRows(taskId); - if (!orphans.isEmpty()) { - int added = 0; - for (Map.Entry entry : orphans.entrySet()) { - if (!result.containsKey(entry.getKey())) { - result.put(entry.getKey(), entry.getValue()); - added++; - } - } - if (added > 0) { - log.warn("[similar-asin] merged orphan llm rows into result taskId={} added={} totalOrphan={}", - taskId, added, orphans.size()); - } - } - return result; - } - - private boolean hasPersistedResultRows(Long taskId) { - if (taskId == null || taskId <= 0) { - return false; - } - Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); - return count != null && count > 0; - } - - private Map loadPersistedResultRowsWithRetry(Long taskId, int expectedParsedRows) { - Map result = loadPersistedResultRows(taskId); - if (expectedParsedRows <= 0 || !result.isEmpty()) { - return result; - } - for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) { - sleepBeforeResultRowsRetry(attempt); - result = loadPersistedResultRows(taskId); - if (!result.isEmpty()) { - log.info("[similar-asin] result rows recovered after retry taskId={} attempt={} rows={}", - taskId, attempt, result.size()); - return result; - } - log.warn("[similar-asin] result rows still empty after retry taskId={} attempt={}/{}", - taskId, attempt, RESULT_ROWS_READ_RETRY_LIMIT); - } - return result; - } - - private void sleepBeforeResultRowsRetry(int attempt) { - try { - Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } - - private void writeResultWorkbook(File xlsx, - List rowsToWrite, - Map resultMap, - Map taskImageCache) { - ensureResultAssemblyNotInterrupted(); - // 单元格内嵌图片：写完 workbook 后 patch WPS DISPIMG 结构（WPS/钉钉/飞书 打开即为单元格图片， - // 与 Excel 365 richData 不同，DISPIMG 是 WPS 原生协议；协议细节见 WpsCellImageWriter 类注释）。 - WpsCellImageWriter.Session cellImageSession = WpsCellImageWriter.createSession(); - SimilarAsinImageEmbedder.ImageSpool imageSpool; - try { - Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-"); - imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir); - } catch (IOException ex) { - throw new BusinessException("create Similar ASIN image spool failed", ex); - } - try { - try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { - Sheet sheet = workbook.createSheet("相似asin检测"); - CellStyle headerStyle = workbook.createCellStyle(); - Font font = workbook.createFont(); - font.setBold(true); - headerStyle.setFont(font); - - // taskImageCache 由 assembleResultWorkbook 跨 workbook 注入，避免 4 源文件 × 同一批 URL 重复下载。 - - sheet.setColumnWidth(IMG_COL_MAIN, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256); - sheet.setColumnWidth(IMG_COL_PUZZLE1, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256); - sheet.setColumnWidth(IMG_COL_PUZZLE2, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256); - - List resultHeaders = new ArrayList<>(RESULT_HEADERS); - Row header = sheet.createRow(0); - for (int i = 0; i < resultHeaders.size(); i++) { - Cell cell = header.createCell(i); - cell.setCellValue(resultHeaders.get(i)); - cell.setCellStyle(headerStyle); - } - - // 预扫图片 URL 并行下载到任务级临时目录；POI 仍按行串行注册 cell image。 - List prefetchUrls = collectImageUrlsForPrefetch(rowsToWrite, resultMap); - if (!prefetchUrls.isEmpty()) { - // DB cache 命中的缩略图也立即落到任务临时目录，避免整批字节停留在堆中。 - long dbCacheStart = System.currentTimeMillis(); - List remainingUrls = new ArrayList<>(prefetchUrls.size()); - int dbHit = 0; - for (String url : prefetchUrls) { - ensureResultAssemblyNotInterrupted(); - if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) { - continue; - } - byte[] cachedBytes = imagePrefetchService.lookup(url); - if (cachedBytes == null) { - remainingUrls.add(url); - continue; - } - SimilarAsinImageEmbedder.ResizedImage thumb = imageEmbedder.decodeCachedThumb(cachedBytes); - if (thumb != null) { - imageSpool.put(url, thumb); - dbHit++; - } else { - remainingUrls.add(url); - } - } - log.info("[similar-asin] image db-cache lookup finished urls={} hit={} miss={} costMs={}", - prefetchUrls.size(), dbHit, remainingUrls.size(), System.currentTimeMillis() - dbCacheStart); - if (!remainingUrls.isEmpty()) { - long prefetchStart = System.currentTimeMillis(); - imageEmbedder.prefetchToDisk(remainingUrls, imageSpool); - ensureResultAssemblyNotInterrupted(); - log.info("[similar-asin] image disk-prefetch finished urls={} spooled={} costMs={}", - remainingUrls.size(), imageSpool.size(), System.currentTimeMillis() - prefetchStart); - } - } - - int rowIndex = 1; - for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.of() : rowsToWrite) { - ensureResultAssemblyNotInterrupted(); - SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap); - Row row = sheet.createRow(rowIndex); - int col = 0; - row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId())); - row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), "")); - row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), "")); - row.createCell(col++).setCellValue(resultRow == null - ? firstNonBlank(parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")) - : firstNonBlank( - resultRow.getPrice(), - firstNonBlank(parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")))); - row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name")); - row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "品牌", "brand")); - String isStock = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getIsStock()); - String similarity = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getSimilarity()); - String isConform = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getIsConform()); - String reason = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getReason()); - String category = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getCategory()); - String status = resultRow == null - || (isSilentOutputFailure(resultRow) && !isAsinNotFound(resultRow.getError())) - ? "" - : resolveResultStatus(resultRow, isStock, similarity, isConform, reason, category); - row.createCell(col++).setCellValue(isStock); - row.createCell(col++).setCellValue(similarity); - row.createCell(col++).setCellValue(isConform); - row.createCell(col++).setCellValue(reason); - row.createCell(col++).setCellValue(category); - row.createCell(col).setCellValue(status); - if (resultRow != null) { - // 行高固定到 IMAGE_ROW_HEIGHT_POINTS（Excel 上限 409pt）： - row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS); - imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN, - resultRow.getMainUrl(), row, taskImageCache, cellImageSession, imageSpool); - imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1, - resultRow.getPuzzleImg1(), row, taskImageCache, cellImageSession, imageSpool); - imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2, - resultRow.getPuzzleImg2(), row, taskImageCache, cellImageSession, imageSpool); - } - rowIndex++; - } - ensureResultAssemblyNotInterrupted(); - workbook.write(fos); - workbook.dispose(); - } - if (!cellImageSession.isEmpty()) { - try { - ensureResultAssemblyNotInterrupted(); - WpsCellImageWriter.patchXlsxFile(xlsx, cellImageSession); - } catch (IOException ex) { - String cause = ex.toString(); - log.error("[similar-asin] excel cell image patch failed xlsx={} images={} err={}", - xlsx.getAbsolutePath(), cellImageSession.size(), cause, ex); - throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex); - } - } - } catch (BusinessException ex) { - throw ex; - } catch (Exception ex) { - throw new BusinessException("生成相似ASIN检测结果失败", ex); - } finally { - try { - imageSpool.close(); - } catch (IOException ex) { - log.warn("[similar-asin] delete image spool failed xlsx={} err={}", - xlsx.getAbsolutePath(), ex.getMessage()); - } - } - } - - private static void cancelResultAssemblyFutures(List> futures) { - for (Future future : futures) { - if (future != null && !future.isDone()) { - future.cancel(true); - } - } - } - - private static void ensureResultAssemblyNotInterrupted() { - if (Thread.currentThread().isInterrupted()) { - throw new BusinessException("生成相似ASIN检测结果中断"); - } - } - - /** - * P2-8：收集本次 xlsx 写入需要嵌入的所有图片 URL（去重交给 prefetch 内部处理）。 - * 仅扫 resultMap 中实际命中的行，避免对 unresolved row 浪费下载配额。 - */ - private List collectImageUrlsForPrefetch(List rowsToWrite, - Map resultMap) { - if (rowsToWrite == null || rowsToWrite.isEmpty() || resultMap == null || resultMap.isEmpty()) { - return List.of(); - } - List urls = new ArrayList<>(rowsToWrite.size() * 3); - for (SimilarAsinParsedRowVo parsedRow : rowsToWrite) { - SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap); - if (resultRow == null) { - continue; - } - addNonBlank(urls, resultRow.getMainUrl()); - addNonBlank(urls, resultRow.getPuzzleImg1()); - addNonBlank(urls, resultRow.getPuzzleImg2()); - } - return urls; - } - - private static void addNonBlank(List sink, String value) { - if (value == null) { - return; - } - String trimmed = value.trim(); - if (!trimmed.isEmpty()) { - sink.add(trimmed); - } - } - - private String imageUrlCellValue(SimilarAsinResultRowDto resultRow, String fallbackUrl) { - if (resultRow == null) { - return firstNonBlank(fallbackUrl, ""); - } - List urls = resultRow.getUrls(); - if (urls != null && !urls.isEmpty()) { - return String.join("\n", urls); - } - return firstNonBlank(resultRow.getUrl(), fallbackUrl); - } - /** * Task 89：解析委托 SimilarAsinExcelParser（表头/单元格读取、别名匹配、空行跳过、字段截断）， * 保留服务侧编排语义：zip 探针、status 列过滤、类目重试判定、行号/分组键/rowToken 组装。 @@ -4785,7 +3086,8 @@ public class SimilarAsinTaskService { log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage()); } try { - SimilarAsinExcelParser.ParsedSheet parsedSheet = excelParser.parse(input, resolveMaxFieldLength()); + SimilarAsinExcelParser.ParsedSheet parsedSheet = excelParser.parse(input, + SimilarAsinLimits.resolveMaxFieldLength(properties)); List headers = parsedSheet.headers(); int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers); String statusHeader = statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : null; @@ -4814,8 +3116,8 @@ public class SimilarAsinTaskService { vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName())); vo.setRowIndex(parsedRow.rowIndex()); vo.setSourceId(id); - vo.setDisplayId(normalizeDisplayId(id)); - String rowBaseId = baseId(vo.getDisplayId()); + vo.setDisplayId(SimilarAsinResultTextSupport.normalizeDisplayId(id)); + String rowBaseId = SimilarAsinResultTextSupport.baseId(vo.getDisplayId()); if (!Objects.equals(currentBlockBaseId, rowBaseId)) { currentBlockBaseId = rowBaseId; currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex()); @@ -4862,131 +3164,6 @@ public class SimilarAsinTaskService { } } - private boolean hasPromptFields(SimilarAsinParsedRowVo row) { - return row != null && (!normalize(row.getTitle()).isBlank() || !normalize(row.getUrl()).isBlank()); - } - - private boolean shouldInspectRow(SimilarAsinResultRowDto row) { - return row != null && (!rowKey(row).isBlank() - || !normalize(row.getTitle()).isBlank() - || row.hasImageUrl()); - } - - private boolean hasResolvedLlmFields(SimilarAsinResultRowDto row) { - if (row == null) { - return false; - } - return hasUsableLlmField(row.getTitleRisk()) - || hasUsableLlmField(row.getAppearanceRisk()) - || hasUsableLlmField(row.getPatentRisk()) - || hasUsableLlmField(row.getConclusion()) - || hasUsableLlmField(row.getIsStock()) - || hasUsableLlmField(row.getSimilarity()) - || hasUsableLlmField(row.getIsConform()) - || hasUsableLlmField(row.getReason()) - || hasUsableLlmField(row.getCategory()) - || hasUsableLlmField(row.getStatus()) - || hasText(row.getMainUrl()) - || hasText(row.getPuzzleImg1()) - || hasText(row.getPuzzleImg2()); - } - - static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) { - if (row == null) { - return "\u5931\u8d25"; - } - if (hasText(row.getMainUrl()) || hasText(row.getPuzzleImg1()) || hasText(row.getPuzzleImg2())) { - return "\u6210\u529f"; - } - // Python \u7aef\u641c\u4e0d\u5230\u5546\u54c1\uff08404\uff09\u7684\u884c\uff1a\u72b6\u6001\u5217\u660e\u786e\u6807\u8bb0\uff0c\u4eba\u5de5\u590d\u6838\u65f6\u4e00\u773c\u53ef\u8fa8\u3002 - // \u653e\u5728\u56fe\u7247/\u72b6\u6001\u5224\u65ad\u4e4b\u524d\uff1a404 \u884c\u5fc5\u7136\u65e0\u56fe\u4e14 status \u5df2\u88ab markRowFailed \u7f6e\u4e3a FAILED\uff0c - // \u4e0d\u80fd\u843d\u56de"\u5931\u8d25"\uff08\u90a3\u662f LLM/\u751f\u6210\u94fe\u8def\u5931\u8d25\u8bed\u4e49\uff0c\u4e0e"ASIN \u641c\u4e0d\u5230"\u662f\u4e24\u56de\u4e8b\uff09\u3002 - if (isAsinNotFound(row.getError())) { - return "ASIN\u672a\u627e\u5230"; - } - String rawStatus = row.getStatus() == null ? "" : row.getStatus().trim(); - if (!rawStatus.isBlank()) { - return isFailedStatus(rawStatus) ? "\u5931\u8d25" : "\u6210\u529f"; - } - if (visibleResultValues != null) { - for (String value : visibleResultValues) { - if (hasText(value)) { - return "\u6210\u529f"; - } - } - } - return hasText(row.getConclusion()) ? "\u6210\u529f" : "\u5931\u8d25"; - } - - /** - * \u5224\u5b9a\u662f\u5426"ASIN \u672a\u627e\u5230"\uff1aPython \u7aef\u6293\u53d6 404\uff08error="404 Not Found"\uff09\u6216\u7b49\u4ef7\u6587\u6848\u3002 - * \u547d\u4e2d\u8be5\u5224\u5b9a\u7684\u884c\u4e0d\u4f1a\u88ab\u5f53\u6210\u751f\u6210\u5931\u8d25\u8ba1\u91cd\u8bd5\uff0c\u4e5f\u4e0d\u4f1a\u88ab\u5bfc\u51fa\u8fc7\u6ee4\uff0c\u72b6\u6001\u5217\u663e\u793a"ASIN\u672a\u627e\u5230"\u3002 - */ - static boolean isAsinNotFound(String error) { - String normalized = error == null ? "" : error.toLowerCase(Locale.ROOT).trim(); - if (normalized.isBlank()) { - return false; - } - return normalized.contains("404") - || normalized.contains("not found") - || normalized.contains("asin \u672a\u627e\u5230") - || normalized.contains("asin\u4e0d\u5b58\u5728") - || normalized.contains("\u5546\u54c1\u672a\u627e\u5230"); - } - - private static boolean isFailedStatus(String status) { - String normalized = status == null ? "" : status.trim().toLowerCase(Locale.ROOT); - return normalized.contains("fail") - || normalized.contains("error") - || normalized.contains("\u5931\u8d25") - || normalized.contains("\u9519\u8bef"); - } - - private static boolean hasText(String value) { - return value != null && !value.trim().isBlank(); - } - - private boolean isExportableResultRow(SimilarAsinResultRowDto row) { - if (row == null) { - return false; - } - if (hasResolvedLlmFields(row)) { - return true; - } - String error = normalize(row.getError()); - if (error.isBlank()) { - return false; - } - // Python 端搜不到商品的 404 行：虽然 title/url/图全空，也必须导出， - // 并在状态列标记"ASIN未找到"，供人工复核。在"其它错误需带 title/url/图"之前判断。 - if (isAsinNotFound(error)) { - return true; - } - return !normalize(row.getTitle()).isBlank() - || !normalize(row.getUrl()).isBlank() - || row.hasImageUrl(); - } - - private boolean hasUsableLlmField(String value) { - String normalized = normalize(value); - return !normalized.isBlank() && !isTechnicalLlmFailure(normalized); - } - - private static boolean isSpreadsheetErrorValue(String value) { - String normalized = value == null ? "" : value.trim(); - return normalized.startsWith("#") && normalized.endsWith("!"); - } - - private String baseId(String id) { - String s = normalize(id); - int idx = s.indexOf('_'); - return idx > 0 ? s.substring(0, idx) : s; - } - - private String normalizeDisplayId(String id) { - return id == null ? "" : id.trim(); - } - private void mergeHeaders(List mergedHeaders, List headers) { if (mergedHeaders == null || headers == null || headers.isEmpty()) { return; @@ -5040,116 +3217,6 @@ public class SimilarAsinTaskService { return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex); } - private long countTask(Long userId, String status) { - Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getUserId, userId) - .eq(FileTaskEntity::getStatus, status)); - return count == null ? 0L : count; - } - - private long countCompletedTasksWithResultFile(Long userId) { - if (userId == null || userId <= 0) { - return 0L; - } - Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getUserId, userId) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .inSql(FileTaskEntity::getId, - "select distinct task_id from biz_file_result " + - "where module_type = '" + MODULE_TYPE + "' " + - "and user_id = " + userId + " " + - "and result_file_url is not null and result_file_url <> ''")); - return count == null ? 0L : count; - } - - private int countChunks(Long taskId, String scopeHash) { - Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .eq(TaskChunkEntity::getScopeHash, scopeHash)); - return count == null ? 0 : count.intValue(); - } - - private int resolveReceivedChunkProgress(Long taskId, String scopeHash, Integer chunkTotal) { - int receivedCount = countChunks(taskId, scopeHash); - Integer maxChunkIndex = taskChunkMapper.selectObjs(new LambdaQueryWrapper() - .select(TaskChunkEntity::getChunkIndex) - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .eq(TaskChunkEntity::getScopeHash, scopeHash) - .orderByDesc(TaskChunkEntity::getChunkIndex) - .last("limit 1")) - .stream() - .filter(Objects::nonNull) - .map(value -> { - if (value instanceof Number number) { - return number.intValue(); - } - try { - return Integer.parseInt(value.toString()); - } catch (Exception ignored) { - return 0; - } - }) - .findFirst() - .orElse(0); - int current = Math.max(receivedCount, maxChunkIndex == null ? 0 : maxChunkIndex); - if (chunkTotal != null && chunkTotal > 0) { - current = Math.min(current, chunkTotal); - } - return Math.max(0, current); - } - - private SimilarAsinTaskItemVo toTaskItem(FileTaskEntity task) { - SimilarAsinTaskItemVo vo = new SimilarAsinTaskItemVo(); - vo.setId(task.getId()); - vo.setTaskNo(task.getTaskNo()); - vo.setStatus(resolveDisplayTaskStatus(task)); - vo.setErrorMessage(task.getErrorMessage()); - vo.setCreatedAt(fmt(task.getCreatedAt())); - vo.setUpdatedAt(fmt(task.getUpdatedAt())); - vo.setFinishedAt(fmt(task.getFinishedAt())); - return vo; - } - - private String resolveDisplayTaskStatus(FileTaskEntity task) { - return resolveDisplayTaskStatus(task, null); - } - - private String resolveDisplayTaskStatus(FileTaskEntity task, FileResultEntity row) { - if (task == null) { - return null; - } - if (canTreatTaskAsCompleted(task, row, null)) { - return STATUS_SUCCESS; - } - return task.getStatus(); - } - - private boolean canTreatTaskAsCompleted(FileTaskEntity task, FileResultEntity row, TaskFileJobEntity job) { - if (task == null) { - return false; - } - if (STATUS_SUCCESS.equals(task.getStatus())) { - return true; - } - if (!STATUS_RUNNING.equals(task.getStatus())) { - return false; - } - if (row == null) { - return false; - } - if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { - return false; - } - if (job != null && "FAILED".equals(job.getStatus())) { - return false; - } - return true; - } - private String fmt(LocalDateTime t) { return t == null ? null : t.toString(); } @@ -5177,8 +3244,8 @@ public class SimilarAsinTaskService { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) { throw new BusinessException("任务不存在"); } - ensureTaskOwnedByCurrentInstance(task, "parsed payload"); - return readParsedPayload(task); + ownershipSupport().ensureTaskOwnedByCurrentInstance(task, "parsed payload"); + return payloadSupport().readParsedPayload(task); } private String normalize(String val) { @@ -5197,406 +3264,10 @@ public class SimilarAsinTaskService { // 兼容 Python 旧链路：groups 需内嵌完整行（Python 端 normalize_groups // 直接返回 groups 并按 gp["items"] 取行；仅有 startIndex/endIndex 引用时 // items 为空、任务空转）。startIndex/endIndex 保留，Java 内部 expandGroupRefs 继续使用。 - payload.setGroups(fillGroupItems(groups, allRows)); + payload.setGroups(SimilarAsinPayloadSupport.fillGroupItems(groups, allRows)); return writeJson(payload, "保存解析结果失败"); } - /** - * 按 group.startIndex/endIndex 半开区间从全量行切片，内嵌回 group.items； - * 无索引或切片越界时退回按 group.items 原样（兼容旧 payload 反序列化场景）。 - */ - static List fillGroupItems(List groups, List allRows) { - if (groups == null || groups.isEmpty()) { - return groups == null ? List.of() : groups; - } - if (allRows == null || allRows.isEmpty()) { - return groups; - } - List rows = new ArrayList<>(allRows); - for (SimilarAsinParsedGroupVo group : groups) { - if (group == null) { - continue; - } - Integer start = group.getStartIndex(); - Integer end = group.getEndIndex(); - if (start != null && end != null) { - int from = Math.max(0, Math.min(start, rows.size())); - int to = Math.max(from, Math.min(end, rows.size())); - if (to > from) { - group.setItems(new ArrayList<>(rows.subList(from, to))); - continue; - } - } - // 索引缺失/无效：保留已有 items（兼容旧 payload），不覆盖 - if (group.getItems() == null) { - group.setItems(new ArrayList<>()); - } - } - return groups; - } - - private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List sourceFiles, String parsedPayloadPointer) { - Map payload = new LinkedHashMap<>(); - payload.put("aiPrompt", normalize(aiPrompt)); - payload.put("apiKey", normalize(apiKey)); - payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch)); - payload.put("categorySwitch", Boolean.TRUE.equals(categorySwitch)); - payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream() - .map(SimilarAsinSourceFileDto::getFileKey) - .filter(Objects::nonNull) - .toList()); - payload.put("parsedPayloadRef", parsedPayloadPointer); - payload.put("ownerInstanceId", currentInstanceId()); - return writeJson(payload, "保存任务结果索引失败"); - } - - private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { - return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, false); - } - - private long elapsedMs(long start, long end) { - return (end - start) / 1_000_000L; - } - - private SimilarAsinParsedPayloadDto readParsedPayload(FileTaskEntity task) { - try { - JsonNode root = objectMapper.readTree(task.getResultJson()); - String pointer = root.path("parsedPayloadRef").asText(""); - if (!pointer.isBlank()) { - String json = transientPayloadStorageService.resolvePayload(pointer, "read similar ASIN parsed payload failed"); - return hydrateParsedPayloadRows(objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class)); - } - if (root.path("allItems").isArray()) { - return hydrateParsedPayloadRows(objectMapper.treeToValue(root, SimilarAsinParsedPayloadDto.class)); - } - } catch (Exception ex) { - throw new BusinessException("读取解析载荷失败"); - } - return hydrateParsedPayloadRows(new SimilarAsinParsedPayloadDto()); - } - - private SimilarAsinParsedPayloadDto hydrateParsedPayloadRows(SimilarAsinParsedPayloadDto payload) { - return hydrateForPythonCompat(payload); - } - - /** 解析载荷 hydrate：回填 allItems/items/groups 内嵌 items（Python 旧链路兼容）。 */ - static SimilarAsinParsedPayloadDto hydrateForPythonCompat(SimilarAsinParsedPayloadDto payload) { - if (payload == null) { - return new SimilarAsinParsedPayloadDto(); - } - List rows = resolveAllRows(payload); - if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) { - payload.setAllItems(rows); - } - if (payload.getItems() == null || payload.getItems().isEmpty()) { - payload.setItems(rows); - } - // Python 旧链路兼容：读取时也回填 groups 内嵌 items（存量任务 payload - // 只有引用），否则 Python normalize_groups 拿到空 items 任务空转。 - if (payload.getGroups() != null && !payload.getGroups().isEmpty()) { - List filledRows = payload.getItems() == null || payload.getItems().isEmpty() - ? rows : payload.getItems(); - payload.setGroups(fillGroupItems(payload.getGroups(), filledRows)); - } - return payload; - } - - private Map readChunkRows(TaskChunkEntity chunk) { - Map rows = new LinkedHashMap<>(); - if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) { - return rows; - } - try { - String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read similar ASIN chunk failed"); - JsonNode array = objectMapper.readTree(payloadJson); - if (!array.isArray()) { - return rows; - } - for (JsonNode node : array) { - SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class); - rows.put(rowKey(row), row); - } - } catch (Exception ex) { - String msg = ex.getMessage() == null ? "" : ex.getMessage(); - // P1-4：识别跨实例 local 指针读不到的场景，把"chunk 在另一实例"的元信息 - // 通过 task_scope_state.last_error 留痕，便于排查"为何 owner 切换后 chunk 读不到"。 - boolean crossInstance = msg.contains("only exists on instance="); - // 类型不兼容（如历史版本序列化数据含当前类加载器不认识的类型标记）： - // 该 chunk 永久不可读，跳过而不是把组装/收尾永久拖死，留痕便于排查。 - boolean typeMismatch = ex instanceof TypeNotPresentException - || msg.contains("not present"); - if (typeMismatch) { - log.warn("[similar-asin] chunk payload type mismatch, skip chunk taskId={} chunk={} err={}", - chunk.getTaskId(), chunk.getChunkIndex(), msg); - recordChunkReadFailure(chunk, false, msg); - return rows; - } - log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}", - chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg); - recordChunkReadFailure(chunk, crossInstance, msg); - throw new BusinessException("similar ASIN chunk payload read failed chunk=" - + chunk.getChunkIndex() + ": " + msg, ex); - } - return rows; - } - - /** - * P1-4：扫描该 task 下所有 task_scope_state.last_error， - * 把 "chunk-read-failed[N]" 标记收集成一行摘要返回；没命中返回空字符串。 - */ - private String collectChunkReadFailureSummary(Long taskId) { - if (taskId == null) { - return ""; - } - try { - List scopes = taskScopeStateMapper.selectList( - new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); - if (scopes == null || scopes.isEmpty()) { - return ""; - } - LinkedHashSet tags = new LinkedHashSet<>(); - for (TaskScopeStateEntity scope : scopes) { - String lastError = scope == null ? null : scope.getLastError(); - if (lastError == null || lastError.isBlank()) { - continue; - } - int from = 0; - while (true) { - int start = lastError.indexOf("chunk-read-failed[", from); - if (start < 0) { - break; - } - int end = lastError.indexOf(']', start); - if (end < 0) { - break; - } - tags.add(lastError.substring(start, end + 1)); - from = end + 1; - } - } - return String.join(", ", tags); - } catch (Exception ex) { - log.warn("[similar-asin] collect chunk-read-failure summary failed taskId={} err={}", taskId, ex.getMessage()); - return ""; - } - } - - /** - * P1-4：把 chunk 读失败信息聚合到 task_scope_state.last_error， - * 形式 "chunk-read-failed[{idx}]" / "chunk-read-failed[{idx}@cross-instance]"。 - * 后续 assembleResultWorkbook 之前抛"结果为空"时可以附带 chunk index 列表，定位更具体。 - * best-effort：查询 / 更新失败时仅记日志，不抛回主流程。 - */ - private void recordChunkReadFailure(TaskChunkEntity chunk, boolean crossInstance, String msg) { - if (chunk == null || chunk.getTaskId() == null || chunk.getScopeHash() == null) { - return; - } - try { - TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, chunk.getTaskId()) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .eq(TaskScopeStateEntity::getScopeHash, chunk.getScopeHash()) - .last("limit 1")); - if (scope == null) { - return; - } - String existing = scope.getLastError() == null ? "" : scope.getLastError(); - String tag = "chunk-read-failed[" + chunk.getChunkIndex() + (crossInstance ? "@cross-instance" : "") + "]"; - if (existing.contains(tag)) { - return; - } - String updated = existing.isBlank() ? tag : existing + "; " + tag; - taskScopeStateMapper.update(null, new LambdaUpdateWrapper() - .eq(TaskScopeStateEntity::getId, scope.getId()) - .set(TaskScopeStateEntity::getLastError, updated) - .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); - } catch (Exception ignored) { - // best-effort：失败不影响主流程 - } - } - - /** - * 跨 chunk merge 时把没有匹配到任何 chunk 的回流行兜底落到 task_scope_state， - * 避免数据被静默丢弃；assembleResult 阶段会在 {@link #loadPersistedResultRows(Long)} - * 中读回这些 orphan 行并合入最终 xlsx。 - * 使用 best-effort 策略：transient storage 出问题时仅记录 warn，不打断主流程。 - */ - private void persistOrphanLlmRows(Long taskId, List orphanRows) { - if (taskId == null || taskId <= 0 || orphanRows == null || orphanRows.isEmpty()) { - return; - } - try { - // 给每条 orphan 行写入 marker，后续导出时可识别。 - for (SimilarAsinResultRowDto row : orphanRows) { - if (row == null) { - continue; - } - String existingError = row.getError(); - String marker = ORPHAN_LLM_ROW_ERROR; - if (existingError == null || existingError.isBlank()) { - row.setError(marker); - } else if (!existingError.contains(marker)) { - row.setError(existingError + " | " + marker); - } - } - String orphanScopeKey = ORPHAN_SCOPE_KEY_PREFIX + UUID.randomUUID(); - String orphanScopeHash = DigestUtil.sha256Hex(orphanScopeKey); - String payloadJson = writeJson(orphanRows, "serialize orphan llm rows failed"); - String storedPayload = transientPayloadStorageService.storeParsedPayloadEntry( - MODULE_TYPE, taskId, orphanScopeHash, "orphan-" + UUID.randomUUID(), payloadJson, true); - LocalDateTime now = LocalDateTime.now(); - TaskScopeStateEntity state = new TaskScopeStateEntity(); - state.setTaskId(taskId); - state.setModuleType(MODULE_TYPE); - state.setScopeKey(orphanScopeKey); - state.setScopeHash(orphanScopeHash); - state.setParsedPayloadJson(storedPayload); - state.setStateJson("{\"phase\":\"ORPHAN_LLM_ROWS\"}"); - // llmStatus 直接置 DONE，避免被 listOwnedPendingLlmStates 扫描到。 - state.setLlmStatus(LLM_STATUS_DONE); - state.setLlmSubmittedAt(now); - state.setLlmCompletedAt(now); - state.setLlmAttemptCount(0); - state.setLlmError(ORPHAN_LLM_ROW_ERROR); - state.setChunkTotal(orphanRows.size()); - state.setReceivedChunkCount(orphanRows.size()); - state.setCompleted(1); - state.setLastChunkAt(now); - state.setCreatedAt(now); - state.setUpdatedAt(now); - try { - taskScopeStateMapper.insert(state); - log.warn("[similar-asin] orphan llm rows persisted taskId={} rows={} scopeKey={}", - taskId, orphanRows.size(), orphanScopeKey); - } catch (DuplicateKeyException ex) { - transientPayloadStorageService.deletePayloadIfPresent(storedPayload); - log.info("[similar-asin] duplicate orphan scope state ignored taskId={} scopeKey={}", - taskId, orphanScopeKey); - } - } catch (Exception ex) { - log.warn("[similar-asin] persist orphan llm rows failed taskId={} rows={} err={}", - taskId, orphanRows.size(), ex.getMessage()); - } - } - - /** - * 读回 {@link #persistOrphanLlmRows(Long, List)} 落库的 orphan 行集合。 - * 仅 assembleResult 阶段调用，与 chunk 行合并时使用 putIfAbsent 语义： - * 同一 rowKey 下 chunk 已有结果时优先用 chunk 行，避免覆盖正常路径数据。 - */ - private Map loadOrphanLlmRows(Long taskId) { - Map rows = new LinkedHashMap<>(); - if (taskId == null || taskId <= 0) { - return rows; - } - List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .likeRight(TaskScopeStateEntity::getScopeKey, ORPHAN_SCOPE_KEY_PREFIX)); - if (states == null || states.isEmpty()) { - return rows; - } - for (TaskScopeStateEntity state : states) { - if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { - continue; - } - try { - String payloadJson = transientPayloadStorageService.resolvePayload( - state.getParsedPayloadJson(), "read similar ASIN orphan rows failed"); - JsonNode array = objectMapper.readTree(payloadJson); - if (!array.isArray()) { - continue; - } - for (JsonNode node : array) { - SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class); - String key = rowKey(row); - if (key.isBlank()) { - continue; - } - rows.putIfAbsent(key, row); - } - } catch (Exception ex) { - log.warn("[similar-asin] read orphan llm rows failed taskId={} stateId={} err={}", - taskId, state.getId(), ex.getMessage()); - } - } - return rows; - } - - private String rowKey(SimilarAsinParsedRowVo row) { - if (row == null) { - return ""; - } - String rowToken = normalize(row.getRowToken()); - if (!rowToken.isBlank()) { - return rowToken; - } - return legacyRowKey(row); - } - - private String rowKey(SimilarAsinResultRowDto row) { - if (row == null) { - return ""; - } - String rowToken = normalize(row.getRowToken()); - if (!rowToken.isBlank()) { - return rowToken; - } - return legacyRowKey(row); - } - - private SimilarAsinResultRowDto findResultRow(SimilarAsinParsedRowVo parsedRow, - Map resultMap) { - if (parsedRow == null || resultMap == null || resultMap.isEmpty()) { - return null; - } - SimilarAsinResultRowDto resultRow = resultMap.get(rowKey(parsedRow)); - if (resultRow != null) { - return resultRow; - } - String legacyKey = legacyRowKey(parsedRow); - if (legacyKey.isBlank()) { - return null; - } - return resultMap.get(legacyKey); - } - - private String legacyRowKey(SimilarAsinParsedRowVo row) { - if (row == null) { - return ""; - } - return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry()); - } - - private String legacyRowKey(SimilarAsinResultRowDto row) { - if (row == null) { - return ""; - } - return rowKey(row.getId(), row.getAsin(), row.getCountry()); - } - - private String rowKey(String id, String asin, String country) { - return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country); - } - - private String readValueByHeader(SimilarAsinParsedRowVo row, String... candidates) { - if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) { - return ""; - } - for (Map.Entry entry : row.getValues().entrySet()) { - String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT); - for (String candidate : candidates) { - String expected = normalize(candidate).toLowerCase(Locale.ROOT); - if (!expected.isBlank() && header.contains(expected)) { - return entry.getValue() == null ? "" : entry.getValue(); - } - } - } - return ""; - } - private boolean isSimilarAsinResultWorkbook(List headers) { return hasHeader(headers, "状态") && (hasHeader(headers, "是否有货") @@ -5628,7 +3299,7 @@ public class SimilarAsinTaskService { for (Map.Entry entry : row.getValues() == null ? Map.of().entrySet() : row.getValues().entrySet()) { if (LLM_RESULT_HEADER_ALIASES.contains(FailedStatusRowFilter.canonicalizeHeader(entry.getKey())) - && !isBlankOrSpreadsheetError(entry.getValue())) { + && !SimilarAsinResultTextSupport.isBlankOrSpreadsheetError(entry.getValue())) { return false; } } @@ -5655,10 +3326,6 @@ public class SimilarAsinTaskService { .orElse(""); } - private static boolean isBlankOrSpreadsheetError(String value) { - return value == null || value.trim().isBlank() || isSpreadsheetErrorValue(value); - } - static boolean shouldEnableCategorySwitchForRetry(List headers, List rows, boolean resultWorkbook) { @@ -5697,76 +3364,6 @@ public class SimilarAsinTaskService { return false; } - private String userFacingLlmCellValue(SimilarAsinResultRowDto row, String value) { - String normalizedValue = normalize(value); - if (isSilentOutputFailure(row) || isSilentOutputFailure(normalizedValue)) { - return ""; - } - if (!normalizedValue.isBlank() && !isTechnicalLlmFailure(normalizedValue)) { - return value; - } - // LLM 技术性失败：有错误信息则放入错误信息，没有则留空 - if (row != null && isTechnicalLlmFailure(row.getError())) { - return firstNonBlank(row.getError(), ""); - } - return firstNonBlank(value, ""); - } - - private String userFacingConclusion(SimilarAsinResultRowDto row) { - if (row == null) { - return ""; - } - String conclusion = normalize(row.getConclusion()); - if (!conclusion.isBlank() && !isTechnicalLlmFailure(conclusion)) { - return row.getConclusion(); - } - // LLM 技术性失败：有错误信息则放入错误信息，没有则留空 - if (isTechnicalLlmFailure(row.getError())) { - return firstNonBlank(row.getError(), ""); - } - return firstNonBlank(row.getConclusion(), ""); - } - - private boolean isSilentOutputFailure(SimilarAsinResultRowDto row) { - return row != null - && (isSilentOutputFailure(row.getError()) - || isSilentOutputFailure(row.getReason()) - || isSilentOutputFailure(row.getConclusion())); - } - - private boolean isSilentOutputFailure(String value) { - String normalized = normalize(value).toLowerCase(Locale.ROOT); - return normalized.contains("required field missing") - && normalized.contains("skip llm"); - } - - private boolean isTechnicalLlmFailure(String value) { - String normalized = normalize(value).toLowerCase(Locale.ROOT); - if (normalized.isBlank()) { - return false; - } - // 确定性输入失败统一通过 DETERMINISTIC_INPUT_FAILURE_PATTERN 判定， - // 其它技术性失败（超时、节点超限等）保持原有子串匹配。 - if (DETERMINISTIC_INPUT_FAILURE_PATTERN.matcher(normalized).find()) { - return true; - } - return normalized.contains("llm") - || normalized.contains("结果不完整") - || normalized.contains("工作流节点执行超限") - || normalized.contains("调用超时") - || normalized.contains("timeout"); - } - - private String safeFileStem(String filename) { - String name = filename == null || filename.isBlank() ? "similar-asin" : filename; - int idx = name.lastIndexOf('.'); - if (idx > 0) { - name = name.substring(0, idx); - } - String safe = name.replaceAll("[\\\\/:*?\"<>|]", "_").trim(); - return safe.isBlank() ? "similar-asin" : safe; - } - private String writeJson(Object value, String message) { try { return objectMapper.writeValueAsString(value); @@ -5843,62 +3440,6 @@ public class SimilarAsinTaskService { boolean terminal) { } - record LlmBatchContext(Long jobId, - Long resultId, - String chunkScopeHash, - Integer chunkIndex, - Integer batchIndex, - Integer batchTotal, - String ownerInstanceId, - Integer submitRetryCount, - String credentialName, - String resultPayloadPointer) { - } - - private static class SourceRowsBuilder { - private final String sourceFileKey; - private String sourceFilename; - private final List rows = new ArrayList<>(); - - private SourceRowsBuilder(String sourceFileKey, String sourceFilename) { - this.sourceFileKey = sourceFileKey; - this.sourceFilename = sourceFilename; - } - - private String sourceFileKey() { - return sourceFileKey; - } - - private String sourceFilename() { - return sourceFilename; - } - - private void setSourceFilename(String sourceFilename) { - this.sourceFilename = sourceFilename; - } - - private List rows() { - return rows; - } - } - - private record SourceRows(String sourceFileKey, - String sourceFilename, - List rows) { - } - - private record SourceResultWorkbook(File file, - String filename, - int rowCount, - int persistedRows, - int resultRows, - long resolvedRows, - int conformPropagated) { - private SourceResultWorkbook(File file, String filename, int rowCount) { - this(file, filename, rowCount, 0, 0, 0L, 0); - } - } - public record ResultDownloadInfo(String url, String filename, String contentType, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/LlmBatchContext.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/LlmBatchContext.java new file mode 100644 index 00000000..9bb8f689 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/LlmBatchContext.java @@ -0,0 +1,18 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +/** + * LLM 批次上下文：持久化在 task_scope_state.state_json 上。 + * spec 05 续（拆分第 6 轮）：从 SimilarAsinTaskService 原样搬出为顶层 record， + * 供实例归属判定与 LLM 流水线共用，避免两族之间形成循环依赖。 + */ +public record LlmBatchContext(Long jobId, + Long resultId, + String chunkScopeHash, + Integer chunkIndex, + Integer batchIndex, + Integer batchTotal, + String ownerInstanceId, + Integer submitRetryCount, + String credentialName, + String resultPayloadPointer) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinChunkMergeSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinChunkMergeSupport.java new file mode 100644 index 00000000..7e30d9f6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinChunkMergeSupport.java @@ -0,0 +1,265 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 4 轮）：chunk 合并纯计算与行键（rowKey）解析。 + *

+ * 从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致： + *

    + *
  • 行键族：rowKey / legacyRowKey（rowToken 优先，缺失回退 {@code id::ASIN::country}）、 + * findResultRow（主键命中后退回 legacy 键）；
    • + *
  • chunk 合并纯计算：keyset 分页加载、rowKey→chunkKey 批量索引、LLM 行去重、 + * 跨 chunk 归属分配、行展开与行数/字节估算。
    • + *
+ * 无实例状态；仅 {@link #assignLlmRowsToChunks} 在行无归属 chunk 时打一条错误日志。 + */ +@Slf4j +public final class SimilarAsinChunkMergeSupport { + + private SimilarAsinChunkMergeSupport() { + } + + // ------------------------------------------------------------------ + // 行键族 + // ------------------------------------------------------------------ + + public static String rowKey(SimilarAsinParsedRowVo row) { + if (row == null) { + return ""; + } + String rowToken = SimilarAsinRowNormalizer.normalize(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + public static String rowKey(SimilarAsinResultRowDto row) { + if (row == null) { + return ""; + } + String rowToken = SimilarAsinRowNormalizer.normalize(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + public static String legacyRowKey(SimilarAsinParsedRowVo row) { + if (row == null) { + return ""; + } + return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry()); + } + + public static String legacyRowKey(SimilarAsinResultRowDto row) { + if (row == null) { + return ""; + } + return rowKey(row.getId(), row.getAsin(), row.getCountry()); + } + + public static String rowKey(String id, String asin, String country) { + return SimilarAsinRowNormalizer.normalize(id) + + "::" + SimilarAsinRowNormalizer.normalize(asin).toUpperCase(Locale.ROOT) + + "::" + SimilarAsinRowNormalizer.normalize(country); + } + + public static SimilarAsinResultRowDto findResultRow(SimilarAsinParsedRowVo parsedRow, + Map resultMap) { + if (parsedRow == null || resultMap == null || resultMap.isEmpty()) { + return null; + } + SimilarAsinResultRowDto resultRow = resultMap.get(rowKey(parsedRow)); + if (resultRow != null) { + return resultRow; + } + String legacyKey = legacyRowKey(parsedRow); + if (legacyKey.isBlank()) { + return null; + } + return resultMap.get(legacyKey); + } + + // ------------------------------------------------------------------ + // chunk 合并纯计算 + // ------------------------------------------------------------------ + + /** + * Task 10：keyset 分页加载 chunk 列表（按 id 递增翻页，避免大 offset）。 + * 中途查询失败抛项目约定异常，不返回半截结果。 + * 使用 QueryWrapper（列名直写）避免对 lambda 元数据缓存的依赖。 + */ + public static List loadChunksKeyset(TaskChunkMapper mapper, Long taskId, + String moduleType, int pageSize) { + if (taskId == null || mapper == null) { + return List.of(); + } + int batch = pageSize > 0 ? pageSize : 500; + List all = new ArrayList<>(); + long lastId = 0L; + while (true) { + List page; + try { + page = mapper.selectList(new QueryWrapper() + .eq("task_id", taskId) + .eq("module_type", moduleType) + .gt("id", lastId) + .orderByAsc("id") + .last("limit " + batch)); + } catch (Exception ex) { + throw new BusinessException("chunk keyset 分页查询失败: " + ex.getMessage(), ex); + } + if (page == null || page.isEmpty()) { + break; + } + all.addAll(page); + lastId = page.getLast().getId(); + if (page.size() < batch) { + break; + } + } + all.sort(Comparator.comparing(TaskChunkEntity::getChunkIndex, + Comparator.nullsLast(Comparator.naturalOrder()))); + return all; + } + + /** + * Task 10：把 chunk 结果行建立成 rowKey → chunkKey 的批量索引。 + * 供 assignLlmRowsToChunks 使用，把跨 chunk 线性扫描降为 O(1) 查找。 + * 同一 rowKey 出现在多个 chunk 时保留第一个（putIfAbsent），行为确定。 + */ + public static Map indexRowsByChunkKey(Map> rowsByChunk) { + Map index = new HashMap<>(); + if (rowsByChunk == null || rowsByChunk.isEmpty()) { + return index; + } + for (Map.Entry> entry : rowsByChunk.entrySet()) { + if (entry.getValue() == null) { + continue; + } + for (String rowKey : entry.getValue().keySet()) { + if (rowKey == null || rowKey.isBlank()) { + continue; + } + index.putIfAbsent(rowKey, entry.getKey()); + } + } + return index; + } + + /** + * Task 11：LLM 结果合并前按稳定 rowKey 一次性去重。 + * 优先 rowToken，缺失时回退归一化的 legacy key（id::ASIN::country）， + * 保留首次出现顺序。合并路径此前对每个重复行都重复 expand/分配/写回，是 O(n²) 热点。 + */ + public static List dedupeRowsByRowKey(List llmRows) { + List deduped = new ArrayList<>(); + if (llmRows == null || llmRows.isEmpty()) { + return deduped; + } + Set seenKeys = new HashSet<>(); + for (SimilarAsinResultRowDto row : llmRows) { + if (row == null) { + continue; + } + String key = rowKey(row); + if (key.isBlank() || !seenKeys.add(key)) { + continue; + } + deduped.add(row); + } + return deduped; + } + + /** + * Task 10：基于 rowKey 索引为 llm 回传行分配归属 chunk。 + * 命中索引 → 归属该 chunk；未命中且有有效 fallback（chunkScopeHash + chunkIndex） + * 且 fallback chunk 存在 → 归属 fallback；否则进 orphan 列表。 + * 与原实现逐 chunk 线性扫描语义完全一致，但每个行查找降为 O(1)。 + */ + public static Map> assignLlmRowsToChunks( + Map> rowsByChunk, + List llmRows, + Map rowKeyIndex, + String chunkScopeHash, + Integer chunkIndex, + List orphans) { + Map> mergeRowsByChunk = new LinkedHashMap<>(); + if (llmRows == null || llmRows.isEmpty() || orphans == null) { + return mergeRowsByChunk; + } + String fallbackKey = chunkScopeHash != null && !chunkScopeHash.isBlank() && chunkIndex != null + ? chunkStorageKey(chunkScopeHash, chunkIndex) : null; + boolean fallbackValid = fallbackKey != null && rowsByChunk != null && rowsByChunk.containsKey(fallbackKey); + for (SimilarAsinResultRowDto expandedRow : llmRows) { + if (expandedRow == null) { + continue; + } + String rowKey = rowKey(expandedRow); + if (rowKey.isBlank()) { + continue; + } + String chunkKey = rowKeyIndex == null ? null : rowKeyIndex.get(rowKey); + if (chunkKey != null) { + mergeRowsByChunk.computeIfAbsent(chunkKey, ignored -> new LinkedHashMap<>()) + .put(rowKey, expandedRow); + } else if (fallbackValid) { + mergeRowsByChunk.computeIfAbsent(fallbackKey, ignored -> new LinkedHashMap<>()) + .put(rowKey, expandedRow); + } else { + orphans.add(expandedRow); + log.error("[similar-asin] llm row has no submitted chunk rowKey={} asin={} country={}", + rowKey, expandedRow.getAsin(), expandedRow.getCountry()); + } + } + return mergeRowsByChunk; + } + + public static String chunkStorageKey(String scopeHash, Integer chunkIndex) { + return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex); + } + + public static int mergedRowCountOf(Map persistedRows, + List rows) { + return persistedRows.size() + rows.size(); + } + + public static List expandRows(List rows, + Map> allRowsByBaseId) { + if (rows == null || rows.isEmpty()) { + return List.of(); + } + List result = new ArrayList<>(); + Set emittedKeys = new LinkedHashSet<>(); + for (SimilarAsinResultRowDto row : rows) { + String key = firstNonBlank(SimilarAsinRowNormalizer.normalize(row.getRowToken()), rowKey(row)); + if (emittedKeys.add(key)) { + result.add(row); + } + } + return result; + } + + private static String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinChunkPayloadSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinChunkPayloadSupport.java new file mode 100644 index 00000000..63f8d5c7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinChunkPayloadSupport.java @@ -0,0 +1,290 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import cn.hutool.crypto.digest.DigestUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DuplicateKeyException; + +import java.time.LocalDateTime; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 5b 轮）：chunk 载荷读取、读失败诊断与 orphan 行兜底。 + *

+ * 从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致： + *

    + *
  • {@link #readChunkRows}：解析 transient storage 指针指向的 chunk 行 JSON， + * 跨实例 / 类型不兼容场景按原语义留痕或跳过；
    • + *
  • {@link #recordChunkReadFailure} / {@link #collectChunkReadFailureSummary}： + * P1-4 的 task_scope_state.last_error 留痕与汇总（best-effort，不打断主流程）；
    • + *
  • {@link #persistOrphanLlmRows} / {@link #loadOrphanLlmRows}：跨 chunk merge 未匹配到 + * 任何 chunk 的回流行兜底落库与读回。
    • + *
+ */ +@Slf4j +public class SimilarAsinChunkPayloadSupport { + + /** 与 SimilarAsinTaskService.MODULE_TYPE 同值。 */ + private static final String MODULE_TYPE = "SIMILAR_ASIN"; + /** orphan-llm-rows: 跨 chunk merge 时未匹配到任何 chunk 的回流行兜底落库前缀。 */ + public static final String ORPHAN_SCOPE_KEY_PREFIX = "orphan-llm-rows:"; + /** orphan 行兜底状态，写入时直接置为 DONE，避免被 stale-recovery 扫描到。 */ + private static final String ORPHAN_LLM_ROW_ERROR = "orphan-llm-row(no submitted chunk)"; + private static final String LLM_STATUS_DONE = "DONE"; + + private final ObjectMapper objectMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + private final TaskScopeStateMapper taskScopeStateMapper; + + public SimilarAsinChunkPayloadSupport(ObjectMapper objectMapper, + TransientPayloadStorageService transientPayloadStorageService, + TaskScopeStateMapper taskScopeStateMapper) { + this.objectMapper = objectMapper; + this.transientPayloadStorageService = transientPayloadStorageService; + this.taskScopeStateMapper = taskScopeStateMapper; + } + + public Map readChunkRows(TaskChunkEntity chunk) { + Map rows = new LinkedHashMap<>(); + if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) { + return rows; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read similar ASIN chunk failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return rows; + } + for (JsonNode node : array) { + SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class); + rows.put(SimilarAsinChunkMergeSupport.rowKey(row), row); + } + } catch (Exception ex) { + String msg = ex.getMessage() == null ? "" : ex.getMessage(); + // P1-4：识别跨实例 local 指针读不到的场景，把"chunk 在另一实例"的元信息 + // 通过 task_scope_state.last_error 留痕，便于排查"为何 owner 切换后 chunk 读不到"。 + boolean crossInstance = msg.contains("only exists on instance="); + // 类型不兼容（如历史版本序列化数据含当前类加载器不认识的类型标记）： + // 该 chunk 永久不可读，跳过而不是把组装/收尾永久拖死，留痕便于排查。 + boolean typeMismatch = ex instanceof TypeNotPresentException + || msg.contains("not present"); + if (typeMismatch) { + log.warn("[similar-asin] chunk payload type mismatch, skip chunk taskId={} chunk={} err={}", + chunk.getTaskId(), chunk.getChunkIndex(), msg); + recordChunkReadFailure(chunk, false, msg); + return rows; + } + log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}", + chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg); + recordChunkReadFailure(chunk, crossInstance, msg); + throw new BusinessException("similar ASIN chunk payload read failed chunk=" + + chunk.getChunkIndex() + ": " + msg, ex); + } + return rows; + } + + /** + * P1-4：扫描该 task 下所有 task_scope_state.last_error， + * 把 "chunk-read-failed[N]" 标记收集成一行摘要返回；没命中返回空字符串。 + */ + public String collectChunkReadFailureSummary(Long taskId) { + if (taskId == null) { + return ""; + } + try { + List scopes = taskScopeStateMapper.selectList( + new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + if (scopes == null || scopes.isEmpty()) { + return ""; + } + LinkedHashSet tags = new LinkedHashSet<>(); + for (TaskScopeStateEntity scope : scopes) { + String lastError = scope == null ? null : scope.getLastError(); + if (lastError == null || lastError.isBlank()) { + continue; + } + int from = 0; + while (true) { + int start = lastError.indexOf("chunk-read-failed[", from); + if (start < 0) { + break; + } + int end = lastError.indexOf(']', start); + if (end < 0) { + break; + } + tags.add(lastError.substring(start, end + 1)); + from = end + 1; + } + } + return String.join(", ", tags); + } catch (Exception ex) { + log.warn("[similar-asin] collect chunk-read-failure summary failed taskId={} err={}", taskId, ex.getMessage()); + return ""; + } + } + + /** + * P1-4：把 chunk 读失败信息聚合到 task_scope_state.last_error， + * 形式 "chunk-read-failed[{idx}]" / "chunk-read-failed[{idx}@cross-instance]"。 + * 后续 assembleResultWorkbook 之前抛"结果为空"时可以附带 chunk index 列表，定位更具体。 + * best-effort：查询 / 更新失败时仅记日志，不抛回主流程。 + */ + public void recordChunkReadFailure(TaskChunkEntity chunk, boolean crossInstance, String msg) { + if (chunk == null || chunk.getTaskId() == null || chunk.getScopeHash() == null) { + return; + } + try { + TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, chunk.getTaskId()) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, chunk.getScopeHash()) + .last("limit 1")); + if (scope == null) { + return; + } + String existing = scope.getLastError() == null ? "" : scope.getLastError(); + String tag = "chunk-read-failed[" + chunk.getChunkIndex() + (crossInstance ? "@cross-instance" : "") + "]"; + if (existing.contains(tag)) { + return; + } + String updated = existing.isBlank() ? tag : existing + "; " + tag; + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, scope.getId()) + .set(TaskScopeStateEntity::getLastError, updated) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + } catch (Exception ignored) { + // best-effort：失败不影响主流程 + } + } + + /** + * 跨 chunk merge 时把没有匹配到任何 chunk 的回流行兜底落到 task_scope_state， + * 避免数据被静默丢弃；assembleResult 阶段由 {@link #loadOrphanLlmRows(Long)} 读回并合入最终 xlsx。 + * 使用 best-effort 策略：transient storage 出问题时仅记录 warn，不打断主流程。 + */ + public void persistOrphanLlmRows(Long taskId, List orphanRows) { + if (taskId == null || taskId <= 0 || orphanRows == null || orphanRows.isEmpty()) { + return; + } + try { + // 给每条 orphan 行写入 marker，后续导出时可识别。 + for (SimilarAsinResultRowDto row : orphanRows) { + if (row == null) { + continue; + } + String existingError = row.getError(); + String marker = ORPHAN_LLM_ROW_ERROR; + if (existingError == null || existingError.isBlank()) { + row.setError(marker); + } else if (!existingError.contains(marker)) { + row.setError(existingError + " | " + marker); + } + } + String orphanScopeKey = ORPHAN_SCOPE_KEY_PREFIX + UUID.randomUUID(); + String orphanScopeHash = DigestUtil.sha256Hex(orphanScopeKey); + String payloadJson = writeJson(orphanRows, "serialize orphan llm rows failed"); + String storedPayload = transientPayloadStorageService.storeParsedPayloadEntry( + MODULE_TYPE, taskId, orphanScopeHash, "orphan-" + UUID.randomUUID(), payloadJson, true); + LocalDateTime now = LocalDateTime.now(); + TaskScopeStateEntity state = new TaskScopeStateEntity(); + state.setTaskId(taskId); + state.setModuleType(MODULE_TYPE); + state.setScopeKey(orphanScopeKey); + state.setScopeHash(orphanScopeHash); + state.setParsedPayloadJson(storedPayload); + state.setStateJson("{\"phase\":\"ORPHAN_LLM_ROWS\"}"); + // llmStatus 直接置 DONE，避免被 listOwnedPendingLlmStates 扫描到。 + state.setLlmStatus(LLM_STATUS_DONE); + state.setLlmSubmittedAt(now); + state.setLlmCompletedAt(now); + state.setLlmAttemptCount(0); + state.setLlmError(ORPHAN_LLM_ROW_ERROR); + state.setChunkTotal(orphanRows.size()); + state.setReceivedChunkCount(orphanRows.size()); + state.setCompleted(1); + state.setLastChunkAt(now); + state.setCreatedAt(now); + state.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(state); + log.warn("[similar-asin] orphan llm rows persisted taskId={} rows={} scopeKey={}", + taskId, orphanRows.size(), orphanScopeKey); + } catch (DuplicateKeyException ex) { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + log.info("[similar-asin] duplicate orphan scope state ignored taskId={} scopeKey={}", + taskId, orphanScopeKey); + } + } catch (Exception ex) { + log.warn("[similar-asin] persist orphan llm rows failed taskId={} rows={} err={}", + taskId, orphanRows.size(), ex.getMessage()); + } + } + + /** + * 读回 {@link #persistOrphanLlmRows(Long, List)} 落库的 orphan 行集合。 + * 仅 assembleResult 阶段调用，与 chunk 行合并时使用 putIfAbsent 语义： + * 同一 rowKey 下 chunk 已有结果时优先用 chunk 行，避免覆盖正常路径数据。 + */ + public Map loadOrphanLlmRows(Long taskId) { + Map rows = new LinkedHashMap<>(); + if (taskId == null || taskId <= 0) { + return rows; + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .likeRight(TaskScopeStateEntity::getScopeKey, ORPHAN_SCOPE_KEY_PREFIX)); + if (states == null || states.isEmpty()) { + return rows; + } + for (TaskScopeStateEntity state : states) { + if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { + continue; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + state.getParsedPayloadJson(), "read similar ASIN orphan rows failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + continue; + } + for (JsonNode node : array) { + SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class); + String key = SimilarAsinChunkMergeSupport.rowKey(row); + if (key.isBlank()) { + continue; + } + rows.putIfAbsent(key, row); + } + } catch (Exception ex) { + log.warn("[similar-asin] read orphan llm rows failed taskId={} stateId={} err={}", + taskId, state.getId(), ex.getMessage()); + } + } + return rows; + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinLimits.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinLimits.java new file mode 100644 index 00000000..89a02849 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinLimits.java @@ -0,0 +1,114 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.nanri.aiimage.config.SimilarAsinProperties; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 1 轮）：阈值与开关解析器。 + *

+ * 全部方法从 {@code SimilarAsinTaskService} 原样搬移，回退默认值与 clamp 语义逐字段一致： + * resolvePreviewLimit / resolveMaxSourceFileBytes / resolveMaxParseRows / resolveMaxFieldLength / + * resolveMaxWorkbookZipEntries / resolveMaxWorkbookUncompressedBytes / getChunkMergeMaxRows / + * getChunkMergePayloadMaxBytes / resolveLlmBatchSize / isLlmResultBufferEnabled / llmFlushPendingMillis。 + * 纯函数无状态，只读 {@link SimilarAsinProperties}，不持有任何可变状态。 + */ +public final class SimilarAsinLimits { + + /** 解析接口响应预览默认上限：配置为 null/0/负值 时回退该值。 */ + private static final int PARSE_RESPONSE_PREVIEW_LIMIT = 100; + /** 解析接口响应预览硬上限：配置超过该值时 clamp，避免响应体无界增长。 */ + private static final int PARSE_RESPONSE_PREVIEW_LIMIT_MAX = 1000; + + private SimilarAsinLimits() { + } + + /** + * 解析接口响应预览上限：优先读取配置 parseResponsePreviewLimit。 + * 0/负值回退默认 100；超过 1000 clamp 到 1000，防止误配导致响应体膨胀。 + */ + public static int resolvePreviewLimit(SimilarAsinProperties properties) { + Integer configured = properties.getParseResponsePreviewLimit(); + if (configured == null || configured <= 0) { + return PARSE_RESPONSE_PREVIEW_LIMIT; + } + return Math.min(configured, PARSE_RESPONSE_PREVIEW_LIMIT_MAX); + } + + /** + * Task 7：单文件大小上限。0/负值回退默认 50MB，防止误配导致解析无界增长。 + */ + public static long resolveMaxSourceFileBytes(SimilarAsinProperties properties) { + Long configured = properties.getMaxSourceFileBytes(); + if (configured == null || configured <= 0) { + return 50L * 1024L * 1024L; + } + return configured; + } + + /** + * Task 7：单次解析最大有效行数。0/负值回退默认 50000，防止任务无界增长。 + */ + public static int resolveMaxParseRows(SimilarAsinProperties properties) { + Integer configured = properties.getMaxParseRows(); + if (configured == null || configured <= 0) { + return 50000; + } + return configured; + } + + /** + * Task 8：单字段最大长度（字符）。0/负值回退默认 2000。 + */ + public static int resolveMaxFieldLength(SimilarAsinProperties properties) { + Integer configured = properties.getMaxFieldLength(); + if (configured == null || configured <= 0) { + return 2000; + } + return configured; + } + + /** + * Task 8：xlsx(zip) 最大条目数。0/负值回退默认 20000。 + */ + public static int resolveMaxWorkbookZipEntries(SimilarAsinProperties properties) { + Integer configured = properties.getMaxWorkbookZipEntries(); + if (configured == null || configured <= 0) { + return 20000; + } + return configured; + } + + /** + * Task 8：xlsx(zip) 解压后总字节数上限。0/负值回退默认 512MB。 + */ + public static long resolveMaxWorkbookUncompressedBytes(SimilarAsinProperties properties) { + Long configured = properties.getMaxWorkbookUncompressedBytes(); + if (configured == null || configured <= 0) { + return 512L * 1024L * 1024L; + } + return configured; + } + + public static int getChunkMergeMaxRows(SimilarAsinProperties properties) { + int limit = properties.getChunkMergeMaxRows(); + return limit > 0 ? limit : 50000; + } + + public static long getChunkMergePayloadMaxBytes(SimilarAsinProperties properties) { + long limit = properties.getChunkMergePayloadMaxBytes(); + return limit > 0 ? limit : 16L * 1024L * 1024L; + } + + public static int resolveLlmBatchSize(SimilarAsinProperties properties, boolean imgSwitch) { + int configured = imgSwitch ? properties.getLlmBatchSize() : properties.getLlmTextOnlyBatchSize(); + return Math.max(1, configured); + } + + public static boolean isLlmResultBufferEnabled(SimilarAsinProperties properties) { + return properties.isLlmResultBufferEnabled(); + } + + public static long llmFlushPendingMillis(SimilarAsinProperties properties) { + int minutes = properties.getLlmFlushPendingMinutes(); + return Math.max(1, minutes) * 60_000L; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPayloadSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPayloadSupport.java new file mode 100644 index 00000000..daeef520 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPayloadSupport.java @@ -0,0 +1,308 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService; +import com.nanri.aiimage.modules.usersecret.support.UserSecretModule; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 2 轮）：解析载荷（parsed payload）编解码与读取。 + *

+ * 全部方法从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致： + *

    + *
  • 纯函数：resolveAllRows / containsGroupRefs / expandGroupRefs / fillGroupItems / + * hydrateForPythonCompat / rowCount / elapsedMs —— 新旧 payload 结构（items / allItems / + * groups 引用）的统一行恢复与 Python 旧链路兼容，无副作用；
    • + *
  • 读取门面：readParsedPayload / readAiPrompt / readApiKey / readImgSwitch / allRowCount， + * 以及落库侧 storeParsedPayload / buildTaskResultJson。
    • + *
+ * 只读 {@code transientPayloadStorageService} 的指针内容，不修改任何任务状态。 + */ +@Slf4j +public class SimilarAsinPayloadSupport { + + private final ObjectMapper objectMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + private final UserApiSecretService userApiSecretService; + private final InstanceMetadata instanceMetadata; + + public SimilarAsinPayloadSupport(ObjectMapper objectMapper, + TransientPayloadStorageService transientPayloadStorageService, + UserApiSecretService userApiSecretService, + InstanceMetadata instanceMetadata) { + this.objectMapper = objectMapper; + this.transientPayloadStorageService = transientPayloadStorageService; + this.userApiSecretService = userApiSecretService; + this.instanceMetadata = instanceMetadata; + } + + // ------------------------------------------------------------------ + // 纯函数：payload 行恢复与兼容 + // ------------------------------------------------------------------ + + public static int rowCount(SimilarAsinParsedPayloadDto payload) { + if (payload == null) { + return 0; + } + return resolveAllRows(payload).size(); + } + + /** + * 从解析载荷统一恢复全量行：优先 items（新规范结构），其次 allItems（旧结构）， + * 最后 groups 展开（最旧结构）。新旧结构均可恢复完整行集合，供 rowCount、 + * LLM 候选加载与结果文件组装复用。 + */ + public static List resolveAllRows(SimilarAsinParsedPayloadDto payload) { + if (payload == null) { + return List.of(); + } + if (containsGroupRefs(payload)) { + // 新格式：分组携带索引引用，信任引用展开结果（越界/非法区间安全跳过） + return expandGroupRefs(payload); + } + List rows = payload.getItems(); + if (rows == null || rows.isEmpty()) { + rows = payload.getAllItems(); + } + if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { + rows = payload.getGroups().stream() + .filter(Objects::nonNull) + .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) + .toList(); + } + return rows == null ? List.of() : rows; + } + + private static boolean containsGroupRefs(SimilarAsinParsedPayloadDto payload) { + if (payload.getGroups() == null) { + return false; + } + for (SimilarAsinParsedGroupVo group : payload.getGroups()) { + if (group != null && (group.getStartIndex() != null || group.getEndIndex() != null)) { + return true; + } + } + return false; + } + + /** + * Task 6：按分组引用 [startIndex, endIndex) 从 items 展开组内行。 + * 引用越界或区间非法时安全跳过，不抛异常；展开不修改 payload 内部状态。 + */ + public static List expandGroupRefs(SimilarAsinParsedPayloadDto payload) { + if (payload == null || payload.getGroups() == null || payload.getGroups().isEmpty()) { + return List.of(); + } + List rows = payload.getItems(); + if (rows == null || rows.isEmpty()) { + rows = payload.getAllItems(); + } + if (rows == null || rows.isEmpty()) { + return List.of(); + } + List expanded = new ArrayList<>(); + for (SimilarAsinParsedGroupVo group : payload.getGroups()) { + if (group == null) { + continue; + } + int start = group.getStartIndex() == null ? 0 : group.getStartIndex(); + int end = group.getEndIndex() == null ? 0 : group.getEndIndex(); + if (start < 0 || end <= start || end > rows.size()) { + continue; + } + expanded.addAll(rows.subList(start, end)); + } + return expanded; + } + + /** + * 按 group.startIndex/endIndex 半开区间从全量行切片，内嵌回 group.items； + * 无索引或切片越界时退回按 group.items 原样（兼容旧 payload 反序列化场景）。 + */ + public static List fillGroupItems(List groups, + List allRows) { + if (groups == null || groups.isEmpty()) { + return groups == null ? List.of() : groups; + } + if (allRows == null || allRows.isEmpty()) { + return groups; + } + List rows = new ArrayList<>(allRows); + for (SimilarAsinParsedGroupVo group : groups) { + if (group == null) { + continue; + } + Integer start = group.getStartIndex(); + Integer end = group.getEndIndex(); + if (start != null && end != null) { + int from = Math.max(0, Math.min(start, rows.size())); + int to = Math.max(from, Math.min(end, rows.size())); + if (to > from) { + group.setItems(new ArrayList<>(rows.subList(from, to))); + continue; + } + } + // 索引缺失/无效：保留已有 items（兼容旧 payload），不覆盖 + if (group.getItems() == null) { + group.setItems(new ArrayList<>()); + } + } + return groups; + } + + /** 解析载荷 hydrate：回填 allItems/items/groups 内嵌 items（Python 旧链路兼容）。 */ + public static SimilarAsinParsedPayloadDto hydrateForPythonCompat(SimilarAsinParsedPayloadDto payload) { + if (payload == null) { + return new SimilarAsinParsedPayloadDto(); + } + List rows = resolveAllRows(payload); + if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) { + payload.setAllItems(rows); + } + if (payload.getItems() == null || payload.getItems().isEmpty()) { + payload.setItems(rows); + } + // Python 旧链路兼容：读取时也回填 groups 内嵌 items（存量任务 payload + // 只有引用），否则 Python normalize_groups 拿到空 items 任务空转。 + if (payload.getGroups() != null && !payload.getGroups().isEmpty()) { + List filledRows = payload.getItems() == null || payload.getItems().isEmpty() + ? rows : payload.getItems(); + payload.setGroups(fillGroupItems(payload.getGroups(), filledRows)); + } + return payload; + } + + public static long elapsedMs(long start, long end) { + return (end - start) / 1_000_000L; + } + + // ------------------------------------------------------------------ + // 读取门面 + // ------------------------------------------------------------------ + + public int allRowCount(FileTaskEntity task) { + try { + return rowCount(readParsedPayload(task)); + } catch (Exception ex) { + log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage()); + return 0; + } + } + + public String readAiPrompt(FileTaskEntity task) { + try { + return readParsedPayload(task).getAiPrompt(); + } catch (Exception ignored) { + return ""; + } + } + + /** 密钥读取：任务 payload 优先（兼容老任务与显式覆盖），为空时按任务归属用户从服务端密钥表兜底。 */ + public String readApiKey(FileTaskEntity task) { + try { + String fromPayload = SimilarAsinRowNormalizer.normalize(readParsedPayload(task).getApiKey()); + if (!fromPayload.isEmpty()) { + return fromPayload; + } + } catch (Exception ex) { + log.warn("[similar-asin] 读取任务 payload 密钥失败，尝试用户密钥兜底 taskId={} err={}", + task.getId(), ex.getMessage()); + } + String fromUserSecret = userApiSecretService.findPlainValue( + task.getUserId(), UserSecretModule.SIMILAR_ASIN.key()); + if (fromUserSecret.isEmpty()) { + log.warn("[similar-asin] 任务未携带密钥且用户未配置密钥，LLM 检测将保留原始行 taskId={} userId={}", + task.getId(), task.getUserId()); + } + return fromUserSecret; + } + + public boolean readImgSwitch(FileTaskEntity task) { + try { + return Boolean.TRUE.equals(readParsedPayload(task).getImgSwitch()); + } catch (Exception ignored) { + return false; + } + } + + public SimilarAsinParsedPayloadDto readParsedPayload(FileTaskEntity task) { + try { + JsonNode root = objectMapper.readTree(task.getResultJson()); + String pointer = root.path("parsedPayloadRef").asText(""); + if (!pointer.isBlank()) { + String json = transientPayloadStorageService.resolvePayload(pointer, "read similar ASIN parsed payload failed"); + return hydrateForPythonCompat(objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class)); + } + if (root.path("allItems").isArray()) { + return hydrateForPythonCompat(objectMapper.treeToValue(root, SimilarAsinParsedPayloadDto.class)); + } + } catch (Exception ex) { + throw new BusinessException("读取解析载荷失败"); + } + return hydrateForPythonCompat(new SimilarAsinParsedPayloadDto()); + } + + public String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { + return transientPayloadStorageService.storeParsedPayloadFast( + "SIMILAR_ASIN", taskId, scopeHash, parsedPayloadJson, false); + } + + public String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, + List sourceFiles, String parsedPayloadPointer) { + Map payload = new LinkedHashMap<>(); + payload.put("aiPrompt", SimilarAsinRowNormalizer.normalize(aiPrompt)); + payload.put("apiKey", SimilarAsinRowNormalizer.normalize(apiKey)); + payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch)); + payload.put("categorySwitch", Boolean.TRUE.equals(categorySwitch)); + payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream() + .map(SimilarAsinSourceFileDto::getFileKey) + .filter(Objects::nonNull) + .toList()); + payload.put("parsedPayloadRef", parsedPayloadPointer); + payload.put("ownerInstanceId", currentInstanceId()); + return writeJson(payload, "保存任务结果索引失败"); + } + + private String currentInstanceId() { + String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId(); + return instanceId == null || instanceId.isBlank() ? "unknown-instance" : instanceId.trim(); + } + + /** 读取 scope state 上的 LLM 批次上下文（stateJson → LlmBatchContext）；解析失败返回 null。 */ + public static LlmBatchContext readLlmBatchContext(ObjectMapper objectMapper, TaskScopeStateEntity state) { + if (state == null || state.getStateJson() == null || state.getStateJson().isBlank()) { + return null; + } + try { + return objectMapper.readValue(state.getStateJson(), LlmBatchContext.class); + } catch (Exception ex) { + log.warn("[similar-asin] read llm batch context failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + return null; + } + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPoisonTracker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPoisonTracker.java new file mode 100644 index 00000000..aaa45d67 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPoisonTracker.java @@ -0,0 +1,118 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceArray; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 3 轮）：LLM 毒行滑窗熔断状态。 + *

+ * 从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致： + * per-task 最近 {@link #POISON_WINDOW_SIZE} 次提交结果环形滑窗、命中率阈值判定 + * （{@link #POISON_MIN_SAMPLES} / {@link #POISON_HIT_RATIO_PERCENT}）、 + * 以及任务终结/删除时的 per-task 清理。同时接管原 {@code LlmFailureClassifier} 的毒行识别。 + *

+ * 进程内可变状态：每实例一份，不落库、不跨实例共享；由 Service 单例持有。 + */ +public class SimilarAsinPoisonTracker { + + /** + * P1-1：per-task 最近 10 次提交结果（true=720712008 命中）。 + * 容量 10，环形覆盖；与 recentPoisonCursorByTask 配合实现 O(1) record。 + * 当近 5+ 次提交中命中率 ≥ 40% 触发 batch 强制降到 1（隔离毒行）。 + * finalizeTask / deleteTask 时清理，避免内存泄漏。 + */ + private static final int POISON_WINDOW_SIZE = 10; + /** + * 触发降档所需最小样本数：5 → 3。 + * 5000 行任务里 batchSize=10 时 1 次 720712008 会让 1 整批 row markFailed， + * 等不到 5 个样本就已经损失大量行。降到 3 让滑窗更早响应。 + */ + private static final int POISON_MIN_SAMPLES = 3; + /** + * 命中率触发阈值：40% → 33%。 + * 配合 POISON_MIN_SAMPLES=3，最快 1/3 命中率即触发，避免滑窗误判可由 batchSize=1 + * 后的连续成功样本快速恢复（1 次成功就把命中率打到 0/3）。 + */ + private static final int POISON_HIT_RATIO_PERCENT = 33; + + private final ConcurrentHashMap> recentPoisonByTask = new ConcurrentHashMap<>(); + private final ConcurrentHashMap recentPoisonCursorByTask = new ConcurrentHashMap<>(); + + /** + * P1-1：记录一次 LLM 提交结果是否触发了 720712008 类毒行。 + * 滑窗大小 POISON_WINDOW_SIZE=10，环形覆盖：满了之后从 0 重新开始。 + */ + public void recordSubmitOutcome(Long taskId, boolean poisonHit) { + if (taskId == null) { + return; + } + AtomicReferenceArray ring = recentPoisonByTask.computeIfAbsent(taskId, + k -> new AtomicReferenceArray<>(POISON_WINDOW_SIZE)); + AtomicInteger cursor = recentPoisonCursorByTask.computeIfAbsent(taskId, k -> new AtomicInteger()); + int idx = Math.floorMod(cursor.getAndIncrement(), POISON_WINDOW_SIZE); + ring.set(idx, poisonHit); + } + + /** + * P1-1：判断当前 task 是否处于 720712008 风暴中。 + * 阈值：近 POISON_MIN_SAMPLES=3+ 次提交、命中率 ≥ POISON_HIT_RATIO_PERCENT=33% → 触发降档。 + */ + public boolean isStormActive(Long taskId) { + if (taskId == null) { + return false; + } + AtomicReferenceArray ring = recentPoisonByTask.get(taskId); + if (ring == null) { + return false; + } + int hits = 0; + int total = 0; + for (int i = 0; i < POISON_WINDOW_SIZE; i++) { + Boolean v = ring.get(i); + if (v != null) { + total++; + if (Boolean.TRUE.equals(v)) { + hits++; + } + } + } + return total >= POISON_MIN_SAMPLES && hits * 100 / total >= POISON_HIT_RATIO_PERCENT; + } + + /** + * P1-1：任务终结 / 删除时调用，清理 per-task 滑窗记录避免内存泄漏。 + */ + public void clear(Long taskId) { + if (taskId == null) { + return; + } + recentPoisonByTask.remove(taskId); + recentPoisonCursorByTask.remove(taskId); + } + + /** + * P1-5：720712008 / "node executed out of limit" / "工作流节点执行超限" + * 这一类强烈暗示“单点死循环”，命中即隔离降档（batchSize=1）， + * 避免毒行把整批拖垮。 + */ + public static boolean isPoisonRow(String msg) { + String n = norm(msg); + return n.contains("720712008") + || n.contains("node executed out of limit") + || n.contains("execution limit") + || n.contains("工作流节点执行超限"); + } + + private static String norm(String msg) { + if (msg == null) { + return ""; + } + String trimmed = msg.trim(); + if (trimmed.isEmpty()) { + return ""; + } + return trimmed.toLowerCase(Locale.ROOT); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultTextSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultTextSupport.java new file mode 100644 index 00000000..c4ce13bd --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultTextSupport.java @@ -0,0 +1,249 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 5a 轮）：结果行判定与用户可见文本渲染。 + *

+ * 从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致： + *

    + *
  • LLM 输出可用性判定：hasUsableLlmField / hasResolvedLlmFields / isTechnicalLlmFailure / + * isSilentOutputFailure（技术性失败与"静默不输出"两类语义）；
    • + *
  • 结果状态判定：resolveResultStatus（成功 / 失败 / ASIN未找到）、isAsinNotFound、isFailedStatus；
    • + *
  • 导出过滤与文本渲染：isExportableResultRow、userFacingLlmCellValue、 + * hasText / isSpreadsheetErrorValue / isBlankOrSpreadsheetError；
    • + *
  • 标识与文件名工具：baseId、normalizeDisplayId、safeFileStem。
    • + *
+ * 纯函数无状态，字段归一化统一走 {@link SimilarAsinRowNormalizer#normalize}。 + */ +public final class SimilarAsinResultTextSupport { + + /** + * 仅命中确定性输入失败： + * - "fields cannot be extracted from null values"（工作流空输入兜底） + * - 错误码 720712000（显式输入校验失败） + * 注意：曾经裸串匹配 "null values" 会把 "got null values during merge" 等 + * 非确定性场景误判成 deterministic，导致行被直接 markFailed 而不进 retry。 + */ + private static final Pattern DETERMINISTIC_INPUT_FAILURE_PATTERN = Pattern.compile( + "fields\\s+cannot\\s+be\\s+extracted\\s+from\\s+null\\s+values" + + "|\\b720712000\\b"); + + private SimilarAsinResultTextSupport() { + } + + public static boolean hasText(String value) { + return value != null && !value.trim().isBlank(); + } + + public static boolean isFailedStatus(String status) { + String normalized = status == null ? "" : status.trim().toLowerCase(Locale.ROOT); + return normalized.contains("fail") + || normalized.contains("error") + || normalized.contains("失败") + || normalized.contains("错误"); + } + + public static boolean isSpreadsheetErrorValue(String value) { + String normalized = value == null ? "" : value.trim(); + return normalized.startsWith("#") && normalized.endsWith("!"); + } + + public static boolean isBlankOrSpreadsheetError(String value) { + return value == null || value.trim().isBlank() || isSpreadsheetErrorValue(value); + } + + public static boolean isTechnicalLlmFailure(String value) { + String normalized = SimilarAsinRowNormalizer.normalize(value).toLowerCase(Locale.ROOT); + if (normalized.isBlank()) { + return false; + } + // 确定性输入失败统一通过 DETERMINISTIC_INPUT_FAILURE_PATTERN 判定， + // 其它技术性失败（超时、节点超限等）保持原有子串匹配。 + if (DETERMINISTIC_INPUT_FAILURE_PATTERN.matcher(normalized).find()) { + return true; + } + return normalized.contains("llm") + || normalized.contains("结果不完整") + || normalized.contains("工作流节点执行超限") + || normalized.contains("调用超时") + || normalized.contains("timeout"); + } + + public static boolean isSilentOutputFailure(String value) { + String normalized = SimilarAsinRowNormalizer.normalize(value).toLowerCase(Locale.ROOT); + return normalized.contains("required field missing") + && normalized.contains("skip llm"); + } + + public static boolean isSilentOutputFailure(SimilarAsinResultRowDto row) { + return row != null + && (isSilentOutputFailure(row.getError()) + || isSilentOutputFailure(row.getReason()) + || isSilentOutputFailure(row.getConclusion())); + } + + public static boolean hasUsableLlmField(String value) { + String normalized = SimilarAsinRowNormalizer.normalize(value); + return !normalized.isBlank() && !isTechnicalLlmFailure(normalized); + } + + public static boolean hasResolvedLlmFields(SimilarAsinResultRowDto row) { + if (row == null) { + return false; + } + return hasUsableLlmField(row.getTitleRisk()) + || hasUsableLlmField(row.getAppearanceRisk()) + || hasUsableLlmField(row.getPatentRisk()) + || hasUsableLlmField(row.getConclusion()) + || hasUsableLlmField(row.getIsStock()) + || hasUsableLlmField(row.getSimilarity()) + || hasUsableLlmField(row.getIsConform()) + || hasUsableLlmField(row.getReason()) + || hasUsableLlmField(row.getCategory()) + || hasUsableLlmField(row.getStatus()) + || hasText(row.getMainUrl()) + || hasText(row.getPuzzleImg1()) + || hasText(row.getPuzzleImg2()); + } + + /** + * 判定是否"ASIN 未找到"：Python 端抓取 404（error="404 Not Found"）或等价文案。 + * 命中该判定的行不会被当成生成失败计重试，也不会被导出过滤，状态列显示"ASIN未找到"。 + */ + public static boolean isAsinNotFound(String error) { + String normalized = error == null ? "" : error.toLowerCase(Locale.ROOT).trim(); + if (normalized.isBlank()) { + return false; + } + return normalized.contains("404") + || normalized.contains("not found") + || normalized.contains("asin 未找到") + || normalized.contains("asin不存在") + || normalized.contains("商品未找到"); + } + + public static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) { + if (row == null) { + return "失败"; + } + if (hasText(row.getMainUrl()) || hasText(row.getPuzzleImg1()) || hasText(row.getPuzzleImg2())) { + return "成功"; + } + // Python 端搜不到商品（404）的行：状态列明确标记，人工复核时一眼可辨。 + // 放在图片/状态判断之前：404 行必然无图且 status 已被 markRowFailed 置为 FAILED， + // 不能落回"失败"（那是 LLM/生成链路失败语义，与"ASIN 搜不到"是两回事）。 + if (isAsinNotFound(row.getError())) { + return "ASIN未找到"; + } + String rawStatus = row.getStatus() == null ? "" : row.getStatus().trim(); + if (!rawStatus.isBlank()) { + return isFailedStatus(rawStatus) ? "失败" : "成功"; + } + if (visibleResultValues != null) { + for (String value : visibleResultValues) { + if (hasText(value)) { + return "成功"; + } + } + } + return hasText(row.getConclusion()) ? "成功" : "失败"; + } + + public static boolean isExportableResultRow(SimilarAsinResultRowDto row) { + if (row == null) { + return false; + } + if (hasResolvedLlmFields(row)) { + return true; + } + String error = SimilarAsinRowNormalizer.normalize(row.getError()); + if (error.isBlank()) { + return false; + } + // Python 端搜不到商品的 404 行：虽然 title/url/图全空，也必须导出， + // 并在状态列标记"ASIN未找到"，供人工复核。在"其它错误需带 title/url/图"之前判断。 + if (isAsinNotFound(error)) { + return true; + } + return !SimilarAsinRowNormalizer.normalize(row.getTitle()).isBlank() + || !SimilarAsinRowNormalizer.normalize(row.getUrl()).isBlank() + || row.hasImageUrl(); + } + + /** 判定该行是否值得送 LLM 检测（有稳定行键、或有标题、或有图）。 */ + public static boolean shouldInspectRow(SimilarAsinResultRowDto row) { + return row != null && (!SimilarAsinChunkMergeSupport.rowKey(row).isBlank() + || !SimilarAsinRowNormalizer.normalize(row.getTitle()).isBlank() + || row.hasImageUrl()); + } + + /** 从一组结果行里挑出尚未得到 LLM 结论、且值得检测的行。 */ + public static List collectPendingLlmRows(Collection rows) { + if (rows == null) { + return List.of(); + } + List pendingRows = new ArrayList<>(); + for (SimilarAsinResultRowDto row : rows) { + if (row != null && !hasResolvedLlmFields(row) && shouldInspectRow(row)) { + pendingRows.add(row); + } + } + return pendingRows; + } + + public static String userFacingLlmCellValue(SimilarAsinResultRowDto row, String value) { + String normalizedValue = SimilarAsinRowNormalizer.normalize(value); + if (isSilentOutputFailure(row) || isSilentOutputFailure(normalizedValue)) { + return ""; + } + if (!normalizedValue.isBlank() && !isTechnicalLlmFailure(normalizedValue)) { + return value; + } + // LLM 技术性失败：有错误信息则放入错误信息，没有则留空 + if (row != null && isTechnicalLlmFailure(row.getError())) { + return firstNonBlank(row.getError(), ""); + } + return firstNonBlank(value, ""); + } + + public static String baseId(String id) { + String s = SimilarAsinRowNormalizer.normalize(id); + int idx = s.indexOf('_'); + return idx > 0 ? s.substring(0, idx) : s; + } + + public static String normalizeDisplayId(String id) { + return id == null ? "" : id.trim(); + } + + public static String safeFileStem(String filename) { + String name = filename == null || filename.isBlank() ? "similar-asin" : filename; + int idx = name.lastIndexOf('.'); + if (idx > 0) { + name = name.substring(0, idx); + } + String safe = name.replaceAll("[\\\\/:*?\"<>|]", "_").trim(); + return safe.isBlank() ? "similar-asin" : safe; + } + + public static void addNonBlank(List sink, String value) { + if (value == null) { + return; + } + String trimmed = value.trim(); + if (!trimmed.isEmpty()) { + sink.add(trimmed); + } + } + + private static String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultWorkbookAssembler.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultWorkbookAssembler.java new file mode 100644 index 00000000..c5e94afb --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultWorkbookAssembler.java @@ -0,0 +1,995 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.GroupResultPropagator; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinImagePrefetchService; +import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import com.nanri.aiimage.modules.similarasin.util.WpsCellImageWriter; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 5c 轮）：结果文件（xlsx/zip）装配。 + *

+ * 从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致：多源文件并发装配 + * （bounded 路径与 legacy 路径）、POI 写盘与 WPS DISPIMG 单元格图片、结果行读取/回填、 + * ZIP 打包与临时文件清理。 + *

+ * 线程池由调用方（Service）持有并注入：Service 的 {@code shutdownAssembleExecutor()} 语义不变。 + */ +@Slf4j +public class SimilarAsinResultWorkbookAssembler { + + private static final String MODULE_TYPE = "SIMILAR_ASIN"; + private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final String CONTENT_TYPE_ZIP = "application/zip"; + private static final List RESULT_HEADERS = List.of( + "id", + "asin", + "国家", + "价格", + "卖家名称", + "品牌", + "是否有货", + "相似度", + "是否符合类目", + "不符合理由", + "产品类目", + "状态", + "主图", + "阿里巴巴图片1", + "阿里巴巴图片2" + ); + private static final int IMG_COL_MAIN = 12; + private static final int IMG_COL_PUZZLE1 = 13; + private static final int IMG_COL_PUZZLE2 = 14; + private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; + private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; + + private final SimilarAsinProperties properties; + private final StorageProperties storageProperties; + private final OssStorageService ossStorageService; + private final ObjectMapper objectMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + private final SimilarAsinImageEmbedder imageEmbedder; + private final SimilarAsinImagePrefetchService imagePrefetchService; + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final ExecutorService assembleExecutor; + private final SimilarAsinPayloadSupport payloadSupport; + private final SimilarAsinChunkPayloadSupport chunkPayloadSupport; + + public SimilarAsinResultWorkbookAssembler(SimilarAsinProperties properties, + StorageProperties storageProperties, + OssStorageService ossStorageService, + ObjectMapper objectMapper, + TransientPayloadStorageService transientPayloadStorageService, + SimilarAsinImageEmbedder imageEmbedder, + SimilarAsinImagePrefetchService imagePrefetchService, + TaskChunkMapper taskChunkMapper, + TaskScopeStateMapper taskScopeStateMapper, + ExecutorService assembleExecutor, + SimilarAsinPayloadSupport payloadSupport, + SimilarAsinChunkPayloadSupport chunkPayloadSupport) { + this.properties = properties; + this.storageProperties = storageProperties; + this.ossStorageService = ossStorageService; + this.objectMapper = objectMapper; + this.transientPayloadStorageService = transientPayloadStorageService; + this.imageEmbedder = imageEmbedder; + this.imagePrefetchService = imagePrefetchService; + this.taskChunkMapper = taskChunkMapper; + this.taskScopeStateMapper = taskScopeStateMapper; + this.assembleExecutor = assembleExecutor; + this.payloadSupport = payloadSupport; + this.chunkPayloadSupport = chunkPayloadSupport; + } + + private void assembleResultWorkbookBounded(FileTaskEntity task, FileResultEntity result) { + SimilarAsinParsedPayloadDto parsed = payloadSupport.readParsedPayload(task); + List parsedRows = SimilarAsinPayloadSupport.resolveAllRows(parsed); + List sourceRows = splitRowsBySourceFile( + parsed, parsedRows, result.getSourceFilename()); + if (sourceRows.isEmpty()) { + throw new BusinessException("Similar ASIN result rows are empty"); + } + + File outputDir = new File(storageProperties.getLocalTempDir(), "similar-asin-result"); + if (!outputDir.exists() && !outputDir.mkdirs()) { + throw new BusinessException("failed to create result directory"); + } + List workbooks = new ArrayList<>(); + List workbookFiles = new ArrayList<>(sourceRows.size()); + File zip = null; + long imageCacheMaxBytes = properties.getImageCacheMaxBytes() > 0 + ? properties.getImageCacheMaxBytes() + : BoundedImageCache.DEFAULT_MAX_BYTES; + BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes); + try { + int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes()); + long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes); + long assembleStart = System.currentTimeMillis(); + List> futures = new ArrayList<>(sourceRows.size()); + for (SourceRows item : sourceRows) { + final SourceRows captured = item; + final String filename = SimilarAsinResultTextSupport.safeFileStem(captured.sourceFilename()) + "-result.xlsx"; + final String tempFilename = SimilarAsinResultTextSupport.safeFileStem(captured.sourceFilename()) + + "-" + task.getId() + + "-" + result.getId() + + "-" + UUID.randomUUID() + + "-result.xlsx"; + final File xlsx = new File(outputDir, tempFilename); + workbookFiles.add(xlsx); + futures.add(assembleExecutor.submit(() -> { + ensureResultAssemblyNotInterrupted(); + Map sourceResultMap = + loadPersistedResultRowsForRowsWithRetry(task.getId(), captured.rows()); + int persistedRows = sourceResultMap.size(); + sourceResultMap.entrySet().removeIf(entry -> !SimilarAsinResultTextSupport.isExportableResultRow(entry.getValue())); + long resolvedRows = captured.rows().stream() + .filter(row -> SimilarAsinChunkMergeSupport.findResultRow(row, sourceResultMap) != null) + .count(); + int conformPropagated = GroupResultPropagator.propagateByGroup( + captured.rows(), + SimilarAsinParsedRowVo::getDisplayId, + row -> SimilarAsinChunkMergeSupport.findResultRow(row, sourceResultMap), + SimilarAsinResultRowDto::getIsConform, + SimilarAsinResultRowDto::setIsConform, + List.of("不符合"), + "[similar-asin] taskId=" + task.getId() + ); + writeResultWorkbook(xlsx, captured.rows(), sourceResultMap, taskImageCache); + ensureResultAssemblyNotInterrupted(); + return new SourceResultWorkbook( + xlsx, filename, captured.rows().size(), persistedRows, + sourceResultMap.size(), resolvedRows, conformPropagated); + })); + } + try { + for (Future future : futures) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0L) { + throw new TimeoutException("result file deadline reached"); + } + workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS)); + } + } catch (InterruptedException ex) { + cancelResultAssemblyFutures(futures); + Thread.currentThread().interrupt(); + throw new BusinessException("result workbook assembly interrupted", ex); + } catch (TimeoutException ex) { + cancelResultAssemblyFutures(futures); + throw new BusinessException("result workbook assembly timed out", ex); + } catch (CancellationException ex) { + cancelResultAssemblyFutures(futures); + throw new BusinessException("result workbook assembly cancelled", ex); + } catch (ExecutionException ex) { + cancelResultAssemblyFutures(futures); + Throwable cause = ex.getCause(); + if (cause instanceof BusinessException businessException) { + throw businessException; + } + throw new BusinessException("result workbook assembly failed", cause == null ? ex : cause); + } + + int persistedResultRows = workbooks.stream() + .mapToInt(SourceResultWorkbook::persistedRows) + .sum(); + int resultRows = workbooks.stream() + .mapToInt(SourceResultWorkbook::resultRows) + .sum(); + long resolvedRows = workbooks.stream() + .mapToLong(SourceResultWorkbook::resolvedRows) + .sum(); + int conformPropagated = workbooks.stream() + .mapToInt(SourceResultWorkbook::conformPropagated) + .sum(); + log.info("[similar-asin] bounded assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={} propagated={} sources={} costMs={}", + task.getId(), parsedRows.size(), persistedResultRows, resultRows, + resolvedRows, conformPropagated, sourceRows.size(), + System.currentTimeMillis() - assembleStart); + if (resultRows == 0) { + String failureSummary = chunkPayloadSupport.collectChunkReadFailureSummary(task.getId()); + throw new BusinessException(failureSummary.isBlank() + ? "Similar ASIN result rows are empty; please retry result generation" + : "Similar ASIN result rows are empty (" + failureSummary + ")"); + } + + taskImageCache.clear(); + File uploadFile; + String filename; + String contentType; + if (workbooks.size() == 1) { + SourceResultWorkbook workbook = workbooks.get(0); + uploadFile = workbook.file(); + filename = workbook.filename(); + contentType = CONTENT_TYPE_XLSX; + } else { + filename = SimilarAsinResultTextSupport.safeFileStem(result.getSourceFilename()) + "-result.zip"; + String tempFilename = SimilarAsinResultTextSupport.safeFileStem(result.getSourceFilename()) + + "-" + task.getId() + + "-" + result.getId() + + "-" + UUID.randomUUID() + + "-result.zip"; + zip = new File(outputDir, tempFilename); + packageResultWorkbooksAsZip(zip, workbooks); + uploadFile = zip; + contentType = CONTENT_TYPE_ZIP; + } + String objectKey = ossStorageService.uploadResultFile(uploadFile, MODULE_TYPE); + result.setResultFilename(filename); + result.setResultFileUrl(objectKey); + result.setResultFileSize(uploadFile.length()); + result.setResultContentType(contentType); + result.setRowCount(parsedRows.size()); + } finally { + taskImageCache.clear(); + for (File workbookFile : workbookFiles) { + if (workbookFile.exists() && !workbookFile.delete()) { + log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile); + } + } + if (zip != null && zip.exists() && !zip.delete()) { + log.warn("[similar-asin] delete temp zip failed file={}", zip); + } + } + } + + public void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) { + if (properties.isBoundedResultAssemblyEnabled()) { + assembleResultWorkbookBounded(task, result); + return; + } + SimilarAsinParsedPayloadDto parsed = payloadSupport.readParsedPayload(task); + List parsedRows = SimilarAsinPayloadSupport.resolveAllRows(parsed); + Map resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsedRows.size()); + int persistedResultRows = resultMap.size(); + resultMap.entrySet().removeIf(entry -> !SimilarAsinResultTextSupport.isExportableResultRow(entry.getValue())); + long resolvedRows = parsedRows.stream() + .filter(row -> SimilarAsinChunkMergeSupport.findResultRow(row, resultMap) != null) + .count(); + log.info("[similar-asin] assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={}", + task.getId(), parsedRows.size(), persistedResultRows, resultMap.size(), resolvedRows); + if (!parsedRows.isEmpty() && resultMap.isEmpty()) { + // P1-4：检查是否有 chunk-read-failed 标记，把 chunk index 列表附在错误信息里。 + String chunkReadFailureSummary = chunkPayloadSupport.collectChunkReadFailureSummary(task.getId()); + if (!chunkReadFailureSummary.isBlank()) { + throw new BusinessException("相似ASIN检测结果为空（" + chunkReadFailureSummary + "），请稍后重试生成结果文件"); + } + throw new BusinessException("相似ASIN检测结果为空，请稍后重试生成结果文件"); + } + // 公共分组处理：同 baseId 连续行视为一组（如 1、1_1、1_2），组内任一行的"是否符合类目" + // 命中"不符合"（包含匹配），则组内所有行的"是否符合类目"统一为标准值"不符合"。 + // 单行组跳过。仅修改 isConform 列，不影响其他列。 + int conformPropagated = GroupResultPropagator.propagateByGroup( + parsedRows, + SimilarAsinParsedRowVo::getDisplayId, + row -> SimilarAsinChunkMergeSupport.findResultRow(row, resultMap), + SimilarAsinResultRowDto::getIsConform, + SimilarAsinResultRowDto::setIsConform, + List.of("不符合"), + "[similar-asin] taskId=" + task.getId() + ); + if (conformPropagated > 0) { + log.info("[similar-asin] group propagate isConform taskId={} updatedRows={}", + task.getId(), conformPropagated); + } + File outputDir = new File(storageProperties.getLocalTempDir(), "similar-asin-result"); + if (!outputDir.exists() && !outputDir.mkdirs()) { + throw new BusinessException("创建结果目录失败"); + } + List sourceRows = splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename()); + List workbooks = new ArrayList<>(); + List workbookFiles = new ArrayList<>(sourceRows.size()); + File zip = null; + // 跨 workbook 共享 taskImageCache：多源场景下相同 URL 仅下载一次。 + // 配合 embed() 写完即 remove()，cache 仅承载 in-flight 图片； + // 5000 行 × 200 实例规模下用 BoundedImageCache 按字节做 LRU 淘汰，硬上限 256MB + // （配置 aiimage.similar-asin.image-cache-max-bytes 可调），避免堆爆。 + long imageCacheMaxBytes = properties.getImageCacheMaxBytes() > 0 + ? properties.getImageCacheMaxBytes() + : BoundedImageCache.DEFAULT_MAX_BYTES; + BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes); + try { + // 单源和多源统一提交真实 Future，确保超时取消能中断实际 workbook 线程。 + // 所有源文件共享同一个绝对 deadline，避免逐个 Future 各等待一轮完整超时。 + int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes()); + long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes); + long assembleStart = System.currentTimeMillis(); + List> futures = new ArrayList<>(sourceRows.size()); + for (SourceRows item : sourceRows) { + final SourceRows captured = item; + final String filename = SimilarAsinResultTextSupport.safeFileStem(captured.sourceFilename()) + "-result.xlsx"; + final String tempFilename = SimilarAsinResultTextSupport.safeFileStem(captured.sourceFilename()) + + "-" + task.getId() + + "-" + result.getId() + + "-" + UUID.randomUUID() + + "-result.xlsx"; + final File xlsx = new File(outputDir, tempFilename); + workbookFiles.add(xlsx); + futures.add(assembleExecutor.submit(() -> { + ensureResultAssemblyNotInterrupted(); + writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache); + ensureResultAssemblyNotInterrupted(); + return new SourceResultWorkbook(xlsx, filename, captured.rows().size()); + })); + } + try { + for (Future future : futures) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0L) { + throw new TimeoutException("result file deadline reached"); + } + workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS)); + } + } catch (InterruptedException ex) { + cancelResultAssemblyFutures(futures); + Thread.currentThread().interrupt(); + throw new BusinessException("生成相似ASIN检测结果中断", ex); + } catch (TimeoutException ex) { + cancelResultAssemblyFutures(futures); + throw new BusinessException("生成相似ASIN检测结果超时（" + timeoutMinutes + " 分钟）", ex); + } catch (CancellationException ex) { + cancelResultAssemblyFutures(futures); + throw new BusinessException("生成相似ASIN检测结果已取消", ex); + } catch (ExecutionException ex) { + cancelResultAssemblyFutures(futures); + Throwable cause = ex.getCause(); + if (cause instanceof BusinessException businessException) { + throw businessException; + } + throw new BusinessException("生成相似ASIN检测结果失败", cause != null ? cause : ex); + } + log.info("[similar-asin] assemble workbooks finished taskId={} sources={} timeoutMinutes={} costMs={}", + task.getId(), sourceRows.size(), timeoutMinutes, System.currentTimeMillis() - assembleStart); + log.info("[similar-asin] assemble image-cache stats taskId={} sources={} cacheSize={} currentBytes={} evictedCount={} evictedBytes={} maxBytes={}", + task.getId(), sourceRows.size(), taskImageCache.size(), + taskImageCache.currentBytes(), taskImageCache.evictedCount(), + taskImageCache.evictedBytes(), taskImageCache.maxBytes()); + // 主动清空，让 GC 尽早回收图片字节，避免 zip 阶段还占着堆。 + taskImageCache.clear(); + if (workbooks.isEmpty()) { + throw new BusinessException("相似ASIN检测结果为空，请稍后重试生成结果文件"); + } + + File uploadFile; + String filename; + String contentType; + if (workbooks.size() == 1) { + SourceResultWorkbook workbook = workbooks.get(0); + uploadFile = workbook.file(); + filename = workbook.filename(); + contentType = CONTENT_TYPE_XLSX; + } else { + filename = SimilarAsinResultTextSupport.safeFileStem(result.getSourceFilename()) + "-result.zip"; + String tempFilename = SimilarAsinResultTextSupport.safeFileStem(result.getSourceFilename()) + + "-" + task.getId() + + "-" + result.getId() + + "-" + UUID.randomUUID() + + "-result.zip"; + zip = new File(outputDir, tempFilename); + packageResultWorkbooksAsZip(zip, workbooks); + uploadFile = zip; + contentType = CONTENT_TYPE_ZIP; + } + + String objectKey = ossStorageService.uploadResultFile(uploadFile, MODULE_TYPE); + result.setResultFilename(filename); + result.setResultFileUrl(objectKey); + result.setResultFileSize(uploadFile.length()); + result.setResultContentType(contentType); + result.setRowCount(SimilarAsinPayloadSupport.resolveAllRows(parsed).size()); + } finally { + for (File workbookFile : workbookFiles) { + if (workbookFile.exists() && !workbookFile.delete()) { + log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile); + } + } + if (zip != null && zip.exists() && !zip.delete()) { + log.warn("[similar-asin] delete temp zip failed file={}", zip); + } + } + } + + private List splitRowsBySourceFile(SimilarAsinParsedPayloadDto parsed, + List rows, + String fallbackFilename) { + String defaultFilename = firstNonBlank(fallbackFilename, "similar-asin"); + Map builders = new LinkedHashMap<>(); + if (parsed != null && parsed.getSourceFiles() != null) { + for (SimilarAsinSourceFileDto sourceFile : parsed.getSourceFiles()) { + if (sourceFile == null) { + continue; + } + String sourceFileKey = SimilarAsinRowNormalizer.normalize(sourceFile.getFileKey()); + if (sourceFileKey.isBlank()) { + continue; + } + builders.putIfAbsent(sourceFileKey, + new SourceRowsBuilder(sourceFileKey, firstNonBlank(sourceFile.getOriginalFilename(), defaultFilename))); + } + } + + for (SimilarAsinParsedRowVo row : rows == null ? List.of() : rows) { + if (row == null) { + continue; + } + String sourceFileKey = SimilarAsinRowNormalizer.normalize(row.getSourceFileKey()); + if (sourceFileKey.isBlank()) { + sourceFileKey = SimilarAsinRowNormalizer.normalize(row.getSourceFilename()); + } + if (sourceFileKey.isBlank()) { + sourceFileKey = "__default__"; + } + SourceRowsBuilder builder = builders.computeIfAbsent(sourceFileKey, + key -> new SourceRowsBuilder(key, firstNonBlank(row.getSourceFilename(), defaultFilename))); + if (builder.sourceFilename().isBlank() || Objects.equals(builder.sourceFilename(), defaultFilename)) { + builder.setSourceFilename(firstNonBlank(row.getSourceFilename(), defaultFilename)); + } + builder.rows().add(row); + } + + return builders.values().stream() + .filter(builder -> !builder.rows().isEmpty()) + .map(builder -> new SourceRows(builder.sourceFileKey(), firstNonBlank(builder.sourceFilename(), defaultFilename), builder.rows())) + .toList(); + } + + private void packageResultWorkbooksAsZip(File zip, List workbooks) { + Set entryNames = new LinkedHashSet<>(); + try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zip))) { + byte[] buffer = new byte[8192]; + for (SourceResultWorkbook workbook : workbooks) { + writeResultZipEntry(zos, buffer, workbook.file(), uniqueZipEntryName(workbook.filename(), entryNames)); + } + } catch (IOException ex) { + throw new BusinessException("生成相似ASIN检测结果压缩包失败"); + } + } + + private void writeResultZipEntry(ZipOutputStream zos, + byte[] buffer, + File file, + String entryName) throws IOException { + zos.putNextEntry(new ZipEntry(entryName)); + try (FileInputStream inputStream = new FileInputStream(file)) { + int len; + while ((len = inputStream.read(buffer)) > 0) { + zos.write(buffer, 0, len); + } + } + zos.closeEntry(); + } + + private String uniqueZipEntryName(String filename, Set entryNames) { + String safeFilename = normalizeZipFilename(filename); + if (entryNames.add(safeFilename)) { + return safeFilename; + } + int dotIndex = safeFilename.lastIndexOf('.'); + String stem = dotIndex > 0 ? safeFilename.substring(0, dotIndex) : safeFilename; + String extension = dotIndex > 0 ? safeFilename.substring(dotIndex) : ""; + int index = 2; + while (true) { + String candidate = stem + "(" + index + ")" + extension; + if (entryNames.add(candidate)) { + return candidate; + } + index++; + } + } + + private String normalizeZipFilename(String filename) { + String normalized = firstNonBlank(filename, "similar-asin-result.xlsx").replace('\\', '/'); + while (normalized.startsWith("/")) { + normalized = normalized.substring(1); + } + return normalized.isBlank() ? "similar-asin-result.xlsx" : normalized; + } + + private Map loadPersistedResultRowsForRowsWithRetry( + Long taskId, + List requestedRows) { + Map result = + loadPersistedResultRowsForRows(taskId, requestedRows); + if (requestedRows == null || requestedRows.isEmpty() || !result.isEmpty()) { + return result; + } + for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) { + sleepBeforeResultRowsRetry(attempt); + result = loadPersistedResultRowsForRows(taskId, requestedRows); + if (!result.isEmpty()) { + log.info("[similar-asin] source result rows recovered after retry taskId={} attempt={} rows={}", + taskId, attempt, result.size()); + } + } + return result; + } + + /** + * Select and decode one chunk payload at a time. Only rows needed by the + * current source workbook are retained in the returned map. + */ + private Map loadPersistedResultRowsForRows( + Long taskId, + List requestedRows) { + Map result = new LinkedHashMap<>(); + Set requestedKeys = resultLookupKeys(requestedRows); + if (taskId == null || requestedKeys.isEmpty()) { + return result; + } + + Long lastChunkId = null; + while (true) { + LambdaQueryWrapper query = new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getId) + .last("LIMIT 1"); + if (lastChunkId != null) { + query.gt(TaskChunkEntity::getId, lastChunkId); + } + List page = taskChunkMapper.selectList(query); + if (page == null || page.isEmpty()) { + break; + } + TaskChunkEntity chunk = page.getFirst(); + appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(), requestedKeys, false); + lastChunkId = chunk.getId(); + if (lastChunkId == null) { + break; + } + } + appendRequestedOrphanRows(taskId, requestedKeys, result); + return result; + } + + private Set resultLookupKeys(List rows) { + Set keys = new LinkedHashSet<>(); + for (SimilarAsinParsedRowVo row : rows == null ? List.of() : rows) { + String primary = SimilarAsinChunkMergeSupport.rowKey(row); + if (!primary.isBlank()) { + keys.add(primary); + } + String legacy = SimilarAsinChunkMergeSupport.legacyRowKey(row); + if (!legacy.isBlank()) { + keys.add(legacy); + } + } + return keys; + } + + private void appendRequestedResultRows(Map sink, + Iterable rows, + Set requestedKeys, + boolean onlyIfAbsent) { + if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) { + return; + } + for (SimilarAsinResultRowDto row : rows) { + if (row == null) { + continue; + } + String primary = SimilarAsinChunkMergeSupport.rowKey(row); + String legacy = SimilarAsinChunkMergeSupport.legacyRowKey(row); + String matchedKey = requestedKeys.contains(primary) + ? primary + : requestedKeys.contains(legacy) ? legacy : ""; + if (matchedKey.isBlank()) { + continue; + } + if (onlyIfAbsent) { + sink.putIfAbsent(matchedKey, row); + } else { + sink.put(matchedKey, row); + } + } + } + + private void appendRequestedOrphanRows(Long taskId, + Set requestedKeys, + Map sink) { + Long lastStateId = null; + while (true) { + LambdaQueryWrapper query = new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getId, + TaskScopeStateEntity::getParsedPayloadJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .likeRight(TaskScopeStateEntity::getScopeKey, SimilarAsinChunkPayloadSupport.ORPHAN_SCOPE_KEY_PREFIX) + .orderByAsc(TaskScopeStateEntity::getId) + .last("LIMIT 1"); + if (lastStateId != null) { + query.gt(TaskScopeStateEntity::getId, lastStateId); + } + List page = taskScopeStateMapper.selectList(query); + if (page == null || page.isEmpty()) { + return; + } + TaskScopeStateEntity state = page.getFirst(); + lastStateId = state.getId(); + if (lastStateId == null) { + return; + } + if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { + continue; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + state.getParsedPayloadJson(), "read similar ASIN orphan rows failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + continue; + } + List matchingRows = new ArrayList<>(); + for (JsonNode node : array) { + matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); + } + appendRequestedResultRows(sink, matchingRows, requestedKeys, true); + } catch (Exception ex) { + log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}", + taskId, state.getId(), ex.getMessage()); + } + } + } + + private Map loadPersistedResultRows(Long taskId) { + Map result = new LinkedHashMap<>(); + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + for (TaskChunkEntity chunk : chunks) { + result.putAll(chunkPayloadSupport.readChunkRows(chunk)); + } + // 合并 orphan 行（跨 chunk merge 兜底），仅当 chunk 中没有同 rowKey 时才补上，避免覆盖正常结果。 + Map orphans = chunkPayloadSupport.loadOrphanLlmRows(taskId); + if (!orphans.isEmpty()) { + int added = 0; + for (Map.Entry entry : orphans.entrySet()) { + if (!result.containsKey(entry.getKey())) { + result.put(entry.getKey(), entry.getValue()); + added++; + } + } + if (added > 0) { + log.warn("[similar-asin] merged orphan llm rows into result taskId={} added={} totalOrphan={}", + taskId, added, orphans.size()); + } + } + return result; + } + + public boolean hasPersistedResultRows(Long taskId) { + if (taskId == null || taskId <= 0) { + return false; + } + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + return count != null && count > 0; + } + + private Map loadPersistedResultRowsWithRetry(Long taskId, int expectedParsedRows) { + Map result = loadPersistedResultRows(taskId); + if (expectedParsedRows <= 0 || !result.isEmpty()) { + return result; + } + for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) { + sleepBeforeResultRowsRetry(attempt); + result = loadPersistedResultRows(taskId); + if (!result.isEmpty()) { + log.info("[similar-asin] result rows recovered after retry taskId={} attempt={} rows={}", + taskId, attempt, result.size()); + return result; + } + log.warn("[similar-asin] result rows still empty after retry taskId={} attempt={}/{}", + taskId, attempt, RESULT_ROWS_READ_RETRY_LIMIT); + } + return result; + } + + private void sleepBeforeResultRowsRetry(int attempt) { + try { + Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + private void writeResultWorkbook(File xlsx, + List rowsToWrite, + Map resultMap, + Map taskImageCache) { + ensureResultAssemblyNotInterrupted(); + // 单元格内嵌图片：写完 workbook 后 patch WPS DISPIMG 结构（WPS/钉钉/飞书 打开即为单元格图片， + // 与 Excel 365 richData 不同，DISPIMG 是 WPS 原生协议；协议细节见 WpsCellImageWriter 类注释）。 + WpsCellImageWriter.Session cellImageSession = WpsCellImageWriter.createSession(); + SimilarAsinImageEmbedder.ImageSpool imageSpool; + try { + Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-"); + imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir); + } catch (IOException ex) { + throw new BusinessException("create Similar ASIN image spool failed", ex); + } + try { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { + Sheet sheet = workbook.createSheet("相似asin检测"); + CellStyle headerStyle = workbook.createCellStyle(); + Font font = workbook.createFont(); + font.setBold(true); + headerStyle.setFont(font); + + // taskImageCache 由 assembleResultWorkbook 跨 workbook 注入，避免 4 源文件 × 同一批 URL 重复下载。 + + sheet.setColumnWidth(IMG_COL_MAIN, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256); + sheet.setColumnWidth(IMG_COL_PUZZLE1, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256); + sheet.setColumnWidth(IMG_COL_PUZZLE2, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256); + + List resultHeaders = new ArrayList<>(RESULT_HEADERS); + Row header = sheet.createRow(0); + for (int i = 0; i < resultHeaders.size(); i++) { + Cell cell = header.createCell(i); + cell.setCellValue(resultHeaders.get(i)); + cell.setCellStyle(headerStyle); + } + + // 预扫图片 URL 并行下载到任务级临时目录；POI 仍按行串行注册 cell image。 + List prefetchUrls = collectImageUrlsForPrefetch(rowsToWrite, resultMap); + if (!prefetchUrls.isEmpty()) { + // DB cache 命中的缩略图也立即落到任务临时目录，避免整批字节停留在堆中。 + long dbCacheStart = System.currentTimeMillis(); + List remainingUrls = new ArrayList<>(prefetchUrls.size()); + int dbHit = 0; + for (String url : prefetchUrls) { + ensureResultAssemblyNotInterrupted(); + if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) { + continue; + } + byte[] cachedBytes = imagePrefetchService.lookup(url); + if (cachedBytes == null) { + remainingUrls.add(url); + continue; + } + SimilarAsinImageEmbedder.ResizedImage thumb = imageEmbedder.decodeCachedThumb(cachedBytes); + if (thumb != null) { + imageSpool.put(url, thumb); + dbHit++; + } else { + remainingUrls.add(url); + } + } + log.info("[similar-asin] image db-cache lookup finished urls={} hit={} miss={} costMs={}", + prefetchUrls.size(), dbHit, remainingUrls.size(), System.currentTimeMillis() - dbCacheStart); + if (!remainingUrls.isEmpty()) { + long prefetchStart = System.currentTimeMillis(); + imageEmbedder.prefetchToDisk(remainingUrls, imageSpool); + ensureResultAssemblyNotInterrupted(); + log.info("[similar-asin] image disk-prefetch finished urls={} spooled={} costMs={}", + remainingUrls.size(), imageSpool.size(), System.currentTimeMillis() - prefetchStart); + } + } + + int rowIndex = 1; + for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.of() : rowsToWrite) { + ensureResultAssemblyNotInterrupted(); + SimilarAsinResultRowDto resultRow = SimilarAsinChunkMergeSupport.findResultRow(parsedRow, resultMap); + Row row = sheet.createRow(rowIndex); + int col = 0; + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId())); + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), "")); + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), "")); + row.createCell(col++).setCellValue(resultRow == null + ? firstNonBlank(parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")) + : firstNonBlank( + resultRow.getPrice(), + firstNonBlank(parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")))); + row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name")); + row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "品牌", "brand")); + String isStock = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getIsStock()); + String similarity = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getSimilarity()); + String isConform = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getIsConform()); + String reason = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getReason()); + String category = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getCategory()); + String status = resultRow == null + || (SimilarAsinResultTextSupport.isSilentOutputFailure(resultRow) && !SimilarAsinResultTextSupport.isAsinNotFound(resultRow.getError())) + ? "" + : SimilarAsinResultTextSupport.resolveResultStatus(resultRow, isStock, similarity, isConform, reason, category); + row.createCell(col++).setCellValue(isStock); + row.createCell(col++).setCellValue(similarity); + row.createCell(col++).setCellValue(isConform); + row.createCell(col++).setCellValue(reason); + row.createCell(col++).setCellValue(category); + row.createCell(col).setCellValue(status); + if (resultRow != null) { + // 行高固定到 IMAGE_ROW_HEIGHT_POINTS（Excel 上限 409pt）： + row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS); + imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN, + resultRow.getMainUrl(), row, taskImageCache, cellImageSession, imageSpool); + imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1, + resultRow.getPuzzleImg1(), row, taskImageCache, cellImageSession, imageSpool); + imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2, + resultRow.getPuzzleImg2(), row, taskImageCache, cellImageSession, imageSpool); + } + rowIndex++; + } + ensureResultAssemblyNotInterrupted(); + workbook.write(fos); + workbook.dispose(); + } + if (!cellImageSession.isEmpty()) { + try { + ensureResultAssemblyNotInterrupted(); + WpsCellImageWriter.patchXlsxFile(xlsx, cellImageSession); + } catch (IOException ex) { + String cause = ex.toString(); + log.error("[similar-asin] excel cell image patch failed xlsx={} images={} err={}", + xlsx.getAbsolutePath(), cellImageSession.size(), cause, ex); + throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex); + } + } + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("生成相似ASIN检测结果失败", ex); + } finally { + try { + imageSpool.close(); + } catch (IOException ex) { + log.warn("[similar-asin] delete image spool failed xlsx={} err={}", + xlsx.getAbsolutePath(), ex.getMessage()); + } + } + } + + private static void cancelResultAssemblyFutures(List> futures) { + for (Future future : futures) { + if (future != null && !future.isDone()) { + future.cancel(true); + } + } + } + + private static void ensureResultAssemblyNotInterrupted() { + if (Thread.currentThread().isInterrupted()) { + throw new BusinessException("生成相似ASIN检测结果中断"); + } + } + + /** + * P2-8：收集本次 xlsx 写入需要嵌入的所有图片 URL（去重交给 prefetch 内部处理）。 + * 仅扫 resultMap 中实际命中的行，避免对 unresolved row 浪费下载配额。 + */ + private List collectImageUrlsForPrefetch(List rowsToWrite, + Map resultMap) { + if (rowsToWrite == null || rowsToWrite.isEmpty() || resultMap == null || resultMap.isEmpty()) { + return List.of(); + } + List urls = new ArrayList<>(rowsToWrite.size() * 3); + for (SimilarAsinParsedRowVo parsedRow : rowsToWrite) { + SimilarAsinResultRowDto resultRow = SimilarAsinChunkMergeSupport.findResultRow(parsedRow, resultMap); + if (resultRow == null) { + continue; + } + SimilarAsinResultTextSupport.addNonBlank(urls, resultRow.getMainUrl()); + SimilarAsinResultTextSupport.addNonBlank(urls, resultRow.getPuzzleImg1()); + SimilarAsinResultTextSupport.addNonBlank(urls, resultRow.getPuzzleImg2()); + } + return urls; + } + + + + private String readValueByHeader(SimilarAsinParsedRowVo row, String... candidates) { + if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) { + return ""; + } + for (Map.Entry entry : row.getValues().entrySet()) { + String header = SimilarAsinRowNormalizer.normalize(entry.getKey()).toLowerCase(Locale.ROOT); + for (String candidate : candidates) { + String expected = SimilarAsinRowNormalizer.normalize(candidate).toLowerCase(Locale.ROOT); + if (!expected.isBlank() && header.contains(expected)) { + return entry.getValue() == null ? "" : entry.getValue(); + } + } + } + return ""; + } + + + private static class SourceRowsBuilder { + private final String sourceFileKey; + private String sourceFilename; + private final List rows = new ArrayList<>(); + + private SourceRowsBuilder(String sourceFileKey, String sourceFilename) { + this.sourceFileKey = sourceFileKey; + this.sourceFilename = sourceFilename; + } + + private String sourceFileKey() { + return sourceFileKey; + } + + private String sourceFilename() { + return sourceFilename; + } + + private void setSourceFilename(String sourceFilename) { + this.sourceFilename = sourceFilename; + } + + private List rows() { + return rows; + } + } + + private record SourceRows(String sourceFileKey, + String sourceFilename, + List rows) { + } + + private record SourceResultWorkbook(File file, + String filename, + int rowCount, + int persistedRows, + int resultRows, + long resolvedRows, + int conformPropagated) { + private SourceResultWorkbook(File file, String filename, int rowCount) { + this(file, filename, rowCount, 0, 0, 0L, 0); + } + } + + + private static String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskOwnershipSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskOwnershipSupport.java new file mode 100644 index 00000000..cecb7602 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskOwnershipSupport.java @@ -0,0 +1,299 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import lombok.extern.slf4j.Slf4j; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 6 轮）：实例归属判定与 per-task 分布式锁。 + *

+ * 从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致：当前实例 ID、 + * task/scope/llm-state 的 owner 提取与归属判定、owner 绑定兜底、per-task 锁获取， + * 以及按归属筛选待处理 LLM state。无进程内可变状态。 + */ +@Slf4j +public class SimilarAsinTaskOwnershipSupport { + + private static final String MODULE_TYPE = "SIMILAR_ASIN"; + private static final String LLM_STATUS_SUBMITTED = "SUBMITTED"; + private static final String LLM_STATUS_RUNNING = "RUNNING"; + private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5); + + private final FileTaskMapper fileTaskMapper; + private final TaskDistributedLockService taskDistributedLockService; + private final TaskScopeStateMapper taskScopeStateMapper; + private final ObjectMapper objectMapper; + private final InstanceMetadata instanceMetadata; + private final SimilarAsinChunkPayloadSupport chunkPayloadSupport; + + public SimilarAsinTaskOwnershipSupport(FileTaskMapper fileTaskMapper, + TaskDistributedLockService taskDistributedLockService, + TaskScopeStateMapper taskScopeStateMapper, + ObjectMapper objectMapper, + InstanceMetadata instanceMetadata, + SimilarAsinChunkPayloadSupport chunkPayloadSupport) { + this.fileTaskMapper = fileTaskMapper; + this.taskDistributedLockService = taskDistributedLockService; + this.taskScopeStateMapper = taskScopeStateMapper; + this.objectMapper = objectMapper; + this.instanceMetadata = instanceMetadata; + this.chunkPayloadSupport = chunkPayloadSupport; + } + + + public TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) { + TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_TTL, waitMillis); + if (lockHandle == null) { + log.info("[similar-asin] skip task operation because task lock is busy taskId={}", taskId); + } + return lockHandle; + } + + public TaskDistributedLockService.LockHandle requireTaskLock(Long taskId, long waitMillis) { + TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, waitMillis); + if (lockHandle == null) { + throw new BusinessException(40902, "Task is busy, please retry later"); + } + return lockHandle; + } + + public String currentInstanceId() { + String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId(); + return firstNonBlank(instanceId, "unknown-instance"); + } + + public boolean isJobOwnedByCurrentInstance(TaskFileJobEntity job) { + return job == null || isOwnerCurrent(ownerFromScopeKey(job.getScopeKey())); + } + + public void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) { + String owner = ownerFromTask(task); + if (isOwnerCurrent(owner)) { + return; + } + log.warn("[similar-asin] reject task operation because owner is another instance taskId={} operation={} owner={} current={}", + task == null ? null : task.getId(), operation, owner, currentInstanceId()); + throw new TaskOwnerMismatchException( + task == null ? null : task.getId(), + operation, + owner, + currentInstanceId()); + } + + public boolean isLlmStateOwnedByCurrentInstance(TaskScopeStateEntity state) { + return state == null || isOwnerCurrent(ownerFromLlmState(state)); + } + + public boolean isOwnerCurrent(String owner) { + return owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId()); + } + + public String ownerFromLlmState(TaskScopeStateEntity state) { + LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); + return context == null ? null : context.ownerInstanceId(); + } + + public String ownerFromTask(FileTaskEntity task) { + if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) { + return null; + } + try { + String owner = objectMapper.readTree(task.getResultJson()).path("ownerInstanceId").asText(""); + return owner.isBlank() ? null : owner; + } catch (Exception ex) { + log.warn("[similar-asin] read task owner failed taskId={} err={}", task.getId(), ex.getMessage()); + return null; + } + + /** + * P0-3：当 chunk 因 RustFS 失败落到本地时，把 task 锁定到当前实例。 + * 这样后续 assembleResultWorkbook 调度只会路由到这台机器（已有 + * isJobOwnedByCurrentInstance / ownerFromTask 兼容这条 ownerInstanceId）。 + * 同时把 fallback 信号写到 task_scope_state.state_json，便于排查。 + */ + } + + public void bindTaskToCurrentOwnerForLocalFallback(FileTaskEntity task, String scopeHash, Integer chunkIndex) { + if (task == null || task.getId() == null) { + throw new IllegalStateException("Cannot bind local fallback without a task ID"); + } + String currentInstance = currentInstanceId(); + try { + String existingJson = task.getResultJson(); + ObjectNode payload; + if (existingJson == null || existingJson.isBlank()) { + payload = objectMapper.createObjectNode(); + } else { + JsonNode tree = objectMapper.readTree(existingJson); + payload = tree.isObject() ? (ObjectNode) tree : objectMapper.createObjectNode(); + } + String existingOwner = payload.path("ownerInstanceId").asText(""); + if (existingOwner.isBlank()) { + payload.put("ownerInstanceId", currentInstance); + payload.put("ownerInstanceReason", "rustfs-fallback-local"); + String updatedJson = objectMapper.writeValueAsString(payload); + task.setResultJson(updatedJson); + FileTaskEntity update = new FileTaskEntity(); + update.setId(task.getId()); + update.setResultJson(updatedJson); + update.setUpdatedAt(LocalDateTime.now()); + if (fileTaskMapper.updateById(update) != 1) { + throw new IllegalStateException("Task owner update affected no row"); + } + log.warn("[similar-asin] task bound to current instance due to rustfs fallback taskId={} instanceId={} chunk={}", + task.getId(), currentInstance, chunkIndex); + } else if (!Objects.equals(existingOwner, currentInstance)) { + throw new IllegalStateException("Local fallback task is owned by instance=" + existingOwner + + " current=" + currentInstance); + } + } catch (Exception ex) { + throw new IllegalStateException("Failed to bind local fallback task owner taskId=" + task.getId(), ex); + } + // 把 fallback 信号写到 task_scope_state.state_json，方便排查 / 后续告警钩子。 + if (scopeHash == null || scopeHash.isBlank()) { + return; + } + try { + TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, task.getId()) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + if (scope == null) { + throw new IllegalStateException("Local fallback scope state does not exist"); + } + ObjectNode stateNode; + String stateJson = scope.getStateJson(); + if (stateJson == null || stateJson.isBlank()) { + stateNode = objectMapper.createObjectNode(); + stateNode.put("phase", "RECEIVED"); + stateNode.put("llm", "PENDING"); + } else { + JsonNode parsed = objectMapper.readTree(stateJson); + stateNode = parsed.isObject() ? (ObjectNode) parsed : objectMapper.createObjectNode(); + } + ArrayNode fallbackArr; + JsonNode existingArr = stateNode.path("localFallback"); + if (existingArr.isArray()) { + fallbackArr = (ArrayNode) existingArr; + } else { + fallbackArr = stateNode.putArray("localFallback"); + } + String tag = "chunk-" + chunkIndex + "@" + currentInstance; + boolean exists = false; + for (JsonNode node : fallbackArr) { + if (tag.equals(node.asText(""))) { + exists = true; + break; + } + } + if (!exists) { + fallbackArr.add(tag); + scope.setStateJson(objectMapper.writeValueAsString(stateNode)); + scope.setUpdatedAt(LocalDateTime.now()); + if (taskScopeStateMapper.updateById(scope) != 1) { + throw new IllegalStateException("Local fallback scope update affected no row"); + } + } + } catch (Exception ex) { + throw new IllegalStateException("Failed to record local fallback state taskId=" + + task.getId() + " chunk=" + chunkIndex, ex); + } + } + + public String ownerFromScopeKey(String scopeKey) { + if (scopeKey == null || scopeKey.isBlank()) { + return null; + } + String marker = ":owner:"; + int index = scopeKey.lastIndexOf(marker); + if (index < 0) { + return null; + } + String owner = scopeKey.substring(index + marker.length()).trim(); + return owner.isBlank() ? null : owner; + } + + public int countLlmWorkUnits(List chunks, int batchSize) { + if (chunks == null || chunks.isEmpty()) { + return 0; + } + int total = 0; + for (TaskChunkEntity chunk : chunks) { + Map persistedRows = chunkPayloadSupport.readChunkRows(chunk); + if (persistedRows.isEmpty()) { + continue; + } + total += SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()).size(); + } + return total <= 0 ? 0 : Math.max(1, (total + batchSize - 1) / batchSize); + } + + public List listOwnedPendingLlmStates() { + Map merged = new LinkedHashMap<>(); + for (TaskScopeStateEntity state : queryOwnedPendingLlmStates(false, 50)) { + if (state != null && state.getId() != null) { + merged.put(state.getId(), state); + } + } + for (TaskScopeStateEntity state : queryOwnedPendingLlmStates(true, 50)) { + if (state != null && state.getId() != null) { + merged.putIfAbsent(state.getId(), state); + } + } + return new ArrayList<>(merged.values()); + } + + public List queryOwnedPendingLlmStates(boolean oldestFirst, int limit) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)) + .and(query -> query + .apply("JSON_UNQUOTE(JSON_EXTRACT(state_json, '$.ownerInstanceId')) IS NULL") + .or() + .apply("JSON_UNQUOTE(JSON_EXTRACT(state_json, '$.ownerInstanceId')) = ''") + .or() + .apply("JSON_UNQUOTE(JSON_EXTRACT(state_json, '$.ownerInstanceId')) = {0}", currentInstanceId())) + .last("limit " + Math.max(1, Math.min(limit, 100))); + if (oldestFirst) { + wrapper.orderByAsc(TaskScopeStateEntity::getUpdatedAt); + } else { + wrapper.orderByDesc(TaskScopeStateEntity::getUpdatedAt); + } + List states = taskScopeStateMapper.selectList(wrapper); + return states == null ? List.of() : states; + } + + public String buildTaskOwnerScopeKey(FileTaskEntity task) { + Long taskId = task == null ? null : task.getId(); + return "task:" + taskId + ":owner:" + firstNonBlank(ownerFromTask(task), currentInstanceId()); + } + + private static String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } + + +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskProgressSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskProgressSupport.java new file mode 100644 index 00000000..ddb0faaa --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskProgressSupport.java @@ -0,0 +1,286 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import lombok.extern.slf4j.Slf4j; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +/** + * spec 05 续（SimilarAsinTaskService 拆分第 6 轮）：文件构建进度、任务视图映射与计数。 + *

+ * 从 {@code SimilarAsinTaskService} 原样搬移，逻辑与搬移前逐行一致：进度百分比计算与 + * 快照解析、组装进度落库、按 task 的查询计数、任务列表项与显示状态推导。 + */ +@Slf4j +public class SimilarAsinTaskProgressSupport { + + private static final String MODULE_TYPE = "SIMILAR_ASIN"; + private static final String STATUS_PENDING = "PENDING"; + private static final String STATUS_RUNNING = "RUNNING"; + private static final String STATUS_SUCCESS = "SUCCESS"; + private static final String STATUS_FAILED = "FAILED"; + private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final String CONTENT_TYPE_ZIP = "application/zip"; + + private final SimilarAsinProperties properties; + private final ObjectMapper objectMapper; + private final FileTaskMapper fileTaskMapper; + private final TaskChunkMapper taskChunkMapper; + private final TaskProgressSnapshotService taskProgressSnapshotService; + + public SimilarAsinTaskProgressSupport(SimilarAsinProperties properties, + ObjectMapper objectMapper, + FileTaskMapper fileTaskMapper, + TaskChunkMapper taskChunkMapper, + TaskProgressSnapshotService taskProgressSnapshotService) { + this.properties = properties; + this.objectMapper = objectMapper; + this.fileTaskMapper = fileTaskMapper; + this.taskChunkMapper = taskChunkMapper; + this.taskProgressSnapshotService = taskProgressSnapshotService; + } + + + public void saveFileBuildProgress(FileTaskEntity task, + TaskFileJobEntity job, + int total, + int completed, + String message) { + if (task == null || task.getId() == null || job == null || job.getId() == null) { + return; + } + int safeTotal = Math.max(1, total); + int safeCompleted = Math.max(0, Math.min(completed, safeTotal)); + TaskProgressSnapshotEntity existing = taskProgressSnapshotService.find(task.getId(), MODULE_TYPE); + int displayPercent = calculateDisplayProgressPercent(safeCompleted, safeTotal, job, LocalDateTime.now()); + boolean terminalProgress = isTerminalFileBuildProgress(task.getStatus(), safeCompleted, safeTotal); + if (terminalProgress) { + displayPercent = 100; + } + int previousDisplayPercent = calculateSnapshotDisplayPercent(existing, job); + if (previousDisplayPercent > displayPercent && safeCompleted < safeTotal) { + displayPercent = previousDisplayPercent; + } + taskProgressSnapshotService.save( + task.getId(), + MODULE_TYPE, + terminalProgress ? task.getStatus() : STATUS_RUNNING, + safeTotal, + safeCompleted, + 0, + job.getScopeKey(), + message, + Map.of("phase", "RESULT_FILE", "jobId", job.getId(), "displayPercent", displayPercent) + ); + } + + + public int calculateDisplayProgressPercent(int current, + int total, + TaskFileJobEntity job, + LocalDateTime baseTime) { + if (total <= 0) { + return 0; + } + current = Math.max(0, Math.min(current, total)); + int percent = Math.max(1, Math.min(99, (int) Math.floor(current * 100.0 / total))); + if (job != null && STATUS_RUNNING.equals(job.getStatus())) { + long elapsedSeconds = baseTime == null ? 0 : Math.max(0, Duration.between(baseTime, LocalDateTime.now()).getSeconds()); + if (current <= 0) { + int firstRealProgressPercent = Math.max(1, Math.min(99, (int) Math.floor(100.0 / total))); + int waitingCap = Math.max(8, Math.min(35, firstRealProgressPercent - 1)); + percent = Math.max(percent, Math.min(waitingCap, 8 + (int) (elapsedSeconds / 6))); + } else if (current < total) { + percent = Math.max(percent, Math.min(92, percent + (int) (elapsedSeconds / 10))); + } + } + return percent; + } + + + public int calculateSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot, TaskFileJobEntity job) { + if (snapshot == null) { + return 0; + } + int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount(); + int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount(); + int percent = total <= 0 ? 0 : calculateDisplayProgressPercent(current, total, job, snapshot.getUpdatedAt()); + return Math.max(percent, extractSnapshotDisplayPercent(snapshot)); + } + + + public int extractSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot) { + if (snapshot == null || snapshot.getSnapshotJson() == null || snapshot.getSnapshotJson().isBlank()) { + return 0; + } + try { + JsonNode displayPercent = objectMapper.readTree(snapshot.getSnapshotJson()).path("displayPercent"); + if (!displayPercent.isNumber()) { + return 0; + } + return Math.max(0, Math.min(100, displayPercent.asInt())); + } catch (Exception ignored) { + return 0; + } + } + + + public int countChunks(Long taskId, String scopeHash) { + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash)); + return count == null ? 0 : count.intValue(); + } + + + public int resolveReceivedChunkProgress(Long taskId, String scopeHash, Integer chunkTotal) { + int receivedCount = countChunks(taskId, scopeHash); + Integer maxChunkIndex = taskChunkMapper.selectObjs(new LambdaQueryWrapper() + .select(TaskChunkEntity::getChunkIndex) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .orderByDesc(TaskChunkEntity::getChunkIndex) + .last("limit 1")) + .stream() + .filter(Objects::nonNull) + .map(value -> { + if (value instanceof Number number) { + return number.intValue(); + } + try { + return Integer.parseInt(value.toString()); + } catch (Exception ignored) { + return 0; + } + }) + .findFirst() + .orElse(0); + int current = Math.max(receivedCount, maxChunkIndex == null ? 0 : maxChunkIndex); + if (chunkTotal != null && chunkTotal > 0) { + current = Math.min(current, chunkTotal); + } + return Math.max(0, current); + } + + + public long countTask(Long userId, String status) { + Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getUserId, userId) + .eq(FileTaskEntity::getStatus, status)); + return count == null ? 0L : count; + } + + + public long countCompletedTasksWithResultFile(Long userId) { + if (userId == null || userId <= 0) { + return 0L; + } + Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getUserId, userId) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .inSql(FileTaskEntity::getId, + "select distinct task_id from biz_file_result " + + "where module_type = '" + MODULE_TYPE + "' " + + "and user_id = " + userId + " " + + "and result_file_url is not null and result_file_url <> ''")); + return count == null ? 0L : count; + } + + + public SimilarAsinTaskItemVo toTaskItem(FileTaskEntity task) { + SimilarAsinTaskItemVo vo = new SimilarAsinTaskItemVo(); + vo.setId(task.getId()); + vo.setTaskNo(task.getTaskNo()); + vo.setStatus(resolveDisplayTaskStatus(task)); + vo.setErrorMessage(task.getErrorMessage()); + vo.setCreatedAt(fmt(task.getCreatedAt())); + vo.setUpdatedAt(fmt(task.getUpdatedAt())); + vo.setFinishedAt(fmt(task.getFinishedAt())); + return vo; + } + + + public String resolveDisplayTaskStatus(FileTaskEntity task) { + return resolveDisplayTaskStatus(task, null); + } + + + public String resolveDisplayTaskStatus(FileTaskEntity task, FileResultEntity row) { + if (task == null) { + return null; + } + if (canTreatTaskAsCompleted(task, row, null)) { + return STATUS_SUCCESS; + } + return task.getStatus(); + } + + + public boolean canTreatTaskAsCompleted(FileTaskEntity task, FileResultEntity row, TaskFileJobEntity job) { + if (task == null) { + return false; + } + if (STATUS_SUCCESS.equals(task.getStatus())) { + return true; + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + return false; + } + if (row == null) { + return false; + } + if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { + return false; + } + if (job != null && "FAILED".equals(job.getStatus())) { + return false; + } + return true; + } + + + public String guessDownloadContentType(String filename) { + String normalized = filename == null ? "" : filename.trim().toLowerCase(Locale.ROOT); + if (normalized.endsWith(".zip")) { + return CONTENT_TYPE_ZIP; + } + return CONTENT_TYPE_XLSX; + } + + /** 任务状态与进度是否已达终态（任务终态 + 进度满格）。 */ + public static boolean isTerminalFileBuildProgress(String taskStatus, int completed, int total) { + return total > 0 + && completed >= total + && (STATUS_SUCCESS.equals(taskStatus) || STATUS_FAILED.equals(taskStatus)); + } + + private static String fmt(LocalDateTime t) { + return t == null ? null : t.toString(); + } + + private static String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkKeysetTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkKeysetTest.java index 9d7cf142..b6e3a944 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkKeysetTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkKeysetTest.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.service; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinChunkMergeSupport; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; import org.junit.jupiter.api.Test; @@ -92,7 +93,7 @@ class SimilarAsinTaskServiceChunkKeysetTest { // 正常输入：chunk 数小于 pageSize，一轮拉完，结果全且按 chunkIndex 有序 List all = chunks(1, 1, 2, 2, 3, 3); stubKeysetPages(all, 500); - List result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); + List result = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); assertEquals(3, result.size()); assertEquals(List.of(1, 2, 3), result.stream().map(TaskChunkEntity::getChunkIndex).toList()); assertEquals(List.of(0L), keysetIdsFromRounds(1), "首轮 keyset 为 0"); @@ -103,7 +104,7 @@ class SimilarAsinTaskServiceChunkKeysetTest { // 超过 pageSize：多轮拉取，keyset 逐轮推进，全部合并且按 chunkIndex 升序、无重复 List all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); stubKeysetPages(all, 3); - List result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3); + List result = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3); assertEquals(7, result.size()); assertEquals(List.of(1, 2, 3, 4, 5, 6, 7), result.stream().map(TaskChunkEntity::getChunkIndex).toList()); long distinctIds = result.stream().map(TaskChunkEntity::getId).distinct().count(); @@ -116,8 +117,8 @@ class SimilarAsinTaskServiceChunkKeysetTest { // 重复执行同一输入：每轮都从 keyset=0 开始，结果一致 List all = chunks(1, 1, 2, 2, 3, 3); stubKeysetPages(all, 500); - List first = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); - List second = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); + List first = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); + List second = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); assertEquals(first.size(), second.size()); for (int i = 0; i < first.size(); i++) { assertEquals(first.get(i).getId(), second.get(i).getId()); @@ -129,7 +130,7 @@ class SimilarAsinTaskServiceChunkKeysetTest { void test_task_009_chunk_boundary_empty_input() { // 空集合：返回空列表，不创建无效资源，且只查一轮 stubKeysetPages(List.of(), 500); - List result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); + List result = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500); assertNotNull(result); assertEquals(0, result.size()); verify(taskChunkMapper, times(1)).selectList(any()); @@ -140,7 +141,7 @@ class SimilarAsinTaskServiceChunkKeysetTest { // 单 chunk：一轮返回后 keyset 推进即拉空，不依赖批量路径 List all = chunks(42, 9); stubKeysetPages(all, 1); - List result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 1); + List result = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 1); assertEquals(1, result.size()); assertEquals(9, result.get(0).getChunkIndex()); assertEquals(42L, result.get(0).getId()); @@ -151,28 +152,28 @@ class SimilarAsinTaskServiceChunkKeysetTest { // chunk 数恰好等于 pageSize 的倍数：最后一轮仍返回非空才继续，全部取回 List all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6); stubKeysetPages(all, 3); - List result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3); + List result = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3); assertEquals(6, result.size()); // pageSize 为 0/负数：回退默认 500，不抛异常 stubKeysetPages(all, 0); - List fallback = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 0); + List fallback = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 0); assertEquals(6, fallback.size()); stubKeysetPages(all, -5); - List negative = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, -5); + List negative = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, -5); assertEquals(6, negative.size()); } @Test void test_task_009_chunk_invalid_input_rejected() { // taskId 为 null：安全返回空列表，不发起查询 - List result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, null, MODULE, 500); + List result = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, null, MODULE, 500); assertNotNull(result); assertEquals(0, result.size()); verify(taskChunkMapper, times(0)).selectList(any()); // mapper 查询抛异常：转项目约定异常，消息可识别 when(taskChunkMapper.selectList(any())).thenThrow(new IllegalStateException("db down")); BusinessException ex = assertThrows(BusinessException.class, - () -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500)); + () -> SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500)); assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"), "异常消息必须可识别，实际: " + ex.getMessage()); } @@ -192,12 +193,12 @@ class SimilarAsinTaskServiceChunkKeysetTest { return all.subList(3, 5); }).when(taskChunkMapper).selectList(any()); BusinessException ex = assertThrows(BusinessException.class, - () -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3)); + () -> SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3)); assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"), "异常消息必须可识别，实际: " + ex.getMessage()); // 恢复后重试成功：5 个 chunk 全部取回 stubKeysetPages(all, 3); - List recovered = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3); + List recovered = SimilarAsinChunkMergeSupport.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3); assertEquals(5, recovered.size()); assertEquals(List.of(1, 2, 3, 4, 5), recovered.stream().map(TaskChunkEntity::getChunkIndex).toList()); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java index 73a8d843..b3f63a80 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.similarasin.service; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.similarasin.service.support.LlmBatchContext; import com.nanri.aiimage.config.SimilarAsinProperties; import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; @@ -189,8 +190,8 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { return new ObjectMapper().writeValueAsString(rows); } - private SimilarAsinTaskService.LlmBatchContext context(int batchTotal) { - return new SimilarAsinTaskService.LlmBatchContext( + private LlmBatchContext context(int batchTotal) { + return new LlmBatchContext( 7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null); } @@ -227,7 +228,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, - SimilarAsinTaskService.LlmBatchContext.class, + LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); @@ -281,7 +282,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, - SimilarAsinTaskService.LlmBatchContext.class, + LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); TaskScopeStateEntity state = state(task, 1L, "DONE", 2); @@ -300,7 +301,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { FileTaskEntity task = task(); Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, - SimilarAsinTaskService.LlmBatchContext.class, + LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of()); @@ -317,7 +318,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, - SimilarAsinTaskService.LlmBatchContext.class, + LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); @@ -336,7 +337,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, - SimilarAsinTaskService.LlmBatchContext.class, + LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); @@ -356,7 +357,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, - SimilarAsinTaskService.LlmBatchContext.class, + LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceGroupRefTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceGroupRefTest.java index 22778ed2..d2913350 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceGroupRefTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceGroupRefTest.java @@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPayloadSupport; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; @@ -244,7 +245,7 @@ class SimilarAsinTaskServiceGroupRefTest { SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); payload.setItems(List.of()); payload.setGroups(List.of()); - List restored = SimilarAsinTaskService.resolveAllRows(payload); + List restored = SimilarAsinPayloadSupport.resolveAllRows(payload); assertNotNull(restored); assertEquals(0, restored.size()); // 引用越界（startIndex 超出 items 范围）：安全跳过该组，不抛异常 @@ -255,7 +256,7 @@ class SimilarAsinTaskServiceGroupRefTest { group.setStartIndex(5); group.setEndIndex(7); badRef.setGroups(List.of(group)); - List outOfRange = SimilarAsinTaskService.resolveAllRows(badRef); + List outOfRange = SimilarAsinPayloadSupport.resolveAllRows(badRef); assertEquals(0, outOfRange.size(), "越界引用必须安全跳过"); } @@ -288,7 +289,7 @@ class SimilarAsinTaskServiceGroupRefTest { String legacy = "{\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0OLD00001\"}]," + "\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":1,\"itemCount\":1}]}"; SimilarAsinParsedPayloadDto legacyPayload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class); - List restored = SimilarAsinTaskService.resolveAllRows(legacyPayload); + List restored = SimilarAsinPayloadSupport.resolveAllRows(legacyPayload); assertEquals(1, restored.size()); assertEquals("B0OLD00001", restored.get(0).getAsin()); } @@ -303,7 +304,7 @@ class SimilarAsinTaskServiceGroupRefTest { bad.setStartIndex(1); bad.setEndIndex(1); payload.setGroups(List.of(bad)); - assertEquals(0, SimilarAsinTaskService.resolveAllRows(payload).size()); + assertEquals(0, SimilarAsinPayloadSupport.resolveAllRows(payload).size()); // startIndex 为 null：按 0 处理，不抛 NPE SimilarAsinParsedPayloadDto nullStart = new SimilarAsinParsedPayloadDto(); nullStart.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1"))); @@ -312,7 +313,7 @@ class SimilarAsinTaskServiceGroupRefTest { g.setStartIndex(null); g.setEndIndex(1); nullStart.setGroups(List.of(g)); - assertEquals(1, SimilarAsinTaskService.resolveAllRows(nullStart).size()); + assertEquals(1, SimilarAsinPayloadSupport.resolveAllRows(nullStart).size()); } @Test @@ -363,7 +364,7 @@ class SimilarAsinTaskServiceGroupRefTest { "组 items 首行必须是 items[startIndex]"); } // Java 内部展开路径不受内嵌 items 影响：仍按引用切片 - List expanded = SimilarAsinTaskService.expandGroupRefs(stored); + List expanded = SimilarAsinPayloadSupport.expandGroupRefs(stored); assertEquals(12, expanded.size(), "Java 内部展开仍按引用，不受内嵌 items 影响"); } @@ -376,7 +377,7 @@ class SimilarAsinTaskServiceGroupRefTest { + "{\"rowToken\":\"t2\",\"asin\":\"B0GRP00002\",\"sourceFileKey\":\"f.xlsx\",\"country\":\"英国\"}" + "],\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":2,\"itemCount\":2}]}"; SimilarAsinParsedPayloadDto payload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class); - SimilarAsinParsedPayloadDto hydrated = SimilarAsinTaskService.hydrateForPythonCompat(payload); + SimilarAsinParsedPayloadDto hydrated = SimilarAsinPayloadSupport.hydrateForPythonCompat(payload); assertEquals(1, hydrated.getGroups().size()); assertNotNull(hydrated.getGroups().get(0).getItems()); assertEquals(2, hydrated.getGroups().get(0).getItems().size(), "hydrate 后组内嵌 items 必须为 2 行"); @@ -423,6 +424,6 @@ class SimilarAsinTaskServiceGroupRefTest { copy.setItems(payload.getItems()); copy.setAllItems(payload.getAllItems()); copy.setGroups(payload.getGroups()); - return SimilarAsinTaskService.expandGroupRefs(copy); + return SimilarAsinPayloadSupport.expandGroupRefs(copy); } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java index 6c7a00f6..cc1bc2cb 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPayloadSupport; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -61,7 +62,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); List items = rows("uploads/20260829/base.xlsx", 100); payload.setItems(items); - List restored = SimilarAsinTaskService.resolveAllRows(payload); + List restored = SimilarAsinPayloadSupport.resolveAllRows(payload); assertEquals(100, restored.size()); assertEquals("uploads/20260829/base.xlsx::row::1", restored.get(0).getRowToken()); } @@ -72,7 +73,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); List items = rows("uploads/20260829/multi.xlsx", 1000); payload.setItems(items); - List restored = SimilarAsinTaskService.resolveAllRows(payload); + List restored = SimilarAsinPayloadSupport.resolveAllRows(payload); assertEquals(1000, restored.size()); for (int i = 0; i < restored.size(); i++) { assertEquals(i + 1, restored.get(i).getRowIndex()); @@ -80,7 +81,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { // 旧结构 1000 行：allItems 全量恢复 SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto(); legacy.setAllItems(items); - List restoredLegacy = SimilarAsinTaskService.resolveAllRows(legacy); + List restoredLegacy = SimilarAsinPayloadSupport.resolveAllRows(legacy); assertEquals(1000, restoredLegacy.size()); assertEquals("uploads/20260829/multi.xlsx::row::1", restoredLegacy.get(0).getRowToken()); } @@ -91,8 +92,8 @@ class SimilarAsinTaskServiceResolveAllRowsTest { SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); List items = rows("uploads/20260829/idem.xlsx", 50); payload.setItems(items); - List first = SimilarAsinTaskService.resolveAllRows(payload); - List second = SimilarAsinTaskService.resolveAllRows(payload); + List first = SimilarAsinPayloadSupport.resolveAllRows(payload); + List second = SimilarAsinPayloadSupport.resolveAllRows(payload); assertEquals(first.size(), second.size()); for (int i = 0; i < first.size(); i++) { assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken()); @@ -105,7 +106,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { void test_task_003_payload_boundary_empty_input() { // 空 payload：返回空列表而非 null，不创建无效资源 SimilarAsinParsedPayloadDto empty = new SimilarAsinParsedPayloadDto(); - List restored = SimilarAsinTaskService.resolveAllRows(empty); + List restored = SimilarAsinPayloadSupport.resolveAllRows(empty); assertNotNull(restored); assertEquals(0, restored.size()); // items/allItems/groups 均为空的 payload @@ -113,7 +114,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { allEmpty.setItems(List.of()); allEmpty.setAllItems(List.of()); allEmpty.setGroups(List.of()); - assertEquals(0, SimilarAsinTaskService.resolveAllRows(allEmpty).size()); + assertEquals(0, SimilarAsinPayloadSupport.resolveAllRows(allEmpty).size()); } @Test @@ -121,11 +122,11 @@ class SimilarAsinTaskServiceResolveAllRowsTest { // 单行新结构 SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); payload.setItems(rows("uploads/20260829/single.xlsx", 1)); - assertEquals(1, SimilarAsinTaskService.resolveAllRows(payload).size()); + assertEquals(1, SimilarAsinPayloadSupport.resolveAllRows(payload).size()); // 单行旧结构（仅 groups） SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto(); legacy.setGroups(List.of(group(rows("uploads/20260829/single.xlsx", 1)))); - assertEquals(1, SimilarAsinTaskService.resolveAllRows(legacy).size()); + assertEquals(1, SimilarAsinPayloadSupport.resolveAllRows(legacy).size()); } @Test @@ -133,35 +134,35 @@ class SimilarAsinTaskServiceResolveAllRowsTest { // 旧结构 allItems 5000 行全量恢复，不丢失 SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto(); legacy.setAllItems(rows("uploads/20260829/max.xlsx", 5000)); - assertEquals(5000, SimilarAsinTaskService.resolveAllRows(legacy).size()); + assertEquals(5000, SimilarAsinPayloadSupport.resolveAllRows(legacy).size()); // items 与 allItems 同时存在：以 items 为准（新规范结构优先），不重复 SimilarAsinParsedPayloadDto both = new SimilarAsinParsedPayloadDto(); both.setItems(rows("uploads/20260829/both.xlsx", 10)); both.setAllItems(rows("uploads/20260829/both.xlsx", 20)); - List restored = SimilarAsinTaskService.resolveAllRows(both); + List restored = SimilarAsinPayloadSupport.resolveAllRows(both); assertEquals(10, restored.size()); // groups 也同时存在：仍以 items 为准 both.setGroups(List.of(group(rows("uploads/20260829/both.xlsx", 30)))); - assertEquals(10, SimilarAsinTaskService.resolveAllRows(both).size()); + assertEquals(10, SimilarAsinPayloadSupport.resolveAllRows(both).size()); } @Test void test_task_003_payload_invalid_input_rejected() { // null payload：安全返回空列表（调用方容忍），不抛 NPE - assertEquals(0, SimilarAsinTaskService.resolveAllRows(null).size()); + assertEquals(0, SimilarAsinPayloadSupport.resolveAllRows(null).size()); // groups 中含 null 元素：跳过不抛异常 SimilarAsinParsedPayloadDto messy = new SimilarAsinParsedPayloadDto(); List groups = new ArrayList<>(); groups.add(null); groups.add(group(rows("uploads/20260829/messy.xlsx", 5))); messy.setGroups(groups); - assertEquals(5, SimilarAsinTaskService.resolveAllRows(messy).size()); + assertEquals(5, SimilarAsinPayloadSupport.resolveAllRows(messy).size()); // group.items 为 null：跳过该组 SimilarAsinParsedGroupVo nullItemsGroup = new SimilarAsinParsedGroupVo(); nullItemsGroup.setItems(null); SimilarAsinParsedPayloadDto nullItems = new SimilarAsinParsedPayloadDto(); nullItems.setGroups(List.of(nullItemsGroup, group(rows("uploads/20260829/n2.xlsx", 3)))); - assertEquals(3, SimilarAsinTaskService.resolveAllRows(nullItems).size()); + assertEquals(3, SimilarAsinPayloadSupport.resolveAllRows(nullItems).size()); } @Test @@ -173,7 +174,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { + "{\"rowToken\":\"t2\",\"asin\":\"B0AAA00002\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":2}]}"; SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class); // 未 hydrate 时：allItems 恢复（items 为空走 allItems） - List fromLegacy = SimilarAsinTaskService.resolveAllRows(payload); + List fromLegacy = SimilarAsinPayloadSupport.resolveAllRows(payload); assertEquals(2, fromLegacy.size()); assertEquals("t1", fromLegacy.get(0).getRowToken()); assertEquals("B0AAA00002", fromLegacy.get(1).getAsin()); @@ -183,7 +184,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { + "\"items\":[{\"rowToken\":\"n1\",\"asin\":\"B0NEW00001\",\"sourceFileKey\":\"uploads/20260829/b.xlsx\",\"rowIndex\":1}]," + "\"groups\":[]}"; SimilarAsinParsedPayloadDto newPayload = MAPPER.readValue(newJson, SimilarAsinParsedPayloadDto.class); - List fromNew = SimilarAsinTaskService.resolveAllRows(newPayload); + List fromNew = SimilarAsinPayloadSupport.resolveAllRows(newPayload); assertEquals(1, fromNew.size()); assertEquals("n1", fromNew.get(0).getRowToken()); // 两种格式行数之和互不影响，恢复结果稳定 @@ -199,7 +200,7 @@ class SimilarAsinTaskServiceResolveAllRowsTest { // 组内行号各自独立从 1 开始（真实解析语义），第二组用不同 fileKey 区分来源 List g2b = rows("uploads/20260829/g2.xlsx", 3); legacy.setGroups(List.of(group(g1), group(g2b))); - List restored = SimilarAsinTaskService.resolveAllRows(legacy); + List restored = SimilarAsinPayloadSupport.resolveAllRows(legacy); assertEquals(5, restored.size()); assertEquals("uploads/20260829/g.xlsx::row::1", restored.get(0).getRowToken()); assertEquals("uploads/20260829/g.xlsx::row::2", restored.get(1).getRowToken()); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java index 6e4f7903..a8df11db 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.config.SimilarAsinProperties; import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinChunkMergeSupport; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; @@ -166,7 +167,7 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")); - List deduped = service.dedupeRowsByRowKey(llmRows); + List deduped = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); assertEquals(1, deduped.size(), "重复行必须按稳定 rowKey 去重"); assertEquals("r1", deduped.get(0).getRowToken()); @@ -200,7 +201,7 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { llmRows.add(row("r1", "1", "B0A0000001", "英国")); llmRows.add(row("r2", "2", "B0A0000002", "英国")); } - List deduped = service.dedupeRowsByRowKey(llmRows); + List deduped = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); assertEquals(2, deduped.size(), "3 轮重复输入去重后只剩 2 个唯一行"); assertEquals(List.of("r1", "r2"), deduped.stream().map(SimilarAsinResultRowDto::getRowToken).toList(), "去重必须保留首次出现顺序"); @@ -221,8 +222,8 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国"), row("r1", "1", "B0A0000001", "英国")); - List first = service.dedupeRowsByRowKey(llmRows); - List second = service.dedupeRowsByRowKey(llmRows); + List first = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); + List second = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); assertEquals(first.size(), second.size()); for (int i = 0; i < first.size(); i++) { assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken()); @@ -233,21 +234,21 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { @Test void test_task_011_merge_row_key_boundary_empty_input() { // 空输入：null/空列表安全返回空结果，不创建无效资源 - assertNotNull(service.dedupeRowsByRowKey(null)); - assertTrue(service.dedupeRowsByRowKey(null).isEmpty()); - assertTrue(service.dedupeRowsByRowKey(List.of()).isEmpty()); + assertNotNull(SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(null)); + assertTrue(SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(null).isEmpty()); + assertTrue(SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(List.of()).isEmpty()); // null 元素：跳过不抛异常 List withNull = new ArrayList<>(); withNull.add(null); withNull.add(row("r1", "1", "B0A0000001", "英国")); - assertEquals(1, service.dedupeRowsByRowKey(withNull).size()); + assertEquals(1, SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(withNull).size()); } @Test void test_task_011_merge_row_key_boundary_single_item() { // 单行：不依赖批量路径，去重后结果正确 List single = List.of(row("r1", "1", "B0A0000001", "英国")); - List deduped = service.dedupeRowsByRowKey(single); + List deduped = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(single); assertEquals(1, deduped.size()); assertEquals("r1", deduped.get(0).getRowToken()); } @@ -259,14 +260,14 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { for (int i = 0; i < 1000; i++) { llmRows.add(row("r1", "1", "B0A0000001", "英国")); } - List deduped = service.dedupeRowsByRowKey(llmRows); + List deduped = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); assertEquals(1, deduped.size()); // 1000 行唯一：全部保留且顺序稳定 List unique = new ArrayList<>(); for (int i = 0; i < 1000; i++) { unique.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0U" + String.format("%06d", i), "英国")); } - List dedupedUnique = service.dedupeRowsByRowKey(unique); + List dedupedUnique = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(unique); assertEquals(1000, dedupedUnique.size()); assertEquals("r0001", dedupedUnique.get(1).getRowToken()); } @@ -277,13 +278,13 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { List noToken = List.of( row("", "1", "B0A0000001", "英国"), row("", "1", "b0a0000001", " 英国 ")); - List deduped = service.dedupeRowsByRowKey(noToken); + List deduped = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(noToken); assertEquals(1, deduped.size(), "legacy key 归一化（ASIN 大写、country trim）后应识别为同一行"); // 不同 ASIN：不误判为重复 List diffAsin = List.of( row("", "1", "B0A0000001", "英国"), row("", "2", "B0A0000002", "英国")); - assertEquals(2, service.dedupeRowsByRowKey(diffAsin).size()); + assertEquals(2, SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(diffAsin).size()); } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java index 4fbff56f..3985b0bd 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java @@ -5,6 +5,7 @@ import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.SimilarAsinProperties; import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinChunkMergeSupport; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; @@ -217,8 +218,8 @@ class SimilarAsinTaskServiceRowKeyIndexTest { } } List orphans = new ArrayList<>(); - Map index = service.indexRowsByChunkKey(rowsByChunk); - Map> merged = service.assignLlmRowsToChunks( + Map index = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); + Map> merged = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, llmRows, index, null, null, orphans); assertEquals(3, merged.size()); assertEquals(9, assignedRowKeys(merged).size()); @@ -234,10 +235,10 @@ class SimilarAsinTaskServiceRowKeyIndexTest { Map> rowsByChunk = rowsByChunkOf("hashA", 1, List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国"))); List llmRows = List.of(row("r1", "1", "B0A0000001", "英国")); - Map index = service.indexRowsByChunkKey(rowsByChunk); - Map> first = service.assignLlmRowsToChunks( + Map index = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); + Map> first = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, llmRows, index, null, null, new ArrayList<>()); - Map> second = service.assignLlmRowsToChunks( + Map> second = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, llmRows, index, null, null, new ArrayList<>()); assertEquals(assignedRowKeys(first), assignedRowKeys(second)); assertEquals(first.size(), second.size()); @@ -249,20 +250,20 @@ class SimilarAsinTaskServiceRowKeyIndexTest { @Test void test_task_010_chunk_row_key_boundary_empty_input() { // 空输入：null/空 rowsByChunk 与 llmRows 均安全返回空结果，不创建无效资源 - assertNotNull(service.indexRowsByChunkKey(null)); - assertTrue(service.indexRowsByChunkKey(null).isEmpty()); - assertTrue(service.indexRowsByChunkKey(Map.of()).isEmpty()); - Map> emptyAssign = service.assignLlmRowsToChunks( + assertNotNull(SimilarAsinChunkMergeSupport.indexRowsByChunkKey(null)); + assertTrue(SimilarAsinChunkMergeSupport.indexRowsByChunkKey(null).isEmpty()); + assertTrue(SimilarAsinChunkMergeSupport.indexRowsByChunkKey(Map.of()).isEmpty()); + Map> emptyAssign = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( Map.of(), List.of(), Map.of(), null, null, new ArrayList<>()); assertTrue(emptyAssign.isEmpty()); - assertTrue(service.assignLlmRowsToChunks( + assertTrue(SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( Map.of(), null, Map.of(), null, null, new ArrayList<>()).isEmpty()); // 无可匹配行（rowKey 不存在于任何 chunk）→ 进 orphan 兜底，不产生 merge Map> rowsByChunk = rowsByChunkOf("hashA", 1, List.of(row("r1", "1", "B0A0000001", "英国"))); List blankRow = List.of(row("", "", "", "")); List orphans = new ArrayList<>(); - Map index = service.indexRowsByChunkKey(rowsByChunk); - Map> merged = service.assignLlmRowsToChunks( + Map index = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); + Map> merged = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, blankRow, index, null, null, orphans); assertTrue(assignedRowKeys(merged).isEmpty()); assertEquals(1, orphans.size(), "全空行生成 legacy key :::: 不命中任何 chunk，按既有语义进 orphan"); @@ -274,8 +275,8 @@ class SimilarAsinTaskServiceRowKeyIndexTest { Map> rowsByChunk = rowsByChunkOf("hashA", 1, List.of(row("r1", "1", "B0A0000001", "英国"))); List orphans = new ArrayList<>(); - Map index = service.indexRowsByChunkKey(rowsByChunk); - Map> merged = service.assignLlmRowsToChunks( + Map index = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); + Map> merged = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, List.of(row("r1", "1", "B0A0000001", "英国")), index, null, null, orphans); assertEquals(1, merged.size()); assertEquals(List.of("r1"), assignedRowKeys(merged)); @@ -294,14 +295,14 @@ class SimilarAsinTaskServiceRowKeyIndexTest { bigChunk.put("r" + String.format("%04d", i), row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国")); } rowsByChunk.put("hashBig:1", bigChunk); - Map index = service.indexRowsByChunkKey(rowsByChunk); + Map index = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); assertEquals(1000, index.size()); List llmRows = new ArrayList<>(); for (int i = 1; i <= 500; i++) { llmRows.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国")); } List orphans = new ArrayList<>(); - Map> merged = service.assignLlmRowsToChunks( + Map> merged = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, llmRows, index, null, null, orphans); assertEquals(1, merged.size()); assertEquals(500, assignedRowKeys(merged).size()); @@ -314,11 +315,11 @@ class SimilarAsinTaskServiceRowKeyIndexTest { Map> rowsByChunk = new LinkedHashMap<>(); rowsByChunk.putAll(rowsByChunkOf("hashA", 1, List.of(row("dup", "1", "B0A0000001", "英国")))); rowsByChunk.putAll(rowsByChunkOf("hashB", 2, List.of(row("dup", "1", "B0A0000001", "英国")))); - Map index = service.indexRowsByChunkKey(rowsByChunk); + Map index = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); assertEquals("hashA:1", index.get("dup"), "重复 rowKey 应保留第一个 chunk"); // fallback 缺失：llm 行未命中且无有效 fallback → 进 orphan，不产生 merge List orphans = new ArrayList<>(); - Map> merged = service.assignLlmRowsToChunks( + Map> merged = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, List.of(row("ghost", "9", "B0A0000009", "英国")), Map.of(), "missingHash", 99, orphans); assertTrue(assignedRowKeys(merged).isEmpty()); assertEquals(1, orphans.size()); @@ -328,8 +329,8 @@ class SimilarAsinTaskServiceRowKeyIndexTest { withNull.add(null); withNull.add(row("dup", "1", "B0A0000001", "英国")); List orphans2 = new ArrayList<>(); - Map index2 = service.indexRowsByChunkKey(rowsByChunk); - Map> merged2 = service.assignLlmRowsToChunks( + Map index2 = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); + Map> merged2 = SimilarAsinChunkMergeSupport.assignLlmRowsToChunks( rowsByChunk, withNull, index2, null, null, orphans2); assertEquals(1, merged2.size()); assertEquals(List.of("dup"), assignedRowKeys(merged2)); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java index 81419bc9..43129467 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java @@ -2,6 +2,9 @@ package com.nanri.aiimage.modules.similarasin.service; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultTextSupport; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinTaskProgressSupport; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultWorkbookAssembler; import org.junit.jupiter.api.Test; import java.lang.reflect.Field; @@ -17,7 +20,7 @@ class SimilarAsinTaskServiceTest { @Test @SuppressWarnings("unchecked") void resultWorkbookRestoresPriceAfterCountryAndShiftsImageColumns() throws Exception { - Field headersField = SimilarAsinTaskService.class.getDeclaredField("RESULT_HEADERS"); + Field headersField = SimilarAsinResultWorkbookAssembler.class.getDeclaredField("RESULT_HEADERS"); headersField.setAccessible(true); List headers = (List) headersField.get(null); @@ -32,26 +35,26 @@ class SimilarAsinTaskServiceTest { @Test void resultStatusUsesReturnedCozeDataAndImages() { - assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(null)); + assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(null)); SimilarAsinResultRowDto empty = new SimilarAsinResultRowDto(); - assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(empty)); + assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(empty)); SimilarAsinResultRowDto withImage = new SimilarAsinResultRowDto(); withImage.setStatus("FAILED"); withImage.setMainUrl("https://example.com/main.jpg"); - assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withImage)); + assertEquals("\u6210\u529f", SimilarAsinResultTextSupport.resolveResultStatus(withImage)); SimilarAsinResultRowDto withCozeStatus = new SimilarAsinResultRowDto(); withCozeStatus.setStatus("success"); - assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withCozeStatus)); + assertEquals("\u6210\u529f", SimilarAsinResultTextSupport.resolveResultStatus(withCozeStatus)); SimilarAsinResultRowDto failedStatus = new SimilarAsinResultRowDto(); failedStatus.setStatus("FAILED"); - assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(failedStatus)); + assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(failedStatus)); SimilarAsinResultRowDto withVisibleResultData = new SimilarAsinResultRowDto(); - assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", "")); + assertEquals("\u6210\u529f", SimilarAsinResultTextSupport.resolveResultStatus(withVisibleResultData, "", "80%", "", "", "")); } @Test @@ -62,39 +65,39 @@ class SimilarAsinTaskServiceTest { notFound.setError("404 Not Found"); notFound.setStatus("FAILED"); notFound.setReason("required field missing (asin/title/url), skip LLM"); - assertEquals("ASIN\u672a\u627e\u5230", SimilarAsinTaskService.resolveResultStatus(notFound)); + assertEquals("ASIN\u672a\u627e\u5230", SimilarAsinResultTextSupport.resolveResultStatus(notFound)); // \u5176\u5b83\u7b49\u4ef7\u6587\u6848\u540c\u6837\u547d\u4e2d SimilarAsinResultRowDto chinese = new SimilarAsinResultRowDto(); chinese.setError("\u5546\u54c1\u672a\u627e\u5230"); - assertEquals("ASIN\u672a\u627e\u5230", SimilarAsinTaskService.resolveResultStatus(chinese)); + assertEquals("ASIN\u672a\u627e\u5230", SimilarAsinResultTextSupport.resolveResultStatus(chinese)); // \u666e\u901a\u6280\u672f\u6027\u5931\u8d25\u4e0d\u8bef\u5224\uff1aerror \u4e0d\u542b 404/not found SimilarAsinResultRowDto llmFailure = new SimilarAsinResultRowDto(); llmFailure.setError("LLM \u8c03\u7528\u8d85\u65f6"); llmFailure.setStatus("FAILED"); - assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(llmFailure)); + assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(llmFailure)); } @Test void asinNotFoundDetectionMatchesPythonErrors() { - assertTrue(SimilarAsinTaskService.isAsinNotFound("404 Not Found")); - assertTrue(SimilarAsinTaskService.isAsinNotFound("404 not found")); - assertTrue(SimilarAsinTaskService.isAsinNotFound("404")); - assertTrue(SimilarAsinTaskService.isAsinNotFound("\u5546\u54c1\u672a\u627e\u5230")); - assertTrue(SimilarAsinTaskService.isAsinNotFound("ASIN \u672a\u627e\u5230")); - assertFalse(SimilarAsinTaskService.isAsinNotFound(null)); - assertFalse(SimilarAsinTaskService.isAsinNotFound("")); - assertFalse(SimilarAsinTaskService.isAsinNotFound("LLM \u8c03\u7528\u8d85\u65f6")); - assertFalse(SimilarAsinTaskService.isAsinNotFound("\u4e0a\u4f20\u5931\u8d25")); + assertTrue(SimilarAsinResultTextSupport.isAsinNotFound("404 Not Found")); + assertTrue(SimilarAsinResultTextSupport.isAsinNotFound("404 not found")); + assertTrue(SimilarAsinResultTextSupport.isAsinNotFound("404")); + assertTrue(SimilarAsinResultTextSupport.isAsinNotFound("\u5546\u54c1\u672a\u627e\u5230")); + assertTrue(SimilarAsinResultTextSupport.isAsinNotFound("ASIN \u672a\u627e\u5230")); + assertFalse(SimilarAsinResultTextSupport.isAsinNotFound(null)); + assertFalse(SimilarAsinResultTextSupport.isAsinNotFound("")); + assertFalse(SimilarAsinResultTextSupport.isAsinNotFound("LLM \u8c03\u7528\u8d85\u65f6")); + assertFalse(SimilarAsinResultTextSupport.isAsinNotFound("\u4e0a\u4f20\u5931\u8d25")); } @Test void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() { - assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3)); - assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("FAILED", 3, 3)); - assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("RUNNING", 3, 3)); - assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3)); + assertTrue(SimilarAsinTaskProgressSupport.isTerminalFileBuildProgress("SUCCESS", 3, 3)); + assertTrue(SimilarAsinTaskProgressSupport.isTerminalFileBuildProgress("FAILED", 3, 3)); + assertFalse(SimilarAsinTaskProgressSupport.isTerminalFileBuildProgress("RUNNING", 3, 3)); + assertFalse(SimilarAsinTaskProgressSupport.isTerminalFileBuildProgress("SUCCESS", 2, 3)); } @Test @@ -149,7 +152,7 @@ class SimilarAsinTaskServiceTest { } private int staticIntField(String name) throws Exception { - Field field = SimilarAsinTaskService.class.getDeclaredField(name); + Field field = SimilarAsinResultWorkbookAssembler.class.getDeclaredField(name); field.setAccessible(true); return field.getInt(null); }