From b05bba50fa90cc4ea206f93079d556892555dab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 14 Sep 2026 01:51:36 +0800 Subject: [PATCH] =?UTF-8?q?refactor(similar-asin):=20=E6=8A=BD=E5=87=BA=20?= =?UTF-8?q?LlmPipelineSupport=EF=BC=8CService=20=E9=99=8D=E8=87=B3=201735?= =?UTF-8?q?=20=E8=A1=8C=EF=BC=88=E7=B4=AF=E8=AE=A1=20-70.7%=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在上一提交（3455 行）基础上，把「Python 结果回传 → 分片落库 → LLM 检测」整条流水线 （57 个方法 / 1694 行）抽为 SimilarAsinPipelineSupport。等价搬移，未改行为。 采用依赖倒置消除循环依赖：support 包声明 SimilarAsinPipelineHost 接口 （finalizeTask / findOrCreateResultRecordForAssembly / readCategorySwitch / finalizeExhaustedResultFileJob），由 SimilarAsinTaskService 实现；这几项保留在宿主 是因为它们属于编排与事务边界（handleResultFileJobFailure 带 @Transactional）。 同时把 5 个被两侧共用的内部 record 提为 support 包顶层类型 （SubmittedTaskMetadata / FinalizeTaskResult / SubmitContext / PersistSubmittedChunkResult / PreparedSubmittedChunk），3 个仅流水线内部使用的 record 内联进流水线类；LlmBatchContext 一并归位。 测试适配：4 个测试类里对 mergeLlmRowsIntoChunk / bufferLlmRowsOrMerge / flushLlmBufferedResults 的反射改指向流水线实例；RollbackSemanticsContractTest 通过 pipelineSupport() 反射取得实例（跨包，保持封装）。 验证：干净工作区叠加本改动跑 similarasin 386 + task 引用方 166 测试， 结果与基线一致（仅既有失败），零新增失败。 --- .../service/SimilarAsinTaskService.java | 1828 +--------------- .../service/support/FinalizeTaskResult.java | 9 + .../support/PersistSubmittedChunkResult.java | 10 + .../support/PreparedSubmittedChunk.java | 19 + .../support/SimilarAsinPipelineHost.java | 31 + .../support/SimilarAsinPipelineSupport.java | 1911 +++++++++++++++++ .../service/support/SubmitContext.java | 16 + .../support/SubmittedTaskMetadata.java | 14 + ...larAsinTaskServiceChunkMergeLimitTest.java | 5 +- ...larAsinTaskServiceCozeBufferScopeTest.java | 39 +- ...imilarAsinTaskServiceRowKeyDedupeTest.java | 15 +- ...SimilarAsinTaskServiceRowKeyIndexTest.java | 11 +- .../SimilarAsinTaskServiceTxBoundaryTest.java | 7 +- 13 files changed, 2105 insertions(+), 1810 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/FinalizeTaskResult.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PersistSubmittedChunkResult.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PreparedSubmittedChunk.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmitContext.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmittedTaskMetadata.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index d769584e..ccb95b26 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -42,6 +42,13 @@ import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGrouping import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler; import com.nanri.aiimage.modules.similarasin.service.support.LlmBatchContext; import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinLimits; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineHost; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport; +import com.nanri.aiimage.modules.similarasin.service.support.SubmitContext; +import com.nanri.aiimage.modules.similarasin.service.support.SubmittedTaskMetadata; +import com.nanri.aiimage.modules.similarasin.service.support.FinalizeTaskResult; +import com.nanri.aiimage.modules.similarasin.service.support.PersistSubmittedChunkResult; +import com.nanri.aiimage.modules.similarasin.service.support.PreparedSubmittedChunk; import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPayloadSupport; import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPoisonTracker; import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultTextSupport; @@ -105,7 +112,7 @@ import java.util.function.Supplier; @Service @RequiredArgsConstructor @Slf4j -public class SimilarAsinTaskService { +public class SimilarAsinTaskService implements SimilarAsinPipelineHost { public static final String MODULE_TYPE = "SIMILAR_ASIN"; private static final String STATUS_PENDING = "PENDING"; @@ -233,6 +240,18 @@ public class SimilarAsinTaskService { objectMapper, transientPayloadStorageService, taskScopeStateMapper); } + /** + * spec 05 续（拆分第 7 轮）：Python 回传 → 分片落库 → LLM 检测流水线。 + * 宿主能力（收尾状态机 / 结果记录 / 类目开关 / 结果任务失败处理）以本服务自身实现。 + */ + SimilarAsinPipelineSupport pipelineSupport() { + return new SimilarAsinPipelineSupport( + properties, objectMapper, similarAsinLlmService, fileTaskMapper, fileResultMapper, + taskChunkMapper, taskScopeStateMapper, transientPayloadStorageService, taskFileJobService, + distributedJobLockService, taskQueueExecutor, imagePrefetchService, poisonTracker, + payloadSupport(), chunkPayloadSupport(), ownershipSupport(), progressSupport(), this); + } + /** * spec 05 续（拆分第 6 轮）：文件构建进度、任务视图映射与计数。 */ @@ -688,23 +707,23 @@ public class SimilarAsinTaskService { if (request == null) { throw new BusinessException("结果请求不能为空"); } - PreparedSubmittedChunk prepared = prepareSubmittedChunk(taskId, request); + PreparedSubmittedChunk prepared = pipelineSupport().prepareSubmittedChunk(taskId, request); PersistSubmittedChunkResult persisted = transactionManager == null - ? persistSubmittedChunk(prepared) - : inNewTransaction(() -> persistSubmittedChunk(prepared)); + ? pipelineSupport().persistSubmittedChunk(prepared) + : inNewTransaction(() -> pipelineSupport().persistSubmittedChunk(prepared)); SubmitContext context = persisted.context(); // A thrown transaction may already have committed but lost its ACK. Keep // that candidate; cleanup is only safe after a successful transaction // has proved that another chunk won the unique key. if (!persisted.payloadPersisted()) { - cleanupPreparedSubmittedChunkIfUnreferenced(prepared); + pipelineSupport().cleanupPreparedSubmittedChunkIfUnreferenced(prepared); } if (persisted.finalizeResult() == null) { taskCacheService.touchTaskHeartbeat(taskId); } else { applyFinalizeSideEffects(persisted.finalizeResult()); } - scheduleLlmPipelineForSubmittedChunk(context); + pipelineSupport().scheduleLlmPipelineForSubmittedChunk(context); } public void deleteTask(Long taskId, Long userId) { @@ -755,7 +774,7 @@ public class SimilarAsinTaskService { } for (String payload : new LinkedHashSet<>(payloads)) { try { - if (isTransientPayloadStillReferenced(payload)) { + if (pipelineSupport().isTransientPayloadStillReferenced(payload)) { log.info("[similar-asin] skip task payload delete because it is still referenced taskId={} pointer={}", taskId, transientPayloadStorageService.extractPointer(payload)); continue; @@ -768,30 +787,6 @@ public class SimilarAsinTaskService { } } - private boolean isTransientPayloadStillReferenced(String payload) { - String pointer = transientPayloadStorageService.extractPointer(payload); - if (pointer == null) { - return false; - } - LinkedHashSet values = new LinkedHashSet<>(); - values.add(payload); - values.add(pointer); - try { - values.add(objectMapper.writeValueAsString(pointer)); - } catch (Exception ignored) { - } - Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper() - .in(TaskChunkEntity::getPayloadJson, values)); - if (chunkCount != null && chunkCount > 0L) { - return true; - } - Long scopeCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() - .and(wrapper -> wrapper.in(TaskScopeStateEntity::getParsedPayloadJson, values) - .or() - .in(TaskScopeStateEntity::getStateJson, values))); - return scopeCount != null && scopeCount > 0L; - } - public ResultDownloadInfo resolveResultDownloadInfo(Long resultId, Long userId) { FileResultEntity row = fileResultMapper.selectById(resultId); if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { @@ -901,226 +896,6 @@ public class SimilarAsinTaskService { return updatedMillis <= thresholdMillis; } - private PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, SimilarAsinSubmitResultRequest request) { - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { - throw new BusinessException("任务不存在"); - } - if (!STATUS_RUNNING.equals(task.getStatus())) { - throw new BusinessException("任务不是运行中状态"); - } - - ownershipSupport().ensureTaskOwnedByCurrentInstance(task, "submit result"); - int chunkIndex = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics - .chunkIndex(request.getChunkIndex()); - int chunkTotal = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics - .chunkTotal(request.getChunkTotal()); - boolean done = Boolean.TRUE.equals(request.getDone()); - String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId); - String scopeHash = DigestUtil.sha256Hex(scopeKey); - boolean terminalCallback = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics - .isTerminalRequest(done, request.getError()); - SubmittedTaskMetadata taskMetadata = terminalCallback - ? readSubmittedTaskMetadata(task) - : null; - TaskChunkEntity existing = findSubmittedChunk(taskId, scopeHash, chunkIndex); - if (existing != null) { - return new PreparedSubmittedChunk( - taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, done, request.getError(), - null, null, false, taskMetadata, null); - } - String payloadJson = writeJson(flattenSubmittedRows(request), "结果序列化失败"); - String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned( - MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); - boolean localFallback = transientPayloadStorageService.wasLastStoreLocalFallback(); - // 纯计算在事务外完成：payload 哈希预计算，persist 落库时直接引用 - return new PreparedSubmittedChunk( - taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, done, request.getError(), - payloadJson, storedPayload, localFallback, taskMetadata, - DigestUtil.sha256Hex(payloadJson)); - } - - private SubmittedTaskMetadata readSubmittedTaskMetadata(FileTaskEntity task) { - SimilarAsinParsedPayloadDto payload = payloadSupport().readParsedPayload(task); - List sourceFiles = payload.getSourceFiles() == null - ? List.of() - : List.copyOf(payload.getSourceFiles()); - return new SubmittedTaskMetadata(SimilarAsinPayloadSupport.rowCount(payload), sourceFiles); - } - - private PersistSubmittedChunkResult persistSubmittedChunk(PreparedSubmittedChunk prepared) { - Long taskId = prepared.taskId(); - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { - throw new BusinessException("任务不存在"); - } - if (!STATUS_RUNNING.equals(task.getStatus())) { - throw new BusinessException("任务不是运行中状态"); - } - ownershipSupport().ensureTaskOwnedByCurrentInstance(task, "submit result"); - - boolean payloadPersisted = prepared.storedPayload() == null; - TaskChunkEntity existing = findSubmittedChunk(taskId, prepared.scopeHash(), prepared.chunkIndex()); - if (existing == null && prepared.storedPayload() != null) { - TaskChunkEntity chunk = new TaskChunkEntity(); - chunk.setTaskId(taskId); - chunk.setModuleType(MODULE_TYPE); - chunk.setScopeKey(prepared.scopeKey()); - chunk.setScopeHash(prepared.scopeHash()); - chunk.setChunkIndex(prepared.chunkIndex()); - chunk.setChunkTotal(prepared.chunkTotal()); - chunk.setPayloadJson(prepared.storedPayload()); - chunk.setPayloadHash(prepared.payloadHash()); - chunk.setCreatedAt(LocalDateTime.now()); - chunk.setUpdatedAt(LocalDateTime.now()); - try { - if (taskChunkMapper.insert(chunk) != 1) { - throw new IllegalStateException("Chunk insert affected no row taskId=" + taskId - + " chunk=" + prepared.chunkIndex()); - } - payloadPersisted = true; - } catch (DuplicateKeyException ex) { - log.info("[similar-asin] duplicate chunk inserted concurrently taskId={} scope={} chunk={} cleanupLoser={}", - taskId, prepared.scopeKey(), prepared.chunkIndex(), prepared.storedPayload() != null); - } - } else { - log.info("[similar-asin] duplicate chunk ignored taskId={} scope={} chunk={}", - taskId, prepared.scopeKey(), prepared.chunkIndex()); - } - - upsertScopeState(taskId, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkTotal(), - prepared.error(), prepared.done(), false); - if (prepared.localFallback() && payloadPersisted) { - ownershipSupport().bindTaskToCurrentOwnerForLocalFallback( - task, prepared.scopeHash(), prepared.chunkIndex()); - } - SubmitContext context = new SubmitContext(task, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkIndex(), - prepared.done(), prepared.error(), prepared.taskMetadata()); - FinalizeTaskResult finalizeResult = completeSubmittedChunk(context); - return new PersistSubmittedChunkResult(context, payloadPersisted, finalizeResult); - } - - private TaskChunkEntity findSubmittedChunk(Long taskId, String scopeHash, Integer chunkIndex) { - return taskChunkMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .eq(TaskChunkEntity::getScopeHash, scopeHash) - .eq(TaskChunkEntity::getChunkIndex, chunkIndex) - .last("limit 1")); - } - - private void cleanupPreparedSubmittedChunkIfUnreferenced(PreparedSubmittedChunk prepared) { - if (prepared == null || prepared.storedPayload() == null) { - return; - } - try { - if (isTransientPayloadStillReferenced(prepared.storedPayload())) { - log.info("[similar-asin] keep prepared chunk payload because it is referenced taskId={} chunk={}", - prepared.taskId(), prepared.chunkIndex()); - return; - } - transientPayloadStorageService.deletePayloadIfPresent(prepared.storedPayload()); - } catch (Exception ex) { - log.warn("[similar-asin] cleanup uncommitted chunk payload failed taskId={} chunk={} err={}", - prepared.taskId(), prepared.chunkIndex(), ex.getMessage()); - } - } - - private FinalizeTaskResult completeSubmittedChunk(SubmitContext context) { - FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); - if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { - throw new BusinessException("任务不存在"); - } - if (!STATUS_RUNNING.equals(task.getStatus())) { - log.info("[similar-asin] skip completion because task already finalized taskId={} status={}", - task.getId(), task.getStatus()); - return null; - } - if (context.forceFlush() || context.error() != null && !context.error().isBlank()) { - return finalizeTask(task, context.error(), context.taskMetadata(), true); - } - touchJavaSideTaskActivity(task.getId()); - return null; - } - - private void submitLlmForSubmittedChunk(SubmitContext context) { - if (context == null || context.task() == null || context.task().getId() == null) { - return; - } - FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { - return; - } - List chunks; - if (context.scopeHash() == null || context.scopeHash().isBlank() || context.chunkIndex() == null) { - chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, task.getId()) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getChunkIndex)); - } else { - TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, task.getId()) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .eq(TaskChunkEntity::getScopeHash, context.scopeHash()) - .eq(TaskChunkEntity::getChunkIndex, context.chunkIndex()) - .last("limit 1")); - chunks = chunk == null ? List.of() : List.of(chunk); - } - chunks = chunks == null ? List.of() : chunks.stream() - .filter(Objects::nonNull) - .filter(chunk -> !chunkPayloadSupport().readChunkRows(chunk).isEmpty()) - .toList(); - if (chunks.isEmpty()) { - return; - } - FileResultEntity result = findOrCreateResultRecordForAssembly(task, payloadSupport().allRowCount(task)); - if (result == null) { - return; - } - TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( - task.getId(), MODULE_TYPE, result.getId(), ownershipSupport().buildTaskOwnerScopeKey(task)); - if (job == null || "SUCCESS".equals(job.getStatus())) { - return; - } - Map> allRowsByBaseId = loadAllRowsByBaseId(task); - boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId); - if (pendingLlm) { - taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); - touchJavaSideTaskActivity(task.getId()); - } else if (isResultSubmissionComplete(task.getId())) { - maybeFinalizeLlmJobLocked(task.getId(), new LlmBatchContext( - job.getId(), result.getId(), context.scopeHash(), context.chunkIndex(), 1, 1, ownershipSupport().currentInstanceId(), 0, null, null)); - } - } - - private void scheduleLlmPipelineForSubmittedChunk(SubmitContext context) { - if (context == null || context.task() == null || context.task().getId() == null) { - return; - } - FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { - return; - } - List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, task.getId()) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getChunkIndex)); - if (chunks.isEmpty()) { - return; - } - FileResultEntity result = findOrCreateResultRecordForAssembly(task, payloadSupport().allRowCount(task)); - if (result == null) { - return; - } - TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( - task.getId(), MODULE_TYPE, result.getId(), ownershipSupport().buildTaskOwnerScopeKey(task)); - if (job == null || "SUCCESS".equals(job.getStatus())) { - return; - } - taskFileJobService.requeue(job.getId(), "Similar ASIN result uploaded, scheduling LLM/file assembly"); - touchJavaSideTaskActivity(task.getId()); - } - private void finalizeStaleTask(Long taskId, String error) { FileTaskEntity task = fileTaskMapper.selectById(taskId); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) { @@ -1129,7 +904,7 @@ public class SimilarAsinTaskService { if (tryRecoverTimedOutPythonTask(task)) { return; } - SubmittedTaskMetadata metadata = readSubmittedTaskMetadata(task); + SubmittedTaskMetadata metadata = pipelineSupport().readSubmittedTaskMetadata(task); FinalizeTaskResult result = transactionManager == null ? finalizeTask(task, error, metadata, true) : inNewTransaction(() -> finalizeTask(task, error, metadata, true)); @@ -1141,8 +916,8 @@ public class SimilarAsinTaskService { return false; } Long taskId = task.getId(); - boolean uploadComplete = isResultSubmissionComplete(taskId); - long pendingLlmStates = countPendingLlmStates(taskId); + boolean uploadComplete = pipelineSupport().isResultSubmissionComplete(taskId); + long pendingLlmStates = pipelineSupport().countPendingLlmStates(taskId); long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE); if (!resultAssembler().hasPersistedResultRows(taskId)) { return false; @@ -1158,8 +933,8 @@ public class SimilarAsinTaskService { log.warn("[similar-asin] Python 超时恢复继续推进 LLM/文件收尾 taskId={} pendingLlmStates={} activeAssembleJobs={}", taskId, pendingLlmStates, activeAssembleJobs); } - submitLlmForSubmittedChunk(new SubmitContext(task, null, null, null, true, null, null)); - touchJavaSideTaskActivity(taskId); + pipelineSupport().submitLlmForSubmittedChunk(new SubmitContext(task, null, null, null, true, null, null)); + pipelineSupport().touchJavaSideTaskActivity(taskId); return true; } @@ -1181,7 +956,7 @@ public class SimilarAsinTaskService { if (latestChunk == null || latestChunk.getScopeHash() == null || latestChunk.getScopeHash().isBlank()) { return 0; } - upsertScopeState(taskId, + pipelineSupport().upsertScopeState(taskId, firstNonBlank(latestChunk.getScopeKey(), "task:" + taskId), latestChunk.getScopeHash(), latestChunk.getChunkTotal(), @@ -1208,272 +983,12 @@ public class SimilarAsinTaskService { return updated; } - private void upsertScopeState(Long taskId, - String scopeKey, - String scopeHash, - Integer chunkTotal, - String error, - boolean completed, - boolean llmDone) { - TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .eq(TaskScopeStateEntity::getScopeHash, scopeHash) - .last("limit 1")); - LocalDateTime now = LocalDateTime.now(); - if (scope == null) { - scope = new TaskScopeStateEntity(); - scope.setTaskId(taskId); - scope.setModuleType(MODULE_TYPE); - scope.setScopeKey(scopeKey); - scope.setScopeHash(scopeHash); - scope.setCreatedAt(now); - } - if (chunkTotal != null) { - scope.setChunkTotal(chunkTotal); - } - scope.setReceivedChunkCount(progressSupport().countChunks(taskId, scopeHash)); - scope.setLastChunkAt(now); - scope.setLastError(error); - scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0); - scope.setUpdatedAt(now); - scope.setStateJson(llmDone - ? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}" - : "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}"); - if (scope.getId() == null) { - try { - taskScopeStateMapper.insert(scope); - return; - } catch (DuplicateKeyException ex) { - log.info("[similar-asin] duplicate scope state inserted concurrently taskId={} scope={}", taskId, scopeKey); - scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .eq(TaskScopeStateEntity::getScopeHash, scopeHash) - .last("limit 1")); - if (scope == null) { - log.info("[similar-asin] scope state winner not committed yet, skip duplicate updater taskId={} scope={}", - taskId, scopeKey); - return; - } - if (chunkTotal != null) { - scope.setChunkTotal(chunkTotal); - } - scope.setReceivedChunkCount(progressSupport().resolveReceivedChunkProgress(taskId, scopeHash, scope.getChunkTotal())); - scope.setLastChunkAt(now); - scope.setLastError(error); - scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0); - scope.setUpdatedAt(now); - scope.setStateJson(llmDone - ? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}" - : "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}"); - } - } - taskScopeStateMapper.updateById(scope); - } - private T inNewTransaction(Supplier action) { TransactionTemplate template = new TransactionTemplate(transactionManager); template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); return template.execute(status -> action.get()); } - private List applyLlmInBatches(List items, FileTaskEntity task) { - return applyLlmInBatches(items, task, null); - } - - private List applyLlmInBatches(List items, - FileTaskEntity task, - Runnable progressHook) { - if (items == null || items.isEmpty()) { - return List.of(); - } - String prompt = payloadSupport().readAiPrompt(task); - String apiKey = payloadSupport().readApiKey(task); - boolean imgSwitch = payloadSupport().readImgSwitch(task); - boolean categorySwitch = readCategorySwitch(task); - int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, imgSwitch); - List result = new ArrayList<>(); - for (int i = 0; i < items.size(); i += batchSize) { - List batch = items.subList(i, Math.min(i + batchSize, items.size())); - // 批次内所有 LLM 请求按任务归属用户计次（无归属时上下文自动跳过） - result.addAll(SecretUsageContext.call(task.getUserId(), UserSecretModule.SIMILAR_ASIN.key(), - () -> similarAsinLlmService.inspectRows(batch, prompt, apiKey, imgSwitch, categorySwitch))); - if (progressHook != null) { - progressHook.run(); - } - } - return result; - } - - private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List rows) { - if (rows == null || rows.isEmpty()) { - return; - } - int maxAttempts = 3; - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .eq(TaskChunkEntity::getScopeHash, scopeHash) - .eq(TaskChunkEntity::getChunkIndex, chunkIndex) - .last("limit 1")); - if (chunk == null) { - return; - } - Map persistedRows = chunkPayloadSupport().readChunkRows(chunk); - int mergedRowCount = persistedRows.size() + rows.size(); - // Task 13：单次合并后总行数超过上限时，从最旧行（存量优先）开始降级到 orphan 兜底， - // chunk 保持在上限内不无界增长；assemble 阶段 putIfAbsent 合并回结果不丢数据。 - if (mergedRowCount > SimilarAsinLimits.getChunkMergeMaxRows(properties)) { - splitChunkMergeOverflow(taskId, persistedRows, rows, chunk, - mergedRowCount - SimilarAsinLimits.getChunkMergeMaxRows(properties)); - if (rows.isEmpty()) { - return; - } - } - for (SimilarAsinResultRowDto row : rows) { - persistedRows.put(SimilarAsinChunkMergeSupport.rowKey(row), row); - } - String payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败"); - // Task 13：合并后 payload 字节超过上限时，从最旧行开始降级到 orphan 兜底； - // 降到只剩一行仍超上限时抛异常拒绝合并，防止无界 payload。 - long payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length; - if (payloadBytes > SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)) { - demoteRowsToOrphan(taskId, persistedRows, - payloadBytes - SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)); - payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败"); - payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length; - if (payloadBytes > SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties) && persistedRows.size() <= 1) { - throw new BusinessException("相似ASIN分片载荷超字节上限 taskId=" + taskId - + " scopeHash=" + scopeHash + " chunk=" + chunkIndex - + " bytes=" + payloadBytes + " limit=" + SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)); - } - } - String oldPayload = chunk.getPayloadJson(); - String oldPayloadHash = chunk.getPayloadHash(); - String newPayloadHash = DigestUtil.sha256Hex(payloadJson); - final String storedPayload; - try { - storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); - } catch (Exception storeEx) { - throw new BusinessException("相似ASIN分片载荷存储失败 taskId=" + taskId + " chunk=" + chunkIndex - + ": " + (storeEx.getMessage() == null ? "" : storeEx.getMessage()), storeEx); - } - int updated = taskChunkMapper.update(null, new LambdaUpdateWrapper() - .eq(TaskChunkEntity::getId, chunk.getId()) - .eq(TaskChunkEntity::getPayloadHash, oldPayloadHash) - .set(TaskChunkEntity::getPayloadJson, storedPayload) - .set(TaskChunkEntity::getPayloadHash, newPayloadHash) - .set(TaskChunkEntity::getUpdatedAt, LocalDateTime.now())); - if (updated > 0) { - log.debug("[similar-asin] chunk payload replaced taskId={} scopeHash={} chunk={} oldPayload={} newPayload={} attempt={}", - taskId, scopeHash, chunkIndex, oldPayload, storedPayload, attempt); - transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload); - return; - } - transientPayloadStorageService.deletePayloadIfPresent(storedPayload); - if (attempt < maxAttempts) { - log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}", - taskId, scopeHash, chunkIndex, attempt, maxAttempts); - } - } - throw new IllegalStateException("相似ASIN分片载荷更新失败"); - } - - /** - * Task 13：按行数上限从最旧行开始降级到 orphan 兜底，保证合并后 chunk 行数不超过上限。 - * 最旧优先：先降级存量行（LinkedHashMap 表头），不够再从新增行表头补齐； - * 传入的 rows 会被就地修改（保留未降级部分）。降级失败仅记日志，不阻断合并主流程。 - */ - private void splitChunkMergeOverflow(Long taskId, Map persistedRows, - List rows, TaskChunkEntity chunk, - int demoteCount) { - if (demoteCount <= 0 || rows.isEmpty()) { - return; - } - List demoted = new ArrayList<>(); - while (demoted.size() < demoteCount && !persistedRows.isEmpty()) { - String oldestKey = persistedRows.keySet().iterator().next(); - SimilarAsinResultRowDto removed = persistedRows.remove(oldestKey); - if (removed != null) { - demoted.add(removed); - } - } - for (Iterator it = rows.iterator(); it.hasNext() && demoted.size() < demoteCount; ) { - SimilarAsinResultRowDto row = it.next(); - if (row != null) { - demoted.add(row); - it.remove(); - } - } - chunkPayloadSupport().persistOrphanLlmRows(taskId, demoted); - log.warn("[similar-asin] chunk merge row-limit exceeded taskId={} chunk={} mergedRows={} limit={} demoted={}", - taskId, chunk.getChunkIndex(), SimilarAsinChunkMergeSupport.mergedRowCountOf(persistedRows, rows), - SimilarAsinLimits.getChunkMergeMaxRows(properties), demoted.size()); - } - - /** - * Task 13：按字节上限从最旧行（LinkedHashMap 表头）开始降级到 orphan 兜底， - * 直到 payload 字节不超过上限或只剩一行；降级失败仅记日志，不阻断合并主流程。 - */ - private void demoteRowsToOrphan(Long taskId, Map persistedRows, long excessBytes) { - List demoted = new ArrayList<>(); - long releasedBytes = 0L; - while (persistedRows.size() > 1 && releasedBytes < excessBytes) { - String oldestKey = persistedRows.keySet().iterator().next(); - SimilarAsinResultRowDto removed = persistedRows.remove(oldestKey); - if (removed != null) { - demoted.add(removed); - releasedBytes += estimateRowBytes(oldestKey, removed); - } - } - if (!demoted.isEmpty()) { - chunkPayloadSupport().persistOrphanLlmRows(taskId, demoted); - log.warn("[similar-asin] chunk merge byte-limit exceeded taskId={} rows={} demoted={} releasedBytes={}", - taskId, demoted.size(), demoted.size(), releasedBytes); - } - } - - private long estimateRowBytes(String rowKey, SimilarAsinResultRowDto row) { - try { - String json = objectMapper.writeValueAsString(row); - return json == null ? 128L : json.getBytes(StandardCharsets.UTF_8).length; - } catch (Exception ex) { - return 128L + (rowKey == null ? 0 : rowKey.getBytes(StandardCharsets.UTF_8).length); - } - } - - private Map> loadAllRowsByBaseId(FileTaskEntity task) { - try { - SimilarAsinParsedPayloadDto payload = payloadSupport().readParsedPayload(task); - return SimilarAsinGroupingConverter.groupRowsByBaseId(SimilarAsinPayloadSupport.resolveAllRows(payload)); - } catch (Exception ex) { - log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage()); - return new LinkedHashMap<>(); - } - } - - private List collectPendingLlmCandidates(List chunks, - Map> allRowsByBaseId) { - if (chunks == null || chunks.isEmpty()) { - return List.of(); - } - List candidates = new ArrayList<>(); - for (TaskChunkEntity chunk : chunks) { - Map persistedRows = chunkPayloadSupport().readChunkRows(chunk); - if (persistedRows.isEmpty()) { - continue; - } - List unresolvedRows = - enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()), allRowsByBaseId); - for (SimilarAsinResultRowDto row : unresolvedRows) { - candidates.add(new LlmCandidate(chunk.getScopeHash(), chunk.getChunkIndex(), row)); - } - } - return candidates; - } - private List buildResponsePreviewGroups(List groups, List allRows) { if (groups == null || groups.isEmpty()) { return List.of(); @@ -1540,91 +1055,8 @@ public class SimilarAsinTaskService { return vo; } - private List flattenSubmittedRows(SimilarAsinSubmitResultRequest request) { - if (request == null) { - return List.of(); - } - List groups = request.getGroups(); - if (groups != null && !groups.isEmpty()) { - List rows = new ArrayList<>(); - for (SimilarAsinResultGroupDto group : groups) { - if (group == null || group.getItems() == null || group.getItems().isEmpty()) { - continue; - } - for (SimilarAsinResultRowDto row : group.getItems()) { - if (row == null) { - continue; - } - if (normalize(row.getGroupKey()).isBlank()) { - row.setGroupKey(group.getGroupKey()); - } - if (normalize(row.getSourceFileKey()).isBlank()) { - row.setSourceFileKey(group.getSourceFileKey()); - } - if (normalize(row.getSourceFilename()).isBlank()) { - row.setSourceFilename(group.getSourceFilename()); - } - rows.add(row); - } - } - logPythonInboundRows(rows); - return rows; - } - return List.of(); - } - - /** - * 打印 Python 端回传给 Java 的每一行 row 关键字段，确认 url（主图）/ urls（同类商品图）/ title / sku - * 是否按预期到达。该日志与直连 LLM 的行级结果日志成对， - * 便于排查"Python 回传了什么、Java 又把什么提交给 LLM"。 - * Task 19：改为 DEBUG 级别并按行采样（每 20 行记一行），减少大任务日志量。 - */ - private void logPythonInboundRows(List rows) { - if (rows == null || rows.isEmpty()) { - return; - } - log.debug("[similar-asin] python inbound start size={}", rows.size()); - for (int i = 0; i < rows.size(); i++) { - SimilarAsinResultRowDto row = rows.get(i); - if (row == null) { - continue; - } - if (!SimilarAsinLogSupport.shouldLog(i, PYTHON_INBOUND_LOG_EVERY_N)) { - continue; - } - String url = row.getUrl(); - List urls = row.getUrls(); - List alibaba = row.getAlibaba(); - log.debug("[similar-asin] python inbound idx={} groupKey={} rowToken={} id={} asin={} country={} title={} sku={} price={} url={} urlsSize={} alibabaSize={} urlsHead={} urlsTail={}", - i, - normalize(row.getGroupKey()), - normalize(row.getRowToken()), - normalize(row.getId()), - normalize(row.getAsin()), - normalize(row.getCountry()), - abbreviateForLog(row.getTitle(), 80), - normalize(row.getSku()), - normalize(row.getPrice()), - abbreviateForLog(url, 200), - urls == null ? 0 : urls.size(), - alibaba == null ? 0 : alibaba.size(), - urls == null || urls.isEmpty() ? "" : abbreviateForLog(urls.get(0), 200), - urls == null || urls.size() <= 1 ? "" : abbreviateForLog(urls.get(urls.size() - 1), 200)); - } - } - - private String abbreviateForLog(String value, int maxLength) { - if (value == null) { - return ""; - } - String trimmed = value.trim(); - if (maxLength <= 0 || trimmed.length() <= maxLength) { - return trimmed; - } - return trimmed.substring(0, maxLength) + "..."; - } - - private FinalizeTaskResult finalizeTask(FileTaskEntity task, + @Override + public FinalizeTaskResult finalizeTask(FileTaskEntity task, String error, SubmittedTaskMetadata taskMetadata, boolean assembleWorkbook) { @@ -1688,7 +1120,8 @@ public class SimilarAsinTaskService { } } - private FileResultEntity findOrCreateResultRecordForAssembly(FileTaskEntity task, int rowCount) { + @Override + public FileResultEntity findOrCreateResultRecordForAssembly(FileTaskEntity task, int rowCount) { if (task == null || task.getId() == null) { return null; } @@ -1711,7 +1144,7 @@ public class SimilarAsinTaskService { if (dispatchWhenIdle && job != null && "RUNNING".equals(job.getStatus()) - && countPendingLlmStates(task.getId()) == 0) { + && pipelineSupport().countPendingLlmStates(task.getId()) == 0) { taskFileJobService.requeue(job.getId(), "Python 上传完成，正在组装 xlsx"); } return job; @@ -1766,23 +1199,23 @@ public class SimilarAsinTaskService { .eq(TaskChunkEntity::getTaskId, task.getId()) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) .orderByAsc(TaskChunkEntity::getChunkIndex)); - Map> allRowsByBaseId = loadAllRowsByBaseId(task); + Map> allRowsByBaseId = pipelineSupport().loadAllRowsByBaseId(task); int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, payloadSupport().readImgSwitch(task)); int llmWorkUnits = ownershipSupport().countLlmWorkUnits(chunks, batchSize); int totalProgressUnits = Math.max(3, llmWorkUnits + 3); - if (countPendingLlmStates(task.getId()) > 0) { - finalizeTimedOutLlmStatesForTask(task.getId()); + if (pipelineSupport().countPendingLlmStates(task.getId()) > 0) { + pipelineSupport().finalizeTimedOutLlmStatesForTask(task.getId()); } - if (countPendingLlmStates(task.getId()) > 0) { + if (pipelineSupport().countPendingLlmStates(task.getId()) > 0) { taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); - touchJavaSideTaskActivity(task.getId()); + pipelineSupport().touchJavaSideTaskActivity(task.getId()); progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, 1, "LLM 已提交，等待结果"); return false; } // P0-3：stale-recovery / 异常路径兜底 —— maybeFinalizeLlmJob 可能未能成功 flush // （Redis 锁竞争 / 异常退出），在 assemble 阶段读 chunk 之前再 flush 一次，幂等。 try { - flushLlmBufferedResults(task.getId()); + pipelineSupport().flushLlmBufferedResults(task.getId()); chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() .eq(TaskChunkEntity::getTaskId, task.getId()) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) @@ -1794,16 +1227,16 @@ public class SimilarAsinTaskService { + firstNonBlank(flushEx.getMessage(), "未知错误")); } progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, 0, "正在提交 LLM"); - boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId); + boolean pendingLlm = pipelineSupport().submitLlmBatches(task, result, job, chunks, allRowsByBaseId); if (pendingLlm) { taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); - touchJavaSideTaskActivity(task.getId()); + pipelineSupport().touchJavaSideTaskActivity(task.getId()); progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, 1, "LLM 已提交，等待结果"); return false; } - if (STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(task.getId())) { + if (STATUS_RUNNING.equals(task.getStatus()) && !pipelineSupport().isResultSubmissionComplete(task.getId())) { taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); - touchJavaSideTaskActivity(task.getId()); + pipelineSupport().touchJavaSideTaskActivity(task.getId()); progressSupport().saveFileBuildProgress(task, job, totalProgressUnits, Math.max(1, llmWorkUnits), "等待 Python 上传，每 " + batchSize + " 行提交一次 LLM"); return false; @@ -1812,7 +1245,8 @@ public class SimilarAsinTaskService { return true; } - private boolean readCategorySwitch(FileTaskEntity task) { + @Override + public boolean readCategorySwitch(FileTaskEntity task) { try { JsonNode root = objectMapper.readTree(task.getResultJson()); JsonNode savedSwitch = root == null ? null : root.get("categorySwitch"); @@ -1874,7 +1308,8 @@ public class SimilarAsinTaskService { return task.getId(); } - private void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message) { + @Override + public void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message) { handleResultFileJobFailure(job, message); taskFileJobService.markFailureFinalized(job.getId(), message); } @@ -1889,7 +1324,7 @@ public class SimilarAsinTaskService { try (lockHandle) { // 直连模式：轮询器退化为"兜底调度器"，把还挂着 PENDING 的任务 // 重新调度一次批量提交（submitLlmBatch 同步直连），新任务本就走直连。 - schedulePendingLlmBatches(); + pipelineSupport().schedulePendingLlmBatches(); List states = ownershipSupport().listOwnedPendingLlmStates(); if (states == null || states.isEmpty()) { return; @@ -1905,579 +1340,16 @@ public class SimilarAsinTaskService { for (Map.Entry> entry : stateIdsByTaskId.entrySet()) { Long taskId = entry.getKey(); List stateIds = new ArrayList<>(entry.getValue()); - taskQueueExecutor.execute(() -> resubmitPendingLlmStates(taskId, stateIds)); + taskQueueExecutor.execute(() -> pipelineSupport().resubmitPendingLlmStates(taskId, stateIds)); } } } - - /** - * 直连模式兜底调度：把已封口（提交完成）但仍有 PENDING 状态的任务重新调度一次 - * 批量提交，新批次走 submitLlmBatch 同步直连，由提交路径落 DONE 缓冲/merge。 - */ - private void schedulePendingLlmBatches() { - List states = ownershipSupport().listOwnedPendingLlmStates(); - if (states == null || states.isEmpty()) { - return; - } - Set taskIds = new LinkedHashSet<>(); - for (TaskScopeStateEntity state : states) { - if (state != null && state.getTaskId() != null) { - taskIds.add(state.getTaskId()); - } - } - log.info("[similar-asin] direct-llm poll fallback scheduling pending tasks count={}", - taskIds.size()); - for (Long taskId : taskIds) { - taskQueueExecutor.execute(() -> { - TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(taskId, 0L); - if (taskLockHandle == null) { - return; - } - try (taskLockHandle) { - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { - return; - } - FileResultEntity result = findOrCreateResultRecordForAssembly(task, payloadSupport().allRowCount(task)); - TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( - task.getId(), MODULE_TYPE, result.getId(), ownershipSupport().buildTaskOwnerScopeKey(task)); - if (job == null || "SUCCESS".equals(job.getStatus())) { - return; - } - List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, task.getId()) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getChunkIndex)); - submitLlmBatches(task, result, job, chunks, loadAllRowsByBaseId(task)); - } - }); - } - } - - private void resubmitPendingLlmStates(Long taskId, List stateIds) { - if (taskId == null || stateIds == null || stateIds.isEmpty()) { - return; - } - TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(taskId, 0L); - if (taskLockHandle == null) { - return; - } - try (taskLockHandle) { - for (Long stateId : stateIds) { - if (stateId == null) { - continue; - } - TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId); - if (state == null) { - continue; - } - try { - submitLlmBatchForPendingState(state); - } catch (Exception ex) { - log.warn("[similar-asin] llm pending state resubmit failed taskId={} stateId={} err={}", - taskId, stateId, firstNonBlank(ex.getMessage(), ex.getClass().getSimpleName())); - } - } - } - } - private boolean submitLlmBatches(FileTaskEntity task, - FileResultEntity result, - TaskFileJobEntity job, - List chunks, - Map> allRowsByBaseId) { - if (chunks == null || chunks.isEmpty()) { - return countPendingLlmStates(task.getId()) > 0; - } - String prompt = payloadSupport().readAiPrompt(task); - String apiKey = payloadSupport().readApiKey(task); - boolean imgSwitch = payloadSupport().readImgSwitch(task); - boolean categorySwitch = readCategorySwitch(task); - int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, imgSwitch); - // P1-1：检测到 720712008 风暴时强制把 batch 降到 1，隔离毒行； - // 持续 5+ 次提交命中率 ≥ 40% 才会触发，正常波动不影响吞吐。 - if (poisonTracker.isStormActive(task.getId())) { - log.warn("[similar-asin] poison-storm detected, force batchSize=1 taskId={} originalBatchSize={}", - task.getId(), batchSize); - batchSize = 1; - } - List candidates = collectPendingLlmCandidates(chunks, allRowsByBaseId); - if (candidates.isEmpty()) { - return countPendingLlmStates(task.getId()) > 0; - } - // P1-2：把"仅过滤 hasImageUrl"扩展为必填字段集中校验。 - // 缺失 asin / title / 图片 url 任意一项即直接 markFailed，不进入 LLM 提交链路。 - // 原因：Python 端偶发空字段会触发 720701002 "fields cannot be extracted from null values"， - // 浪费 LLM 配额且把整 batch 拖垮；提前过滤更显式、易于排错。 - // 不打算把"必填字段集合"做成 properties——LLM 工作流签名固定，过度可配置反而把错配藏起来。 - java.util.function.Predicate isMissingRequired = row -> - row == null - || !row.hasImageUrl() - || normalize(row.getAsin()).isBlank() - || normalize(row.getTitle()).isBlank(); - List missingFieldCandidates = candidates.stream() - .filter(candidate -> candidate != null && isMissingRequired.test(candidate.row())) - .toList(); - if (!missingFieldCandidates.isEmpty()) { - mergeLlmRowsIntoChunk(task, - null, - null, - markRowsFailed(missingFieldCandidates.stream().map(LlmCandidate::row).toList(), - "required field missing (asin/title/url), skip LLM"), - allRowsByBaseId); - log.warn("[similar-asin] skip llm rows missing required fields taskId={} jobId={} rows={}", - task.getId(), job.getId(), missingFieldCandidates.size()); - } - List readyCandidates = candidates.stream() - .filter(candidate -> candidate != null && !isMissingRequired.test(candidate.row())) - .toList(); - boolean flushRemainder = isResultSubmissionComplete(task.getId()); - // P1-6: 防止 Python 端长时间慢回传时零头永久挂着：job.updatedAt 距今 ≥ llmFlushPendingMinutes 分钟则强制 flush。 - if (!flushRemainder && readyCandidates.size() > 0) { - LocalDateTime jobUpdatedAt = job.getUpdatedAt(); - long pendingFlushMillis = SimilarAsinLimits.llmFlushPendingMillis(properties); - if (jobUpdatedAt != null - && Duration.between(jobUpdatedAt, LocalDateTime.now()).toMillis() >= pendingFlushMillis) { - flushRemainder = true; - log.warn("[similar-asin] llm batch flush triggered by stale timer taskId={} jobId={} pendingRows={} batchSize={} jobUpdatedAt={} flushAfterMillis={}", - task.getId(), job.getId(), readyCandidates.size(), batchSize, jobUpdatedAt, pendingFlushMillis); - } - } - int submitLimit = (readyCandidates.size() / batchSize) * batchSize; - if (flushRemainder && submitLimit < readyCandidates.size()) { - submitLimit = readyCandidates.size(); - } - if (submitLimit <= 0) { - log.info("[similar-asin] llm batch waiting for more rows taskId={} jobId={} pendingRows={} batchSize={} finalUpload={}", - task.getId(), job.getId(), readyCandidates.size(), batchSize, flushRemainder); - return countPendingLlmStates(task.getId()) > 0; - } - boolean pending = false; - int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize); - int batchIndex = 1; - for (int i = 0; i < submitLimit; i += batchSize) { - List batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit)); - pending |= submitLlmBatchEntry(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId); - batchIndex++; - } - return pending || countPendingLlmStates(task.getId()) > 0; - } - - private List enrichRowsForLlm(List rows, - Map> allRowsByBaseId) { - if (rows == null || rows.isEmpty()) { - return List.of(); - } - List enrichedRows = new ArrayList<>(rows.size()); - for (SimilarAsinResultRowDto row : rows) { - enrichedRows.add(enrichRowForLlm(row, allRowsByBaseId)); - } - return enrichedRows; - } - - private SimilarAsinResultRowDto enrichRowForLlm(SimilarAsinResultRowDto row, - Map> allRowsByBaseId) { - if (row == null || allRowsByBaseId == null || allRowsByBaseId.isEmpty()) { - return row; - } - SimilarAsinParsedRowVo parsedRow = findParsedRow(row, allRowsByBaseId); - if (parsedRow == null) { - return row; - } - // 不再回填 url：Python 端会同时回传 url（主图）与 urls（同类商品图）， - // 缺失场景应在 Python 侧定位，Java 不再合成新的 url 字段以避免覆盖原始数据。 - if (normalize(row.getTitle()).isBlank()) { - row.setTitle(parsedRow.getTitle()); - } - if (normalize(row.getSku()).isBlank()) { - row.setSku(parsedRow.getSku()); - } - if (normalize(row.getPrice()).isBlank()) { - row.setPrice(parsedRow.getPrice()); - } - if (normalize(row.getCountry()).isBlank()) { - row.setCountry(parsedRow.getCountry()); - } - if (normalize(row.getId()).isBlank()) { - row.setId(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId())); - } - if (normalize(row.getRowToken()).isBlank()) { - row.setRowToken(parsedRow.getRowToken()); - } - if (normalize(row.getGroupKey()).isBlank()) { - row.setGroupKey(parsedRow.getGroupKey()); - } - return row; - } - - private SimilarAsinParsedRowVo findParsedRow(SimilarAsinResultRowDto row, - Map> allRowsByBaseId) { - if (row == null || allRowsByBaseId == null || allRowsByBaseId.isEmpty()) { - return null; - } - String groupKey = normalize(row.getGroupKey()); - List candidates = !groupKey.isBlank() - ? allRowsByBaseId.getOrDefault(groupKey, List.of()) - : List.of(); - SimilarAsinParsedRowVo matched = findParsedRowInCandidates(row, candidates); - if (matched != null) { - return matched; - } - for (List rows : allRowsByBaseId.values()) { - matched = findParsedRowInCandidates(row, rows); - if (matched != null) { - return matched; - } - } - return null; - } - - private SimilarAsinParsedRowVo findParsedRowInCandidates(SimilarAsinResultRowDto row, - List candidates) { - if (row == null || candidates == null || candidates.isEmpty()) { - return null; - } - String rowToken = normalize(row.getRowToken()); - String resultKey = SimilarAsinChunkMergeSupport.rowKey(row); - for (SimilarAsinParsedRowVo candidate : candidates) { - if (candidate == null) { - continue; - } - if (!rowToken.isBlank() && rowToken.equals(normalize(candidate.getRowToken()))) { - return candidate; - } - if (!resultKey.isBlank() && resultKey.equals(SimilarAsinChunkMergeSupport.rowKey(candidate))) { - return candidate; - } - } - String asin = normalize(row.getAsin()).toUpperCase(Locale.ROOT); - String country = normalize(row.getCountry()); - if (asin.isBlank()) { - return candidates.getFirst(); - } - for (SimilarAsinParsedRowVo candidate : candidates) { - if (asin.equals(normalize(candidate.getAsin()).toUpperCase(Locale.ROOT)) - && (country.isBlank() || country.equals(normalize(candidate.getCountry())))) { - return candidate; - } - } - return candidates.getFirst(); - } - - private boolean submitLlmBatchEntry(FileTaskEntity task, - FileResultEntity result, - TaskFileJobEntity job, - List batchCandidates, - int batchIndex, - int batchTotal, - String prompt, - String apiKey, - boolean imgSwitch, - boolean categorySwitch, - Map> allRowsByBaseId) { - if (batchCandidates == null || batchCandidates.isEmpty()) { - return false; - } - List batchRows = batchCandidates.stream() - .map(LlmCandidate::row) - .filter(Objects::nonNull) - .toList(); - if (batchRows.isEmpty()) { - return false; - } - taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); - String batchScopeKey = buildLlmBatchScopeKey(task.getId(), batchRows); - String batchScopeHash = DigestUtil.sha256Hex(batchScopeKey); - TaskScopeStateEntity existing = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, task.getId()) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .eq(TaskScopeStateEntity::getScopeHash, batchScopeHash) - .last("limit 1")); - if (existing != null) { - return LLM_STATUS_SUBMITTED.equals(existing.getLlmStatus()) - || LLM_STATUS_RUNNING.equals(existing.getLlmStatus()); - } - // 直连 LLM 模式：同步跑完行级链路后落 DONE 缓冲/merge。 - return submitLlmBatch(task, result, job, batchRows, batchScopeKey, batchScopeHash, - batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId); - } - - /** - * 直连 LLM 模式（directLlmEnabled=true）下的批提交：跳过工作流中转， - * 由 SimilarAsinLlmService 逐行跑完整链路（拼图/合规/对比），成功后按 - * 原同步 immediate DONE 结果路径集成：scope 去重 → 缓冲或立即 merge。 - * 行级失败信息经空结果检测保留，与同步提交失败行为对齐。 - */ - private boolean submitLlmBatch(FileTaskEntity task, - FileResultEntity result, - TaskFileJobEntity job, - List batchRows, - String batchScopeKey, - String batchScopeHash, - int batchIndex, - int batchTotal, - String prompt, - String apiKey, - boolean imgSwitch, - boolean categorySwitch, - Map> allRowsByBaseId) { - List llmRows; - try { - llmRows = SecretUsageContext.call(task.getUserId(), UserSecretModule.SIMILAR_ASIN.key(), - () -> similarAsinLlmService.inspectRows(batchRows, prompt, apiKey, imgSwitch, categorySwitch)); - } catch (Exception ex) { - String message = firstNonBlank(ex.getMessage(), "LLM submit failed"); - log.warn("[similar-asin] llm submit failed taskId={} jobId={} rows={} batch={}/{} err={}", - task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message); - poisonTracker.recordSubmitOutcome(task.getId(), SimilarAsinPoisonTracker.isPoisonRow(message)); - mergeLlmRowsIntoChunk(task, - null, - null, - markRowsFailed(batchRows, message), - allRowsByBaseId); - return false; - } - if (llmRows == null || llmRows.isEmpty()) { - String message = "LLM submit returned empty result rows"; - log.warn("[similar-asin] llm submit empty taskId={} jobId={} rows={} batch={}/{}", - task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal); - mergeLlmRowsIntoChunk(task, - null, - null, - markRowsFailed(batchRows, message), - allRowsByBaseId); - return false; - } - String emptyResultMessage = emptyLlmResultMessage(llmRows, batchRows.size()); - if (!emptyResultMessage.isBlank()) { - mergeLlmRowsIntoChunk(task, - null, - null, - markRowsFailed(batchRows, emptyResultMessage), - allRowsByBaseId); - return false; - } - // 落一条 DONE state 承载缓冲 pointer（对齐直连同步 immediate 路径）； - // 缓冲失败/关闭时回退立即 merge，结果不丢失。 - if (SimilarAsinLimits.isLlmResultBufferEnabled(properties)) { - TaskScopeStateEntity doneState = persistImmediateLlmDoneState(task, result, job, batchRows, - batchScopeKey, batchScopeHash, batchIndex, batchTotal, "llm-direct"); - if (doneState != null) { - bufferLlmRowsOrMerge(doneState, SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, doneState), llmRows, task, allRowsByBaseId); - return false; - } - } - mergeLlmRowsIntoChunk(task, null, null, llmRows, allRowsByBaseId); - return false; - } - - /** - * 直连模式下清存量 PENDING 状态：把该 state 的批次载荷重跑一遍直连 LLM， - * 结果落 DONE 缓冲/merge，由 maybeFinalizeLlmJob 触发收尾，最后把 state 置为终态。 - * 返回 true 表示本批次已被本轮处理完。 - */ - private boolean submitLlmBatchForPendingState(TaskScopeStateEntity state) { - if (state == null || state.getId() == null || state.getTaskId() == null) { - return false; - } - FileTaskEntity task = taskForPoll(state.getTaskId()); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { - return false; - } - List batchRows = readLlmBatchRows(state); - if (batchRows == null || batchRows.isEmpty()) { - markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 批次载荷缺失"); - maybeFinalizeLlmJob(state.getTaskId(), SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state)); - return true; - } - LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); - String prompt = payloadSupport().readAiPrompt(task); - String apiKey = payloadSupport().readApiKey(task); - boolean imgSwitch = payloadSupport().readImgSwitch(task); - boolean categorySwitch = readCategorySwitch(task); - boolean submitted = submitLlmBatch(task, null, taskFileJobService.findById( - context == null ? null : context.jobId()), - batchRows, state.getScopeKey(), state.getScopeHash(), - context == null ? 1 : context.batchIndex(), - context == null ? 1 : context.batchTotal(), - prompt, apiKey, imgSwitch, categorySwitch, - allRowsByBaseIdForPoll(task)); - if (submitted) { - return true; - } - // 直连重跑未真正提交（提交异常已被 submitLlmBatch 内部消化为失败 merge）： - // 直接把 state 置为终态并触发收尾，避免 PENDING 永远挂着。 - markLlmStateTerminal(state, LLM_STATUS_FAILED, "直连模式重跑批次失败"); - maybeFinalizeLlmJob(state.getTaskId(), context); - return true; - } - - private void finalizeTimedOutLlmStatesForTask(Long taskId) { - if (taskId == null || taskId <= 0) { - return; - } - List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)) - .orderByAsc(TaskScopeStateEntity::getLlmSubmittedAt) - .last("limit 50")); - if (states == null || states.isEmpty()) { - return; - } - for (TaskScopeStateEntity state : states) { - if (state == null || state.getId() == null || !isLlmStateTimedOut(state)) { - continue; - } - LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); - if (context == null || context.resultId() == null) { - markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 批次上下文缺失"); - continue; - } - List batchRows = readLlmBatchRows(state); - FileTaskEntity task = taskForPoll(taskId); - if (task != null) { - Map> allRowsByBaseId = allRowsByBaseIdForPoll(task); - mergeLlmRowsIntoChunk(task, - context.chunkScopeHash(), - context.chunkIndex(), - markRowsFailed(batchRows, "LLM 异步工作流轮询超时"), - allRowsByBaseId); - } - markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 异步工作流轮询超时"); - maybeFinalizeLlmJob(taskId, context); - log.warn("[similar-asin] 文件任务超时兜底已将 LLM pending 批次置为失败 taskId={} stateId={} jobId={}", - taskId, state.getId(), context.jobId()); - } - } - - /** - * P0-4 / P2-9：pending 重跑链路直接查 DB 取 task（poll 链移除后无同线程复用上下文）。 - */ - private FileTaskEntity taskForPoll(Long taskId) { - if (taskId == null) { - return null; - } - return fileTaskMapper.selectById(taskId); - } - - /** - * P0-4：pending 重跑链路直接加载 allRowsByBaseId（poll 链移除后无同线程复用上下文）。 - */ - private Map> allRowsByBaseIdForPoll(FileTaskEntity task) { - if (task == null) { - return Map.of(); - } - return loadAllRowsByBaseId(task); - } - - private String emptyLlmResultMessage(List llmRows, int expectedRows) { - if (llmRows == null || llmRows.isEmpty()) { - return expectedRows > 0 ? "LLM async workflow returned empty result rows" : ""; - } - long unresolved = llmRows.stream() - .filter(row -> row != null - && !SimilarAsinResultTextSupport.hasResolvedLlmFields(row) - && !SimilarAsinResultTextSupport.isTechnicalLlmFailure(row.getError())) - .count(); - if (unresolved <= 0) { - return ""; - } - return "LLM async workflow returned empty result rows: " + unresolved + "/" + Math.max(expectedRows, llmRows.size()); - } - - private void markLlmStateTerminal(TaskScopeStateEntity state, String status, String error) { - taskScopeStateMapper.update(null, new LambdaUpdateWrapper() - .eq(TaskScopeStateEntity::getId, state.getId()) - .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)) - .set(TaskScopeStateEntity::getLlmStatus, status) - .set(TaskScopeStateEntity::getLlmCompletedAt, LocalDateTime.now()) - .set(TaskScopeStateEntity::getLlmLastPolledAt, LocalDateTime.now()) - .set(TaskScopeStateEntity::getLlmAttemptCount, llmAttemptCount(state) + 1) - .set(TaskScopeStateEntity::getLlmError, error) - .set(TaskScopeStateEntity::getCompleted, 1) - .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); - } - - private void maybeFinalizeLlmJob(Long taskId, LlmBatchContext context) { - if (taskId == null || context == null || countPendingLlmStates(taskId) > 0) { - return; - } - if (!ownershipSupport().isOwnerCurrent(context.ownerInstanceId())) { - return; - } - TaskDistributedLockService.LockHandle lockHandle = ownershipSupport().acquireTaskLock(taskId, 0L); - if (lockHandle == null) { - return; - } - try (lockHandle) { - maybeFinalizeLlmJobLocked(taskId, context); - } catch (Exception ex) { - TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); - if (job != null) { - taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "相似ASIN结果文件生成失败")); - if (taskFileJobService.isRetryExhausted(job.getId())) { - finalizeExhaustedResultFileJob(job, ex.getMessage()); - } - } - log.warn("[相似ASIN] LLM 异步收尾失败 任务ID={} 结果ID={} 错误={}", - taskId, context.resultId(), ex.getMessage(), ex); - } - } - - private void maybeFinalizeLlmJobLocked(Long taskId, LlmBatchContext context) { - if (taskId == null || context == null || countPendingLlmStates(taskId) > 0) { - return; - } - DistributedJobLockService.LockHandle lockHandle = - distributedJobLockService.tryLock("similar-asin:llm-finalize:" + taskId, Duration.ofMinutes(5)); - if (lockHandle == null) { - return; - } - try (lockHandle) { - if (countPendingLlmStates(taskId) > 0) { - return; - } - TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); - if (job == null || "SUCCESS".equals(job.getStatus())) { - return; - } - if (taskFileJobService.isRetryExhausted(job.getId())) { - finalizeExhaustedResultFileJob(job, job.getErrorMessage()); - return; - } - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) { - taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); - touchJavaSideTaskActivity(taskId); - return; - } - // P0-3：requeue assemble 之前一次性把缓冲的 llmRows 合并到 chunk。 - // 失败时 markFailed job 并阻止 requeue，避免 assemble 阶段读到不完整 chunk。 - try { - flushLlmBufferedResults(taskId); - } catch (Exception flushEx) { - String message = firstNonBlank(flushEx.getMessage(), "刷新缓冲区 LLM 结果失败"); - log.warn("[相似ASIN] LLM 收尾刷新缓冲区失败，已标记文件任务失败 任务ID={} 文件任务ID={} 错误={}", - taskId, job.getId(), message, flushEx); - taskFileJobService.markFailed(job, message); - return; - } - boolean requeued = taskFileJobService.requeue(job.getId(), "LLM 结果已就绪，正在组装 xlsx"); - if (requeued) { - log.info("[相似ASIN] LLM 异步结果已就绪，结果文件任务已重新入队 任务ID={} 文件任务ID={} 结果ID={}", - taskId, job.getId(), context.resultId()); - } else if (taskFileJobService.isRetryExhausted(job.getId())) { - finalizeExhaustedResultFileJob(job, job.getErrorMessage()); - } - } - } - private void completeLlmFileJob(FileTaskEntity task, FileResultEntity result, TaskFileJobEntity job, int totalProgressUnits, int llmWorkUnits) { - if (countPendingLlmStates(task.getId()) > 0) { + if (pipelineSupport().countPendingLlmStates(task.getId()) > 0) { throw new BusinessException("LLM result is still processing, cannot generate result file yet"); } int assembleProgress = Math.max(1, Math.min(totalProgressUnits - 2, llmWorkUnits)); @@ -2501,562 +1373,6 @@ public class SimilarAsinTaskService { poisonTracker.clear(task.getId()); } - private void mergeLlmRowsIntoChunk(FileTaskEntity task, - String chunkScopeHash, - Integer chunkIndex, - List llmRows, - Map> allRowsByBaseId) { - if (task == null || llmRows == null || llmRows.isEmpty()) { - return; - } - List chunks = loadSubmittedChunks(task.getId()); - if (chunks.isEmpty()) { - return; - } - // P2-11：把当前 batch 命中的图片 url 异步丢入预热队列。 - // 预热失败不影响主流程，assemble 阶段无 DB cache 命中也会走原下载链路兜底。 - try { - List prefetchUrls = new ArrayList<>(llmRows.size() * 3); - for (SimilarAsinResultRowDto llmRow : llmRows) { - if (llmRow == null) { - continue; - } - SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getMainUrl()); - SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getPuzzleImg1()); - SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getPuzzleImg2()); - } - imagePrefetchService.enqueue(task.getId(), prefetchUrls); - } catch (Exception ex) { - // 预热入队是 best-effort，任何异常都不能阻断 merge 主路径。 - log.debug("[similar-asin] enqueue prefetch failed taskId={} err={}", task.getId(), ex.getMessage()); - } - Map> rowsByChunk = new LinkedHashMap<>(); - Map chunkByKey = new LinkedHashMap<>(); - for (TaskChunkEntity chunk : chunks) { - String chunkKey = SimilarAsinChunkMergeSupport.chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()); - rowsByChunk.put(chunkKey, chunkPayloadSupport().readChunkRows(chunk)); - chunkByKey.put(chunkKey, chunk); - } - // Task 11：合并前按稳定 rowKey 去重，消除重复行逐行 expand/分配/写回 的 O(n²) 热点。 - List uniqueRows = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); - List expandedAll = new ArrayList<>(); - for (SimilarAsinResultRowDto resultRow : uniqueRows) { - expandedAll.addAll(SimilarAsinChunkMergeSupport.expandRows(List.of(resultRow), allRowsByBaseId)); - } - // Task 10：先建 rowKey → chunkKey 批量索引，把逐 chunk 线性扫描降为 O(1) 查找。 - Map rowKeyIndex = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); - List orphanRows = new ArrayList<>(); - Map> mergeRowsByChunk = - SimilarAsinChunkMergeSupport.assignLlmRowsToChunks(rowsByChunk, expandedAll, rowKeyIndex, chunkScopeHash, chunkIndex, orphanRows); - if (!orphanRows.isEmpty()) { - chunkPayloadSupport().persistOrphanLlmRows(task.getId(), orphanRows); - } - for (Map.Entry> entry : mergeRowsByChunk.entrySet()) { - TaskChunkEntity chunk = chunkByKey.get(entry.getKey()); - if (chunk == null || entry.getValue().isEmpty()) { - continue; - } - mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), new ArrayList<>(entry.getValue().values())); - } - } - - private List loadSubmittedChunks(Long taskId) { - if (taskId == null || taskId <= 0) { - return List.of(); - } - List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getChunkIndex)); - if (chunks == null || chunks.isEmpty()) { - return List.of(); - } - return chunks.stream() - .filter(Objects::nonNull) - .filter(chunk -> !chunkPayloadSupport().readChunkRows(chunk).isEmpty()) - .toList(); - } - - private List markRowsFailed(List rows, String failureMessage) { - if (rows == null || rows.isEmpty()) { - return List.of(); - } - return rows.stream() - .map(this::copyRowForFailure) - .map(row -> markRowFailed(row, failureMessage)) - .toList(); - } - - private SimilarAsinResultRowDto copyRowForFailure(SimilarAsinResultRowDto source) { - SimilarAsinResultRowDto row = new SimilarAsinResultRowDto(); - row.setSourceFileKey(source.getSourceFileKey()); - row.setSourceFilename(source.getSourceFilename()); - row.setRowToken(source.getRowToken()); - row.setGroupKey(source.getGroupKey()); - row.setId(source.getId()); - row.setAsin(source.getAsin()); - row.setCountry(source.getCountry()); - row.setSku(source.getSku()); - row.setPrice(source.getPrice()); - row.setUrls(source.getUrls()); - row.setAlibaba(source.getAlibaba()); - row.setTitle(source.getTitle()); - row.setError(source.getError()); - row.setDone(source.getDone()); - row.setStatus(source.getStatus()); - row.setIsConform(source.getIsConform()); - row.setReason(source.getReason()); - row.setCategory(source.getCategory()); - row.setTitleRisk(source.getTitleRisk()); - row.setAppearanceRisk(source.getAppearanceRisk()); - row.setPatentRisk(source.getPatentRisk()); - row.setConclusion(source.getConclusion()); - row.setIsStock(source.getIsStock()); - row.setSimilarity(source.getSimilarity()); - row.setTitleReason(source.getTitleReason()); - row.setAppearanceReason(source.getAppearanceReason()); - row.setPatentReason(source.getPatentReason()); - row.setMainUrl(source.getMainUrl()); - row.setPuzzleImg1(source.getPuzzleImg1()); - row.setPuzzleImg2(source.getPuzzleImg2()); - return row; - } - - private SimilarAsinResultRowDto markRowFailed(SimilarAsinResultRowDto row, String failureMessage) { - String reviewMessage = failureMessage == null || failureMessage.isBlank() - ? "检测失败" - : "检测失败：" + failureMessage; - if (row.getError() == null || row.getError().isBlank()) { - row.setError(failureMessage); - } - if (row.getReason() == null || row.getReason().isBlank()) { - row.setReason(failureMessage); - } - if (row.getStatus() == null || row.getStatus().isBlank()) { - row.setStatus("FAILED"); - } - if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) { - row.setTitleRisk(reviewMessage); - } - if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) { - row.setAppearanceRisk(reviewMessage); - } - if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) { - row.setPatentRisk(reviewMessage); - } - if (row.getConclusion() == null || row.getConclusion().isBlank()) { - row.setConclusion(failureMessage); - } - return row; - } - - private List readLlmBatchRows(TaskScopeStateEntity state) { - if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { - return List.of(); - } - try { - String payloadJson = transientPayloadStorageService.resolvePayload( - state.getParsedPayloadJson(), "read similar ASIN llm batch failed"); - JsonNode array = objectMapper.readTree(payloadJson); - if (!array.isArray()) { - return List.of(); - } - List rows = new ArrayList<>(); - for (JsonNode node : array) { - rows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); - } - return rows; - } catch (Exception ex) { - log.warn("[similar-asin] read llm batch failed taskId={} stateId={} err={}", - state.getTaskId(), state.getId(), ex.getMessage()); - return List.of(); - } - } - - /** - * P0-3：读取 state 缓冲的 llmRows。 - * 与 readLlmBatchRows（读输入 batchRows）不同，这里读的是 - * 通过 bufferLlmResultForFlush 写入 transient storage 的 DONE 结果。 - */ - private List readLlmBufferedRows(String resultPayloadPointer) { - if (resultPayloadPointer == null || resultPayloadPointer.isBlank()) { - return List.of(); - } - try { - String payloadJson = transientPayloadStorageService.resolvePayload( - resultPayloadPointer, "read similar ASIN llm result buffer failed"); - if (payloadJson == null || payloadJson.isBlank()) { - return List.of(); - } - JsonNode array = objectMapper.readTree(payloadJson); - if (!array.isArray()) { - return List.of(); - } - List rows = new ArrayList<>(); - for (JsonNode node : array) { - rows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); - } - return rows; - } catch (Exception ex) { - log.warn("[similar-asin] read llm result buffer failed pointer={} err={}", - resultPayloadPointer, ex.getMessage()); - return List.of(); - } - } - - /** - * P0-3：把单个 DONE batch 的 llmRows 缓冲到 transient storage（不立即写 chunk）。 - * 返回更新后的 LlmBatchContext（含 resultPayloadPointer），调用方需写回 stateJson。 - * 缓冲失败时返回原 context，调用方应回退到立即 merge 路径。 - */ - private LlmBatchContext bufferLlmResultForFlush(TaskScopeStateEntity state, - LlmBatchContext context, - List llmRows) { - if (state == null || context == null || llmRows == null || llmRows.isEmpty()) { - return context; - } - try { - String payloadJson = writeJson(llmRows, "serialize llm result buffer failed"); - String pointer = transientPayloadStorageService.storeParsedPayloadEntry( - MODULE_TYPE, state.getTaskId(), state.getScopeHash(), - "llm-result-" + state.getId(), payloadJson, true); - if (pointer == null || pointer.isBlank()) { - return context; - } - LlmBatchContext bufferedContext = withResultPayloadPointer(context, pointer); - String updatedStateJson = writeJson(bufferedContext, "serialize llm result buffer context failed"); - int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper() - .eq(TaskScopeStateEntity::getId, state.getId()) - .set(TaskScopeStateEntity::getStateJson, updatedStateJson) - .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); - if (updated <= 0) { - transientPayloadStorageService.deletePayloadIfPresent(pointer); - return context; - } - state.setStateJson(updatedStateJson); - return bufferedContext; - } catch (Exception ex) { - log.warn("[similar-asin] buffer llm result failed taskId={} stateId={} err={}", - state.getTaskId(), state.getId(), ex.getMessage()); - return context; - } - } - - /** - * P0-3：清掉 state 上缓冲的 pointer + 删除 transient payload。 - * 在 flush 成功合并到 chunk 后调用。 - */ - private void clearLlmBufferedResult(TaskScopeStateEntity state, LlmBatchContext context) { - if (state == null || context == null) { - return; - } - String pointer = context.resultPayloadPointer(); - if (pointer == null || pointer.isBlank()) { - return; - } - try { - LlmBatchContext clearedContext = withResultPayloadPointer(context, null); - String updatedStateJson = writeJson(clearedContext, "serialize llm result buffer cleared context failed"); - taskScopeStateMapper.update(null, new LambdaUpdateWrapper() - .eq(TaskScopeStateEntity::getId, state.getId()) - .set(TaskScopeStateEntity::getStateJson, updatedStateJson) - .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); - state.setStateJson(updatedStateJson); - transientPayloadStorageService.deletePayloadIfPresent(pointer); - } catch (Exception ex) { - log.warn("[similar-asin] clear llm result buffer failed taskId={} stateId={} err={}", - state.getTaskId(), state.getId(), ex.getMessage()); - } - } - - /** - * Task 12：统一 LLM DONE 结果落库入口。 - * 缓冲开关开启时把 llmRows 写入 transient storage（pointer 存进 state.stateJson）， - * 由 flushLlmBufferedResults 在 finalize/assemble 前一次性合并到 chunk； - * 缓冲失败（存储异常 / state 更新失败 / 开关关闭）回退立即 merge，结果不丢失。 - * 空 rows / 空 state / 空 context 直接返回，不产生任何写入。 - */ - private void bufferLlmRowsOrMerge(TaskScopeStateEntity state, - LlmBatchContext context, - List llmRows, - FileTaskEntity task, - Map> allRowsByBaseId) { - if (state == null || context == null || llmRows == null || llmRows.isEmpty()) { - return; - } - if (!SimilarAsinLimits.isLlmResultBufferEnabled(properties)) { - mergeLlmRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), llmRows, allRowsByBaseId); - return; - } - LlmBatchContext bufferedContext = bufferLlmResultForFlush(state, context, llmRows); - if (bufferedContext == null - || bufferedContext.resultPayloadPointer() == null - || bufferedContext.resultPayloadPointer().isBlank()) { - // 缓冲失败：回退立即 merge，避免结果悬挂在 transient storage 之外。 - mergeLlmRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), llmRows, allRowsByBaseId); - } - } - - /** - * Task 12：为 submit 同步 immediate DONE 结果落一条 DONE state 承载缓冲 pointer。 - * 与 saveLlmBatchState（SUBMITTED 异步）不同，该 state 直接以 DONE 终态插入， - * 不会被 countPendingLlmStates 扫描；缓冲失败/重复插入时返回 null，调用方回退立即 merge。 - */ - private TaskScopeStateEntity persistImmediateLlmDoneState(FileTaskEntity task, - FileResultEntity result, - TaskFileJobEntity job, - List batchRows, - String batchScopeKey, - String batchScopeHash, - int batchIndex, - int batchTotal, - String credentialName) { - LocalDateTime now = LocalDateTime.now(); - LlmBatchContext context = new LlmBatchContext( - job.getId(), - result.getId(), - null, - null, - batchIndex, - batchTotal, - ownershipSupport().currentInstanceId(), - 0, - credentialName, - null - ); - TaskScopeStateEntity state = new TaskScopeStateEntity(); - state.setTaskId(task.getId()); - state.setModuleType(MODULE_TYPE); - state.setScopeKey(batchScopeKey); - state.setScopeHash(batchScopeHash); - state.setStateJson(writeJson(context, "serialize immediate llm done state context failed")); - state.setLlmStatus(LLM_STATUS_DONE); - state.setLlmSubmittedAt(now); - state.setLlmCompletedAt(now); - state.setLlmAttemptCount(0); - state.setChunkTotal(batchTotal); - state.setReceivedChunkCount(batchIndex); - state.setCompleted(1); - state.setCreatedAt(now); - state.setUpdatedAt(now); - try { - taskScopeStateMapper.insert(state); - return state; - } catch (DuplicateKeyException ex) { - log.info("[similar-asin] duplicate immediate done llm state ignored taskId={} scope={}", - task.getId(), batchScopeKey); - return null; - } - } - - /** - * P0-3：在 finalize 之前一次性把所有 DONE/FAILED state 上缓冲的 llmRows - * 按 chunkScopeHash 分组合并到 chunk。把每个 batch 的"loadSubmittedChunks + - * readChunkRows × N + writeChunkPayload × M"压缩成"读 chunk 一次 + - * 写 chunk 一次"。 - */ - private void flushLlmBufferedResults(Long taskId) { - if (taskId == null || taskId <= 0) { - return; - } - // 只取 flush 需要的列：parsed_payload_json/llm_* 等大字段不拉，避免跨库大结果集传输（每任务可达数百行大 JSON） - List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() - .select(TaskScopeStateEntity::getId, - TaskScopeStateEntity::getTaskId, - TaskScopeStateEntity::getModuleType, - TaskScopeStateEntity::getLlmStatus, - TaskScopeStateEntity::getStateJson) - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_DONE, LLM_STATUS_FAILED))); - if (states == null || states.isEmpty()) { - return; - } - List entries = new ArrayList<>(); - for (TaskScopeStateEntity state : states) { - LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); - if (context == null) { - continue; - } - String pointer = context.resultPayloadPointer(); - if (pointer == null || pointer.isBlank()) { - continue; - } - List rows = readLlmBufferedRows(pointer); - if (rows == null || rows.isEmpty()) { - // pointer 存在但 payload 已被清理（被 GC 或上一次 flush 部分成功），直接清 pointer。 - clearLlmBufferedResult(state, context); - continue; - } - entries.add(new BufferedFlushEntry(state, context, rows)); - } - if (entries.isEmpty()) { - return; - } - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { - return; - } - Map> allRowsByBaseId = loadAllRowsByBaseId(task); - // 按 chunkScopeHash 分组（空字符串占位 null，让 mergeLlmRowsIntoChunk 走全 chunk 扫描匹配） - Map> entriesByChunk = new LinkedHashMap<>(); - for (BufferedFlushEntry entry : entries) { - String key = entry.context().chunkScopeHash() == null ? "" : entry.context().chunkScopeHash(); - entriesByChunk.computeIfAbsent(key, ignored -> new ArrayList<>()).add(entry); - } - int totalFlushed = 0; - int failedGroups = 0; - for (Map.Entry> group : entriesByChunk.entrySet()) { - List groupEntries = group.getValue(); - List aggregated = new ArrayList<>(); - for (BufferedFlushEntry entry : groupEntries) { - aggregated.addAll(entry.rows()); - } - if (aggregated.isEmpty()) { - continue; - } - String chunkScopeHash = group.getKey().isEmpty() ? null : group.getKey(); - Integer chunkIndex = null; - if (chunkScopeHash != null) { - for (BufferedFlushEntry entry : groupEntries) { - if (entry.context().chunkIndex() != null) { - chunkIndex = entry.context().chunkIndex(); - break; - } - } - } - try { - mergeLlmRowsIntoChunk(task, chunkScopeHash, chunkIndex, aggregated, allRowsByBaseId); - for (BufferedFlushEntry entry : groupEntries) { - clearLlmBufferedResult(entry.state(), entry.context()); - } - totalFlushed += aggregated.size(); - } catch (Exception ex) { - failedGroups++; - log.warn("[similar-asin] flush buffered llm results group failed taskId={} chunkScopeHash={} err={}", - taskId, chunkScopeHash, ex.getMessage(), ex); - // 不 clear pointer，留待下次 finalize 或 stale-recovery 重试 - } - } - if (totalFlushed > 0 || failedGroups > 0) { - log.info("[similar-asin] flushed buffered llm results taskId={} entries={} totalRows={} chunkGroups={} failedGroups={}", - taskId, entries.size(), totalFlushed, entriesByChunk.size(), failedGroups); - } - if (failedGroups > 0) { - // 让 finalize 能感知到失败，调用方应避免继续 requeue assemble。 - throw new IllegalStateException("刷新缓冲区 LLM 结果失败，失败分组数=" + failedGroups); - } - } - - private record BufferedFlushEntry(TaskScopeStateEntity state, - LlmBatchContext context, - List rows) { - } - - - private LlmBatchContext withResultPayloadPointer(LlmBatchContext context, String resultPayloadPointer) { - return new LlmBatchContext( - context.jobId(), - context.resultId(), - context.chunkScopeHash(), - context.chunkIndex(), - context.batchIndex(), - context.batchTotal(), - context.ownerInstanceId(), - context.submitRetryCount(), - context.credentialName(), - resultPayloadPointer - ); - } - - private boolean isLlmStateTimedOut(TaskScopeStateEntity state) { - if (state == null || state.getLlmSubmittedAt() == null) { - return false; - } - long timeoutMillis = Math.max(10000L, properties.getStaleTimeoutMinutes() * 60_000L); - return Duration.between(state.getLlmSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis; - } - - private int llmAttemptCount(TaskScopeStateEntity state) { - return state == null || state.getLlmAttemptCount() == null ? 0 : state.getLlmAttemptCount(); - } - - private int countPendingLlmStates(Long taskId) { - if (taskId == null || taskId <= 0) { - return 0; - } - Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING))); - return count == null ? 0 : count.intValue(); - } - - private boolean isResultSubmissionComplete(Long taskId) { - if (taskId == null || taskId <= 0) { - return false; - } - List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) - .isNull(TaskScopeStateEntity::getLlmStatus) - .isNotNull(TaskScopeStateEntity::getLastChunkAt)); - if (states == null || states.isEmpty()) { - return false; - } - boolean hasCompletedScope = false; - for (TaskScopeStateEntity state : states) { - if (state == null) { - continue; - } - if (Integer.valueOf(1).equals(state.getCompleted())) { - hasCompletedScope = true; - continue; - } - return false; - } - return hasCompletedScope; - } - - private void touchJavaSideTaskActivity(Long taskId) { - if (taskId == null || taskId <= 0) { - return; - } - LocalDateTime now = LocalDateTime.now(); - LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, properties.getDbTaskTouchIntervalMillis()))); - fileTaskMapper.update(null, new LambdaUpdateWrapper() - .eq(FileTaskEntity::getId, taskId) - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .and(wrapper -> wrapper.isNull(FileTaskEntity::getUpdatedAt) - .or() - .le(FileTaskEntity::getUpdatedAt, cutoff)) - .set(FileTaskEntity::getUpdatedAt, now)); - } - - private String buildLlmBatchScopeKey(Long taskId, List batchRows) { - StringBuilder rowKeys = new StringBuilder(); - if (batchRows != null) { - for (SimilarAsinResultRowDto row : batchRows) { - String key = SimilarAsinChunkMergeSupport.rowKey(row); - if (!key.isBlank()) { - if (!rowKeys.isEmpty()) { - rowKeys.append('|'); - } - rowKeys.append(key); - } - } - } - return "llm:task:" + taskId + ":rows:" + DigestUtil.sha256Hex(rowKeys.toString()); - } - - record LlmCandidate(String chunkScopeHash, - Integer chunkIndex, - SimilarAsinResultRowDto row) { - } - - private record PythonUploadProgress(int current, int total, String unit) { - } public void cleanupResultFileJob(TaskFileJobEntity job) { if (job == null || job.getTaskId() == null) { @@ -3404,42 +1720,6 @@ public class SimilarAsinTaskService { return payloads; } - private record PreparedSubmittedChunk(Long taskId, - String scopeKey, - String scopeHash, - Integer chunkIndex, - Integer chunkTotal, - boolean done, - String error, - String payloadJson, - String storedPayload, - boolean localFallback, - SubmittedTaskMetadata taskMetadata, - String payloadHash) { - } - - private record PersistSubmittedChunkResult(SubmitContext context, - boolean payloadPersisted, - FinalizeTaskResult finalizeResult) { - } - - private record SubmitContext(FileTaskEntity task, - String scopeKey, - String scopeHash, - Integer chunkIndex, - boolean forceFlush, - String error, - SubmittedTaskMetadata taskMetadata) { - } - - private record SubmittedTaskMetadata(int rowCount, - List sourceFiles) { - } - - private record FinalizeTaskResult(Long taskId, - boolean terminal) { - } - public record ResultDownloadInfo(String url, String filename, String contentType, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/FinalizeTaskResult.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/FinalizeTaskResult.java new file mode 100644 index 00000000..2bb5bd13 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/FinalizeTaskResult.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +/** + * 任务收尾结果：taskId 与是否已达终态。 + * spec 05 续（拆分第 7 轮）：从 SimilarAsinTaskService 原样搬出为顶层 record。 + */ +public record FinalizeTaskResult(Long taskId, + boolean terminal) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PersistSubmittedChunkResult.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PersistSubmittedChunkResult.java new file mode 100644 index 00000000..91b38893 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PersistSubmittedChunkResult.java @@ -0,0 +1,10 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +/** + * 分片落库结果：提交上下文、payload 是否落库、以及收尾结果。 + * spec 05 续（拆分第 7 轮）：从 SimilarAsinTaskService 原样搬出为顶层 record。 + */ +public record PersistSubmittedChunkResult(SubmitContext context, + boolean payloadPersisted, + FinalizeTaskResult finalizeResult) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PreparedSubmittedChunk.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PreparedSubmittedChunk.java new file mode 100644 index 00000000..527aee7d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/PreparedSubmittedChunk.java @@ -0,0 +1,19 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +/** + * 预处理后的待落库分片：行裁剪/校验后的最小写入单元。 + * spec 05 续（拆分第 7 轮）：从 SimilarAsinTaskService 原样搬出为顶层 record。 + */ +public record PreparedSubmittedChunk(Long taskId, + String scopeKey, + String scopeHash, + Integer chunkIndex, + Integer chunkTotal, + boolean done, + String error, + String payloadJson, + String storedPayload, + boolean localFallback, + SubmittedTaskMetadata taskMetadata, + String payloadHash) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java new file mode 100644 index 00000000..ff3580f0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java @@ -0,0 +1,31 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; + +/** + * spec 05 续（拆分第 7 轮）：Python 回传 → 分片落库 → LLM 检测流水线需要宿主服务提供的能力。 + *

+ * 采用依赖倒置：由宿主（SimilarAsinTaskService）实现本接口，流水线只依赖接口， + * 避免 support 包与 Service 之间形成循环依赖。这些方法保留在宿主的两个原因： + * 一是属于任务/结果文件任务的编排与事务边界（{@code handleResultFileJobFailure} 带 @Transactional）， + * 二是被宿主自身的门面路径复用。 + */ +public interface SimilarAsinPipelineHost { + + /** 任务收尾状态机（成功/失败判定、结果文件装配入队、任务状态落库）。 */ + FinalizeTaskResult finalizeTask(FileTaskEntity task, + String error, + SubmittedTaskMetadata taskMetadata, + boolean assembleWorkbook); + + /** 取回或创建结果文件记录（用于装配阶段回填）。 */ + FileResultEntity findOrCreateResultRecordForAssembly(FileTaskEntity task, int rowCount); + + /** 读取任务的类目重试开关。 */ + boolean readCategorySwitch(FileTaskEntity task); + + /** 结果文件任务重试耗尽后的失败处理（宿主侧带事务边界）。 */ + void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java new file mode 100644 index 00000000..3f46952b --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java @@ -0,0 +1,1911 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import cn.hutool.crypto.digest.DigestUtil; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultGroupDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinLogSupport; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinImagePrefetchService; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.usersecret.support.SecretUsageContext; +import com.nanri.aiimage.modules.usersecret.support.UserSecretModule; +import org.springframework.dao.DuplicateKeyException; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.task.TaskExecutor; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +/** + * spec 05 續（拆分第 7 輪）：Python 結果回傳 → 分片落庫 → LLM 檢測流水線。 + *

+ * 從 {@code SimilarAsinTaskService} 原樣搬移 57 個方法，邏輯與搬移前逐行一致： + * 分片接收與合併（含 orphan 兜底）、LLM 批次派發/輪詢/重試與毒行熔斷、結果緩衝與 flush。 + *

+ * 對宿主服務的能力需求通過 {@link SimilarAsinPipelineHost} 依賴倒置，避免與 Service 循環依賴。 + */ +@Slf4j +public class SimilarAsinPipelineSupport { + + private static final String MODULE_TYPE = "SIMILAR_ASIN"; + private static final String LLM_STATUS_SUBMITTED = "SUBMITTED"; + private static final String LLM_STATUS_RUNNING = "RUNNING"; + private static final String LLM_STATUS_DONE = "DONE"; + private static final String LLM_STATUS_FAILED = "FAILED"; + private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5); + private static final String STATUS_PENDING = "PENDING"; + private static final String STATUS_RUNNING = "RUNNING"; + private static final String STATUS_SUCCESS = "SUCCESS"; + private static final String STATUS_FAILED = "FAILED"; + /** Task 19：Python 回传逐行日志采样频率（每 N 行记一行）。 */ + private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L; + + private final SimilarAsinProperties properties; + private final ObjectMapper objectMapper; + private final SimilarAsinLlmService similarAsinLlmService; + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + private final TaskFileJobService taskFileJobService; + private final DistributedJobLockService distributedJobLockService; + private final TaskExecutor taskQueueExecutor; + private final SimilarAsinImagePrefetchService imagePrefetchService; + private final SimilarAsinPoisonTracker poisonTracker; + private final SimilarAsinPayloadSupport payloadSupport; + private final SimilarAsinChunkPayloadSupport chunkPayloadSupport; + private final SimilarAsinTaskOwnershipSupport ownershipSupport; + private final SimilarAsinTaskProgressSupport progressSupport; + private final SimilarAsinPipelineHost host; + + private record BufferedFlushEntry(TaskScopeStateEntity state, + LlmBatchContext context, + List rows) { + } + + record LlmCandidate(String chunkScopeHash, + Integer chunkIndex, + SimilarAsinResultRowDto row) { + } + + private record PythonUploadProgress(int current, int total, String unit) { + } + + public SimilarAsinPipelineSupport(SimilarAsinProperties properties, + ObjectMapper objectMapper, + SimilarAsinLlmService similarAsinLlmService, + FileTaskMapper fileTaskMapper, + FileResultMapper fileResultMapper, + TaskChunkMapper taskChunkMapper, + TaskScopeStateMapper taskScopeStateMapper, + TransientPayloadStorageService transientPayloadStorageService, + TaskFileJobService taskFileJobService, + DistributedJobLockService distributedJobLockService, + TaskExecutor taskQueueExecutor, + SimilarAsinImagePrefetchService imagePrefetchService, + SimilarAsinPoisonTracker poisonTracker, + SimilarAsinPayloadSupport payloadSupport, + SimilarAsinChunkPayloadSupport chunkPayloadSupport, + SimilarAsinTaskOwnershipSupport ownershipSupport, + SimilarAsinTaskProgressSupport progressSupport, + SimilarAsinPipelineHost host) { + this.properties = properties; + this.objectMapper = objectMapper; + this.similarAsinLlmService = similarAsinLlmService; + this.fileTaskMapper = fileTaskMapper; + this.fileResultMapper = fileResultMapper; + this.taskChunkMapper = taskChunkMapper; + this.taskScopeStateMapper = taskScopeStateMapper; + this.transientPayloadStorageService = transientPayloadStorageService; + this.taskFileJobService = taskFileJobService; + this.distributedJobLockService = distributedJobLockService; + this.taskQueueExecutor = taskQueueExecutor; + this.imagePrefetchService = imagePrefetchService; + this.poisonTracker = poisonTracker; + this.payloadSupport = payloadSupport; + this.chunkPayloadSupport = chunkPayloadSupport; + this.ownershipSupport = ownershipSupport; + this.progressSupport = progressSupport; + this.host = host; + } + + + + + public void mergeLlmRowsIntoChunk(FileTaskEntity task, + String chunkScopeHash, + Integer chunkIndex, + List llmRows, + Map> allRowsByBaseId) { + if (task == null || llmRows == null || llmRows.isEmpty()) { + return; + } + List chunks = loadSubmittedChunks(task.getId()); + if (chunks.isEmpty()) { + return; + } + // P2-11：把当前 batch 命中的图片 url 异步丢入预热队列。 + // 预热失败不影响主流程，assemble 阶段无 DB cache 命中也会走原下载链路兜底。 + try { + List prefetchUrls = new ArrayList<>(llmRows.size() * 3); + for (SimilarAsinResultRowDto llmRow : llmRows) { + if (llmRow == null) { + continue; + } + SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getMainUrl()); + SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getPuzzleImg1()); + SimilarAsinResultTextSupport.addNonBlank(prefetchUrls, llmRow.getPuzzleImg2()); + } + imagePrefetchService.enqueue(task.getId(), prefetchUrls); + } catch (Exception ex) { + // 预热入队是 best-effort，任何异常都不能阻断 merge 主路径。 + log.debug("[similar-asin] enqueue prefetch failed taskId={} err={}", task.getId(), ex.getMessage()); + } + Map> rowsByChunk = new LinkedHashMap<>(); + Map chunkByKey = new LinkedHashMap<>(); + for (TaskChunkEntity chunk : chunks) { + String chunkKey = SimilarAsinChunkMergeSupport.chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()); + rowsByChunk.put(chunkKey, chunkPayloadSupport.readChunkRows(chunk)); + chunkByKey.put(chunkKey, chunk); + } + // Task 11：合并前按稳定 rowKey 去重，消除重复行逐行 expand/分配/写回 的 O(n²) 热点。 + List uniqueRows = SimilarAsinChunkMergeSupport.dedupeRowsByRowKey(llmRows); + List expandedAll = new ArrayList<>(); + for (SimilarAsinResultRowDto resultRow : uniqueRows) { + expandedAll.addAll(SimilarAsinChunkMergeSupport.expandRows(List.of(resultRow), allRowsByBaseId)); + } + // Task 10：先建 rowKey → chunkKey 批量索引，把逐 chunk 线性扫描降为 O(1) 查找。 + Map rowKeyIndex = SimilarAsinChunkMergeSupport.indexRowsByChunkKey(rowsByChunk); + List orphanRows = new ArrayList<>(); + Map> mergeRowsByChunk = + SimilarAsinChunkMergeSupport.assignLlmRowsToChunks(rowsByChunk, expandedAll, rowKeyIndex, chunkScopeHash, chunkIndex, orphanRows); + if (!orphanRows.isEmpty()) { + chunkPayloadSupport.persistOrphanLlmRows(task.getId(), orphanRows); + } + for (Map.Entry> entry : mergeRowsByChunk.entrySet()) { + TaskChunkEntity chunk = chunkByKey.get(entry.getKey()); + if (chunk == null || entry.getValue().isEmpty()) { + continue; + } + mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), new ArrayList<>(entry.getValue().values())); + } + } + + + public void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List rows) { + if (rows == null || rows.isEmpty()) { + return; + } + int maxAttempts = 3; + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (chunk == null) { + return; + } + Map persistedRows = chunkPayloadSupport.readChunkRows(chunk); + int mergedRowCount = persistedRows.size() + rows.size(); + // Task 13：单次合并后总行数超过上限时，从最旧行（存量优先）开始降级到 orphan 兜底， + // chunk 保持在上限内不无界增长；assemble 阶段 putIfAbsent 合并回结果不丢数据。 + if (mergedRowCount > SimilarAsinLimits.getChunkMergeMaxRows(properties)) { + splitChunkMergeOverflow(taskId, persistedRows, rows, chunk, + mergedRowCount - SimilarAsinLimits.getChunkMergeMaxRows(properties)); + if (rows.isEmpty()) { + return; + } + } + for (SimilarAsinResultRowDto row : rows) { + persistedRows.put(SimilarAsinChunkMergeSupport.rowKey(row), row); + } + String payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败"); + // Task 13：合并后 payload 字节超过上限时，从最旧行开始降级到 orphan 兜底； + // 降到只剩一行仍超上限时抛异常拒绝合并，防止无界 payload。 + long payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length; + if (payloadBytes > SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)) { + demoteRowsToOrphan(taskId, persistedRows, + payloadBytes - SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)); + payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败"); + payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length; + if (payloadBytes > SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties) && persistedRows.size() <= 1) { + throw new BusinessException("相似ASIN分片载荷超字节上限 taskId=" + taskId + + " scopeHash=" + scopeHash + " chunk=" + chunkIndex + + " bytes=" + payloadBytes + " limit=" + SimilarAsinLimits.getChunkMergePayloadMaxBytes(properties)); + } + } + String oldPayload = chunk.getPayloadJson(); + String oldPayloadHash = chunk.getPayloadHash(); + String newPayloadHash = DigestUtil.sha256Hex(payloadJson); + final String storedPayload; + try { + storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + } catch (Exception storeEx) { + throw new BusinessException("相似ASIN分片载荷存储失败 taskId=" + taskId + " chunk=" + chunkIndex + + ": " + (storeEx.getMessage() == null ? "" : storeEx.getMessage()), storeEx); + } + int updated = taskChunkMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskChunkEntity::getId, chunk.getId()) + .eq(TaskChunkEntity::getPayloadHash, oldPayloadHash) + .set(TaskChunkEntity::getPayloadJson, storedPayload) + .set(TaskChunkEntity::getPayloadHash, newPayloadHash) + .set(TaskChunkEntity::getUpdatedAt, LocalDateTime.now())); + if (updated > 0) { + log.debug("[similar-asin] chunk payload replaced taskId={} scopeHash={} chunk={} oldPayload={} newPayload={} attempt={}", + taskId, scopeHash, chunkIndex, oldPayload, storedPayload, attempt); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload); + return; + } + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + if (attempt < maxAttempts) { + log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}", + taskId, scopeHash, chunkIndex, attempt, maxAttempts); + } + } + throw new IllegalStateException("相似ASIN分片载荷更新失败"); + } + + + /** + * Task 13：按行数上限从最旧行开始降级到 orphan 兜底，保证合并后 chunk 行数不超过上限。 + * 最旧优先：先降级存量行（LinkedHashMap 表头），不够再从新增行表头补齐； + * 传入的 rows 会被就地修改（保留未降级部分）。降级失败仅记日志，不阻断合并主流程。 + */ + public void splitChunkMergeOverflow(Long taskId, Map persistedRows, + List rows, TaskChunkEntity chunk, + int demoteCount) { + if (demoteCount <= 0 || rows.isEmpty()) { + return; + } + List demoted = new ArrayList<>(); + while (demoted.size() < demoteCount && !persistedRows.isEmpty()) { + String oldestKey = persistedRows.keySet().iterator().next(); + SimilarAsinResultRowDto removed = persistedRows.remove(oldestKey); + if (removed != null) { + demoted.add(removed); + } + } + for (Iterator it = rows.iterator(); it.hasNext() && demoted.size() < demoteCount; ) { + SimilarAsinResultRowDto row = it.next(); + if (row != null) { + demoted.add(row); + it.remove(); + } + } + chunkPayloadSupport.persistOrphanLlmRows(taskId, demoted); + log.warn("[similar-asin] chunk merge row-limit exceeded taskId={} chunk={} mergedRows={} limit={} demoted={}", + taskId, chunk.getChunkIndex(), SimilarAsinChunkMergeSupport.mergedRowCountOf(persistedRows, rows), + SimilarAsinLimits.getChunkMergeMaxRows(properties), demoted.size()); + } + + + /** + * Task 13：按字节上限从最旧行（LinkedHashMap 表头）开始降级到 orphan 兜底， + * 直到 payload 字节不超过上限或只剩一行；降级失败仅记日志，不阻断合并主流程。 + */ + public void demoteRowsToOrphan(Long taskId, Map persistedRows, long excessBytes) { + List demoted = new ArrayList<>(); + long releasedBytes = 0L; + while (persistedRows.size() > 1 && releasedBytes < excessBytes) { + String oldestKey = persistedRows.keySet().iterator().next(); + SimilarAsinResultRowDto removed = persistedRows.remove(oldestKey); + if (removed != null) { + demoted.add(removed); + releasedBytes += estimateRowBytes(oldestKey, removed); + } + } + if (!demoted.isEmpty()) { + chunkPayloadSupport.persistOrphanLlmRows(taskId, demoted); + log.warn("[similar-asin] chunk merge byte-limit exceeded taskId={} rows={} demoted={} releasedBytes={}", + taskId, demoted.size(), demoted.size(), releasedBytes); + } + } + + + public long estimateRowBytes(String rowKey, SimilarAsinResultRowDto row) { + try { + String json = objectMapper.writeValueAsString(row); + return json == null ? 128L : json.getBytes(StandardCharsets.UTF_8).length; + } catch (Exception ex) { + return 128L + (rowKey == null ? 0 : rowKey.getBytes(StandardCharsets.UTF_8).length); + } + } + + + public Map> loadAllRowsByBaseId(FileTaskEntity task) { + try { + SimilarAsinParsedPayloadDto payload = payloadSupport.readParsedPayload(task); + return SimilarAsinGroupingConverter.groupRowsByBaseId(SimilarAsinPayloadSupport.resolveAllRows(payload)); + } catch (Exception ex) { + log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage()); + return new LinkedHashMap<>(); + } + } + + + public List loadSubmittedChunks(Long taskId) { + if (taskId == null || taskId <= 0) { + return List.of(); + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + if (chunks == null || chunks.isEmpty()) { + return List.of(); + } + return chunks.stream() + .filter(Objects::nonNull) + .filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty()) + .toList(); + } + + + public List markRowsFailed(List rows, String failureMessage) { + if (rows == null || rows.isEmpty()) { + return List.of(); + } + return rows.stream() + .map(this::copyRowForFailure) + .map(row -> markRowFailed(row, failureMessage)) + .toList(); + } + + + public SimilarAsinResultRowDto copyRowForFailure(SimilarAsinResultRowDto source) { + SimilarAsinResultRowDto row = new SimilarAsinResultRowDto(); + row.setSourceFileKey(source.getSourceFileKey()); + row.setSourceFilename(source.getSourceFilename()); + row.setRowToken(source.getRowToken()); + row.setGroupKey(source.getGroupKey()); + row.setId(source.getId()); + row.setAsin(source.getAsin()); + row.setCountry(source.getCountry()); + row.setSku(source.getSku()); + row.setPrice(source.getPrice()); + row.setUrls(source.getUrls()); + row.setAlibaba(source.getAlibaba()); + row.setTitle(source.getTitle()); + row.setError(source.getError()); + row.setDone(source.getDone()); + row.setStatus(source.getStatus()); + row.setIsConform(source.getIsConform()); + row.setReason(source.getReason()); + row.setCategory(source.getCategory()); + row.setTitleRisk(source.getTitleRisk()); + row.setAppearanceRisk(source.getAppearanceRisk()); + row.setPatentRisk(source.getPatentRisk()); + row.setConclusion(source.getConclusion()); + row.setIsStock(source.getIsStock()); + row.setSimilarity(source.getSimilarity()); + row.setTitleReason(source.getTitleReason()); + row.setAppearanceReason(source.getAppearanceReason()); + row.setPatentReason(source.getPatentReason()); + row.setMainUrl(source.getMainUrl()); + row.setPuzzleImg1(source.getPuzzleImg1()); + row.setPuzzleImg2(source.getPuzzleImg2()); + return row; + } + + + public SimilarAsinResultRowDto markRowFailed(SimilarAsinResultRowDto row, String failureMessage) { + String reviewMessage = failureMessage == null || failureMessage.isBlank() + ? "检测失败" + : "检测失败：" + failureMessage; + if (row.getError() == null || row.getError().isBlank()) { + row.setError(failureMessage); + } + if (row.getReason() == null || row.getReason().isBlank()) { + row.setReason(failureMessage); + } + if (row.getStatus() == null || row.getStatus().isBlank()) { + row.setStatus("FAILED"); + } + if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) { + row.setTitleRisk(reviewMessage); + } + if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) { + row.setAppearanceRisk(reviewMessage); + } + if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) { + row.setPatentRisk(reviewMessage); + } + if (row.getConclusion() == null || row.getConclusion().isBlank()) { + row.setConclusion(failureMessage); + } + return row; + } + + + public TaskChunkEntity findSubmittedChunk(Long taskId, String scopeHash, Integer chunkIndex) { + return taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + } + + + public PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, SimilarAsinSubmitResultRequest request) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + throw new BusinessException("任务不是运行中状态"); + } + + ownershipSupport.ensureTaskOwnedByCurrentInstance(task, "submit result"); + int chunkIndex = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics + .chunkIndex(request.getChunkIndex()); + int chunkTotal = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics + .chunkTotal(request.getChunkTotal()); + boolean done = Boolean.TRUE.equals(request.getDone()); + String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId); + String scopeHash = DigestUtil.sha256Hex(scopeKey); + boolean terminalCallback = com.nanri.aiimage.modules.similarasin.model.SimilarAsinSubmitResultSemantics + .isTerminalRequest(done, request.getError()); + SubmittedTaskMetadata taskMetadata = terminalCallback + ? readSubmittedTaskMetadata(task) + : null; + TaskChunkEntity existing = findSubmittedChunk(taskId, scopeHash, chunkIndex); + if (existing != null) { + return new PreparedSubmittedChunk( + taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, done, request.getError(), + null, null, false, taskMetadata, null); + } + String payloadJson = writeJson(flattenSubmittedRows(request), "结果序列化失败"); + String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned( + MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + boolean localFallback = transientPayloadStorageService.wasLastStoreLocalFallback(); + // 纯计算在事务外完成：payload 哈希预计算，persist 落库时直接引用 + return new PreparedSubmittedChunk( + taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, done, request.getError(), + payloadJson, storedPayload, localFallback, taskMetadata, + DigestUtil.sha256Hex(payloadJson)); + } + + + public PersistSubmittedChunkResult persistSubmittedChunk(PreparedSubmittedChunk prepared) { + Long taskId = prepared.taskId(); + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + throw new BusinessException("任务不是运行中状态"); + } + ownershipSupport.ensureTaskOwnedByCurrentInstance(task, "submit result"); + + boolean payloadPersisted = prepared.storedPayload() == null; + TaskChunkEntity existing = findSubmittedChunk(taskId, prepared.scopeHash(), prepared.chunkIndex()); + if (existing == null && prepared.storedPayload() != null) { + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setTaskId(taskId); + chunk.setModuleType(MODULE_TYPE); + chunk.setScopeKey(prepared.scopeKey()); + chunk.setScopeHash(prepared.scopeHash()); + chunk.setChunkIndex(prepared.chunkIndex()); + chunk.setChunkTotal(prepared.chunkTotal()); + chunk.setPayloadJson(prepared.storedPayload()); + chunk.setPayloadHash(prepared.payloadHash()); + chunk.setCreatedAt(LocalDateTime.now()); + chunk.setUpdatedAt(LocalDateTime.now()); + try { + if (taskChunkMapper.insert(chunk) != 1) { + throw new IllegalStateException("Chunk insert affected no row taskId=" + taskId + + " chunk=" + prepared.chunkIndex()); + } + payloadPersisted = true; + } catch (DuplicateKeyException ex) { + log.info("[similar-asin] duplicate chunk inserted concurrently taskId={} scope={} chunk={} cleanupLoser={}", + taskId, prepared.scopeKey(), prepared.chunkIndex(), prepared.storedPayload() != null); + } + } else { + log.info("[similar-asin] duplicate chunk ignored taskId={} scope={} chunk={}", + taskId, prepared.scopeKey(), prepared.chunkIndex()); + } + + upsertScopeState(taskId, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkTotal(), + prepared.error(), prepared.done(), false); + if (prepared.localFallback() && payloadPersisted) { + ownershipSupport.bindTaskToCurrentOwnerForLocalFallback( + task, prepared.scopeHash(), prepared.chunkIndex()); + } + SubmitContext context = new SubmitContext(task, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkIndex(), + prepared.done(), prepared.error(), prepared.taskMetadata()); + FinalizeTaskResult finalizeResult = completeSubmittedChunk(context); + return new PersistSubmittedChunkResult(context, payloadPersisted, finalizeResult); + } + + + public FinalizeTaskResult completeSubmittedChunk(SubmitContext context) { + FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + log.info("[similar-asin] skip completion because task already finalized taskId={} status={}", + task.getId(), task.getStatus()); + return null; + } + if (context.forceFlush() || context.error() != null && !context.error().isBlank()) { + return host.finalizeTask(task, context.error(), context.taskMetadata(), true); + } + touchJavaSideTaskActivity(task.getId()); + return null; + } + + + public void upsertScopeState(Long taskId, + String scopeKey, + String scopeHash, + Integer chunkTotal, + String error, + boolean completed, + boolean llmDone) { + TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + LocalDateTime now = LocalDateTime.now(); + if (scope == null) { + scope = new TaskScopeStateEntity(); + scope.setTaskId(taskId); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(scopeKey); + scope.setScopeHash(scopeHash); + scope.setCreatedAt(now); + } + if (chunkTotal != null) { + scope.setChunkTotal(chunkTotal); + } + scope.setReceivedChunkCount(progressSupport.countChunks(taskId, scopeHash)); + scope.setLastChunkAt(now); + scope.setLastError(error); + scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0); + scope.setUpdatedAt(now); + scope.setStateJson(llmDone + ? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}" + : "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}"); + if (scope.getId() == null) { + try { + taskScopeStateMapper.insert(scope); + return; + } catch (DuplicateKeyException ex) { + log.info("[similar-asin] duplicate scope state inserted concurrently taskId={} scope={}", taskId, scopeKey); + scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + if (scope == null) { + log.info("[similar-asin] scope state winner not committed yet, skip duplicate updater taskId={} scope={}", + taskId, scopeKey); + return; + } + if (chunkTotal != null) { + scope.setChunkTotal(chunkTotal); + } + scope.setReceivedChunkCount(progressSupport.resolveReceivedChunkProgress(taskId, scopeHash, scope.getChunkTotal())); + scope.setLastChunkAt(now); + scope.setLastError(error); + scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0); + scope.setUpdatedAt(now); + scope.setStateJson(llmDone + ? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}" + : "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}"); + } + } + taskScopeStateMapper.updateById(scope); + } + + + public List flattenSubmittedRows(SimilarAsinSubmitResultRequest request) { + if (request == null) { + return List.of(); + } + List groups = request.getGroups(); + if (groups != null && !groups.isEmpty()) { + List rows = new ArrayList<>(); + for (SimilarAsinResultGroupDto group : groups) { + if (group == null || group.getItems() == null || group.getItems().isEmpty()) { + continue; + } + for (SimilarAsinResultRowDto row : group.getItems()) { + if (row == null) { + continue; + } + if (SimilarAsinRowNormalizer.normalize(row.getGroupKey()).isBlank()) { + row.setGroupKey(group.getGroupKey()); + } + if (SimilarAsinRowNormalizer.normalize(row.getSourceFileKey()).isBlank()) { + row.setSourceFileKey(group.getSourceFileKey()); + } + if (SimilarAsinRowNormalizer.normalize(row.getSourceFilename()).isBlank()) { + row.setSourceFilename(group.getSourceFilename()); + } + rows.add(row); + } + } + logPythonInboundRows(rows); + return rows; + } + return List.of(); + } + + + /** + * 打印 Python 端回传给 Java 的每一行 row 关键字段，确认 url（主图）/ urls（同类商品图）/ title / sku + * 是否按预期到达。该日志与直连 LLM 的行级结果日志成对， + * 便于排查"Python 回传了什么、Java 又把什么提交给 LLM"。 + * Task 19：改为 DEBUG 级别并按行采样（每 20 行记一行），减少大任务日志量。 + */ + public void logPythonInboundRows(List rows) { + if (rows == null || rows.isEmpty()) { + return; + } + log.debug("[similar-asin] python inbound start size={}", rows.size()); + for (int i = 0; i < rows.size(); i++) { + SimilarAsinResultRowDto row = rows.get(i); + if (row == null) { + continue; + } + if (!SimilarAsinLogSupport.shouldLog(i, PYTHON_INBOUND_LOG_EVERY_N)) { + continue; + } + String url = row.getUrl(); + List urls = row.getUrls(); + List alibaba = row.getAlibaba(); + log.debug("[similar-asin] python inbound idx={} groupKey={} rowToken={} id={} asin={} country={} title={} sku={} price={} url={} urlsSize={} alibabaSize={} urlsHead={} urlsTail={}", + i, + SimilarAsinRowNormalizer.normalize(row.getGroupKey()), + SimilarAsinRowNormalizer.normalize(row.getRowToken()), + SimilarAsinRowNormalizer.normalize(row.getId()), + SimilarAsinRowNormalizer.normalize(row.getAsin()), + SimilarAsinRowNormalizer.normalize(row.getCountry()), + abbreviateForLog(row.getTitle(), 80), + SimilarAsinRowNormalizer.normalize(row.getSku()), + SimilarAsinRowNormalizer.normalize(row.getPrice()), + abbreviateForLog(url, 200), + urls == null ? 0 : urls.size(), + alibaba == null ? 0 : alibaba.size(), + urls == null || urls.isEmpty() ? "" : abbreviateForLog(urls.get(0), 200), + urls == null || urls.size() <= 1 ? "" : abbreviateForLog(urls.get(urls.size() - 1), 200)); + } + } + + + public String abbreviateForLog(String value, int maxLength) { + if (value == null) { + return ""; + } + String trimmed = value.trim(); + if (maxLength <= 0 || trimmed.length() <= maxLength) { + return trimmed; + } + return trimmed.substring(0, maxLength) + "..."; + } + + + public SubmittedTaskMetadata readSubmittedTaskMetadata(FileTaskEntity task) { + SimilarAsinParsedPayloadDto payload = payloadSupport.readParsedPayload(task); + List sourceFiles = payload.getSourceFiles() == null + ? List.of() + : List.copyOf(payload.getSourceFiles()); + return new SubmittedTaskMetadata(SimilarAsinPayloadSupport.rowCount(payload), sourceFiles); + } + + + public void cleanupPreparedSubmittedChunkIfUnreferenced(PreparedSubmittedChunk prepared) { + if (prepared == null || prepared.storedPayload() == null) { + return; + } + try { + if (isTransientPayloadStillReferenced(prepared.storedPayload())) { + log.info("[similar-asin] keep prepared chunk payload because it is referenced taskId={} chunk={}", + prepared.taskId(), prepared.chunkIndex()); + return; + } + transientPayloadStorageService.deletePayloadIfPresent(prepared.storedPayload()); + } catch (Exception ex) { + log.warn("[similar-asin] cleanup uncommitted chunk payload failed taskId={} chunk={} err={}", + prepared.taskId(), prepared.chunkIndex(), ex.getMessage()); + } + } + + + public boolean isTransientPayloadStillReferenced(String payload) { + String pointer = transientPayloadStorageService.extractPointer(payload); + if (pointer == null) { + return false; + } + LinkedHashSet values = new LinkedHashSet<>(); + values.add(payload); + values.add(pointer); + try { + values.add(objectMapper.writeValueAsString(pointer)); + } catch (Exception ignored) { + } + Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .in(TaskChunkEntity::getPayloadJson, values)); + if (chunkCount != null && chunkCount > 0L) { + return true; + } + Long scopeCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .and(wrapper -> wrapper.in(TaskScopeStateEntity::getParsedPayloadJson, values) + .or() + .in(TaskScopeStateEntity::getStateJson, values))); + return scopeCount != null && scopeCount > 0L; + } + + + public void submitLlmForSubmittedChunk(SubmitContext context) { + if (context == null || context.task() == null || context.task().getId() == null) { + return; + } + FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { + return; + } + List chunks; + if (context.scopeHash() == null || context.scopeHash().isBlank() || context.chunkIndex() == null) { + chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, task.getId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + } else { + TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, task.getId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, context.scopeHash()) + .eq(TaskChunkEntity::getChunkIndex, context.chunkIndex()) + .last("limit 1")); + chunks = chunk == null ? List.of() : List.of(chunk); + } + chunks = chunks == null ? List.of() : chunks.stream() + .filter(Objects::nonNull) + .filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty()) + .toList(); + if (chunks.isEmpty()) { + return; + } + FileResultEntity result = host.findOrCreateResultRecordForAssembly(task, payloadSupport.allRowCount(task)); + if (result == null) { + return; + } + TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( + task.getId(), MODULE_TYPE, result.getId(), ownershipSupport.buildTaskOwnerScopeKey(task)); + if (job == null || "SUCCESS".equals(job.getStatus())) { + return; + } + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId); + if (pendingLlm) { + taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); + touchJavaSideTaskActivity(task.getId()); + } else if (isResultSubmissionComplete(task.getId())) { + maybeFinalizeLlmJobLocked(task.getId(), new LlmBatchContext( + job.getId(), result.getId(), context.scopeHash(), context.chunkIndex(), 1, 1, ownershipSupport.currentInstanceId(), 0, null, null)); + } + } + + + public void scheduleLlmPipelineForSubmittedChunk(SubmitContext context) { + if (context == null || context.task() == null || context.task().getId() == null) { + return; + } + FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { + return; + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, task.getId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + if (chunks.isEmpty()) { + return; + } + FileResultEntity result = host.findOrCreateResultRecordForAssembly(task, payloadSupport.allRowCount(task)); + if (result == null) { + return; + } + TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( + task.getId(), MODULE_TYPE, result.getId(), ownershipSupport.buildTaskOwnerScopeKey(task)); + if (job == null || "SUCCESS".equals(job.getStatus())) { + return; + } + taskFileJobService.requeue(job.getId(), "Similar ASIN result uploaded, scheduling LLM/file assembly"); + touchJavaSideTaskActivity(task.getId()); + } + + + public List applyLlmInBatches(List items, FileTaskEntity task) { + return applyLlmInBatches(items, task, null); + } + + + public List applyLlmInBatches(List items, + FileTaskEntity task, + Runnable progressHook) { + if (items == null || items.isEmpty()) { + return List.of(); + } + String prompt = payloadSupport.readAiPrompt(task); + String apiKey = payloadSupport.readApiKey(task); + boolean imgSwitch = payloadSupport.readImgSwitch(task); + boolean categorySwitch = host.readCategorySwitch(task); + int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, imgSwitch); + List result = new ArrayList<>(); + for (int i = 0; i < items.size(); i += batchSize) { + List batch = items.subList(i, Math.min(i + batchSize, items.size())); + // 批次内所有 LLM 请求按任务归属用户计次（无归属时上下文自动跳过） + result.addAll(SecretUsageContext.call(task.getUserId(), UserSecretModule.SIMILAR_ASIN.key(), + () -> similarAsinLlmService.inspectRows(batch, prompt, apiKey, imgSwitch, categorySwitch))); + if (progressHook != null) { + progressHook.run(); + } + } + return result; + } + + + public List collectPendingLlmCandidates(List chunks, + Map> allRowsByBaseId) { + if (chunks == null || chunks.isEmpty()) { + return List.of(); + } + List candidates = new ArrayList<>(); + for (TaskChunkEntity chunk : chunks) { + Map persistedRows = chunkPayloadSupport.readChunkRows(chunk); + if (persistedRows.isEmpty()) { + continue; + } + List unresolvedRows = + enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()), allRowsByBaseId); + for (SimilarAsinResultRowDto row : unresolvedRows) { + candidates.add(new LlmCandidate(chunk.getScopeHash(), chunk.getChunkIndex(), row)); + } + } + return candidates; + } + + + /** + * 直连模式兜底调度：把已封口（提交完成）但仍有 PENDING 状态的任务重新调度一次 + * 批量提交，新批次走 submitLlmBatch 同步直连，由提交路径落 DONE 缓冲/merge。 + */ + public void schedulePendingLlmBatches() { + List states = ownershipSupport.listOwnedPendingLlmStates(); + if (states == null || states.isEmpty()) { + return; + } + Set taskIds = new LinkedHashSet<>(); + for (TaskScopeStateEntity state : states) { + if (state != null && state.getTaskId() != null) { + taskIds.add(state.getTaskId()); + } + } + log.info("[similar-asin] direct-llm poll fallback scheduling pending tasks count={}", + taskIds.size()); + for (Long taskId : taskIds) { + taskQueueExecutor.execute(() -> { + TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport.acquireTaskLock(taskId, 0L); + if (taskLockHandle == null) { + return; + } + try (taskLockHandle) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { + return; + } + FileResultEntity result = host.findOrCreateResultRecordForAssembly(task, payloadSupport.allRowCount(task)); + TaskFileJobEntity job = taskFileJobService.enqueueAssembleResult( + task.getId(), MODULE_TYPE, result.getId(), ownershipSupport.buildTaskOwnerScopeKey(task)); + if (job == null || "SUCCESS".equals(job.getStatus())) { + return; + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, task.getId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + submitLlmBatches(task, result, job, chunks, loadAllRowsByBaseId(task)); + } + }); + } + } + + + public void resubmitPendingLlmStates(Long taskId, List stateIds) { + if (taskId == null || stateIds == null || stateIds.isEmpty()) { + return; + } + TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport.acquireTaskLock(taskId, 0L); + if (taskLockHandle == null) { + return; + } + try (taskLockHandle) { + for (Long stateId : stateIds) { + if (stateId == null) { + continue; + } + TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId); + if (state == null) { + continue; + } + try { + submitLlmBatchForPendingState(state); + } catch (Exception ex) { + log.warn("[similar-asin] llm pending state resubmit failed taskId={} stateId={} err={}", + taskId, stateId, firstNonBlank(ex.getMessage(), ex.getClass().getSimpleName())); + } + } + } + } + + public boolean submitLlmBatches(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + List chunks, + Map> allRowsByBaseId) { + if (chunks == null || chunks.isEmpty()) { + return countPendingLlmStates(task.getId()) > 0; + } + String prompt = payloadSupport.readAiPrompt(task); + String apiKey = payloadSupport.readApiKey(task); + boolean imgSwitch = payloadSupport.readImgSwitch(task); + boolean categorySwitch = host.readCategorySwitch(task); + int batchSize = SimilarAsinLimits.resolveLlmBatchSize(properties, imgSwitch); + // P1-1：检测到 720712008 风暴时强制把 batch 降到 1，隔离毒行； + // 持续 5+ 次提交命中率 ≥ 40% 才会触发，正常波动不影响吞吐。 + if (poisonTracker.isStormActive(task.getId())) { + log.warn("[similar-asin] poison-storm detected, force batchSize=1 taskId={} originalBatchSize={}", + task.getId(), batchSize); + batchSize = 1; + } + List candidates = collectPendingLlmCandidates(chunks, allRowsByBaseId); + if (candidates.isEmpty()) { + return countPendingLlmStates(task.getId()) > 0; + } + // P1-2：把"仅过滤 hasImageUrl"扩展为必填字段集中校验。 + // 缺失 asin / title / 图片 url 任意一项即直接 markFailed，不进入 LLM 提交链路。 + // 原因：Python 端偶发空字段会触发 720701002 "fields cannot be extracted from null values"， + // 浪费 LLM 配额且把整 batch 拖垮；提前过滤更显式、易于排错。 + // 不打算把"必填字段集合"做成 properties——LLM 工作流签名固定，过度可配置反而把错配藏起来。 + java.util.function.Predicate isMissingRequired = row -> + row == null + || !row.hasImageUrl() + || SimilarAsinRowNormalizer.normalize(row.getAsin()).isBlank() + || SimilarAsinRowNormalizer.normalize(row.getTitle()).isBlank(); + List missingFieldCandidates = candidates.stream() + .filter(candidate -> candidate != null && isMissingRequired.test(candidate.row())) + .toList(); + if (!missingFieldCandidates.isEmpty()) { + mergeLlmRowsIntoChunk(task, + null, + null, + markRowsFailed(missingFieldCandidates.stream().map(LlmCandidate::row).toList(), + "required field missing (asin/title/url), skip LLM"), + allRowsByBaseId); + log.warn("[similar-asin] skip llm rows missing required fields taskId={} jobId={} rows={}", + task.getId(), job.getId(), missingFieldCandidates.size()); + } + List readyCandidates = candidates.stream() + .filter(candidate -> candidate != null && !isMissingRequired.test(candidate.row())) + .toList(); + boolean flushRemainder = isResultSubmissionComplete(task.getId()); + // P1-6: 防止 Python 端长时间慢回传时零头永久挂着：job.updatedAt 距今 ≥ llmFlushPendingMinutes 分钟则强制 flush。 + if (!flushRemainder && readyCandidates.size() > 0) { + LocalDateTime jobUpdatedAt = job.getUpdatedAt(); + long pendingFlushMillis = SimilarAsinLimits.llmFlushPendingMillis(properties); + if (jobUpdatedAt != null + && Duration.between(jobUpdatedAt, LocalDateTime.now()).toMillis() >= pendingFlushMillis) { + flushRemainder = true; + log.warn("[similar-asin] llm batch flush triggered by stale timer taskId={} jobId={} pendingRows={} batchSize={} jobUpdatedAt={} flushAfterMillis={}", + task.getId(), job.getId(), readyCandidates.size(), batchSize, jobUpdatedAt, pendingFlushMillis); + } + } + int submitLimit = (readyCandidates.size() / batchSize) * batchSize; + if (flushRemainder && submitLimit < readyCandidates.size()) { + submitLimit = readyCandidates.size(); + } + if (submitLimit <= 0) { + log.info("[similar-asin] llm batch waiting for more rows taskId={} jobId={} pendingRows={} batchSize={} finalUpload={}", + task.getId(), job.getId(), readyCandidates.size(), batchSize, flushRemainder); + return countPendingLlmStates(task.getId()) > 0; + } + boolean pending = false; + int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize); + int batchIndex = 1; + for (int i = 0; i < submitLimit; i += batchSize) { + List batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit)); + pending |= submitLlmBatchEntry(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId); + batchIndex++; + } + return pending || countPendingLlmStates(task.getId()) > 0; + } + + + public List enrichRowsForLlm(List rows, + Map> allRowsByBaseId) { + if (rows == null || rows.isEmpty()) { + return List.of(); + } + List enrichedRows = new ArrayList<>(rows.size()); + for (SimilarAsinResultRowDto row : rows) { + enrichedRows.add(enrichRowForLlm(row, allRowsByBaseId)); + } + return enrichedRows; + } + + + public SimilarAsinResultRowDto enrichRowForLlm(SimilarAsinResultRowDto row, + Map> allRowsByBaseId) { + if (row == null || allRowsByBaseId == null || allRowsByBaseId.isEmpty()) { + return row; + } + SimilarAsinParsedRowVo parsedRow = findParsedRow(row, allRowsByBaseId); + if (parsedRow == null) { + return row; + } + // 不再回填 url：Python 端会同时回传 url（主图）与 urls（同类商品图）， + // 缺失场景应在 Python 侧定位，Java 不再合成新的 url 字段以避免覆盖原始数据。 + if (SimilarAsinRowNormalizer.normalize(row.getTitle()).isBlank()) { + row.setTitle(parsedRow.getTitle()); + } + if (SimilarAsinRowNormalizer.normalize(row.getSku()).isBlank()) { + row.setSku(parsedRow.getSku()); + } + if (SimilarAsinRowNormalizer.normalize(row.getPrice()).isBlank()) { + row.setPrice(parsedRow.getPrice()); + } + if (SimilarAsinRowNormalizer.normalize(row.getCountry()).isBlank()) { + row.setCountry(parsedRow.getCountry()); + } + if (SimilarAsinRowNormalizer.normalize(row.getId()).isBlank()) { + row.setId(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId())); + } + if (SimilarAsinRowNormalizer.normalize(row.getRowToken()).isBlank()) { + row.setRowToken(parsedRow.getRowToken()); + } + if (SimilarAsinRowNormalizer.normalize(row.getGroupKey()).isBlank()) { + row.setGroupKey(parsedRow.getGroupKey()); + } + return row; + } + + + public SimilarAsinParsedRowVo findParsedRow(SimilarAsinResultRowDto row, + Map> allRowsByBaseId) { + if (row == null || allRowsByBaseId == null || allRowsByBaseId.isEmpty()) { + return null; + } + String groupKey = SimilarAsinRowNormalizer.normalize(row.getGroupKey()); + List candidates = !groupKey.isBlank() + ? allRowsByBaseId.getOrDefault(groupKey, List.of()) + : List.of(); + SimilarAsinParsedRowVo matched = findParsedRowInCandidates(row, candidates); + if (matched != null) { + return matched; + } + for (List rows : allRowsByBaseId.values()) { + matched = findParsedRowInCandidates(row, rows); + if (matched != null) { + return matched; + } + } + return null; + } + + + public SimilarAsinParsedRowVo findParsedRowInCandidates(SimilarAsinResultRowDto row, + List candidates) { + if (row == null || candidates == null || candidates.isEmpty()) { + return null; + } + String rowToken = SimilarAsinRowNormalizer.normalize(row.getRowToken()); + String resultKey = SimilarAsinChunkMergeSupport.rowKey(row); + for (SimilarAsinParsedRowVo candidate : candidates) { + if (candidate == null) { + continue; + } + if (!rowToken.isBlank() && rowToken.equals(SimilarAsinRowNormalizer.normalize(candidate.getRowToken()))) { + return candidate; + } + if (!resultKey.isBlank() && resultKey.equals(SimilarAsinChunkMergeSupport.rowKey(candidate))) { + return candidate; + } + } + String asin = SimilarAsinRowNormalizer.normalize(row.getAsin()).toUpperCase(Locale.ROOT); + String country = SimilarAsinRowNormalizer.normalize(row.getCountry()); + if (asin.isBlank()) { + return candidates.getFirst(); + } + for (SimilarAsinParsedRowVo candidate : candidates) { + if (asin.equals(SimilarAsinRowNormalizer.normalize(candidate.getAsin()).toUpperCase(Locale.ROOT)) + && (country.isBlank() || country.equals(SimilarAsinRowNormalizer.normalize(candidate.getCountry())))) { + return candidate; + } + } + return candidates.getFirst(); + } + + + public boolean submitLlmBatchEntry(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + List batchCandidates, + int batchIndex, + int batchTotal, + String prompt, + String apiKey, + boolean imgSwitch, + boolean categorySwitch, + Map> allRowsByBaseId) { + if (batchCandidates == null || batchCandidates.isEmpty()) { + return false; + } + List batchRows = batchCandidates.stream() + .map(LlmCandidate::row) + .filter(Objects::nonNull) + .toList(); + if (batchRows.isEmpty()) { + return false; + } + taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); + String batchScopeKey = buildLlmBatchScopeKey(task.getId(), batchRows); + String batchScopeHash = DigestUtil.sha256Hex(batchScopeKey); + TaskScopeStateEntity existing = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, task.getId()) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, batchScopeHash) + .last("limit 1")); + if (existing != null) { + return LLM_STATUS_SUBMITTED.equals(existing.getLlmStatus()) + || LLM_STATUS_RUNNING.equals(existing.getLlmStatus()); + } + // 直连 LLM 模式：同步跑完行级链路后落 DONE 缓冲/merge。 + return submitLlmBatch(task, result, job, batchRows, batchScopeKey, batchScopeHash, + batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId); + } + + + /** + * 直连 LLM 模式（directLlmEnabled=true）下的批提交：跳过工作流中转， + * 由 SimilarAsinLlmService 逐行跑完整链路（拼图/合规/对比），成功后按 + * 原同步 immediate DONE 结果路径集成：scope 去重 → 缓冲或立即 merge。 + * 行级失败信息经空结果检测保留，与同步提交失败行为对齐。 + */ + public boolean submitLlmBatch(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + List batchRows, + String batchScopeKey, + String batchScopeHash, + int batchIndex, + int batchTotal, + String prompt, + String apiKey, + boolean imgSwitch, + boolean categorySwitch, + Map> allRowsByBaseId) { + List llmRows; + try { + llmRows = SecretUsageContext.call(task.getUserId(), UserSecretModule.SIMILAR_ASIN.key(), + () -> similarAsinLlmService.inspectRows(batchRows, prompt, apiKey, imgSwitch, categorySwitch)); + } catch (Exception ex) { + String message = firstNonBlank(ex.getMessage(), "LLM submit failed"); + log.warn("[similar-asin] llm submit failed taskId={} jobId={} rows={} batch={}/{} err={}", + task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message); + poisonTracker.recordSubmitOutcome(task.getId(), SimilarAsinPoisonTracker.isPoisonRow(message)); + mergeLlmRowsIntoChunk(task, + null, + null, + markRowsFailed(batchRows, message), + allRowsByBaseId); + return false; + } + if (llmRows == null || llmRows.isEmpty()) { + String message = "LLM submit returned empty result rows"; + log.warn("[similar-asin] llm submit empty taskId={} jobId={} rows={} batch={}/{}", + task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal); + mergeLlmRowsIntoChunk(task, + null, + null, + markRowsFailed(batchRows, message), + allRowsByBaseId); + return false; + } + String emptyResultMessage = emptyLlmResultMessage(llmRows, batchRows.size()); + if (!emptyResultMessage.isBlank()) { + mergeLlmRowsIntoChunk(task, + null, + null, + markRowsFailed(batchRows, emptyResultMessage), + allRowsByBaseId); + return false; + } + // 落一条 DONE state 承载缓冲 pointer（对齐直连同步 immediate 路径）； + // 缓冲失败/关闭时回退立即 merge，结果不丢失。 + if (SimilarAsinLimits.isLlmResultBufferEnabled(properties)) { + TaskScopeStateEntity doneState = persistImmediateLlmDoneState(task, result, job, batchRows, + batchScopeKey, batchScopeHash, batchIndex, batchTotal, "llm-direct"); + if (doneState != null) { + bufferLlmRowsOrMerge(doneState, SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, doneState), llmRows, task, allRowsByBaseId); + return false; + } + } + mergeLlmRowsIntoChunk(task, null, null, llmRows, allRowsByBaseId); + return false; + } + + + /** + * 直连模式下清存量 PENDING 状态：把该 state 的批次载荷重跑一遍直连 LLM， + * 结果落 DONE 缓冲/merge，由 maybeFinalizeLlmJob 触发收尾，最后把 state 置为终态。 + * 返回 true 表示本批次已被本轮处理完。 + */ + public boolean submitLlmBatchForPendingState(TaskScopeStateEntity state) { + if (state == null || state.getId() == null || state.getTaskId() == null) { + return false; + } + FileTaskEntity task = taskForPoll(state.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) { + return false; + } + List batchRows = readLlmBatchRows(state); + if (batchRows == null || batchRows.isEmpty()) { + markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 批次载荷缺失"); + maybeFinalizeLlmJob(state.getTaskId(), SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state)); + return true; + } + LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); + String prompt = payloadSupport.readAiPrompt(task); + String apiKey = payloadSupport.readApiKey(task); + boolean imgSwitch = payloadSupport.readImgSwitch(task); + boolean categorySwitch = host.readCategorySwitch(task); + boolean submitted = submitLlmBatch(task, null, taskFileJobService.findById( + context == null ? null : context.jobId()), + batchRows, state.getScopeKey(), state.getScopeHash(), + context == null ? 1 : context.batchIndex(), + context == null ? 1 : context.batchTotal(), + prompt, apiKey, imgSwitch, categorySwitch, + allRowsByBaseIdForPoll(task)); + if (submitted) { + return true; + } + // 直连重跑未真正提交（提交异常已被 submitLlmBatch 内部消化为失败 merge）： + // 直接把 state 置为终态并触发收尾，避免 PENDING 永远挂着。 + markLlmStateTerminal(state, LLM_STATUS_FAILED, "直连模式重跑批次失败"); + maybeFinalizeLlmJob(state.getTaskId(), context); + return true; + } + + + public void finalizeTimedOutLlmStatesForTask(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)) + .orderByAsc(TaskScopeStateEntity::getLlmSubmittedAt) + .last("limit 50")); + if (states == null || states.isEmpty()) { + return; + } + for (TaskScopeStateEntity state : states) { + if (state == null || state.getId() == null || !isLlmStateTimedOut(state)) { + continue; + } + LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); + if (context == null || context.resultId() == null) { + markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 批次上下文缺失"); + continue; + } + List batchRows = readLlmBatchRows(state); + FileTaskEntity task = taskForPoll(taskId); + if (task != null) { + Map> allRowsByBaseId = allRowsByBaseIdForPoll(task); + mergeLlmRowsIntoChunk(task, + context.chunkScopeHash(), + context.chunkIndex(), + markRowsFailed(batchRows, "LLM 异步工作流轮询超时"), + allRowsByBaseId); + } + markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 异步工作流轮询超时"); + maybeFinalizeLlmJob(taskId, context); + log.warn("[similar-asin] 文件任务超时兜底已将 LLM pending 批次置为失败 taskId={} stateId={} jobId={}", + taskId, state.getId(), context.jobId()); + } + } + + + /** + * P0-4 / P2-9：pending 重跑链路直接查 DB 取 task（poll 链移除后无同线程复用上下文）。 + */ + public FileTaskEntity taskForPoll(Long taskId) { + if (taskId == null) { + return null; + } + return fileTaskMapper.selectById(taskId); + } + + + /** + * P0-4：pending 重跑链路直接加载 allRowsByBaseId（poll 链移除后无同线程复用上下文）。 + */ + public Map> allRowsByBaseIdForPoll(FileTaskEntity task) { + if (task == null) { + return Map.of(); + } + return loadAllRowsByBaseId(task); + } + + + public String emptyLlmResultMessage(List llmRows, int expectedRows) { + if (llmRows == null || llmRows.isEmpty()) { + return expectedRows > 0 ? "LLM async workflow returned empty result rows" : ""; + } + long unresolved = llmRows.stream() + .filter(row -> row != null + && !SimilarAsinResultTextSupport.hasResolvedLlmFields(row) + && !SimilarAsinResultTextSupport.isTechnicalLlmFailure(row.getError())) + .count(); + if (unresolved <= 0) { + return ""; + } + return "LLM async workflow returned empty result rows: " + unresolved + "/" + Math.max(expectedRows, llmRows.size()); + } + + + public void markLlmStateTerminal(TaskScopeStateEntity state, String status, String error) { + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)) + .set(TaskScopeStateEntity::getLlmStatus, status) + .set(TaskScopeStateEntity::getLlmCompletedAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getLlmLastPolledAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getLlmAttemptCount, llmAttemptCount(state) + 1) + .set(TaskScopeStateEntity::getLlmError, error) + .set(TaskScopeStateEntity::getCompleted, 1) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + } + + + public void maybeFinalizeLlmJob(Long taskId, LlmBatchContext context) { + if (taskId == null || context == null || countPendingLlmStates(taskId) > 0) { + return; + } + if (!ownershipSupport.isOwnerCurrent(context.ownerInstanceId())) { + return; + } + TaskDistributedLockService.LockHandle lockHandle = ownershipSupport.acquireTaskLock(taskId, 0L); + if (lockHandle == null) { + return; + } + try (lockHandle) { + maybeFinalizeLlmJobLocked(taskId, context); + } catch (Exception ex) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); + if (job != null) { + taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "相似ASIN结果文件生成失败")); + if (taskFileJobService.isRetryExhausted(job.getId())) { + host.finalizeExhaustedResultFileJob(job, ex.getMessage()); + } + } + log.warn("[相似ASIN] LLM 异步收尾失败 任务ID={} 结果ID={} 错误={}", + taskId, context.resultId(), ex.getMessage(), ex); + } + } + + + public void maybeFinalizeLlmJobLocked(Long taskId, LlmBatchContext context) { + if (taskId == null || context == null || countPendingLlmStates(taskId) > 0) { + return; + } + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("similar-asin:llm-finalize:" + taskId, Duration.ofMinutes(5)); + if (lockHandle == null) { + return; + } + try (lockHandle) { + if (countPendingLlmStates(taskId) > 0) { + return; + } + TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); + if (job == null || "SUCCESS".equals(job.getStatus())) { + return; + } + if (taskFileJobService.isRetryExhausted(job.getId())) { + host.finalizeExhaustedResultFileJob(job, job.getErrorMessage()); + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) { + taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); + touchJavaSideTaskActivity(taskId); + return; + } + // P0-3：requeue assemble 之前一次性把缓冲的 llmRows 合并到 chunk。 + // 失败时 markFailed job 并阻止 requeue，避免 assemble 阶段读到不完整 chunk。 + try { + flushLlmBufferedResults(taskId); + } catch (Exception flushEx) { + String message = firstNonBlank(flushEx.getMessage(), "刷新缓冲区 LLM 结果失败"); + log.warn("[相似ASIN] LLM 收尾刷新缓冲区失败，已标记文件任务失败 任务ID={} 文件任务ID={} 错误={}", + taskId, job.getId(), message, flushEx); + taskFileJobService.markFailed(job, message); + return; + } + boolean requeued = taskFileJobService.requeue(job.getId(), "LLM 结果已就绪，正在组装 xlsx"); + if (requeued) { + log.info("[相似ASIN] LLM 异步结果已就绪，结果文件任务已重新入队 任务ID={} 文件任务ID={} 结果ID={}", + taskId, job.getId(), context.resultId()); + } else if (taskFileJobService.isRetryExhausted(job.getId())) { + host.finalizeExhaustedResultFileJob(job, job.getErrorMessage()); + } + } + } + + + public List readLlmBatchRows(TaskScopeStateEntity state) { + if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { + return List.of(); + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + state.getParsedPayloadJson(), "read similar ASIN llm batch failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return List.of(); + } + List rows = new ArrayList<>(); + for (JsonNode node : array) { + rows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); + } + return rows; + } catch (Exception ex) { + log.warn("[similar-asin] read llm batch failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + return List.of(); + } + } + + + /** + * P0-3：读取 state 缓冲的 llmRows。 + * 与 readLlmBatchRows（读输入 batchRows）不同，这里读的是 + * 通过 bufferLlmResultForFlush 写入 transient storage 的 DONE 结果。 + */ + public List readLlmBufferedRows(String resultPayloadPointer) { + if (resultPayloadPointer == null || resultPayloadPointer.isBlank()) { + return List.of(); + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + resultPayloadPointer, "read similar ASIN llm result buffer failed"); + if (payloadJson == null || payloadJson.isBlank()) { + return List.of(); + } + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return List.of(); + } + List rows = new ArrayList<>(); + for (JsonNode node : array) { + rows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); + } + return rows; + } catch (Exception ex) { + log.warn("[similar-asin] read llm result buffer failed pointer={} err={}", + resultPayloadPointer, ex.getMessage()); + return List.of(); + } + } + + + /** + * P0-3：把单个 DONE batch 的 llmRows 缓冲到 transient storage（不立即写 chunk）。 + * 返回更新后的 LlmBatchContext（含 resultPayloadPointer），调用方需写回 stateJson。 + * 缓冲失败时返回原 context，调用方应回退到立即 merge 路径。 + */ + public LlmBatchContext bufferLlmResultForFlush(TaskScopeStateEntity state, + LlmBatchContext context, + List llmRows) { + if (state == null || context == null || llmRows == null || llmRows.isEmpty()) { + return context; + } + try { + String payloadJson = writeJson(llmRows, "serialize llm result buffer failed"); + String pointer = transientPayloadStorageService.storeParsedPayloadEntry( + MODULE_TYPE, state.getTaskId(), state.getScopeHash(), + "llm-result-" + state.getId(), payloadJson, true); + if (pointer == null || pointer.isBlank()) { + return context; + } + LlmBatchContext bufferedContext = withResultPayloadPointer(context, pointer); + String updatedStateJson = writeJson(bufferedContext, "serialize llm result buffer context failed"); + int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getStateJson, updatedStateJson) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + if (updated <= 0) { + transientPayloadStorageService.deletePayloadIfPresent(pointer); + return context; + } + state.setStateJson(updatedStateJson); + return bufferedContext; + } catch (Exception ex) { + log.warn("[similar-asin] buffer llm result failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + return context; + } + } + + + /** + * P0-3：清掉 state 上缓冲的 pointer + 删除 transient payload。 + * 在 flush 成功合并到 chunk 后调用。 + */ + public void clearLlmBufferedResult(TaskScopeStateEntity state, LlmBatchContext context) { + if (state == null || context == null) { + return; + } + String pointer = context.resultPayloadPointer(); + if (pointer == null || pointer.isBlank()) { + return; + } + try { + LlmBatchContext clearedContext = withResultPayloadPointer(context, null); + String updatedStateJson = writeJson(clearedContext, "serialize llm result buffer cleared context failed"); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getStateJson, updatedStateJson) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + state.setStateJson(updatedStateJson); + transientPayloadStorageService.deletePayloadIfPresent(pointer); + } catch (Exception ex) { + log.warn("[similar-asin] clear llm result buffer failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + } + } + + + /** + * Task 12：统一 LLM DONE 结果落库入口。 + * 缓冲开关开启时把 llmRows 写入 transient storage（pointer 存进 state.stateJson）， + * 由 flushLlmBufferedResults 在 finalize/assemble 前一次性合并到 chunk； + * 缓冲失败（存储异常 / state 更新失败 / 开关关闭）回退立即 merge，结果不丢失。 + * 空 rows / 空 state / 空 context 直接返回，不产生任何写入。 + */ + public void bufferLlmRowsOrMerge(TaskScopeStateEntity state, + LlmBatchContext context, + List llmRows, + FileTaskEntity task, + Map> allRowsByBaseId) { + if (state == null || context == null || llmRows == null || llmRows.isEmpty()) { + return; + } + if (!SimilarAsinLimits.isLlmResultBufferEnabled(properties)) { + mergeLlmRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), llmRows, allRowsByBaseId); + return; + } + LlmBatchContext bufferedContext = bufferLlmResultForFlush(state, context, llmRows); + if (bufferedContext == null + || bufferedContext.resultPayloadPointer() == null + || bufferedContext.resultPayloadPointer().isBlank()) { + // 缓冲失败：回退立即 merge，避免结果悬挂在 transient storage 之外。 + mergeLlmRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), llmRows, allRowsByBaseId); + } + } + + + /** + * Task 12：为 submit 同步 immediate DONE 结果落一条 DONE state 承载缓冲 pointer。 + * 与 saveLlmBatchState（SUBMITTED 异步）不同，该 state 直接以 DONE 终态插入， + * 不会被 countPendingLlmStates 扫描；缓冲失败/重复插入时返回 null，调用方回退立即 merge。 + */ + public TaskScopeStateEntity persistImmediateLlmDoneState(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + List batchRows, + String batchScopeKey, + String batchScopeHash, + int batchIndex, + int batchTotal, + String credentialName) { + LocalDateTime now = LocalDateTime.now(); + LlmBatchContext context = new LlmBatchContext( + job.getId(), + result.getId(), + null, + null, + batchIndex, + batchTotal, + ownershipSupport.currentInstanceId(), + 0, + credentialName, + null + ); + TaskScopeStateEntity state = new TaskScopeStateEntity(); + state.setTaskId(task.getId()); + state.setModuleType(MODULE_TYPE); + state.setScopeKey(batchScopeKey); + state.setScopeHash(batchScopeHash); + state.setStateJson(writeJson(context, "serialize immediate llm done state context failed")); + state.setLlmStatus(LLM_STATUS_DONE); + state.setLlmSubmittedAt(now); + state.setLlmCompletedAt(now); + state.setLlmAttemptCount(0); + state.setChunkTotal(batchTotal); + state.setReceivedChunkCount(batchIndex); + state.setCompleted(1); + state.setCreatedAt(now); + state.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(state); + return state; + } catch (DuplicateKeyException ex) { + log.info("[similar-asin] duplicate immediate done llm state ignored taskId={} scope={}", + task.getId(), batchScopeKey); + return null; + } + } + + + /** + * P0-3：在 finalize 之前一次性把所有 DONE/FAILED state 上缓冲的 llmRows + * 按 chunkScopeHash 分组合并到 chunk。把每个 batch 的"loadSubmittedChunks + + * readChunkRows × N + writeChunkPayload × M"压缩成"读 chunk 一次 + + * 写 chunk 一次"。 + */ + public void flushLlmBufferedResults(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + // 只取 flush 需要的列：parsed_payload_json/llm_* 等大字段不拉，避免跨库大结果集传输（每任务可达数百行大 JSON） + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getId, + TaskScopeStateEntity::getTaskId, + TaskScopeStateEntity::getModuleType, + TaskScopeStateEntity::getLlmStatus, + TaskScopeStateEntity::getStateJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_DONE, LLM_STATUS_FAILED))); + if (states == null || states.isEmpty()) { + return; + } + List entries = new ArrayList<>(); + for (TaskScopeStateEntity state : states) { + LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); + if (context == null) { + continue; + } + String pointer = context.resultPayloadPointer(); + if (pointer == null || pointer.isBlank()) { + continue; + } + List rows = readLlmBufferedRows(pointer); + if (rows == null || rows.isEmpty()) { + // pointer 存在但 payload 已被清理（被 GC 或上一次 flush 部分成功），直接清 pointer。 + clearLlmBufferedResult(state, context); + continue; + } + entries.add(new BufferedFlushEntry(state, context, rows)); + } + if (entries.isEmpty()) { + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + return; + } + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + // 按 chunkScopeHash 分组（空字符串占位 null，让 mergeLlmRowsIntoChunk 走全 chunk 扫描匹配） + Map> entriesByChunk = new LinkedHashMap<>(); + for (BufferedFlushEntry entry : entries) { + String key = entry.context().chunkScopeHash() == null ? "" : entry.context().chunkScopeHash(); + entriesByChunk.computeIfAbsent(key, ignored -> new ArrayList<>()).add(entry); + } + int totalFlushed = 0; + int failedGroups = 0; + for (Map.Entry> group : entriesByChunk.entrySet()) { + List groupEntries = group.getValue(); + List aggregated = new ArrayList<>(); + for (BufferedFlushEntry entry : groupEntries) { + aggregated.addAll(entry.rows()); + } + if (aggregated.isEmpty()) { + continue; + } + String chunkScopeHash = group.getKey().isEmpty() ? null : group.getKey(); + Integer chunkIndex = null; + if (chunkScopeHash != null) { + for (BufferedFlushEntry entry : groupEntries) { + if (entry.context().chunkIndex() != null) { + chunkIndex = entry.context().chunkIndex(); + break; + } + } + } + try { + mergeLlmRowsIntoChunk(task, chunkScopeHash, chunkIndex, aggregated, allRowsByBaseId); + for (BufferedFlushEntry entry : groupEntries) { + clearLlmBufferedResult(entry.state(), entry.context()); + } + totalFlushed += aggregated.size(); + } catch (Exception ex) { + failedGroups++; + log.warn("[similar-asin] flush buffered llm results group failed taskId={} chunkScopeHash={} err={}", + taskId, chunkScopeHash, ex.getMessage(), ex); + // 不 clear pointer，留待下次 finalize 或 stale-recovery 重试 + } + } + if (totalFlushed > 0 || failedGroups > 0) { + log.info("[similar-asin] flushed buffered llm results taskId={} entries={} totalRows={} chunkGroups={} failedGroups={}", + taskId, entries.size(), totalFlushed, entriesByChunk.size(), failedGroups); + } + if (failedGroups > 0) { + // 让 finalize 能感知到失败，调用方应避免继续 requeue assemble。 + throw new IllegalStateException("刷新缓冲区 LLM 结果失败，失败分组数=" + failedGroups); + } + } + + + public LlmBatchContext withResultPayloadPointer(LlmBatchContext context, String resultPayloadPointer) { + return new LlmBatchContext( + context.jobId(), + context.resultId(), + context.chunkScopeHash(), + context.chunkIndex(), + context.batchIndex(), + context.batchTotal(), + context.ownerInstanceId(), + context.submitRetryCount(), + context.credentialName(), + resultPayloadPointer + ); + } + + + public boolean isLlmStateTimedOut(TaskScopeStateEntity state) { + if (state == null || state.getLlmSubmittedAt() == null) { + return false; + } + long timeoutMillis = Math.max(10000L, properties.getStaleTimeoutMinutes() * 60_000L); + return Duration.between(state.getLlmSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis; + } + + + public int llmAttemptCount(TaskScopeStateEntity state) { + return state == null || state.getLlmAttemptCount() == null ? 0 : state.getLlmAttemptCount(); + } + + + public int countPendingLlmStates(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING))); + return count == null ? 0 : count.intValue(); + } + + + public boolean isResultSubmissionComplete(Long taskId) { + if (taskId == null || taskId <= 0) { + return false; + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .isNull(TaskScopeStateEntity::getLlmStatus) + .isNotNull(TaskScopeStateEntity::getLastChunkAt)); + if (states == null || states.isEmpty()) { + return false; + } + boolean hasCompletedScope = false; + for (TaskScopeStateEntity state : states) { + if (state == null) { + continue; + } + if (Integer.valueOf(1).equals(state.getCompleted())) { + hasCompletedScope = true; + continue; + } + return false; + } + return hasCompletedScope; + } + + + public void touchJavaSideTaskActivity(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + LocalDateTime now = LocalDateTime.now(); + LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, properties.getDbTaskTouchIntervalMillis()))); + fileTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(FileTaskEntity::getId, taskId) + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .and(wrapper -> wrapper.isNull(FileTaskEntity::getUpdatedAt) + .or() + .le(FileTaskEntity::getUpdatedAt, cutoff)) + .set(FileTaskEntity::getUpdatedAt, now)); + } + + + public String buildLlmBatchScopeKey(Long taskId, List batchRows) { + StringBuilder rowKeys = new StringBuilder(); + if (batchRows != null) { + for (SimilarAsinResultRowDto row : batchRows) { + String key = SimilarAsinChunkMergeSupport.rowKey(row); + if (!key.isBlank()) { + if (!rowKeys.isEmpty()) { + rowKeys.append('|'); + } + rowKeys.append(key); + } + } + } + return "llm:task:" + taskId + ":rows:" + DigestUtil.sha256Hex(rowKeys.toString()); + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } + + private static String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmitContext.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmitContext.java new file mode 100644 index 00000000..d29b7aad --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmitContext.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; + +/** + * 一次 Python 结果提交的上下文。 + * spec 05 续（拆分第 7 轮）：从 SimilarAsinTaskService 原样搬出为顶层 record。 + */ +public record SubmitContext(FileTaskEntity task, + String scopeKey, + String scopeHash, + Integer chunkIndex, + boolean forceFlush, + String error, + SubmittedTaskMetadata taskMetadata) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmittedTaskMetadata.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmittedTaskMetadata.java new file mode 100644 index 00000000..9826f13a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SubmittedTaskMetadata.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.similarasin.service.support; + +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; + +import java.util.List; + +/** + * 提交阶段的任务元信息：总行数与源文件清单。 + * spec 05 续（拆分第 7 轮）：从 SimilarAsinTaskService 原样搬出为顶层 record， + * 供流水线与其宿主服务共用。 + */ +public record SubmittedTaskMetadata(int rowCount, + List sourceFiles) { +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkMergeLimitTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkMergeLimitTest.java index dbfcc54d..d70c3854 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkMergeLimitTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkMergeLimitTest.java @@ -20,6 +20,7 @@ import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -166,10 +167,10 @@ class SimilarAsinTaskServiceChunkMergeLimitTest { private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task, String scopeHash, Integer chunkIndex, List llmRows) throws Exception { - Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk", + Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk", FileTaskEntity.class, String.class, Integer.class, List.class, Map.class); merge.setAccessible(true); - merge.invoke(service, task, scopeHash, chunkIndex, llmRows, Map.of()); + merge.invoke(service.pipelineSupport(), task, scopeHash, chunkIndex, llmRows, Map.of()); } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java index b3f63a80..84692004 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceCozeBufferScopeTest.java @@ -23,6 +23,7 @@ import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -226,12 +227,12 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { FileTaskEntity task = task(); List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); - Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", + Method bufferOrMerge = SimilarAsinPipelineSupport.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); - bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry( eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)); @@ -264,9 +265,9 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1"))); when(taskChunkMapper.update(any(), any())).thenReturn(1); - Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class); + Method flush = SimilarAsinPipelineSupport.class.getDeclaredMethod("flushLlmBufferedResults", Long.class); flush.setAccessible(true); - flush.invoke(service, 7104L); + flush.invoke(service.pipelineSupport(), 7104L); verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); verify(taskChunkMapper, atLeastOnce()).update(any(), any()); @@ -280,14 +281,14 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { FileTaskEntity task = task(); List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); - Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", + Method bufferOrMerge = SimilarAsinPipelineSupport.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); TaskScopeStateEntity state = state(task, 1L, "DONE", 2); - bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of()); - bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state, context(2), rows, task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state, context(2), rows, task, Map.of()); // 缓冲 2 次（每次重新写 pointer 是幂等语义：同一 state 覆盖写，无重复行） verify(transientPayloadStorageService, times(2)).storeParsedPayloadEntry( @@ -299,13 +300,13 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { void test_task_012_payload_chunk_boundary_empty_input() throws Exception { // 空输入：无行时缓冲与 merge 都不发生，不创建无效资源 FileTaskEntity task = task(); - Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", + Method bufferOrMerge = SimilarAsinPipelineSupport.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); - bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of()); - bufferOrMerge.invoke(service, state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state(task, 1L, "DONE", 1), context(1), null, task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of()); verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true)); verify(taskChunkMapper, never()).update(any(), any()); @@ -316,12 +317,12 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { // 单 batch（batchTotal=1）：原 P0-3 例外，现在也缓冲 FileTaskEntity task = task(); List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); - Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", + Method bufferOrMerge = SimilarAsinPipelineSupport.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); - bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry( eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)); @@ -335,12 +336,12 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { stubChunkMerge(chunkRowsJson()); when(properties.isLlmResultBufferEnabled()).thenReturn(false); List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); - Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", + Method bufferOrMerge = SimilarAsinPipelineSupport.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); - bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true)); verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); @@ -355,12 +356,12 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true))) .thenThrow(new IllegalStateException("rustfs full")); List rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")); - Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge", + Method bufferOrMerge = SimilarAsinPipelineSupport.class.getDeclaredMethod("bufferLlmRowsOrMerge", TaskScopeStateEntity.class, LlmBatchContext.class, List.class, FileTaskEntity.class, Map.class); bufferOrMerge.setAccessible(true); - bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); + bufferOrMerge.invoke(service.pipelineSupport(), state(task, 1L, "DONE", 1), context(1), rows, task, Map.of()); verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); verify(taskChunkMapper, atLeastOnce()).update(any(), any()); @@ -394,11 +395,11 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { return "ptr:stored-" + invocation.getArgument(3); }).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); - Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class); + Method flush = SimilarAsinPipelineSupport.class.getDeclaredMethod("flushLlmBufferedResults", Long.class); flush.setAccessible(true); Exception ex = assertThrows(Exception.class, () -> { try { - flush.invoke(service, 7104L); + flush.invoke(service.pipelineSupport(), 7104L); } catch (java.lang.reflect.InvocationTargetException e) { throw e.getCause(); } @@ -410,7 +411,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest { verify(taskScopeStateMapper, never()).update(any(), any()); // 恢复后重试 flush：chunk 合并成功一次，pointer 清理 - flush.invoke(service, 7104L); + flush.invoke(service.pipelineSupport(), 7104L); assertEquals(2, storeCalls.get(), "恢复后重试应再次写 chunk"); verify(taskScopeStateMapper, atLeastOnce()).update(any(), any()); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java index a8df11db..4ec576b1 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyDedupeTest.java @@ -20,6 +20,7 @@ import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -171,10 +172,10 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { assertEquals(1, deduped.size(), "重复行必须按稳定 rowKey 去重"); assertEquals("r1", deduped.get(0).getRowToken()); - Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk", + Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk", FileTaskEntity.class, String.class, Integer.class, List.class, Map.class); merge.setAccessible(true); - merge.invoke(service, task, null, null, llmRows, Map.of()); + merge.invoke(service.pipelineSupport(), task, null, null, llmRows, Map.of()); assertEquals(1, storedCounter.get(), "去重后 chunk 只写一次"); verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); } @@ -208,10 +209,10 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { FileTaskEntity task = new FileTaskEntity(); task.setId(7004L); - Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk", + Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk", FileTaskEntity.class, String.class, Integer.class, List.class, Map.class); merge.setAccessible(true); - merge.invoke(service, task, null, null, llmRows, Map.of()); + merge.invoke(service.pipelineSupport(), task, null, null, llmRows, Map.of()); verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); } @@ -297,12 +298,12 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { .thenThrow(new IllegalStateException("rustfs down")); FileTaskEntity task = new FileTaskEntity(); task.setId(7004L); - Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk", + Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk", FileTaskEntity.class, String.class, Integer.class, List.class, Map.class); merge.setAccessible(true); Exception ex = assertThrows(Exception.class, () -> { try { - merge.invoke(service, task, null, null, + merge.invoke(service.pipelineSupport(), task, null, null, List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of()); } catch (java.lang.reflect.InvocationTargetException e) { throw e.getCause(); @@ -314,7 +315,7 @@ class SimilarAsinTaskServiceRowKeyDedupeTest { // 恢复后重试成功：只写一次，无重复记录 AtomicLong storedCounter = new AtomicLong(0); stubSingleChunkMerge(chunk, rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))), storedCounter); - merge.invoke(service, task, null, null, + merge.invoke(service.pipelineSupport(), task, null, null, List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of()); assertEquals(1, storedCounter.get()); verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java index 3985b0bd..555dbb5b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java @@ -22,6 +22,7 @@ import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -191,10 +192,10 @@ class SimilarAsinTaskServiceRowKeyIndexTest { task.setId(7004L); List llmRows = List.of(row("r1", "1", "B0A0000001", "英国"), row("r3", "3", "B0A0000003", "美国")); - Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk", + Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk", FileTaskEntity.class, String.class, Integer.class, List.class, Map.class); merge.setAccessible(true); - merge.invoke(service, task, null, null, llmRows, Map.of()); + merge.invoke(service.pipelineSupport(), task, null, null, llmRows, Map.of()); verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString()); verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); @@ -347,12 +348,12 @@ class SimilarAsinTaskServiceRowKeyIndexTest { .thenThrow(new IllegalStateException("rustfs down")); FileTaskEntity task = new FileTaskEntity(); task.setId(7004L); - Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk", + Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk", FileTaskEntity.class, String.class, Integer.class, List.class, Map.class); merge.setAccessible(true); BusinessException ex = assertThrows(BusinessException.class, () -> { try { - merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of()); + merge.invoke(service.pipelineSupport(), task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of()); } catch (java.lang.reflect.InvocationTargetException e) { throw e.getCause(); } @@ -367,7 +368,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest { .thenReturn("stored:retry"); when(taskChunkMapper.selectOne(any())).thenReturn(chunks.get(0)); when(taskChunkMapper.update(any(), any())).thenReturn(1); - merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of()); + merge.invoke(service.pipelineSupport(), task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of()); verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTxBoundaryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTxBoundaryTest.java index cd3e03e0..39c3b4a7 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTxBoundaryTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTxBoundaryTest.java @@ -28,6 +28,7 @@ import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -169,7 +170,7 @@ class SimilarAsinTaskServiceTxBoundaryTest { when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); Object prepared = ReflectionTestUtils.invokeMethod( - service, "prepareSubmittedChunk", TASK_ID, request(false)); + service.pipelineSupport(), "prepareSubmittedChunk", TASK_ID, request(false)); assertEquals(DigestUtil.sha256Hex(storedPayloadJson.get()), ReflectionTestUtils.getField(prepared, "payloadHash"), "prepare 阶段必须产出预计算的 payload 哈希（事务开始前可用）"); @@ -263,11 +264,11 @@ class SimilarAsinTaskServiceTxBoundaryTest { @Test void movedComputationMethodsCarryNoTransactionAnnotation() throws Exception { - assertNull(SimilarAsinTaskService.class + assertNull(SimilarAsinPipelineSupport.class .getDeclaredMethod("prepareSubmittedChunk", Long.class, SimilarAsinSubmitResultRequest.class) .getAnnotation(Transactional.class), "prepareSubmittedChunk 不得带 @Transactional"); - boolean persistUnannotated = java.util.Arrays.stream(SimilarAsinTaskService.class.getDeclaredMethods()) + boolean persistUnannotated = java.util.Arrays.stream(SimilarAsinPipelineSupport.class.getDeclaredMethods()) .filter(method -> method.getName().equals("persistSubmittedChunk")) .allMatch(method -> method.getAnnotation(Transactional.class) == null); assertTrue(persistUnannotated, "persistSubmittedChunk 保持无注解（事务由调用方 inNewTransaction 控制）");