From f5e5717443c952acd4bc58d35bf26153b9850879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 21 Oct 2026 10:19:34 +0800 Subject: [PATCH] =?UTF-8?q?fix(similar-asin):=20=E8=BF=9F=E5=88=B0?= =?UTF-8?q?=E5=9B=9E=E4=BC=A0=E9=87=8D=E5=BC=80=E8=B6=85=E6=97=B6=E5=88=A4?= =?UTF-8?q?=E6=AD=BB=E4=BB=BB=E5=8A=A1=EF=BC=8C=E6=94=B6=E5=B0=BE=E5=A4=8D?= =?UTF-8?q?=E4=BD=8D=E7=BB=84=E8=A3=85=20job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 心跳判死巡检只改任务状态,客户端继续补发时被 40901 拒绝并删本地 副本;线上 31577/31472 的 214 行结果分片整批丢失。24 小时内、仅 心跳/回传停滞类判死文案的 FAILED 任务,收到迟到分片时条件重开为 RUNNING;业务失败与 SUCCESS 仍按终态拒绝。 判死巡检会把组装 job 一并写成 FAILED+retry=max,底座不会自动重排; 收尾进入等组装态时复位它,避免重开后卡在 RUNNING。 --- .../service/SimilarAsinTaskService.java | 85 ++++++++++++++++++ .../SimilarAsinTaskServiceSubmitTest.java | 88 +++++++++++++++++++ 2 files changed, 173 insertions(+) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 72341e5c5..5a11d4fc4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -117,6 +117,15 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { private static final String LLM_STATUS_FAILED = "FAILED"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_ZIP = "application/zip"; + /** 心跳/回传停滞类判死文案:客户端迟到回传可把任务重开,避免 40901 直接丢数据。 */ + private static final List STALE_FAILURE_MARKERS = List.of( + "任务心跳超时,已自动失败", + "长时间未收到 Python 心跳,任务已自动失败", + "客户端未回传任何检测结果", + "客户端回传中断", + "任务创建至今超过", + "Python heartbeat alive but no result chunk uploaded"); + private static final Duration STALE_REOPEN_WINDOW = Duration.ofHours(24); private static final int MAX_LIGHT_TASK_IDS = 200; /** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */ private static final int MAX_BATCH_TASK_IDS = 1000; @@ -754,6 +763,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { if (request == null) { throw new BusinessException("结果请求不能为空"); } + reopenStaleFailedTask(taskId); PreparedSubmittedChunk prepared = pipelineSupport().prepareSubmittedChunk(taskId, request); PersistSubmittedChunkResult persisted = transactionManager == null ? pipelineSupport().persistSubmittedChunk(prepared) @@ -769,10 +779,85 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { taskCacheService.touchTaskHeartbeat(taskId); } else { applyFinalizeSideEffects(persisted.finalizeResult()); + // 中间分片重开后任务已是 RUNNING,finalize 时不能再靠「本次是否重开」判断; + // terminal=false 才是「收尾后等组装」,此时才需要复位判死时一起失败的组装 job。 + if (!persisted.finalizeResult().terminal()) { + recoverTerminalFailedAssemble(taskId); + } } pipelineSupport().scheduleLlmPipelineForSubmittedChunk(context); } + /** + * 判死巡检会把在跑的组装 job 一起写成 FAILED + retryCount=max(见 + * {@code StaleTaskRepairService.markDeadDownstream})。底座对重试耗尽的 job 不会自动重排, + * 迟到回传重开后若不复位,任务会永远挂在 RUNNING 等一个不会跑的组装。 + * {@code resetTerminalFailedForRecovery} 自带「非终态失败不动作」守卫,重复调用安全。 + */ + private void recoverTerminalFailedAssemble(Long taskId) { + List results = fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .last("limit 1")); + if (results == null || results.isEmpty() || results.getFirst().getId() == null) { + return; + } + taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, results.getFirst().getId()); + } + + /** + * 被心跳/回传超时判死(含首心跳宽限)的任务收到客户端迟到分片时先重开,再走正常落库。 + * + *

判死巡检只改 {@code biz_file_task},客户端并不知道;它继续补发时旧逻辑按终态回 40901, + * 客户端收到 {@code success=true} 即删本地副本,任务已采集的数据整批丢掉(生产 31577/31472)。 + * 只重开归因于客户端停滞的判死文案、24 小时内、owner 为本实例的记录;业务性失败 + * (如结果文件生成失败)与 SUCCESS 仍按终态拒绝。 + */ + private void reopenStaleFailedTask(Long taskId) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_FAILED.equals(task.getStatus())) { + return; + } + if (!isStaleFailure(task.getErrorMessage())) { + return; + } + String reason = task.getErrorMessage(); + LocalDateTime anchor = task.getFinishedAt() == null ? task.getUpdatedAt() : task.getFinishedAt(); + if (anchor == null || anchor.isBefore(LocalDateTime.now().minus(STALE_REOPEN_WINDOW))) { + return; + } + if (!ownershipSupport().isOwnerCurrent(ownershipSupport().ownerFromTask(task))) { + return; + } + int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(FileTaskEntity::getId, taskId) + .eq(FileTaskEntity::getStatus, STATUS_FAILED) + .eq(FileTaskEntity::getErrorMessage, reason) + .set(FileTaskEntity::getStatus, STATUS_RUNNING) + .set(FileTaskEntity::getErrorMessage, null) + .set(FileTaskEntity::getFinishedAt, null) + .set(FileTaskEntity::getSuccessFileCount, 0) + .set(FileTaskEntity::getFailedFileCount, 0) + .set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())); + if (updated == 1) { + taskCacheService.deleteTaskCache(taskId); + log.warn("[similar-asin] 迟到结果回传,重开心跳/回传超时判死的任务 taskId={} reason={}", + taskId, reason); + } + } + + private boolean isStaleFailure(String errorMessage) { + if (errorMessage == null || errorMessage.isBlank()) { + return false; + } + for (String marker : STALE_FAILURE_MARKERS) { + if (errorMessage.contains(marker)) { + return true; + } + } + return false; + } + public void deleteTask(Long taskId, Long userId) { List payloads; try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceSubmitTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceSubmitTest.java index fdd2c5c94..e38adf5c9 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceSubmitTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceSubmitTest.java @@ -45,6 +45,7 @@ import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; import org.apache.ibatis.builder.MapperBuilderAssistant; import java.time.Duration; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -274,6 +275,93 @@ class SimilarAsinTaskServiceSubmitTest { anyString(), anyLong(), anyString(), any(), anyString()); } + @Test + void lateResultReopensStaleFailedTaskInsteadOfDroppingIt() throws Exception { + // 生产 31577/31472:心跳超时判死后客户端继续补发,旧逻辑回 40901,客户端删本地副本, + // 已采数据整批丢失。迟到分片到达时应先把任务拉回 RUNNING,再正常落库。 + FileTaskEntity task = runningTask("instance-a"); + task.setStatus("FAILED"); + task.setErrorMessage("任务心跳超时,已自动失败"); + task.setFinishedAt(LocalDateTime.now().minusMinutes(30)); + task.setFailedFileCount(1); + AtomicBoolean reopened = new AtomicBoolean(); + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); + when(fileTaskMapper.update(any(), any())).thenAnswer(invocation -> { + reopened.set(true); + task.setStatus("RUNNING"); + task.setErrorMessage(null); + task.setFinishedAt(null); + return 1; + }); + configureNewChunkAndScope(); + configureChunkStore(false); + when(taskChunkMapper.selectCount(any())).thenReturn(1L); + + service.submitResult(TASK_ID, request(false)); + + assertTrue(reopened.get(), "心跳超时判死的任务收到迟到分片必须先重开,而不是 40901 丢数据"); + assertEquals("RUNNING", task.getStatus()); + verify(taskCacheService).touchTaskHeartbeat(TASK_ID); + } + + @Test + void lateFinalResultResetsTerminalFailedAssembleJob() throws Exception { + // 判死巡检把组装 job 一起写成 FAILED+retryCount=max;迟到 done 重开后若不复位, + // finalizeTask 只把任务挂回 RUNNING,组装永远不会再跑。31577/31472 正是这个形状。 + FileTaskEntity task = runningTask("instance-a"); + task.setStatus("FAILED"); + task.setErrorMessage("任务心跳超时,已自动失败"); + task.setFinishedAt(LocalDateTime.now().minusMinutes(30)); + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); + when(fileTaskMapper.update(any(), any())).thenAnswer(invocation -> { + task.setStatus("RUNNING"); + task.setErrorMessage(null); + task.setFinishedAt(null); + return 1; + }); + configureNewChunkAndScope(); + when(taskChunkMapper.selectCount(any())).thenReturn(1L); + when(transientPayloadStorageService.storeChunkPayloadVersioned( + eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString())) + .thenReturn(STORED_CHUNK_POINTER); + when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(false); + String parsedPayload = parsedPayloadJson(List.of(new SimilarAsinParsedRowVo())); + when(transientPayloadStorageService.resolvePayload(eq(PARSED_POINTER), anyString())) + .thenReturn(parsedPayload); + FileResultEntity result = new FileResultEntity(); + result.setId(501L); + result.setTaskId(TASK_ID); + result.setModuleType(SimilarAsinTaskService.MODULE_TYPE); + result.setSourceFilename("germany.xlsx"); + when(fileResultMapper.selectList(any())).thenReturn(List.of(result)); + + // 先来一个中间分片:任务在这里被重开;随后 done 到达时状态已是 RUNNING, + // 不能再用「本次请求是否发生重开」决定要不要复位组装 job。 + service.submitResult(TASK_ID, request(false)); + assertEquals("RUNNING", task.getStatus(), "中间分片应先把判死任务重开"); + + service.submitResult(TASK_ID, request(true)); + + assertEquals("RUNNING", task.getStatus(), "迟到 done 收尾后应进入等待组装态"); + verify(taskFileJobService).resetTerminalFailedForRecovery( + TASK_ID, SimilarAsinTaskService.MODULE_TYPE, 501L); + } + + @Test + void businessFailedTaskStillRejectsLateResultWithTerminalCode() throws Exception { + // 只有客户端停滞类判死可重开;结果文件生成失败等业务失败仍按终态收口。 + FileTaskEntity task = runningTask("instance-a"); + task.setStatus("FAILED"); + task.setErrorMessage("相似ASIN结果文件生成失败"); + task.setFinishedAt(LocalDateTime.now().minusMinutes(30)); + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); + + BusinessException ex = assertThrows(BusinessException.class, + () -> service.submitResult(TASK_ID, request(false))); + assertEquals(BusinessCodes.TASK_ALREADY_FINISHED, ex.getCode()); + verify(fileTaskMapper, never()).update(any(), any()); + } + @Test void submitResultOnFinishedTaskIsRejectedWithTerminalCode() throws Exception { // 真终态被拒必须带 40901:异常处理器对它幂等回 success=true,客户端据此删本地副本、停止补发。