From bd0c0a08109b8147819443bf9a68d473b48bb76c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Tue, 10 Nov 2026 11:49:43 +0800 Subject: [PATCH] =?UTF-8?q?fix(task):=20=E5=A4=B1=E8=B4=A5=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E4=B9=9F=E5=A4=8D=E4=BD=8D=E5=B7=B2=E4=BA=A4=E4=BB=98?= =?UTF-8?q?=E5=87=BA=E8=A1=A8=E4=BD=9C=E4=B8=9A=EF=BC=8C=E5=88=A4=E6=AD=BB?= =?UTF-8?q?=E6=96=87=E6=A1=88=E6=8C=89=E3=80=8C=E5=AE=A2=E6=88=B7=E7=AB=AF?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E4=BA=A4=E5=AE=8C=E3=80=8D=E5=88=86=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/task/StaleTaskDeathReason.java | 42 +++++++++++++++-- .../service/AppearancePatentTaskService.java | 2 +- .../AppearancePatentStaleTaskScanner.java | 11 ++++- .../task/service/TaskInPlaceRetryService.java | 8 +++- .../common/task/StaleTaskDeathReasonTest.java | 43 ++++++++++++++--- .../service/TaskInPlaceRetryServiceTest.java | 46 +++++++++++++++++++ 6 files changed, 138 insertions(+), 14 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/task/StaleTaskDeathReason.java b/backend-java/src/main/java/com/nanri/aiimage/common/task/StaleTaskDeathReason.java index 4fd3227a2..1d9a26a33 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/common/task/StaleTaskDeathReason.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/task/StaleTaskDeathReason.java @@ -19,6 +19,8 @@ import java.util.List; * @param lastHeartbeatAt 最后一次客户端心跳时间,Redis 无记录时为 null * @param lastProgress 客户端最后一次上报的位置/阶段文本,老客户端或无位置信息时为 null * @param fileJobExhausted 结果文件作业是否已重试耗尽(true 时责任不在客户端回传) + * @param submissionComplete 客户端是否已交完全部结果(末分片 done=true/scope 已封口)。 + * true 时责任在出表侧:采集早就干完了,是结果文件没生成出来 * @param timeoutMinutes 判死阈值(分钟) */ public record StaleTaskDeathReason(long chunkCount, @@ -26,17 +28,51 @@ public record StaleTaskDeathReason(long chunkCount, LocalDateTime lastHeartbeatAt, String lastProgress, boolean fileJobExhausted, + boolean submissionComplete, int timeoutMinutes) { + /** + * 没有「客户端是否交完」信号的模块用这一档:等价于 {@code submissionComplete=false}, + * 文案与加这个字段之前逐字相同。 + * + *

留在重载里而不是让调用方自己写 {@code false},是为了不动相似ASIN那一支—— + * {@code SimilarAsinTaskService} 是在册的超长文件(2066 行,可读棘轮只许减), + * 多一行参数就把它顶到 2067 并当场判红。外观专利有真实现场(2026-11-10 33828/33758), + * 那条链路已经接上真信号;相似ASIN这一格要接,得等它先拆出判死文案那一段。 + */ + public StaleTaskDeathReason(long chunkCount, + LocalDateTime lastChunkAt, + LocalDateTime lastHeartbeatAt, + String lastProgress, + boolean fileJobExhausted, + int timeoutMinutes) { + this(chunkCount, lastChunkAt, lastHeartbeatAt, lastProgress, fileJobExhausted, false, timeoutMinutes); + } + private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("MM-dd HH:mm"); public String describe() { if (chunkCount > 0 && fileJobExhausted) { return "已收到 " + chunkCount + " 个结果分片,但结果文件生成多次失败,任务已自动结束"; } - String headline = chunkCount <= 0L - ? "客户端未回传任何检测结果,超过 " + Math.max(1, timeoutMinutes) + " 分钟无进展" - : "客户端回传中断"; + String headline; + if (submissionComplete) { + // 2026-11-10 现网 33828/33758 的形状:采集侧末分片 done=true 已上传成功 + // (worker 日志「任务处理完成 99/99,抓取失败 0 行」),出表作业却停在上一代的 SUCCESS 上 + // 没被复位,于是永不出表、心跳停了 30 分钟被判死。此时说「客户端回传中断」与事实相反—— + // 客户端早就传完了,卡住的是结果文件生成。 + headline = "客户端已上传完全部结果,但结果文件未能生成"; + } else if (chunkCount <= 0L) { + headline = "客户端未回传任何检测结果,超过 " + Math.max(1, timeoutMinutes) + " 分钟无进展"; + if (lastProgress == null || lastProgress.isBlank()) { + // 有位置文本时客户端至少开过工(如「排队等待空闲槽位」),这半句只加在「零证据」那一格: + // 2026-11-10 慧赢2 的 33827 就是没人领到(采集机 02:50→09:30 全程无日志)却写成 + // 「无进展」的,用户据此以为是自己数据有问题。 + headline += "(全程无处理位置上报,可能一直在等待采集槽位)"; + } + } else { + headline = "客户端回传中断"; + } List evidence = new ArrayList<>(); if (chunkCount > 0L) { evidence.add("已收到 " + chunkCount + " 个分片,最后一次 " + at(lastChunkAt)); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 17b41f69a..1d288fc13 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -389,7 +389,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline submissionComplete; public AppearancePatentStaleTaskScanner(String moduleType, Duration taskLockTtl, @@ -78,7 +81,8 @@ public class AppearancePatentStaleTaskScanner { TaskFileJobService taskFileJobService, TaskDistributedLockService taskDistributedLockService, AppearancePatentProperties properties, - AppearancePatentTaskOwnership ownership) { + AppearancePatentTaskOwnership ownership, + Predicate submissionComplete) { this.moduleType = moduleType; this.taskLockTtl = taskLockTtl; this.fileTaskMapper = fileTaskMapper; @@ -90,6 +94,7 @@ public class AppearancePatentStaleTaskScanner { this.taskDistributedLockService = taskDistributedLockService; this.properties = properties; this.ownership = ownership; + this.submissionComplete = submissionComplete; } public void sweep(StaleDeathFinalizer finalizer) { @@ -144,7 +149,8 @@ public class AppearancePatentStaleTaskScanner { } /** - * 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。 + * 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、客户端已交完但出表没跑、 + * 还是文件生成侧失败。 * 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result", * 把「没开跑」说成「跑挂了」。 */ @@ -158,6 +164,7 @@ public class AppearancePatentStaleTaskScanner { : null, taskHeartbeatPositionService.describe(taskId), taskFileJobService.hasExhaustedAssembleJob(taskId, moduleType), + submissionComplete.test(taskId), properties.getStaleTimeoutMinutes()).describe(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryService.java index 2b8c5722d..07e37091a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryService.java @@ -94,8 +94,14 @@ public class TaskInPlaceRetryService { // 中断已交付过部分结果:先清直链再复位作业,顺序反了会出现「作业已 PENDING、 // 已交付闸还短着」的一轮空转。直链清掉后下载按钮收起,新一轮组装完成再挂新文件。 clearDeliveredResultUrl(task.getId(), task.getModuleType()); - resetDeliveredAssembleJob(task.getId(), task.getModuleType()); } + // 复位「上一代已出过表」的作业:失败代与中断代**都要做**,差别只在直链清不清。 + // 只给中断代做会留下一个永不出表的死循环——那行 SUCCESS 留在原地时, + // enqueueAssembleResult 每次都原样拿回它(底座的 .ne SUCCESS 是刻意的:没人要求重做时 + // 不许替换用户可能已下载的成品),于是采集侧把结果全部传完也轮不到出表、心跳停了 + // 30 分钟再被陈旧扫描判死一次,且文案写成「客户端回传中断」。2026-11-10 现网三单同型: + // 33828 二代/三代(worker 两次「任务处理完成 99/99,抓取失败 0 行」)、33758(657/657)。 + resetDeliveredAssembleJob(task.getId(), task.getModuleType()); clearResultError(task.getId(), task.getModuleType()); resetAssembleJob(task.getId(), task.getModuleType()); log.info("[task-retry] 已在原任务上复位重试 taskId={} moduleType={} 代数={} 上次状态={} 上次失败={}", diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/task/StaleTaskDeathReasonTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/task/StaleTaskDeathReasonTest.java index 6fca24553..e6a8d7c10 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/common/task/StaleTaskDeathReasonTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/common/task/StaleTaskDeathReasonTest.java @@ -13,7 +13,7 @@ class StaleTaskDeathReasonTest { @Test void neverUploadedSaysNoResultUploadedNotThatPythonCrashed() { - String reason = new StaleTaskDeathReason(0L, null, T, null, false, 30).describe(); + String reason = new StaleTaskDeathReason(0L, null, T, null, false, false, 30).describe(); assertTrue(reason.contains("未回传任何检测结果"), reason); assertTrue(reason.contains("超过 30 分钟无进展"), reason); assertTrue(reason.contains("最后心跳 10-01 12:34"), reason); @@ -23,21 +23,23 @@ class StaleTaskDeathReasonTest { @Test void queuedTaskIsToldApartFromACrashedOneByTheReportedPhase() { - String reason = new StaleTaskDeathReason(0L, null, T, "排队等待空闲槽位", false, 30).describe(); + String reason = new StaleTaskDeathReason(0L, null, T, "排队等待空闲槽位", false, false, 30).describe(); assertTrue(reason.contains("最后上报 排队等待空闲槽位"), reason); assertFalse(reason.contains("已收到"), reason); + // 有位置文本=客户端开过工,不得再挂「等待采集槽位」那半句 + assertFalse(reason.contains("等待采集槽位"), reason); } @Test void missingHeartbeatAndBlankPhaseDegradeQuietly() { - String reason = new StaleTaskDeathReason(0L, null, null, " ", false, 30).describe(); + String reason = new StaleTaskDeathReason(0L, null, null, " ", false, false, 30).describe(); assertTrue(reason.contains("最后心跳 时间未知"), reason); assertFalse(reason.contains("最后上报"), reason); } @Test void interruptedUploadReportsChunksAndLastChunkTime() { - String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", false, 30).describe(); + String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", false, false, 30).describe(); assertTrue(reason.contains("客户端回传中断"), reason); assertTrue(reason.contains("已收到 7 个分片,最后一次 10-01 12:34"), reason); assertTrue(reason.contains("最后上报 第 120/500 组"), reason); @@ -46,21 +48,48 @@ class StaleTaskDeathReasonTest { @Test void exhaustedFileJobBlamesFileGenerationInsteadOfTheClient() { - String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", true, 30).describe(); + String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", true, false, 30).describe(); assertTrue(reason.contains("结果文件生成多次失败"), reason); assertFalse(reason.contains("回传中断"), reason); } @Test void exhaustedFileJobWithoutAnyChunkStillReportsClientSilence() { - String reason = new StaleTaskDeathReason(0L, null, T, null, true, 30).describe(); + String reason = new StaleTaskDeathReason(0L, null, T, null, true, false, 30).describe(); assertTrue(reason.contains("未回传任何检测结果"), reason); assertFalse(reason.contains("结果文件生成"), reason); } @Test void nonPositiveTimeoutNeverPrintsZeroMinutes() { - assertTrue(new StaleTaskDeathReason(0L, null, null, null, false, 0) + assertTrue(new StaleTaskDeathReason(0L, null, null, null, false, false, 0) .describe().contains("超过 1 分钟")); } + + /** + * 2026-11-10 现网 33828/33758 的形状:采集侧末分片 done=true 已上传成功 + * (worker 日志「任务处理完成,抓取失败 0 行」),出表作业却停在上一代的 SUCCESS 上没被复位, + * 于是永不出表、心跳停了 30 分钟被判死。原文案「客户端回传中断」把责任判给了客户端, + * 与其实际行为相反,排查方向被带偏(用户以为是采集没传完)。 + */ + @Test + void submissionCompletedTaskBlamesFileGenerationNotTheClient() { + String reason = new StaleTaskDeathReason(12L, T, null, "Linux后台抓取: 98/99", false, true, 30).describe(); + assertTrue(reason.contains("已上传完全部结果"), reason); + assertTrue(reason.contains("结果文件未能生成"), reason); + assertTrue(reason.contains("已收到 12 个分片,最后一次 10-01 12:34"), reason); + assertFalse(reason.contains("回传中断"), reason); + assertFalse(reason.contains("未回传任何检测结果"), reason); + } + + /** + * 2026-11-10 慧赢2 的 33827:建单后 30 分钟没有任何采集槽位认领(采集机该窗全程无日志), + * 却被写成「无进展」,用户无从判断是自己的数据问题还是排队。零位置文本时补一句成因提示, + * 且措辞只到「可能」——没认领这件事本身无法从这两列证明。 + */ + @Test + void neverUploadedWithoutAnyPositionHintsQueuedForCollectorSlot() { + String reason = new StaleTaskDeathReason(0L, null, T, null, false, false, 30).describe(); + assertTrue(reason.contains("可能一直在等待采集槽位"), reason); + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryServiceTest.java index 0ec572d9e..cafb9a63f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskInPlaceRetryServiceTest.java @@ -156,6 +156,52 @@ class TaskInPlaceRetryServiceTest { verify(handler, never()).onTaskResumed(anyLong()); } + /** + * 2026-11-10 现网 33828/33758:上一代**已经出过表**(组装作业停在 SUCCESS)的任务判失败后, + * 重试若不复位那行作业,enqueueAssembleResult 每次原样拿回它、永不出表——采集侧把结果全部传完 + * (worker 日志「任务处理完成 99/99」)也轮不到出表,30 分钟没心跳再被判一次, + * 还写成「客户端回传中断」。失败代与中断代的差别只在直链清不清,作业复位两边都要做。 + */ + @Test + void 失败任务重试也要复位上一代已交付的出表作业() { + FileTaskMapper mapper = mock(FileTaskMapper.class); + FileResultMapper results = mock(FileResultMapper.class); + TaskFileJobService jobs = mock(TaskFileJobService.class); + TaskFileJobMapper jobMapper = mock(TaskFileJobMapper.class); + when(mapper.update(isNull(), any())).thenReturn(1); + FileResultEntity result = new FileResultEntity(); + result.setId(36758L); + when(results.selectOne(any())).thenReturn(result); + when(jobMapper.selectOne(any())).thenReturn(deliveredJob()); + + boolean ok = service(mapper, results, jobs, jobMapper, + mock(ResultFileJobHandlerRegistry.class)).retryInPlace(failedTask()); + + assertTrue(ok); + // 已交付作业被复位成 PENDING(下一轮才排得上出表) + verify(jobMapper).update(isNull(), any()); + // 失败保留直链:只清旧错误文案,不清下载直链(用户手上那份还能下) + verify(results, times(1)).update(isNull(), any()); + verify(jobs).resetTerminalFailedForRecovery(eq(32967L), eq("APPEARANCE_PATENT"), eq(36758L)); + } + + @Test + void 失败任务没有已交付作业时不动物业行() { + FileTaskMapper mapper = mock(FileTaskMapper.class); + FileResultMapper results = mock(FileResultMapper.class); + TaskFileJobMapper jobMapper = mock(TaskFileJobMapper.class); + when(mapper.update(isNull(), any())).thenReturn(1); + FileResultEntity result = new FileResultEntity(); + result.setId(36758L); + when(results.selectOne(any())).thenReturn(result); + when(jobMapper.selectOne(any())).thenReturn(null); + + service(mapper, results, mock(TaskFileJobService.class), jobMapper, + mock(ResultFileJobHandlerRegistry.class)).retryInPlace(failedTask()); + + verify(jobMapper, never()).update(isNull(), any()); + } + @Test void 成功任务不可继续() { FileTaskMapper mapper = mock(FileTaskMapper.class);