From 583dad92ecd74e8247839ecc763428097eac8042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sun, 25 Oct 2026 14:28:57 +0800 Subject: [PATCH] =?UTF-8?q?fix(task):=20=E7=BB=88=E6=80=81=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E4=BF=9D=E7=95=99=E5=A4=B1=E8=B4=A5=E5=85=9C=E5=BA=95?= =?UTF-8?q?=E5=87=BA=E9=83=A8=E5=88=86=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B28 的终态守卫把父任务已失败的结果作业直接标终态;terminal_callback_at 一打, stuck 扫描就再也捞不到它,fallbackAssembleOnFailure 不再执行,失败任务永远没有结果文件。 改为判死但保留兜底回调,由归属实例收口;已终态任务的补检直接放行,按已有分片出部分结果。 --- .../AppearancePatentDeliverySupport.java | 15 +++++++-- .../task/service/TaskFileJobService.java | 32 ++++++++++++++++--- .../task/service/TaskResultFileJobWorker.java | 15 +++++++-- .../AppearancePatentDeliveryGateTest.java | 17 ++++++++-- .../TaskFileJobTerminalTaskGuardTest.java | 5 +++ .../TaskResultFileJobWorkerStuckScanTest.java | 25 +++++++++++++-- 6 files changed, 94 insertions(+), 15 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentDeliverySupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentDeliverySupport.java index d47dc8dc1..a88d54403 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentDeliverySupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentDeliverySupport.java @@ -159,6 +159,10 @@ public class AppearancePatentDeliverySupport { * 「因为慢而抛满重试」不再等于「永远补不了」。 * *

预算耗尽、或重排没接上时仍按已有结果出表:「起码失败了也可以下载」那条保证不变。 + * + *

父任务已终态时不再排补检:B28 的终态守卫不会让结果作业再跑,重排只会被下一轮 + * claim 立刻打回 FAILED,白转一圈(现网 32217/32220 锁续 86 分钟就是这条循环的旧形态)。 + * 终态任务直接按已有分片出部分结果,缺口由行账文案标在表里。 */ public boolean scheduleFallbackRecheck(FileTaskEntity task, FileResultEntity result, AppearancePatentRowOutcome.Facts facts) { @@ -167,15 +171,20 @@ public class AppearancePatentDeliverySupport { || facts == null || maxRounds <= 0) { return false; } + if (facts.missingRows() <= 0 || facts.missingKeys().isEmpty()) { + return false; + } + if (TaskStatuses.FileTask.TERMINAL.contains(task.getStatus())) { + log.warn("[appearance-patent] 任务已终态,补检不再执行,按已有结果交付 taskId={} status={} missingRows={} keys={}", + task.getId(), task.getStatus(), facts.missingRows(), facts.missingKeys().size()); + return false; + } long done = taskCacheService.recheckRounds(task.getId()); if (done >= maxRounds) { log.warn("[appearance-patent] 补检预算已耗尽,按已有结果交付 taskId={} rounds={}/{}", task.getId(), done, maxRounds); return false; } - if (facts.missingRows() <= 0 || facts.missingKeys().isEmpty()) { - return false; - } taskCacheService.releaseRowInspection(task.getId(), facts.missingKeys(), true); TaskFileJobEntity reset = taskFileJobService.resetTerminalFailedForRecovery( task.getId(), moduleType, result.getId()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index f2a1e0a9d..b7865034f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -239,10 +239,16 @@ public class TaskFileJobService { return status != null && TaskStatuses.FileTask.TERMINAL.contains(status); } - /** 终态父任务的 job 直接拉满重试并打终态回调:不执行、不再被下一轮扫到。 */ + /** + * 终态父任务的 job 直接拉满重试:不执行、不再被下一轮扫到。 + * + *

终态回调要留给卡死扫描的失败兜底({@code fallbackAssembleOnFailure}):外观专利等模块 + * 靠它用已回传分片出部分结果文件,「起码失败了也可以下载」才不会被 B28 的终态守卫顺手掐掉。 + * 这里若直接打 {@code terminal_callback_at},兜底扫描再也捞不到这行,用户只剩红字、连半张表都没有。 + */ private void skipJobOfTerminalTask(TaskFileJobEntity job, String stage) { String message = "父任务已终态,结果作业不再执行"; - markFailedPermanent(job, message); + markFailedPermanentAwaitingFallback(job, message); log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage={}", job.getId(), job.getTaskId(), job.getModuleType(), stage); } @@ -801,15 +807,31 @@ public class TaskFileJobService { * 适用于 task/result 已经不存在等无法通过重试恢复的场景。 */ public void markFailedPermanent(TaskFileJobEntity job, String message) { + markFailedPermanent(job, message, true); + } + + /** + * 父任务已终态时的作业收口:同样判死并拉满重试,但**不打 terminal_callback_at**。 + * + *

不打标不是漏收尾:卡死扫描专门捞「FAILED + retry 到顶 + terminal_callback_at 为空」的行, + * 由它调用模块的失败兜底出部分结果(外观专利任务 31302/31354 的「起码失败了也可以下载」)。 + * 这里一旦打标,兜底路径就被永久跳过,失败任务的 result_file_url 永远为空,前端连下载按钮都没有。 + */ + public void markFailedPermanentAwaitingFallback(TaskFileJobEntity job, String message) { + markFailedPermanent(job, message, false); + } + + private void markFailedPermanent(TaskFileJobEntity job, String message, boolean finalizeCallback) { + LocalDateTime now = LocalDateTime.now(); taskFileJobMapper.update(null, new LambdaUpdateWrapper() .eq(TaskFileJobEntity::getId, job.getId()) .ne(TaskFileJobEntity::getStatus, "SUCCESS") .set(TaskFileJobEntity::getStatus, "FAILED") .set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT) .set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message) - .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()) - .set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()) - .set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now())); + .set(TaskFileJobEntity::getUpdatedAt, now) + .set(TaskFileJobEntity::getFinishedAt, now) + .set(TaskFileJobEntity::getTerminalCallbackAt, finalizeCallback ? now : null)); } /** diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index df0696fa0..d5ae3336f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -102,6 +102,14 @@ public class TaskResultFileJobWorker { result.resetCount(), result.exhaustedJobs().size(), stuckTimeoutMinutes); } for (TaskFileJobEntity job : result.exhaustedJobs()) { + // 终态兜底出表必须落在归属实例:fallbackAssembleResultFile 在非归属机会直接返回 false, + // 但 finalizeRetryExhausted 随后仍会打 terminal_callback_at,归属机的兜底就永远跑不到了。 + if (isOwnerScopedJob(job) && !isOwnedByCurrentInstance(job)) { + log.info("[task-file-job] skip exhausted owner-scoped job of another instance jobId={} taskId={} moduleType={} owner={} current={} stage=SKIP_OWNER", + job.getId(), job.getTaskId(), job.getModuleType(), + ownerFromScopeKey(job.getScopeKey()), currentInstanceId()); + continue; + } String message = job.getErrorMessage() == null || job.getErrorMessage().isBlank() ? "文件生成任务运行超时,已达到最大重试次数" : job.getErrorMessage(); @@ -215,7 +223,8 @@ public class TaskResultFileJobWorker { } // 终态后不再 touchRunning:否则 job 的 updated_at 一直被刷新, // job 级 stuck 扫描永远看不到这行,成为不可回收的 RUNNING 遗骸。 - taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行"); + // 判死但不打 terminal_callback_at:留给兜底扫描出部分结果,失败任务仍可下载。 + taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行"); log.info("[task-file-job] parent task terminal; stopped renewal and failed job jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW", job.getId(), job.getTaskId(), job.getModuleType()); } @@ -231,7 +240,7 @@ public class TaskResultFileJobWorker { private void processClaimedWithHeartbeat(TaskFileJobEntity job, TaskFileJobEntity claim) { if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) { - taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行"); + taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行"); log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_SKIP", job.getId(), job.getTaskId(), job.getModuleType()); return; @@ -279,7 +288,7 @@ public class TaskResultFileJobWorker { } if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) { try (lockHandle) { - taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行"); + taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行"); } log.info("[task-file-job] skipped after acquiring lock because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_AFTER_LOCK", job.getId(), job.getTaskId(), job.getModuleType()); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentDeliveryGateTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentDeliveryGateTest.java index 60f06f115..2fb95523c 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentDeliveryGateTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentDeliveryGateTest.java @@ -371,7 +371,7 @@ class AppearancePatentDeliveryGateTest { when(taskFileJobService.resetTerminalFailedForRecovery(TASK_ID, "APPEARANCE_PATENT", 700L)) .thenReturn(pendingJob(500L)); - assertTrue(scheduleFallbackRecheck(task("FAILED"), result(700L), facts)); + assertTrue(scheduleFallbackRecheck(task("RUNNING"), result(700L), facts)); verify(taskCacheService).releaseRowInspection(eq(TASK_ID), anyList(), eq(true)); verify(taskCacheService).countRecheckRound(TASK_ID); } @@ -382,7 +382,20 @@ class AppearancePatentDeliveryGateTest { AppearancePatentRowOutcome.Facts facts = summarize(rows, submittedRows(rows, 1, true)); when(taskCacheService.recheckRounds(TASK_ID)).thenReturn(2L); + assertFalse(scheduleFallbackRecheck(task("RUNNING"), result(700L), facts)); + verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), any(), anyLong()); + } + + @Test + void terminalTaskWithGapShipsPartialInsteadOfRechecking() throws Exception { + // B28 起终态任务的结果作业不会再被 claim;再排补检只会被下一轮终态守卫立刻打回 FAILED。 + // 终态任务直接按已有分片出部分结果,缺口由行账文案标在表里。 + List rows = sheetRows(2); + AppearancePatentRowOutcome.Facts facts = summarize(rows, submittedRows(rows, 1, true)); + assertFalse(scheduleFallbackRecheck(task("FAILED"), result(700L), facts)); + verify(taskCacheService, never()).recheckRounds(anyLong()); + verify(taskCacheService, never()).releaseRowInspection(anyLong(), anyList(), eq(true)); verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), any(), anyLong()); } @@ -406,7 +419,7 @@ class AppearancePatentDeliveryGateTest { when(taskFileJobService.resetTerminalFailedForRecovery(TASK_ID, "APPEARANCE_PATENT", 700L)) .thenReturn(job(500L)); - assertFalse(scheduleFallbackRecheck(task("FAILED"), result(700L), facts)); + assertFalse(scheduleFallbackRecheck(task("RUNNING"), result(700L), facts)); verify(taskCacheService, never()).countRecheckRound(anyLong()); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java index 2689764b3..66afdeeb8 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java @@ -92,6 +92,9 @@ class TaskFileJobTerminalTaskGuardTest { assertTrue(hasParam(wrapper, "FAILED"), "必须把作业拉成 FAILED,下一轮扫描才捞不到"); assertTrue(hasParam(wrapper, 5), "retry_count 必须一次拉满到上限"); assertFalse(hasParam(wrapper, "RUNNING"), "不得再翻转成 RUNNING"); + assertTrue(wrapper.getSqlSet().contains("terminal_callback_at"), + "terminal_callback_at 必须显式清空:留给兜底扫描出部分结果,失败任务仍可下载"); + assertTrue(hasParam(wrapper, null), "terminal_callback_at 置空后兜底扫描才捞得到这一行"); } @Test @@ -118,6 +121,8 @@ class TaskFileJobTerminalTaskGuardTest { verify(taskFileJobMapper, times(2)).update(isNull(), captor.capture()); assertTrue(hasParam(captor.getAllValues().get(0), "FAILED"), "第一笔是终态收尾"); assertTrue(hasParam(captor.getAllValues().get(1), "RUNNING"), "第二笔才是正常 claim"); + assertTrue(captor.getAllValues().get(0).getSqlSet().contains("terminal_callback_at"), + "终态作业必须留给兜底扫描,不能提前打终态回调"); } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java index 74bf531ad..ee564588f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java @@ -355,7 +355,7 @@ class TaskResultFileJobWorkerStuckScanTest { @Test void terminalParentHeartbeatStopsTouchingAndFailsJob() throws Exception { // 父任务在 worker 执行中变终态:心跳不得再 touchRunning(否则 stuck 扫描永远看不到), - // 并主动把 job 收口为 FAILED(终态回调由 markFailedPermanent 打标)。 + // 并主动把 job 收口为 FAILED;终态回调留给兜底扫描出部分结果,失败任务仍可下载。 TaskResultFileJobWorker worker = buildWorker(); setLong(worker, "heartbeatIntervalMillis", 10L); TaskFileJobEntity job = new TaskFileJobEntity(); @@ -373,13 +373,34 @@ class TaskResultFileJobWorkerStuckScanTest { try { // heartbeat 间隔下限 1000ms,首轮在 1s 后才跑;给足调度余量 verify(taskFileJobService, timeout(3000)) - .markFailedPermanent(eq(job), contains("父任务已终态")); + .markFailedPermanentAwaitingFallback(eq(job), contains("父任务已终态")); verify(taskFileJobService, after(150).never()).touchRunning(77L); } finally { future.cancel(true); } } + /** + * 终态兜底出表只在归属实例做:非归属实例若抢先 markFailureFinalized,归属机的 + * fallbackAssembleOnFailure 就永远跑不到,失败任务的结果文件也就永远不生成。 + */ + @Test + void exhaustedOwnerScopedJobOfAnotherInstanceIsLeftToItsOwner() throws Exception { + TaskResultFileJobWorker worker = buildWorker(); + when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + TaskFileJobEntity job = exhaustedJob(9L, 19L, "父任务已终态,结果作业不再执行"); + job.setModuleType("APPEARANCE_PATENT"); + job.setScopeKey("appearance-patent:19:owner:instance-b"); + TaskFileJobService.StuckJobResetResult result = + new TaskFileJobService.StuckJobResetResult(0, List.of(job)); + when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0)).thenReturn(result); + + worker.resetStuckJobs(); + + verify(appearance, never()).fallbackAssembleResultFile(any()); + verify(taskFileJobService, never()).markFailureFinalized(any(), anyString()); + } + private static void setLong(TaskResultFileJobWorker worker, String fieldName, long value) throws Exception { Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName); field.setAccessible(true);