From 5f1dc3ac23bbe0010672787d043529453b666862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 24 Oct 2026 16:35:23 +0800 Subject: [PATCH] =?UTF-8?q?fix(task):=20=E8=A1=A5=E6=A3=80=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E4=BB=BB=E5=8A=A1=E4=B8=8D=E5=86=8D=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=85=8B=E9=9A=86=E9=87=8D=E8=B7=91=EF=BC=8C=E5=8D=A1=E6=AD=BB?= =?UTF-8?q?=E9=87=8D=E6=8E=92=E4=B8=8D=E6=B6=88=E8=80=97=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E9=A2=84=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TaskResumeService:名下还有 PENDING/RUNNING 组装作业的 FAILED 任务不克隆 (现网 2026-10-24 一枚 760 行外观专利被连克隆 3 代,每代全量重跑、同秒显示) - resetStuckRunningJobsDetailed:RUNNING 行重排不再 +1 retry_count,判死只剩 markFailed 一条真失败的路——补检一轮几十分钟的慢执行不再被记成失败次数 - 新增 hasUnfinishedAssembleJob;测试按新口径重写并补现网回归两例 --- .../task/service/TaskFileJobService.java | 49 ++++++++++++------- .../task/service/TaskResumeService.java | 17 +++++++ .../task/service/TaskFileJobServiceTest.java | 42 ++++++++++------ .../task/service/TaskResumeServiceTest.java | 36 +++++++++++++- 4 files changed, 110 insertions(+), 34 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index 00bba74d3..498f919f5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.nanri.aiimage.common.task.TaskStatuses; import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; @@ -102,6 +103,25 @@ public class TaskFileJobService { return retry >= MAX_RETRY_COUNT; } + /** + * 该任务名下还有没有**未终态**的组装作业(PENDING/RUNNING)。 + * + *

交付缺口触发的补检轮就是把作业排回这两态({@link #requeue}),所以"有活着的组装作业" + * ="这单还在补检"。而任务行可能早就落成 FAILED(心跳超时或交付判死先落),此时按任务状态 + * 去克隆重跑,等于同一份活开两份:现网 2026-10-24 一枚 760 行的外观专利因此被克隆成 4 枚、 + * 每枚都全量重跑一遍,四枚卡片的开始时间还都是同一秒(克隆件继承原件 created_at)。 + */ + public boolean hasUnfinishedAssembleJob(Long taskId) { + if (taskId == null) { + return false; + } + Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskFileJobEntity::getTaskId, taskId) + .eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT) + .in(TaskFileJobEntity::getStatus, TaskStatuses.Job.PENDING, TaskStatuses.Job.RUNNING)); + return count != null && count > 0; + } + /** * 补传完整后恢复终态失败的组装 job:重置为 PENDING(retryCount 清零)重新派发。 * 幂等:job 不存在、非 FAILED 或未达重试上限时不动作,返回当前 job。 @@ -310,6 +330,10 @@ public class TaskFileJobService { * 每天上百次 "Lock wait timeout exceeded"(biz_task_file_job)。 * 这里的每行更新本就带 status + updatedAt 的 CAS 条件(幂等),逐行独立提交更安全: * 锁即时释放,CAS 不匹配的一方返回 0 行即可,也不会因中途异常回滚掉已修好的行。 + * + *

RUNNING 行的重排**不消耗** retry_count(2026-10-24 分账口径):扫描只负责把卡住的行 + * 放回队列,预算只由真失败({@link #markFailed})推进——"补检一轮要几十分钟"的慢执行不该 + * 被记成失败次数。PENDING 行的累加保留:那条防的是 dispatch 事件无限重发,与执行快慢无关。 */ public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) { LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes)); @@ -367,30 +391,21 @@ public class TaskFileJobService { } continue; } - int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount(); - int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1); + // 重排**不消耗**重试预算(2026-10-24 拍板的分账口径):卡死扫描只负责把卡住的 + // RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在 + // MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行 + // 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后 + // 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、 + // 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。 LocalDateTime now = LocalDateTime.now(); - LambdaUpdateWrapper update = new LambdaUpdateWrapper() + int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper() .eq(TaskFileJobEntity::getId, job.getId()) .eq(TaskFileJobEntity::getStatus, "RUNNING") .eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt()) .isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt) - .set(TaskFileJobEntity::getRetryCount, nextRetryCount) - .set(TaskFileJobEntity::getUpdatedAt, now); - if (nextRetryCount >= MAX_RETRY_COUNT) { - int updated = taskFileJobMapper.update(null, update - .set(TaskFileJobEntity::getStatus, "FAILED") - .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数") - .set(TaskFileJobEntity::getFinishedAt, now) - .set(TaskFileJobEntity::getTerminalCallbackAt, null)); - if (updated > 0) { - exhaustedJobIds.add(job.getId()); - } - continue; - } - int updated = taskFileJobMapper.update(null, update .set(TaskFileJobEntity::getStatus, "PENDING") .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队") + .set(TaskFileJobEntity::getUpdatedAt, now) .set(TaskFileJobEntity::getFinishedAt, null) .set(TaskFileJobEntity::getTerminalCallbackAt, null)); if (updated > 0) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java index fec9e1000..a758047c4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java @@ -65,6 +65,11 @@ public class TaskResumeService { private static final int DUPLICATE_SWEEP_LIMIT = 200; private final FileTaskMapper fileTaskMapper; + /** + * 查"这单名下还有没有活着的组装作业"——补检轮还活着的意思,见 + * {@link TaskFileJobService#hasUnfinishedAssembleJob(Long)}。 + */ + private final TaskFileJobService taskFileJobs; /** moduleType → 克隆重试实现;只有注册了本 SPI 的模块才会自动重试。 */ private final Map retryHandlers; @@ -87,8 +92,10 @@ public class TaskResumeService { private int limit; public TaskResumeService(FileTaskMapper fileTaskMapper, + TaskFileJobService taskFileJobs, List retrySpiHandlers) { this.fileTaskMapper = fileTaskMapper; + this.taskFileJobs = taskFileJobs; Map retryIndex = new LinkedHashMap<>(); if (retrySpiHandlers != null) { for (ClientTaskRetrySpi handler : retrySpiHandlers) { @@ -216,6 +223,16 @@ public class TaskResumeService { stats.skippedTaskCount++; continue; } + // 任务行是 FAILED,不等于这份活干完了:交付缺口会把组装作业排回 PENDING/RUNNING 去补检 + // (见 AppearancePatentDeliverySupport.requestRecheckForMissingRows),此时克隆=同一份活 + // 开两份、还各按全量重跑一遍。作业真卡死另有 TaskResultFileJobWorker 的 stuck 扫描兜底, + // 不会因为这条例外就永远没人管。 + if (taskFileJobs.hasUnfinishedAssembleJob(original.getId())) { + stats.skippedTaskCount++; + log.info("[task-resume] 不重试 taskId={} moduleType={} 原因=组装作业仍在补检(PENDING/RUNNING)", + original.getId(), original.getModuleType()); + continue; + } ClientTaskRetrySpi handler = retryHandlers.get(normalize(original.getModuleType())); if (handler == null) { stats.skippedTaskCount++; diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java index df7c9802b..f6d3789be 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java @@ -44,9 +44,9 @@ class TaskFileJobServiceTest { } @Test - void stuckJobIsRequeuedAndRetryCountIsIncremented() { + void stuckJobIsRequeuedWithoutConsumingRetryBudget() { TaskFileJobEntity running = runningJob(101L, 3, LocalDateTime.now().minusHours(1)); - TaskFileJobEntity pending = runningJob(101L, 4, LocalDateTime.now()); + TaskFileJobEntity pending = runningJob(101L, 3, LocalDateTime.now()); pending.setStatus("PENDING"); when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running)); when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); @@ -58,29 +58,29 @@ class TaskFileJobServiceTest { assertEquals(1, result.resetCount()); assertTrue(result.exhaustedJobs().isEmpty()); verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class)); - assertUpdateContains(4, running.getUpdatedAt()); + assertUpdateLeavesRetryCountIntact(running.getUpdatedAt()); } + /** + * 现网 2026-10-24:一枚 760 行的外观专利组装作业在补检轮里被卡死扫描连扣预算, + * RUNNING + retryCount=4 一进扫描就被就地判死「文件生成任务运行超时,已达到最大重试次数」, + * 结果文件 0 字节收场。改口径后扫描只放回队列:到达 MAX 的终态只能由真失败(markFailed)走出。 + */ @Test - void stuckJobAtLastRetryWaitsForDurableFailureFinalization() { + void stuckJobAtLastRetryIsRequeuedWithoutTerminalKill() { TaskFileJobEntity running = runningJob(102L, 4, LocalDateTime.now().minusHours(1)); - TaskFileJobEntity failed = runningJob(102L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now()); - failed.setStatus("FAILED"); - /* - failed.setErrorMessage("文件生成任务运行超时,已达到最大重试次数"); - */ - failed.setErrorMessage("result file job timeout"); + TaskFileJobEntity pending = runningJob(102L, 4, LocalDateTime.now()); + pending.setStatus("PENDING"); when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running)); when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); - when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(failed)); + when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(pending)); TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher); TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20); - assertEquals(0, result.resetCount()); - assertEquals(List.of(failed), result.exhaustedJobs()); - verify(applicationEventPublisher, never()).publishEvent(any()); - assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt()); + assertEquals(1, result.resetCount()); + assertTrue(result.exhaustedJobs().isEmpty()); + assertUpdateLeavesRetryCountIntact(running.getUpdatedAt()); } @Test @@ -125,7 +125,7 @@ class TaskFileJobServiceTest { assertTrue(result.exhaustedJobs().isEmpty()); verify(taskFileJobMapper, never()).selectById(any()); verify(applicationEventPublisher, never()).publishEvent(any()); - assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt()); + assertUpdateLeavesRetryCountIntact(running.getUpdatedAt()); } @Test @@ -308,6 +308,16 @@ class TaskFileJobServiceTest { assertTrue(update.getValue().getParamNameValuePairs().containsValue(nextRetry)); } + /** 卡死重排的分账口径:SET 片段里不许出现 retry_count,预算只由真失败(markFailed)推进。 */ + private void assertUpdateLeavesRetryCountIntact(LocalDateTime updatedAt) { + ArgumentCaptor> update = updateCaptor(); + verify(taskFileJobMapper).update(isNull(), update.capture()); + assertTrue(update.getValue().getSqlSegment().contains("updated_at")); + assertTrue(update.getValue().getParamNameValuePairs().containsValue(updatedAt)); + assertFalse(update.getValue().getSqlSet().contains("retry_count"), + "卡死重排不许碰 retry_count"); + } + @SuppressWarnings({"rawtypes", "unchecked"}) private static ArgumentCaptor> updateCaptor() { return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java index c3cfadda2..f4659e9d9 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java @@ -60,7 +60,15 @@ class TaskResumeServiceTest { private static TaskResumeService service(FileTaskMapper mapper, List handlers, boolean enabled, int maxAttempt) { - TaskResumeService service = new TaskResumeService(mapper, handlers); + // 默认"名下没有活着的组装作业"(Mockito 对 boolean 返 false):既有用例判的都是克隆本体, + // 补检豁免那条单独用 serviceWithLiveAssembleJob 覆盖。 + return service(mapper, mock(TaskFileJobService.class), handlers, enabled, maxAttempt); + } + + private static TaskResumeService service(FileTaskMapper mapper, TaskFileJobService jobs, + List handlers, + boolean enabled, int maxAttempt) { + TaskResumeService service = new TaskResumeService(mapper, jobs, handlers); ReflectionTestUtils.setField(service, "enabled", enabled); ReflectionTestUtils.setField(service, "maxAttempt", maxAttempt); ReflectionTestUtils.setField(service, "windowMinutes", 30L); @@ -138,6 +146,32 @@ class TaskResumeServiceTest { verify(spi, never()).cloneForRetry(any()); } + /** + * 现网 2026-10-24:一枚 760 行的外观专利任务因交付缺口把组装作业排回补检(作业还在 + * PENDING/RUNNING),任务行却已是 FAILED,于是被巡检连克隆成 4 枚、每枚全量重跑一遍, + * 四枚卡片开始时间还同秒(克隆件继承原件 created_at)。非零对照=上面那条 + * 「失败任务会被克隆重试」:作业已终态时照旧克隆。 + */ + @Test + void 组装作业仍在补检时不克隆() { + FileTaskMapper mapper = mock(FileTaskMapper.class); + when(mapper.selectList(any())).thenReturn(List.of( + failedTask(32201L, "APPEARANCE_PATENT", "有 3 行抓取失败(未取得商品页数据):B07P5TR7Q5"))); + when(mapper.selectCount(any())).thenReturn(0L); + TaskFileJobService jobs = mock(TaskFileJobService.class); + when(jobs.hasUnfinishedAssembleJob(32201L)).thenReturn(true); + ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class); + when(spi.moduleType()).thenReturn("APPEARANCE_PATENT"); + TaskResumeService service = service(mapper, jobs, List.of(spi), true, 3); + + TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks(); + + assertEquals(1, stats.scannedTaskCount); + assertEquals(1, stats.skippedTaskCount); + assertEquals(0, stats.resumedTaskCount); + verify(spi, never()).cloneForRetry(any()); + } + @Test void 重试代数到上限后跳过() { FileTaskMapper mapper = mock(FileTaskMapper.class);