diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java index 0631481d7..dac41e12a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java @@ -4,7 +4,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -40,6 +42,8 @@ public class StaleTaskRepairService { private static final int MAX_BRAND_STALE_BATCH = 500; private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final TaskFileJobService taskFileJobService; private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi; private final DistributedJobLockService distributedJobLockService; /** 心跳时钟改为「从首次心跳起算」:排队中的 RUNNING 任务不在这条线上判死。 */ @@ -78,16 +82,17 @@ public class StaleTaskRepairService { return; } String reason = "任务长期未被领取,已自动失败"; + List ids = stale.stream().map(FileTaskEntity::getId).toList(); LocalDateTime now = LocalDateTime.now(); int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() - .in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList()) + .in(FileTaskEntity::getId, ids) .eq(FileTaskEntity::getStatus, STATUS_PENDING) .set(FileTaskEntity::getStatus, STATUS_FAILED) .set(FileTaskEntity::getErrorMessage, reason) .set(FileTaskEntity::getFinishedAt, now)); if (updated > 0) { - log.warn("[stale-task-repair] 陈旧中间态已标失败 count={} ids={}", - updated, stale.stream().map(FileTaskEntity::getId).toList()); + log.warn("[stale-task-repair] 陈旧中间态已标失败 count={} ids={}", updated, ids); + markDeadDownstream(ids, reason); } } @@ -114,16 +119,17 @@ public class StaleTaskRepairService { return; } String reason = "定时任务到点后长时间未启动,已自动失败"; + List ids = stale.stream().map(FileTaskEntity::getId).toList(); LocalDateTime now = LocalDateTime.now(); int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() - .in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList()) + .in(FileTaskEntity::getId, ids) .eq(FileTaskEntity::getStatus, STATUS_SCHEDULED) .set(FileTaskEntity::getStatus, STATUS_FAILED) .set(FileTaskEntity::getErrorMessage, reason) .set(FileTaskEntity::getFinishedAt, now)); if (updated > 0) { - log.warn("[stale-task-repair] 陈旧定时任务已标失败 count={} ids={}", - updated, stale.stream().map(FileTaskEntity::getId).toList()); + log.warn("[stale-task-repair] 陈旧定时任务已标失败 count={} ids={}", updated, ids); + markDeadDownstream(ids, reason); } } @@ -144,16 +150,17 @@ public class StaleTaskRepairService { return; } String reason = "任务心跳超时,已自动失败"; + List ids = stale.stream().map(FileTaskEntity::getId).toList(); LocalDateTime now = LocalDateTime.now(); int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() - .in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList()) + .in(FileTaskEntity::getId, ids) .eq(FileTaskEntity::getStatus, STATUS_RUNNING) .set(FileTaskEntity::getStatus, STATUS_FAILED) .set(FileTaskEntity::getErrorMessage, reason) .set(FileTaskEntity::getFinishedAt, now)); if (updated > 0) { - log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}", - updated, stale.stream().map(FileTaskEntity::getId).toList()); + log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}", updated, ids); + markDeadDownstream(ids, reason); } } @@ -191,4 +198,35 @@ public class StaleTaskRepairService { log.warn("[stale-task-repair] brand 陈旧任务已标失败 count={} ids={}", failedIds.size(), failedIds); } } + + /** + * 判死必须连带改口下游两处,否则只改 {@code biz_file_task} 会留下两种假象: + * + *

1){@code biz_file_result.success} 是 {@code NOT NULL DEFAULT 1} 的列,只有走正常收尾才回写。 + * 被兜底判死的任务这一位仍是 1,而历史列表拿它当「已完成」的判据之一,于是显示「已完成」却没有任何 + * 结果文件(2026-10-16 任务 31060:客户端只回传 51/510 个分片,判死后结果行仍是 success=1)。 + * 只改没有结果文件的行——文件已经补出来的任务,其部分交付提示(error_message)不能被这里覆盖。 + * + *

2)结果文件作业仍 RUNNING 会被 worker 反复 defer 续命、永不出栈,见 + * {@link TaskFileJobService#failUnfinishedJobs}。 + * + *

批内并发:下游按「本轮 SELECT 到的陈旧 id」写,而任务行的 UPDATE 带 status 条件,理论上可波及 + * 同批里刚被别的写入方推走的任务。窗口是同一次方法调用内 SELECT→UPDATE 的毫秒级,且候选已被要求 + * 2 小时无任何更新,故按批处理、不逐行复核。 + */ + private void markDeadDownstream(List taskIds, String reason) { + if (taskIds.isEmpty()) { + return; + } + fileResultMapper.update(null, new LambdaUpdateWrapper() + .in(FileResultEntity::getTaskId, taskIds) + .eq(FileResultEntity::getSuccess, 1) + .and(w -> w.isNull(FileResultEntity::getResultFileUrl) + .or().eq(FileResultEntity::getResultFileUrl, "")) + .set(FileResultEntity::getSuccess, 0) + .set(FileResultEntity::getErrorMessage, reason)); + int jobs = taskFileJobService.failUnfinishedJobs(taskIds, reason); + log.warn("[stale-task-repair] 判死下游已改口(不影响本轮判死)tasks={} jobs={} reason={}", + taskIds.size(), jobs, reason); + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index a81d62935..00bba74d3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -734,6 +734,33 @@ public class TaskFileJobService { .set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now())); } + /** + * 父任务已被判死时,终结它仍未结束的组装作业。 + * + *

不终结的后果(2026-10-16 线上作业 22136 / 任务 31060):worker 每 15 秒把作业捞起来走 + * {@code touchRunning} 续命,{@code updated_at} 因此一直被刷新,而 job 级 stuck 扫描正是按 + * {@code updated_at} 判陈旧的 → 永不命中,作业随已死的父任务空转 28 小时。 + * + *

与 {@link #markFailedPermanent} 同口径:重试拉满(FAILED 而 retry_count 未达上限的作业会被 + * {@code listRunnableJobs} 再次捞起)+ 打 {@code terminal_callback_at}(否则失败收尾扫描会去回调 + * 已被判死的父任务与结果行)。 + */ + public int failUnfinishedJobs(List taskIds, String message) { + if (taskIds == null || taskIds.isEmpty()) { + return 0; + } + LocalDateTime now = LocalDateTime.now(); + return taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .in(TaskFileJobEntity::getTaskId, taskIds) + .in(TaskFileJobEntity::getStatus, "RUNNING", "PENDING") + .set(TaskFileJobEntity::getStatus, "FAILED") + .set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT) + .set(TaskFileJobEntity::getErrorMessage, message == null ? "父任务已判死,结果文件作业同步结束" : message) + .set(TaskFileJobEntity::getUpdatedAt, now) + .set(TaskFileJobEntity::getFinishedAt, now) + .set(TaskFileJobEntity::getTerminalCallbackAt, now)); + } + /** * 该任务是否已有「重试耗尽且已走完终态回调」的组装 job。 * diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairServiceTest.java index 7f5b6dff8..6a418f17d 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairServiceTest.java @@ -5,7 +5,9 @@ import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi; import org.apache.ibatis.builder.MapperBuilderAssistant; @@ -39,16 +41,20 @@ import static org.mockito.Mockito.when; class StaleTaskRepairServiceTest { private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class); + private final FileResultMapper fileResultMapper = mock(FileResultMapper.class); + private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class); private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi = mock(BrandTaskStaleRepairSpi.class); private final DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class); private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy = mock(StaleTaskHeartbeatPolicy.class); private final StaleTaskRepairService service = new StaleTaskRepairService( - fileTaskMapper, brandTaskStaleRepairSpi, distributedJobLockService, staleTaskHeartbeatPolicy); + fileTaskMapper, fileResultMapper, taskFileJobService, brandTaskStaleRepairSpi, distributedJobLockService, + staleTaskHeartbeatPolicy); @BeforeEach void initTableInfo() { // LambdaWrapper 需要实体列缓存,单测无 Spring/MyBatis 上下文,手动初始化 TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class); + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class); } @Test @@ -96,9 +102,55 @@ class StaleTaskRepairServiceTest { ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleIdle"); ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleScheduled"); + ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning"); verify(fileTaskMapper, never()).update(any(), any()); + verify(fileResultMapper, never()).update(any(), any()); verifyNoMoreInteractions(brandTaskStaleRepairSpi); + verify(taskFileJobService, never()).failUnfinishedJobs(any(), any()); + } + + // 2026-10-16 任务 31060:客户端只回传 51/510 个分片后被本方法判死,但 biz_file_result.success + // 是 `NOT NULL DEFAULT 1` 的列、只有正常收尾才回写,于是历史列表拿这一位显示「已完成」, + // 而结果文件从未存在(下载按钮的判据 result_file_url 为空)。判死必须同时把结果行改口。 + @Test + void 心跳判死必须同步把无结果文件的结果行改判失败() { + when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(31060L, "RUNNING"))); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); + when(fileResultMapper.update(isNull(), any())).thenReturn(1); + + ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning"); + + ArgumentCaptor> captor = resultUpdateCaptor(); + verify(fileResultMapper).update(isNull(), captor.capture()); + LambdaUpdateWrapper wrapper = captor.getValue(); + String set = wrapper.getSqlSet(); + assertTrue(set.contains("success"), "结果行的 success 位必须被改写,否则前端仍读到默认值 1"); + assertTrue(set.contains("error_message"), "判死原因要留痕到结果行,页面才说得出为什么失败"); + assertFalse(set.contains("updated_at"), + "updated_at 由数据库维护,显式写会冻结结果行的版本信号(见 FileResultEntity 注释)"); + Map params = paramsOf(wrapper); + assertTrue(params.containsValue(0), "success 写 0"); + assertTrue(containsMessage(params, "心跳超时"), "留痕文案与任务行同源"); + + String where = wrapper.getSqlSegment(); + assertTrue(where.contains("result_file_url"), + "只改没有结果文件的行:文件已生成的任务不许被抹掉部分交付提示"); + assertTrue(where.contains("task_id"), "按本次判死的任务圈定"); + } + + @Test + void 心跳判死必须终结仍_RUNNING_的组装作业() { + when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(31060L, "RUNNING"))); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); + + ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning"); + + ArgumentCaptor> ids = listCaptor(); + ArgumentCaptor reason = ArgumentCaptor.forClass(String.class); + verify(taskFileJobService).failUnfinishedJobs(ids.capture(), reason.capture()); + assertEquals(List.of(31060L), ids.getValue()); + assertTrue(reason.getValue().contains("心跳超时"), "作业行的判死原因与任务行同源"); } // 2026-10-17 实录:任务一被认领就记 RUNNING,而客户端要等执行器真开始跑才发心跳(同店占用 / @@ -117,6 +169,17 @@ class StaleTaskRepairServiceTest { verify(fileTaskMapper, never()).update(isNull(), any()); } + @Test + void 未被领取判死同样要回写结果行并终结作业() { + when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(31176L, "PENDING"))); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); + + ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleIdle"); + + verify(fileResultMapper).update(isNull(), any()); + verify(taskFileJobService).failUnfinishedJobs(eq(List.of(31176L)), any()); + } + private Map selectParams() { ArgumentCaptor> captor = wrapperCaptor(); verify(fileTaskMapper).selectList(captor.capture()); @@ -134,12 +197,22 @@ class StaleTaskRepairServiceTest { return ArgumentCaptor.forClass(Wrapper.class); } + @SuppressWarnings({"rawtypes", "unchecked"}) + private static ArgumentCaptor> resultUpdateCaptor() { + return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static ArgumentCaptor> listCaptor() { + return ArgumentCaptor.forClass((Class) List.class); + } + /** * 读取 wrapper 参数容器:用来断言判死谓词与写入值。 * 注意 MyBatis-Plus 是在生成 SQL 片段时才把参数写进 paramNameValuePairs,故先触发一次。 */ @SuppressWarnings("unchecked") - private static Map paramsOf(Wrapper wrapper) { + private static Map paramsOf(Wrapper wrapper) { wrapper.getSqlSegment(); if (wrapper instanceof LambdaUpdateWrapper) { ((LambdaUpdateWrapper) wrapper).getSqlSet(); @@ -164,4 +237,12 @@ class StaleTaskRepairServiceTest { task.setScheduledAt(LocalDateTime.now().plusHours(3)); return task; } + + private static FileTaskEntity task(Long id, String status) { + FileTaskEntity task = task(id); + task.setModuleType("SIMILAR_ASIN"); + task.setStatus(status); + task.setScheduledAt(null); + return task; + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java index ba2eed8cb..df7c9802b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java @@ -261,6 +261,45 @@ class TaskFileJobServiceTest { verify(applicationEventPublisher, times(2)).publishEvent(any(Object.class)); } + @Test + void 父任务判死后未结束的组装作业必须被终结而不是被无限defer() { + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher); + + int failed = service.failUnfinishedJobs(List.of(31060L), "任务心跳超时,已自动失败"); + + assertEquals(1, failed, "返回值即被终结的作业数,修复日志要靠它"); + ArgumentCaptor> update = updateCaptor(); + verify(taskFileJobMapper).update(isNull(), update.capture()); + LambdaUpdateWrapper wrapper = update.getValue(); + String where = wrapper.getSqlSegment(); + assertTrue(where.contains("task_id"), "必须按父任务圈定作业"); + assertTrue(where.contains("status"), "必须按作业状态圈定"); + assertFalse(where.contains("SUCCESS"), "已成功的作业不许被改口"); + + String set = wrapper.getSqlSet(); + assertTrue(set.contains("retry_count"), + "重试必须拉满:FAILED 而 retry_count=0 的作业仍会被 listRunnableJobs 捞起重试"); + assertTrue(set.contains("terminal_callback_at"), + "必须打终态回调标记,否则失败收尾扫描会再去回调已被判死的父任务"); + assertTrue(set.contains("finished_at"), "终态作业要有结束时间"); + + var params = wrapper.getParamNameValuePairs(); + assertTrue(params.containsValue("FAILED"), "作业状态写 FAILED"); + assertTrue(params.containsValue(TaskFileJobService.MAX_RETRY_COUNT), "重试次数拉满到上限"); + assertTrue(params.values().stream().anyMatch(value -> value instanceof String text && text.contains("心跳超时")), + "判死原因要落到作业行,否则排查时只知道它红了"); + } + + @Test + void 空任务清单不去碰作业表() { + TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher); + + assertEquals(0, service.failUnfinishedJobs(List.of(), "任务心跳超时,已自动失败")); + + verify(taskFileJobMapper, never()).update(any(), any()); + } + private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) { ArgumentCaptor> update = updateCaptor(); verify(taskFileJobMapper).update(isNull(), update.capture());