feat(任务判死): 判死时把下游一并收口——写 biz_file_result 并 fail 未完 jobs
代收在飞件落地(原作者那一路已不在跑;本笔在我到场前就在工作树里,一字未改)。 StaleTaskRepairService 新增 markDeadDownstream,挂进三条判死路径(长期未被领取/定时到点未启动/ 心跳超时),新增 FileResultMapper 与 TaskFileJobService 两枚依赖;TaskFileJobService 补 failUnfinishedJobs。同名测试各 +85/+39 行。 验证依据=四面闸门 ts=2026-10-18T09:49:23 overall=pass(后端 34 格全绿,含 mvn -o test 整量), 量的是这份未提交内容;落地提交后内容逐字不变。
This commit is contained in:
+47
-9
@@ -4,7 +4,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -40,6 +42,8 @@ public class StaleTaskRepairService {
|
||||
private static final int MAX_BRAND_STALE_BATCH = 500;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
/** 心跳时钟改为「从首次心跳起算」:排队中的 RUNNING 任务不在这条线上判死。 */
|
||||
@@ -78,16 +82,17 @@ public class StaleTaskRepairService {
|
||||
return;
|
||||
}
|
||||
String reason = "任务长期未被领取,已自动失败";
|
||||
List<Long> ids = stale.stream().map(FileTaskEntity::getId).toList();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.in(FileTaskEntity::getId, ids)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_PENDING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 陈旧中间态已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
log.warn("[stale-task-repair] 陈旧中间态已标失败 count={} ids={}", updated, ids);
|
||||
markDeadDownstream(ids, reason);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,16 +119,17 @@ public class StaleTaskRepairService {
|
||||
return;
|
||||
}
|
||||
String reason = "定时任务到点后长时间未启动,已自动失败";
|
||||
List<Long> ids = stale.stream().map(FileTaskEntity::getId).toList();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.in(FileTaskEntity::getId, ids)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_SCHEDULED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 陈旧定时任务已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
log.warn("[stale-task-repair] 陈旧定时任务已标失败 count={} ids={}", updated, ids);
|
||||
markDeadDownstream(ids, reason);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,16 +150,17 @@ public class StaleTaskRepairService {
|
||||
return;
|
||||
}
|
||||
String reason = "任务心跳超时,已自动失败";
|
||||
List<Long> ids = stale.stream().map(FileTaskEntity::getId).toList();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.in(FileTaskEntity::getId, ids)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}", updated, ids);
|
||||
markDeadDownstream(ids, reason);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,4 +198,35 @@ public class StaleTaskRepairService {
|
||||
log.warn("[stale-task-repair] brand 陈旧任务已标失败 count={} ids={}", failedIds.size(), failedIds);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死必须连带改口下游两处,否则只改 {@code biz_file_task} 会留下两种假象:
|
||||
*
|
||||
* <p>1){@code biz_file_result.success} 是 {@code NOT NULL DEFAULT 1} 的列,只有走正常收尾才回写。
|
||||
* 被兜底判死的任务这一位仍是 1,而历史列表拿它当「已完成」的判据之一,于是显示「已完成」却没有任何
|
||||
* 结果文件(2026-10-16 任务 31060:客户端只回传 51/510 个分片,判死后结果行仍是 success=1)。
|
||||
* 只改没有结果文件的行——文件已经补出来的任务,其部分交付提示(error_message)不能被这里覆盖。
|
||||
*
|
||||
* <p>2)结果文件作业仍 RUNNING 会被 worker 反复 defer 续命、永不出栈,见
|
||||
* {@link TaskFileJobService#failUnfinishedJobs}。
|
||||
*
|
||||
* <p>批内并发:下游按「本轮 SELECT 到的陈旧 id」写,而任务行的 UPDATE 带 status 条件,理论上可波及
|
||||
* 同批里刚被别的写入方推走的任务。窗口是同一次方法调用内 SELECT→UPDATE 的毫秒级,且候选已被要求
|
||||
* 2 小时无任何更新,故按批处理、不逐行复核。
|
||||
*/
|
||||
private void markDeadDownstream(List<Long> taskIds, String reason) {
|
||||
if (taskIds.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getTaskId, taskIds)
|
||||
.eq(FileResultEntity::getSuccess, 1)
|
||||
.and(w -> w.isNull(FileResultEntity::getResultFileUrl)
|
||||
.or().eq(FileResultEntity::getResultFileUrl, ""))
|
||||
.set(FileResultEntity::getSuccess, 0)
|
||||
.set(FileResultEntity::getErrorMessage, reason));
|
||||
int jobs = taskFileJobService.failUnfinishedJobs(taskIds, reason);
|
||||
log.warn("[stale-task-repair] 判死下游已改口(不影响本轮判死)tasks={} jobs={} reason={}",
|
||||
taskIds.size(), jobs, reason);
|
||||
}
|
||||
}
|
||||
|
||||
+27
@@ -734,6 +734,33 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 父任务已被判死时,终结它仍未结束的组装作业。
|
||||
*
|
||||
* <p>不终结的后果(2026-10-16 线上作业 22136 / 任务 31060):worker 每 15 秒把作业捞起来走
|
||||
* {@code touchRunning} 续命,{@code updated_at} 因此一直被刷新,而 job 级 stuck 扫描正是按
|
||||
* {@code updated_at} 判陈旧的 → 永不命中,作业随已死的父任务空转 28 小时。
|
||||
*
|
||||
* <p>与 {@link #markFailedPermanent} 同口径:重试拉满(FAILED 而 retry_count 未达上限的作业会被
|
||||
* {@code listRunnableJobs} 再次捞起)+ 打 {@code terminal_callback_at}(否则失败收尾扫描会去回调
|
||||
* 已被判死的父任务与结果行)。
|
||||
*/
|
||||
public int failUnfinishedJobs(List<Long> taskIds, String message) {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getTaskId, taskIds)
|
||||
.in(TaskFileJobEntity::getStatus, "RUNNING", "PENDING")
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "父任务已判死,结果文件作业同步结束" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, now));
|
||||
}
|
||||
|
||||
/**
|
||||
* 该任务是否已有「重试耗尽且已走完终态回调」的组装 job。
|
||||
*
|
||||
|
||||
+83
-2
@@ -5,7 +5,9 @@ import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
@@ -39,16 +41,20 @@ import static org.mockito.Mockito.when;
|
||||
class StaleTaskRepairServiceTest {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||
private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi = mock(BrandTaskStaleRepairSpi.class);
|
||||
private final DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class);
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy = mock(StaleTaskHeartbeatPolicy.class);
|
||||
private final StaleTaskRepairService service = new StaleTaskRepairService(
|
||||
fileTaskMapper, brandTaskStaleRepairSpi, distributedJobLockService, staleTaskHeartbeatPolicy);
|
||||
fileTaskMapper, fileResultMapper, taskFileJobService, brandTaskStaleRepairSpi, distributedJobLockService,
|
||||
staleTaskHeartbeatPolicy);
|
||||
|
||||
@BeforeEach
|
||||
void initTableInfo() {
|
||||
// LambdaWrapper 需要实体列缓存,单测无 Spring/MyBatis 上下文,手动初始化
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -96,9 +102,55 @@ class StaleTaskRepairServiceTest {
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleIdle");
|
||||
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleScheduled");
|
||||
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning");
|
||||
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
verify(fileResultMapper, never()).update(any(), any());
|
||||
verifyNoMoreInteractions(brandTaskStaleRepairSpi);
|
||||
verify(taskFileJobService, never()).failUnfinishedJobs(any(), any());
|
||||
}
|
||||
|
||||
// 2026-10-16 任务 31060:客户端只回传 51/510 个分片后被本方法判死,但 biz_file_result.success
|
||||
// 是 `NOT NULL DEFAULT 1` 的列、只有正常收尾才回写,于是历史列表拿这一位显示「已完成」,
|
||||
// 而结果文件从未存在(下载按钮的判据 result_file_url 为空)。判死必须同时把结果行改口。
|
||||
@Test
|
||||
void 心跳判死必须同步把无结果文件的结果行改判失败() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(31060L, "RUNNING")));
|
||||
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
|
||||
when(fileResultMapper.update(isNull(), any())).thenReturn(1);
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning");
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<FileResultEntity>> captor = resultUpdateCaptor();
|
||||
verify(fileResultMapper).update(isNull(), captor.capture());
|
||||
LambdaUpdateWrapper<FileResultEntity> wrapper = captor.getValue();
|
||||
String set = wrapper.getSqlSet();
|
||||
assertTrue(set.contains("success"), "结果行的 success 位必须被改写,否则前端仍读到默认值 1");
|
||||
assertTrue(set.contains("error_message"), "判死原因要留痕到结果行,页面才说得出为什么失败");
|
||||
assertFalse(set.contains("updated_at"),
|
||||
"updated_at 由数据库维护,显式写会冻结结果行的版本信号(见 FileResultEntity 注释)");
|
||||
Map<String, Object> params = paramsOf(wrapper);
|
||||
assertTrue(params.containsValue(0), "success 写 0");
|
||||
assertTrue(containsMessage(params, "心跳超时"), "留痕文案与任务行同源");
|
||||
|
||||
String where = wrapper.getSqlSegment();
|
||||
assertTrue(where.contains("result_file_url"),
|
||||
"只改没有结果文件的行:文件已生成的任务不许被抹掉部分交付提示");
|
||||
assertTrue(where.contains("task_id"), "按本次判死的任务圈定");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 心跳判死必须终结仍_RUNNING_的组装作业() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(31060L, "RUNNING")));
|
||||
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning");
|
||||
|
||||
ArgumentCaptor<List<Long>> ids = listCaptor();
|
||||
ArgumentCaptor<String> reason = ArgumentCaptor.forClass(String.class);
|
||||
verify(taskFileJobService).failUnfinishedJobs(ids.capture(), reason.capture());
|
||||
assertEquals(List.of(31060L), ids.getValue());
|
||||
assertTrue(reason.getValue().contains("心跳超时"), "作业行的判死原因与任务行同源");
|
||||
}
|
||||
|
||||
// 2026-10-17 实录:任务一被认领就记 RUNNING,而客户端要等执行器真开始跑才发心跳(同店占用 /
|
||||
@@ -117,6 +169,17 @@ class StaleTaskRepairServiceTest {
|
||||
verify(fileTaskMapper, never()).update(isNull(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 未被领取判死同样要回写结果行并终结作业() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(31176L, "PENDING")));
|
||||
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleIdle");
|
||||
|
||||
verify(fileResultMapper).update(isNull(), any());
|
||||
verify(taskFileJobService).failUnfinishedJobs(eq(List.of(31176L)), any());
|
||||
}
|
||||
|
||||
private Map<String, Object> selectParams() {
|
||||
ArgumentCaptor<Wrapper<FileTaskEntity>> captor = wrapperCaptor();
|
||||
verify(fileTaskMapper).selectList(captor.capture());
|
||||
@@ -134,12 +197,22 @@ class StaleTaskRepairServiceTest {
|
||||
return ArgumentCaptor.forClass(Wrapper.class);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
private static ArgumentCaptor<LambdaUpdateWrapper<FileResultEntity>> resultUpdateCaptor() {
|
||||
return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
private static ArgumentCaptor<List<Long>> listCaptor() {
|
||||
return ArgumentCaptor.forClass((Class) List.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取 wrapper 参数容器:用来断言判死谓词与写入值。
|
||||
* 注意 MyBatis-Plus 是在生成 SQL 片段时才把参数写进 paramNameValuePairs,故先触发一次。
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, Object> paramsOf(Wrapper<FileTaskEntity> wrapper) {
|
||||
private static Map<String, Object> paramsOf(Wrapper<?> wrapper) {
|
||||
wrapper.getSqlSegment();
|
||||
if (wrapper instanceof LambdaUpdateWrapper) {
|
||||
((LambdaUpdateWrapper<FileTaskEntity>) wrapper).getSqlSet();
|
||||
@@ -164,4 +237,12 @@ class StaleTaskRepairServiceTest {
|
||||
task.setScheduledAt(LocalDateTime.now().plusHours(3));
|
||||
return task;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task(Long id, String status) {
|
||||
FileTaskEntity task = task(id);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setStatus(status);
|
||||
task.setScheduledAt(null);
|
||||
return task;
|
||||
}
|
||||
}
|
||||
|
||||
+39
@@ -261,6 +261,45 @@ class TaskFileJobServiceTest {
|
||||
verify(applicationEventPublisher, times(2)).publishEvent(any(Object.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 父任务判死后未结束的组装作业必须被终结而不是被无限defer() {
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
int failed = service.failUnfinishedJobs(List.of(31060L), "任务心跳超时,已自动失败");
|
||||
|
||||
assertEquals(1, failed, "返回值即被终结的作业数,修复日志要靠它");
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> wrapper = update.getValue();
|
||||
String where = wrapper.getSqlSegment();
|
||||
assertTrue(where.contains("task_id"), "必须按父任务圈定作业");
|
||||
assertTrue(where.contains("status"), "必须按作业状态圈定");
|
||||
assertFalse(where.contains("SUCCESS"), "已成功的作业不许被改口");
|
||||
|
||||
String set = wrapper.getSqlSet();
|
||||
assertTrue(set.contains("retry_count"),
|
||||
"重试必须拉满:FAILED 而 retry_count=0 的作业仍会被 listRunnableJobs 捞起重试");
|
||||
assertTrue(set.contains("terminal_callback_at"),
|
||||
"必须打终态回调标记,否则失败收尾扫描会再去回调已被判死的父任务");
|
||||
assertTrue(set.contains("finished_at"), "终态作业要有结束时间");
|
||||
|
||||
var params = wrapper.getParamNameValuePairs();
|
||||
assertTrue(params.containsValue("FAILED"), "作业状态写 FAILED");
|
||||
assertTrue(params.containsValue(TaskFileJobService.MAX_RETRY_COUNT), "重试次数拉满到上限");
|
||||
assertTrue(params.values().stream().anyMatch(value -> value instanceof String text && text.contains("心跳超时")),
|
||||
"判死原因要落到作业行,否则排查时只知道它红了");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 空任务清单不去碰作业表() {
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
assertEquals(0, service.failUnfinishedJobs(List.of(), "任务心跳超时,已自动失败"));
|
||||
|
||||
verify(taskFileJobMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) {
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
|
||||
Reference in New Issue
Block a user