feat(任务判死): 判死时把下游一并收口——写 biz_file_result 并 fail 未完 jobs
代收在飞件落地(原作者那一路已不在跑;本笔在我到场前就在工作树里,一字未改)。 StaleTaskRepairService 新增 markDeadDownstream,挂进三条判死路径(长期未被领取/定时到点未启动/ 心跳超时),新增 FileResultMapper 与 TaskFileJobService 两枚依赖;TaskFileJobService 补 failUnfinishedJobs。同名测试各 +85/+39 行。 验证依据=四面闸门 ts=2026-10-18T09:49:23 overall=pass(后端 34 格全绿,含 mvn -o test 整量), 量的是这份未提交内容;落地提交后内容逐字不变。
This commit is contained in:
+47
-9
@@ -4,7 +4,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -40,6 +42,8 @@ public class StaleTaskRepairService {
|
||||
private static final int MAX_BRAND_STALE_BATCH = 500;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
/** 心跳时钟改为「从首次心跳起算」:排队中的 RUNNING 任务不在这条线上判死。 */
|
||||
@@ -78,16 +82,17 @@ public class StaleTaskRepairService {
|
||||
return;
|
||||
}
|
||||
String reason = "任务长期未被领取,已自动失败";
|
||||
List<Long> ids = stale.stream().map(FileTaskEntity::getId).toList();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.in(FileTaskEntity::getId, ids)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_PENDING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 陈旧中间态已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
log.warn("[stale-task-repair] 陈旧中间态已标失败 count={} ids={}", updated, ids);
|
||||
markDeadDownstream(ids, reason);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,16 +119,17 @@ public class StaleTaskRepairService {
|
||||
return;
|
||||
}
|
||||
String reason = "定时任务到点后长时间未启动,已自动失败";
|
||||
List<Long> ids = stale.stream().map(FileTaskEntity::getId).toList();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.in(FileTaskEntity::getId, ids)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_SCHEDULED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 陈旧定时任务已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
log.warn("[stale-task-repair] 陈旧定时任务已标失败 count={} ids={}", updated, ids);
|
||||
markDeadDownstream(ids, reason);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,16 +150,17 @@ public class StaleTaskRepairService {
|
||||
return;
|
||||
}
|
||||
String reason = "任务心跳超时,已自动失败";
|
||||
List<Long> ids = stale.stream().map(FileTaskEntity::getId).toList();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.in(FileTaskEntity::getId, ids)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}", updated, ids);
|
||||
markDeadDownstream(ids, reason);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,4 +198,35 @@ public class StaleTaskRepairService {
|
||||
log.warn("[stale-task-repair] brand 陈旧任务已标失败 count={} ids={}", failedIds.size(), failedIds);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死必须连带改口下游两处,否则只改 {@code biz_file_task} 会留下两种假象:
|
||||
*
|
||||
* <p>1){@code biz_file_result.success} 是 {@code NOT NULL DEFAULT 1} 的列,只有走正常收尾才回写。
|
||||
* 被兜底判死的任务这一位仍是 1,而历史列表拿它当「已完成」的判据之一,于是显示「已完成」却没有任何
|
||||
* 结果文件(2026-10-16 任务 31060:客户端只回传 51/510 个分片,判死后结果行仍是 success=1)。
|
||||
* 只改没有结果文件的行——文件已经补出来的任务,其部分交付提示(error_message)不能被这里覆盖。
|
||||
*
|
||||
* <p>2)结果文件作业仍 RUNNING 会被 worker 反复 defer 续命、永不出栈,见
|
||||
* {@link TaskFileJobService#failUnfinishedJobs}。
|
||||
*
|
||||
* <p>批内并发:下游按「本轮 SELECT 到的陈旧 id」写,而任务行的 UPDATE 带 status 条件,理论上可波及
|
||||
* 同批里刚被别的写入方推走的任务。窗口是同一次方法调用内 SELECT→UPDATE 的毫秒级,且候选已被要求
|
||||
* 2 小时无任何更新,故按批处理、不逐行复核。
|
||||
*/
|
||||
private void markDeadDownstream(List<Long> taskIds, String reason) {
|
||||
if (taskIds.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getTaskId, taskIds)
|
||||
.eq(FileResultEntity::getSuccess, 1)
|
||||
.and(w -> w.isNull(FileResultEntity::getResultFileUrl)
|
||||
.or().eq(FileResultEntity::getResultFileUrl, ""))
|
||||
.set(FileResultEntity::getSuccess, 0)
|
||||
.set(FileResultEntity::getErrorMessage, reason));
|
||||
int jobs = taskFileJobService.failUnfinishedJobs(taskIds, reason);
|
||||
log.warn("[stale-task-repair] 判死下游已改口(不影响本轮判死)tasks={} jobs={} reason={}",
|
||||
taskIds.size(), jobs, reason);
|
||||
}
|
||||
}
|
||||
|
||||
+27
@@ -734,6 +734,33 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 父任务已被判死时,终结它仍未结束的组装作业。
|
||||
*
|
||||
* <p>不终结的后果(2026-10-16 线上作业 22136 / 任务 31060):worker 每 15 秒把作业捞起来走
|
||||
* {@code touchRunning} 续命,{@code updated_at} 因此一直被刷新,而 job 级 stuck 扫描正是按
|
||||
* {@code updated_at} 判陈旧的 → 永不命中,作业随已死的父任务空转 28 小时。
|
||||
*
|
||||
* <p>与 {@link #markFailedPermanent} 同口径:重试拉满(FAILED 而 retry_count 未达上限的作业会被
|
||||
* {@code listRunnableJobs} 再次捞起)+ 打 {@code terminal_callback_at}(否则失败收尾扫描会去回调
|
||||
* 已被判死的父任务与结果行)。
|
||||
*/
|
||||
public int failUnfinishedJobs(List<Long> taskIds, String message) {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getTaskId, taskIds)
|
||||
.in(TaskFileJobEntity::getStatus, "RUNNING", "PENDING")
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "父任务已判死,结果文件作业同步结束" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, now));
|
||||
}
|
||||
|
||||
/**
|
||||
* 该任务是否已有「重试耗尽且已走完终态回调」的组装 job。
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user