fix(task): 补检中的任务不再自动克隆重跑,卡死重排不消耗重试预算
- TaskResumeService:名下还有 PENDING/RUNNING 组装作业的 FAILED 任务不克隆 (现网 2026-10-24 一枚 760 行外观专利被连克隆 3 代,每代全量重跑、同秒显示) - resetStuckRunningJobsDetailed:RUNNING 行重排不再 +1 retry_count,判死只剩 markFailed 一条真失败的路——补检一轮几十分钟的慢执行不再被记成失败次数 - 新增 hasUnfinishedAssembleJob;测试按新口径重写并补现网回归两例
This commit is contained in:
+32
-17
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
@@ -102,6 +103,25 @@ public class TaskFileJobService {
|
||||
return retry >= MAX_RETRY_COUNT;
|
||||
}
|
||||
|
||||
/**
|
||||
* 该任务名下还有没有**未终态**的组装作业(PENDING/RUNNING)。
|
||||
*
|
||||
* <p>交付缺口触发的补检轮就是把作业排回这两态({@link #requeue}),所以"有活着的组装作业"
|
||||
* ="这单还在补检"。而任务行可能早就落成 FAILED(心跳超时或交付判死先落),此时按任务状态
|
||||
* 去克隆重跑,等于同一份活开两份:现网 2026-10-24 一枚 760 行的外观专利因此被克隆成 4 枚、
|
||||
* 每枚都全量重跑一遍,四枚卡片的开始时间还都是同一秒(克隆件继承原件 created_at)。
|
||||
*/
|
||||
public boolean hasUnfinishedAssembleJob(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return false;
|
||||
}
|
||||
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.in(TaskFileJobEntity::getStatus, TaskStatuses.Job.PENDING, TaskStatuses.Job.RUNNING));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 补传完整后恢复终态失败的组装 job:重置为 PENDING(retryCount 清零)重新派发。
|
||||
* 幂等:job 不存在、非 FAILED 或未达重试上限时不动作,返回当前 job。
|
||||
@@ -310,6 +330,10 @@ public class TaskFileJobService {
|
||||
* 每天上百次 "Lock wait timeout exceeded"(biz_task_file_job)。
|
||||
* 这里的每行更新本就带 status + updatedAt 的 CAS 条件(幂等),逐行独立提交更安全:
|
||||
* 锁即时释放,CAS 不匹配的一方返回 0 行即可,也不会因中途异常回滚掉已修好的行。
|
||||
*
|
||||
* <p>RUNNING 行的重排**不消耗** retry_count(2026-10-24 分账口径):扫描只负责把卡住的行
|
||||
* 放回队列,预算只由真失败({@link #markFailed})推进——"补检一轮要几十分钟"的慢执行不该
|
||||
* 被记成失败次数。PENDING 行的累加保留:那条防的是 dispatch 事件无限重发,与执行快慢无关。
|
||||
*/
|
||||
public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) {
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
|
||||
@@ -367,30 +391,21 @@ public class TaskFileJobService {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
||||
// 重排**不消耗**重试预算(2026-10-24 拍板的分账口径):卡死扫描只负责把卡住的
|
||||
// RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在
|
||||
// MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行
|
||||
// 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后
|
||||
// 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、
|
||||
// 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now);
|
||||
if (nextRetryCount >= MAX_RETRY_COUNT) {
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数")
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
exhaustedJobIds.add(job.getId());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
|
||||
+17
@@ -65,6 +65,11 @@ public class TaskResumeService {
|
||||
private static final int DUPLICATE_SWEEP_LIMIT = 200;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
/**
|
||||
* 查"这单名下还有没有活着的组装作业"——补检轮还活着的意思,见
|
||||
* {@link TaskFileJobService#hasUnfinishedAssembleJob(Long)}。
|
||||
*/
|
||||
private final TaskFileJobService taskFileJobs;
|
||||
/** moduleType → 克隆重试实现;只有注册了本 SPI 的模块才会自动重试。 */
|
||||
private final Map<String, ClientTaskRetrySpi> retryHandlers;
|
||||
|
||||
@@ -87,8 +92,10 @@ public class TaskResumeService {
|
||||
private int limit;
|
||||
|
||||
public TaskResumeService(FileTaskMapper fileTaskMapper,
|
||||
TaskFileJobService taskFileJobs,
|
||||
List<ClientTaskRetrySpi> retrySpiHandlers) {
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskFileJobs = taskFileJobs;
|
||||
Map<String, ClientTaskRetrySpi> retryIndex = new LinkedHashMap<>();
|
||||
if (retrySpiHandlers != null) {
|
||||
for (ClientTaskRetrySpi handler : retrySpiHandlers) {
|
||||
@@ -216,6 +223,16 @@ public class TaskResumeService {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
// 任务行是 FAILED,不等于这份活干完了:交付缺口会把组装作业排回 PENDING/RUNNING 去补检
|
||||
// (见 AppearancePatentDeliverySupport.requestRecheckForMissingRows),此时克隆=同一份活
|
||||
// 开两份、还各按全量重跑一遍。作业真卡死另有 TaskResultFileJobWorker 的 stuck 扫描兜底,
|
||||
// 不会因为这条例外就永远没人管。
|
||||
if (taskFileJobs.hasUnfinishedAssembleJob(original.getId())) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[task-resume] 不重试 taskId={} moduleType={} 原因=组装作业仍在补检(PENDING/RUNNING)",
|
||||
original.getId(), original.getModuleType());
|
||||
continue;
|
||||
}
|
||||
ClientTaskRetrySpi handler = retryHandlers.get(normalize(original.getModuleType()));
|
||||
if (handler == null) {
|
||||
stats.skippedTaskCount++;
|
||||
|
||||
Reference in New Issue
Block a user