fix(断点续传): 外观/结果作业/去重三处中断语义补齐,重启不再静默丢行
- 外观专利行标改两态(in-flight 15min / processed 24h):只有 CAS 写回 payload 成功 才转「已处理」,异常、覆盖率不足、CAS 打满等未落库出口一律撤销标记 —— 修「发 LLM 请求前就盖章,进程被杀的那一批永不再补检」 - 结果文件作业开机自回收:按 owner_instance_id 只收本机名下 RUNNING 且心跳超 90s 的行; 卡死阈值 30 → 5 分钟,重启接回从最长半小时压到 5 分钟 - 外观结果表缺检行显式写「未取得检测结果」并判失败,不再留空靠 0.5 覆盖率阈值蒙过 - 去重进度改 NodeSharedStore 跨节点共享,runId 用已落库的 task_no:轮询打到另一节点或 发版重启后仍能读到真实进度、已完成文件仍可下载;共享态缺失时按 task_no 从库重建; 逐文件心跳防 StaleTaskRepair 把长任务误判死;开机把本机残留 RUNNING 判失败并给提示
This commit is contained in:
+62
@@ -428,6 +428,68 @@ public class TaskFileJobService {
|
||||
.collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a));
|
||||
}
|
||||
|
||||
/**
|
||||
* 开机自回收:把本机 owner 名下「RUNNING 且心跳已停」的作业退回 PENDING 并补发派发事件。
|
||||
*
|
||||
* <p>进程被杀(发版、崩溃、容器重建)时这些行不会自己退出 RUNNING,只能等 60 秒一轮的
|
||||
* stuck 扫描按时间阈值判死——阈值历史上是 30 分钟,等于每次发版都把在跑的结果文件作业
|
||||
* 冻结一整个阈值窗口,而这批行其实早已无人持有。进程刚起来时本机不可能还在跑它们,
|
||||
* 所以这一刻就能直接接回来,把等待从「阈值」压到「启动」。
|
||||
*
|
||||
* <p>两道保险:只取 owner 等值等于自己的行(绝不碰另一台实例在跑的),且必须已停止心跳
|
||||
* 一个完整周期以上(正常心跳是 60 秒一次,误抢会让在跑的作业被双份派发)。
|
||||
*
|
||||
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,每发一次版
|
||||
* 吃掉它那 5 次重试中的一次,反复发版会把正常任务判成"已达到最大重试次数"。
|
||||
*
|
||||
* <p>刻意不加事务,与 {@link #resetStuckRunningJobsDetailed} 同理:每行都带
|
||||
* status + updated_at 的 CAS 条件,逐行独立提交,锁即时释放。
|
||||
*/
|
||||
public int reclaimOwnedRunningJobs(String owner, long staleSeconds, int limit) {
|
||||
String normalizedOwner = owner == null ? "" : owner.trim();
|
||||
if (normalizedOwner.isBlank()) {
|
||||
return 0;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusSeconds(Math.max(60L, staleSeconds));
|
||||
int safeLimit = Math.max(1, Math.min(limit, 100));
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(TaskFileJobEntity::getOwner, normalizedOwner)
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, cutoff)
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
if (jobs.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
List<Long> resetIds = new ArrayList<>();
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
if (job == null || job.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "实例重启,作业已重新排队")
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
resetIds.add(job.getId());
|
||||
}
|
||||
}
|
||||
if (resetIds.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
Map<Long, TaskFileJobEntity> refreshedById = refreshJobsByIds(resetIds);
|
||||
for (Long jobId : resetIds) {
|
||||
publishDispatchEvent(refreshedById.get(jobId));
|
||||
}
|
||||
log.warn("[task-file-job] 开机自回收本机遗留作业 count={} owner={} staleSeconds={} limit={}",
|
||||
resetIds.size(), normalizedOwner, staleSeconds, safeLimit);
|
||||
return resetIds.size();
|
||||
}
|
||||
|
||||
public record StuckJobResetResult(int resetCount, List<TaskFileJobEntity> exhaustedJobs) {
|
||||
}
|
||||
|
||||
|
||||
+39
-1
@@ -12,6 +12,8 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -43,12 +45,22 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.batch-size:20}")
|
||||
private int batchSize;
|
||||
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
|
||||
/**
|
||||
* 卡死判定阈值。作业在跑期间每 60 秒 touch 一次 {@code updated_at},所以"超过 5 分钟没心跳"
|
||||
* 就等于持有者已经没了;取 30 分钟时一次实例被销毁重建(owner 变了、开机回收够不着)
|
||||
* 会把结果文件作业冻结半小时。队列积压导致的"认领后迟迟没执行"不会误伤:
|
||||
* 真正开跑前的 {@code activateRunningClaim} 用 updated_at 做 CAS,被复位的作业会直接放弃执行。
|
||||
*/
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:5}")
|
||||
private int stuckTimeoutMinutes;
|
||||
|
||||
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
|
||||
private long heartbeatIntervalMillis = 60000L;
|
||||
|
||||
/** 开机回收的"心跳已停"判据:默认 90 秒 ≈ 1.5 个心跳周期,容得下一次正常心跳延迟。 */
|
||||
@Value("${aiimage.result-file-job.boot-reclaim-stale-seconds:90}")
|
||||
private long bootReclaimStaleSeconds;
|
||||
|
||||
/** 多个结果文件 Job 并行运行时,心跳不能被单个 DB 慢调用串行阻塞。 */
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
|
||||
@@ -99,6 +111,32 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用就绪即回收本机遗留作业。
|
||||
*
|
||||
* <p>发版杀进程时正在跑的 job 会一直挂在 RUNNING,要等卡死扫描按 stuck-timeout-minutes
|
||||
* 判死才重新排队——那段时间里用户看到的是"任务卡住不动",而实际上没有任何进程在跑它。
|
||||
* 本机刚起来,这些 owner=自己的 RUNNING 行不可能还在本机执行,所以此刻就能直接接回来,
|
||||
* 不必等阈值;回收不消耗 retry_count(发版不是作业的错)。
|
||||
*
|
||||
* <p>只处理 owner 等于本机实例 id 的行,另一台实例在跑的作业不受影响。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void reclaimOwnedJobsOnBoot() {
|
||||
try {
|
||||
int reclaimed = taskFileJobService.reclaimOwnedRunningJobs(
|
||||
currentInstanceId(), bootReclaimStaleSeconds, batchSize);
|
||||
if (reclaimed > 0) {
|
||||
log.warn("[task-file-job] booted and reclaimed abandoned jobs of this instance reclaimed={} instanceId={} staleSeconds={}",
|
||||
reclaimed, currentInstanceId(), bootReclaimStaleSeconds);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// 回收只是加速恢复;失败不影响启动,兜底仍是 stuck 扫描。
|
||||
log.error("[task-file-job] boot reclaim failed, falling back to stuck scan instanceId={} timeoutMinutes={} err={}",
|
||||
currentInstanceId(), stuckTimeoutMinutes, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void process(TaskFileJobEntity job) {
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user