From eb6f082674159daff3c6bde8053c92b4f5ea66a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Tue, 20 Oct 2026 12:30:09 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E5=BF=83=E8=B7=B3):=20=E5=BD=B1=E5=AD=90?= =?UTF-8?q?=E5=88=A4=E5=AE=98=E5=85=88=E8=B7=91=E8=AF=BB=E6=95=B0=E2=80=94?= =?UTF-8?q?=E2=80=94=E5=BF=83=E8=B7=B3=E6=96=AD/=E8=BF=9B=E5=BA=A6?= =?UTF-8?q?=E5=81=9C=E6=BB=9E/=E6=8E=92=E9=98=9F/=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E7=AB=AF=E5=A4=84=E7=90=86=E5=88=86=E5=BC=80=E5=88=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死(长任务/排队被 判死)同时存在。新增只读影子扫描:每 5 分钟扫 RUNNING 文件任务,按「心跳中断 / 进度停滞 / 排队宽限 / 服务端结果任务 / 24h 硬顶」输出 would-finalize 日志, 不改任何任务状态。进度用「计数+文本+阶段」指纹,写影子独立快照键,不影响正式 判死线的未变化起点。规则纯函数单测 9 例 + 只读边界守卫 3 例。 --- .../service/TaskHeartbeatPositionService.java | 64 ++++- .../task/service/TaskLivenessShadowRules.java | 132 ++++++++++ .../service/TaskLivenessShadowService.java | 236 ++++++++++++++++++ .../src/main/resources/application.yml | 12 + .../TaskHeartbeatPositionServiceTest.java | 32 +++ .../service/TaskLivenessShadowGuardTest.java | 76 ++++++ .../service/TaskLivenessShadowRulesTest.java | 125 ++++++++++ 7 files changed, 671 insertions(+), 6 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowGuardTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRulesTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java index 37ab015d8..cccf33c79 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java @@ -28,6 +28,8 @@ public class TaskHeartbeatPositionService { private static final String KEY_PREFIX = "task:heartbeat:position:"; private static final String STALL_KEY_PREFIX = "task:progress:stall:"; + /** 影子判官的独立快照键:影子的观测时刻不得改变正式判死线的「未变化起点」。 */ + private static final String SHADOW_STALL_KEY_PREFIX = "task:progress:shadow-stall:"; private static final Duration POSITION_TTL = Duration.ofHours(24); private static final int POSITION_LABEL_MAX_LEN = 60; /** 计数字符串的最长位数:9 位以内 parseInt 必不溢出,所以这一处不需要 catch。 */ @@ -303,7 +305,57 @@ public class TaskHeartbeatPositionService { if (position == null || position.isBlank()) { return StallState.UNKNOWN; } - String stallKey = STALL_KEY_PREFIX + taskId; + return evaluateAgainstSnapshot(taskId, STALL_KEY_PREFIX + taskId, position, staleWindowMinutes); + } + + /** + * 影子判官专用:把「计数 + 文本 + 阶段」压成指纹判断推进,快照写独立键。 + * + *

与正式线只比文本不同:计数变了也是推进(长页面里文本可能长时间不变), + * 阶段变化同样算推进。影子每 5 分钟扫一次,若与正式线共用快照键,影子可能比正式线早几分钟 + * 看到变化,从而把正式线的未变化起点提前、间接影响封口时刻;所以这里只写 shadow 键。 + */ + public StallState evaluateProgressForShadow(Long taskId, long staleWindowMinutes) { + if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) { + return StallState.UNKNOWN; + } + String fingerprint = readProgressFingerprint(taskId); + if (fingerprint == null) { + return StallState.UNKNOWN; + } + return evaluateAgainstSnapshot(taskId, SHADOW_STALL_KEY_PREFIX + taskId, + fingerprint, staleWindowMinutes); + } + + /** + * 影子判官的推进指纹:位置哈希一次读出,计数/文本/阶段任一变化都算推进。 + * 返回 null 表示这个任务没有任何位置信息(老客户端不传位置),不能作为判死证据。 + */ + private String readProgressFingerprint(Long taskId) { + Map entries; + try { + entries = stringRedisTemplate.opsForHash().entries(KEY_PREFIX + taskId); + } catch (Exception ex) { + log.warn("[heartbeat-position] 读取影子位置失败 taskId={} msg={}", taskId, ex.getMessage()); + return null; + } + if (entries == null || entries.isEmpty()) { + return null; + } + String text = value(entries, "progressText"); + String current = value(entries, "current"); + String total = value(entries, "total"); + if (text.isBlank() && current.isBlank() && total.isBlank()) { + return null; + } + return value(entries, "phase") + "|" + value(entries, "grain") + + "|" + current + "|" + total + "|" + text; + } + + private StallState evaluateAgainstSnapshot(Long taskId, + String stallKey, + String fingerprint, + long staleWindowMinutes) { Map stored; try { stored = stringRedisTemplate.opsForHash().entries(stallKey); @@ -312,18 +364,18 @@ public class TaskHeartbeatPositionService { return StallState.UNKNOWN; } long now = Instant.now().toEpochMilli(); - String storedPosition = stored == null ? "" : value(stored, "pos"); + String storedFingerprint = stored == null ? "" : value(stored, "pos"); long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at")); - if (!position.equals(storedPosition) || unchangedSince <= 0L) { - writeStallSnapshot(stallKey, position, now, staleWindowMinutes); + if (!fingerprint.equals(storedFingerprint) || unchangedSince <= 0L) { + writeStallSnapshot(stallKey, fingerprint, now, staleWindowMinutes); return StallState.ADVANCING; } long unchangedMillis = now - unchangedSince; if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) { return StallState.STALLED; } - log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}", - taskId, unchangedMillis, position); + log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} fingerprint={}", + taskId, unchangedMillis, fingerprint); return StallState.ADVANCING; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java new file mode 100644 index 000000000..1a1326946 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java @@ -0,0 +1,132 @@ +package com.nanri.aiimage.modules.task.service; + +import java.util.Objects; + +/** + * 心跳影子判官的纯规则:只回答「按新规则会怎么判」,不碰任何任务状态。 + * + *

为什么要影子期:现有判死线各看各的证据(Redis 心跳、file_task.updated_at、 + * last_chunk_at、各模块超时),既有「心跳线程活着但主线程卡死」的假活,也有长任务/排队 + * 被判死的假死。新规则把「心跳是否断」与「进度是否推进」分开,并显式豁免服务端处理中的 + * 任务;先用真实 RUNNING 任务跑一周 would-finalize 读数,再决定是否接管判死。 + * + *

规则优先级(先命中先返回): + *

    + *
  1. 服务端结果任务仍在跑 → KEEP_SERVER_SIDE(客户端已交棒,心跳停了是正常的);
  2. + *
  3. 从未收到心跳 → 未启动宽限内 KEEP_QUEUED,超宽限 WOULD_FINALIZE(no-first-heartbeat);
  4. + *
  5. 心跳中断超过阈值 → WOULD_FINALIZE(beat-lost);
  6. + *
  7. 任务总时长超过硬顶 → WOULD_FINALIZE(hard-cap)(即使进度仍在推进);
  8. + *
  9. 心跳正常但进度停滞 → WOULD_FINALIZE(no-progress);
  10. + *
  11. 进度在推进 → KEEP_ADVANCING;
  12. + *
  13. 没有位置信息 → UNKNOWN(证据不足,不参与判死)。
  14. + *
+ */ +public final class TaskLivenessShadowRules { + + public static final String RULE_SERVER_SIDE_JOB = "server-side-job"; + public static final String RULE_NO_FIRST_HEARTBEAT = "no-first-heartbeat"; + public static final String RULE_BEAT_LOST = "beat-lost"; + public static final String RULE_HARD_CAP = "hard-cap"; + public static final String RULE_NO_PROGRESS = "no-progress"; + public static final String RULE_ADVANCING = "advancing"; + public static final String RULE_NO_POSITION = "no-position"; + + public enum Verdict { + KEEP_ADVANCING, + KEEP_QUEUED, + KEEP_SERVER_SIDE, + UNKNOWN, + WOULD_FINALIZE + } + + /** 客户端上报的进度位置是否还在变;UNKNOWN 表示老客户端没传位置,不能当判死证据。 */ + public enum ProgressState { + ADVANCING, + STALLED, + UNKNOWN + } + + /** + * @param beatLostMinutes 心跳中断多少分钟算断 + * @param noProgressMinutes 进度位置多少分钟没变算停滞 + * @param firstHeartbeatGraceMinutes 从未心跳的排队宽限;<=0 表示该规则停用(判 UNKNOWN) + * @param hardCapHours 任务总时长硬顶;<=0 表示该规则停用 + */ + public record Thresholds(int beatLostMinutes, + int noProgressMinutes, + int firstHeartbeatGraceMinutes, + int hardCapHours) { + } + + /** + * @param beatMillis 最近一次心跳(0=从未收到) + * @param queuedSinceMillis 未启动宽限的锚(file_task.updated_at) + * @param createdAtMillis 任务创建时刻,用于总量硬顶 + */ + public record Evidence(long taskId, + String moduleType, + long nowMillis, + long beatMillis, + long queuedSinceMillis, + long createdAtMillis, + boolean serverSideJob, + ProgressState progress) { + } + + public record Decision(Verdict verdict, String rule, String detail) { + } + + private TaskLivenessShadowRules() { + } + + public static Decision evaluate(Evidence evidence, Thresholds thresholds) { + Objects.requireNonNull(evidence, "evidence"); + Objects.requireNonNull(thresholds, "thresholds"); + if (evidence.serverSideJob()) { + return new Decision(Verdict.KEEP_SERVER_SIDE, RULE_SERVER_SIDE_JOB, + "服务端结果任务仍在运行,客户端心跳停止属正常"); + } + if (evidence.beatMillis() <= 0L) { + long waitedMinutes = ageMinutes(evidence.nowMillis(), evidence.queuedSinceMillis()); + if (thresholds.firstHeartbeatGraceMinutes() <= 0) { + return new Decision(Verdict.UNKNOWN, RULE_NO_FIRST_HEARTBEAT, + "从未收到心跳,但排队宽限规则已停用,无法判定"); + } + if (waitedMinutes < thresholds.firstHeartbeatGraceMinutes()) { + return new Decision(Verdict.KEEP_QUEUED, RULE_NO_FIRST_HEARTBEAT, + "从未收到心跳,仍在排队宽限内(已等待 " + waitedMinutes + + " 分钟 / 宽限 " + thresholds.firstHeartbeatGraceMinutes() + " 分钟)"); + } + return new Decision(Verdict.WOULD_FINALIZE, RULE_NO_FIRST_HEARTBEAT, + "从未收到心跳,已等待 " + waitedMinutes + + " 分钟(宽限 " + thresholds.firstHeartbeatGraceMinutes() + " 分钟)"); + } + long beatAgeMinutes = ageMinutes(evidence.nowMillis(), evidence.beatMillis()); + if (thresholds.beatLostMinutes() > 0 && beatAgeMinutes > thresholds.beatLostMinutes()) { + return new Decision(Verdict.WOULD_FINALIZE, RULE_BEAT_LOST, + "心跳已中断 " + beatAgeMinutes + " 分钟(阈值 " + thresholds.beatLostMinutes() + " 分钟)"); + } + long taskAgeMinutes = ageMinutes(evidence.nowMillis(), evidence.createdAtMillis()); + long taskAgeHours = taskAgeMinutes / 60L; + if (thresholds.hardCapHours() > 0 && taskAgeHours >= thresholds.hardCapHours()) { + return new Decision(Verdict.WOULD_FINALIZE, RULE_HARD_CAP, + "任务已运行 " + taskAgeHours + " 小时(硬顶 " + thresholds.hardCapHours() + " 小时)"); + } + if (evidence.progress() == ProgressState.STALLED) { + return new Decision(Verdict.WOULD_FINALIZE, RULE_NO_PROGRESS, + "心跳正常但进度至少 " + thresholds.noProgressMinutes() + " 分钟未推进"); + } + if (evidence.progress() == ProgressState.ADVANCING) { + return new Decision(Verdict.KEEP_ADVANCING, RULE_ADVANCING, "进度仍在推进"); + } + return new Decision(Verdict.UNKNOWN, RULE_NO_POSITION, + "心跳正常但没有进度位置,证据不足"); + } + + private static long ageMinutes(long nowMillis, long sinceMillis) { + if (sinceMillis <= 0L) { + return 0L; + } + return Math.max(0L, (nowMillis - sinceMillis) / 60_000L); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java new file mode 100644 index 000000000..f96bff1cf --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java @@ -0,0 +1,236 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * 心跳影子判官:只读扫描 RUNNING 任务,按 {@link TaskLivenessShadowRules} 输出 + * would-finalize 读数,不修改任何任务状态。 + * + *

存在的理由:现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死 + * (长任务/排队被判死)同时存在。新规则先把「心跳断」与「进度停滞」分开,并显式豁免 + * 服务端结果任务;影子期只把这些判断写进日志,攒真实分布再决定是否接管判死。 + * + *

状态边界:本类不许出现 update/delete/setStatus 一类任务写入;唯一的写入是 + * {@link TaskHeartbeatPositionService#evaluateProgressForShadow} 落在影子自己的进度快照键上, + * 不影响正式判死线的「未变化起点」。这条由 {@code TaskLivenessShadowGuardTest} 按源码钉住, + * 防止影子线悄悄变成执行线。 + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class TaskLivenessShadowService { + + public static final String LOG_ANCHOR = "[task-liveness-shadow]"; + /** 单轮最多逐条打多少条 would-finalize;其余只进汇总计数,避免日志被长任务尾巴刷屏。 */ + private static final int MAX_LOGGED_WOULD_FINALIZE = 20; + private static final String SCAN_LOCK_NAME = "task-liveness-shadow"; + + private final FileTaskMapper fileTaskMapper; + private final TaskFileJobMapper taskFileJobMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TaskHeartbeatLivenessService livenessService; + private final TaskHeartbeatPositionService positionService; + private final DistributedJobLockService distributedJobLockService; + + @Value("${aiimage.task-liveness-shadow.enabled:true}") + private boolean enabled; + @Value("${aiimage.task-liveness-shadow.scan-limit:500}") + private int scanLimit; + @Value("${aiimage.task-liveness-shadow.beat-lost-minutes:10}") + private int beatLostMinutes; + @Value("${aiimage.task-liveness-shadow.no-progress-minutes:30}") + private int noProgressMinutes; + @Value("${aiimage.task-liveness-shadow.first-heartbeat-grace-minutes:240}") + private int firstHeartbeatGraceMinutes; + @Value("${aiimage.task-liveness-shadow.hard-cap-hours:24}") + private int hardCapHours; + + @Scheduled(fixedDelayString = "${aiimage.task-liveness-shadow.scan-fixed-delay-ms:300000}", + initialDelayString = "${aiimage.task-liveness-shadow.scan-initial-delay-ms:120000}") + public void scheduledScan() { + if (!enabled) { + return; + } + DistributedJobLockService.LockHandle lock; + try { + lock = distributedJobLockService.tryLock(SCAN_LOCK_NAME, Duration.ofMinutes(5)); + } catch (Exception ex) { + log.warn("{} 获取扫描锁失败,本轮跳过 msg={}", LOG_ANCHOR, ex.getMessage()); + return; + } + if (lock == null) { + log.info("{} 另一实例持有扫描锁,本轮跳过", LOG_ANCHOR); + return; + } + try (lock) { + scanOnce(); + } catch (Exception ex) { + // 影子线只读:扫描失败绝不能影响任务本身 + log.warn("{} 扫描失败(只读,不影响任务) msg={}", LOG_ANCHOR, ex.getMessage(), ex); + } + } + + private void scanOnce() { + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getId, FileTaskEntity::getModuleType, + FileTaskEntity::getCreatedAt, FileTaskEntity::getUpdatedAt) + .eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING) + .orderByAsc(FileTaskEntity::getUpdatedAt) + .last("limit " + Math.max(1, scanLimit))); + if (tasks == null || tasks.isEmpty()) { + log.info("{} scanned=0(当前没有 RUNNING 任务)", LOG_ANCHOR); + return; + } + List taskIds = tasks.stream() + .map(FileTaskEntity::getId) + .filter(Objects::nonNull) + .toList(); + Set serverSideTaskIds = activeJobTaskIds(taskIds); + Map lastChunkAtByTaskId = lastChunkAtByTaskId(taskIds); + TaskLivenessShadowRules.Thresholds thresholds = new TaskLivenessShadowRules.Thresholds( + beatLostMinutes, noProgressMinutes, firstHeartbeatGraceMinutes, hardCapHours); + long now = System.currentTimeMillis(); + int keepAdvancing = 0; + int keepQueued = 0; + int keepServerSide = 0; + int unknown = 0; + int wouldFinalize = 0; + int logged = 0; + Map ruleCounts = new LinkedHashMap<>(); + for (FileTaskEntity task : tasks) { + Long taskId = task.getId(); + if (taskId == null) { + continue; + } + // 服务端结果任务在跑时不读心跳/位置:客户端已交棒,心跳停了属正常,读取只会白打 Redis。 + boolean serverSide = serverSideTaskIds.contains(taskId); + long beatMillis = serverSide ? 0L : livenessService.lastHeartbeatMillis(taskId); + // 从未心跳的任务由排队规则直接定论,不必再读一次位置(省一轮 Redis)。 + TaskLivenessShadowRules.ProgressState progress = !serverSide && beatMillis > 0L + ? toProgress(positionService.evaluateProgressForShadow(taskId, noProgressMinutes)) + : TaskLivenessShadowRules.ProgressState.UNKNOWN; + TaskLivenessShadowRules.Evidence evidence = new TaskLivenessShadowRules.Evidence( + taskId, task.getModuleType(), now, beatMillis, + toMillis(task.getUpdatedAt()), toMillis(task.getCreatedAt()), serverSide, progress); + TaskLivenessShadowRules.Decision decision = + TaskLivenessShadowRules.evaluate(evidence, thresholds); + switch (decision.verdict()) { + case KEEP_ADVANCING -> keepAdvancing++; + case KEEP_QUEUED -> keepQueued++; + case KEEP_SERVER_SIDE -> keepServerSide++; + case UNKNOWN -> unknown++; + case WOULD_FINALIZE -> { + wouldFinalize++; + ruleCounts.merge(decision.rule(), 1, Integer::sum); + if (logged < MAX_LOGGED_WOULD_FINALIZE) { + logWouldFinalize(evidence, decision, lastChunkAtByTaskId.get(taskId), now); + logged++; + } + } + } + } + log.info("{} scanned={} keepAdvancing={} keepQueued={} keepServerSide={} unknown={} wouldFinalize={} byRule={} " + + "thresholds=beatLost{}m/noProgress{}m/grace{}m/hardCap{}h", + LOG_ANCHOR, tasks.size(), keepAdvancing, keepQueued, keepServerSide, unknown, + wouldFinalize, ruleCounts, beatLostMinutes, noProgressMinutes, + firstHeartbeatGraceMinutes, hardCapHours); + } + + private void logWouldFinalize(TaskLivenessShadowRules.Evidence evidence, + TaskLivenessShadowRules.Decision decision, + LocalDateTime lastChunkAt, + long now) { + log.warn("{} would-finalize taskId={} module={} rule={} beatAgeMin={} queuedAgeMin={} taskAgeMin={} " + + "lastChunkAgeMin={} progress={} detail={}", + LOG_ANCHOR, evidence.taskId(), evidence.moduleType(), decision.rule(), + ageMinutesOrMissing(now, evidence.beatMillis()), + ageMinutesOrMissing(now, evidence.queuedSinceMillis()), + ageMinutesOrMissing(now, evidence.createdAtMillis()), + lastChunkAt == null ? -1L : ageMinutesOrMissing(now, toMillis(lastChunkAt)), + evidence.progress(), decision.detail()); + } + + private Set activeJobTaskIds(List taskIds) { + if (taskIds.isEmpty()) { + return Set.of(); + } + List jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper() + .select(TaskFileJobEntity::getTaskId) + .in(TaskFileJobEntity::getTaskId, taskIds) + .in(TaskFileJobEntity::getStatus, TaskStatuses.Job.ACTIVE_ORDERED)); + if (jobs == null || jobs.isEmpty()) { + return Set.of(); + } + Set result = new LinkedHashSet<>(); + for (TaskFileJobEntity job : jobs) { + if (job != null && job.getTaskId() != null) { + result.add(job.getTaskId()); + } + } + return result; + } + + private Map lastChunkAtByTaskId(List taskIds) { + Map result = new HashMap<>(); + if (taskIds.isEmpty()) { + return result; + } + List rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(taskIds); + if (rows == null) { + return result; + } + for (TaskScopeLastChunkDto row : rows) { + if (row != null && row.taskId() != null && row.lastChunkAt() != null) { + result.put(row.taskId(), row.lastChunkAt()); + } + } + return result; + } + + private static TaskLivenessShadowRules.ProgressState toProgress( + TaskHeartbeatPositionService.StallState stallState) { + if (stallState == null) { + return TaskLivenessShadowRules.ProgressState.UNKNOWN; + } + return switch (stallState) { + case ADVANCING -> TaskLivenessShadowRules.ProgressState.ADVANCING; + case STALLED -> TaskLivenessShadowRules.ProgressState.STALLED; + case UNKNOWN -> TaskLivenessShadowRules.ProgressState.UNKNOWN; + }; + } + + private static long toMillis(LocalDateTime time) { + return time == null ? 0L : time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + } + + private static long ageMinutesOrMissing(long nowMillis, long sinceMillis) { + if (sinceMillis <= 0L) { + return -1L; + } + return Math.max(0L, (nowMillis - sinceMillis) / 60_000L); + } +} diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 9ea53f7f0..38db3be85 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -240,6 +240,18 @@ aiimage: # 置 0 即整体关闭闸门、退回旧行为(只需改环境变量,不用重新构建)。 task-stale: first-heartbeat-grace-minutes: ${AIIMAGE_TASK_STALE_FIRST_HEARTBEAT_GRACE_MINUTES:240} + # 心跳影子判官(2026-10-20):只读扫描 RUNNING 文件任务,按「心跳中断 / 进度停滞 / 排队宽限 / + # 服务端处理中 / 总量硬顶」输出 would-finalize 日志,不修改任何任务状态。先用真实读数校准阈值, + # 再决定是否接进判死线;enabled=false 即整体停用。 + task-liveness-shadow: + enabled: ${AIIMAGE_TASK_LIVENESS_SHADOW_ENABLED:true} + scan-fixed-delay-ms: ${AIIMAGE_TASK_LIVENESS_SHADOW_SCAN_FIXED_DELAY_MS:300000} + scan-initial-delay-ms: ${AIIMAGE_TASK_LIVENESS_SHADOW_SCAN_INITIAL_DELAY_MS:120000} + scan-limit: ${AIIMAGE_TASK_LIVENESS_SHADOW_SCAN_LIMIT:500} + beat-lost-minutes: ${AIIMAGE_TASK_LIVENESS_SHADOW_BEAT_LOST_MINUTES:10} + no-progress-minutes: ${AIIMAGE_TASK_LIVENESS_SHADOW_NO_PROGRESS_MINUTES:30} + first-heartbeat-grace-minutes: ${AIIMAGE_TASK_LIVENESS_SHADOW_FIRST_HEARTBEAT_GRACE_MINUTES:240} + hard-cap-hours: ${AIIMAGE_TASK_LIVENESS_SHADOW_HARD_CAP_HOURS:24} module-cleanup: enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true} cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java index d2062c0ae..2ba46aa89 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java @@ -329,6 +329,38 @@ class TaskHeartbeatPositionServiceTest { assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L)); } + @Test + void shadowStallUsesItsOwnSnapshotKey() { + // 影子每 5 分钟一跑;共用正式快照键会把「未变化起点」提前,间接改变封口时刻 + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)", + "current", "401", "total", "1300")); + when(hashOperations.entries("task:progress:shadow-stall:123")) + .thenReturn(Map.of("pos", "crawling||400|1300|ASIN B0F44Y3YYQ(400/1300)", + "at", String.valueOf(Long.MAX_VALUE))); + + assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, + service().evaluateProgressForShadow(123L, 30L)); + verify(hashOperations).putAll(eq("task:progress:shadow-stall:123"), any()); + verify(hashOperations, never()).putAll(eq("task:progress:stall:123"), any()); + } + + @Test + void shadowProgressCountsChangesEvenWhenTextIsStatic() { + // 长页面里文本可能长时间不变,但计数在涨 ⇒ 必须算推进 + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("progressText", "明细处理中", + "current", "401", "total", "1300")); + when(hashOperations.entries("task:progress:shadow-stall:123")) + .thenReturn(Map.of("pos", "crawling||400|1300|明细处理中", + "at", String.valueOf(Long.MAX_VALUE))); + + assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, + service().evaluateProgressForShadow(123L, 30L)); + } + @Test void clientAdvancingOnlyForAdvancingState() { when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowGuardTest.java new file mode 100644 index 000000000..479f00f8d --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowGuardTest.java @@ -0,0 +1,76 @@ +package com.nanri.aiimage.modules.task.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * 影子判官必须是纯只读:它现在的价值是攒读数,不是接管判死。 + * + *

这条守卫防的是最危险的那种「顺手」——把影子里的 would-finalize 改成真的 finalize, + * 在没有一周真实分布之前就把长任务杀掉。规则类也必须保持无 Spring/无 Mapper 的纯函数, + * 否则规则本身没法逐条离线复核。 + */ +class TaskLivenessShadowGuardTest { + + private static final List FORBIDDEN_WRITES = List.of( + ".updateById(", ".update(", ".delete(", "setStatus(", "setErrorMessage(", "setFinishedAt("); + + @Test + void 影子服务源码里不许出现任务写入() throws IOException { + String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java"); + for (String forbidden : FORBIDDEN_WRITES) { + assertThat(source) + .as("影子线只能读:出现了 %s,说明它已经越过影子边界", forbidden) + .doesNotContain(forbidden); + } + assertThat(source) + .as("影子扫描必须有日志锚点,否则生产上无法检索读数") + .contains("LOG_ANCHOR = \"[task-liveness-shadow]\""); + } + + @Test + void 规则类必须是可离线复核的纯函数() throws IOException { + String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java"); + assertThat(source) + .as("规则类不许依赖 Spring/Redis/Mapper,否则不能按纯函数单测") + .doesNotContain("import org.springframework") + .doesNotContain("import com.nanri.aiimage.modules.task.mapper") + .doesNotContain("StringRedisTemplate"); + } + + @Test + void 开关默认开启且可以只改环境变量停用() throws IOException { + String yaml = readSource("resources/application.yml"); + assertThat(yaml) + .contains("task-liveness-shadow:") + .contains("${AIIMAGE_TASK_LIVENESS_SHADOW_ENABLED:true}"); + } + + private static String readSource(String moduleRelativePath) throws IOException { + Path root = backendJavaRoot(); + Path file = root.resolve("src/main/" + moduleRelativePath); + if (!Files.isRegularFile(file)) { + fail("找不到 " + file + " —— 守卫依赖以 backend-java 为工作目录运行,别让它静默通过"); + } + return Files.readString(file, StandardCharsets.UTF_8); + } + + private static Path backendJavaRoot() { + Path here = Path.of("").toAbsolutePath(); + if (Files.isDirectory(here.resolve("src/main"))) { + return here; + } + Path nested = here.resolve("backend-java"); + if (Files.isDirectory(nested.resolve("src/main"))) { + return nested; + } + throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRulesTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRulesTest.java new file mode 100644 index 000000000..8fde4f88e --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRulesTest.java @@ -0,0 +1,125 @@ +package com.nanri.aiimage.modules.task.service; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Decision; +import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Evidence; +import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.ProgressState; +import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Thresholds; +import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Verdict; +import org.junit.jupiter.api.Test; + +/** + * 影子规则:把「心跳断」与「进度停滞」拆开,并把排队/服务端处理中排除在判死之外。 + * + *

这条线只产出 would-finalize 读数,所以规则必须能逐条解释:命中哪条、 + * 用了哪份证据、为什么不是 KEEP。这里把每条规则与边界都钉住,影子期读数才可信。 + */ +class TaskLivenessShadowRulesTest { + + private static final long NOW = 1_800_000_000_000L; + private static final Thresholds DEFAULT = new Thresholds(10, 30, 240, 24); + + @Test + void 服务端结果任务在跑时即使没有心跳也保持存活() { + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(null, 5_000, 5_000, true, ProgressState.UNKNOWN), DEFAULT); + + assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_SERVER_SIDE); + assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_SERVER_SIDE_JOB); + } + + @Test + void 从未心跳但仍在排队宽限内不判死() { + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(null, 100, 100, false, ProgressState.UNKNOWN), DEFAULT); + + assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_QUEUED); + assertThat(decision.detail()).contains("已等待 100 分钟").contains("宽限 240 分钟"); + } + + @Test + void 从未心跳且超过排队宽限才判死() { + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(null, 300, 300, false, ProgressState.UNKNOWN), DEFAULT); + + assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE); + assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_FIRST_HEARTBEAT); + } + + @Test + void 排队宽限停用时不产出判死读数() { + Thresholds disabled = new Thresholds(10, 30, 0, 24); + + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(null, 300, 300, false, ProgressState.UNKNOWN), disabled); + + assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN); + } + + @Test + void 心跳中断超过阈值判死_边界值本身不判() { + Decision stale = TaskLivenessShadowRules.evaluate( + evidence(11L, 100, 100, false, ProgressState.ADVANCING), DEFAULT); + Decision boundary = TaskLivenessShadowRules.evaluate( + evidence(10L, 100, 100, false, ProgressState.ADVANCING), DEFAULT); + + assertThat(stale.verdict()).isEqualTo(Verdict.WOULD_FINALIZE); + assertThat(stale.rule()).isEqualTo(TaskLivenessShadowRules.RULE_BEAT_LOST); + assertThat(boundary.verdict()).isEqualTo(Verdict.KEEP_ADVANCING); + } + + @Test + void 长任务只要还在推进就不判死() { + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(1L, 10 * 60, 10 * 60, false, ProgressState.ADVANCING), DEFAULT); + + assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_ADVANCING); + assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_ADVANCING); + } + + @Test + void 总量硬顶压过推进状态() { + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(1L, 24 * 60, 24 * 60, false, ProgressState.ADVANCING), DEFAULT); + + assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE); + assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_HARD_CAP); + assertThat(decision.detail()).contains("24 小时"); + } + + @Test + void 心跳正常但进度停滞判死() { + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(1L, 100, 100, false, ProgressState.STALLED), DEFAULT); + + assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE); + assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_PROGRESS); + assertThat(decision.detail()).contains("30 分钟未推进"); + } + + @Test + void 没有进度位置只算证据不足不判死() { + Decision decision = TaskLivenessShadowRules.evaluate( + evidence(1L, 100, 100, false, ProgressState.UNKNOWN), DEFAULT); + + assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN); + assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_POSITION); + } + + private static Evidence evidence(Long beatAgoMinutes, + long createdAgoMinutes, + long queuedAgoMinutes, + boolean serverSideJob, + ProgressState progress) { + return new Evidence(1L, "SIMILAR_ASIN", NOW, + ago(beatAgoMinutes), + NOW - queuedAgoMinutes * 60_000L, + NOW - createdAgoMinutes * 60_000L, + serverSideJob, progress); + } + + private static long ago(Long minutes) { + return minutes == null ? 0L : NOW - minutes * 60_000L; + } +}