feat(心跳): 影子判官先跑读数——心跳断/进度停滞/排队/服务端处理分开判

现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死(长任务/排队被
判死)同时存在。新增只读影子扫描:每 5 分钟扫 RUNNING 文件任务,按「心跳中断 /
进度停滞 / 排队宽限 / 服务端结果任务 / 24h 硬顶」输出 would-finalize 日志,
不改任何任务状态。进度用「计数+文本+阶段」指纹,写影子独立快照键,不影响正式
判死线的未变化起点。规则纯函数单测 9 例 + 只读边界守卫 3 例。
This commit is contained in:
2026-10-20 12:30:09 +08:00
parent 4c1a2a9ffb
commit eb6f082674
7 changed files with 671 additions and 6 deletions
@@ -28,6 +28,8 @@ public class TaskHeartbeatPositionService {
private static final String KEY_PREFIX = "task:heartbeat:position:";
private static final String STALL_KEY_PREFIX = "task:progress:stall:";
/** 影子判官的独立快照键:影子的观测时刻不得改变正式判死线的「未变化起点」。 */
private static final String SHADOW_STALL_KEY_PREFIX = "task:progress:shadow-stall:";
private static final Duration POSITION_TTL = Duration.ofHours(24);
private static final int POSITION_LABEL_MAX_LEN = 60;
/** 计数字符串的最长位数:9 位以内 parseInt 必不溢出,所以这一处不需要 catch。 */
@@ -303,7 +305,57 @@ public class TaskHeartbeatPositionService {
if (position == null || position.isBlank()) {
return StallState.UNKNOWN;
}
String stallKey = STALL_KEY_PREFIX + taskId;
return evaluateAgainstSnapshot(taskId, STALL_KEY_PREFIX + taskId, position, staleWindowMinutes);
}
/**
* 影子判官专用:把「计数 + 文本 + 阶段」压成指纹判断推进,快照写独立键。
*
* <p>与正式线只比文本不同:计数变了也是推进(长页面里文本可能长时间不变),
* 阶段变化同样算推进。影子每 5 分钟扫一次,若与正式线共用快照键,影子可能比正式线早几分钟
* 看到变化,从而把正式线的未变化起点提前、间接影响封口时刻;所以这里只写 shadow 键。
*/
public StallState evaluateProgressForShadow(Long taskId, long staleWindowMinutes) {
if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) {
return StallState.UNKNOWN;
}
String fingerprint = readProgressFingerprint(taskId);
if (fingerprint == null) {
return StallState.UNKNOWN;
}
return evaluateAgainstSnapshot(taskId, SHADOW_STALL_KEY_PREFIX + taskId,
fingerprint, staleWindowMinutes);
}
/**
* 影子判官的推进指纹:位置哈希一次读出,计数/文本/阶段任一变化都算推进。
* 返回 null 表示这个任务没有任何位置信息(老客户端不传位置),不能作为判死证据。
*/
private String readProgressFingerprint(Long taskId) {
Map<Object, Object> entries;
try {
entries = stringRedisTemplate.opsForHash().entries(KEY_PREFIX + taskId);
} catch (Exception ex) {
log.warn("[heartbeat-position] 读取影子位置失败 taskId={} msg={}", taskId, ex.getMessage());
return null;
}
if (entries == null || entries.isEmpty()) {
return null;
}
String text = value(entries, "progressText");
String current = value(entries, "current");
String total = value(entries, "total");
if (text.isBlank() && current.isBlank() && total.isBlank()) {
return null;
}
return value(entries, "phase") + "|" + value(entries, "grain")
+ "|" + current + "|" + total + "|" + text;
}
private StallState evaluateAgainstSnapshot(Long taskId,
String stallKey,
String fingerprint,
long staleWindowMinutes) {
Map<Object, Object> stored;
try {
stored = stringRedisTemplate.opsForHash().entries(stallKey);
@@ -312,18 +364,18 @@ public class TaskHeartbeatPositionService {
return StallState.UNKNOWN;
}
long now = Instant.now().toEpochMilli();
String storedPosition = stored == null ? "" : value(stored, "pos");
String storedFingerprint = stored == null ? "" : value(stored, "pos");
long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at"));
if (!position.equals(storedPosition) || unchangedSince <= 0L) {
writeStallSnapshot(stallKey, position, now, staleWindowMinutes);
if (!fingerprint.equals(storedFingerprint) || unchangedSince <= 0L) {
writeStallSnapshot(stallKey, fingerprint, now, staleWindowMinutes);
return StallState.ADVANCING;
}
long unchangedMillis = now - unchangedSince;
if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) {
return StallState.STALLED;
}
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}",
taskId, unchangedMillis, position);
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} fingerprint={}",
taskId, unchangedMillis, fingerprint);
return StallState.ADVANCING;
}
@@ -0,0 +1,132 @@
package com.nanri.aiimage.modules.task.service;
import java.util.Objects;
/**
* 心跳影子判官的纯规则:只回答「按新规则会怎么判」,不碰任何任务状态。
*
* <p>为什么要影子期:现有判死线各看各的证据(Redis 心跳、file_task.updated_at、
* last_chunk_at、各模块超时),既有「心跳线程活着但主线程卡死」的假活,也有长任务/排队
* 被判死的假死。新规则把「心跳是否断」与「进度是否推进」分开,并显式豁免服务端处理中的
* 任务;先用真实 RUNNING 任务跑一周 would-finalize 读数,再决定是否接管判死。
*
* <p>规则优先级(先命中先返回):
* <ol>
* <li>服务端结果任务仍在跑 → KEEP_SERVER_SIDE(客户端已交棒,心跳停了是正常的);</li>
* <li>从未收到心跳 → 未启动宽限内 KEEP_QUEUED,超宽限 WOULD_FINALIZE(no-first-heartbeat);</li>
* <li>心跳中断超过阈值 → WOULD_FINALIZE(beat-lost);</li>
* <li>任务总时长超过硬顶 → WOULD_FINALIZE(hard-cap)(即使进度仍在推进);</li>
* <li>心跳正常但进度停滞 → WOULD_FINALIZE(no-progress);</li>
* <li>进度在推进 → KEEP_ADVANCING;</li>
* <li>没有位置信息 → UNKNOWN(证据不足,不参与判死)。</li>
* </ol>
*/
public final class TaskLivenessShadowRules {
public static final String RULE_SERVER_SIDE_JOB = "server-side-job";
public static final String RULE_NO_FIRST_HEARTBEAT = "no-first-heartbeat";
public static final String RULE_BEAT_LOST = "beat-lost";
public static final String RULE_HARD_CAP = "hard-cap";
public static final String RULE_NO_PROGRESS = "no-progress";
public static final String RULE_ADVANCING = "advancing";
public static final String RULE_NO_POSITION = "no-position";
public enum Verdict {
KEEP_ADVANCING,
KEEP_QUEUED,
KEEP_SERVER_SIDE,
UNKNOWN,
WOULD_FINALIZE
}
/** 客户端上报的进度位置是否还在变;UNKNOWN 表示老客户端没传位置,不能当判死证据。 */
public enum ProgressState {
ADVANCING,
STALLED,
UNKNOWN
}
/**
* @param beatLostMinutes 心跳中断多少分钟算断
* @param noProgressMinutes 进度位置多少分钟没变算停滞
* @param firstHeartbeatGraceMinutes 从未心跳的排队宽限;&lt;=0 表示该规则停用(判 UNKNOWN)
* @param hardCapHours 任务总时长硬顶;&lt;=0 表示该规则停用
*/
public record Thresholds(int beatLostMinutes,
int noProgressMinutes,
int firstHeartbeatGraceMinutes,
int hardCapHours) {
}
/**
* @param beatMillis 最近一次心跳(0=从未收到)
* @param queuedSinceMillis 未启动宽限的锚(file_task.updated_at)
* @param createdAtMillis 任务创建时刻,用于总量硬顶
*/
public record Evidence(long taskId,
String moduleType,
long nowMillis,
long beatMillis,
long queuedSinceMillis,
long createdAtMillis,
boolean serverSideJob,
ProgressState progress) {
}
public record Decision(Verdict verdict, String rule, String detail) {
}
private TaskLivenessShadowRules() {
}
public static Decision evaluate(Evidence evidence, Thresholds thresholds) {
Objects.requireNonNull(evidence, "evidence");
Objects.requireNonNull(thresholds, "thresholds");
if (evidence.serverSideJob()) {
return new Decision(Verdict.KEEP_SERVER_SIDE, RULE_SERVER_SIDE_JOB,
"服务端结果任务仍在运行,客户端心跳停止属正常");
}
if (evidence.beatMillis() <= 0L) {
long waitedMinutes = ageMinutes(evidence.nowMillis(), evidence.queuedSinceMillis());
if (thresholds.firstHeartbeatGraceMinutes() <= 0) {
return new Decision(Verdict.UNKNOWN, RULE_NO_FIRST_HEARTBEAT,
"从未收到心跳,但排队宽限规则已停用,无法判定");
}
if (waitedMinutes < thresholds.firstHeartbeatGraceMinutes()) {
return new Decision(Verdict.KEEP_QUEUED, RULE_NO_FIRST_HEARTBEAT,
"从未收到心跳,仍在排队宽限内(已等待 " + waitedMinutes
+ " 分钟 / 宽限 " + thresholds.firstHeartbeatGraceMinutes() + " 分钟)");
}
return new Decision(Verdict.WOULD_FINALIZE, RULE_NO_FIRST_HEARTBEAT,
"从未收到心跳,已等待 " + waitedMinutes
+ " 分钟(宽限 " + thresholds.firstHeartbeatGraceMinutes() + " 分钟)");
}
long beatAgeMinutes = ageMinutes(evidence.nowMillis(), evidence.beatMillis());
if (thresholds.beatLostMinutes() > 0 && beatAgeMinutes > thresholds.beatLostMinutes()) {
return new Decision(Verdict.WOULD_FINALIZE, RULE_BEAT_LOST,
"心跳已中断 " + beatAgeMinutes + " 分钟(阈值 " + thresholds.beatLostMinutes() + " 分钟)");
}
long taskAgeMinutes = ageMinutes(evidence.nowMillis(), evidence.createdAtMillis());
long taskAgeHours = taskAgeMinutes / 60L;
if (thresholds.hardCapHours() > 0 && taskAgeHours >= thresholds.hardCapHours()) {
return new Decision(Verdict.WOULD_FINALIZE, RULE_HARD_CAP,
"任务已运行 " + taskAgeHours + " 小时(硬顶 " + thresholds.hardCapHours() + " 小时)");
}
if (evidence.progress() == ProgressState.STALLED) {
return new Decision(Verdict.WOULD_FINALIZE, RULE_NO_PROGRESS,
"心跳正常但进度至少 " + thresholds.noProgressMinutes() + " 分钟未推进");
}
if (evidence.progress() == ProgressState.ADVANCING) {
return new Decision(Verdict.KEEP_ADVANCING, RULE_ADVANCING, "进度仍在推进");
}
return new Decision(Verdict.UNKNOWN, RULE_NO_POSITION,
"心跳正常但没有进度位置,证据不足");
}
private static long ageMinutes(long nowMillis, long sinceMillis) {
if (sinceMillis <= 0L) {
return 0L;
}
return Math.max(0L, (nowMillis - sinceMillis) / 60_000L);
}
}
@@ -0,0 +1,236 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.task.TaskStatuses;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/**
* 心跳影子判官:只读扫描 RUNNING 任务,按 {@link TaskLivenessShadowRules} 输出
* would-finalize 读数,<b>不修改任何任务状态</b>。
*
* <p>存在的理由:现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死
* (长任务/排队被判死)同时存在。新规则先把「心跳断」与「进度停滞」分开,并显式豁免
* 服务端结果任务;影子期只把这些判断写进日志,攒真实分布再决定是否接管判死。
*
* <p>状态边界:本类不许出现 update/delete/setStatus 一类任务写入;唯一的写入是
* {@link TaskHeartbeatPositionService#evaluateProgressForShadow} 落在影子自己的进度快照键上,
* 不影响正式判死线的「未变化起点」。这条由 {@code TaskLivenessShadowGuardTest} 按源码钉住,
* 防止影子线悄悄变成执行线。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskLivenessShadowService {
public static final String LOG_ANCHOR = "[task-liveness-shadow]";
/** 单轮最多逐条打多少条 would-finalize;其余只进汇总计数,避免日志被长任务尾巴刷屏。 */
private static final int MAX_LOGGED_WOULD_FINALIZE = 20;
private static final String SCAN_LOCK_NAME = "task-liveness-shadow";
private final FileTaskMapper fileTaskMapper;
private final TaskFileJobMapper taskFileJobMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskHeartbeatLivenessService livenessService;
private final TaskHeartbeatPositionService positionService;
private final DistributedJobLockService distributedJobLockService;
@Value("${aiimage.task-liveness-shadow.enabled:true}")
private boolean enabled;
@Value("${aiimage.task-liveness-shadow.scan-limit:500}")
private int scanLimit;
@Value("${aiimage.task-liveness-shadow.beat-lost-minutes:10}")
private int beatLostMinutes;
@Value("${aiimage.task-liveness-shadow.no-progress-minutes:30}")
private int noProgressMinutes;
@Value("${aiimage.task-liveness-shadow.first-heartbeat-grace-minutes:240}")
private int firstHeartbeatGraceMinutes;
@Value("${aiimage.task-liveness-shadow.hard-cap-hours:24}")
private int hardCapHours;
@Scheduled(fixedDelayString = "${aiimage.task-liveness-shadow.scan-fixed-delay-ms:300000}",
initialDelayString = "${aiimage.task-liveness-shadow.scan-initial-delay-ms:120000}")
public void scheduledScan() {
if (!enabled) {
return;
}
DistributedJobLockService.LockHandle lock;
try {
lock = distributedJobLockService.tryLock(SCAN_LOCK_NAME, Duration.ofMinutes(5));
} catch (Exception ex) {
log.warn("{} 获取扫描锁失败,本轮跳过 msg={}", LOG_ANCHOR, ex.getMessage());
return;
}
if (lock == null) {
log.info("{} 另一实例持有扫描锁,本轮跳过", LOG_ANCHOR);
return;
}
try (lock) {
scanOnce();
} catch (Exception ex) {
// 影子线只读:扫描失败绝不能影响任务本身
log.warn("{} 扫描失败(只读,不影响任务) msg={}", LOG_ANCHOR, ex.getMessage(), ex);
}
}
private void scanOnce() {
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId, FileTaskEntity::getModuleType,
FileTaskEntity::getCreatedAt, FileTaskEntity::getUpdatedAt)
.eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit " + Math.max(1, scanLimit)));
if (tasks == null || tasks.isEmpty()) {
log.info("{} scanned=0(当前没有 RUNNING 任务)", LOG_ANCHOR);
return;
}
List<Long> taskIds = tasks.stream()
.map(FileTaskEntity::getId)
.filter(Objects::nonNull)
.toList();
Set<Long> serverSideTaskIds = activeJobTaskIds(taskIds);
Map<Long, LocalDateTime> lastChunkAtByTaskId = lastChunkAtByTaskId(taskIds);
TaskLivenessShadowRules.Thresholds thresholds = new TaskLivenessShadowRules.Thresholds(
beatLostMinutes, noProgressMinutes, firstHeartbeatGraceMinutes, hardCapHours);
long now = System.currentTimeMillis();
int keepAdvancing = 0;
int keepQueued = 0;
int keepServerSide = 0;
int unknown = 0;
int wouldFinalize = 0;
int logged = 0;
Map<String, Integer> ruleCounts = new LinkedHashMap<>();
for (FileTaskEntity task : tasks) {
Long taskId = task.getId();
if (taskId == null) {
continue;
}
// 服务端结果任务在跑时不读心跳/位置:客户端已交棒,心跳停了属正常,读取只会白打 Redis。
boolean serverSide = serverSideTaskIds.contains(taskId);
long beatMillis = serverSide ? 0L : livenessService.lastHeartbeatMillis(taskId);
// 从未心跳的任务由排队规则直接定论,不必再读一次位置(省一轮 Redis)。
TaskLivenessShadowRules.ProgressState progress = !serverSide && beatMillis > 0L
? toProgress(positionService.evaluateProgressForShadow(taskId, noProgressMinutes))
: TaskLivenessShadowRules.ProgressState.UNKNOWN;
TaskLivenessShadowRules.Evidence evidence = new TaskLivenessShadowRules.Evidence(
taskId, task.getModuleType(), now, beatMillis,
toMillis(task.getUpdatedAt()), toMillis(task.getCreatedAt()), serverSide, progress);
TaskLivenessShadowRules.Decision decision =
TaskLivenessShadowRules.evaluate(evidence, thresholds);
switch (decision.verdict()) {
case KEEP_ADVANCING -> keepAdvancing++;
case KEEP_QUEUED -> keepQueued++;
case KEEP_SERVER_SIDE -> keepServerSide++;
case UNKNOWN -> unknown++;
case WOULD_FINALIZE -> {
wouldFinalize++;
ruleCounts.merge(decision.rule(), 1, Integer::sum);
if (logged < MAX_LOGGED_WOULD_FINALIZE) {
logWouldFinalize(evidence, decision, lastChunkAtByTaskId.get(taskId), now);
logged++;
}
}
}
}
log.info("{} scanned={} keepAdvancing={} keepQueued={} keepServerSide={} unknown={} wouldFinalize={} byRule={} "
+ "thresholds=beatLost{}m/noProgress{}m/grace{}m/hardCap{}h",
LOG_ANCHOR, tasks.size(), keepAdvancing, keepQueued, keepServerSide, unknown,
wouldFinalize, ruleCounts, beatLostMinutes, noProgressMinutes,
firstHeartbeatGraceMinutes, hardCapHours);
}
private void logWouldFinalize(TaskLivenessShadowRules.Evidence evidence,
TaskLivenessShadowRules.Decision decision,
LocalDateTime lastChunkAt,
long now) {
log.warn("{} would-finalize taskId={} module={} rule={} beatAgeMin={} queuedAgeMin={} taskAgeMin={} "
+ "lastChunkAgeMin={} progress={} detail={}",
LOG_ANCHOR, evidence.taskId(), evidence.moduleType(), decision.rule(),
ageMinutesOrMissing(now, evidence.beatMillis()),
ageMinutesOrMissing(now, evidence.queuedSinceMillis()),
ageMinutesOrMissing(now, evidence.createdAtMillis()),
lastChunkAt == null ? -1L : ageMinutesOrMissing(now, toMillis(lastChunkAt)),
evidence.progress(), decision.detail());
}
private Set<Long> activeJobTaskIds(List<Long> taskIds) {
if (taskIds.isEmpty()) {
return Set.of();
}
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.select(TaskFileJobEntity::getTaskId)
.in(TaskFileJobEntity::getTaskId, taskIds)
.in(TaskFileJobEntity::getStatus, TaskStatuses.Job.ACTIVE_ORDERED));
if (jobs == null || jobs.isEmpty()) {
return Set.of();
}
Set<Long> result = new LinkedHashSet<>();
for (TaskFileJobEntity job : jobs) {
if (job != null && job.getTaskId() != null) {
result.add(job.getTaskId());
}
}
return result;
}
private Map<Long, LocalDateTime> lastChunkAtByTaskId(List<Long> taskIds) {
Map<Long, LocalDateTime> result = new HashMap<>();
if (taskIds.isEmpty()) {
return result;
}
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(taskIds);
if (rows == null) {
return result;
}
for (TaskScopeLastChunkDto row : rows) {
if (row != null && row.taskId() != null && row.lastChunkAt() != null) {
result.put(row.taskId(), row.lastChunkAt());
}
}
return result;
}
private static TaskLivenessShadowRules.ProgressState toProgress(
TaskHeartbeatPositionService.StallState stallState) {
if (stallState == null) {
return TaskLivenessShadowRules.ProgressState.UNKNOWN;
}
return switch (stallState) {
case ADVANCING -> TaskLivenessShadowRules.ProgressState.ADVANCING;
case STALLED -> TaskLivenessShadowRules.ProgressState.STALLED;
case UNKNOWN -> TaskLivenessShadowRules.ProgressState.UNKNOWN;
};
}
private static long toMillis(LocalDateTime time) {
return time == null ? 0L : time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
}
private static long ageMinutesOrMissing(long nowMillis, long sinceMillis) {
if (sinceMillis <= 0L) {
return -1L;
}
return Math.max(0L, (nowMillis - sinceMillis) / 60_000L);
}
}
@@ -240,6 +240,18 @@ aiimage:
# 置 0 即整体关闭闸门、退回旧行为(只需改环境变量,不用重新构建)。
task-stale:
first-heartbeat-grace-minutes: ${AIIMAGE_TASK_STALE_FIRST_HEARTBEAT_GRACE_MINUTES:240}
# 心跳影子判官(2026-10-20):只读扫描 RUNNING 文件任务,按「心跳中断 / 进度停滞 / 排队宽限 /
# 服务端处理中 / 总量硬顶」输出 would-finalize 日志,不修改任何任务状态。先用真实读数校准阈值,
# 再决定是否接进判死线;enabled=false 即整体停用。
task-liveness-shadow:
enabled: ${AIIMAGE_TASK_LIVENESS_SHADOW_ENABLED:true}
scan-fixed-delay-ms: ${AIIMAGE_TASK_LIVENESS_SHADOW_SCAN_FIXED_DELAY_MS:300000}
scan-initial-delay-ms: ${AIIMAGE_TASK_LIVENESS_SHADOW_SCAN_INITIAL_DELAY_MS:120000}
scan-limit: ${AIIMAGE_TASK_LIVENESS_SHADOW_SCAN_LIMIT:500}
beat-lost-minutes: ${AIIMAGE_TASK_LIVENESS_SHADOW_BEAT_LOST_MINUTES:10}
no-progress-minutes: ${AIIMAGE_TASK_LIVENESS_SHADOW_NO_PROGRESS_MINUTES:30}
first-heartbeat-grace-minutes: ${AIIMAGE_TASK_LIVENESS_SHADOW_FIRST_HEARTBEAT_GRACE_MINUTES:240}
hard-cap-hours: ${AIIMAGE_TASK_LIVENESS_SHADOW_HARD_CAP_HOURS:24}
module-cleanup:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
@@ -329,6 +329,38 @@ class TaskHeartbeatPositionServiceTest {
assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L));
}
@Test
void shadowStallUsesItsOwnSnapshotKey() {
// 影子每 5 分钟一跑;共用正式快照键会把「未变化起点」提前,间接改变封口时刻
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)",
"current", "401", "total", "1300"));
when(hashOperations.entries("task:progress:shadow-stall:123"))
.thenReturn(Map.of("pos", "crawling||400|1300|ASIN B0F44Y3YYQ(400/1300)",
"at", String.valueOf(Long.MAX_VALUE)));
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING,
service().evaluateProgressForShadow(123L, 30L));
verify(hashOperations).putAll(eq("task:progress:shadow-stall:123"), any());
verify(hashOperations, never()).putAll(eq("task:progress:stall:123"), any());
}
@Test
void shadowProgressCountsChangesEvenWhenTextIsStatic() {
// 长页面里文本可能长时间不变,但计数在涨 ⇒ 必须算推进
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "明细处理中",
"current", "401", "total", "1300"));
when(hashOperations.entries("task:progress:shadow-stall:123"))
.thenReturn(Map.of("pos", "crawling||400|1300|明细处理中",
"at", String.valueOf(Long.MAX_VALUE)));
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING,
service().evaluateProgressForShadow(123L, 30L));
}
@Test
void clientAdvancingOnlyForAdvancingState() {
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
@@ -0,0 +1,76 @@
package com.nanri.aiimage.modules.task.service;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Test;
/**
* 影子判官必须是纯只读:它现在的价值是攒读数,不是接管判死。
*
* <p>这条守卫防的是最危险的那种「顺手」——把影子里的 would-finalize 改成真的 finalize,
* 在没有一周真实分布之前就把长任务杀掉。规则类也必须保持无 Spring/无 Mapper 的纯函数,
* 否则规则本身没法逐条离线复核。
*/
class TaskLivenessShadowGuardTest {
private static final List<String> FORBIDDEN_WRITES = List.of(
".updateById(", ".update(", ".delete(", "setStatus(", "setErrorMessage(", "setFinishedAt(");
@Test
void 影子服务源码里不许出现任务写入() throws IOException {
String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java");
for (String forbidden : FORBIDDEN_WRITES) {
assertThat(source)
.as("影子线只能读:出现了 %s,说明它已经越过影子边界", forbidden)
.doesNotContain(forbidden);
}
assertThat(source)
.as("影子扫描必须有日志锚点,否则生产上无法检索读数")
.contains("LOG_ANCHOR = \"[task-liveness-shadow]\"");
}
@Test
void 规则类必须是可离线复核的纯函数() throws IOException {
String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java");
assertThat(source)
.as("规则类不许依赖 Spring/Redis/Mapper,否则不能按纯函数单测")
.doesNotContain("import org.springframework")
.doesNotContain("import com.nanri.aiimage.modules.task.mapper")
.doesNotContain("StringRedisTemplate");
}
@Test
void 开关默认开启且可以只改环境变量停用() throws IOException {
String yaml = readSource("resources/application.yml");
assertThat(yaml)
.contains("task-liveness-shadow:")
.contains("${AIIMAGE_TASK_LIVENESS_SHADOW_ENABLED:true}");
}
private static String readSource(String moduleRelativePath) throws IOException {
Path root = backendJavaRoot();
Path file = root.resolve("src/main/" + moduleRelativePath);
if (!Files.isRegularFile(file)) {
fail("找不到 " + file + " —— 守卫依赖以 backend-java 为工作目录运行,别让它静默通过");
}
return Files.readString(file, StandardCharsets.UTF_8);
}
private static Path backendJavaRoot() {
Path here = Path.of("").toAbsolutePath();
if (Files.isDirectory(here.resolve("src/main"))) {
return here;
}
Path nested = here.resolve("backend-java");
if (Files.isDirectory(nested.resolve("src/main"))) {
return nested;
}
throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")");
}
}
@@ -0,0 +1,125 @@
package com.nanri.aiimage.modules.task.service;
import static org.assertj.core.api.Assertions.assertThat;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Decision;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Evidence;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.ProgressState;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Thresholds;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Verdict;
import org.junit.jupiter.api.Test;
/**
* 影子规则:把「心跳断」与「进度停滞」拆开,并把排队/服务端处理中排除在判死之外。
*
* <p>这条线只产出 would-finalize 读数,所以规则必须能逐条解释:命中哪条、
* 用了哪份证据、为什么不是 KEEP。这里把每条规则与边界都钉住,影子期读数才可信。
*/
class TaskLivenessShadowRulesTest {
private static final long NOW = 1_800_000_000_000L;
private static final Thresholds DEFAULT = new Thresholds(10, 30, 240, 24);
@Test
void 服务端结果任务在跑时即使没有心跳也保持存活() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 5_000, 5_000, true, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_SERVER_SIDE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_SERVER_SIDE_JOB);
}
@Test
void 从未心跳但仍在排队宽限内不判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 100, 100, false, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_QUEUED);
assertThat(decision.detail()).contains("已等待 100 分钟").contains("宽限 240 分钟");
}
@Test
void 从未心跳且超过排队宽限才判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 300, 300, false, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_FIRST_HEARTBEAT);
}
@Test
void 排队宽限停用时不产出判死读数() {
Thresholds disabled = new Thresholds(10, 30, 0, 24);
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 300, 300, false, ProgressState.UNKNOWN), disabled);
assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN);
}
@Test
void 心跳中断超过阈值判死_边界值本身不判() {
Decision stale = TaskLivenessShadowRules.evaluate(
evidence(11L, 100, 100, false, ProgressState.ADVANCING), DEFAULT);
Decision boundary = TaskLivenessShadowRules.evaluate(
evidence(10L, 100, 100, false, ProgressState.ADVANCING), DEFAULT);
assertThat(stale.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(stale.rule()).isEqualTo(TaskLivenessShadowRules.RULE_BEAT_LOST);
assertThat(boundary.verdict()).isEqualTo(Verdict.KEEP_ADVANCING);
}
@Test
void 长任务只要还在推进就不判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 10 * 60, 10 * 60, false, ProgressState.ADVANCING), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_ADVANCING);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_ADVANCING);
}
@Test
void 总量硬顶压过推进状态() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 24 * 60, 24 * 60, false, ProgressState.ADVANCING), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_HARD_CAP);
assertThat(decision.detail()).contains("24 小时");
}
@Test
void 心跳正常但进度停滞判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 100, 100, false, ProgressState.STALLED), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_PROGRESS);
assertThat(decision.detail()).contains("30 分钟未推进");
}
@Test
void 没有进度位置只算证据不足不判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 100, 100, false, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_POSITION);
}
private static Evidence evidence(Long beatAgoMinutes,
long createdAgoMinutes,
long queuedAgoMinutes,
boolean serverSideJob,
ProgressState progress) {
return new Evidence(1L, "SIMILAR_ASIN", NOW,
ago(beatAgoMinutes),
NOW - queuedAgoMinutes * 60_000L,
NOW - createdAgoMinutes * 60_000L,
serverSideJob, progress);
}
private static long ago(Long minutes) {
return minutes == null ? 0L : NOW - minutes * 60_000L;
}
}