feat(心跳): 影子判官先跑读数——心跳断/进度停滞/排队/服务端处理分开判

现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死(长任务/排队被
判死)同时存在。新增只读影子扫描:每 5 分钟扫 RUNNING 文件任务,按「心跳中断 /
进度停滞 / 排队宽限 / 服务端结果任务 / 24h 硬顶」输出 would-finalize 日志,
不改任何任务状态。进度用「计数+文本+阶段」指纹,写影子独立快照键,不影响正式
判死线的未变化起点。规则纯函数单测 9 例 + 只读边界守卫 3 例。
This commit is contained in:
2026-10-20 12:30:09 +08:00
parent 4c1a2a9ffb
commit eb6f082674
7 changed files with 671 additions and 6 deletions
@@ -329,6 +329,38 @@ class TaskHeartbeatPositionServiceTest {
assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L));
}
@Test
void shadowStallUsesItsOwnSnapshotKey() {
// 影子每 5 分钟一跑;共用正式快照键会把「未变化起点」提前,间接改变封口时刻
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)",
"current", "401", "total", "1300"));
when(hashOperations.entries("task:progress:shadow-stall:123"))
.thenReturn(Map.of("pos", "crawling||400|1300|ASIN B0F44Y3YYQ(400/1300)",
"at", String.valueOf(Long.MAX_VALUE)));
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING,
service().evaluateProgressForShadow(123L, 30L));
verify(hashOperations).putAll(eq("task:progress:shadow-stall:123"), any());
verify(hashOperations, never()).putAll(eq("task:progress:stall:123"), any());
}
@Test
void shadowProgressCountsChangesEvenWhenTextIsStatic() {
// 长页面里文本可能长时间不变,但计数在涨 ⇒ 必须算推进
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "明细处理中",
"current", "401", "total", "1300"));
when(hashOperations.entries("task:progress:shadow-stall:123"))
.thenReturn(Map.of("pos", "crawling||400|1300|明细处理中",
"at", String.valueOf(Long.MAX_VALUE)));
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING,
service().evaluateProgressForShadow(123L, 30L));
}
@Test
void clientAdvancingOnlyForAdvancingState() {
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
@@ -0,0 +1,76 @@
package com.nanri.aiimage.modules.task.service;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Test;
/**
* 影子判官必须是纯只读:它现在的价值是攒读数,不是接管判死。
*
* <p>这条守卫防的是最危险的那种「顺手」——把影子里的 would-finalize 改成真的 finalize,
* 在没有一周真实分布之前就把长任务杀掉。规则类也必须保持无 Spring/无 Mapper 的纯函数,
* 否则规则本身没法逐条离线复核。
*/
class TaskLivenessShadowGuardTest {
private static final List<String> FORBIDDEN_WRITES = List.of(
".updateById(", ".update(", ".delete(", "setStatus(", "setErrorMessage(", "setFinishedAt(");
@Test
void 影子服务源码里不许出现任务写入() throws IOException {
String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java");
for (String forbidden : FORBIDDEN_WRITES) {
assertThat(source)
.as("影子线只能读:出现了 %s,说明它已经越过影子边界", forbidden)
.doesNotContain(forbidden);
}
assertThat(source)
.as("影子扫描必须有日志锚点,否则生产上无法检索读数")
.contains("LOG_ANCHOR = \"[task-liveness-shadow]\"");
}
@Test
void 规则类必须是可离线复核的纯函数() throws IOException {
String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java");
assertThat(source)
.as("规则类不许依赖 Spring/Redis/Mapper,否则不能按纯函数单测")
.doesNotContain("import org.springframework")
.doesNotContain("import com.nanri.aiimage.modules.task.mapper")
.doesNotContain("StringRedisTemplate");
}
@Test
void 开关默认开启且可以只改环境变量停用() throws IOException {
String yaml = readSource("resources/application.yml");
assertThat(yaml)
.contains("task-liveness-shadow:")
.contains("${AIIMAGE_TASK_LIVENESS_SHADOW_ENABLED:true}");
}
private static String readSource(String moduleRelativePath) throws IOException {
Path root = backendJavaRoot();
Path file = root.resolve("src/main/" + moduleRelativePath);
if (!Files.isRegularFile(file)) {
fail("找不到 " + file + " —— 守卫依赖以 backend-java 为工作目录运行,别让它静默通过");
}
return Files.readString(file, StandardCharsets.UTF_8);
}
private static Path backendJavaRoot() {
Path here = Path.of("").toAbsolutePath();
if (Files.isDirectory(here.resolve("src/main"))) {
return here;
}
Path nested = here.resolve("backend-java");
if (Files.isDirectory(nested.resolve("src/main"))) {
return nested;
}
throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")");
}
}
@@ -0,0 +1,125 @@
package com.nanri.aiimage.modules.task.service;
import static org.assertj.core.api.Assertions.assertThat;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Decision;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Evidence;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.ProgressState;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Thresholds;
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Verdict;
import org.junit.jupiter.api.Test;
/**
* 影子规则:把「心跳断」与「进度停滞」拆开,并把排队/服务端处理中排除在判死之外。
*
* <p>这条线只产出 would-finalize 读数,所以规则必须能逐条解释:命中哪条、
* 用了哪份证据、为什么不是 KEEP。这里把每条规则与边界都钉住,影子期读数才可信。
*/
class TaskLivenessShadowRulesTest {
private static final long NOW = 1_800_000_000_000L;
private static final Thresholds DEFAULT = new Thresholds(10, 30, 240, 24);
@Test
void 服务端结果任务在跑时即使没有心跳也保持存活() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 5_000, 5_000, true, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_SERVER_SIDE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_SERVER_SIDE_JOB);
}
@Test
void 从未心跳但仍在排队宽限内不判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 100, 100, false, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_QUEUED);
assertThat(decision.detail()).contains("已等待 100 分钟").contains("宽限 240 分钟");
}
@Test
void 从未心跳且超过排队宽限才判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 300, 300, false, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_FIRST_HEARTBEAT);
}
@Test
void 排队宽限停用时不产出判死读数() {
Thresholds disabled = new Thresholds(10, 30, 0, 24);
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(null, 300, 300, false, ProgressState.UNKNOWN), disabled);
assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN);
}
@Test
void 心跳中断超过阈值判死_边界值本身不判() {
Decision stale = TaskLivenessShadowRules.evaluate(
evidence(11L, 100, 100, false, ProgressState.ADVANCING), DEFAULT);
Decision boundary = TaskLivenessShadowRules.evaluate(
evidence(10L, 100, 100, false, ProgressState.ADVANCING), DEFAULT);
assertThat(stale.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(stale.rule()).isEqualTo(TaskLivenessShadowRules.RULE_BEAT_LOST);
assertThat(boundary.verdict()).isEqualTo(Verdict.KEEP_ADVANCING);
}
@Test
void 长任务只要还在推进就不判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 10 * 60, 10 * 60, false, ProgressState.ADVANCING), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_ADVANCING);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_ADVANCING);
}
@Test
void 总量硬顶压过推进状态() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 24 * 60, 24 * 60, false, ProgressState.ADVANCING), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_HARD_CAP);
assertThat(decision.detail()).contains("24 小时");
}
@Test
void 心跳正常但进度停滞判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 100, 100, false, ProgressState.STALLED), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_PROGRESS);
assertThat(decision.detail()).contains("30 分钟未推进");
}
@Test
void 没有进度位置只算证据不足不判死() {
Decision decision = TaskLivenessShadowRules.evaluate(
evidence(1L, 100, 100, false, ProgressState.UNKNOWN), DEFAULT);
assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN);
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_POSITION);
}
private static Evidence evidence(Long beatAgoMinutes,
long createdAgoMinutes,
long queuedAgoMinutes,
boolean serverSideJob,
ProgressState progress) {
return new Evidence(1L, "SIMILAR_ASIN", NOW,
ago(beatAgoMinutes),
NOW - queuedAgoMinutes * 60_000L,
NOW - createdAgoMinutes * 60_000L,
serverSideJob, progress);
}
private static long ago(Long minutes) {
return minutes == null ? 0L : NOW - minutes * 60_000L;
}
}