feat(心跳): 影子判官先跑读数——心跳断/进度停滞/排队/服务端处理分开判
现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死(长任务/排队被 判死)同时存在。新增只读影子扫描:每 5 分钟扫 RUNNING 文件任务,按「心跳中断 / 进度停滞 / 排队宽限 / 服务端结果任务 / 24h 硬顶」输出 would-finalize 日志, 不改任何任务状态。进度用「计数+文本+阶段」指纹,写影子独立快照键,不影响正式 判死线的未变化起点。规则纯函数单测 9 例 + 只读边界守卫 3 例。
This commit is contained in:
+32
@@ -329,6 +329,38 @@ class TaskHeartbeatPositionServiceTest {
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shadowStallUsesItsOwnSnapshotKey() {
|
||||
// 影子每 5 分钟一跑;共用正式快照键会把「未变化起点」提前,间接改变封口时刻
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)",
|
||||
"current", "401", "total", "1300"));
|
||||
when(hashOperations.entries("task:progress:shadow-stall:123"))
|
||||
.thenReturn(Map.of("pos", "crawling||400|1300|ASIN B0F44Y3YYQ(400/1300)",
|
||||
"at", String.valueOf(Long.MAX_VALUE)));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING,
|
||||
service().evaluateProgressForShadow(123L, 30L));
|
||||
verify(hashOperations).putAll(eq("task:progress:shadow-stall:123"), any());
|
||||
verify(hashOperations, never()).putAll(eq("task:progress:stall:123"), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shadowProgressCountsChangesEvenWhenTextIsStatic() {
|
||||
// 长页面里文本可能长时间不变,但计数在涨 ⇒ 必须算推进
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "明细处理中",
|
||||
"current", "401", "total", "1300"));
|
||||
when(hashOperations.entries("task:progress:shadow-stall:123"))
|
||||
.thenReturn(Map.of("pos", "crawling||400|1300|明细处理中",
|
||||
"at", String.valueOf(Long.MAX_VALUE)));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING,
|
||||
service().evaluateProgressForShadow(123L, 30L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientAdvancingOnlyForAdvancingState() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* 影子判官必须是纯只读:它现在的价值是攒读数,不是接管判死。
|
||||
*
|
||||
* <p>这条守卫防的是最危险的那种「顺手」——把影子里的 would-finalize 改成真的 finalize,
|
||||
* 在没有一周真实分布之前就把长任务杀掉。规则类也必须保持无 Spring/无 Mapper 的纯函数,
|
||||
* 否则规则本身没法逐条离线复核。
|
||||
*/
|
||||
class TaskLivenessShadowGuardTest {
|
||||
|
||||
private static final List<String> FORBIDDEN_WRITES = List.of(
|
||||
".updateById(", ".update(", ".delete(", "setStatus(", "setErrorMessage(", "setFinishedAt(");
|
||||
|
||||
@Test
|
||||
void 影子服务源码里不许出现任务写入() throws IOException {
|
||||
String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowService.java");
|
||||
for (String forbidden : FORBIDDEN_WRITES) {
|
||||
assertThat(source)
|
||||
.as("影子线只能读:出现了 %s,说明它已经越过影子边界", forbidden)
|
||||
.doesNotContain(forbidden);
|
||||
}
|
||||
assertThat(source)
|
||||
.as("影子扫描必须有日志锚点,否则生产上无法检索读数")
|
||||
.contains("LOG_ANCHOR = \"[task-liveness-shadow]\"");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 规则类必须是可离线复核的纯函数() throws IOException {
|
||||
String source = readSource("java/com/nanri/aiimage/modules/task/service/TaskLivenessShadowRules.java");
|
||||
assertThat(source)
|
||||
.as("规则类不许依赖 Spring/Redis/Mapper,否则不能按纯函数单测")
|
||||
.doesNotContain("import org.springframework")
|
||||
.doesNotContain("import com.nanri.aiimage.modules.task.mapper")
|
||||
.doesNotContain("StringRedisTemplate");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 开关默认开启且可以只改环境变量停用() throws IOException {
|
||||
String yaml = readSource("resources/application.yml");
|
||||
assertThat(yaml)
|
||||
.contains("task-liveness-shadow:")
|
||||
.contains("${AIIMAGE_TASK_LIVENESS_SHADOW_ENABLED:true}");
|
||||
}
|
||||
|
||||
private static String readSource(String moduleRelativePath) throws IOException {
|
||||
Path root = backendJavaRoot();
|
||||
Path file = root.resolve("src/main/" + moduleRelativePath);
|
||||
if (!Files.isRegularFile(file)) {
|
||||
fail("找不到 " + file + " —— 守卫依赖以 backend-java 为工作目录运行,别让它静默通过");
|
||||
}
|
||||
return Files.readString(file, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private static Path backendJavaRoot() {
|
||||
Path here = Path.of("").toAbsolutePath();
|
||||
if (Files.isDirectory(here.resolve("src/main"))) {
|
||||
return here;
|
||||
}
|
||||
Path nested = here.resolve("backend-java");
|
||||
if (Files.isDirectory(nested.resolve("src/main"))) {
|
||||
return nested;
|
||||
}
|
||||
throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")");
|
||||
}
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Decision;
|
||||
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Evidence;
|
||||
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.ProgressState;
|
||||
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Thresholds;
|
||||
import com.nanri.aiimage.modules.task.service.TaskLivenessShadowRules.Verdict;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* 影子规则:把「心跳断」与「进度停滞」拆开,并把排队/服务端处理中排除在判死之外。
|
||||
*
|
||||
* <p>这条线只产出 would-finalize 读数,所以规则必须能逐条解释:命中哪条、
|
||||
* 用了哪份证据、为什么不是 KEEP。这里把每条规则与边界都钉住,影子期读数才可信。
|
||||
*/
|
||||
class TaskLivenessShadowRulesTest {
|
||||
|
||||
private static final long NOW = 1_800_000_000_000L;
|
||||
private static final Thresholds DEFAULT = new Thresholds(10, 30, 240, 24);
|
||||
|
||||
@Test
|
||||
void 服务端结果任务在跑时即使没有心跳也保持存活() {
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(null, 5_000, 5_000, true, ProgressState.UNKNOWN), DEFAULT);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_SERVER_SIDE);
|
||||
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_SERVER_SIDE_JOB);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 从未心跳但仍在排队宽限内不判死() {
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(null, 100, 100, false, ProgressState.UNKNOWN), DEFAULT);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_QUEUED);
|
||||
assertThat(decision.detail()).contains("已等待 100 分钟").contains("宽限 240 分钟");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 从未心跳且超过排队宽限才判死() {
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(null, 300, 300, false, ProgressState.UNKNOWN), DEFAULT);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
|
||||
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_FIRST_HEARTBEAT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 排队宽限停用时不产出判死读数() {
|
||||
Thresholds disabled = new Thresholds(10, 30, 0, 24);
|
||||
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(null, 300, 300, false, ProgressState.UNKNOWN), disabled);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 心跳中断超过阈值判死_边界值本身不判() {
|
||||
Decision stale = TaskLivenessShadowRules.evaluate(
|
||||
evidence(11L, 100, 100, false, ProgressState.ADVANCING), DEFAULT);
|
||||
Decision boundary = TaskLivenessShadowRules.evaluate(
|
||||
evidence(10L, 100, 100, false, ProgressState.ADVANCING), DEFAULT);
|
||||
|
||||
assertThat(stale.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
|
||||
assertThat(stale.rule()).isEqualTo(TaskLivenessShadowRules.RULE_BEAT_LOST);
|
||||
assertThat(boundary.verdict()).isEqualTo(Verdict.KEEP_ADVANCING);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 长任务只要还在推进就不判死() {
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(1L, 10 * 60, 10 * 60, false, ProgressState.ADVANCING), DEFAULT);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.KEEP_ADVANCING);
|
||||
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_ADVANCING);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 总量硬顶压过推进状态() {
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(1L, 24 * 60, 24 * 60, false, ProgressState.ADVANCING), DEFAULT);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
|
||||
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_HARD_CAP);
|
||||
assertThat(decision.detail()).contains("24 小时");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 心跳正常但进度停滞判死() {
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(1L, 100, 100, false, ProgressState.STALLED), DEFAULT);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.WOULD_FINALIZE);
|
||||
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_PROGRESS);
|
||||
assertThat(decision.detail()).contains("30 分钟未推进");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有进度位置只算证据不足不判死() {
|
||||
Decision decision = TaskLivenessShadowRules.evaluate(
|
||||
evidence(1L, 100, 100, false, ProgressState.UNKNOWN), DEFAULT);
|
||||
|
||||
assertThat(decision.verdict()).isEqualTo(Verdict.UNKNOWN);
|
||||
assertThat(decision.rule()).isEqualTo(TaskLivenessShadowRules.RULE_NO_POSITION);
|
||||
}
|
||||
|
||||
private static Evidence evidence(Long beatAgoMinutes,
|
||||
long createdAgoMinutes,
|
||||
long queuedAgoMinutes,
|
||||
boolean serverSideJob,
|
||||
ProgressState progress) {
|
||||
return new Evidence(1L, "SIMILAR_ASIN", NOW,
|
||||
ago(beatAgoMinutes),
|
||||
NOW - queuedAgoMinutes * 60_000L,
|
||||
NOW - createdAgoMinutes * 60_000L,
|
||||
serverSideJob, progress);
|
||||
}
|
||||
|
||||
private static long ago(Long minutes) {
|
||||
return minutes == null ? 0L : NOW - minutes * 60_000L;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user