feat(任务判死): 心跳时钟改为从首次心跳起算——排队中的 RUNNING 任务不再被心跳超时线误杀

任务一被认领(页面 activate / 客户端兜底拉取)就记 RUNNING,而客户端要等执行器真开始跑
才发心跳(同店占用、线程池 5 格全忙、浏览器首屏)。这段排队时间里服务端一行心跳都没有、
updated_at 停在认领时刻,被 30/120 分钟心跳线杀掉的是排队而不是死任务
(现网 2026-10-17 查得近 7 天 COLLECT_DATA 9 条 62 分钟判死、SIMILAR_ASIN 2 条一行未产出)。

- 新增 TaskHeartbeatLivenessService:每条心跳无条件打点(Redis,24h TTL),
  读失败一律落 0=没有证据;新增 StaleTaskHeartbeatPolicy:收到过心跳才走心跳时钟,
  从未心跳=仍在排队,超过未启动宽限(默认 240 分钟,置 0 即回退旧行为)才判死。
- 接入 product-risk/跟价/店铺匹配/巡店删除/查 ASIN/提现六条委派判死线、采集 30 分钟线、
  generic 120 分钟线(该文件另一路会话正在改,只提交本笔 HEAD+本改动的切片)。
- 沉默棘轮按逐处记名抬 3 格(total 1002→1003、不重抛 559→561、只 log 172→173),
  两处新增 catch 均为 fail-open 且误差方向是「少杀一个任务」。

验证:backend 全量 4400 例 0 失败;CatchSilenceRatchetTest 9 例绿;
ops 判死签名尺子 6 例绿;ruff check . All checks passed。
This commit is contained in:
2026-10-17 19:46:04 +08:00
parent 531278f861
commit 3c44f686da
18 changed files with 617 additions and 12 deletions
@@ -97,6 +97,7 @@ import java.util.function.Consumer;
import java.util.regex.Pattern;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
@Service
@RequiredArgsConstructor
@@ -190,6 +191,13 @@ public class CollectDataService {
/** 品牌检测失败补测队列:查询失败的品牌在锁外入队,上游恢复后自动重查补结果。 */
private final CollectDataBrandRecheckService collectDataBrandRecheckService;
/**
* 「首次心跳之前不判死」闸门:采集这条 30 分钟心跳线只看 {@code updated_at},而客户端认领任务后
* 要等同店释放、等线程池那 5 格、等浏览器首屏,这段时间一行心跳都没有、{@code updated_at} 又停在
* 认领时刻,被套上 30 分钟超时线杀掉的正是排队任务(现网 2026-10-17 查得近 7 天 9 条)。
*/
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
@@ -580,6 +588,14 @@ public class CollectDataService {
|| !task.getUpdatedAt().isBefore(threshold)) {
return;
}
// 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
taskId, 0L, task.getUpdatedAt(), LocalDateTime.now())) {
log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(),
task.getUpdatedAt());
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
return;
}
@@ -30,6 +30,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskResumeService;
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
@@ -99,6 +100,8 @@ public class DeleteBrandStaleTaskService {
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
/** 客户端中断任务的自动续跑(保留失败记录 + 重排队续跑任务,V129)。 */
private final TaskResumeService taskResumeService;
/** 心跳时钟改为「从首次心跳起算」的闸门:排队中的任务不被各模块心跳超时线误杀。 */
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
@Value("${aiimage.temp-dir.retention-hours:24}")
private long tempDirRetentionHours;
@@ -328,6 +331,26 @@ public class DeleteBrandStaleTaskService {
return sb.toString();
}
/**
* 「首次心跳之前不判死」闸门的接线处:各模块只把自己的最近心跳读数交出去,判据与宽限
* 统一收在 {@link StaleTaskHeartbeatPolicy},这里只补一行可计数的日志(措辞带「暂不判死」,
* 与 ops 那条判死签名尺子的口径对齐——它要求判死相关日志非「真死」即「设计内」,不留第三种)。
*
* @param lastHeartbeatMillis 该模块 Redis 里的最近心跳;0=没有这份证据,闸门会改读共享心跳打点
* @return true=本轮暂不判死(任务还在客户端队列里排队,心跳时钟尚未开始)
*/
private boolean skipUntilFirstHeartbeat(String moduleTag, FileTaskEntity task,
long lastHeartbeatMillis, long timeoutMinutes) {
if (!staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
task.getId(), lastHeartbeatMillis, task.getUpdatedAt(), LocalDateTime.now())) {
return false;
}
log.info("[stale-check] {} 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
moduleTag, task.getId(), timeoutMinutes,
staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(), task.getUpdatedAt());
return true;
}
private ProductRiskStaleCheckStats failStaleProductRiskResolveTasks() {
ProductRiskStaleCheckStats stats = new ProductRiskStaleCheckStats();
long minutes = Math.max(1L, deleteBrandProgressProperties.getProductRiskStaleTimeoutMinutes());
@@ -358,6 +381,10 @@ public class DeleteBrandStaleTaskService {
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
if (skipUntilFirstHeartbeat("product-risk", task, lastPayloadHeartbeatMillis, minutes)) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
@@ -432,6 +459,10 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++;
continue;
}
if (skipUntilFirstHeartbeat("price-track", task, lastHeartbeatMillis, minutes)) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
@@ -509,6 +540,10 @@ public class DeleteBrandStaleTaskService {
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
if (skipUntilFirstHeartbeat("shop-match", task, lastHeartbeatMillis, minutes)) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
@@ -582,6 +617,10 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++;
continue;
}
if (skipUntilFirstHeartbeat("patrol-delete", task, lastHeartbeatMillis, minutes)) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
@@ -654,6 +693,10 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++;
continue;
}
if (skipUntilFirstHeartbeat("query-asin", task, lastHeartbeatMillis, minutes)) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = queryAsinTaskCacheService.hasAnyShopMergedPayload(task.getId());
@@ -724,6 +767,10 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++;
continue;
}
if (skipUntilFirstHeartbeat("withdraw", task, lastHeartbeatMillis, minutes)) {
stats.skippedTaskCount++;
continue;
}
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_WITHDRAW, task.getId());
if (taskLockHandle == null) {
stats.skippedTaskCount++;
@@ -0,0 +1,76 @@
package com.nanri.aiimage.modules.task.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
/**
* 陈旧判死的「首次心跳」闸——把各条心跳超时线改成<b>从首次心跳起算</b>。
*
* <p>要解决的问题:任务一被认领(页面 activate / 客户端兜底拉取,同一条 PENDING→RUNNING 原子更新)
* 服务端就把它记成 RUNNING,而客户端的心跳要等到执行器真开始跑才发(各执行器里的
* {@code task_heartbeats.add})。同店被占用、线程池 5 格全忙、浏览器还在开首屏,这几段排队时间
* 服务端一行心跳都收不到,{@code updated_at} 也就一直停在认领那一刻。此时按「心跳超时 30 分钟」
* 判死,杀掉的是排队的正常任务(现网 2026-10-17 近 7 天 COLLECT_DATA 那 9 条 62 分钟判死即此类)。
*
* <p>闸门的口径:
* <ul>
* <li>收到过心跳 ⇒ 心跳时钟已经走起来了,本闸放行,交回各模块原有的心跳判据;</li>
* <li>从未收到心跳 ⇒ 任务还没开始跑,未启动宽限(默认 240 分钟)之内一律<b>暂不判死</b>;
* 宽限之外仍然判死,否则「认领后客户端就崩、也没来得及上报中断」的任务会变成永久 RUNNING 孤儿;</li>
* <li>宽限配成 {@code <= 0} ⇒ 闸门整体关闭,退回改造前行为(回滚只需要改环境变量)。</li>
* </ul>
*
* <p>宽限的锚是 {@code updated_at}。对「从未心跳」的任务来说这个等式成立:没有心跳就没有任何
* 刷新 {@code updated_at} 的路径,它停在那里正是认领时刻;万一还有别的写入方碰过这一行,等式变成
* 「最近一次状态写入」,误差方向依旧是多活一会儿。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class StaleTaskHeartbeatPolicy {
/** 未启动宽限(分钟)。&lt;=0=闸门关闭,退回「心跳超时线从认领时刻起算」的旧行为。 */
@Value("${aiimage.task-stale.first-heartbeat-grace-minutes:240}")
private long firstHeartbeatGraceMinutes;
private final TaskHeartbeatLivenessService livenessService;
/**
* @param taskId 任务 id,用于回退读共享心跳打点
* @param moduleLastHeartbeatMillis 调用方模块自己的「最近心跳」读数;0 表示该模块没有这份证据
* @param taskUpdatedAt 任务行的 updated_at(未启动宽限的锚)
* @param now 本轮扫描的统一时刻
* @return true=本轮暂不判死(任务还在排队,心跳时钟尚未开始)
*/
public boolean isStillQueuedBeforeFirstHeartbeat(long taskId, long moduleLastHeartbeatMillis,
LocalDateTime taskUpdatedAt, LocalDateTime now) {
if (firstHeartbeatGraceMinutes <= 0L) {
return false;
}
long beat = moduleLastHeartbeatMillis > 0L
? moduleLastHeartbeatMillis
: livenessService.lastHeartbeatMillis(taskId);
if (beat > 0L) {
return false;
}
if (taskUpdatedAt == null) {
// 没有锚就无从计算宽限,按放宽处理:判死证据不足时不该杀任务
return true;
}
return Duration.between(taskUpdatedAt, now).toMinutes() < firstHeartbeatGraceMinutes;
}
public long firstHeartbeatGraceMinutes() {
return firstHeartbeatGraceMinutes;
}
/** 单测直接塞宽限值,不改生产注入路径。 */
void setFirstHeartbeatGraceMinutes(long minutes) {
this.firstHeartbeatGraceMinutes = minutes;
}
}
@@ -12,6 +12,7 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
/**
@@ -41,6 +42,8 @@ public class StaleTaskRepairService {
private final FileTaskMapper fileTaskMapper;
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi;
private final DistributedJobLockService distributedJobLockService;
/** 心跳时钟改为「从首次心跳起算」:排队中的 RUNNING 任务不在这条线上判死。 */
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
/**
* 每 10 分钟执行一次;双实例下用分布式锁保证只有一台执行修复。
@@ -130,11 +133,13 @@ public class StaleTaskRepairService {
// 先查 id 再按主键更新:此前是一条无 LIMIT、无 module_type 的批量 UPDATE,
// 而 biz_file_task 没有以 status 打头的索引 → 每 10 分钟一次全表扫,
// 且按无索引条件命中行加 X 锁(长事务/锁扩散)。与另两个修复方法口径保持一致。
// 多取 updated_at / module_type:首次心跳闸门要算排队时长,日志要说是哪个模块。
List<FileTaskEntity> stale = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId)
.select(FileTaskEntity::getId, FileTaskEntity::getUpdatedAt, FileTaskEntity::getModuleType)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.lt(FileTaskEntity::getUpdatedAt, cutoff)
.last("limit 500"));
stale = rejectQueuedBeforeFirstHeartbeat(stale);
if (stale.isEmpty()) {
return;
}
@@ -152,6 +157,29 @@ public class StaleTaskRepairService {
}
}
/**
* 把「服务端还一行心跳都没收到、且仍在未启动宽限内」的 RUNNING 任务从候选里摘掉。
*
* <p>这条线是 2 小时无写判死,也是各模块心跳线的天花板:不摘它们,排队超过 2 小时的任务照样会被
* 这里以「任务心跳超时」杀掉,模块侧放宽到多少分钟都白做。判据与措辞统一收在
* {@link StaleTaskHeartbeatPolicy}(那个模块没有自己的心跳读数时,闸门退回共享心跳打点)。
*/
private List<FileTaskEntity> rejectQueuedBeforeFirstHeartbeat(List<FileTaskEntity> candidates) {
LocalDateTime now = LocalDateTime.now();
List<FileTaskEntity> kept = new ArrayList<>(candidates.size());
for (FileTaskEntity task : candidates) {
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
task.getId(), 0L, task.getUpdatedAt(), now)) {
log.info("[stale-task-repair] 未收到首次心跳、排队宽限内暂不判死 taskId={} moduleType={} updatedAt={} graceMinutes={}",
task.getId(), task.getModuleType(), task.getUpdatedAt(),
staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes());
continue;
}
kept.add(task);
}
return kept;
}
/**
* brand_crawl_tasks 的 pending/running 陈旧 → failed。
* 品牌表与状态字面量属于 brand 模块,实现放在 BrandTaskStaleRepairSpiImpl(G5:消除 task → 业务依赖)。
@@ -0,0 +1,84 @@
package com.nanri.aiimage.modules.task.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
/**
* 「服务端最后一次收到该任务心跳」的时刻(Redis String,24h TTL)。
*
* <p>与 {@link TaskHeartbeatPositionService} 的分工必须划清:那边存的是「卡在哪」,位置缺失只退化
* 文案,按既有铁律<b>不得作为任何判死依据</b>;这里只回答一个二元问题——这个任务到底有没有心跳上来过,
* 因此每条心跳都无条件打点(不挑载荷有没有 progressText),老客户端同样能被算成「已开始心跳」。
*
* <p>存在的理由:客户端只在真正开始执行任务时才发心跳(各执行器 {@code task_heartbeats.add}),
* 而任务一被认领就已是 RUNNING。排队期间(同店占用、线程池 5 格全忙、浏览器还在开)服务端一行心跳都
* 收不到,此时拿「心跳超时 30 分钟」去套,杀的是排队而不是死任务。
*
* <p>读不到=没有证据,调用方按「从未心跳」处理。这个方向的误差只会让任务多活一会儿(最终仍由
* {@link StaleTaskHeartbeatPolicy} 的未启动宽限线和 {@link StaleTaskRepairService} 兜底),
* 不会多杀一个任务,所以 Redis 抖动时不需要额外兜底。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskHeartbeatLivenessService {
private static final String KEY_PREFIX = "task:heartbeat:beat:";
private static final Duration BEAT_TTL = Duration.ofHours(24);
private final StringRedisTemplate stringRedisTemplate;
/** 收到一次心跳就打点。打点失败只记日志——心跳接口绝不能因为证据写不下去而报错。 */
public void stamp(long taskId) {
if (taskId <= 0L) {
return;
}
try {
stringRedisTemplate.opsForValue()
.set(KEY_PREFIX + taskId, String.valueOf(Instant.now().toEpochMilli()), BEAT_TTL);
} catch (Exception ex) {
log.warn("[task-heartbeat-beat] 心跳打点失败(不影响心跳)taskId={} msg={}", taskId, ex.getMessage());
}
}
/** 最近一次心跳到达时刻(epoch millis);返回 0 表示从未收到过、或那条证据已过 TTL。 */
public long lastHeartbeatMillis(long taskId) {
if (taskId <= 0L) {
return 0L;
}
try {
return parseUnsignedLong(stringRedisTemplate.opsForValue().get(KEY_PREFIX + taskId));
} catch (Exception ex) {
// 读证据失败=「没有证据」,判死线据此放宽;把异常抛给巡检线换不来一个更准的判定
log.warn("[task-heartbeat-beat] 读心跳打点失败 taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
}
/**
* 脏值一律读成 0,不靠异常控流:这个数唯一的作用是回答「有没有心跳上来过」,
* 非纯数字/溢出/空值都只能落到「没有证据」那一侧。
*/
private static long parseUnsignedLong(String raw) {
if (raw == null || raw.isEmpty()) {
return 0L;
}
long value = 0L;
for (int i = 0; i < raw.length(); i++) {
char digit = raw.charAt(i);
if (digit < '0' || digit > '9') {
return 0L;
}
value = value * 10L + (digit - '0');
if (value < 0L) {
return 0L; // 溢出=脏值
}
}
return value;
}
}
@@ -36,16 +36,19 @@ public class TaskHeartbeatService {
private final FileTaskMapper fileTaskMapper;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi;
private final TaskHeartbeatLivenessService livenessService;
/** moduleType → 模块心跳实现(启动时建索引并校验重复注册) */
private final Map<String, TaskModuleHeartbeatSpi> moduleHandlers;
public TaskHeartbeatService(FileTaskMapper fileTaskMapper,
TaskProgressSnapshotService taskProgressSnapshotService,
BrandTaskHeartbeatSpi brandTaskHeartbeatSpi,
TaskHeartbeatLivenessService livenessService,
List<TaskModuleHeartbeatSpi> moduleHeartbeatHandlers) {
this.fileTaskMapper = fileTaskMapper;
this.taskProgressSnapshotService = taskProgressSnapshotService;
this.brandTaskHeartbeatSpi = brandTaskHeartbeatSpi;
this.livenessService = livenessService;
Map<String, TaskModuleHeartbeatSpi> handlers = new LinkedHashMap<>();
for (TaskModuleHeartbeatSpi handler : moduleHeartbeatHandlers == null ? List.<TaskModuleHeartbeatSpi>of() : moduleHeartbeatHandlers) {
String moduleType = handler.moduleType();
@@ -79,6 +82,13 @@ public class TaskHeartbeatService {
TaskHeartbeatVo fileResult = brandOnly ? null : touchFileTaskIfRunning(selectFileTask(taskId), request);
TaskHeartbeatVo brandResult = skipBrandLookup ? null : brandTaskHeartbeatSpi.heartbeat(taskId, request);
// 心跳到达打点:陈旧判死线用它把「心跳时钟」改成从首次心跳起算(见 StaleTaskHeartbeatPolicy)。
// 只在任务确实存在时打,免得坏 id 把 Redis 写成垃圾键;状态不是 RUNNING 也照打——
// 「客户端已经在管这个任务」本身就是证据,与本轮要不要刷 updated_at 是两件事。
if (fileResult != null || brandResult != null) {
livenessService.stamp(taskId);
}
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
}
@@ -235,6 +235,11 @@ aiimage:
withdraw-stale-timeout-minutes: ${AIIMAGE_WITHDRAW_STALE_TIMEOUT_MINUTES:30}
no-result-upload-timeout-minutes: ${AIIMAGE_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180}
no-result-upload-check-enabled: ${AIIMAGE_NO_RESULT_UPLOAD_CHECK_ENABLED:true}
# 陈旧判死的「首次心跳」闸(2026-10-17):任务被认领即记 RUNNING,而客户端要等执行器真开始跑才发心跳。
# 这段排队时间里各模块的心跳超时线一律不判死,超过本宽限才由「认领后长时间未启动」收口;
# 置 0 即整体关闭闸门、退回旧行为(只需改环境变量,不用重新构建)。
task-stale:
first-heartbeat-grace-minutes: ${AIIMAGE_TASK_STALE_FIRST_HEARTBEAT_GRACE_MINUTES:240}
module-cleanup:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
@@ -45,8 +45,9 @@ class CatchSilenceRatchetTest {
private static final Path TREE = moduleRoot().resolve("src/main/java");
// ==== 验收基线(用户 2026-10-08 12:0x 实测,HEAD 274ad017,单位:处)====
// 10-17 心跳证据两处 fail-open 抬 559 -> 561,逐处出处见 STRICT_NOT_RETHROW 上方注释。
static final int GIVEN_TOTAL = 1000;
static final int GIVEN_NOT_RETHROW = 559;
static final int GIVEN_NOT_RETHROW = 561;
static final int GIVEN_LOG_ONLY = 249;
static final int GIVEN_EMPTY = 56;
@@ -59,8 +60,13 @@ class CatchSilenceRatchetTest {
* BuildInfoLogger 那一处不是吞掉:它打 {@code [build-info-missing]} 的 WARN 并返回可区分的
* "章存在但没盖住"状态,巡检按锚点数得到。抬这一格是为了把出处留在这份 diff 里,
* 而不是让后来人以为基线是自然涨上来的。
* 10-17 再 +2,逐处记名:{@code TaskHeartbeatLivenessService.stamp()}(心跳证据写失败只 WARN,
* 心跳接口不许因证据写不下去而 500)与同类 {@code lastHeartbeatMillis()}(读证据失败一律落到 0=
* 「没有证据」,判死线据此放宽)。两处都是 fail-open 且方向是「少杀一个任务」,与上面
* BuildInfoLogger 同形状;本仓实测基线由 1001 起(原记 1002 留有 1 格无人认领的额度),
* 加这 2 处后实测 1003。
*/
static final int STRICT_TOTAL = 1002;
static final int STRICT_TOTAL = 1003;
/**
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
@@ -75,8 +81,23 @@ class CatchSilenceRatchetTest {
// 的两个零调用者重载,其中 `throw new BusinessException("删除 Excel 匹配数据失败", e)` 是 RETHROW 形状,
// 随死代码一起离开树;实测重抛 386、不重抛与只 log 两数未涨(那两格仍绿)=> 没有任何一处「异常会往外走」被改回吞掉。
static final int RETHROW_FLOOR = 386;
static final int STRICT_NOT_RETHROW = 559;
static final int STRICT_LOG_ONLY = 172; // 严格口径:块内每条顶层语句都是 log 调用才算
/**
* 10-17 抬 559 -> 561,逐处记名(两处都是新增的 fail-open 证据读写,不是把已有出口改回吞掉):
* 1){@code TaskHeartbeatLivenessService.stamp()}:心跳证据写失败只 WARN——心跳接口不许因为
* 「记不下证据」而 500,那会让客户端整轮心跳线程崩掉,反而制造更多孤儿任务;
* 2){@code TaskHeartbeatLivenessService.lastHeartbeatMillis()}:读证据失败一律落到 0=「没有证据」,
* 判死线据此**放宽**(顶多让任务多活一会儿,最终仍由未启动宽限与 stale-repair 兜底)。
* 两处的误差方向都是「少杀一个任务」,与这一格想拦的「异常到此为止、外面看不见」不同性质;
* 抬格是为了把出处留在这份 diff 里,而不是让后来人以为基线自然涨上来的。
*/
static final int STRICT_NOT_RETHROW = 561;
/**
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
* 心跳接口不许因为「记不下证据」而 500。同类的 {@code lastHeartbeatMillis()} 读失败还要
* 落 0 返回,形状是 log+return 默认值,因此不进这一格。
*/
static final int STRICT_LOG_ONLY = 173; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_EMPTY = 56;
/**
@@ -147,7 +147,8 @@ class CollectDataActivateGuardTest {
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
mock(CollectDataBrandRecheckService.class));
mock(CollectDataBrandRecheckService.class),
mock(com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy.class));
}
@SuppressWarnings("unchecked")
@@ -120,7 +120,8 @@ class CollectDataDeleteConsistencyTest {
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
mock(CollectDataBrandRecheckService.class));
mock(CollectDataBrandRecheckService.class),
mock(com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy.class));
}
private FileTaskEntity task(long id, long userId) {
@@ -58,6 +58,13 @@ class CollectDataServiceTest {
@Mock(answer = org.mockito.Answers.RETURNS_MOCKS)
private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
/**
* 「首次心跳之前不判死」闸门:默认 false=不豁免,本类原有 stale 判死用例的行为保持不变;
* 豁免分支另有用例(staleTaskWithoutFirstHeartbeatWithinGraceIsNotJudgedDead)。
*/
@Mock
private com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
@Spy
private ObjectMapper objectMapper = new ObjectMapper();
@@ -190,6 +197,28 @@ class CollectDataServiceTest {
verify(lockHandle).close();
}
// 2026-10-17 现网 9 条误杀的口径:采集这条 30 分钟心跳线只看 updated_at,而客户端认领任务后要先等
// 同店释放、等线程池那 5 格、等浏览器首屏——这段时间一行心跳都没有,updated_at 停在认领时刻。
// 闸门放行时,本条线既不能标失败,也不能封口(封口会让后续回传被拒、数据丢失)。
@Test
void staleTaskWithoutFirstHeartbeatWithinGraceIsNotJudgedDead() throws Exception {
FileTaskEntity queued = staleTask(95L);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(queued));
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, queued.getId(), 0L))
.thenReturn(lockHandle);
when(fileTaskMapper.selectById(queued.getId())).thenReturn(queued);
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
org.mockito.ArgumentMatchers.eq(95L), org.mockito.ArgumentMatchers.eq(0L),
any(), any())).thenReturn(true);
service.finalizeStaleTasks();
assertThat(queued.getStatus()).isEqualTo("RUNNING");
verify(fileTaskMapper, never()).update(any(), any());
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
}
private static FileTaskEntity staleTask(long taskId) {
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
mock(CollectDataBrandRecheckService.class));
mock(CollectDataBrandRecheckService.class),
mock(com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy.class));
}
private FileResultEntity result(long id, long taskId, long userId) {
@@ -62,6 +62,9 @@ class DeleteBrandStaleTaskServiceTest {
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
@Mock private TaskResumeService taskResumeService;
@Mock private PriceTrackLoopRunStaleService priceTrackLoopRunStaleService;
@Mock private com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService priceTrackTaskService;
@Mock private com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService priceTrackTaskCacheService;
@Mock private com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
@BeforeAll
static void initializeTableInfo() {
@@ -448,7 +451,67 @@ class DeleteBrandStaleTaskServiceTest {
null, null, priceTrackLoopRunStaleService, null, null, null, null, null, null, null,
null, null, null, null,
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService, null,
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService);
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService, staleTaskHeartbeatPolicy);
}
/** 只喂跟价那条线需要的两个 mock,其余仍走 service() 的窄桩口径。 */
private DeleteBrandStaleTaskService priceTrackService() {
return new DeleteBrandStaleTaskService(
fileTaskMapper, deleteBrandTaskCacheService, deleteBrandTaskStorageService, deleteBrandRunService,
productRiskTaskService, productRiskTaskCacheService,
priceTrackTaskService, priceTrackTaskCacheService, priceTrackLoopRunStaleService,
null, null, null, null, null, null, null,
null, null, null, null,
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService, null,
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService, staleTaskHeartbeatPolicy);
}
@SuppressWarnings({"rawtypes", "unchecked"})
private FileTaskEntity priceTrackCandidate(Long id) {
FileTaskEntity task = runningTask(id);
task.setModuleType("PRICE_TRACK");
// 认领后再没有别的写入:updated_at 就是排队起点
task.setUpdatedAt(LocalDateTime.now().minusMinutes(40));
return task;
}
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void priceTrackCandidateWithoutFirstHeartbeatIsNotJudgedDead() {
FileTaskEntity queued = priceTrackCandidate(9001L);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(queued));
when(deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes()).thenReturn(30L);
when(priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(any())).thenReturn(Map.of());
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
org.mockito.ArgumentMatchers.eq(9001L), org.mockito.ArgumentMatchers.eq(0L),
any(), any())).thenReturn(true);
ReflectionTestUtils.invokeMethod(priceTrackService(), "failStalePriceTrackTasks");
// 排队中的任务:既不该标失败,也不该被 finalize 封口(封口会让后续回传被拒、数据丢失)
verify(fileTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
verify(priceTrackTaskService, never()).tryFinalizeTask(anyLong(), anyBoolean());
}
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void priceTrackCandidatePastTheGraceWindowIsStillJudgedDead() {
FileTaskEntity orphan = priceTrackCandidate(9002L);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(orphan));
when(deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes()).thenReturn(30L);
when(priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(any())).thenReturn(Map.of());
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
org.mockito.ArgumentMatchers.eq(9002L), org.mockito.ArgumentMatchers.eq(0L),
any(), any())).thenReturn(false);
when(taskDistributedLockService.acquire(any(), any(), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(priceTrackTaskService.tryFinalizeTask(org.mockito.ArgumentMatchers.eq(9002L), anyBoolean()))
.thenReturn(false);
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
ReflectionTestUtils.invokeMethod(priceTrackService(), "failStalePriceTrackTasks");
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
}
private void lockAvailable() {
@@ -82,7 +82,7 @@ class TaskModuleCoverageTest {
mock(SimilarAsinTaskService.class),
null, null, null, null,
mock(ShopDataCrawlTaskService.class),
null, null, null);
null, null, null, null);
}
@Test
@@ -0,0 +1,79 @@
package com.nanri.aiimage.modules.task.service;
import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 「心跳时钟从首次心跳起算」的判据契约。
*
* <p>要守住的现场(2026-10-17 生产实录):任务一被认领就记 RUNNING,而客户端要等执行器真开始跑才发
* 心跳(同店占用 / 线程池 5 格全忙 / 浏览器首屏)。这段排队时间里服务端一行心跳都没有,
* 拿各模块 30 分钟心跳超时线去套,杀的是排队任务。
*/
class StaleTaskHeartbeatPolicyTest {
private final TaskHeartbeatLivenessService livenessService = mock(TaskHeartbeatLivenessService.class);
private final StaleTaskHeartbeatPolicy policy = new StaleTaskHeartbeatPolicy(livenessService);
private StaleTaskHeartbeatPolicy withGrace(long minutes) {
policy.setFirstHeartbeatGraceMinutes(minutes);
return policy;
}
@Test
void 从未心跳且在宽限内_还在排队本轮不判死() {
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
assertTrue(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
31279L, 0L, LocalDateTime.now().minusMinutes(40), LocalDateTime.now()));
}
@Test
void 从未心跳且超出宽限_照旧判死不许留成永久孤儿() {
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
assertFalse(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
31279L, 0L, LocalDateTime.now().minusMinutes(241), LocalDateTime.now()));
}
@Test
void 模块自带心跳读数时不回落共享打点() {
assertFalse(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
31279L, System.currentTimeMillis(), LocalDateTime.now().minusMinutes(90), LocalDateTime.now()));
verify(livenessService, never()).lastHeartbeatMillis(anyLong());
}
@Test
void 模块没有心跳读数时回落共享打点() {
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(System.currentTimeMillis());
assertFalse(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
31279L, 0L, LocalDateTime.now().minusMinutes(40), LocalDateTime.now()));
}
@Test
void 宽限置零_闸门关闭退回旧行为() {
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
assertFalse(withGrace(0).isStillQueuedBeforeFirstHeartbeat(
31279L, 0L, LocalDateTime.now().minusMinutes(31), LocalDateTime.now()));
}
@Test
void 没有updated_at就无从算宽限() {
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
assertTrue(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
31279L, 0L, null, LocalDateTime.now()));
}
}
@@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -40,8 +41,9 @@ class StaleTaskRepairServiceTest {
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi = mock(BrandTaskStaleRepairSpi.class);
private final DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class);
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy = mock(StaleTaskHeartbeatPolicy.class);
private final StaleTaskRepairService service = new StaleTaskRepairService(
fileTaskMapper, brandTaskStaleRepairSpi, distributedJobLockService);
fileTaskMapper, brandTaskStaleRepairSpi, distributedJobLockService, staleTaskHeartbeatPolicy);
@BeforeEach
void initTableInfo() {
@@ -99,6 +101,22 @@ class StaleTaskRepairServiceTest {
verifyNoMoreInteractions(brandTaskStaleRepairSpi);
}
// 2026-10-17 实录:任务一被认领就记 RUNNING,而客户端要等执行器真开始跑才发心跳(同店占用 /
// 线程池 5 格全忙 / 浏览器首屏)。本线是各模块心跳线的天花板——不在这里放行,模块侧把宽限
// 放到多少分钟都会被这里的 2 小时重新杀成「任务心跳超时」。
@Test
void 未收到首次心跳的_RUNNING_任务在排队宽限内暂不判死() {
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
eq(31279L), eq(0L), any(), any())).thenReturn(true);
FileTaskEntity queued = task(31279L);
queued.setStatus("RUNNING");
when(fileTaskMapper.selectList(any())).thenReturn(List.of(queued));
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning");
verify(fileTaskMapper, never()).update(isNull(), any());
}
private Map<String, Object> selectParams() {
ArgumentCaptor<Wrapper<FileTaskEntity>> captor = wrapperCaptor();
verify(fileTaskMapper).selectList(captor.capture());
@@ -0,0 +1,93 @@
package com.nanri.aiimage.modules.task.service;
import org.junit.jupiter.api.Test;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import java.time.Duration;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 心跳打点契约:判死线把它当「这任务到底有没有心跳上来过」的唯一通用证据,
* 因此读取侧的每一个失败分支都必须落到 0(=没有证据=放宽),绝不能把异常抛给判死线。
*/
@SuppressWarnings("unchecked")
class TaskHeartbeatLivenessServiceTest {
private final StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
private final ValueOperations<String, String> valueOperations = mock(ValueOperations.class);
private final TaskHeartbeatLivenessService service = new TaskHeartbeatLivenessService(redisTemplate);
private TaskHeartbeatLivenessServiceTest withRedis() {
lenient().when(redisTemplate.opsForValue()).thenReturn(valueOperations);
return this;
}
@Test
void 收到心跳即打点并带过期() {
withRedis();
service.stamp(31279L);
verify(valueOperations).set(eq("task:heartbeat:beat:31279"), anyString(), any(Duration.class));
}
@Test
void 打点失败不许影响心跳本身() {
withRedis();
doThrow(new IllegalStateException("redis down"))
.when(valueOperations).set(anyString(), anyString(), any(Duration.class));
service.stamp(31279L); // 不抛即通过:心跳接口不能因为证据写不下去而 500
}
@Test
void 读回打点时刻() {
withRedis();
when(valueOperations.get("task:heartbeat:beat:31279")).thenReturn("1792228880835");
assertEquals(1792228880835L, service.lastHeartbeatMillis(31279L));
}
@Test
void 没有打点时读成从未心跳() {
withRedis();
when(valueOperations.get(anyString())).thenReturn(null);
assertEquals(0L, service.lastHeartbeatMillis(31279L));
}
@Test
void 脏值按没有证据处理而不是把判死线带崩() {
withRedis();
when(valueOperations.get(anyString())).thenReturn("not-a-timestamp");
assertEquals(0L, service.lastHeartbeatMillis(31279L));
}
@Test
void Redis异常时读成从未心跳() {
withRedis();
when(valueOperations.get(anyString())).thenThrow(new IllegalStateException("redis down"));
assertEquals(0L, service.lastHeartbeatMillis(31279L));
}
@Test
void 非法taskId不打点也不读() {
service.stamp(0L);
assertEquals(0L, service.lastHeartbeatMillis(-1L));
verify(redisTemplate, never()).opsForValue();
}
}
@@ -29,8 +29,10 @@ import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
@@ -55,10 +57,11 @@ class TaskHeartbeatServiceTest {
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi = mock(BrandTaskHeartbeatSpi.class);
private final TaskHeartbeatLivenessService livenessService = mock(TaskHeartbeatLivenessService.class);
private TaskHeartbeatService serviceWith(TaskModuleHeartbeatSpi... handlers) {
return new TaskHeartbeatService(fileTaskMapper, taskProgressSnapshotService, brandTaskHeartbeatSpi,
List.of(handlers));
livenessService, List.of(handlers));
}
private static FileTaskEntity runningTask(long taskId, String moduleType) {
@@ -184,4 +187,34 @@ class TaskHeartbeatServiceTest {
assertTrue(result.isAlive(), "未注册 SPI 的模块不应影响通用心跳(保持向后兼容)");
assertEquals("UNREGISTERED_MODULE", result.getModuleType());
}
@Test
@SuppressWarnings("unchecked")
void heartbeatStampsLivenessEvenWhenTaskIsNotRunningYet() {
long taskId = 20151L;
FileTaskEntity justClaimed = runningTask(taskId, "PRICE_TRACK");
justClaimed.setStatus("PENDING");
TaskHeartbeatService service = serviceWith();
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(justClaimed);
TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest());
assertFalse(result.isAlive(), "非 RUNNING 的心跳不得把任务改成存活(既有契约不变)");
verify(livenessService).stamp(taskId);
}
@Test
@SuppressWarnings("unchecked")
void heartbeatForUnknownTaskDoesNotStampLiveness() {
long taskId = 20152L;
TaskHeartbeatService service = serviceWith();
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
service.heartbeat(taskId, new TaskHeartbeatRequest());
// 坏 id 的轮询不能把 Redis 写成垃圾键
verify(livenessService, never()).stamp(anyLong());
}
}