feat(任务判死): 心跳时钟改为从首次心跳起算——排队中的 RUNNING 任务不再被心跳超时线误杀
任务一被认领(页面 activate / 客户端兜底拉取)就记 RUNNING,而客户端要等执行器真开始跑 才发心跳(同店占用、线程池 5 格全忙、浏览器首屏)。这段排队时间里服务端一行心跳都没有、 updated_at 停在认领时刻,被 30/120 分钟心跳线杀掉的是排队而不是死任务 (现网 2026-10-17 查得近 7 天 COLLECT_DATA 9 条 62 分钟判死、SIMILAR_ASIN 2 条一行未产出)。 - 新增 TaskHeartbeatLivenessService:每条心跳无条件打点(Redis,24h TTL), 读失败一律落 0=没有证据;新增 StaleTaskHeartbeatPolicy:收到过心跳才走心跳时钟, 从未心跳=仍在排队,超过未启动宽限(默认 240 分钟,置 0 即回退旧行为)才判死。 - 接入 product-risk/跟价/店铺匹配/巡店删除/查 ASIN/提现六条委派判死线、采集 30 分钟线、 generic 120 分钟线(该文件另一路会话正在改,只提交本笔 HEAD+本改动的切片)。 - 沉默棘轮按逐处记名抬 3 格(total 1002→1003、不重抛 559→561、只 log 172→173), 两处新增 catch 均为 fail-open 且误差方向是「少杀一个任务」。 验证:backend 全量 4400 例 0 失败;CatchSilenceRatchetTest 9 例绿; ops 判死签名尺子 6 例绿;ruff check . All checks passed。
This commit is contained in:
+16
@@ -97,6 +97,7 @@ import java.util.function.Consumer;
|
||||
import java.util.regex.Pattern;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -190,6 +191,13 @@ public class CollectDataService {
|
||||
/** 品牌检测失败补测队列:查询失败的品牌在锁外入队,上游恢复后自动重查补结果。 */
|
||||
private final CollectDataBrandRecheckService collectDataBrandRecheckService;
|
||||
|
||||
/**
|
||||
* 「首次心跳之前不判死」闸门:采集这条 30 分钟心跳线只看 {@code updated_at},而客户端认领任务后
|
||||
* 要等同店释放、等线程池那 5 格、等浏览器首屏,这段时间一行心跳都没有、{@code updated_at} 又停在
|
||||
* 认领时刻,被套上 30 分钟超时线杀掉的正是排队任务(现网 2026-10-17 查得近 7 天 9 条)。
|
||||
*/
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
|
||||
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
||||
private long staleTimeoutMinutes;
|
||||
|
||||
@@ -580,6 +588,14 @@ public class CollectDataService {
|
||||
|| !task.getUpdatedAt().isBefore(threshold)) {
|
||||
return;
|
||||
}
|
||||
// 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点
|
||||
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
taskId, 0L, task.getUpdatedAt(), LocalDateTime.now())) {
|
||||
log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
|
||||
taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(),
|
||||
task.getUpdatedAt());
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
|
||||
+47
@@ -30,6 +30,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -99,6 +100,8 @@ public class DeleteBrandStaleTaskService {
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
/** 客户端中断任务的自动续跑(保留失败记录 + 重排队续跑任务,V129)。 */
|
||||
private final TaskResumeService taskResumeService;
|
||||
/** 心跳时钟改为「从首次心跳起算」的闸门:排队中的任务不被各模块心跳超时线误杀。 */
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
|
||||
@Value("${aiimage.temp-dir.retention-hours:24}")
|
||||
private long tempDirRetentionHours;
|
||||
@@ -328,6 +331,26 @@ public class DeleteBrandStaleTaskService {
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 「首次心跳之前不判死」闸门的接线处:各模块只把自己的最近心跳读数交出去,判据与宽限
|
||||
* 统一收在 {@link StaleTaskHeartbeatPolicy},这里只补一行可计数的日志(措辞带「暂不判死」,
|
||||
* 与 ops 那条判死签名尺子的口径对齐——它要求判死相关日志非「真死」即「设计内」,不留第三种)。
|
||||
*
|
||||
* @param lastHeartbeatMillis 该模块 Redis 里的最近心跳;0=没有这份证据,闸门会改读共享心跳打点
|
||||
* @return true=本轮暂不判死(任务还在客户端队列里排队,心跳时钟尚未开始)
|
||||
*/
|
||||
private boolean skipUntilFirstHeartbeat(String moduleTag, FileTaskEntity task,
|
||||
long lastHeartbeatMillis, long timeoutMinutes) {
|
||||
if (!staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
task.getId(), lastHeartbeatMillis, task.getUpdatedAt(), LocalDateTime.now())) {
|
||||
return false;
|
||||
}
|
||||
log.info("[stale-check] {} 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
|
||||
moduleTag, task.getId(), timeoutMinutes,
|
||||
staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(), task.getUpdatedAt());
|
||||
return true;
|
||||
}
|
||||
|
||||
private ProductRiskStaleCheckStats failStaleProductRiskResolveTasks() {
|
||||
ProductRiskStaleCheckStats stats = new ProductRiskStaleCheckStats();
|
||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getProductRiskStaleTimeoutMinutes());
|
||||
@@ -358,6 +381,10 @@ public class DeleteBrandStaleTaskService {
|
||||
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
|
||||
continue;
|
||||
}
|
||||
if (skipUntilFirstHeartbeat("product-risk", task, lastPayloadHeartbeatMillis, minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
boolean hasStartedProgress = hasFinishedRows;
|
||||
if (!hasStartedProgress) {
|
||||
hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||
@@ -432,6 +459,10 @@ public class DeleteBrandStaleTaskService {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
if (skipUntilFirstHeartbeat("price-track", task, lastHeartbeatMillis, minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
boolean hasStartedProgress = hasFinishedRows;
|
||||
if (!hasStartedProgress) {
|
||||
hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||
@@ -509,6 +540,10 @@ public class DeleteBrandStaleTaskService {
|
||||
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
|
||||
continue;
|
||||
}
|
||||
if (skipUntilFirstHeartbeat("shop-match", task, lastHeartbeatMillis, minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
boolean hasStartedProgress = hasFinishedRows;
|
||||
if (!hasStartedProgress) {
|
||||
hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||
@@ -582,6 +617,10 @@ public class DeleteBrandStaleTaskService {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
if (skipUntilFirstHeartbeat("patrol-delete", task, lastHeartbeatMillis, minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
boolean hasStartedProgress = hasFinishedRows;
|
||||
if (!hasStartedProgress) {
|
||||
hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||
@@ -654,6 +693,10 @@ public class DeleteBrandStaleTaskService {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
if (skipUntilFirstHeartbeat("query-asin", task, lastHeartbeatMillis, minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
boolean hasStartedProgress = hasFinishedRows;
|
||||
if (!hasStartedProgress) {
|
||||
hasStartedProgress = queryAsinTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||
@@ -724,6 +767,10 @@ public class DeleteBrandStaleTaskService {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
if (skipUntilFirstHeartbeat("withdraw", task, lastHeartbeatMillis, minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_WITHDRAW, task.getId());
|
||||
if (taskLockHandle == null) {
|
||||
stats.skippedTaskCount++;
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 陈旧判死的「首次心跳」闸——把各条心跳超时线改成<b>从首次心跳起算</b>。
|
||||
*
|
||||
* <p>要解决的问题:任务一被认领(页面 activate / 客户端兜底拉取,同一条 PENDING→RUNNING 原子更新)
|
||||
* 服务端就把它记成 RUNNING,而客户端的心跳要等到执行器真开始跑才发(各执行器里的
|
||||
* {@code task_heartbeats.add})。同店被占用、线程池 5 格全忙、浏览器还在开首屏,这几段排队时间
|
||||
* 服务端一行心跳都收不到,{@code updated_at} 也就一直停在认领那一刻。此时按「心跳超时 30 分钟」
|
||||
* 判死,杀掉的是排队的正常任务(现网 2026-10-17 近 7 天 COLLECT_DATA 那 9 条 62 分钟判死即此类)。
|
||||
*
|
||||
* <p>闸门的口径:
|
||||
* <ul>
|
||||
* <li>收到过心跳 ⇒ 心跳时钟已经走起来了,本闸放行,交回各模块原有的心跳判据;</li>
|
||||
* <li>从未收到心跳 ⇒ 任务还没开始跑,未启动宽限(默认 240 分钟)之内一律<b>暂不判死</b>;
|
||||
* 宽限之外仍然判死,否则「认领后客户端就崩、也没来得及上报中断」的任务会变成永久 RUNNING 孤儿;</li>
|
||||
* <li>宽限配成 {@code <= 0} ⇒ 闸门整体关闭,退回改造前行为(回滚只需要改环境变量)。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>宽限的锚是 {@code updated_at}。对「从未心跳」的任务来说这个等式成立:没有心跳就没有任何
|
||||
* 刷新 {@code updated_at} 的路径,它停在那里正是认领时刻;万一还有别的写入方碰过这一行,等式变成
|
||||
* 「最近一次状态写入」,误差方向依旧是多活一会儿。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class StaleTaskHeartbeatPolicy {
|
||||
|
||||
/** 未启动宽限(分钟)。<=0=闸门关闭,退回「心跳超时线从认领时刻起算」的旧行为。 */
|
||||
@Value("${aiimage.task-stale.first-heartbeat-grace-minutes:240}")
|
||||
private long firstHeartbeatGraceMinutes;
|
||||
|
||||
private final TaskHeartbeatLivenessService livenessService;
|
||||
|
||||
/**
|
||||
* @param taskId 任务 id,用于回退读共享心跳打点
|
||||
* @param moduleLastHeartbeatMillis 调用方模块自己的「最近心跳」读数;0 表示该模块没有这份证据
|
||||
* @param taskUpdatedAt 任务行的 updated_at(未启动宽限的锚)
|
||||
* @param now 本轮扫描的统一时刻
|
||||
* @return true=本轮暂不判死(任务还在排队,心跳时钟尚未开始)
|
||||
*/
|
||||
public boolean isStillQueuedBeforeFirstHeartbeat(long taskId, long moduleLastHeartbeatMillis,
|
||||
LocalDateTime taskUpdatedAt, LocalDateTime now) {
|
||||
if (firstHeartbeatGraceMinutes <= 0L) {
|
||||
return false;
|
||||
}
|
||||
long beat = moduleLastHeartbeatMillis > 0L
|
||||
? moduleLastHeartbeatMillis
|
||||
: livenessService.lastHeartbeatMillis(taskId);
|
||||
if (beat > 0L) {
|
||||
return false;
|
||||
}
|
||||
if (taskUpdatedAt == null) {
|
||||
// 没有锚就无从计算宽限,按放宽处理:判死证据不足时不该杀任务
|
||||
return true;
|
||||
}
|
||||
return Duration.between(taskUpdatedAt, now).toMinutes() < firstHeartbeatGraceMinutes;
|
||||
}
|
||||
|
||||
public long firstHeartbeatGraceMinutes() {
|
||||
return firstHeartbeatGraceMinutes;
|
||||
}
|
||||
|
||||
/** 单测直接塞宽限值,不改生产注入路径。 */
|
||||
void setFirstHeartbeatGraceMinutes(long minutes) {
|
||||
this.firstHeartbeatGraceMinutes = minutes;
|
||||
}
|
||||
}
|
||||
+29
-1
@@ -12,6 +12,7 @@ import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
@@ -41,6 +42,8 @@ public class StaleTaskRepairService {
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
/** 心跳时钟改为「从首次心跳起算」:排队中的 RUNNING 任务不在这条线上判死。 */
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
|
||||
/**
|
||||
* 每 10 分钟执行一次;双实例下用分布式锁保证只有一台执行修复。
|
||||
@@ -130,11 +133,13 @@ public class StaleTaskRepairService {
|
||||
// 先查 id 再按主键更新:此前是一条无 LIMIT、无 module_type 的批量 UPDATE,
|
||||
// 而 biz_file_task 没有以 status 打头的索引 → 每 10 分钟一次全表扫,
|
||||
// 且按无索引条件命中行加 X 锁(长事务/锁扩散)。与另两个修复方法口径保持一致。
|
||||
// 多取 updated_at / module_type:首次心跳闸门要算排队时长,日志要说是哪个模块。
|
||||
List<FileTaskEntity> stale = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId)
|
||||
.select(FileTaskEntity::getId, FileTaskEntity::getUpdatedAt, FileTaskEntity::getModuleType)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, cutoff)
|
||||
.last("limit 500"));
|
||||
stale = rejectQueuedBeforeFirstHeartbeat(stale);
|
||||
if (stale.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
@@ -152,6 +157,29 @@ public class StaleTaskRepairService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 把「服务端还一行心跳都没收到、且仍在未启动宽限内」的 RUNNING 任务从候选里摘掉。
|
||||
*
|
||||
* <p>这条线是 2 小时无写判死,也是各模块心跳线的天花板:不摘它们,排队超过 2 小时的任务照样会被
|
||||
* 这里以「任务心跳超时」杀掉,模块侧放宽到多少分钟都白做。判据与措辞统一收在
|
||||
* {@link StaleTaskHeartbeatPolicy}(那个模块没有自己的心跳读数时,闸门退回共享心跳打点)。
|
||||
*/
|
||||
private List<FileTaskEntity> rejectQueuedBeforeFirstHeartbeat(List<FileTaskEntity> candidates) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
List<FileTaskEntity> kept = new ArrayList<>(candidates.size());
|
||||
for (FileTaskEntity task : candidates) {
|
||||
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
task.getId(), 0L, task.getUpdatedAt(), now)) {
|
||||
log.info("[stale-task-repair] 未收到首次心跳、排队宽限内暂不判死 taskId={} moduleType={} updatedAt={} graceMinutes={}",
|
||||
task.getId(), task.getModuleType(), task.getUpdatedAt(),
|
||||
staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes());
|
||||
continue;
|
||||
}
|
||||
kept.add(task);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
/**
|
||||
* brand_crawl_tasks 的 pending/running 陈旧 → failed。
|
||||
* 品牌表与状态字面量属于 brand 模块,实现放在 BrandTaskStaleRepairSpiImpl(G5:消除 task → 业务依赖)。
|
||||
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* 「服务端最后一次收到该任务心跳」的时刻(Redis String,24h TTL)。
|
||||
*
|
||||
* <p>与 {@link TaskHeartbeatPositionService} 的分工必须划清:那边存的是「卡在哪」,位置缺失只退化
|
||||
* 文案,按既有铁律<b>不得作为任何判死依据</b>;这里只回答一个二元问题——这个任务到底有没有心跳上来过,
|
||||
* 因此每条心跳都无条件打点(不挑载荷有没有 progressText),老客户端同样能被算成「已开始心跳」。
|
||||
*
|
||||
* <p>存在的理由:客户端只在真正开始执行任务时才发心跳(各执行器 {@code task_heartbeats.add}),
|
||||
* 而任务一被认领就已是 RUNNING。排队期间(同店占用、线程池 5 格全忙、浏览器还在开)服务端一行心跳都
|
||||
* 收不到,此时拿「心跳超时 30 分钟」去套,杀的是排队而不是死任务。
|
||||
*
|
||||
* <p>读不到=没有证据,调用方按「从未心跳」处理。这个方向的误差只会让任务多活一会儿(最终仍由
|
||||
* {@link StaleTaskHeartbeatPolicy} 的未启动宽限线和 {@link StaleTaskRepairService} 兜底),
|
||||
* 不会多杀一个任务,所以 Redis 抖动时不需要额外兜底。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskHeartbeatLivenessService {
|
||||
|
||||
private static final String KEY_PREFIX = "task:heartbeat:beat:";
|
||||
private static final Duration BEAT_TTL = Duration.ofHours(24);
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
|
||||
/** 收到一次心跳就打点。打点失败只记日志——心跳接口绝不能因为证据写不下去而报错。 */
|
||||
public void stamp(long taskId) {
|
||||
if (taskId <= 0L) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.opsForValue()
|
||||
.set(KEY_PREFIX + taskId, String.valueOf(Instant.now().toEpochMilli()), BEAT_TTL);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-heartbeat-beat] 心跳打点失败(不影响心跳)taskId={} msg={}", taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/** 最近一次心跳到达时刻(epoch millis);返回 0 表示从未收到过、或那条证据已过 TTL。 */
|
||||
public long lastHeartbeatMillis(long taskId) {
|
||||
if (taskId <= 0L) {
|
||||
return 0L;
|
||||
}
|
||||
try {
|
||||
return parseUnsignedLong(stringRedisTemplate.opsForValue().get(KEY_PREFIX + taskId));
|
||||
} catch (Exception ex) {
|
||||
// 读证据失败=「没有证据」,判死线据此放宽;把异常抛给巡检线换不来一个更准的判定
|
||||
log.warn("[task-heartbeat-beat] 读心跳打点失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 脏值一律读成 0,不靠异常控流:这个数唯一的作用是回答「有没有心跳上来过」,
|
||||
* 非纯数字/溢出/空值都只能落到「没有证据」那一侧。
|
||||
*/
|
||||
private static long parseUnsignedLong(String raw) {
|
||||
if (raw == null || raw.isEmpty()) {
|
||||
return 0L;
|
||||
}
|
||||
long value = 0L;
|
||||
for (int i = 0; i < raw.length(); i++) {
|
||||
char digit = raw.charAt(i);
|
||||
if (digit < '0' || digit > '9') {
|
||||
return 0L;
|
||||
}
|
||||
value = value * 10L + (digit - '0');
|
||||
if (value < 0L) {
|
||||
return 0L; // 溢出=脏值
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
+10
@@ -36,16 +36,19 @@ public class TaskHeartbeatService {
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi;
|
||||
private final TaskHeartbeatLivenessService livenessService;
|
||||
/** moduleType → 模块心跳实现(启动时建索引并校验重复注册) */
|
||||
private final Map<String, TaskModuleHeartbeatSpi> moduleHandlers;
|
||||
|
||||
public TaskHeartbeatService(FileTaskMapper fileTaskMapper,
|
||||
TaskProgressSnapshotService taskProgressSnapshotService,
|
||||
BrandTaskHeartbeatSpi brandTaskHeartbeatSpi,
|
||||
TaskHeartbeatLivenessService livenessService,
|
||||
List<TaskModuleHeartbeatSpi> moduleHeartbeatHandlers) {
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskProgressSnapshotService = taskProgressSnapshotService;
|
||||
this.brandTaskHeartbeatSpi = brandTaskHeartbeatSpi;
|
||||
this.livenessService = livenessService;
|
||||
Map<String, TaskModuleHeartbeatSpi> handlers = new LinkedHashMap<>();
|
||||
for (TaskModuleHeartbeatSpi handler : moduleHeartbeatHandlers == null ? List.<TaskModuleHeartbeatSpi>of() : moduleHeartbeatHandlers) {
|
||||
String moduleType = handler.moduleType();
|
||||
@@ -79,6 +82,13 @@ public class TaskHeartbeatService {
|
||||
TaskHeartbeatVo fileResult = brandOnly ? null : touchFileTaskIfRunning(selectFileTask(taskId), request);
|
||||
TaskHeartbeatVo brandResult = skipBrandLookup ? null : brandTaskHeartbeatSpi.heartbeat(taskId, request);
|
||||
|
||||
// 心跳到达打点:陈旧判死线用它把「心跳时钟」改成从首次心跳起算(见 StaleTaskHeartbeatPolicy)。
|
||||
// 只在任务确实存在时打,免得坏 id 把 Redis 写成垃圾键;状态不是 RUNNING 也照打——
|
||||
// 「客户端已经在管这个任务」本身就是证据,与本轮要不要刷 updated_at 是两件事。
|
||||
if (fileResult != null || brandResult != null) {
|
||||
livenessService.stamp(taskId);
|
||||
}
|
||||
|
||||
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
|
||||
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
@@ -235,6 +235,11 @@ aiimage:
|
||||
withdraw-stale-timeout-minutes: ${AIIMAGE_WITHDRAW_STALE_TIMEOUT_MINUTES:30}
|
||||
no-result-upload-timeout-minutes: ${AIIMAGE_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180}
|
||||
no-result-upload-check-enabled: ${AIIMAGE_NO_RESULT_UPLOAD_CHECK_ENABLED:true}
|
||||
# 陈旧判死的「首次心跳」闸(2026-10-17):任务被认领即记 RUNNING,而客户端要等执行器真开始跑才发心跳。
|
||||
# 这段排队时间里各模块的心跳超时线一律不判死,超过本宽限才由「认领后长时间未启动」收口;
|
||||
# 置 0 即整体关闭闸门、退回旧行为(只需改环境变量,不用重新构建)。
|
||||
task-stale:
|
||||
first-heartbeat-grace-minutes: ${AIIMAGE_TASK_STALE_FIRST_HEARTBEAT_GRACE_MINUTES:240}
|
||||
module-cleanup:
|
||||
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
|
||||
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
||||
|
||||
@@ -45,8 +45,9 @@ class CatchSilenceRatchetTest {
|
||||
private static final Path TREE = moduleRoot().resolve("src/main/java");
|
||||
|
||||
// ==== 验收基线(用户 2026-10-08 12:0x 实测,HEAD 274ad017,单位:处)====
|
||||
// 10-17 心跳证据两处 fail-open 抬 559 -> 561,逐处出处见 STRICT_NOT_RETHROW 上方注释。
|
||||
static final int GIVEN_TOTAL = 1000;
|
||||
static final int GIVEN_NOT_RETHROW = 559;
|
||||
static final int GIVEN_NOT_RETHROW = 561;
|
||||
static final int GIVEN_LOG_ONLY = 249;
|
||||
static final int GIVEN_EMPTY = 56;
|
||||
|
||||
@@ -59,8 +60,13 @@ class CatchSilenceRatchetTest {
|
||||
* BuildInfoLogger 那一处不是吞掉:它打 {@code [build-info-missing]} 的 WARN 并返回可区分的
|
||||
* "章存在但没盖住"状态,巡检按锚点数得到。抬这一格是为了把出处留在这份 diff 里,
|
||||
* 而不是让后来人以为基线是自然涨上来的。
|
||||
* 10-17 再 +2,逐处记名:{@code TaskHeartbeatLivenessService.stamp()}(心跳证据写失败只 WARN,
|
||||
* 心跳接口不许因证据写不下去而 500)与同类 {@code lastHeartbeatMillis()}(读证据失败一律落到 0=
|
||||
* 「没有证据」,判死线据此放宽)。两处都是 fail-open 且方向是「少杀一个任务」,与上面
|
||||
* BuildInfoLogger 同形状;本仓实测基线由 1001 起(原记 1002 留有 1 格无人认领的额度),
|
||||
* 加这 2 处后实测 1003。
|
||||
*/
|
||||
static final int STRICT_TOTAL = 1002;
|
||||
static final int STRICT_TOTAL = 1003;
|
||||
/**
|
||||
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
|
||||
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
|
||||
@@ -75,8 +81,23 @@ class CatchSilenceRatchetTest {
|
||||
// 的两个零调用者重载,其中 `throw new BusinessException("删除 Excel 匹配数据失败", e)` 是 RETHROW 形状,
|
||||
// 随死代码一起离开树;实测重抛 386、不重抛与只 log 两数未涨(那两格仍绿)=> 没有任何一处「异常会往外走」被改回吞掉。
|
||||
static final int RETHROW_FLOOR = 386;
|
||||
static final int STRICT_NOT_RETHROW = 559;
|
||||
static final int STRICT_LOG_ONLY = 172; // 严格口径:块内每条顶层语句都是 log 调用才算
|
||||
/**
|
||||
* 10-17 抬 559 -> 561,逐处记名(两处都是新增的 fail-open 证据读写,不是把已有出口改回吞掉):
|
||||
* 1){@code TaskHeartbeatLivenessService.stamp()}:心跳证据写失败只 WARN——心跳接口不许因为
|
||||
* 「记不下证据」而 500,那会让客户端整轮心跳线程崩掉,反而制造更多孤儿任务;
|
||||
* 2){@code TaskHeartbeatLivenessService.lastHeartbeatMillis()}:读证据失败一律落到 0=「没有证据」,
|
||||
* 判死线据此**放宽**(顶多让任务多活一会儿,最终仍由未启动宽限与 stale-repair 兜底)。
|
||||
* 两处的误差方向都是「少杀一个任务」,与这一格想拦的「异常到此为止、外面看不见」不同性质;
|
||||
* 抬格是为了把出处留在这份 diff 里,而不是让后来人以为基线自然涨上来的。
|
||||
*/
|
||||
static final int STRICT_NOT_RETHROW = 561;
|
||||
/**
|
||||
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
|
||||
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
|
||||
* 心跳接口不许因为「记不下证据」而 500。同类的 {@code lastHeartbeatMillis()} 读失败还要
|
||||
* 落 0 返回,形状是 log+return 默认值,因此不进这一格。
|
||||
*/
|
||||
static final int STRICT_LOG_ONLY = 173; // 严格口径:块内每条顶层语句都是 log 调用才算
|
||||
static final int STRICT_EMPTY = 56;
|
||||
|
||||
/**
|
||||
|
||||
+2
-1
@@ -147,7 +147,8 @@ class CollectDataActivateGuardTest {
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
|
||||
mock(CollectDataBrandRecheckService.class));
|
||||
mock(CollectDataBrandRecheckService.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy.class));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
+2
-1
@@ -120,7 +120,8 @@ class CollectDataDeleteConsistencyTest {
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
|
||||
mock(CollectDataBrandRecheckService.class));
|
||||
mock(CollectDataBrandRecheckService.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity task(long id, long userId) {
|
||||
|
||||
+29
@@ -58,6 +58,13 @@ class CollectDataServiceTest {
|
||||
@Mock(answer = org.mockito.Answers.RETURNS_MOCKS)
|
||||
private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
|
||||
/**
|
||||
* 「首次心跳之前不判死」闸门:默认 false=不豁免,本类原有 stale 判死用例的行为保持不变;
|
||||
* 豁免分支另有用例(staleTaskWithoutFirstHeartbeatWithinGraceIsNotJudgedDead)。
|
||||
*/
|
||||
@Mock
|
||||
private com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
|
||||
@Spy
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@@ -190,6 +197,28 @@ class CollectDataServiceTest {
|
||||
verify(lockHandle).close();
|
||||
}
|
||||
|
||||
// 2026-10-17 现网 9 条误杀的口径:采集这条 30 分钟心跳线只看 updated_at,而客户端认领任务后要先等
|
||||
// 同店释放、等线程池那 5 格、等浏览器首屏——这段时间一行心跳都没有,updated_at 停在认领时刻。
|
||||
// 闸门放行时,本条线既不能标失败,也不能封口(封口会让后续回传被拒、数据丢失)。
|
||||
@Test
|
||||
void staleTaskWithoutFirstHeartbeatWithinGraceIsNotJudgedDead() throws Exception {
|
||||
FileTaskEntity queued = staleTask(95L);
|
||||
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(queued));
|
||||
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, queued.getId(), 0L))
|
||||
.thenReturn(lockHandle);
|
||||
when(fileTaskMapper.selectById(queued.getId())).thenReturn(queued);
|
||||
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
org.mockito.ArgumentMatchers.eq(95L), org.mockito.ArgumentMatchers.eq(0L),
|
||||
any(), any())).thenReturn(true);
|
||||
|
||||
service.finalizeStaleTasks();
|
||||
|
||||
assertThat(queued.getStatus()).isEqualTo("RUNNING");
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
|
||||
}
|
||||
|
||||
private static FileTaskEntity staleTask(long taskId) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
|
||||
+2
-1
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
|
||||
mock(CollectDataBrandRecheckService.class));
|
||||
mock(CollectDataBrandRecheckService.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy.class));
|
||||
}
|
||||
|
||||
private FileResultEntity result(long id, long taskId, long userId) {
|
||||
|
||||
+64
-1
@@ -62,6 +62,9 @@ class DeleteBrandStaleTaskServiceTest {
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TaskResumeService taskResumeService;
|
||||
@Mock private PriceTrackLoopRunStaleService priceTrackLoopRunStaleService;
|
||||
@Mock private com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService priceTrackTaskService;
|
||||
@Mock private com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
@@ -448,7 +451,67 @@ class DeleteBrandStaleTaskServiceTest {
|
||||
null, null, priceTrackLoopRunStaleService, null, null, null, null, null, null, null,
|
||||
null, null, null, null,
|
||||
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService, null,
|
||||
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService);
|
||||
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService, staleTaskHeartbeatPolicy);
|
||||
}
|
||||
|
||||
/** 只喂跟价那条线需要的两个 mock,其余仍走 service() 的窄桩口径。 */
|
||||
private DeleteBrandStaleTaskService priceTrackService() {
|
||||
return new DeleteBrandStaleTaskService(
|
||||
fileTaskMapper, deleteBrandTaskCacheService, deleteBrandTaskStorageService, deleteBrandRunService,
|
||||
productRiskTaskService, productRiskTaskCacheService,
|
||||
priceTrackTaskService, priceTrackTaskCacheService, priceTrackLoopRunStaleService,
|
||||
null, null, null, null, null, null, null,
|
||||
null, null, null, null,
|
||||
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService, null,
|
||||
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService, staleTaskHeartbeatPolicy);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
private FileTaskEntity priceTrackCandidate(Long id) {
|
||||
FileTaskEntity task = runningTask(id);
|
||||
task.setModuleType("PRICE_TRACK");
|
||||
// 认领后再没有别的写入:updated_at 就是排队起点
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(40));
|
||||
return task;
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
void priceTrackCandidateWithoutFirstHeartbeatIsNotJudgedDead() {
|
||||
FileTaskEntity queued = priceTrackCandidate(9001L);
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(queued));
|
||||
when(deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes()).thenReturn(30L);
|
||||
when(priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(any())).thenReturn(Map.of());
|
||||
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
org.mockito.ArgumentMatchers.eq(9001L), org.mockito.ArgumentMatchers.eq(0L),
|
||||
any(), any())).thenReturn(true);
|
||||
|
||||
ReflectionTestUtils.invokeMethod(priceTrackService(), "failStalePriceTrackTasks");
|
||||
|
||||
// 排队中的任务:既不该标失败,也不该被 finalize 封口(封口会让后续回传被拒、数据丢失)
|
||||
verify(fileTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
verify(priceTrackTaskService, never()).tryFinalizeTask(anyLong(), anyBoolean());
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
void priceTrackCandidatePastTheGraceWindowIsStillJudgedDead() {
|
||||
FileTaskEntity orphan = priceTrackCandidate(9002L);
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(orphan));
|
||||
when(deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes()).thenReturn(30L);
|
||||
when(priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(any())).thenReturn(Map.of());
|
||||
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
org.mockito.ArgumentMatchers.eq(9002L), org.mockito.ArgumentMatchers.eq(0L),
|
||||
any(), any())).thenReturn(false);
|
||||
when(taskDistributedLockService.acquire(any(), any(), anyLong()))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
when(priceTrackTaskService.tryFinalizeTask(org.mockito.ArgumentMatchers.eq(9002L), anyBoolean()))
|
||||
.thenReturn(false);
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
ReflectionTestUtils.invokeMethod(priceTrackService(), "failStalePriceTrackTasks");
|
||||
|
||||
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
|
||||
+1
-1
@@ -82,7 +82,7 @@ class TaskModuleCoverageTest {
|
||||
mock(SimilarAsinTaskService.class),
|
||||
null, null, null, null,
|
||||
mock(ShopDataCrawlTaskService.class),
|
||||
null, null, null);
|
||||
null, null, null, null);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 「心跳时钟从首次心跳起算」的判据契约。
|
||||
*
|
||||
* <p>要守住的现场(2026-10-17 生产实录):任务一被认领就记 RUNNING,而客户端要等执行器真开始跑才发
|
||||
* 心跳(同店占用 / 线程池 5 格全忙 / 浏览器首屏)。这段排队时间里服务端一行心跳都没有,
|
||||
* 拿各模块 30 分钟心跳超时线去套,杀的是排队任务。
|
||||
*/
|
||||
class StaleTaskHeartbeatPolicyTest {
|
||||
|
||||
private final TaskHeartbeatLivenessService livenessService = mock(TaskHeartbeatLivenessService.class);
|
||||
private final StaleTaskHeartbeatPolicy policy = new StaleTaskHeartbeatPolicy(livenessService);
|
||||
|
||||
private StaleTaskHeartbeatPolicy withGrace(long minutes) {
|
||||
policy.setFirstHeartbeatGraceMinutes(minutes);
|
||||
return policy;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 从未心跳且在宽限内_还在排队本轮不判死() {
|
||||
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
|
||||
|
||||
assertTrue(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
|
||||
31279L, 0L, LocalDateTime.now().minusMinutes(40), LocalDateTime.now()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 从未心跳且超出宽限_照旧判死不许留成永久孤儿() {
|
||||
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
|
||||
|
||||
assertFalse(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
|
||||
31279L, 0L, LocalDateTime.now().minusMinutes(241), LocalDateTime.now()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 模块自带心跳读数时不回落共享打点() {
|
||||
assertFalse(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
|
||||
31279L, System.currentTimeMillis(), LocalDateTime.now().minusMinutes(90), LocalDateTime.now()));
|
||||
|
||||
verify(livenessService, never()).lastHeartbeatMillis(anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 模块没有心跳读数时回落共享打点() {
|
||||
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(System.currentTimeMillis());
|
||||
|
||||
assertFalse(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
|
||||
31279L, 0L, LocalDateTime.now().minusMinutes(40), LocalDateTime.now()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 宽限置零_闸门关闭退回旧行为() {
|
||||
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
|
||||
|
||||
assertFalse(withGrace(0).isStillQueuedBeforeFirstHeartbeat(
|
||||
31279L, 0L, LocalDateTime.now().minusMinutes(31), LocalDateTime.now()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有updated_at就无从算宽限() {
|
||||
when(livenessService.lastHeartbeatMillis(31279L)).thenReturn(0L);
|
||||
|
||||
assertTrue(withGrace(240).isStillQueuedBeforeFirstHeartbeat(
|
||||
31279L, 0L, null, LocalDateTime.now()));
|
||||
}
|
||||
}
|
||||
+19
-1
@@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -40,8 +41,9 @@ class StaleTaskRepairServiceTest {
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi = mock(BrandTaskStaleRepairSpi.class);
|
||||
private final DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class);
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy = mock(StaleTaskHeartbeatPolicy.class);
|
||||
private final StaleTaskRepairService service = new StaleTaskRepairService(
|
||||
fileTaskMapper, brandTaskStaleRepairSpi, distributedJobLockService);
|
||||
fileTaskMapper, brandTaskStaleRepairSpi, distributedJobLockService, staleTaskHeartbeatPolicy);
|
||||
|
||||
@BeforeEach
|
||||
void initTableInfo() {
|
||||
@@ -99,6 +101,22 @@ class StaleTaskRepairServiceTest {
|
||||
verifyNoMoreInteractions(brandTaskStaleRepairSpi);
|
||||
}
|
||||
|
||||
// 2026-10-17 实录:任务一被认领就记 RUNNING,而客户端要等执行器真开始跑才发心跳(同店占用 /
|
||||
// 线程池 5 格全忙 / 浏览器首屏)。本线是各模块心跳线的天花板——不在这里放行,模块侧把宽限
|
||||
// 放到多少分钟都会被这里的 2 小时重新杀成「任务心跳超时」。
|
||||
@Test
|
||||
void 未收到首次心跳的_RUNNING_任务在排队宽限内暂不判死() {
|
||||
when(staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
eq(31279L), eq(0L), any(), any())).thenReturn(true);
|
||||
FileTaskEntity queued = task(31279L);
|
||||
queued.setStatus("RUNNING");
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(queued));
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "repairFileTaskStaleRunning");
|
||||
|
||||
verify(fileTaskMapper, never()).update(isNull(), any());
|
||||
}
|
||||
|
||||
private Map<String, Object> selectParams() {
|
||||
ArgumentCaptor<Wrapper<FileTaskEntity>> captor = wrapperCaptor();
|
||||
verify(fileTaskMapper).selectList(captor.capture());
|
||||
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 心跳打点契约:判死线把它当「这任务到底有没有心跳上来过」的唯一通用证据,
|
||||
* 因此读取侧的每一个失败分支都必须落到 0(=没有证据=放宽),绝不能把异常抛给判死线。
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
class TaskHeartbeatLivenessServiceTest {
|
||||
|
||||
private final StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
|
||||
private final ValueOperations<String, String> valueOperations = mock(ValueOperations.class);
|
||||
private final TaskHeartbeatLivenessService service = new TaskHeartbeatLivenessService(redisTemplate);
|
||||
|
||||
private TaskHeartbeatLivenessServiceTest withRedis() {
|
||||
lenient().when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 收到心跳即打点并带过期() {
|
||||
withRedis();
|
||||
|
||||
service.stamp(31279L);
|
||||
|
||||
verify(valueOperations).set(eq("task:heartbeat:beat:31279"), anyString(), any(Duration.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 打点失败不许影响心跳本身() {
|
||||
withRedis();
|
||||
doThrow(new IllegalStateException("redis down"))
|
||||
.when(valueOperations).set(anyString(), anyString(), any(Duration.class));
|
||||
|
||||
service.stamp(31279L); // 不抛即通过:心跳接口不能因为证据写不下去而 500
|
||||
}
|
||||
|
||||
@Test
|
||||
void 读回打点时刻() {
|
||||
withRedis();
|
||||
when(valueOperations.get("task:heartbeat:beat:31279")).thenReturn("1792228880835");
|
||||
|
||||
assertEquals(1792228880835L, service.lastHeartbeatMillis(31279L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有打点时读成从未心跳() {
|
||||
withRedis();
|
||||
when(valueOperations.get(anyString())).thenReturn(null);
|
||||
|
||||
assertEquals(0L, service.lastHeartbeatMillis(31279L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 脏值按没有证据处理而不是把判死线带崩() {
|
||||
withRedis();
|
||||
when(valueOperations.get(anyString())).thenReturn("not-a-timestamp");
|
||||
|
||||
assertEquals(0L, service.lastHeartbeatMillis(31279L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void Redis异常时读成从未心跳() {
|
||||
withRedis();
|
||||
when(valueOperations.get(anyString())).thenThrow(new IllegalStateException("redis down"));
|
||||
|
||||
assertEquals(0L, service.lastHeartbeatMillis(31279L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 非法taskId不打点也不读() {
|
||||
service.stamp(0L);
|
||||
assertEquals(0L, service.lastHeartbeatMillis(-1L));
|
||||
|
||||
verify(redisTemplate, never()).opsForValue();
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -29,8 +29,10 @@ import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
@@ -55,10 +57,11 @@ class TaskHeartbeatServiceTest {
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
|
||||
private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi = mock(BrandTaskHeartbeatSpi.class);
|
||||
private final TaskHeartbeatLivenessService livenessService = mock(TaskHeartbeatLivenessService.class);
|
||||
|
||||
private TaskHeartbeatService serviceWith(TaskModuleHeartbeatSpi... handlers) {
|
||||
return new TaskHeartbeatService(fileTaskMapper, taskProgressSnapshotService, brandTaskHeartbeatSpi,
|
||||
List.of(handlers));
|
||||
livenessService, List.of(handlers));
|
||||
}
|
||||
|
||||
private static FileTaskEntity runningTask(long taskId, String moduleType) {
|
||||
@@ -184,4 +187,34 @@ class TaskHeartbeatServiceTest {
|
||||
assertTrue(result.isAlive(), "未注册 SPI 的模块不应影响通用心跳(保持向后兼容)");
|
||||
assertEquals("UNREGISTERED_MODULE", result.getModuleType());
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void heartbeatStampsLivenessEvenWhenTaskIsNotRunningYet() {
|
||||
long taskId = 20151L;
|
||||
FileTaskEntity justClaimed = runningTask(taskId, "PRICE_TRACK");
|
||||
justClaimed.setStatus("PENDING");
|
||||
TaskHeartbeatService service = serviceWith();
|
||||
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(justClaimed);
|
||||
|
||||
TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest());
|
||||
|
||||
assertFalse(result.isAlive(), "非 RUNNING 的心跳不得把任务改成存活(既有契约不变)");
|
||||
verify(livenessService).stamp(taskId);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void heartbeatForUnknownTaskDoesNotStampLiveness() {
|
||||
long taskId = 20152L;
|
||||
TaskHeartbeatService service = serviceWith();
|
||||
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
|
||||
|
||||
service.heartbeat(taskId, new TaskHeartbeatRequest());
|
||||
|
||||
// 坏 id 的轮询不能把 Redis 写成垃圾键
|
||||
verify(livenessService, never()).stamp(anyLong());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user