feat(心跳): 影子判官先跑读数——心跳断/进度停滞/排队/服务端处理分开判
现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死(长任务/排队被 判死)同时存在。新增只读影子扫描:每 5 分钟扫 RUNNING 文件任务,按「心跳中断 / 进度停滞 / 排队宽限 / 服务端结果任务 / 24h 硬顶」输出 would-finalize 日志, 不改任何任务状态。进度用「计数+文本+阶段」指纹,写影子独立快照键,不影响正式 判死线的未变化起点。规则纯函数单测 9 例 + 只读边界守卫 3 例。
This commit is contained in:
+58
-6
@@ -28,6 +28,8 @@ public class TaskHeartbeatPositionService {
|
||||
|
||||
private static final String KEY_PREFIX = "task:heartbeat:position:";
|
||||
private static final String STALL_KEY_PREFIX = "task:progress:stall:";
|
||||
/** 影子判官的独立快照键:影子的观测时刻不得改变正式判死线的「未变化起点」。 */
|
||||
private static final String SHADOW_STALL_KEY_PREFIX = "task:progress:shadow-stall:";
|
||||
private static final Duration POSITION_TTL = Duration.ofHours(24);
|
||||
private static final int POSITION_LABEL_MAX_LEN = 60;
|
||||
/** 计数字符串的最长位数:9 位以内 parseInt 必不溢出,所以这一处不需要 catch。 */
|
||||
@@ -303,7 +305,57 @@ public class TaskHeartbeatPositionService {
|
||||
if (position == null || position.isBlank()) {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
String stallKey = STALL_KEY_PREFIX + taskId;
|
||||
return evaluateAgainstSnapshot(taskId, STALL_KEY_PREFIX + taskId, position, staleWindowMinutes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 影子判官专用:把「计数 + 文本 + 阶段」压成指纹判断推进,快照写独立键。
|
||||
*
|
||||
* <p>与正式线只比文本不同:计数变了也是推进(长页面里文本可能长时间不变),
|
||||
* 阶段变化同样算推进。影子每 5 分钟扫一次,若与正式线共用快照键,影子可能比正式线早几分钟
|
||||
* 看到变化,从而把正式线的未变化起点提前、间接影响封口时刻;所以这里只写 shadow 键。
|
||||
*/
|
||||
public StallState evaluateProgressForShadow(Long taskId, long staleWindowMinutes) {
|
||||
if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
String fingerprint = readProgressFingerprint(taskId);
|
||||
if (fingerprint == null) {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
return evaluateAgainstSnapshot(taskId, SHADOW_STALL_KEY_PREFIX + taskId,
|
||||
fingerprint, staleWindowMinutes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 影子判官的推进指纹:位置哈希一次读出,计数/文本/阶段任一变化都算推进。
|
||||
* 返回 null 表示这个任务没有任何位置信息(老客户端不传位置),不能作为判死证据。
|
||||
*/
|
||||
private String readProgressFingerprint(Long taskId) {
|
||||
Map<Object, Object> entries;
|
||||
try {
|
||||
entries = stringRedisTemplate.opsForHash().entries(KEY_PREFIX + taskId);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[heartbeat-position] 读取影子位置失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
if (entries == null || entries.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String text = value(entries, "progressText");
|
||||
String current = value(entries, "current");
|
||||
String total = value(entries, "total");
|
||||
if (text.isBlank() && current.isBlank() && total.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value(entries, "phase") + "|" + value(entries, "grain")
|
||||
+ "|" + current + "|" + total + "|" + text;
|
||||
}
|
||||
|
||||
private StallState evaluateAgainstSnapshot(Long taskId,
|
||||
String stallKey,
|
||||
String fingerprint,
|
||||
long staleWindowMinutes) {
|
||||
Map<Object, Object> stored;
|
||||
try {
|
||||
stored = stringRedisTemplate.opsForHash().entries(stallKey);
|
||||
@@ -312,18 +364,18 @@ public class TaskHeartbeatPositionService {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
long now = Instant.now().toEpochMilli();
|
||||
String storedPosition = stored == null ? "" : value(stored, "pos");
|
||||
String storedFingerprint = stored == null ? "" : value(stored, "pos");
|
||||
long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at"));
|
||||
if (!position.equals(storedPosition) || unchangedSince <= 0L) {
|
||||
writeStallSnapshot(stallKey, position, now, staleWindowMinutes);
|
||||
if (!fingerprint.equals(storedFingerprint) || unchangedSince <= 0L) {
|
||||
writeStallSnapshot(stallKey, fingerprint, now, staleWindowMinutes);
|
||||
return StallState.ADVANCING;
|
||||
}
|
||||
long unchangedMillis = now - unchangedSince;
|
||||
if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) {
|
||||
return StallState.STALLED;
|
||||
}
|
||||
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}",
|
||||
taskId, unchangedMillis, position);
|
||||
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} fingerprint={}",
|
||||
taskId, unchangedMillis, fingerprint);
|
||||
return StallState.ADVANCING;
|
||||
}
|
||||
|
||||
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* 心跳影子判官的纯规则:只回答「按新规则会怎么判」,不碰任何任务状态。
|
||||
*
|
||||
* <p>为什么要影子期:现有判死线各看各的证据(Redis 心跳、file_task.updated_at、
|
||||
* last_chunk_at、各模块超时),既有「心跳线程活着但主线程卡死」的假活,也有长任务/排队
|
||||
* 被判死的假死。新规则把「心跳是否断」与「进度是否推进」分开,并显式豁免服务端处理中的
|
||||
* 任务;先用真实 RUNNING 任务跑一周 would-finalize 读数,再决定是否接管判死。
|
||||
*
|
||||
* <p>规则优先级(先命中先返回):
|
||||
* <ol>
|
||||
* <li>服务端结果任务仍在跑 → KEEP_SERVER_SIDE(客户端已交棒,心跳停了是正常的);</li>
|
||||
* <li>从未收到心跳 → 未启动宽限内 KEEP_QUEUED,超宽限 WOULD_FINALIZE(no-first-heartbeat);</li>
|
||||
* <li>心跳中断超过阈值 → WOULD_FINALIZE(beat-lost);</li>
|
||||
* <li>任务总时长超过硬顶 → WOULD_FINALIZE(hard-cap)(即使进度仍在推进);</li>
|
||||
* <li>心跳正常但进度停滞 → WOULD_FINALIZE(no-progress);</li>
|
||||
* <li>进度在推进 → KEEP_ADVANCING;</li>
|
||||
* <li>没有位置信息 → UNKNOWN(证据不足,不参与判死)。</li>
|
||||
* </ol>
|
||||
*/
|
||||
public final class TaskLivenessShadowRules {
|
||||
|
||||
public static final String RULE_SERVER_SIDE_JOB = "server-side-job";
|
||||
public static final String RULE_NO_FIRST_HEARTBEAT = "no-first-heartbeat";
|
||||
public static final String RULE_BEAT_LOST = "beat-lost";
|
||||
public static final String RULE_HARD_CAP = "hard-cap";
|
||||
public static final String RULE_NO_PROGRESS = "no-progress";
|
||||
public static final String RULE_ADVANCING = "advancing";
|
||||
public static final String RULE_NO_POSITION = "no-position";
|
||||
|
||||
public enum Verdict {
|
||||
KEEP_ADVANCING,
|
||||
KEEP_QUEUED,
|
||||
KEEP_SERVER_SIDE,
|
||||
UNKNOWN,
|
||||
WOULD_FINALIZE
|
||||
}
|
||||
|
||||
/** 客户端上报的进度位置是否还在变;UNKNOWN 表示老客户端没传位置,不能当判死证据。 */
|
||||
public enum ProgressState {
|
||||
ADVANCING,
|
||||
STALLED,
|
||||
UNKNOWN
|
||||
}
|
||||
|
||||
/**
|
||||
* @param beatLostMinutes 心跳中断多少分钟算断
|
||||
* @param noProgressMinutes 进度位置多少分钟没变算停滞
|
||||
* @param firstHeartbeatGraceMinutes 从未心跳的排队宽限;<=0 表示该规则停用(判 UNKNOWN)
|
||||
* @param hardCapHours 任务总时长硬顶;<=0 表示该规则停用
|
||||
*/
|
||||
public record Thresholds(int beatLostMinutes,
|
||||
int noProgressMinutes,
|
||||
int firstHeartbeatGraceMinutes,
|
||||
int hardCapHours) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param beatMillis 最近一次心跳(0=从未收到)
|
||||
* @param queuedSinceMillis 未启动宽限的锚(file_task.updated_at)
|
||||
* @param createdAtMillis 任务创建时刻,用于总量硬顶
|
||||
*/
|
||||
public record Evidence(long taskId,
|
||||
String moduleType,
|
||||
long nowMillis,
|
||||
long beatMillis,
|
||||
long queuedSinceMillis,
|
||||
long createdAtMillis,
|
||||
boolean serverSideJob,
|
||||
ProgressState progress) {
|
||||
}
|
||||
|
||||
public record Decision(Verdict verdict, String rule, String detail) {
|
||||
}
|
||||
|
||||
private TaskLivenessShadowRules() {
|
||||
}
|
||||
|
||||
public static Decision evaluate(Evidence evidence, Thresholds thresholds) {
|
||||
Objects.requireNonNull(evidence, "evidence");
|
||||
Objects.requireNonNull(thresholds, "thresholds");
|
||||
if (evidence.serverSideJob()) {
|
||||
return new Decision(Verdict.KEEP_SERVER_SIDE, RULE_SERVER_SIDE_JOB,
|
||||
"服务端结果任务仍在运行,客户端心跳停止属正常");
|
||||
}
|
||||
if (evidence.beatMillis() <= 0L) {
|
||||
long waitedMinutes = ageMinutes(evidence.nowMillis(), evidence.queuedSinceMillis());
|
||||
if (thresholds.firstHeartbeatGraceMinutes() <= 0) {
|
||||
return new Decision(Verdict.UNKNOWN, RULE_NO_FIRST_HEARTBEAT,
|
||||
"从未收到心跳,但排队宽限规则已停用,无法判定");
|
||||
}
|
||||
if (waitedMinutes < thresholds.firstHeartbeatGraceMinutes()) {
|
||||
return new Decision(Verdict.KEEP_QUEUED, RULE_NO_FIRST_HEARTBEAT,
|
||||
"从未收到心跳,仍在排队宽限内(已等待 " + waitedMinutes
|
||||
+ " 分钟 / 宽限 " + thresholds.firstHeartbeatGraceMinutes() + " 分钟)");
|
||||
}
|
||||
return new Decision(Verdict.WOULD_FINALIZE, RULE_NO_FIRST_HEARTBEAT,
|
||||
"从未收到心跳,已等待 " + waitedMinutes
|
||||
+ " 分钟(宽限 " + thresholds.firstHeartbeatGraceMinutes() + " 分钟)");
|
||||
}
|
||||
long beatAgeMinutes = ageMinutes(evidence.nowMillis(), evidence.beatMillis());
|
||||
if (thresholds.beatLostMinutes() > 0 && beatAgeMinutes > thresholds.beatLostMinutes()) {
|
||||
return new Decision(Verdict.WOULD_FINALIZE, RULE_BEAT_LOST,
|
||||
"心跳已中断 " + beatAgeMinutes + " 分钟(阈值 " + thresholds.beatLostMinutes() + " 分钟)");
|
||||
}
|
||||
long taskAgeMinutes = ageMinutes(evidence.nowMillis(), evidence.createdAtMillis());
|
||||
long taskAgeHours = taskAgeMinutes / 60L;
|
||||
if (thresholds.hardCapHours() > 0 && taskAgeHours >= thresholds.hardCapHours()) {
|
||||
return new Decision(Verdict.WOULD_FINALIZE, RULE_HARD_CAP,
|
||||
"任务已运行 " + taskAgeHours + " 小时(硬顶 " + thresholds.hardCapHours() + " 小时)");
|
||||
}
|
||||
if (evidence.progress() == ProgressState.STALLED) {
|
||||
return new Decision(Verdict.WOULD_FINALIZE, RULE_NO_PROGRESS,
|
||||
"心跳正常但进度至少 " + thresholds.noProgressMinutes() + " 分钟未推进");
|
||||
}
|
||||
if (evidence.progress() == ProgressState.ADVANCING) {
|
||||
return new Decision(Verdict.KEEP_ADVANCING, RULE_ADVANCING, "进度仍在推进");
|
||||
}
|
||||
return new Decision(Verdict.UNKNOWN, RULE_NO_POSITION,
|
||||
"心跳正常但没有进度位置,证据不足");
|
||||
}
|
||||
|
||||
private static long ageMinutes(long nowMillis, long sinceMillis) {
|
||||
if (sinceMillis <= 0L) {
|
||||
return 0L;
|
||||
}
|
||||
return Math.max(0L, (nowMillis - sinceMillis) / 60_000L);
|
||||
}
|
||||
}
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 心跳影子判官:只读扫描 RUNNING 任务,按 {@link TaskLivenessShadowRules} 输出
|
||||
* would-finalize 读数,<b>不修改任何任务状态</b>。
|
||||
*
|
||||
* <p>存在的理由:现有判死线各看各的证据,假活(心跳线程活着、主线程卡死)与假死
|
||||
* (长任务/排队被判死)同时存在。新规则先把「心跳断」与「进度停滞」分开,并显式豁免
|
||||
* 服务端结果任务;影子期只把这些判断写进日志,攒真实分布再决定是否接管判死。
|
||||
*
|
||||
* <p>状态边界:本类不许出现 update/delete/setStatus 一类任务写入;唯一的写入是
|
||||
* {@link TaskHeartbeatPositionService#evaluateProgressForShadow} 落在影子自己的进度快照键上,
|
||||
* 不影响正式判死线的「未变化起点」。这条由 {@code TaskLivenessShadowGuardTest} 按源码钉住,
|
||||
* 防止影子线悄悄变成执行线。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskLivenessShadowService {
|
||||
|
||||
public static final String LOG_ANCHOR = "[task-liveness-shadow]";
|
||||
/** 单轮最多逐条打多少条 would-finalize;其余只进汇总计数,避免日志被长任务尾巴刷屏。 */
|
||||
private static final int MAX_LOGGED_WOULD_FINALIZE = 20;
|
||||
private static final String SCAN_LOCK_NAME = "task-liveness-shadow";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskFileJobMapper taskFileJobMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final TaskHeartbeatLivenessService livenessService;
|
||||
private final TaskHeartbeatPositionService positionService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Value("${aiimage.task-liveness-shadow.enabled:true}")
|
||||
private boolean enabled;
|
||||
@Value("${aiimage.task-liveness-shadow.scan-limit:500}")
|
||||
private int scanLimit;
|
||||
@Value("${aiimage.task-liveness-shadow.beat-lost-minutes:10}")
|
||||
private int beatLostMinutes;
|
||||
@Value("${aiimage.task-liveness-shadow.no-progress-minutes:30}")
|
||||
private int noProgressMinutes;
|
||||
@Value("${aiimage.task-liveness-shadow.first-heartbeat-grace-minutes:240}")
|
||||
private int firstHeartbeatGraceMinutes;
|
||||
@Value("${aiimage.task-liveness-shadow.hard-cap-hours:24}")
|
||||
private int hardCapHours;
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.task-liveness-shadow.scan-fixed-delay-ms:300000}",
|
||||
initialDelayString = "${aiimage.task-liveness-shadow.scan-initial-delay-ms:120000}")
|
||||
public void scheduledScan() {
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
DistributedJobLockService.LockHandle lock;
|
||||
try {
|
||||
lock = distributedJobLockService.tryLock(SCAN_LOCK_NAME, Duration.ofMinutes(5));
|
||||
} catch (Exception ex) {
|
||||
log.warn("{} 获取扫描锁失败,本轮跳过 msg={}", LOG_ANCHOR, ex.getMessage());
|
||||
return;
|
||||
}
|
||||
if (lock == null) {
|
||||
log.info("{} 另一实例持有扫描锁,本轮跳过", LOG_ANCHOR);
|
||||
return;
|
||||
}
|
||||
try (lock) {
|
||||
scanOnce();
|
||||
} catch (Exception ex) {
|
||||
// 影子线只读:扫描失败绝不能影响任务本身
|
||||
log.warn("{} 扫描失败(只读,不影响任务) msg={}", LOG_ANCHOR, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void scanOnce() {
|
||||
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId, FileTaskEntity::getModuleType,
|
||||
FileTaskEntity::getCreatedAt, FileTaskEntity::getUpdatedAt)
|
||||
.eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit " + Math.max(1, scanLimit)));
|
||||
if (tasks == null || tasks.isEmpty()) {
|
||||
log.info("{} scanned=0(当前没有 RUNNING 任务)", LOG_ANCHOR);
|
||||
return;
|
||||
}
|
||||
List<Long> taskIds = tasks.stream()
|
||||
.map(FileTaskEntity::getId)
|
||||
.filter(Objects::nonNull)
|
||||
.toList();
|
||||
Set<Long> serverSideTaskIds = activeJobTaskIds(taskIds);
|
||||
Map<Long, LocalDateTime> lastChunkAtByTaskId = lastChunkAtByTaskId(taskIds);
|
||||
TaskLivenessShadowRules.Thresholds thresholds = new TaskLivenessShadowRules.Thresholds(
|
||||
beatLostMinutes, noProgressMinutes, firstHeartbeatGraceMinutes, hardCapHours);
|
||||
long now = System.currentTimeMillis();
|
||||
int keepAdvancing = 0;
|
||||
int keepQueued = 0;
|
||||
int keepServerSide = 0;
|
||||
int unknown = 0;
|
||||
int wouldFinalize = 0;
|
||||
int logged = 0;
|
||||
Map<String, Integer> ruleCounts = new LinkedHashMap<>();
|
||||
for (FileTaskEntity task : tasks) {
|
||||
Long taskId = task.getId();
|
||||
if (taskId == null) {
|
||||
continue;
|
||||
}
|
||||
// 服务端结果任务在跑时不读心跳/位置:客户端已交棒,心跳停了属正常,读取只会白打 Redis。
|
||||
boolean serverSide = serverSideTaskIds.contains(taskId);
|
||||
long beatMillis = serverSide ? 0L : livenessService.lastHeartbeatMillis(taskId);
|
||||
// 从未心跳的任务由排队规则直接定论,不必再读一次位置(省一轮 Redis)。
|
||||
TaskLivenessShadowRules.ProgressState progress = !serverSide && beatMillis > 0L
|
||||
? toProgress(positionService.evaluateProgressForShadow(taskId, noProgressMinutes))
|
||||
: TaskLivenessShadowRules.ProgressState.UNKNOWN;
|
||||
TaskLivenessShadowRules.Evidence evidence = new TaskLivenessShadowRules.Evidence(
|
||||
taskId, task.getModuleType(), now, beatMillis,
|
||||
toMillis(task.getUpdatedAt()), toMillis(task.getCreatedAt()), serverSide, progress);
|
||||
TaskLivenessShadowRules.Decision decision =
|
||||
TaskLivenessShadowRules.evaluate(evidence, thresholds);
|
||||
switch (decision.verdict()) {
|
||||
case KEEP_ADVANCING -> keepAdvancing++;
|
||||
case KEEP_QUEUED -> keepQueued++;
|
||||
case KEEP_SERVER_SIDE -> keepServerSide++;
|
||||
case UNKNOWN -> unknown++;
|
||||
case WOULD_FINALIZE -> {
|
||||
wouldFinalize++;
|
||||
ruleCounts.merge(decision.rule(), 1, Integer::sum);
|
||||
if (logged < MAX_LOGGED_WOULD_FINALIZE) {
|
||||
logWouldFinalize(evidence, decision, lastChunkAtByTaskId.get(taskId), now);
|
||||
logged++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
log.info("{} scanned={} keepAdvancing={} keepQueued={} keepServerSide={} unknown={} wouldFinalize={} byRule={} "
|
||||
+ "thresholds=beatLost{}m/noProgress{}m/grace{}m/hardCap{}h",
|
||||
LOG_ANCHOR, tasks.size(), keepAdvancing, keepQueued, keepServerSide, unknown,
|
||||
wouldFinalize, ruleCounts, beatLostMinutes, noProgressMinutes,
|
||||
firstHeartbeatGraceMinutes, hardCapHours);
|
||||
}
|
||||
|
||||
private void logWouldFinalize(TaskLivenessShadowRules.Evidence evidence,
|
||||
TaskLivenessShadowRules.Decision decision,
|
||||
LocalDateTime lastChunkAt,
|
||||
long now) {
|
||||
log.warn("{} would-finalize taskId={} module={} rule={} beatAgeMin={} queuedAgeMin={} taskAgeMin={} "
|
||||
+ "lastChunkAgeMin={} progress={} detail={}",
|
||||
LOG_ANCHOR, evidence.taskId(), evidence.moduleType(), decision.rule(),
|
||||
ageMinutesOrMissing(now, evidence.beatMillis()),
|
||||
ageMinutesOrMissing(now, evidence.queuedSinceMillis()),
|
||||
ageMinutesOrMissing(now, evidence.createdAtMillis()),
|
||||
lastChunkAt == null ? -1L : ageMinutesOrMissing(now, toMillis(lastChunkAt)),
|
||||
evidence.progress(), decision.detail());
|
||||
}
|
||||
|
||||
private Set<Long> activeJobTaskIds(List<Long> taskIds) {
|
||||
if (taskIds.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.select(TaskFileJobEntity::getTaskId)
|
||||
.in(TaskFileJobEntity::getTaskId, taskIds)
|
||||
.in(TaskFileJobEntity::getStatus, TaskStatuses.Job.ACTIVE_ORDERED));
|
||||
if (jobs == null || jobs.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<Long> result = new LinkedHashSet<>();
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
if (job != null && job.getTaskId() != null) {
|
||||
result.add(job.getTaskId());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private Map<Long, LocalDateTime> lastChunkAtByTaskId(List<Long> taskIds) {
|
||||
Map<Long, LocalDateTime> result = new HashMap<>();
|
||||
if (taskIds.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(taskIds);
|
||||
if (rows == null) {
|
||||
return result;
|
||||
}
|
||||
for (TaskScopeLastChunkDto row : rows) {
|
||||
if (row != null && row.taskId() != null && row.lastChunkAt() != null) {
|
||||
result.put(row.taskId(), row.lastChunkAt());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static TaskLivenessShadowRules.ProgressState toProgress(
|
||||
TaskHeartbeatPositionService.StallState stallState) {
|
||||
if (stallState == null) {
|
||||
return TaskLivenessShadowRules.ProgressState.UNKNOWN;
|
||||
}
|
||||
return switch (stallState) {
|
||||
case ADVANCING -> TaskLivenessShadowRules.ProgressState.ADVANCING;
|
||||
case STALLED -> TaskLivenessShadowRules.ProgressState.STALLED;
|
||||
case UNKNOWN -> TaskLivenessShadowRules.ProgressState.UNKNOWN;
|
||||
};
|
||||
}
|
||||
|
||||
private static long toMillis(LocalDateTime time) {
|
||||
return time == null ? 0L : time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
}
|
||||
|
||||
private static long ageMinutesOrMissing(long nowMillis, long sinceMillis) {
|
||||
if (sinceMillis <= 0L) {
|
||||
return -1L;
|
||||
}
|
||||
return Math.max(0L, (nowMillis - sinceMillis) / 60_000L);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user