fix(任务): 开机自回收改以本进程启动时刻为界,发版不再蚕食结果文件作业的重试次数
原判据是 now-max(60,staleSeconds),但 ApplicationReadyEvent 只在重启后几秒响一次, 那时遗留行的心跳也只停了几秒,条件结构上永不成立——线上"实例重启,作业已重新排队"累计 0 条。 遗骸改由 stuck 扫描按 +1 重试处理(今天 A 发了 7 版、重试预算只有 5), 20833 已被吃掉一次,20836/任务29475 就是被这条链判成"已达到最大重试次数"的。
This commit is contained in:
@@ -4,7 +4,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.net.InetAddress;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
@@ -14,8 +18,16 @@ public class InstanceMetadata {
|
||||
private final String hostname;
|
||||
private final String source;
|
||||
private final boolean stable;
|
||||
/**
|
||||
* 本进程的启动时刻。开机自回收用它当界:owner=本机且 status=RUNNING 的行若
|
||||
* {@code updated_at} 早于这一刻,只可能是上一个进程留下的遗骸,不可能是本进程在跑的。
|
||||
*/
|
||||
private final LocalDateTime startedAt;
|
||||
|
||||
public InstanceMetadata(@Value("${aiimage.instance-id:}") String configuredInstanceId) {
|
||||
this.startedAt = LocalDateTime.ofInstant(
|
||||
Instant.ofEpochMilli(ManagementFactory.getRuntimeMXBean().getStartTime()),
|
||||
ZoneId.systemDefault());
|
||||
this.hostname = resolveHostname();
|
||||
String normalizedConfiguredInstanceId = configuredInstanceId == null ? "" : configuredInstanceId.trim();
|
||||
if (!normalizedConfiguredInstanceId.isBlank()) {
|
||||
@@ -51,6 +63,10 @@ public class InstanceMetadata {
|
||||
return stable;
|
||||
}
|
||||
|
||||
public LocalDateTime getStartedAt() {
|
||||
return startedAt;
|
||||
}
|
||||
|
||||
private static String resolveHostname() {
|
||||
try {
|
||||
return InetAddress.getLocalHost().getHostName();
|
||||
|
||||
+11
-7
@@ -454,8 +454,12 @@ public class TaskFileJobService {
|
||||
* 冻结一整个阈值窗口,而这批行其实早已无人持有。进程刚起来时本机不可能还在跑它们,
|
||||
* 所以这一刻就能直接接回来,把等待从「阈值」压到「启动」。
|
||||
*
|
||||
* <p>两道保险:只取 owner 等值等于自己的行(绝不碰另一台实例在跑的),且必须已停止心跳
|
||||
* 一个完整周期以上(正常心跳是 60 秒一次,误抢会让在跑的作业被双份派发)。
|
||||
* <p>两道保险:只取 owner 等值等于自己的行(绝不碰另一台实例在跑的),且只取
|
||||
* {@code updated_at} 早于本进程启动时刻的行——旧进程已经退出,它留下的行必然停在这个时刻之前,
|
||||
* 而本进程自己派发的行只会晚于它。用「本进程启动」而不是「距今 N 秒」做界是必须的:
|
||||
* 开机事件发生在重启后几秒,那时遗留行的心跳也只停了几天秒,按固定 staleness 阈值
|
||||
* (曾经是 60/90 秒)**永远取不到任何行**,回收等于不存在,只能等 stuck 扫描按 +1 重试判死。
|
||||
* updated_at 全部由应用侧 {@code LocalDateTime.now()} 写入(不是 DB 时钟),与本界同源,不存在跨机时钟偏移。
|
||||
*
|
||||
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,每发一次版
|
||||
* 吃掉它那 5 次重试中的一次,反复发版会把正常任务判成"已达到最大重试次数"。
|
||||
@@ -463,12 +467,12 @@ public class TaskFileJobService {
|
||||
* <p>刻意不加事务,与 {@link #resetStuckRunningJobsDetailed} 同理:每行都带
|
||||
* status + updated_at 的 CAS 条件,逐行独立提交,锁即时释放。
|
||||
*/
|
||||
public int reclaimOwnedRunningJobs(String owner, long staleSeconds, int limit) {
|
||||
public int reclaimOwnedRunningJobs(String owner, LocalDateTime orphanedBefore, int limit) {
|
||||
String normalizedOwner = owner == null ? "" : owner.trim();
|
||||
if (normalizedOwner.isBlank()) {
|
||||
if (normalizedOwner.isBlank() || orphanedBefore == null) {
|
||||
return 0;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusSeconds(Math.max(60L, staleSeconds));
|
||||
LocalDateTime cutoff = orphanedBefore;
|
||||
int safeLimit = Math.max(1, Math.min(limit, 100));
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
@@ -503,8 +507,8 @@ public class TaskFileJobService {
|
||||
for (Long jobId : resetIds) {
|
||||
publishDispatchEvent(refreshedById.get(jobId));
|
||||
}
|
||||
log.warn("[task-file-job] 开机自回收本机遗留作业 count={} owner={} staleSeconds={} limit={}",
|
||||
resetIds.size(), normalizedOwner, staleSeconds, safeLimit);
|
||||
log.warn("[task-file-job] 开机自回收本机遗留作业 count={} owner={} cutoff={} limit={}",
|
||||
resetIds.size(), normalizedOwner, cutoff, safeLimit);
|
||||
return resetIds.size();
|
||||
}
|
||||
|
||||
|
||||
+10
-7
@@ -18,6 +18,7 @@ import org.springframework.core.task.TaskExecutor;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
@@ -57,10 +58,6 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
|
||||
private long heartbeatIntervalMillis = 60000L;
|
||||
|
||||
/** 开机回收的"心跳已停"判据:默认 90 秒 ≈ 1.5 个心跳周期,容得下一次正常心跳延迟。 */
|
||||
@Value("${aiimage.result-file-job.boot-reclaim-stale-seconds:90}")
|
||||
private long bootReclaimStaleSeconds;
|
||||
|
||||
/** 多个结果文件 Job 并行运行时,心跳不能被单个 DB 慢调用串行阻塞。 */
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
|
||||
@@ -119,16 +116,22 @@ public class TaskResultFileJobWorker {
|
||||
* 本机刚起来,这些 owner=自己的 RUNNING 行不可能还在本机执行,所以此刻就能直接接回来,
|
||||
* 不必等阈值;回收不消耗 retry_count(发版不是作业的错)。
|
||||
*
|
||||
* <p>判据用「本进程启动时刻」而不是「距今 N 秒」:开机事件只在重启后几秒响一次,
|
||||
* 那时刻遗留行的心跳也只停了几秒,按固定 staleness 阈值永远筛不出它们。曾经就是这个样子,
|
||||
* 结果线上 {@code 实例重启,作业已重新排队} 一条都没有,遗骸全部由 stuck 扫描按 +1 重试处理,
|
||||
* 反复发版把还能补传的组装作业推上"已达到最大重试次数"。
|
||||
*
|
||||
* <p>只处理 owner 等于本机实例 id 的行,另一台实例在跑的作业不受影响。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void reclaimOwnedJobsOnBoot() {
|
||||
try {
|
||||
LocalDateTime jvmStartedAt = instanceMetadata == null ? null : instanceMetadata.getStartedAt();
|
||||
int reclaimed = taskFileJobService.reclaimOwnedRunningJobs(
|
||||
currentInstanceId(), bootReclaimStaleSeconds, batchSize);
|
||||
currentInstanceId(), jvmStartedAt, batchSize);
|
||||
if (reclaimed > 0) {
|
||||
log.warn("[task-file-job] booted and reclaimed abandoned jobs of this instance reclaimed={} instanceId={} staleSeconds={}",
|
||||
reclaimed, currentInstanceId(), bootReclaimStaleSeconds);
|
||||
log.warn("[task-file-job] booted and reclaimed abandoned jobs of this instance reclaimed={} instanceId={} cutoff={}",
|
||||
reclaimed, currentInstanceId(), jvmStartedAt);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// 回收只是加速恢复;失败不影响启动,兜底仍是 stuck 扫描。
|
||||
|
||||
@@ -281,8 +281,6 @@ aiimage:
|
||||
# 作业运行期间每 60s touch 一次 updated_at,所以"5 分钟没心跳"就等于持有者已经没了;
|
||||
# 30 分钟是发版/容器重建后结果文件作业被白白冻结的时长。
|
||||
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:5}
|
||||
# 开机自回收的"已停止心跳"判据(≈1.5 个心跳周期),只回收 owner=本机 的 RUNNING 作业。
|
||||
boot-reclaim-stale-seconds: ${AIIMAGE_RESULT_FILE_JOB_BOOT_RECLAIM_STALE_SECONDS:90}
|
||||
heartbeat-interval-ms: ${AIIMAGE_RESULT_FILE_JOB_HEARTBEAT_INTERVAL_MS:60000}
|
||||
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
|
||||
# 客户端兜底拉取:页面没推送成功(只解析没点启动 / 推送前关页面 / 纯浏览器打开)、
|
||||
|
||||
+60
-52
@@ -31,11 +31,11 @@ import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 开机自回收:本机 owner 名下 RUNNING 且已停止心跳的作业,在进程起来时立刻退回 PENDING。
|
||||
* 开机自回收:本机 owner 名下、心跳停在本进程启动时刻之前的 RUNNING 作业,起来时立刻退回 PENDING。
|
||||
*
|
||||
* <p>发版/崩溃杀掉的 job 会一直挂在 RUNNING,等 60 秒一轮的 stuck 扫描按时间判死才能复位
|
||||
* (阈值历史上是 30 分钟)。也就是说每次发版都会把在跑的结果文件作业冻结掉一整个阈值窗口,
|
||||
* 而这批 job 其实早已无人持有。开机回收把这个等待从「阈值」压到「进程起来那一刻」。
|
||||
* <p>发版/崩溃杀掉的 job 会一直挂在 RUNNING,等 60 秒一轮的 stuck 扫描按时间判死才能复位,
|
||||
* 也就是说每次发版都会把在跑的结果文件作业冻结掉一整个阈值窗口,而这批 job 其实早已无人持有。
|
||||
* 开机回收把这个等待从「阈值」压到「进程起来那一刻」。
|
||||
*
|
||||
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,
|
||||
* 每发一次版扣掉它 5 次重试中的一次,反复发版会把正常任务判死。
|
||||
@@ -77,15 +77,33 @@ class TaskFileJobBootReclaimTest {
|
||||
return job;
|
||||
}
|
||||
|
||||
private LambdaQueryWrapper<TaskFileJobEntity> captureQuery() {
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query =
|
||||
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper).selectList(query.capture());
|
||||
return query.getValue();
|
||||
}
|
||||
|
||||
private static LocalDateTime cutoffOf(LambdaQueryWrapper<TaskFileJobEntity> query) {
|
||||
// 参数表要先把 SQL 段渲染出来才落值(MyBatis-Plus 惰性填充),否则读到空 map
|
||||
query.getCustomSqlSegment();
|
||||
return query.getParamNameValuePairs().values().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(v -> v instanceof LocalDateTime)
|
||||
.map(v -> (LocalDateTime) v)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
/** 心跳已停的自家 RUNNING 作业退回 PENDING 并补发派发事件,且不得消耗 retry_count。 */
|
||||
@Test
|
||||
void staleOwnedRunningJobsGoBackToPendingWithoutBurningRetry() {
|
||||
void abandonedOwnedRunningJobsGoBackToPendingWithoutBurningRetry() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
|
||||
|
||||
int reclaimed = service.reclaimOwnedRunningJobs(OWNER, 90, 20);
|
||||
int reclaimed = service.reclaimOwnedRunningJobs(OWNER, LocalDateTime.now(), 20);
|
||||
|
||||
assertEquals(2, reclaimed);
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updates =
|
||||
@@ -99,34 +117,43 @@ class TaskFileJobBootReclaimTest {
|
||||
verify(applicationEventPublisher, times(2)).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
|
||||
/** 只回收「心跳已停」的:阈值必须是 now-staleSeconds,否则会抢另一个仍在跑同 owner 的进程的作业。 */
|
||||
/**
|
||||
* 回归:线上遗留行的心跳只停了**几秒**(进程刚重启),旧实现把界锚在 now-90s 于是永远取不到,
|
||||
* {@code 实例重启,作业已重新排队} 一条都没出现,遗骸全交给 stuck 扫描按 +1 重试判死。
|
||||
* 界必须是调用方传进来的那个时刻(本进程启动时刻),原样使用、不得再加任何 staleness 下限。
|
||||
*/
|
||||
@Test
|
||||
void onlyJobsStalerThanTheThresholdAreCandidates() {
|
||||
void cutoffIsUsedVerbatimSoSecondsOldOrphansAreStillReclaimed() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
LocalDateTime before = LocalDateTime.now();
|
||||
LocalDateTime jvmStartedAt = LocalDateTime.now().minusSeconds(12);
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 20);
|
||||
service.reclaimOwnedRunningJobs(OWNER, jvmStartedAt, 20);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper).selectList(query.capture());
|
||||
String segment = query.getValue().getCustomSqlSegment();
|
||||
assertTrue(segment.contains("owner = "), "必须只回收本机 owner: " + segment);
|
||||
assertTrue(segment.contains("status"), "必须只回收 RUNNING: " + segment);
|
||||
assertTrue(segment.contains("updated_at"), "必须按心跳时间设阈值: " + segment);
|
||||
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
|
||||
.filter(v -> v instanceof LocalDateTime)
|
||||
.map(v -> (LocalDateTime) v)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
assertTrue(cutoff != null && !cutoff.isAfter(before.minusSeconds(90)),
|
||||
"阈值不得晚于 now-90s,否则会抢仍在心跳的作业: " + cutoff);
|
||||
LocalDateTime cutoff = cutoffOf(captureQuery());
|
||||
assertEquals(jvmStartedAt, cutoff,
|
||||
"界必须等于本进程启动时刻;一旦掺进 now-N 秒的下限,重启后几秒的遗骸就永远筛不出来");
|
||||
}
|
||||
|
||||
/** 没有 owner 就没有"本机"可言:不查库、不改写,交给通用 stuck 扫描。 */
|
||||
/** 只回收「心跳停在本进程启动之前」的:界之后的行是本进程自己派发出去的,不能抢。 */
|
||||
@Test
|
||||
void blankOwnerReclaimsNothing() {
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(" ", 90, 20));
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(null, 90, 20));
|
||||
void queryIsScopedToOwnerRunningAndBoundedByCutoff() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, LocalDateTime.now(), 20);
|
||||
|
||||
LambdaQueryWrapper<TaskFileJobEntity> query = captureQuery();
|
||||
String segment = query.getCustomSqlSegment();
|
||||
assertTrue(segment.contains("owner = "), "必须只回收本机 owner: " + segment);
|
||||
assertTrue(segment.contains("status"), "必须只回收 RUNNING: " + segment);
|
||||
assertTrue(segment.contains("updated_at <"), "必须按界筛(严格小于,等于界的留给本进程): " + segment);
|
||||
}
|
||||
|
||||
/** 没有 owner 或没有界就没有"本机遗骸"可言:不查库、不改写,交给通用 stuck 扫描。 */
|
||||
@Test
|
||||
void blankOwnerOrMissingCutoffReclaimsNothing() {
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(" ", LocalDateTime.now(), 20));
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(null, LocalDateTime.now(), 20));
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, null, 20));
|
||||
|
||||
verify(taskFileJobMapper, times(0)).selectList(any(LambdaQueryWrapper.class));
|
||||
verify(taskFileJobMapper, times(0)).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
@@ -138,7 +165,7 @@ class TaskFileJobBootReclaimTest {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(abandonedJob(3L)));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, 90, 20));
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, LocalDateTime.now(), 20));
|
||||
|
||||
verify(applicationEventPublisher, times(0)).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
@@ -147,36 +174,17 @@ class TaskFileJobBootReclaimTest {
|
||||
@Test
|
||||
void limitIsClamped() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
LocalDateTime cutoff = LocalDateTime.now();
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 0);
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 5000);
|
||||
service.reclaimOwnedRunningJobs(OWNER, cutoff, 0);
|
||||
service.reclaimOwnedRunningJobs(OWNER, cutoff, 5000);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query =
|
||||
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper, times(2)).selectList(query.capture());
|
||||
List<String> segments = query.getAllValues().stream()
|
||||
.map(LambdaQueryWrapper::getCustomSqlSegment).toList();
|
||||
assertTrue(segments.get(0).contains("limit 1"), "limit=0 回退到 1: " + segments.get(0));
|
||||
assertTrue(segments.get(1).contains("limit 100"), "超大 limit 钳制到 100: " + segments.get(1));
|
||||
}
|
||||
|
||||
/** staleSeconds 给得太小时取不到任何作业:至少留出一个心跳周期的余量。 */
|
||||
@Test
|
||||
void staleThresholdNeverBelowHeartbeatInterval() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 1, 20);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper).selectList(query.capture());
|
||||
// 参数表要先把 SQL 段渲染出来才落值(MyBatis-Plus 惰性填充),否则读到空 map
|
||||
query.getValue().getCustomSqlSegment();
|
||||
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(v -> v instanceof LocalDateTime)
|
||||
.map(v -> (LocalDateTime) v)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
assertTrue(cutoff != null && cutoff.isBefore(LocalDateTime.now().minusSeconds(30)),
|
||||
"阈值过短会把仍在正常心跳的作业抢走: " + cutoff);
|
||||
}
|
||||
}
|
||||
|
||||
+19
-8
@@ -38,15 +38,17 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
@@ -280,7 +282,10 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
String config = new String(Files.readAllBytes(yml), StandardCharsets.UTF_8);
|
||||
// 运行期真正生效的是 yml 的兜底值,只改 @Value 默认等于没改
|
||||
assertTrue(config.contains("STUCK_TIMEOUT_MINUTES:5}"), "application.yml 卡死超时 5 分钟: " + config);
|
||||
assertTrue(config.contains("BOOT_RECLAIM_STALE_SECONDS:90}"), "开机回收阈值可在 env 覆盖");
|
||||
// 开机回收的界改锚在"本进程启动时刻"上;这个可调项必须彻底消失,
|
||||
// 否则谁把它调大一点,重启遗骸就又筛不出来了(线上就是因为它而一条都没回收过)。
|
||||
assertFalse(config.contains("BOOT_RECLAIM_STALE_SECONDS"), "开机回收 staleness 可调项应已删除: " + config);
|
||||
assertFalse(content.contains("boot-reclaim-stale-seconds"), "worker 不应再读 staleness 配置");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -298,17 +303,22 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
verify(taskFileJobService).markFailureFinalized(9L, "文件生成任务运行超时,已达到最大重试次数");
|
||||
}
|
||||
|
||||
/** 开机回收必须只针对"本机 instanceId 名下"的遗留作业,否则会抢另一台实例正在跑的 job。 */
|
||||
/**
|
||||
* 开机回收必须只针对"本机 instanceId 名下、且心跳停在本进程启动之前"的遗留作业。
|
||||
* 界必须是 InstanceMetadata.startedAt(进程启动时刻),不能是 now-N 秒:
|
||||
* 本事件只在重启后几秒响一次,那时遗骸的心跳也只停了几秒,now-N 永远筛不出它们。
|
||||
*/
|
||||
@Test
|
||||
void bootReclaimTargetsOwnAbandonedJobs() throws Exception {
|
||||
void bootReclaimTargetsOwnAbandonedJobsSinceProcessStart() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
LocalDateTime jvmStartedAt = LocalDateTime.now().minusSeconds(37);
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
|
||||
when(instanceMetadata.getStartedAt()).thenReturn(jvmStartedAt);
|
||||
setInt(worker, "batchSize", 20);
|
||||
setLong(worker, "bootReclaimStaleSeconds", 90L);
|
||||
|
||||
worker.reclaimOwnedJobsOnBoot();
|
||||
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 90L, 20);
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", jvmStartedAt, 20);
|
||||
}
|
||||
|
||||
/** 回收只是加速恢复,DB 抽风时绝不能把应用启动一起带下水。 */
|
||||
@@ -316,12 +326,13 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
void bootReclaimFailureDoesNotBreakStartup() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
|
||||
when(taskFileJobService.reclaimOwnedRunningJobs(anyString(), anyLong(), anyInt()))
|
||||
when(instanceMetadata.getStartedAt()).thenReturn(LocalDateTime.now());
|
||||
when(taskFileJobService.reclaimOwnedRunningJobs(anyString(), any(LocalDateTime.class), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
worker.reclaimOwnedJobsOnBoot();
|
||||
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 0L, 0);
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs(eq("java-server"), any(LocalDateTime.class), eq(0));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user