fix(任务): 开机自回收改以本进程启动时刻为界,发版不再蚕食结果文件作业的重试次数

原判据是 now-max(60,staleSeconds),但 ApplicationReadyEvent 只在重启后几秒响一次,
那时遗留行的心跳也只停了几秒,条件结构上永不成立——线上"实例重启,作业已重新排队"累计 0 条。
遗骸改由 stuck 扫描按 +1 重试处理(今天 A 发了 7 版、重试预算只有 5),
20833 已被吃掉一次,20836/任务29475 就是被这条链判成"已达到最大重试次数"的。
This commit is contained in:
2026-09-28 15:59:47 +08:00
parent 2689fd5e1f
commit 7bb03466b0
7 changed files with 182 additions and 76 deletions
@@ -31,11 +31,11 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 开机自回收:本机 owner 名下 RUNNING 且已停止心跳的作业,在进程起来时立刻退回 PENDING。
* 开机自回收:本机 owner 名下、心跳停在本进程启动时刻之前的 RUNNING 作业,起来时立刻退回 PENDING。
*
* <p>发版/崩溃杀掉的 job 会一直挂在 RUNNING,等 60 秒一轮的 stuck 扫描按时间判死才能复位
* (阈值历史上是 30 分钟)。也就是说每次发版都会把在跑的结果文件作业冻结掉一整个阈值窗口,
* 而这批 job 其实早已无人持有。开机回收把这个等待从「阈值」压到「进程起来那一刻」。
* <p>发版/崩溃杀掉的 job 会一直挂在 RUNNING,等 60 秒一轮的 stuck 扫描按时间判死才能复位,
* 也就是说每次发版都会把在跑的结果文件作业冻结掉一整个阈值窗口,而这批 job 其实早已无人持有。
* 开机回收把这个等待从「阈值」压到「进程起来那一刻」。
*
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,
* 每发一次版扣掉它 5 次重试中的一次,反复发版会把正常任务判死。
@@ -77,15 +77,33 @@ class TaskFileJobBootReclaimTest {
return job;
}
private LambdaQueryWrapper<TaskFileJobEntity> captureQuery() {
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query =
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(taskFileJobMapper).selectList(query.capture());
return query.getValue();
}
private static LocalDateTime cutoffOf(LambdaQueryWrapper<TaskFileJobEntity> query) {
// 参数表要先把 SQL 段渲染出来才落值(MyBatis-Plus 惰性填充),否则读到空 map
query.getCustomSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Objects::nonNull)
.filter(v -> v instanceof LocalDateTime)
.map(v -> (LocalDateTime) v)
.findFirst()
.orElse(null);
}
/** 心跳已停的自家 RUNNING 作业退回 PENDING 并补发派发事件,且不得消耗 retry_count。 */
@Test
void staleOwnedRunningJobsGoBackToPendingWithoutBurningRetry() {
void abandonedOwnedRunningJobsGoBackToPendingWithoutBurningRetry() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
.thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1);
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
int reclaimed = service.reclaimOwnedRunningJobs(OWNER, 90, 20);
int reclaimed = service.reclaimOwnedRunningJobs(OWNER, LocalDateTime.now(), 20);
assertEquals(2, reclaimed);
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updates =
@@ -99,34 +117,43 @@ class TaskFileJobBootReclaimTest {
verify(applicationEventPublisher, times(2)).publishEvent(any(TaskFileJobDispatchEvent.class));
}
/** 只回收「心跳已停」的:阈值必须是 now-staleSeconds,否则会抢另一个仍在跑同 owner 的进程的作业。 */
/**
* 回归:线上遗留行的心跳只停了**几秒**(进程刚重启),旧实现把界锚在 now-90s 于是永远取不到,
* {@code 实例重启,作业已重新排队} 一条都没出现,遗骸全交给 stuck 扫描按 +1 重试判死。
* 界必须是调用方传进来的那个时刻(本进程启动时刻),原样使用、不得再加任何 staleness 下限。
*/
@Test
void onlyJobsStalerThanTheThresholdAreCandidates() {
void cutoffIsUsedVerbatimSoSecondsOldOrphansAreStillReclaimed() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
LocalDateTime before = LocalDateTime.now();
LocalDateTime jvmStartedAt = LocalDateTime.now().minusSeconds(12);
service.reclaimOwnedRunningJobs(OWNER, 90, 20);
service.reclaimOwnedRunningJobs(OWNER, jvmStartedAt, 20);
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(taskFileJobMapper).selectList(query.capture());
String segment = query.getValue().getCustomSqlSegment();
assertTrue(segment.contains("owner = "), "必须只回收本机 owner: " + segment);
assertTrue(segment.contains("status"), "必须只回收 RUNNING: " + segment);
assertTrue(segment.contains("updated_at"), "必须按心跳时间设阈值: " + segment);
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
.filter(v -> v instanceof LocalDateTime)
.map(v -> (LocalDateTime) v)
.findFirst()
.orElse(null);
assertTrue(cutoff != null && !cutoff.isAfter(before.minusSeconds(90)),
"阈值不得晚于 now-90s,否则会抢仍在心跳的作业: " + cutoff);
LocalDateTime cutoff = cutoffOf(captureQuery());
assertEquals(jvmStartedAt, cutoff,
"界必须等于本进程启动时刻;一旦掺进 now-N 秒的下限,重启后几秒的遗骸就永远筛不出来");
}
/** 没有 owner 就没有"本机"可言:不查库、不改写,交给通用 stuck 扫描。 */
/** 只回收「心跳停在本进程启动之前」的:界之后的行是本进程自己派发出去的,不能抢。 */
@Test
void blankOwnerReclaimsNothing() {
assertEquals(0, service.reclaimOwnedRunningJobs(" ", 90, 20));
assertEquals(0, service.reclaimOwnedRunningJobs(null, 90, 20));
void queryIsScopedToOwnerRunningAndBoundedByCutoff() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
service.reclaimOwnedRunningJobs(OWNER, LocalDateTime.now(), 20);
LambdaQueryWrapper<TaskFileJobEntity> query = captureQuery();
String segment = query.getCustomSqlSegment();
assertTrue(segment.contains("owner = "), "必须只回收本机 owner: " + segment);
assertTrue(segment.contains("status"), "必须只回收 RUNNING: " + segment);
assertTrue(segment.contains("updated_at <"), "必须按界筛(严格小于,等于界的留给本进程): " + segment);
}
/** 没有 owner 或没有界就没有"本机遗骸"可言:不查库、不改写,交给通用 stuck 扫描。 */
@Test
void blankOwnerOrMissingCutoffReclaimsNothing() {
assertEquals(0, service.reclaimOwnedRunningJobs(" ", LocalDateTime.now(), 20));
assertEquals(0, service.reclaimOwnedRunningJobs(null, LocalDateTime.now(), 20));
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, null, 20));
verify(taskFileJobMapper, times(0)).selectList(any(LambdaQueryWrapper.class));
verify(taskFileJobMapper, times(0)).update(isNull(), any(LambdaUpdateWrapper.class));
@@ -138,7 +165,7 @@ class TaskFileJobBootReclaimTest {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(abandonedJob(3L)));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, 90, 20));
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, LocalDateTime.now(), 20));
verify(applicationEventPublisher, times(0)).publishEvent(any(TaskFileJobDispatchEvent.class));
}
@@ -147,36 +174,17 @@ class TaskFileJobBootReclaimTest {
@Test
void limitIsClamped() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
LocalDateTime cutoff = LocalDateTime.now();
service.reclaimOwnedRunningJobs(OWNER, 90, 0);
service.reclaimOwnedRunningJobs(OWNER, 90, 5000);
service.reclaimOwnedRunningJobs(OWNER, cutoff, 0);
service.reclaimOwnedRunningJobs(OWNER, cutoff, 5000);
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query =
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(taskFileJobMapper, times(2)).selectList(query.capture());
List<String> segments = query.getAllValues().stream()
.map(LambdaQueryWrapper::getCustomSqlSegment).toList();
assertTrue(segments.get(0).contains("limit 1"), "limit=0 回退到 1: " + segments.get(0));
assertTrue(segments.get(1).contains("limit 100"), "超大 limit 钳制到 100: " + segments.get(1));
}
/** staleSeconds 给得太小时取不到任何作业:至少留出一个心跳周期的余量。 */
@Test
void staleThresholdNeverBelowHeartbeatInterval() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
service.reclaimOwnedRunningJobs(OWNER, 1, 20);
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(taskFileJobMapper).selectList(query.capture());
// 参数表要先把 SQL 段渲染出来才落值(MyBatis-Plus 惰性填充),否则读到空 map
query.getValue().getCustomSqlSegment();
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
.filter(Objects::nonNull)
.filter(v -> v instanceof LocalDateTime)
.map(v -> (LocalDateTime) v)
.findFirst()
.orElse(null);
assertTrue(cutoff != null && cutoff.isBefore(LocalDateTime.now().minusSeconds(30)),
"阈值过短会把仍在正常心跳的作业抢走: " + cutoff);
}
}
@@ -38,15 +38,17 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -280,7 +282,10 @@ class TaskResultFileJobWorkerStuckScanTest {
String config = new String(Files.readAllBytes(yml), StandardCharsets.UTF_8);
// 运行期真正生效的是 yml 的兜底值,只改 @Value 默认等于没改
assertTrue(config.contains("STUCK_TIMEOUT_MINUTES:5}"), "application.yml 卡死超时 5 分钟: " + config);
assertTrue(config.contains("BOOT_RECLAIM_STALE_SECONDS:90}"), "开机回收阈值可在 env 覆盖");
// 开机回收的界改锚在"本进程启动时刻"上;这个可调项必须彻底消失,
// 否则谁把它调大一点,重启遗骸就又筛不出来了(线上就是因为它而一条都没回收过)。
assertFalse(config.contains("BOOT_RECLAIM_STALE_SECONDS"), "开机回收 staleness 可调项应已删除: " + config);
assertFalse(content.contains("boot-reclaim-stale-seconds"), "worker 不应再读 staleness 配置");
}
@Test
@@ -298,17 +303,22 @@ class TaskResultFileJobWorkerStuckScanTest {
verify(taskFileJobService).markFailureFinalized(9L, "文件生成任务运行超时,已达到最大重试次数");
}
/** 开机回收必须只针对"本机 instanceId 名下"的遗留作业,否则会抢另一台实例正在跑的 job。 */
/**
* 开机回收必须只针对"本机 instanceId 名下、且心跳停在本进程启动之前"的遗留作业。
* 界必须是 InstanceMetadata.startedAt(进程启动时刻),不能是 now-N 秒:
* 本事件只在重启后几秒响一次,那时遗骸的心跳也只停了几秒,now-N 永远筛不出它们。
*/
@Test
void bootReclaimTargetsOwnAbandonedJobs() throws Exception {
void bootReclaimTargetsOwnAbandonedJobsSinceProcessStart() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
LocalDateTime jvmStartedAt = LocalDateTime.now().minusSeconds(37);
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
when(instanceMetadata.getStartedAt()).thenReturn(jvmStartedAt);
setInt(worker, "batchSize", 20);
setLong(worker, "bootReclaimStaleSeconds", 90L);
worker.reclaimOwnedJobsOnBoot();
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 90L, 20);
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", jvmStartedAt, 20);
}
/** 回收只是加速恢复,DB 抽风时绝不能把应用启动一起带下水。 */
@@ -316,12 +326,13 @@ class TaskResultFileJobWorkerStuckScanTest {
void bootReclaimFailureDoesNotBreakStartup() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
when(taskFileJobService.reclaimOwnedRunningJobs(anyString(), anyLong(), anyInt()))
when(instanceMetadata.getStartedAt()).thenReturn(LocalDateTime.now());
when(taskFileJobService.reclaimOwnedRunningJobs(anyString(), any(LocalDateTime.class), anyInt()))
.thenThrow(new IllegalStateException("db down"));
worker.reclaimOwnedJobsOnBoot();
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 0L, 0);
verify(taskFileJobService).reclaimOwnedRunningJobs(eq("java-server"), any(LocalDateTime.class), eq(0));
}
/**