fix(task): 补检中的任务不再自动克隆重跑,卡死重排不消耗重试预算

- TaskResumeService:名下还有 PENDING/RUNNING 组装作业的 FAILED 任务不克隆
  (现网 2026-10-24 一枚 760 行外观专利被连克隆 3 代,每代全量重跑、同秒显示)
- resetStuckRunningJobsDetailed:RUNNING 行重排不再 +1 retry_count,判死只剩
  markFailed 一条真失败的路——补检一轮几十分钟的慢执行不再被记成失败次数
- 新增 hasUnfinishedAssembleJob;测试按新口径重写并补现网回归两例
This commit is contained in:
2026-10-24 16:35:23 +08:00
parent ccb30aa99e
commit 5f1dc3ac23
4 changed files with 110 additions and 34 deletions
@@ -44,9 +44,9 @@ class TaskFileJobServiceTest {
}
@Test
void stuckJobIsRequeuedAndRetryCountIsIncremented() {
void stuckJobIsRequeuedWithoutConsumingRetryBudget() {
TaskFileJobEntity running = runningJob(101L, 3, LocalDateTime.now().minusHours(1));
TaskFileJobEntity pending = runningJob(101L, 4, LocalDateTime.now());
TaskFileJobEntity pending = runningJob(101L, 3, LocalDateTime.now());
pending.setStatus("PENDING");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
@@ -58,29 +58,29 @@ class TaskFileJobServiceTest {
assertEquals(1, result.resetCount());
assertTrue(result.exhaustedJobs().isEmpty());
verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class));
assertUpdateContains(4, running.getUpdatedAt());
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
}
/**
* 现网 2026-10-24:一枚 760 行的外观专利组装作业在补检轮里被卡死扫描连扣预算,
* RUNNING + retryCount=4 一进扫描就被就地判死「文件生成任务运行超时,已达到最大重试次数」,
* 结果文件 0 字节收场。改口径后扫描只放回队列:到达 MAX 的终态只能由真失败(markFailed)走出。
*/
@Test
void stuckJobAtLastRetryWaitsForDurableFailureFinalization() {
void stuckJobAtLastRetryIsRequeuedWithoutTerminalKill() {
TaskFileJobEntity running = runningJob(102L, 4, LocalDateTime.now().minusHours(1));
TaskFileJobEntity failed = runningJob(102L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
failed.setStatus("FAILED");
/*
failed.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
*/
failed.setErrorMessage("result file job timeout");
TaskFileJobEntity pending = runningJob(102L, 4, LocalDateTime.now());
pending.setStatus("PENDING");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(failed));
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(pending));
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(0, result.resetCount());
assertEquals(List.of(failed), result.exhaustedJobs());
verify(applicationEventPublisher, never()).publishEvent(any());
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
assertEquals(1, result.resetCount());
assertTrue(result.exhaustedJobs().isEmpty());
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
}
@Test
@@ -125,7 +125,7 @@ class TaskFileJobServiceTest {
assertTrue(result.exhaustedJobs().isEmpty());
verify(taskFileJobMapper, never()).selectById(any());
verify(applicationEventPublisher, never()).publishEvent(any());
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
}
@Test
@@ -308,6 +308,16 @@ class TaskFileJobServiceTest {
assertTrue(update.getValue().getParamNameValuePairs().containsValue(nextRetry));
}
/** 卡死重排的分账口径:SET 片段里不许出现 retry_count,预算只由真失败(markFailed)推进。 */
private void assertUpdateLeavesRetryCountIntact(LocalDateTime updatedAt) {
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getSqlSegment().contains("updated_at"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue(updatedAt));
assertFalse(update.getValue().getSqlSet().contains("retry_count"),
"卡死重排不许碰 retry_count");
}
@SuppressWarnings({"rawtypes", "unchecked"})
private static ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updateCaptor() {
return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class);
@@ -60,7 +60,15 @@ class TaskResumeServiceTest {
private static TaskResumeService service(FileTaskMapper mapper, List<ClientTaskRetrySpi> handlers,
boolean enabled, int maxAttempt) {
TaskResumeService service = new TaskResumeService(mapper, handlers);
// 默认"名下没有活着的组装作业"(Mockito 对 boolean 返 false):既有用例判的都是克隆本体,
// 补检豁免那条单独用 serviceWithLiveAssembleJob 覆盖。
return service(mapper, mock(TaskFileJobService.class), handlers, enabled, maxAttempt);
}
private static TaskResumeService service(FileTaskMapper mapper, TaskFileJobService jobs,
List<ClientTaskRetrySpi> handlers,
boolean enabled, int maxAttempt) {
TaskResumeService service = new TaskResumeService(mapper, jobs, handlers);
ReflectionTestUtils.setField(service, "enabled", enabled);
ReflectionTestUtils.setField(service, "maxAttempt", maxAttempt);
ReflectionTestUtils.setField(service, "windowMinutes", 30L);
@@ -138,6 +146,32 @@ class TaskResumeServiceTest {
verify(spi, never()).cloneForRetry(any());
}
/**
* 现网 2026-10-24:一枚 760 行的外观专利任务因交付缺口把组装作业排回补检(作业还在
* PENDING/RUNNING),任务行却已是 FAILED,于是被巡检连克隆成 4 枚、每枚全量重跑一遍,
* 四枚卡片开始时间还同秒(克隆件继承原件 created_at)。非零对照=上面那条
* 「失败任务会被克隆重试」:作业已终态时照旧克隆。
*/
@Test
void 组装作业仍在补检时不克隆() {
FileTaskMapper mapper = mock(FileTaskMapper.class);
when(mapper.selectList(any())).thenReturn(List.of(
failedTask(32201L, "APPEARANCE_PATENT", "有 3 行抓取失败(未取得商品页数据):B07P5TR7Q5")));
when(mapper.selectCount(any())).thenReturn(0L);
TaskFileJobService jobs = mock(TaskFileJobService.class);
when(jobs.hasUnfinishedAssembleJob(32201L)).thenReturn(true);
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
when(spi.moduleType()).thenReturn("APPEARANCE_PATENT");
TaskResumeService service = service(mapper, jobs, List.of(spi), true, 3);
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
assertEquals(1, stats.scannedTaskCount);
assertEquals(1, stats.skippedTaskCount);
assertEquals(0, stats.resumedTaskCount);
verify(spi, never()).cloneForRetry(any());
}
@Test
void 重试代数到上限后跳过() {
FileTaskMapper mapper = mock(FileTaskMapper.class);