fix(task): 补检中的任务不再自动克隆重跑,卡死重排不消耗重试预算
- TaskResumeService:名下还有 PENDING/RUNNING 组装作业的 FAILED 任务不克隆 (现网 2026-10-24 一枚 760 行外观专利被连克隆 3 代,每代全量重跑、同秒显示) - resetStuckRunningJobsDetailed:RUNNING 行重排不再 +1 retry_count,判死只剩 markFailed 一条真失败的路——补检一轮几十分钟的慢执行不再被记成失败次数 - 新增 hasUnfinishedAssembleJob;测试按新口径重写并补现网回归两例
This commit is contained in:
+26
-16
@@ -44,9 +44,9 @@ class TaskFileJobServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckJobIsRequeuedAndRetryCountIsIncremented() {
|
||||
void stuckJobIsRequeuedWithoutConsumingRetryBudget() {
|
||||
TaskFileJobEntity running = runningJob(101L, 3, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity pending = runningJob(101L, 4, LocalDateTime.now());
|
||||
TaskFileJobEntity pending = runningJob(101L, 3, LocalDateTime.now());
|
||||
pending.setStatus("PENDING");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
@@ -58,29 +58,29 @@ class TaskFileJobServiceTest {
|
||||
assertEquals(1, result.resetCount());
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
assertUpdateContains(4, running.getUpdatedAt());
|
||||
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
|
||||
}
|
||||
|
||||
/**
|
||||
* 现网 2026-10-24:一枚 760 行的外观专利组装作业在补检轮里被卡死扫描连扣预算,
|
||||
* RUNNING + retryCount=4 一进扫描就被就地判死「文件生成任务运行超时,已达到最大重试次数」,
|
||||
* 结果文件 0 字节收场。改口径后扫描只放回队列:到达 MAX 的终态只能由真失败(markFailed)走出。
|
||||
*/
|
||||
@Test
|
||||
void stuckJobAtLastRetryWaitsForDurableFailureFinalization() {
|
||||
void stuckJobAtLastRetryIsRequeuedWithoutTerminalKill() {
|
||||
TaskFileJobEntity running = runningJob(102L, 4, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity failed = runningJob(102L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
failed.setStatus("FAILED");
|
||||
/*
|
||||
failed.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
|
||||
*/
|
||||
failed.setErrorMessage("result file job timeout");
|
||||
TaskFileJobEntity pending = runningJob(102L, 4, LocalDateTime.now());
|
||||
pending.setStatus("PENDING");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(failed));
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(pending));
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(0, result.resetCount());
|
||||
assertEquals(List.of(failed), result.exhaustedJobs());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
|
||||
assertEquals(1, result.resetCount());
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -125,7 +125,7 @@ class TaskFileJobServiceTest {
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
verify(taskFileJobMapper, never()).selectById(any());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
|
||||
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -308,6 +308,16 @@ class TaskFileJobServiceTest {
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(nextRetry));
|
||||
}
|
||||
|
||||
/** 卡死重排的分账口径:SET 片段里不许出现 retry_count,预算只由真失败(markFailed)推进。 */
|
||||
private void assertUpdateLeavesRetryCountIntact(LocalDateTime updatedAt) {
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSegment().contains("updated_at"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(updatedAt));
|
||||
assertFalse(update.getValue().getSqlSet().contains("retry_count"),
|
||||
"卡死重排不许碰 retry_count");
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
private static ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updateCaptor() {
|
||||
return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class);
|
||||
|
||||
+35
-1
@@ -60,7 +60,15 @@ class TaskResumeServiceTest {
|
||||
|
||||
private static TaskResumeService service(FileTaskMapper mapper, List<ClientTaskRetrySpi> handlers,
|
||||
boolean enabled, int maxAttempt) {
|
||||
TaskResumeService service = new TaskResumeService(mapper, handlers);
|
||||
// 默认"名下没有活着的组装作业"(Mockito 对 boolean 返 false):既有用例判的都是克隆本体,
|
||||
// 补检豁免那条单独用 serviceWithLiveAssembleJob 覆盖。
|
||||
return service(mapper, mock(TaskFileJobService.class), handlers, enabled, maxAttempt);
|
||||
}
|
||||
|
||||
private static TaskResumeService service(FileTaskMapper mapper, TaskFileJobService jobs,
|
||||
List<ClientTaskRetrySpi> handlers,
|
||||
boolean enabled, int maxAttempt) {
|
||||
TaskResumeService service = new TaskResumeService(mapper, jobs, handlers);
|
||||
ReflectionTestUtils.setField(service, "enabled", enabled);
|
||||
ReflectionTestUtils.setField(service, "maxAttempt", maxAttempt);
|
||||
ReflectionTestUtils.setField(service, "windowMinutes", 30L);
|
||||
@@ -138,6 +146,32 @@ class TaskResumeServiceTest {
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 现网 2026-10-24:一枚 760 行的外观专利任务因交付缺口把组装作业排回补检(作业还在
|
||||
* PENDING/RUNNING),任务行却已是 FAILED,于是被巡检连克隆成 4 枚、每枚全量重跑一遍,
|
||||
* 四枚卡片开始时间还同秒(克隆件继承原件 created_at)。非零对照=上面那条
|
||||
* 「失败任务会被克隆重试」:作业已终态时照旧克隆。
|
||||
*/
|
||||
@Test
|
||||
void 组装作业仍在补检时不克隆() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(32201L, "APPEARANCE_PATENT", "有 3 行抓取失败(未取得商品页数据):B07P5TR7Q5")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
TaskFileJobService jobs = mock(TaskFileJobService.class);
|
||||
when(jobs.hasUnfinishedAssembleJob(32201L)).thenReturn(true);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("APPEARANCE_PATENT");
|
||||
TaskResumeService service = service(mapper, jobs, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.scannedTaskCount);
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 重试代数到上限后跳过() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
|
||||
Reference in New Issue
Block a user