fix(任务): 卡死自愈扫描不再被一行的锁等待作废整轮

线上 12:58–13:13 每 110 秒一次 1205:扫描按 updated_at 升序取到的第一行正被另一台
持着行锁,一抛就把本轮已复位成功的 job 的 dispatch 事件一起带走(它们已是 PENDING
却没人派发),该判终态失败的作业也不 finalize。改成循环体逐行 try/catch、派发也逐个包,
只跳过这一行。刻意不加事务:每行本就带 status+updated_at 的幂等 CAS 条件。
This commit is contained in:
2026-09-28 14:11:26 +08:00
parent d70c472cab
commit 3a416f3935
2 changed files with 104 additions and 44 deletions
@@ -14,6 +14,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.dao.CannotAcquireLockException;
import java.time.LocalDateTime;
import java.util.List;
@@ -23,7 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -201,6 +204,45 @@ class TaskFileJobServiceTest {
assertTrue(service.isRetryExhausted(110L));
}
@Test
void lockedJobRowDoesNotAbortTheRestOfStuckScan() {
TaskFileJobEntity locked = runningJob(201L, 1, LocalDateTime.now().minusHours(1));
TaskFileJobEntity healthy = runningJob(202L, 1, LocalDateTime.now().minusHours(1));
TaskFileJobEntity refreshed = runningJob(202L, 2, LocalDateTime.now());
refreshed.setStatus("PENDING");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(locked, healthy));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class)))
.thenThrow(new CannotAcquireLockException("Lock wait timeout exceeded"))
.thenReturn(1);
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(refreshed));
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(1, result.resetCount());
ArgumentCaptor<TaskFileJobDispatchEvent> events = ArgumentCaptor.forClass(TaskFileJobDispatchEvent.class);
verify(applicationEventPublisher).publishEvent(events.capture());
assertEquals(202L, events.getValue().jobId());
}
@Test
void alreadyResetJobsStillDispatchWhenOnePublishFails() {
TaskFileJobEntity first = runningJob(203L, 1, LocalDateTime.now().minusHours(1));
TaskFileJobEntity second = runningJob(204L, 1, LocalDateTime.now().minusHours(1));
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(first, second));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(first, second));
doThrow(new IllegalStateException("listener boom"))
.doNothing()
.when(applicationEventPublisher).publishEvent(any(Object.class));
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(2, result.resetCount());
verify(applicationEventPublisher, times(2)).publishEvent(any(Object.class));
}
private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) {
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());