fix(任务): 卡死自愈扫描不再被一行的锁等待作废整轮
线上 12:58–13:13 每 110 秒一次 1205:扫描按 updated_at 升序取到的第一行正被另一台 持着行锁,一抛就把本轮已复位成功的 job 的 dispatch 事件一起带走(它们已是 PENDING 却没人派发),该判终态失败的作业也不 finalize。改成循环体逐行 try/catch、派发也逐个包, 只跳过这一行。刻意不加事务:每行本就带 status+updated_at 的幂等 CAS 条件。
This commit is contained in:
+42
@@ -14,6 +14,7 @@ import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.dao.CannotAcquireLockException;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
@@ -23,7 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@@ -201,6 +204,45 @@ class TaskFileJobServiceTest {
|
||||
assertTrue(service.isRetryExhausted(110L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lockedJobRowDoesNotAbortTheRestOfStuckScan() {
|
||||
TaskFileJobEntity locked = runningJob(201L, 1, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity healthy = runningJob(202L, 1, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity refreshed = runningJob(202L, 2, LocalDateTime.now());
|
||||
refreshed.setStatus("PENDING");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(locked, healthy));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class)))
|
||||
.thenThrow(new CannotAcquireLockException("Lock wait timeout exceeded"))
|
||||
.thenReturn(1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(refreshed));
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(1, result.resetCount());
|
||||
ArgumentCaptor<TaskFileJobDispatchEvent> events = ArgumentCaptor.forClass(TaskFileJobDispatchEvent.class);
|
||||
verify(applicationEventPublisher).publishEvent(events.capture());
|
||||
assertEquals(202L, events.getValue().jobId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void alreadyResetJobsStillDispatchWhenOnePublishFails() {
|
||||
TaskFileJobEntity first = runningJob(203L, 1, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity second = runningJob(204L, 1, LocalDateTime.now().minusHours(1));
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(first, second));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(first, second));
|
||||
doThrow(new IllegalStateException("listener boom"))
|
||||
.doNothing()
|
||||
.when(applicationEventPublisher).publishEvent(any(Object.class));
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(2, result.resetCount());
|
||||
verify(applicationEventPublisher, times(2)).publishEvent(any(Object.class));
|
||||
}
|
||||
|
||||
private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) {
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
|
||||
Reference in New Issue
Block a user