fix(任务): 卡死自愈扫描不再被一行的锁等待作废整轮
线上 12:58–13:13 每 110 秒一次 1205:扫描按 updated_at 升序取到的第一行正被另一台 持着行锁,一抛就把本轮已复位成功的 job 的 dispatch 事件一起带走(它们已是 PENDING 却没人派发),该判终态失败的作业也不 finalize。改成循环体逐行 try/catch、派发也逐个包, 只跳过这一行。刻意不加事务:每行本就带 status+updated_at 的幂等 CAS 条件。
This commit is contained in:
+62
-44
@@ -342,65 +342,83 @@ public class TaskFileJobService {
|
|||||||
Map<Long, TaskFileJobEntity> originalJobsById = jobs.stream()
|
Map<Long, TaskFileJobEntity> originalJobsById = jobs.stream()
|
||||||
.collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a));
|
.collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a));
|
||||||
for (TaskFileJobEntity job : jobs) {
|
for (TaskFileJobEntity job : jobs) {
|
||||||
if ("FAILED".equals(job.getStatus())
|
try {
|
||||||
&& job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) {
|
if ("FAILED".equals(job.getStatus())
|
||||||
exhaustedJobIds.add(job.getId());
|
&& job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) {
|
||||||
originalExhaustedById.put(job.getId(), job);
|
exhaustedJobIds.add(job.getId());
|
||||||
continue;
|
originalExhaustedById.put(job.getId(), job);
|
||||||
}
|
continue;
|
||||||
if ("PENDING".equals(job.getStatus())) {
|
}
|
||||||
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),
|
if ("PENDING".equals(job.getStatus())) {
|
||||||
// 并累加 retryCount 防无限重发;注意不翻转状态,
|
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),
|
||||||
// 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。
|
// 并累加 retryCount 防无限重发;注意不翻转状态,
|
||||||
|
// 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。
|
||||||
|
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||||
|
int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1);
|
||||||
|
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, job.getId())
|
||||||
|
.eq(TaskFileJobEntity::getStatus, "PENDING")
|
||||||
|
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||||
|
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||||
|
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
if (updated > 0) {
|
||||||
|
eventJobIds.add(job.getId());
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1);
|
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
||||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
.eq(TaskFileJobEntity::getId, job.getId())
|
.eq(TaskFileJobEntity::getId, job.getId())
|
||||||
.eq(TaskFileJobEntity::getStatus, "PENDING")
|
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
.set(TaskFileJobEntity::getUpdatedAt, now);
|
||||||
if (updated > 0) {
|
if (nextRetryCount >= MAX_RETRY_COUNT) {
|
||||||
eventJobIds.add(job.getId());
|
int updated = taskFileJobMapper.update(null, update
|
||||||
|
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||||
|
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数")
|
||||||
|
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||||
|
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||||
|
if (updated > 0) {
|
||||||
|
exhaustedJobIds.add(job.getId());
|
||||||
|
}
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
continue;
|
|
||||||
}
|
|
||||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
|
||||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
|
||||||
LocalDateTime now = LocalDateTime.now();
|
|
||||||
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
|
||||||
.eq(TaskFileJobEntity::getId, job.getId())
|
|
||||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
|
||||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
|
||||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
|
||||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
|
||||||
.set(TaskFileJobEntity::getUpdatedAt, now);
|
|
||||||
if (nextRetryCount >= MAX_RETRY_COUNT) {
|
|
||||||
int updated = taskFileJobMapper.update(null, update
|
int updated = taskFileJobMapper.update(null, update
|
||||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数")
|
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||||
if (updated > 0) {
|
if (updated > 0) {
|
||||||
exhaustedJobIds.add(job.getId());
|
reset++;
|
||||||
|
eventJobIds.add(job.getId());
|
||||||
}
|
}
|
||||||
continue;
|
} catch (RuntimeException ex) {
|
||||||
}
|
// 这一行为什么必须单独吞:它正被别的事务占着时会等满 innodb_lock_wait_timeout
|
||||||
int updated = taskFileJobMapper.update(null, update
|
// 再抛 1205。整轮向外抛的代价不是这一行没修好,而是同批其余 199 行这轮全不看、
|
||||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
// 本轮已复位成功的 job 也发不出 dispatch(它们在 400 行之后才发布),
|
||||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
// 于是被锁的行每轮都排在最前(ORDER BY updated_at ASC)——线上实测每 110 秒
|
||||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
// 重复一次「等 50 秒 + 整轮作废」,卡住的任务在这段时间里根本没人管。
|
||||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
// jobId 必须打出来:它是下一轮定位持锁方的唯一线索。
|
||||||
if (updated > 0) {
|
log.warn("[task-file-job] stuck reset 单行失败已跳过,继续处理其余行 jobId={} scanned={} taskId={} moduleType={} status={} retryCount={} err={}",
|
||||||
reset++;
|
job.getId(), jobs.size(), job.getTaskId(), job.getModuleType(), job.getStatus(),
|
||||||
eventJobIds.add(job.getId());
|
job.getRetryCount(), ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!eventJobIds.isEmpty()) {
|
if (!eventJobIds.isEmpty()) {
|
||||||
Map<Long, TaskFileJobEntity> refreshedById = refreshJobsByIds(eventJobIds);
|
Map<Long, TaskFileJobEntity> refreshedById = refreshJobsByIds(eventJobIds);
|
||||||
for (Long jobId : eventJobIds) {
|
for (Long jobId : eventJobIds) {
|
||||||
publishDispatchEvent(refreshedById.get(jobId));
|
try {
|
||||||
|
publishDispatchEvent(refreshedById.get(jobId));
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
// 派发监听器是同步执行的,一个 job 派发失败不能带走同批其余 job
|
||||||
|
log.warn("[task-file-job] stuck reset 派发事件发布失败,不影响其余 job jobId={} err={}",
|
||||||
|
jobId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!exhaustedJobIds.isEmpty()) {
|
if (!exhaustedJobIds.isEmpty()) {
|
||||||
|
|||||||
+42
@@ -14,6 +14,7 @@ import org.mockito.ArgumentCaptor;
|
|||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
import org.springframework.context.ApplicationEventPublisher;
|
import org.springframework.context.ApplicationEventPublisher;
|
||||||
|
import org.springframework.dao.CannotAcquireLockException;
|
||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -23,7 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.ArgumentMatchers.isNull;
|
import static org.mockito.ArgumentMatchers.isNull;
|
||||||
|
import static org.mockito.Mockito.doThrow;
|
||||||
import static org.mockito.Mockito.never;
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
@@ -201,6 +204,45 @@ class TaskFileJobServiceTest {
|
|||||||
assertTrue(service.isRetryExhausted(110L));
|
assertTrue(service.isRetryExhausted(110L));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void lockedJobRowDoesNotAbortTheRestOfStuckScan() {
|
||||||
|
TaskFileJobEntity locked = runningJob(201L, 1, LocalDateTime.now().minusHours(1));
|
||||||
|
TaskFileJobEntity healthy = runningJob(202L, 1, LocalDateTime.now().minusHours(1));
|
||||||
|
TaskFileJobEntity refreshed = runningJob(202L, 2, LocalDateTime.now());
|
||||||
|
refreshed.setStatus("PENDING");
|
||||||
|
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(locked, healthy));
|
||||||
|
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class)))
|
||||||
|
.thenThrow(new CannotAcquireLockException("Lock wait timeout exceeded"))
|
||||||
|
.thenReturn(1);
|
||||||
|
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(refreshed));
|
||||||
|
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||||
|
|
||||||
|
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||||
|
|
||||||
|
assertEquals(1, result.resetCount());
|
||||||
|
ArgumentCaptor<TaskFileJobDispatchEvent> events = ArgumentCaptor.forClass(TaskFileJobDispatchEvent.class);
|
||||||
|
verify(applicationEventPublisher).publishEvent(events.capture());
|
||||||
|
assertEquals(202L, events.getValue().jobId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void alreadyResetJobsStillDispatchWhenOnePublishFails() {
|
||||||
|
TaskFileJobEntity first = runningJob(203L, 1, LocalDateTime.now().minusHours(1));
|
||||||
|
TaskFileJobEntity second = runningJob(204L, 1, LocalDateTime.now().minusHours(1));
|
||||||
|
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(first, second));
|
||||||
|
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||||
|
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(first, second));
|
||||||
|
doThrow(new IllegalStateException("listener boom"))
|
||||||
|
.doNothing()
|
||||||
|
.when(applicationEventPublisher).publishEvent(any(Object.class));
|
||||||
|
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||||
|
|
||||||
|
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||||
|
|
||||||
|
assertEquals(2, result.resetCount());
|
||||||
|
verify(applicationEventPublisher, times(2)).publishEvent(any(Object.class));
|
||||||
|
}
|
||||||
|
|
||||||
private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) {
|
private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) {
|
||||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||||
|
|||||||
Reference in New Issue
Block a user