From 3a416f393529013042aaaeba155669aa24a874b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 28 Sep 2026 14:11:26 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E4=BB=BB=E5=8A=A1):=20=E5=8D=A1=E6=AD=BB?= =?UTF-8?q?=E8=87=AA=E6=84=88=E6=89=AB=E6=8F=8F=E4=B8=8D=E5=86=8D=E8=A2=AB?= =?UTF-8?q?=E4=B8=80=E8=A1=8C=E7=9A=84=E9=94=81=E7=AD=89=E5=BE=85=E4=BD=9C?= =?UTF-8?q?=E5=BA=9F=E6=95=B4=E8=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 线上 12:58–13:13 每 110 秒一次 1205:扫描按 updated_at 升序取到的第一行正被另一台 持着行锁,一抛就把本轮已复位成功的 job 的 dispatch 事件一起带走(它们已是 PENDING 却没人派发),该判终态失败的作业也不 finalize。改成循环体逐行 try/catch、派发也逐个包, 只跳过这一行。刻意不加事务:每行本就带 status+updated_at 的幂等 CAS 条件。 --- .../task/service/TaskFileJobService.java | 106 ++++++++++-------- .../task/service/TaskFileJobServiceTest.java | 42 +++++++ 2 files changed, 104 insertions(+), 44 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index 7bc0e63c2..fa66f2e32 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -342,65 +342,83 @@ public class TaskFileJobService { Map originalJobsById = jobs.stream() .collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a)); for (TaskFileJobEntity job : jobs) { - if ("FAILED".equals(job.getStatus()) - && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) { - exhaustedJobIds.add(job.getId()); - originalExhaustedById.put(job.getId(), job); - continue; - } - if ("PENDING".equals(job.getStatus())) { - // PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等), - // 并累加 retryCount 防无限重发;注意不翻转状态, - // 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。 + try { + if ("FAILED".equals(job.getStatus()) + && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) { + exhaustedJobIds.add(job.getId()); + originalExhaustedById.put(job.getId(), job); + continue; + } + if ("PENDING".equals(job.getStatus())) { + // PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等), + // 并累加 retryCount 防无限重发;注意不翻转状态, + // 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。 + int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount(); + int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1); + int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, job.getId()) + .eq(TaskFileJobEntity::getStatus, "PENDING") + .eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt()) + .isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt) + .set(TaskFileJobEntity::getRetryCount, nextRetryCount) + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())); + if (updated > 0) { + eventJobIds.add(job.getId()); + } + continue; + } int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount(); - int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1); - int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper() + int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1); + LocalDateTime now = LocalDateTime.now(); + LambdaUpdateWrapper update = new LambdaUpdateWrapper() .eq(TaskFileJobEntity::getId, job.getId()) - .eq(TaskFileJobEntity::getStatus, "PENDING") + .eq(TaskFileJobEntity::getStatus, "RUNNING") .eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt()) .isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt) .set(TaskFileJobEntity::getRetryCount, nextRetryCount) - .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())); - if (updated > 0) { - eventJobIds.add(job.getId()); + .set(TaskFileJobEntity::getUpdatedAt, now); + if (nextRetryCount >= MAX_RETRY_COUNT) { + int updated = taskFileJobMapper.update(null, update + .set(TaskFileJobEntity::getStatus, "FAILED") + .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数") + .set(TaskFileJobEntity::getFinishedAt, now) + .set(TaskFileJobEntity::getTerminalCallbackAt, null)); + if (updated > 0) { + exhaustedJobIds.add(job.getId()); + } + continue; } - continue; - } - int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount(); - int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1); - LocalDateTime now = LocalDateTime.now(); - LambdaUpdateWrapper update = new LambdaUpdateWrapper() - .eq(TaskFileJobEntity::getId, job.getId()) - .eq(TaskFileJobEntity::getStatus, "RUNNING") - .eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt()) - .isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt) - .set(TaskFileJobEntity::getRetryCount, nextRetryCount) - .set(TaskFileJobEntity::getUpdatedAt, now); - if (nextRetryCount >= MAX_RETRY_COUNT) { int updated = taskFileJobMapper.update(null, update - .set(TaskFileJobEntity::getStatus, "FAILED") - .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数") - .set(TaskFileJobEntity::getFinishedAt, now) + .set(TaskFileJobEntity::getStatus, "PENDING") + .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队") + .set(TaskFileJobEntity::getFinishedAt, null) .set(TaskFileJobEntity::getTerminalCallbackAt, null)); if (updated > 0) { - exhaustedJobIds.add(job.getId()); + reset++; + eventJobIds.add(job.getId()); } - continue; - } - int updated = taskFileJobMapper.update(null, update - .set(TaskFileJobEntity::getStatus, "PENDING") - .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队") - .set(TaskFileJobEntity::getFinishedAt, null) - .set(TaskFileJobEntity::getTerminalCallbackAt, null)); - if (updated > 0) { - reset++; - eventJobIds.add(job.getId()); + } catch (RuntimeException ex) { + // 这一行为什么必须单独吞:它正被别的事务占着时会等满 innodb_lock_wait_timeout + // 再抛 1205。整轮向外抛的代价不是这一行没修好,而是同批其余 199 行这轮全不看、 + // 本轮已复位成功的 job 也发不出 dispatch(它们在 400 行之后才发布), + // 于是被锁的行每轮都排在最前(ORDER BY updated_at ASC)——线上实测每 110 秒 + // 重复一次「等 50 秒 + 整轮作废」,卡住的任务在这段时间里根本没人管。 + // jobId 必须打出来:它是下一轮定位持锁方的唯一线索。 + log.warn("[task-file-job] stuck reset 单行失败已跳过,继续处理其余行 jobId={} scanned={} taskId={} moduleType={} status={} retryCount={} err={}", + job.getId(), jobs.size(), job.getTaskId(), job.getModuleType(), job.getStatus(), + job.getRetryCount(), ex.getMessage()); } } if (!eventJobIds.isEmpty()) { Map refreshedById = refreshJobsByIds(eventJobIds); for (Long jobId : eventJobIds) { - publishDispatchEvent(refreshedById.get(jobId)); + try { + publishDispatchEvent(refreshedById.get(jobId)); + } catch (RuntimeException ex) { + // 派发监听器是同步执行的,一个 job 派发失败不能带走同批其余 job + log.warn("[task-file-job] stuck reset 派发事件发布失败,不影响其余 job jobId={} err={}", + jobId, ex.getMessage()); + } } } if (!exhaustedJobIds.isEmpty()) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java index eff68a26a..3a343e576 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobServiceTest.java @@ -14,6 +14,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.context.ApplicationEventPublisher; +import org.springframework.dao.CannotAcquireLockException; import java.time.LocalDateTime; import java.util.List; @@ -23,7 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -201,6 +204,45 @@ class TaskFileJobServiceTest { assertTrue(service.isRetryExhausted(110L)); } + @Test + void lockedJobRowDoesNotAbortTheRestOfStuckScan() { + TaskFileJobEntity locked = runningJob(201L, 1, LocalDateTime.now().minusHours(1)); + TaskFileJobEntity healthy = runningJob(202L, 1, LocalDateTime.now().minusHours(1)); + TaskFileJobEntity refreshed = runningJob(202L, 2, LocalDateTime.now()); + refreshed.setStatus("PENDING"); + when(taskFileJobMapper.selectList(any())).thenReturn(List.of(locked, healthy)); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))) + .thenThrow(new CannotAcquireLockException("Lock wait timeout exceeded")) + .thenReturn(1); + when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(refreshed)); + TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher); + + TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20); + + assertEquals(1, result.resetCount()); + ArgumentCaptor events = ArgumentCaptor.forClass(TaskFileJobDispatchEvent.class); + verify(applicationEventPublisher).publishEvent(events.capture()); + assertEquals(202L, events.getValue().jobId()); + } + + @Test + void alreadyResetJobsStillDispatchWhenOnePublishFails() { + TaskFileJobEntity first = runningJob(203L, 1, LocalDateTime.now().minusHours(1)); + TaskFileJobEntity second = runningJob(204L, 1, LocalDateTime.now().minusHours(1)); + when(taskFileJobMapper.selectList(any())).thenReturn(List.of(first, second)); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(first, second)); + doThrow(new IllegalStateException("listener boom")) + .doNothing() + .when(applicationEventPublisher).publishEvent(any(Object.class)); + TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher); + + TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20); + + assertEquals(2, result.resetCount()); + verify(applicationEventPublisher, times(2)).publishEvent(any(Object.class)); + } + private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) { ArgumentCaptor> update = updateCaptor(); verify(taskFileJobMapper).update(isNull(), update.capture());