From ac7412f77d892c7aed24cd8611b230a07cf7a284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 28 Sep 2026 15:20:59 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E4=BB=BB=E5=8A=A1):=20=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=BD=9C=E4=B8=9A=E6=92=9E=E9=94=81=E5=90=8E?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E8=87=AA=E5=B8=A6=E4=B8=8B=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E6=8A=95=E9=80=92=EF=BC=8C=E6=B6=88=E9=99=A4=E9=9B=B6=E9=80=80?= =?UTF-8?q?=E9=81=BF=E8=87=AA=E5=BE=AA=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit requeue() 在把行从 RUNNING 翻回 PENDING 的同一刻 publish 一条 dispatch 事件, 于是投递链立刻重新 claim 同一个 job、再撞同一把锁(持锁方是分钟级的 LLM 批次)。 线上实测 20 分钟 667 次、占该实例日志 8.7%,且每次要占满 taskQueueExecutor 的 10 秒探锁 —— 该池只有 12 个许可且与货源 LLM 重提、图像视频轮询共用,按 Little 定律 相当于长期吃掉近一半容量。 撞锁分支改用同语义但不发事件的 deferRunning(),重投节奏交回 runPendingJobs 的 10 秒周期;本地轮询关闭的 profile 没有周期兜底,继续走 requeue 以免作业搁死在 PENDING。 --- .../task/service/TaskResultFileJobWorker.java | 13 ++++- ...askResultFileJobWorkerSpecialRuleTest.java | 50 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index 88a99da34..4455f6726 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -232,7 +232,18 @@ public class TaskResultFileJobWorker { TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS); if (lockHandle == null) { - taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation"); + // 撞锁后不能再走 requeue():它在把行翻回 PENDING 的同一刻 publish 一条 dispatch + // 事件,投递链立刻重新 claim 同一个 job 再撞同一把锁(持锁方是分钟级的 LLM 批次), + // 形成零退避自循环——线上 20 分钟 667 次、占该实例日志 8.7%,且每次都占满 + // taskQueueExecutor 的 10 秒探锁(全池 12 槽,与货源 LLM 重提、图像视频轮询共用)。 + // deferRunning 同样退回 PENDING 但不发事件,重投节奏交回 runPendingJobs 的固定周期。 + // 本地轮询关闭的形态没有周期兜底,必须继续用 requeue 立刻重投,否则作业搁死在 PENDING。 + String busyMessage = "Task is busy, waiting for previous task operation"; + if (localWorkerEnabled) { + taskFileJobService.deferRunning(job.getId(), busyMessage); + } else { + taskFileJobService.requeue(job.getId(), busyMessage); + } log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={} stage=REQUEUE", job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId()); return; diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerSpecialRuleTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerSpecialRuleTest.java index 956e44e69..7a07a2d75 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerSpecialRuleTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerSpecialRuleTest.java @@ -324,4 +324,54 @@ class TaskResultFileJobWorkerSpecialRuleTest { verify(taskFileJobService).deferRunning(11L, "Waiting for LLM/file assembly to continue"); verify(lock).close(); } + + /** + * 锁忙回投必须**不再自带下一次投递**:requeue() 在把行翻回 PENDING 的同一刻 publish 一条 + * dispatch 事件 → 归属实例立刻重新 claim → 再撞同一把锁(持锁方是分钟级的 LLM 批次)。 + * 线上实测 20 分钟 667 次自循环,每次还要占住与货源/图像视频共用的执行器槽位 10 秒。 + * 本地轮询开着时,重投的节奏应由 runPendingJobs 的固定周期给出,这里只停车不点火。 + */ + @Test + void lockBusyWithLocalWorkerPollEnabledParksWithoutRepublishing() throws Exception { + TaskResultFileJobWorker worker = buildWorker(); + setLocalWorkerEnabled(worker, true); + ResultFileJobHandler handler = mock(ResultFileJobHandler.class); + when(handler.moduleType()).thenReturn("APPEARANCE_PATENT"); + stubHandler(handler, true, true); + TaskFileJobEntity job = job("APPEARANCE_PATENT", 12L, 22L, 32L); + allowClaim(job); + when(taskDistributedLockService.acquire("APPEARANCE_PATENT", 22L, + TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(null); + + worker.process(job); + + verify(taskFileJobService).deferRunning(12L, "Task is busy, waiting for previous task operation"); + verify(taskFileJobService, never()).requeue(any(), any()); + verify(handler, never()).process(any()); + } + + /** 关掉本地轮询的形态没有周期性兜底,锁忙仍必须重投,否则作业会永久搁死在 PENDING。 */ + @Test + void lockBusyWithoutLocalWorkerPollStillRepublishes() throws Exception { + TaskResultFileJobWorker worker = buildWorker(); + setLocalWorkerEnabled(worker, false); + ResultFileJobHandler handler = mock(ResultFileJobHandler.class); + when(handler.moduleType()).thenReturn("APPEARANCE_PATENT"); + stubHandler(handler, true, true); + TaskFileJobEntity job = job("APPEARANCE_PATENT", 13L, 23L, 33L); + allowClaim(job); + when(taskDistributedLockService.acquire("APPEARANCE_PATENT", 23L, + TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(null); + + worker.process(job); + + verify(taskFileJobService).requeue(13L, "Task is busy, waiting for previous task operation"); + verify(taskFileJobService, never()).deferRunning(any(), any()); + } + + private static void setLocalWorkerEnabled(TaskResultFileJobWorker worker, boolean enabled) throws Exception { + java.lang.reflect.Field field = TaskResultFileJobWorker.class.getDeclaredField("localWorkerEnabled"); + field.setAccessible(true); + field.setBoolean(worker, enabled); + } }