fix(任务): 结果文件作业撞锁后不再自带下一次投递,消除零退避自循环
requeue() 在把行从 RUNNING 翻回 PENDING 的同一刻 publish 一条 dispatch 事件, 于是投递链立刻重新 claim 同一个 job、再撞同一把锁(持锁方是分钟级的 LLM 批次)。 线上实测 20 分钟 667 次、占该实例日志 8.7%,且每次要占满 taskQueueExecutor 的 10 秒探锁 —— 该池只有 12 个许可且与货源 LLM 重提、图像视频轮询共用,按 Little 定律 相当于长期吃掉近一半容量。 撞锁分支改用同语义但不发事件的 deferRunning(),重投节奏交回 runPendingJobs 的 10 秒周期;本地轮询关闭的 profile 没有周期兜底,继续走 requeue 以免作业搁死在 PENDING。
This commit is contained in:
+12
-1
@@ -232,7 +232,18 @@ public class TaskResultFileJobWorker {
|
||||
TaskDistributedLockService.LockHandle lockHandle =
|
||||
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
|
||||
// 撞锁后不能再走 requeue():它在把行翻回 PENDING 的同一刻 publish 一条 dispatch
|
||||
// 事件,投递链立刻重新 claim 同一个 job 再撞同一把锁(持锁方是分钟级的 LLM 批次),
|
||||
// 形成零退避自循环——线上 20 分钟 667 次、占该实例日志 8.7%,且每次都占满
|
||||
// taskQueueExecutor 的 10 秒探锁(全池 12 槽,与货源 LLM 重提、图像视频轮询共用)。
|
||||
// deferRunning 同样退回 PENDING 但不发事件,重投节奏交回 runPendingJobs 的固定周期。
|
||||
// 本地轮询关闭的形态没有周期兜底,必须继续用 requeue 立刻重投,否则作业搁死在 PENDING。
|
||||
String busyMessage = "Task is busy, waiting for previous task operation";
|
||||
if (localWorkerEnabled) {
|
||||
taskFileJobService.deferRunning(job.getId(), busyMessage);
|
||||
} else {
|
||||
taskFileJobService.requeue(job.getId(), busyMessage);
|
||||
}
|
||||
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={} stage=REQUEUE",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
return;
|
||||
|
||||
+50
@@ -324,4 +324,54 @@ class TaskResultFileJobWorkerSpecialRuleTest {
|
||||
verify(taskFileJobService).deferRunning(11L, "Waiting for LLM/file assembly to continue");
|
||||
verify(lock).close();
|
||||
}
|
||||
|
||||
/**
|
||||
* 锁忙回投必须**不再自带下一次投递**:requeue() 在把行翻回 PENDING 的同一刻 publish 一条
|
||||
* dispatch 事件 → 归属实例立刻重新 claim → 再撞同一把锁(持锁方是分钟级的 LLM 批次)。
|
||||
* 线上实测 20 分钟 667 次自循环,每次还要占住与货源/图像视频共用的执行器槽位 10 秒。
|
||||
* 本地轮询开着时,重投的节奏应由 runPendingJobs 的固定周期给出,这里只停车不点火。
|
||||
*/
|
||||
@Test
|
||||
void lockBusyWithLocalWorkerPollEnabledParksWithoutRepublishing() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
setLocalWorkerEnabled(worker, true);
|
||||
ResultFileJobHandler handler = mock(ResultFileJobHandler.class);
|
||||
when(handler.moduleType()).thenReturn("APPEARANCE_PATENT");
|
||||
stubHandler(handler, true, true);
|
||||
TaskFileJobEntity job = job("APPEARANCE_PATENT", 12L, 22L, 32L);
|
||||
allowClaim(job);
|
||||
when(taskDistributedLockService.acquire("APPEARANCE_PATENT", 22L,
|
||||
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(null);
|
||||
|
||||
worker.process(job);
|
||||
|
||||
verify(taskFileJobService).deferRunning(12L, "Task is busy, waiting for previous task operation");
|
||||
verify(taskFileJobService, never()).requeue(any(), any());
|
||||
verify(handler, never()).process(any());
|
||||
}
|
||||
|
||||
/** 关掉本地轮询的形态没有周期性兜底,锁忙仍必须重投,否则作业会永久搁死在 PENDING。 */
|
||||
@Test
|
||||
void lockBusyWithoutLocalWorkerPollStillRepublishes() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
setLocalWorkerEnabled(worker, false);
|
||||
ResultFileJobHandler handler = mock(ResultFileJobHandler.class);
|
||||
when(handler.moduleType()).thenReturn("APPEARANCE_PATENT");
|
||||
stubHandler(handler, true, true);
|
||||
TaskFileJobEntity job = job("APPEARANCE_PATENT", 13L, 23L, 33L);
|
||||
allowClaim(job);
|
||||
when(taskDistributedLockService.acquire("APPEARANCE_PATENT", 23L,
|
||||
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(null);
|
||||
|
||||
worker.process(job);
|
||||
|
||||
verify(taskFileJobService).requeue(13L, "Task is busy, waiting for previous task operation");
|
||||
verify(taskFileJobService, never()).deferRunning(any(), any());
|
||||
}
|
||||
|
||||
private static void setLocalWorkerEnabled(TaskResultFileJobWorker worker, boolean enabled) throws Exception {
|
||||
java.lang.reflect.Field field = TaskResultFileJobWorker.class.getDeclaredField("localWorkerEnabled");
|
||||
field.setAccessible(true);
|
||||
field.setBoolean(worker, enabled);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user