From aa7e07cf0205d5b382ed2beab4ad5bebd698bfe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 28 Sep 2026 14:11:36 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E5=A4=96=E8=A7=82=E4=B8=93=E5=88=A9):=20st?= =?UTF-8?q?ale=20=E6=81=A2=E5=A4=8D=E4=B8=8D=E5=86=8D=E6=8A=8A=E6=95=B4?= =?UTF-8?q?=E6=89=B9=20LLM=20=E5=8C=85=E8=BF=9B=E4=BA=8B=E5=8A=A1=EF=BC=8C?= =?UTF-8?q?=E8=A1=8C=E9=94=81=E4=B8=8D=E5=86=8D=E6=8C=81=E5=8D=81=E5=87=A0?= =?UTF-8?q?=E5=88=86=E9=92=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 那 50 秒锁等待的持锁方就是这里:三个调用点把整个 finalizeStaleTask 包进 PROPAGATION_REQUIRES_NEW,而恢复分支里 submitLlmBatches 每批开头的 touchRunning 是 UPDATE biz_task_file_job ... WHERE id=? AND status='RUNNING',锁要活到串行发完 整批 LLM 请求(线上实测单批 costMs=1077959)之后才释放。改为只把封口和 finalizeTask 各自包成短事务,LLM 批次留在事务外;另在发批前查 isActualTransactionActive 做回归哨兵。 同源的 SimilarAsinTaskService 一直是正确形状,这里只是回到一致。 --- .../service/AppearancePatentTaskService.java | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index a67c54e89..46814ccd8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -652,14 +652,7 @@ public class AppearancePatentTaskService { continue; } try (lockHandle) { - if (transactionManager != null) { - inNewTransaction(() -> { - finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result"); - return null; - }); - } else { - finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result"); - } + finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result"); } } finalizeNoUploadStaleTasks(); @@ -669,19 +662,6 @@ public class AppearancePatentTaskService { if (taskId == null || taskId <= 0) { return; } - if (transactionManager != null) { - TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L); - if (lockHandle == null) { - return; - } - try (lockHandle) { - inNewTransaction(() -> { - finalizeStaleTask(taskId, "Python interrupted before uploading final appearance patent result"); - return null; - }); - } - return; - } TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L); if (lockHandle == null) { return; @@ -777,14 +757,7 @@ public class AppearancePatentTaskService { log.warn("[appearance-patent] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}", minutes, task.getId(), lastResultAt); String error = "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes"; - if (transactionManager != null) { - inNewTransaction(() -> { - finalizeStaleTask(task.getId(), error); - return null; - }); - } else { - finalizeStaleTask(task.getId(), error); - } + finalizeStaleTask(task.getId(), error); } } } @@ -929,6 +902,15 @@ public class AppearancePatentTaskService { task.getId(), job.getId(), candidates.size(), batchSize, flushRemainder); return; } + if (TransactionSynchronizationManager.isActualTransactionActive()) { + // 守住"整批 LLM 跑在事务里"这一类回归:下面循环每批开头的 touchRunning 是 + // UPDATE biz_task_file_job ... WHERE id=? AND status='RUNNING',本方法一旦处在事务中, + // 这把行锁就要持到全部批次结束才释放(线上实测一批 18 分钟),另一台的卡死自愈扫描 + // 于是每 110 秒撞一次 50 秒锁等待、并且整轮作废。历史上真发生过,见 + // docs/ops/prod-decisions-2026-09-28.md §9。 + log.error("[appearance-patent] llm 批次正跑在事务里,行锁会持到批次结束 taskId={} jobId={} pendingRows={}", + task.getId(), job.getId(), submitLimit); + } int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize); int batchIndex = 1; for (int i = 0; i < submitLimit; i += batchSize) { @@ -1196,6 +1178,17 @@ public class AppearancePatentTaskService { touchJavaSideTaskActivity(task.getId()); } + /** + * stale 判死收尾:事务只包住真正需要原子的两段(封口状态批量更新、任务终态 + 结果 + 组装作业入队), + * **LLM 批次留在事务之外**。 + * + *

外层调用点曾把整个方法包进 {@code inNewTransaction},于是恢复分支里 + * {@code submitLlmBatches} 每批开头 {@code touchRunning} 对 biz_task_file_job 取的 X 锁, + * 要一直活到串行发完整批 LLM 请求 + * (线上实测单批 costMs=1077959)之后才释放——两台的卡死自愈扫描 UPDATE 同一行, + * 每 110 秒撞一次 50 秒锁等待,还顺手按住一条 Hikari 连接十几分钟、并把已扣过 LLM 次数的状态 + * 置于可回滚的位置。同源的 {@code SimilarAsinTaskService#finalizeStaleTask} 一直是这个形状。 + */ private void finalizeStaleTask(Long taskId, String error) { FileTaskEntity task = fileTaskMapper.selectById(taskId); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) { @@ -1204,7 +1197,14 @@ public class AppearancePatentTaskService { if (tryRecoverTimedOutPythonTask(task)) { return; } - finalizeTask(task, error, allRowCount(task), true); + if (transactionManager == null) { + finalizeTask(task, error, allRowCount(task), true); + return; + } + inNewTransaction(() -> { + finalizeTask(task, error, allRowCount(task), true); + return null; + }); } private boolean tryRecoverTimedOutPythonTask(FileTaskEntity task) { @@ -1229,7 +1229,10 @@ public class AppearancePatentTaskService { return false; } if (!uploadComplete) { - int forcedScopes = markSubmissionCompleteOnPythonTimeout(taskId); + // 封口是多行 scope 状态更新,单独占一个短事务;紧随其后的 LLM 批次不在事务里 + int forcedScopes = transactionManager == null + ? markSubmissionCompleteOnPythonTimeout(taskId) + : inNewTransaction(() -> markSubmissionCompleteOnPythonTimeout(taskId)); if (forcedScopes <= 0) { log.warn("[appearance-patent] stale recovery aborted because submission could not be marked complete taskId={}", taskId);