fix(外观专利): stale 恢复不再把整批 LLM 包进事务,行锁不再持十几分钟
那 50 秒锁等待的持锁方就是这里:三个调用点把整个 finalizeStaleTask 包进 PROPAGATION_REQUIRES_NEW,而恢复分支里 submitLlmBatches 每批开头的 touchRunning 是 UPDATE biz_task_file_job ... WHERE id=? AND status='RUNNING',锁要活到串行发完 整批 LLM 请求(线上实测单批 costMs=1077959)之后才释放。改为只把封口和 finalizeTask 各自包成短事务,LLM 批次留在事务外;另在发批前查 isActualTransactionActive 做回归哨兵。 同源的 SimilarAsinTaskService 一直是正确形状,这里只是回到一致。
This commit is contained in:
+34
-31
@@ -652,14 +652,7 @@ public class AppearancePatentTaskService {
|
||||
continue;
|
||||
}
|
||||
try (lockHandle) {
|
||||
if (transactionManager != null) {
|
||||
inNewTransaction(() -> {
|
||||
finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result");
|
||||
return null;
|
||||
});
|
||||
} else {
|
||||
finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result");
|
||||
}
|
||||
finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result");
|
||||
}
|
||||
}
|
||||
finalizeNoUploadStaleTasks();
|
||||
@@ -669,19 +662,6 @@ public class AppearancePatentTaskService {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
if (transactionManager != null) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L);
|
||||
if (lockHandle == null) {
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
inNewTransaction(() -> {
|
||||
finalizeStaleTask(taskId, "Python interrupted before uploading final appearance patent result");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L);
|
||||
if (lockHandle == null) {
|
||||
return;
|
||||
@@ -777,14 +757,7 @@ public class AppearancePatentTaskService {
|
||||
log.warn("[appearance-patent] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}",
|
||||
minutes, task.getId(), lastResultAt);
|
||||
String error = "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes";
|
||||
if (transactionManager != null) {
|
||||
inNewTransaction(() -> {
|
||||
finalizeStaleTask(task.getId(), error);
|
||||
return null;
|
||||
});
|
||||
} else {
|
||||
finalizeStaleTask(task.getId(), error);
|
||||
}
|
||||
finalizeStaleTask(task.getId(), error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -929,6 +902,15 @@ public class AppearancePatentTaskService {
|
||||
task.getId(), job.getId(), candidates.size(), batchSize, flushRemainder);
|
||||
return;
|
||||
}
|
||||
if (TransactionSynchronizationManager.isActualTransactionActive()) {
|
||||
// 守住"整批 LLM 跑在事务里"这一类回归:下面循环每批开头的 touchRunning 是
|
||||
// UPDATE biz_task_file_job ... WHERE id=? AND status='RUNNING',本方法一旦处在事务中,
|
||||
// 这把行锁就要持到全部批次结束才释放(线上实测一批 18 分钟),另一台的卡死自愈扫描
|
||||
// 于是每 110 秒撞一次 50 秒锁等待、并且整轮作废。历史上真发生过,见
|
||||
// docs/ops/prod-decisions-2026-09-28.md §9。
|
||||
log.error("[appearance-patent] llm 批次正跑在事务里,行锁会持到批次结束 taskId={} jobId={} pendingRows={}",
|
||||
task.getId(), job.getId(), submitLimit);
|
||||
}
|
||||
int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize);
|
||||
int batchIndex = 1;
|
||||
for (int i = 0; i < submitLimit; i += batchSize) {
|
||||
@@ -1196,6 +1178,17 @@ public class AppearancePatentTaskService {
|
||||
touchJavaSideTaskActivity(task.getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* stale 判死收尾:事务只包住真正需要原子的两段(封口状态批量更新、任务终态 + 结果 + 组装作业入队),
|
||||
* **LLM 批次留在事务之外**。
|
||||
*
|
||||
* <p>外层调用点曾把整个方法包进 {@code inNewTransaction},于是恢复分支里
|
||||
* {@code submitLlmBatches} 每批开头 {@code touchRunning} 对 biz_task_file_job 取的 X 锁,
|
||||
* 要一直活到串行发完整批 LLM 请求
|
||||
* (线上实测单批 costMs=1077959)之后才释放——两台的卡死自愈扫描 UPDATE 同一行,
|
||||
* 每 110 秒撞一次 50 秒锁等待,还顺手按住一条 Hikari 连接十几分钟、并把已扣过 LLM 次数的状态
|
||||
* 置于可回滚的位置。同源的 {@code SimilarAsinTaskService#finalizeStaleTask} 一直是这个形状。
|
||||
*/
|
||||
private void finalizeStaleTask(Long taskId, String error) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
|
||||
@@ -1204,7 +1197,14 @@ public class AppearancePatentTaskService {
|
||||
if (tryRecoverTimedOutPythonTask(task)) {
|
||||
return;
|
||||
}
|
||||
finalizeTask(task, error, allRowCount(task), true);
|
||||
if (transactionManager == null) {
|
||||
finalizeTask(task, error, allRowCount(task), true);
|
||||
return;
|
||||
}
|
||||
inNewTransaction(() -> {
|
||||
finalizeTask(task, error, allRowCount(task), true);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private boolean tryRecoverTimedOutPythonTask(FileTaskEntity task) {
|
||||
@@ -1229,7 +1229,10 @@ public class AppearancePatentTaskService {
|
||||
return false;
|
||||
}
|
||||
if (!uploadComplete) {
|
||||
int forcedScopes = markSubmissionCompleteOnPythonTimeout(taskId);
|
||||
// 封口是多行 scope 状态更新,单独占一个短事务;紧随其后的 LLM 批次不在事务里
|
||||
int forcedScopes = transactionManager == null
|
||||
? markSubmissionCompleteOnPythonTimeout(taskId)
|
||||
: inNewTransaction(() -> markSubmissionCompleteOnPythonTimeout(taskId));
|
||||
if (forcedScopes <= 0) {
|
||||
log.warn("[appearance-patent] stale recovery aborted because submission could not be marked complete taskId={}",
|
||||
taskId);
|
||||
|
||||
Reference in New Issue
Block a user