fix(任务判死): 外观/货源判死文案按证据陈述,不再断言 Python 中断

2026-10-01 三条外观任务被心跳判死时一行业务结果都没回传过(Linux worker 一次领 5 条却只有
2 个采集槽,多出来的任务零心跳零分片直到钟面走完),而文案固定写 "Python interrupted before
uploading final result",把「没开跑」说成「跑挂了」,排查方向被带偏过一次。现在按已落库分片数、
最后回传/心跳时间、客户端最后一次上报的位置分述;结果文件作业重试耗尽也不再写成客户端中断。
This commit is contained in:
2026-10-02 09:21:11 +08:00
parent fa4bd57016
commit 5ad0f5f685
6 changed files with 197 additions and 6 deletions
@@ -0,0 +1,54 @@
package com.nanri.aiimage.common.util;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
/**
* 客户端型任务(外观专利 / 货源)心跳判死文案:只写证据支持的话。
*
* <p>历史上这里固定写 "Python interrupted before uploading final result",而 2026-10-01 线上三条
* 外观任务其实一行业务结果都没回传过——Linux worker 一次领 5 条却只有 2 个采集槽,多出来的任务
* 状态是 RUNNING、零心跳零分片,两小时后被写成「Python 中断」。「没开工」被说成「跑挂了」,
* 排查方向被带偏过一次,故文案按证据分述,并把客户端最后一次上报的位置一起带出来
* (排队中的任务会上报「排队等待空闲槽位」,与真在采集是两种成因)。
*
* @param chunkCount 已落库的结果分片数(biz_task_chunk)
* @param lastChunkAt 最后一次结果分片回传时间,从未回传时为 null
* @param lastHeartbeatAt 最后一次客户端心跳时间,Redis 无记录时为 null
* @param lastProgress 客户端最后一次上报的位置/阶段文本,老客户端或无位置信息时为 null
* @param fileJobExhausted 结果文件作业是否已重试耗尽(true 时责任不在客户端回传)
* @param timeoutMinutes 判死阈值(分钟)
*/
public record StaleTaskDeathReason(long chunkCount,
LocalDateTime lastChunkAt,
LocalDateTime lastHeartbeatAt,
String lastProgress,
boolean fileJobExhausted,
int timeoutMinutes) {
private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("MM-dd HH:mm");
public String describe() {
if (chunkCount > 0 && fileJobExhausted) {
return "已收到 " + chunkCount + " 个结果分片,但结果文件生成多次失败,任务已自动结束";
}
String headline = chunkCount <= 0L
? "客户端未回传任何检测结果,超过 " + Math.max(1, timeoutMinutes) + " 分钟无进展"
: "客户端回传中断";
List<String> evidence = new ArrayList<>();
if (chunkCount > 0L) {
evidence.add("已收到 " + chunkCount + " 个分片,最后一次 " + at(lastChunkAt));
}
evidence.add("最后心跳 " + at(lastHeartbeatAt));
if (lastProgress != null && !lastProgress.isBlank()) {
evidence.add("最后上报 " + lastProgress.trim());
}
return headline + ",任务已自动结束(" + String.join(";", evidence) + ")";
}
private static String at(LocalDateTime time) {
return time == null ? "时间未知" : time.format(TIME);
}
}
@@ -28,7 +28,7 @@ public class AppearancePatentHistoryItemVo {
private String taskStatus;
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true")
private Boolean success;
@Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final appearance patent result")
@Schema(description = "错误信息。任务失败时返回,例如客户端未回传任何检测结果、结果文件生成失败等。", example = "客户端未回传任何检测结果,超过 30 分钟无进展,任务已自动结束(最后心跳 10-01 12:34)")
private String error;
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
private Integer rowCount;
@@ -13,6 +13,7 @@ import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.util.GroupResultPropagator;
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
import com.nanri.aiimage.common.util.StaleTaskDeathReason;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
@@ -650,7 +651,7 @@ public class AppearancePatentTaskService {
continue;
}
try (lockHandle) {
finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result");
finalizeStaleTask(task.getId(), describeStaleDeath(task.getId()));
}
}
finalizeNoUploadStaleTasks();
@@ -665,10 +666,44 @@ public class AppearancePatentTaskService {
return;
}
try (lockHandle) {
finalizeStaleTask(taskId, "Python interrupted before uploading final appearance patent result");
finalizeStaleTask(taskId, describeStaleDeath(taskId));
}
}
/**
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。
* 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result",
* 把「没开跑」说成「跑挂了」。
*/
private String describeStaleDeath(Long taskId) {
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId);
return new StaleTaskDeathReason(
persistedResultChunkCount(taskId),
lastResultChunkAt(taskId),
heartbeatMillis > 0L
? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault())
: null,
taskHeartbeatPositionService.describe(taskId),
taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE),
properties.getStaleTimeoutMinutes()).describe();
}
private long persistedResultChunkCount(Long taskId) {
if (taskId == null) {
return 0L;
}
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
return count == null ? 0L : count;
}
/** 最后一次结果分片回传时间;从未回传时为 null。 */
private LocalDateTime lastResultChunkAt(Long taskId) {
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt();
}
private List<FileTaskEntity> listStaleFinalizeCandidates() {
// P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。
// APPEARANCE_PATENT 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。
@@ -28,7 +28,7 @@ public class SimilarAsinHistoryItemVo {
private String taskStatus;
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true")
private Boolean success;
@Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final similar ASIN result")
@Schema(description = "错误信息。任务失败时返回,例如客户端未回传任何检测结果、结果文件生成失败等。", example = "客户端回传中断,任务已自动结束(已收到 3 个分片,最后一次 10-01 12:34)")
private String error;
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
private Integer rowCount;
@@ -13,6 +13,7 @@ import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
import com.nanri.aiimage.common.util.StaleTaskDeathReason;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.config.StorageProperties;
@@ -94,6 +95,7 @@ import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.nio.charset.StandardCharsets;
@@ -867,7 +869,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
continue;
}
try (taskLockHandle) {
finalizeStaleTask(task.getId(), "Python interrupted before uploading final similar ASIN result");
finalizeStaleTask(task.getId(), describeStaleDeath(task.getId()));
}
}
finalizeNoUploadStaleTasks();
@@ -882,10 +884,44 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
return;
}
try (taskLockHandle) {
finalizeStaleTask(taskId, "Python interrupted before uploading final similar ASIN result");
finalizeStaleTask(taskId, describeStaleDeath(taskId));
}
}
/**
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。
* 与 {@code AppearancePatentTaskService#describeStaleDeath} 同型,共用
* {@link StaleTaskDeathReason} 的措辞。
*/
private String describeStaleDeath(Long taskId) {
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId);
return new StaleTaskDeathReason(
persistedResultChunkCount(taskId),
lastResultChunkAt(taskId),
heartbeatMillis > 0L
? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault())
: null,
taskHeartbeatPositionService.describe(taskId),
taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE),
properties.getStaleTimeoutMinutes()).describe();
}
private long persistedResultChunkCount(Long taskId) {
if (taskId == null) {
return 0L;
}
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
return count == null ? 0L : count;
}
/** 最后一次结果分片回传时间;从未回传时为 null。 */
private LocalDateTime lastResultChunkAt(Long taskId) {
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt();
}
private List<FileTaskEntity> listStaleFinalizeCandidates() {
// P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。
// SIMILAR_ASIN 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。