diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/util/StaleTaskDeathReason.java b/backend-java/src/main/java/com/nanri/aiimage/common/util/StaleTaskDeathReason.java new file mode 100644 index 000000000..96f80c4e3 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/util/StaleTaskDeathReason.java @@ -0,0 +1,54 @@ +package com.nanri.aiimage.common.util; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +/** + * 客户端型任务(外观专利 / 货源)心跳判死文案:只写证据支持的话。 + * + *

历史上这里固定写 "Python interrupted before uploading final result",而 2026-10-01 线上三条 + * 外观任务其实一行业务结果都没回传过——Linux worker 一次领 5 条却只有 2 个采集槽,多出来的任务 + * 状态是 RUNNING、零心跳零分片,两小时后被写成「Python 中断」。「没开工」被说成「跑挂了」, + * 排查方向被带偏过一次,故文案按证据分述,并把客户端最后一次上报的位置一起带出来 + * (排队中的任务会上报「排队等待空闲槽位」,与真在采集是两种成因)。 + * + * @param chunkCount 已落库的结果分片数(biz_task_chunk) + * @param lastChunkAt 最后一次结果分片回传时间,从未回传时为 null + * @param lastHeartbeatAt 最后一次客户端心跳时间,Redis 无记录时为 null + * @param lastProgress 客户端最后一次上报的位置/阶段文本,老客户端或无位置信息时为 null + * @param fileJobExhausted 结果文件作业是否已重试耗尽(true 时责任不在客户端回传) + * @param timeoutMinutes 判死阈值(分钟) + */ +public record StaleTaskDeathReason(long chunkCount, + LocalDateTime lastChunkAt, + LocalDateTime lastHeartbeatAt, + String lastProgress, + boolean fileJobExhausted, + int timeoutMinutes) { + + private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("MM-dd HH:mm"); + + public String describe() { + if (chunkCount > 0 && fileJobExhausted) { + return "已收到 " + chunkCount + " 个结果分片,但结果文件生成多次失败,任务已自动结束"; + } + String headline = chunkCount <= 0L + ? "客户端未回传任何检测结果,超过 " + Math.max(1, timeoutMinutes) + " 分钟无进展" + : "客户端回传中断"; + List evidence = new ArrayList<>(); + if (chunkCount > 0L) { + evidence.add("已收到 " + chunkCount + " 个分片,最后一次 " + at(lastChunkAt)); + } + evidence.add("最后心跳 " + at(lastHeartbeatAt)); + if (lastProgress != null && !lastProgress.isBlank()) { + evidence.add("最后上报 " + lastProgress.trim()); + } + return headline + ",任务已自动结束(" + String.join(";", evidence) + ")"; + } + + private static String at(LocalDateTime time) { + return time == null ? "时间未知" : time.format(TIME); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java index adc6395aa..2b00e60fb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java @@ -28,7 +28,7 @@ public class AppearancePatentHistoryItemVo { private String taskStatus; @Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true") private Boolean success; - @Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final appearance patent result") + @Schema(description = "错误信息。任务失败时返回,例如客户端未回传任何检测结果、结果文件生成失败等。", example = "客户端未回传任何检测结果,超过 30 分钟无进展,任务已自动结束(最后心跳 10-01 12:34)") private String error; @Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716") private Integer rowCount; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 04aa5ef5b..26e012514 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -13,6 +13,7 @@ import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.common.util.GroupResultPropagator; import com.nanri.aiimage.common.util.FailedStatusRowFilter; +import com.nanri.aiimage.common.util.StaleTaskDeathReason; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.StorageProperties; @@ -650,7 +651,7 @@ public class AppearancePatentTaskService { continue; } try (lockHandle) { - finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result"); + finalizeStaleTask(task.getId(), describeStaleDeath(task.getId())); } } finalizeNoUploadStaleTasks(); @@ -665,10 +666,44 @@ public class AppearancePatentTaskService { return; } try (lockHandle) { - finalizeStaleTask(taskId, "Python interrupted before uploading final appearance patent result"); + finalizeStaleTask(taskId, describeStaleDeath(taskId)); } } + /** + * 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。 + * 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result", + * 把「没开跑」说成「跑挂了」。 + */ + private String describeStaleDeath(Long taskId) { + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId); + return new StaleTaskDeathReason( + persistedResultChunkCount(taskId), + lastResultChunkAt(taskId), + heartbeatMillis > 0L + ? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault()) + : null, + taskHeartbeatPositionService.describe(taskId), + taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE), + properties.getStaleTimeoutMinutes()).describe(); + } + + private long persistedResultChunkCount(Long taskId) { + if (taskId == null) { + return 0L; + } + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + return count == null ? 0L : count; + } + + /** 最后一次结果分片回传时间;从未回传时为 null。 */ + private LocalDateTime lastResultChunkAt(Long taskId) { + List rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId)); + return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt(); + } + private List listStaleFinalizeCandidates() { // P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。 // APPEARANCE_PATENT 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java index a0b6eb66c..f94c78f97 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java @@ -28,7 +28,7 @@ public class SimilarAsinHistoryItemVo { private String taskStatus; @Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true") private Boolean success; - @Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final similar ASIN result") + @Schema(description = "错误信息。任务失败时返回,例如客户端未回传任何检测结果、结果文件生成失败等。", example = "客户端回传中断,任务已自动结束(已收到 3 个分片,最后一次 10-01 12:34)") private String error; @Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716") private Integer rowCount; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index c7f20e713..9f1abdd93 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -13,6 +13,7 @@ import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; import com.nanri.aiimage.common.security.AdminAuthSupport; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.common.util.FailedStatusRowFilter; +import com.nanri.aiimage.common.util.StaleTaskDeathReason; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.SimilarAsinProperties; import com.nanri.aiimage.config.StorageProperties; @@ -94,6 +95,7 @@ import org.springframework.transaction.support.TransactionTemplate; import java.io.File; import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.nio.charset.StandardCharsets; @@ -867,7 +869,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { continue; } try (taskLockHandle) { - finalizeStaleTask(task.getId(), "Python interrupted before uploading final similar ASIN result"); + finalizeStaleTask(task.getId(), describeStaleDeath(task.getId())); } } finalizeNoUploadStaleTasks(); @@ -882,10 +884,44 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { return; } try (taskLockHandle) { - finalizeStaleTask(taskId, "Python interrupted before uploading final similar ASIN result"); + finalizeStaleTask(taskId, describeStaleDeath(taskId)); } } + /** + * 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。 + * 与 {@code AppearancePatentTaskService#describeStaleDeath} 同型,共用 + * {@link StaleTaskDeathReason} 的措辞。 + */ + private String describeStaleDeath(Long taskId) { + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId); + return new StaleTaskDeathReason( + persistedResultChunkCount(taskId), + lastResultChunkAt(taskId), + heartbeatMillis > 0L + ? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault()) + : null, + taskHeartbeatPositionService.describe(taskId), + taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE), + properties.getStaleTimeoutMinutes()).describe(); + } + + private long persistedResultChunkCount(Long taskId) { + if (taskId == null) { + return 0L; + } + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + return count == null ? 0L : count; + } + + /** 最后一次结果分片回传时间;从未回传时为 null。 */ + private LocalDateTime lastResultChunkAt(Long taskId) { + List rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId)); + return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt(); + } + private List listStaleFinalizeCandidates() { // P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。 // SIMILAR_ASIN 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。 diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/util/StaleTaskDeathReasonTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/util/StaleTaskDeathReasonTest.java new file mode 100644 index 000000000..778ec42e3 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/common/util/StaleTaskDeathReasonTest.java @@ -0,0 +1,66 @@ +package com.nanri.aiimage.common.util; + +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StaleTaskDeathReasonTest { + + private static final LocalDateTime T = LocalDateTime.of(2026, 10, 1, 12, 34, 5); + + @Test + void neverUploadedSaysNoResultUploadedNotThatPythonCrashed() { + String reason = new StaleTaskDeathReason(0L, null, T, null, false, 30).describe(); + assertTrue(reason.contains("未回传任何检测结果"), reason); + assertTrue(reason.contains("超过 30 分钟无进展"), reason); + assertTrue(reason.contains("最后心跳 10-01 12:34"), reason); + assertFalse(reason.contains("Python"), reason); + assertFalse(reason.contains("最后上报"), reason); + } + + @Test + void queuedTaskIsToldApartFromACrashedOneByTheReportedPhase() { + String reason = new StaleTaskDeathReason(0L, null, T, "排队等待空闲槽位", false, 30).describe(); + assertTrue(reason.contains("最后上报 排队等待空闲槽位"), reason); + assertFalse(reason.contains("已收到"), reason); + } + + @Test + void missingHeartbeatAndBlankPhaseDegradeQuietly() { + String reason = new StaleTaskDeathReason(0L, null, null, " ", false, 30).describe(); + assertTrue(reason.contains("最后心跳 时间未知"), reason); + assertFalse(reason.contains("最后上报"), reason); + } + + @Test + void interruptedUploadReportsChunksAndLastChunkTime() { + String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", false, 30).describe(); + assertTrue(reason.contains("客户端回传中断"), reason); + assertTrue(reason.contains("已收到 7 个分片,最后一次 10-01 12:34"), reason); + assertTrue(reason.contains("最后上报 第 120/500 组"), reason); + assertFalse(reason.contains("结果文件生成"), reason); + } + + @Test + void exhaustedFileJobBlamesFileGenerationInsteadOfTheClient() { + String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", true, 30).describe(); + assertTrue(reason.contains("结果文件生成多次失败"), reason); + assertFalse(reason.contains("回传中断"), reason); + } + + @Test + void exhaustedFileJobWithoutAnyChunkStillReportsClientSilence() { + String reason = new StaleTaskDeathReason(0L, null, T, null, true, 30).describe(); + assertTrue(reason.contains("未回传任何检测结果"), reason); + assertFalse(reason.contains("结果文件生成"), reason); + } + + @Test + void nonPositiveTimeoutNeverPrintsZeroMinutes() { + assertTrue(new StaleTaskDeathReason(0L, null, null, null, false, 0) + .describe().contains("超过 1 分钟")); + } +}