fix(任务判死): 外观/货源判死文案按证据陈述,不再断言 Python 中断
2026-10-01 三条外观任务被心跳判死时一行业务结果都没回传过(Linux worker 一次领 5 条却只有 2 个采集槽,多出来的任务零心跳零分片直到钟面走完),而文案固定写 "Python interrupted before uploading final result",把「没开跑」说成「跑挂了」,排查方向被带偏过一次。现在按已落库分片数、 最后回传/心跳时间、客户端最后一次上报的位置分述;结果文件作业重试耗尽也不再写成客户端中断。
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 客户端型任务(外观专利 / 货源)心跳判死文案:只写证据支持的话。
|
||||
*
|
||||
* <p>历史上这里固定写 "Python interrupted before uploading final result",而 2026-10-01 线上三条
|
||||
* 外观任务其实一行业务结果都没回传过——Linux worker 一次领 5 条却只有 2 个采集槽,多出来的任务
|
||||
* 状态是 RUNNING、零心跳零分片,两小时后被写成「Python 中断」。「没开工」被说成「跑挂了」,
|
||||
* 排查方向被带偏过一次,故文案按证据分述,并把客户端最后一次上报的位置一起带出来
|
||||
* (排队中的任务会上报「排队等待空闲槽位」,与真在采集是两种成因)。
|
||||
*
|
||||
* @param chunkCount 已落库的结果分片数(biz_task_chunk)
|
||||
* @param lastChunkAt 最后一次结果分片回传时间,从未回传时为 null
|
||||
* @param lastHeartbeatAt 最后一次客户端心跳时间,Redis 无记录时为 null
|
||||
* @param lastProgress 客户端最后一次上报的位置/阶段文本,老客户端或无位置信息时为 null
|
||||
* @param fileJobExhausted 结果文件作业是否已重试耗尽(true 时责任不在客户端回传)
|
||||
* @param timeoutMinutes 判死阈值(分钟)
|
||||
*/
|
||||
public record StaleTaskDeathReason(long chunkCount,
|
||||
LocalDateTime lastChunkAt,
|
||||
LocalDateTime lastHeartbeatAt,
|
||||
String lastProgress,
|
||||
boolean fileJobExhausted,
|
||||
int timeoutMinutes) {
|
||||
|
||||
private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("MM-dd HH:mm");
|
||||
|
||||
public String describe() {
|
||||
if (chunkCount > 0 && fileJobExhausted) {
|
||||
return "已收到 " + chunkCount + " 个结果分片,但结果文件生成多次失败,任务已自动结束";
|
||||
}
|
||||
String headline = chunkCount <= 0L
|
||||
? "客户端未回传任何检测结果,超过 " + Math.max(1, timeoutMinutes) + " 分钟无进展"
|
||||
: "客户端回传中断";
|
||||
List<String> evidence = new ArrayList<>();
|
||||
if (chunkCount > 0L) {
|
||||
evidence.add("已收到 " + chunkCount + " 个分片,最后一次 " + at(lastChunkAt));
|
||||
}
|
||||
evidence.add("最后心跳 " + at(lastHeartbeatAt));
|
||||
if (lastProgress != null && !lastProgress.isBlank()) {
|
||||
evidence.add("最后上报 " + lastProgress.trim());
|
||||
}
|
||||
return headline + ",任务已自动结束(" + String.join(";", evidence) + ")";
|
||||
}
|
||||
|
||||
private static String at(LocalDateTime time) {
|
||||
return time == null ? "时间未知" : time.format(TIME);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -28,7 +28,7 @@ public class AppearancePatentHistoryItemVo {
|
||||
private String taskStatus;
|
||||
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true")
|
||||
private Boolean success;
|
||||
@Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final appearance patent result")
|
||||
@Schema(description = "错误信息。任务失败时返回,例如客户端未回传任何检测结果、结果文件生成失败等。", example = "客户端未回传任何检测结果,超过 30 分钟无进展,任务已自动结束(最后心跳 10-01 12:34)")
|
||||
private String error;
|
||||
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
|
||||
private Integer rowCount;
|
||||
|
||||
+37
-2
@@ -13,6 +13,7 @@ import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.util.GroupResultPropagator;
|
||||
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
|
||||
import com.nanri.aiimage.common.util.StaleTaskDeathReason;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
@@ -650,7 +651,7 @@ public class AppearancePatentTaskService {
|
||||
continue;
|
||||
}
|
||||
try (lockHandle) {
|
||||
finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result");
|
||||
finalizeStaleTask(task.getId(), describeStaleDeath(task.getId()));
|
||||
}
|
||||
}
|
||||
finalizeNoUploadStaleTasks();
|
||||
@@ -665,10 +666,44 @@ public class AppearancePatentTaskService {
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
finalizeStaleTask(taskId, "Python interrupted before uploading final appearance patent result");
|
||||
finalizeStaleTask(taskId, describeStaleDeath(taskId));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。
|
||||
* 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result",
|
||||
* 把「没开跑」说成「跑挂了」。
|
||||
*/
|
||||
private String describeStaleDeath(Long taskId) {
|
||||
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId);
|
||||
return new StaleTaskDeathReason(
|
||||
persistedResultChunkCount(taskId),
|
||||
lastResultChunkAt(taskId),
|
||||
heartbeatMillis > 0L
|
||||
? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault())
|
||||
: null,
|
||||
taskHeartbeatPositionService.describe(taskId),
|
||||
taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE),
|
||||
properties.getStaleTimeoutMinutes()).describe();
|
||||
}
|
||||
|
||||
private long persistedResultChunkCount(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
/** 最后一次结果分片回传时间;从未回传时为 null。 */
|
||||
private LocalDateTime lastResultChunkAt(Long taskId) {
|
||||
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
|
||||
return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt();
|
||||
}
|
||||
|
||||
private List<FileTaskEntity> listStaleFinalizeCandidates() {
|
||||
// P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。
|
||||
// APPEARANCE_PATENT 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ public class SimilarAsinHistoryItemVo {
|
||||
private String taskStatus;
|
||||
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true")
|
||||
private Boolean success;
|
||||
@Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final similar ASIN result")
|
||||
@Schema(description = "错误信息。任务失败时返回,例如客户端未回传任何检测结果、结果文件生成失败等。", example = "客户端回传中断,任务已自动结束(已收到 3 个分片,最后一次 10-01 12:34)")
|
||||
private String error;
|
||||
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
|
||||
private Integer rowCount;
|
||||
|
||||
+38
-2
@@ -13,6 +13,7 @@ import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
|
||||
import com.nanri.aiimage.common.util.StaleTaskDeathReason;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
@@ -94,6 +95,7 @@ import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.File;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -867,7 +869,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
continue;
|
||||
}
|
||||
try (taskLockHandle) {
|
||||
finalizeStaleTask(task.getId(), "Python interrupted before uploading final similar ASIN result");
|
||||
finalizeStaleTask(task.getId(), describeStaleDeath(task.getId()));
|
||||
}
|
||||
}
|
||||
finalizeNoUploadStaleTasks();
|
||||
@@ -882,10 +884,44 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
return;
|
||||
}
|
||||
try (taskLockHandle) {
|
||||
finalizeStaleTask(taskId, "Python interrupted before uploading final similar ASIN result");
|
||||
finalizeStaleTask(taskId, describeStaleDeath(taskId));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。
|
||||
* 与 {@code AppearancePatentTaskService#describeStaleDeath} 同型,共用
|
||||
* {@link StaleTaskDeathReason} 的措辞。
|
||||
*/
|
||||
private String describeStaleDeath(Long taskId) {
|
||||
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId);
|
||||
return new StaleTaskDeathReason(
|
||||
persistedResultChunkCount(taskId),
|
||||
lastResultChunkAt(taskId),
|
||||
heartbeatMillis > 0L
|
||||
? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault())
|
||||
: null,
|
||||
taskHeartbeatPositionService.describe(taskId),
|
||||
taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE),
|
||||
properties.getStaleTimeoutMinutes()).describe();
|
||||
}
|
||||
|
||||
private long persistedResultChunkCount(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
/** 最后一次结果分片回传时间;从未回传时为 null。 */
|
||||
private LocalDateTime lastResultChunkAt(Long taskId) {
|
||||
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
|
||||
return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt();
|
||||
}
|
||||
|
||||
private List<FileTaskEntity> listStaleFinalizeCandidates() {
|
||||
// P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。
|
||||
// SIMILAR_ASIN 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。
|
||||
|
||||
Reference in New Issue
Block a user