feat(任务判死): 心跳正常但 180 分钟无结果上报的二次判死线(13 模块)+ 同期待发改动

判死线(治 28131 型「主线程卡死、心跳线程照发」):
- 判据改看 biz_task_scope_state.last_chunk_at(HTTP 心跳不刷新它);从未上报跳过不判
- 中央线覆盖 DELETE_BRAND/PRODUCT_RISK_RESOLVE/PRICE_TRACK/SHOP_MATCH/PATROL_DELETE/QUERY_ASIN/WITHDRAW
- 自带线接入 COLLECT_DATA/SIMILAR_ASIN/APPEARANCE_PATENT/SHOP_DATA_CRAWL/PUBLISH/BRAND
- 客户端心跳带处理位置 progressText,判死文案含最后位置;no-result-upload-timeout-minutes 默认 180(0 关闭)

同期带上另一工作流的待发改动:跟价换 IP 重试、品牌检测重试上限与 LLM 并发下调、
教程包后台管理页与 V126 迁移、admin-vue 教程记录页。
This commit is contained in:
2026-09-14 17:56:20 +08:00
parent b2b9983b07
commit 792b5f8bce
36 changed files with 1538 additions and 57 deletions
@@ -36,6 +36,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
@@ -70,6 +71,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Collectors;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@@ -116,6 +118,10 @@ public class PublishTaskService {
@Value("${aiimage.publish.stale-timeout-minutes:30}")
private int staleTimeoutMinutes;
/** 二次判死线(心跳正常但连续 N 分钟无结果分片上报)阈值,分钟;<=0 关闭本线。 */
@Value("${aiimage.publish.no-result-upload-timeout-minutes:180}")
private int noResultUploadTimeoutMinutes;
public PublishParseVo parseAndCreateTask(PublishParseRequest request) {
validateParseRequest(request);
List<PreparedFile> preparedFiles = new ArrayList<>();
@@ -459,6 +465,7 @@ public class PublishTaskService {
candidate.getId(), ex.getMessage());
}
}
failNoUploadStaleTasks();
}
}
@@ -685,10 +692,93 @@ public class PublishTaskService {
}
private void failStaleTaskLocked(Long taskId, LocalDateTime threshold) {
failStaleTaskLockedInternal(taskId,
task -> task.getUpdatedAt() != null && task.getUpdatedAt().isBefore(threshold),
"任务心跳超时");
}
/**
* 二次判死:Python 心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
*
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
* 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。
* owner 过滤与 A 线保持一致(避免双节点同时处理非本实例任务);
* 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报跳过,避免误杀首批较慢的正常任务。
*/
private void failNoUploadStaleTasks() {
long minutes = noResultUploadTimeoutMinutes;
if (minutes <= 0) {
return;
}
LocalDateTime heartbeatThreshold = LocalDateTime.now().minusMinutes(Math.max(5, staleTimeoutMinutes));
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.and(owner -> owner
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId()))
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
.lt(FileTaskEntity::getCreatedAt, cutoff)
.orderByAsc(FileTaskEntity::getCreatedAt)
.last("limit 100"));
if (candidates.isEmpty()) {
return;
}
Map<Long, LocalDateTime> lastResultAtByTaskId = new LinkedHashMap<>();
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
candidates.stream().map(FileTaskEntity::getId).toList())) {
if (dto.taskId() != null && dto.lastChunkAt() != null) {
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
}
}
for (FileTaskEntity candidate : candidates) {
LocalDateTime lastResultAt = lastResultAtByTaskId.get(candidate.getId());
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
continue;
}
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
continue;
}
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
if (lock == null) {
continue;
}
String error = "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")";
transactionTemplate.executeWithoutResult(status -> failStaleTaskLockedInternal(
candidate.getId(),
task -> {
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现新结果上报 → 任务已恢复
if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) {
return false;
}
LocalDateTime freshLastChunk = queryLastChunkAt(task.getId());
return freshLastChunk != null && !freshLastChunk.isAfter(cutoff);
},
error));
} catch (Exception ex) {
log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}",
candidate.getId(), ex.getMessage());
}
}
}
/** 单任务最后一次结果分片上报时间;无记录返回 null。 */
private LocalDateTime queryLastChunkAt(Long taskId) {
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
return rows.isEmpty() ? null : rows.get(0).lastChunkAt();
}
/** 收尾条件由调用方判定(A 线=心跳陈旧;B 线=心跳新鲜但结果上报陈旧),条件通过后走同一收尾路径。 */
private void failStaleTaskLockedInternal(Long taskId, Predicate<FileTaskEntity> staleCheck, String error) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())
|| !STATUS_RUNNING.equals(task.getStatus())
|| task.getUpdatedAt() == null || !task.getUpdatedAt().isBefore(threshold)) {
|| !staleCheck.test(task)) {
return;
}
ensureTaskOwnedByCurrentInstance(task, "cleanup stale publish task");
@@ -696,7 +786,6 @@ public class PublishTaskService {
return;
}
LocalDateTime now = LocalDateTime.now();
String error = "任务心跳超时";
List<PublishFileEntity> files = listTaskFiles(taskId);
for (PublishFileEntity file : files) {
if (!STATUS_PENDING.equals(file.getStatus()) && !STATUS_RUNNING.equals(file.getStatus())) {