feat(任务判死): 心跳正常但 180 分钟无结果上报的二次判死线(13 模块)+ 同期待发改动
判死线(治 28131 型「主线程卡死、心跳线程照发」): - 判据改看 biz_task_scope_state.last_chunk_at(HTTP 心跳不刷新它);从未上报跳过不判 - 中央线覆盖 DELETE_BRAND/PRODUCT_RISK_RESOLVE/PRICE_TRACK/SHOP_MATCH/PATROL_DELETE/QUERY_ASIN/WITHDRAW - 自带线接入 COLLECT_DATA/SIMILAR_ASIN/APPEARANCE_PATENT/SHOP_DATA_CRAWL/PUBLISH/BRAND - 客户端心跳带处理位置 progressText,判死文案含最后位置;no-result-upload-timeout-minutes 默认 180(0 关闭) 同期带上另一工作流的待发改动:跟价换 IP 重试、品牌检测重试上限与 LLM 并发下调、 教程包后台管理页与 V126 迁移、admin-vue 教程记录页。
This commit is contained in:
+91
-2
@@ -36,6 +36,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
@@ -70,6 +71,7 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Collectors;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
@@ -116,6 +118,10 @@ public class PublishTaskService {
|
||||
@Value("${aiimage.publish.stale-timeout-minutes:30}")
|
||||
private int staleTimeoutMinutes;
|
||||
|
||||
/** 二次判死线(心跳正常但连续 N 分钟无结果分片上报)阈值,分钟;<=0 关闭本线。 */
|
||||
@Value("${aiimage.publish.no-result-upload-timeout-minutes:180}")
|
||||
private int noResultUploadTimeoutMinutes;
|
||||
|
||||
public PublishParseVo parseAndCreateTask(PublishParseRequest request) {
|
||||
validateParseRequest(request);
|
||||
List<PreparedFile> preparedFiles = new ArrayList<>();
|
||||
@@ -459,6 +465,7 @@ public class PublishTaskService {
|
||||
candidate.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
failNoUploadStaleTasks();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -685,10 +692,93 @@ public class PublishTaskService {
|
||||
}
|
||||
|
||||
private void failStaleTaskLocked(Long taskId, LocalDateTime threshold) {
|
||||
failStaleTaskLockedInternal(taskId,
|
||||
task -> task.getUpdatedAt() != null && task.getUpdatedAt().isBefore(threshold),
|
||||
"任务心跳超时");
|
||||
}
|
||||
|
||||
/**
|
||||
* 二次判死:Python 心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
|
||||
*
|
||||
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||
* 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。
|
||||
* owner 过滤与 A 线保持一致(避免双节点同时处理非本实例任务);
|
||||
* 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报跳过,避免误杀首批较慢的正常任务。
|
||||
*/
|
||||
private void failNoUploadStaleTasks() {
|
||||
long minutes = noResultUploadTimeoutMinutes;
|
||||
if (minutes <= 0) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime heartbeatThreshold = LocalDateTime.now().minusMinutes(Math.max(5, staleTimeoutMinutes));
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.and(owner -> owner
|
||||
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL")
|
||||
.or()
|
||||
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''")
|
||||
.or()
|
||||
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId()))
|
||||
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
|
||||
.lt(FileTaskEntity::getCreatedAt, cutoff)
|
||||
.orderByAsc(FileTaskEntity::getCreatedAt)
|
||||
.last("limit 100"));
|
||||
if (candidates.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Map<Long, LocalDateTime> lastResultAtByTaskId = new LinkedHashMap<>();
|
||||
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||
candidates.stream().map(FileTaskEntity::getId).toList())) {
|
||||
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||
}
|
||||
}
|
||||
for (FileTaskEntity candidate : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(candidate.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
|
||||
continue;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
|
||||
if (lock == null) {
|
||||
continue;
|
||||
}
|
||||
String error = "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")";
|
||||
transactionTemplate.executeWithoutResult(status -> failStaleTaskLockedInternal(
|
||||
candidate.getId(),
|
||||
task -> {
|
||||
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现新结果上报 → 任务已恢复
|
||||
if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) {
|
||||
return false;
|
||||
}
|
||||
LocalDateTime freshLastChunk = queryLastChunkAt(task.getId());
|
||||
return freshLastChunk != null && !freshLastChunk.isAfter(cutoff);
|
||||
},
|
||||
error));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}",
|
||||
candidate.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 单任务最后一次结果分片上报时间;无记录返回 null。 */
|
||||
private LocalDateTime queryLastChunkAt(Long taskId) {
|
||||
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
|
||||
return rows.isEmpty() ? null : rows.get(0).lastChunkAt();
|
||||
}
|
||||
|
||||
/** 收尾条件由调用方判定(A 线=心跳陈旧;B 线=心跳新鲜但结果上报陈旧),条件通过后走同一收尾路径。 */
|
||||
private void failStaleTaskLockedInternal(Long taskId, Predicate<FileTaskEntity> staleCheck, String error) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())
|
||||
|| !STATUS_RUNNING.equals(task.getStatus())
|
||||
|| task.getUpdatedAt() == null || !task.getUpdatedAt().isBefore(threshold)) {
|
||||
|| !staleCheck.test(task)) {
|
||||
return;
|
||||
}
|
||||
ensureTaskOwnedByCurrentInstance(task, "cleanup stale publish task");
|
||||
@@ -696,7 +786,6 @@ public class PublishTaskService {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
String error = "任务心跳超时";
|
||||
List<PublishFileEntity> files = listTaskFiles(taskId);
|
||||
for (PublishFileEntity file : files) {
|
||||
if (!STATUS_PENDING.equals(file.getStatus()) && !STATUS_RUNNING.equals(file.getStatus())) {
|
||||
|
||||
Reference in New Issue
Block a user