feat(任务判死): 心跳正常但 180 分钟无结果上报的二次判死线(13 模块)+ 同期待发改动
判死线(治 28131 型「主线程卡死、心跳线程照发」): - 判据改看 biz_task_scope_state.last_chunk_at(HTTP 心跳不刷新它);从未上报跳过不判 - 中央线覆盖 DELETE_BRAND/PRODUCT_RISK_RESOLVE/PRICE_TRACK/SHOP_MATCH/PATROL_DELETE/QUERY_ASIN/WITHDRAW - 自带线接入 COLLECT_DATA/SIMILAR_ASIN/APPEARANCE_PATENT/SHOP_DATA_CRAWL/PUBLISH/BRAND - 客户端心跳带处理位置 progressText,判死文案含最后位置;no-result-upload-timeout-minutes 默认 180(0 关闭) 同期带上另一工作流的待发改动:跟价换 IP 重试、品牌检测重试上限与 LLM 并发下调、 教程包后台管理页与 V126 迁移、admin-vue 教程记录页。
This commit is contained in:
+86
@@ -51,6 +51,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
@@ -179,6 +180,10 @@ public class CollectDataService {
|
||||
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
||||
private long staleTimeoutMinutes;
|
||||
|
||||
/** 二次判死线(心跳正常但连续 N 分钟无结果分片上报)阈值,分钟;<=0 关闭本线。 */
|
||||
@Value("${aiimage.collect-data.no-result-upload-timeout-minutes:180}")
|
||||
private long noResultUploadTimeoutMinutes;
|
||||
|
||||
@Value("${aiimage.collect-data.max-source-file-bytes:0}")
|
||||
private Long maxSourceFileBytes;
|
||||
|
||||
@@ -526,6 +531,7 @@ public class CollectDataService {
|
||||
for (FileTaskEntity task : tasks) {
|
||||
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
|
||||
}
|
||||
finalizeNoUploadStaleTasks(threshold);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -565,6 +571,86 @@ public class CollectDataService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
|
||||
*
|
||||
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。
|
||||
* 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用
|
||||
* biz_task_scope_state.last_chunk_at(只随结果分片上报刷新);
|
||||
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||
*/
|
||||
private void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) {
|
||||
long minutes = noResultUploadTimeoutMinutes;
|
||||
if (minutes <= 0) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
|
||||
.lt(FileTaskEntity::getCreatedAt, cutoff)
|
||||
.orderByAsc(FileTaskEntity::getCreatedAt)
|
||||
.last("limit 200"));
|
||||
if (candidates.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
|
||||
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||
candidates.stream().map(FileTaskEntity::getId).toList())) {
|
||||
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||
}
|
||||
}
|
||||
for (FileTaskEntity task : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
finalizeNoUploadStaleTask(task.getId(), lastResultAt, minutes);
|
||||
}
|
||||
}
|
||||
|
||||
/** 单个任务的二次判死收尾:锁内复查心跳后,有分片→组装部分工作簿;无分片→标失败。 */
|
||||
private void finalizeNoUploadStaleTask(Long taskId, LocalDateTime lastResultAt, long minutes) {
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
|
||||
if (lock == null) {
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes));
|
||||
if (task == null
|
||||
|| !MODULE_TYPE.equals(task.getModuleType())
|
||||
|| !STATUS_RUNNING.equals(task.getStatus())
|
||||
|| task.getUpdatedAt() == null
|
||||
|| task.getUpdatedAt().isBefore(heartbeatFreshAfter)) {
|
||||
// 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理)
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
if (hasReceivedChunks(taskId)) {
|
||||
enqueueFinalWorkbook(task, result, stats);
|
||||
log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}",
|
||||
taskId, lastResultAt, minutes, stats.finalRowCount);
|
||||
return;
|
||||
}
|
||||
markTaskFailed(task, result,
|
||||
"连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")",
|
||||
stats);
|
||||
log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}",
|
||||
taskId, lastResultAt, minutes);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}",
|
||||
taskId, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public CollectDataDashboardVo dashboard(Long userId) {
|
||||
CollectDataDashboardVo vo = new CollectDataDashboardVo();
|
||||
vo.setPendingTaskCount(countActiveTasks(userId));
|
||||
|
||||
Reference in New Issue
Block a user