feat(task-retry): 失败任务克隆重试,最多 3 代并跳过欠费/宕机
- 废弃只复制 request_json 的空壳续跑:三类任务的解析输入都在模块侧状态里, 线上历史 19 条续跑因此全部失败;改为由 ClientTaskRetrySpi 克隆任务头 + 采集明细 / 解析 scope state / 源结果行,新任务直接可执行。 - 不可抗力不重试:LLM 密钥或代理 IP 欠费、服务宕机、账号自动登录失败。 - 新增内部回填端点 POST /api/internal/tasks/retry-failed(内部令牌, dryRun 默认 true),用于补扫 30 分钟窗口之外的历史失败。
This commit is contained in:
+42
@@ -0,0 +1,42 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskRetryCloneSupport;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskRetrySpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* 外观专利的失败重试:与相似ASIN同构,克隆任务头 + 解析 scope state,
|
||||
* 后台 worker 按新 taskId 重新取 queue-payload 执行。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AppearancePatentTaskRetrySpiImpl implements ClientTaskRetrySpi {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskRetryCloneSupport cloneSupport;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return AppearancePatentTaskService.MODULE_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Long cloneForRetry(FileTaskEntity original) {
|
||||
String resultJson = cloneSupport.resultJsonWithoutOwner(original.getResultJson());
|
||||
FileTaskEntity child = cloneSupport.buildChild(original, resultJson);
|
||||
fileTaskMapper.insert(child);
|
||||
if (cloneSupport.copyParsedScope(original.getId(), child.getId(), moduleType()) == null) {
|
||||
throw new IllegalStateException("外观专利任务缺少解析 scope,无法重试");
|
||||
}
|
||||
cloneSupport.copySourceResult(original.getId(), child.getId(), moduleType());
|
||||
log.info("[appearance-patent][retry] 已克隆任务 original={} new={}", original.getId(), child.getId());
|
||||
return child.getId();
|
||||
}
|
||||
}
|
||||
+6
@@ -5,10 +5,16 @@ import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface CollectDataItemMapper extends BaseMapper<CollectDataItemEntity> {
|
||||
|
||||
int insertBatch(@Param("rows") List<CollectDataItemEntity> rows);
|
||||
|
||||
/** 失败任务重试:把原任务的明细行整体复制到新任务,避免重新解析源 Excel。 */
|
||||
int copyItemsForRetry(@Param("sourceTaskId") Long sourceTaskId,
|
||||
@Param("targetTaskId") Long targetTaskId,
|
||||
@Param("createdAt") LocalDateTime createdAt);
|
||||
}
|
||||
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskRetryCloneSupport;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskRetrySpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 采集数据的失败重试:明细行已在解析阶段落库(biz_collect_data_item),直接复制到新任务,
|
||||
* 不重新解析源 Excel——源文件本地保留 24 小时后会被清理,重试不能依赖它。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class CollectDataTaskRetrySpiImpl implements ClientTaskRetrySpi {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final TaskRetryCloneSupport cloneSupport;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return CollectDataService.MODULE_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Long cloneForRetry(FileTaskEntity original) {
|
||||
Long itemCount = collectDataItemMapper.selectCount(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, original.getId()));
|
||||
if (itemCount == null || itemCount <= 0) {
|
||||
log.warn("[collect-data][retry] 原任务没有可复制的明细行 taskId={}", original.getId());
|
||||
return null;
|
||||
}
|
||||
int totalRows = itemCount > Integer.MAX_VALUE ? Integer.MAX_VALUE : itemCount.intValue();
|
||||
String resultJson = initialStatsJson(totalRows);
|
||||
FileTaskEntity child = cloneSupport.buildChild(original, resultJson);
|
||||
fileTaskMapper.insert(child);
|
||||
|
||||
int copied = collectDataItemMapper.copyItemsForRetry(
|
||||
original.getId(), child.getId(), LocalDateTime.now());
|
||||
if (copied != itemCount.intValue()) {
|
||||
throw new IllegalStateException("采集明细复制数量不一致 source=" + itemCount + " copied=" + copied);
|
||||
}
|
||||
cloneSupport.copySourceResult(original.getId(), child.getId(), moduleType(), totalRows);
|
||||
log.info("[collect-data][retry] 已克隆任务 original={} new={} items={}",
|
||||
original.getId(), child.getId(), copied);
|
||||
return child.getId();
|
||||
}
|
||||
|
||||
/** 与解析落库时的初始统计同形状,让客户端从第 1 行重新跑而不是继承上次的进度。 */
|
||||
private String initialStatsJson(int totalRows) {
|
||||
Map<String, Object> stats = new LinkedHashMap<>();
|
||||
stats.put("totalRows", totalRows);
|
||||
stats.put("receivedRows", 0);
|
||||
stats.put("processedRows", 0);
|
||||
stats.put("currentChunkRows", 0);
|
||||
stats.put("dedupeFilteredCount", 0);
|
||||
stats.put("invalidFilteredCount", 0);
|
||||
stats.put("brandRejectedCount", 0);
|
||||
stats.put("brandQueryFailedCount", 0);
|
||||
stats.put("finalRowCount", 0);
|
||||
stats.put("summaries", java.util.List.of());
|
||||
try {
|
||||
return objectMapper.writeValueAsString(stats);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("序列化采集重试统计失败", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskRetryCloneSupport;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskRetrySpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* 相似ASIN的失败重试:解析载荷指针在 resultJson 里、载荷本体在共享临时存储(72h),
|
||||
* 克隆任务头 + 解析 scope state 即可让客户端按新 taskId 重新回拉 parsed-payload 执行。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SimilarAsinTaskRetrySpiImpl implements ClientTaskRetrySpi {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskRetryCloneSupport cloneSupport;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return SimilarAsinTaskService.MODULE_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Long cloneForRetry(FileTaskEntity original) {
|
||||
String resultJson = cloneSupport.resultJsonWithoutOwner(original.getResultJson());
|
||||
FileTaskEntity child = cloneSupport.buildChild(original, resultJson);
|
||||
fileTaskMapper.insert(child);
|
||||
if (cloneSupport.copyParsedScope(original.getId(), child.getId(), moduleType()) == null) {
|
||||
throw new IllegalStateException("相似ASIN任务缺少解析 scope,无法重试");
|
||||
}
|
||||
cloneSupport.copySourceResult(original.getId(), child.getId(), moduleType());
|
||||
log.info("[similar-asin][retry] 已克隆任务 original={} new={}", original.getId(), child.getId());
|
||||
return child.getId();
|
||||
}
|
||||
}
|
||||
+37
@@ -3,12 +3,15 @@ package com.nanri.aiimage.modules.task.controller;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskClientPullVo;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskRetryBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskClientPullService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
@@ -31,6 +34,7 @@ import java.util.Locale;
|
||||
public class InternalTaskPullController {
|
||||
|
||||
private final TaskClientPullService taskClientPullService;
|
||||
private final TaskResumeService taskResumeService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
@GetMapping("/pull-pending")
|
||||
@@ -55,4 +59,37 @@ public class InternalTaskPullController {
|
||||
}
|
||||
return ApiResponse.success(taskClientPullService.pullPendingTasksInternal(moduleTypes));
|
||||
}
|
||||
|
||||
/**
|
||||
* 运维回填:扫描最近 N 小时失败的任务并克隆重试,最多重试 3 代(沿 resume_attempt)。
|
||||
*
|
||||
* <p>dryRun 默认 true,先看清会克隆哪些、哪些因欠费/宕机被跳过,再显式传 false 落库。
|
||||
* 定时扫描只看 30 分钟窗口,这个端点是给"窗口已经过去才发现"的历史失败收口用的。
|
||||
*/
|
||||
@PostMapping("/retry-failed")
|
||||
@Operation(
|
||||
summary = "失败任务克隆重试(运维回填)",
|
||||
description = "sinceHours 回看小时数(必填,1~336),limit 单轮上限(必填,1~500),dryRun 默认 true;"
|
||||
+ "types 为逗号分隔的模块白名单,缺省用已注册的克隆重试模块。")
|
||||
public ApiResponse<TaskRetryBatchVo> retryFailed(
|
||||
HttpServletRequest request,
|
||||
@RequestParam("sinceHours") long sinceHours,
|
||||
@RequestParam("limit") int limit,
|
||||
@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun,
|
||||
@RequestParam(value = "types", required = false) String types) {
|
||||
if (!adminAuthSupport.isTrustedInternalToken(request)) {
|
||||
return ApiResponse.fail(401, "内部令牌无效");
|
||||
}
|
||||
List<String> moduleTypes = new ArrayList<>();
|
||||
if (types != null && !types.isBlank()) {
|
||||
for (String raw : types.split(",")) {
|
||||
String value = raw.trim().toUpperCase(Locale.ROOT);
|
||||
if (!value.isEmpty() && !moduleTypes.contains(value)) {
|
||||
moduleTypes.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ApiResponse.success(TaskRetryBatchVo.from(
|
||||
taskResumeService.retryFailedTasks(sinceHours, limit, dryRun, moduleTypes)));
|
||||
}
|
||||
}
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package com.nanri.aiimage.modules.task.model.vo;
|
||||
|
||||
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** 失败任务克隆重试(运维回填)的单轮结果。 */
|
||||
@Data
|
||||
public class TaskRetryBatchVo {
|
||||
|
||||
private boolean dryRun;
|
||||
private int scanned;
|
||||
private int retried;
|
||||
private int skippedExisting;
|
||||
private int skippedForceMajeure;
|
||||
private int failed;
|
||||
private List<Long> candidateTaskIds;
|
||||
private List<Long> retriedTaskIds;
|
||||
|
||||
public static TaskRetryBatchVo from(TaskResumeService.ResumeStats stats) {
|
||||
TaskRetryBatchVo vo = new TaskRetryBatchVo();
|
||||
vo.setDryRun(stats.dryRun);
|
||||
vo.setScanned(stats.scannedTaskCount);
|
||||
vo.setRetried(stats.resumedTaskCount);
|
||||
vo.setSkippedExisting(stats.skippedTaskCount);
|
||||
vo.setSkippedForceMajeure(stats.forceMajeureSkippedCount);
|
||||
vo.setFailed(stats.cloneFailedCount);
|
||||
vo.setCandidateTaskIds(List.copyOf(stats.candidateTaskIds));
|
||||
vo.setRetriedTaskIds(List.copyOf(stats.retriedTaskIds));
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
+201
-117
@@ -1,219 +1,303 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskRetrySpi;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 客户端中断任务的自动续跑:**保留失败记录 + 重新排队一条 PENDING 续跑任务**。
|
||||
* 失败任务的自动重试:把仍可执行的原任务**克隆**成一条新的 PENDING 任务,最多重试 3 代。
|
||||
*
|
||||
* <p>场景:客户端被更新脚本/任务管理器杀掉时,{@code TaskHeartbeatService.markInterrupted} 会把在跑的任务
|
||||
* 标 FAILED(原因「客户端异常中断: …」),用户能看到发生过什么;但长任务(采集/相似ASIN/外观专利/单次跟价)
|
||||
* 一遇客户端重启就整个白跑,需要有人把它重新放回队列。本服务负责这件事。
|
||||
* <p>为什么不再沿用"只复制 request_json 的新任务":采集明细行、相似ASIN/外观专利的解析载荷
|
||||
* 都在解析阶段落成了模块侧状态,空壳续跑任务在客户端看来是一张空表。线上 2026-10-15~21
|
||||
* 实测 19 条续跑任务全部失败(长时间无心跳 / 结果为空 / 长期未被领取),根因即此。
|
||||
* 现在由各模块的 {@link ClientTaskRetrySpi} 负责把解析输入与载荷一起克隆过去。
|
||||
*
|
||||
* <p>续跑方式刻意复用已有链路:新建的任务落 PENDING,由 {@link TaskClientPullService} 的兜底拉取
|
||||
* (客户端每分钟 {@code GET /api/tasks/pull-pending})领走执行——不再新造第二套派发机制。
|
||||
* 只有注册了 {@link ClientTaskPullSpi} 的模块才会被续跑(这些模块的载荷能由服务端自行组装);
|
||||
* 上架/改价等写操作模块不在其中,避免盲目重跑。
|
||||
* <p>不可抗力不重试:LLM 密钥/代理 IP 欠费、服务宕机、账号自动登录失败等,重跑不会改变结果,
|
||||
* 还会白烧外部额度。判据集中在 {@link TaskRetryEligibility}。
|
||||
*
|
||||
* <p>封顶 {@code max-attempt}:会话持续不可用(账号被风控、紫鸟未就绪)时,不封顶会无限重排队。
|
||||
* 计数写在 {@code biz_file_task.resume_attempt}(V129),随续跑任务代代递增。
|
||||
*
|
||||
* <p>幂等:同一原任务已存在续跑任务({@code resume_of_task_id} 反查)则跳过,重复扫描安全。
|
||||
* <p>幂等:同一原任务已有重试子任务({@code resume_of_task_id} 反查)则跳过,重复扫描安全。
|
||||
* 代数封顶写在 {@code biz_file_task.resume_attempt}(V129),沿重试链递增。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class TaskResumeService {
|
||||
|
||||
private static final String STATUS_FAILED = "FAILED";
|
||||
private static final String STATUS_PENDING = "PENDING";
|
||||
/** 客户端重启中断的失败原因前缀,由 TaskHeartbeatService.markInterrupted 写入。 */
|
||||
private static final String CLIENT_INTERRUPT_PREFIX = "客户端异常中断";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
/** moduleType → 模块兜底载荷实现;只有注册了 SPI 的模块才能被自动续跑。 */
|
||||
private final Map<String, ClientTaskPullSpi> resumeHandlers;
|
||||
/** moduleType → 克隆重试实现;只有注册了本 SPI 的模块才会自动重试。 */
|
||||
private final Map<String, ClientTaskRetrySpi> retryHandlers;
|
||||
|
||||
/**
|
||||
* 续跑总开关。默认值必须与 {@code application.yml} 的
|
||||
* {@code ${AIIMAGE_TASK_RESUME_ENABLED:${AIIMAGE_CLIENT_TASK_PULL_ENABLED:false}}} 保持同一语义:
|
||||
* 继承兜底拉取的开关、两者都没配时为 <b>false</b>。
|
||||
*
|
||||
* <p>这里曾经是 {@code :true},而 yml 永远会给出值,所以注解里的默认在生效路径上是死代码 ——
|
||||
* 但它误导了读代码的人(以为默认开着),线上因此把「48 轮全零」当成了「恰好没有中断任务」。
|
||||
* {@code ${AIIMAGE_TASK_RESUME_ENABLED:${AIIMAGE_CLIENT_TASK_PULL_ENABLED:false}}} 保持同一语义。
|
||||
*/
|
||||
@Value("${aiimage.task-resume.enabled:${aiimage.client-task-pull.enabled:false}}")
|
||||
private boolean enabled;
|
||||
|
||||
/** 续跑代数上限:达到后不再重排队,任务保持 FAILED 等人工介入。 */
|
||||
/** 重试代数上限:达到后不再克隆,任务保持 FAILED 等人工介入。 */
|
||||
@Value("${aiimage.task-resume.max-attempt:3}")
|
||||
private int maxAttempt;
|
||||
|
||||
/** 只处理最近这段时间内中断的任务,避免开机扫描历史积压。 */
|
||||
/** 定时扫描只处理最近这段时间内失败的任务,避免开机扫描历史积压。 */
|
||||
@Value("${aiimage.task-resume.window-minutes:30}")
|
||||
private long windowMinutes;
|
||||
|
||||
@Value("${aiimage.task-resume.limit:20}")
|
||||
private int limit;
|
||||
|
||||
public TaskResumeService(FileTaskMapper fileTaskMapper, List<ClientTaskPullSpi> pullSpiHandlers) {
|
||||
public TaskResumeService(FileTaskMapper fileTaskMapper,
|
||||
List<ClientTaskRetrySpi> retrySpiHandlers) {
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
Map<String, ClientTaskPullSpi> index = new LinkedHashMap<>();
|
||||
if (pullSpiHandlers != null) {
|
||||
for (ClientTaskPullSpi handler : pullSpiHandlers) {
|
||||
String moduleType = handler.moduleType();
|
||||
if (moduleType == null || moduleType.isBlank()) {
|
||||
Map<String, ClientTaskRetrySpi> retryIndex = new LinkedHashMap<>();
|
||||
if (retrySpiHandlers != null) {
|
||||
for (ClientTaskRetrySpi handler : retrySpiHandlers) {
|
||||
String moduleType = normalize(handler.moduleType());
|
||||
if (moduleType == null) {
|
||||
continue;
|
||||
}
|
||||
index.put(moduleType.trim().toUpperCase(Locale.ROOT), handler);
|
||||
retryIndex.put(moduleType, handler);
|
||||
}
|
||||
}
|
||||
this.resumeHandlers = Map.copyOf(index);
|
||||
log.info("[task-resume] 可自动续跑模块注册完成 count={} modules={}", index.size(), index.keySet());
|
||||
this.retryHandlers = Map.copyOf(retryIndex);
|
||||
log.info("[task-resume] 可克隆重试模块注册完成 count={} modules={}",
|
||||
retryIndex.size(), retryIndex.keySet());
|
||||
}
|
||||
|
||||
private static String normalize(String moduleType) {
|
||||
if (moduleType == null || moduleType.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return moduleType.trim().toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动时打印本轮真正生效的配置,而不是让运维去反推 yml 的嵌套占位符。
|
||||
*
|
||||
* <p>{@code enabled=false} 会让巡检里的 resume 统计恒为 0,与"恰好没有中断任务"同形;
|
||||
* 没有这一行,"续跑从未执行过"这件事在日志里是不可见的。
|
||||
* <p>{@code enabled=false} 会让巡检里的 resume 统计恒为 0,与"恰好没有失败任务"同形;
|
||||
* 没有这一行,"重试从未执行过"这件事在日志里是不可见的。
|
||||
*/
|
||||
@PostConstruct
|
||||
public void logEffectiveConfig() {
|
||||
log.info("[task-resume] 生效配置 enabled={} maxAttempt={} windowMinutes={} limit={} modules={}",
|
||||
enabled, maxAttempt, windowMinutes, limit, resumeHandlers.keySet());
|
||||
enabled, maxAttempt, windowMinutes, limit, retryHandlers.keySet());
|
||||
}
|
||||
|
||||
/** 扫描一轮:把最近因客户端中断而失败、且未续跑过的任务重新排队。 */
|
||||
/** 定时扫描一轮:把最近失败且未重试过的任务克隆成新的 PENDING 任务。 */
|
||||
public ResumeStats resumeInterruptedTasks() {
|
||||
ResumeStats stats = new ResumeStats();
|
||||
if (!enabled) {
|
||||
return stats;
|
||||
}
|
||||
if (resumeHandlers.isEmpty()) {
|
||||
log.warn("[task-resume] 没有注册任何 ClientTaskPullSpi,跳过本轮续跑");
|
||||
if (retryHandlers.isEmpty()) {
|
||||
log.warn("[task-resume] 没有注册任何 ClientTaskRetrySpi,跳过本轮重试");
|
||||
return stats;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(Math.max(1L, windowMinutes));
|
||||
int safeLimit = Math.max(1, limit);
|
||||
stats.ran = true;
|
||||
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.likeRight(FileTaskEntity::getErrorMessage, CLIENT_INTERRUPT_PREFIX)
|
||||
.in(FileTaskEntity::getModuleType, resumeHandlers.keySet())
|
||||
.lt(FileTaskEntity::getResumeAttempt, maxAttempt)
|
||||
.ge(FileTaskEntity::getFinishedAt, cutoff)
|
||||
.orderByAsc(FileTaskEntity::getId)
|
||||
.last("limit " + safeLimit));
|
||||
if (candidates.isEmpty()) {
|
||||
stats.unsupportedTaskCount = countUnsupportedInterrupts(cutoff);
|
||||
return stats;
|
||||
}
|
||||
int safeLimit = Math.max(1, limit);
|
||||
stats.unsupportedTaskCount = countUnsupportedInterrupts(cutoff);
|
||||
stats.scannedTaskCount = candidates.size();
|
||||
for (FileTaskEntity original : candidates) {
|
||||
if (original.getUserId() == null || original.getUserId() <= 0) {
|
||||
log.warn("[task-resume] 原任务没有归属用户,跳过 taskId={}", original.getId());
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
// 幂等:同一原任务已经排过一次续跑就跳过(重复扫描 / 双实例竞态下不会重复建单)
|
||||
if (hasResumeChild(original.getId())) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
FileTaskEntity resume = buildResumeTask(original);
|
||||
try {
|
||||
fileTaskMapper.insert(resume);
|
||||
stats.resumedTaskCount++;
|
||||
log.warn("[task-resume] 已自动重新排队 taskId={} moduleType={} userId={} 续跑任务={} 代数={}/{} 中断原因={}",
|
||||
original.getId(), original.getModuleType(), original.getUserId(),
|
||||
resume.getId(), resume.getResumeAttempt(), maxAttempt, original.getErrorMessage());
|
||||
} catch (Exception ex) {
|
||||
stats.skippedTaskCount++;
|
||||
log.error("[task-resume] 重新排队失败 taskId={} moduleType={} err={}",
|
||||
original.getId(), original.getModuleType(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
return retryFailedSince(cutoff, safeLimit, false, retryHandlers.keySet(), stats);
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计「因客户端中断而失败、但模块不支持自动续跑」的任务数。
|
||||
* 运维回填:按小时窗口扫描失败任务并克隆重试(内部接口调用)。
|
||||
*
|
||||
* <p>这些模块(上架/改价/审批/商品管理采集/跟价的非循环任务等)的续跑载荷需要**用户在页面上选的
|
||||
* 执行参数**(如 ziniao_version),而这份选择只存在于派发那一刻的浏览器里、没落到服务端
|
||||
* request_json —— 自动重排队会用错参数。因此它们只做**可见**:数量进巡检 summary,
|
||||
* 运维据此人工重跑;将来把这类参数回写落库后即可纳入续跑白名单。
|
||||
* <p>不受 {@code enabled} 开关限制——这是显式的人工动作;dryRun 只统计不落库。
|
||||
* 窗口与单次条数都设上限,避免一次误调用把整张表克隆一遍。
|
||||
*/
|
||||
public ResumeStats retryFailedTasks(long sinceHours, int limit, boolean dryRun, List<String> moduleTypes) {
|
||||
ResumeStats stats = new ResumeStats();
|
||||
stats.dryRun = dryRun;
|
||||
if (retryHandlers.isEmpty()) {
|
||||
log.warn("[task-resume] 回填失败:没有注册任何 ClientTaskRetrySpi");
|
||||
return stats;
|
||||
}
|
||||
Set<String> types = resolveRequestedTypes(moduleTypes);
|
||||
if (types.isEmpty()) {
|
||||
log.warn("[task-resume] 回填失败:模块白名单为空");
|
||||
return stats;
|
||||
}
|
||||
long safeHours = Math.max(1L, Math.min(sinceHours, 24L * 14));
|
||||
int safeLimit = Math.max(1, Math.min(limit, 500));
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusHours(safeHours);
|
||||
stats.ran = true;
|
||||
stats.unsupportedTaskCount = countUnsupportedInterrupts(cutoff);
|
||||
log.warn("[task-resume] 回填开始 dryRun={} sinceHours={} limit={} types={}",
|
||||
dryRun, safeHours, safeLimit, types);
|
||||
return retryFailedSince(cutoff, safeLimit, dryRun, types, stats);
|
||||
}
|
||||
|
||||
private ResumeStats retryFailedSince(LocalDateTime cutoff,
|
||||
int safeLimit,
|
||||
boolean dryRun,
|
||||
Set<String> types,
|
||||
ResumeStats stats) {
|
||||
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.in(FileTaskEntity::getModuleType, types)
|
||||
.and(w -> w.isNull(FileTaskEntity::getResumeAttempt)
|
||||
.or().lt(FileTaskEntity::getResumeAttempt, Math.max(1, maxAttempt)))
|
||||
.apply("COALESCE(finished_at, updated_at) >= {0}", cutoff)
|
||||
.orderByAsc(FileTaskEntity::getId)
|
||||
.last("limit " + safeLimit));
|
||||
if (candidates.isEmpty()) {
|
||||
logRetrySummary(stats, cutoff, safeLimit);
|
||||
return stats;
|
||||
}
|
||||
stats.scannedTaskCount = candidates.size();
|
||||
for (FileTaskEntity original : candidates) {
|
||||
if (original.getUserId() == null || original.getUserId() <= 0) {
|
||||
stats.skippedTaskCount++;
|
||||
log.warn("[task-resume] 原任务没有归属用户,跳过 taskId={} moduleType={}",
|
||||
original.getId(), original.getModuleType());
|
||||
continue;
|
||||
}
|
||||
int attempt = original.getResumeAttempt() == null ? 0 : original.getResumeAttempt();
|
||||
if (attempt >= Math.max(1, maxAttempt)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
TaskRetryEligibility.Category forceMajeure =
|
||||
TaskRetryEligibility.forceMajeureCategory(original.getErrorMessage());
|
||||
if (forceMajeure != null) {
|
||||
stats.forceMajeureSkippedCount++;
|
||||
log.info("[task-resume] 不可抗力不重试 taskId={} moduleType={} category={} reason={}",
|
||||
original.getId(), original.getModuleType(), forceMajeure,
|
||||
abbreviate(original.getErrorMessage()));
|
||||
continue;
|
||||
}
|
||||
if (hasRetryChild(original.getId())) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
ClientTaskRetrySpi handler = retryHandlers.get(normalize(original.getModuleType()));
|
||||
if (handler == null) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
if (dryRun) {
|
||||
stats.candidateTaskIds.add(original.getId());
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
Long childTaskId = handler.cloneForRetry(original);
|
||||
if (childTaskId == null) {
|
||||
stats.cloneFailedCount++;
|
||||
log.warn("[task-resume] 克隆返回空(缺解析输入) taskId={} moduleType={}",
|
||||
original.getId(), original.getModuleType());
|
||||
continue;
|
||||
}
|
||||
stats.resumedTaskCount++;
|
||||
stats.retriedTaskIds.add(childTaskId);
|
||||
log.warn("[task-resume] 已克隆重试任务 original={} new={} moduleType={} userId={} 代数={}/{} 失败原因={}",
|
||||
original.getId(), childTaskId, original.getModuleType(), original.getUserId(),
|
||||
attempt + 1, maxAttempt, abbreviate(original.getErrorMessage()));
|
||||
} catch (Exception ex) {
|
||||
stats.cloneFailedCount++;
|
||||
log.error("[task-resume] 克隆重试任务失败 taskId={} moduleType={} err={}",
|
||||
original.getId(), original.getModuleType(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
logRetrySummary(stats, cutoff, safeLimit);
|
||||
return stats;
|
||||
}
|
||||
|
||||
private void logRetrySummary(ResumeStats stats, LocalDateTime cutoff, int safeLimit) {
|
||||
log.info("[task-resume] 本轮完成 dryRun={} cutoff={} limit={} scanned={} retried={} skippedExisting={} "
|
||||
+ "skippedForceMajeure={} cloneFailed={} unsupportedInterrupts={}",
|
||||
stats.dryRun, cutoff, safeLimit, stats.scannedTaskCount, stats.resumedTaskCount,
|
||||
stats.skippedTaskCount, stats.forceMajeureSkippedCount, stats.cloneFailedCount,
|
||||
stats.unsupportedTaskCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计「因客户端中断而失败、但模块不支持克隆重试」的任务数。
|
||||
*
|
||||
* <p>这些模块的执行参数需要用户在页面上选择、服务端 request_json 里没有,盲目克隆会用错参数。
|
||||
* 因此它们只做**可见**:数量进巡检 summary,运维据此人工重跑。
|
||||
*/
|
||||
private int countUnsupportedInterrupts(LocalDateTime cutoff) {
|
||||
try {
|
||||
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
Set<String> supported = retryHandlers.keySet();
|
||||
LambdaQueryWrapper<FileTaskEntity> query = new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.likeRight(FileTaskEntity::getErrorMessage, CLIENT_INTERRUPT_PREFIX)
|
||||
.notIn(FileTaskEntity::getModuleType, resumeHandlers.keySet())
|
||||
.ge(FileTaskEntity::getFinishedAt, cutoff));
|
||||
.ge(FileTaskEntity::getFinishedAt, cutoff);
|
||||
if (!supported.isEmpty()) {
|
||||
query.notIn(FileTaskEntity::getModuleType, supported);
|
||||
}
|
||||
Long count = fileTaskMapper.selectCount(query);
|
||||
return count == null ? 0 : count.intValue();
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-resume] 统计不支持续跑的中断任务失败(忽略): {}", ex.getMessage());
|
||||
log.warn("[task-resume] 统计不支持重试的中断任务失败(忽略): {}", ex.getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** 该原任务是否已经有续跑任务(反查 resume_of_task_id)。 */
|
||||
private boolean hasResumeChild(Long originalTaskId) {
|
||||
/** 该原任务是否已经有重试任务(反查 resume_of_task_id)。 */
|
||||
private boolean hasRetryChild(Long originalTaskId) {
|
||||
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getResumeOfTaskId, originalTaskId));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
private FileTaskEntity buildResumeTask(FileTaskEntity original) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
FileTaskEntity resume = new FileTaskEntity();
|
||||
resume.setTaskNo(original.getModuleType() + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
resume.setModuleType(original.getModuleType());
|
||||
resume.setTaskMode(original.getTaskMode());
|
||||
resume.setStatus(STATUS_PENDING);
|
||||
resume.setSourceFileCount(original.getSourceFileCount());
|
||||
resume.setSuccessFileCount(0);
|
||||
resume.setFailedFileCount(0);
|
||||
// 原任务的请求参数整体带走:各模块的 ClientTaskPullSpi 从 request_json / 关联表还原执行参数
|
||||
resume.setRequestJson(original.getRequestJson());
|
||||
resume.setCreatedBy(original.getCreatedBy());
|
||||
resume.setUserId(original.getUserId());
|
||||
// 续跑任务继承原任务设备:续跑由服务端发起,无 JWT 可解析
|
||||
resume.setDeviceId(original.getDeviceId());
|
||||
resume.setResumeOfTaskId(original.getId());
|
||||
int attempt = original.getResumeAttempt() == null ? 0 : original.getResumeAttempt();
|
||||
resume.setResumeAttempt(attempt + 1);
|
||||
resume.setCreatedAt(now);
|
||||
resume.setUpdatedAt(now);
|
||||
return resume;
|
||||
private Set<String> resolveRequestedTypes(List<String> requested) {
|
||||
Set<String> types = new LinkedHashSet<>();
|
||||
if (requested == null || requested.isEmpty()) {
|
||||
types.addAll(retryHandlers.keySet());
|
||||
return types;
|
||||
}
|
||||
for (String raw : requested) {
|
||||
String value = normalize(raw);
|
||||
if (value != null && retryHandlers.containsKey(value)) {
|
||||
types.add(value);
|
||||
}
|
||||
}
|
||||
return types;
|
||||
}
|
||||
|
||||
private static String abbreviate(String text) {
|
||||
if (text == null) {
|
||||
return "";
|
||||
}
|
||||
String trimmed = text.trim();
|
||||
return trimmed.length() <= 160 ? trimmed : trimmed.substring(0, 160) + "…";
|
||||
}
|
||||
|
||||
/** 单轮统计(合并进 stale-check 的 summary 日志,避免定期刷屏)。 */
|
||||
public static final class ResumeStats {
|
||||
/**
|
||||
* 本轮是否真的执行过扫描。开关关闭或模块未注册时为 false,此时其余计数虽同为 0,
|
||||
* 含义却是"没跑"而不是"跑了没活" —— 必须让日志能把这两件事区分开。
|
||||
*/
|
||||
/** 本轮是否真的执行过扫描。开关关闭或模块未注册时为 false,与"跑了没活"区分。 */
|
||||
public boolean ran;
|
||||
/** 本次是回填 dry-run(只统计不落库)。 */
|
||||
public boolean dryRun;
|
||||
public int scannedTaskCount;
|
||||
public int resumedTaskCount;
|
||||
public int skippedTaskCount;
|
||||
/** 模块不支持自动续跑的中断任务数(仅计数,供运维人工重跑)。 */
|
||||
/** 未注册克隆实现的中断任务数(仅计数,供运维人工重跑)。 */
|
||||
public int unsupportedTaskCount;
|
||||
/** 命中不可抗力(欠费/服务宕机/登录不可用)而跳过的数量。 */
|
||||
public int forceMajeureSkippedCount;
|
||||
/** 克隆过程中失败的数量(缺解析输入、库异常等)。 */
|
||||
public int cloneFailedCount;
|
||||
/** dry-run 下本可克隆的原任务 id。 */
|
||||
public final List<Long> candidateTaskIds = new ArrayList<>();
|
||||
/** 实际克隆出的新任务 id。 */
|
||||
public final List<Long> retriedTaskIds = new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 失败任务克隆重试的公共骨架:任务头、源结果行、解析 scope state。
|
||||
*
|
||||
* <p>只依赖 task 侧的表与 Mapper,业务模块在自己的 {@code ClientTaskRetrySpi} 里
|
||||
* 调用这里完成公共部分,再补各自的模块状态(如采集明细行)。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskRetryCloneSupport {
|
||||
|
||||
private static final String STATUS_PENDING = "PENDING";
|
||||
private static final String INITIAL_SCOPE_STATE_JSON = "{\"phase\":\"PARSED\"}";
|
||||
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* 建一条待执行的重试任务。
|
||||
*
|
||||
* <p>{@code createdAt} 刻意继承原任务:兜底拉取只领「创建超过 30 分钟」的 PENDING 任务,
|
||||
* 若写成当前时间,重试要白等半小时才可能被客户端看见。updated_at 写当前时间,
|
||||
* 2 小时无心跳的陈旧判死从重试这一刻重新起算。
|
||||
*/
|
||||
public FileTaskEntity buildChild(FileTaskEntity original, String resultJson) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
FileTaskEntity child = new FileTaskEntity();
|
||||
child.setTaskNo(original.getModuleType() + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
child.setModuleType(original.getModuleType());
|
||||
child.setTaskMode(original.getTaskMode());
|
||||
child.setStatus(STATUS_PENDING);
|
||||
child.setSourceFileCount(original.getSourceFileCount());
|
||||
child.setSuccessFileCount(0);
|
||||
child.setFailedFileCount(0);
|
||||
child.setRequestJson(original.getRequestJson());
|
||||
child.setResultJson(resultJson);
|
||||
child.setCreatedBy(original.getCreatedBy());
|
||||
child.setUserId(original.getUserId());
|
||||
child.setDeviceId(original.getDeviceId());
|
||||
child.setResumeOfTaskId(original.getId());
|
||||
child.setResumeAttempt((original.getResumeAttempt() == null ? 0 : original.getResumeAttempt()) + 1);
|
||||
child.setCreatedAt(original.getCreatedAt() == null ? now : original.getCreatedAt());
|
||||
child.setUpdatedAt(now);
|
||||
return child;
|
||||
}
|
||||
|
||||
/**
|
||||
* 复制源结果行(每个任务在解析阶段恰好一行),只带源文件与行数,清空上次的结果文件字段——
|
||||
* 否则新任务一创建就显示"已有可下载结果",但那份文件属于上一次失败的执行。
|
||||
*/
|
||||
public void copySourceResult(Long originalTaskId, Long newTaskId, String moduleType) {
|
||||
copySourceResult(originalTaskId, newTaskId, moduleType, null);
|
||||
}
|
||||
|
||||
/** rowCountOverride 非空时按解析行数写(失败任务的源结果行 rowCount 可能是上次的最终行数)。 */
|
||||
public void copySourceResult(Long originalTaskId, Long newTaskId, String moduleType,
|
||||
Integer rowCountOverride) {
|
||||
FileResultEntity source = fileResultMapper.selectOne(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, originalTaskId)
|
||||
.eq(FileResultEntity::getModuleType, moduleType)
|
||||
.orderByAsc(FileResultEntity::getId)
|
||||
.last("limit 1"));
|
||||
if (source == null) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
FileResultEntity copy = new FileResultEntity();
|
||||
copy.setTaskId(newTaskId);
|
||||
copy.setModuleType(moduleType);
|
||||
copy.setSourceFilename(source.getSourceFilename());
|
||||
copy.setSourceFileUrl(source.getSourceFileUrl());
|
||||
copy.setRowCount(rowCountOverride == null ? source.getRowCount() : rowCountOverride);
|
||||
copy.setSuccess(1);
|
||||
copy.setUserId(source.getUserId());
|
||||
copy.setCreatedAt(now);
|
||||
fileResultMapper.insert(copy);
|
||||
}
|
||||
|
||||
/**
|
||||
* 复制解析阶段创建的 scope state(状态机重置为 PARSED、分片/LLM 计数清零),
|
||||
* 保留同一个解析载荷指针——载荷本体在共享临时存储里,72 小时内仍可读,不需要重新解析源 Excel。
|
||||
*
|
||||
* @return 复制出的 scope;原任务没有解析 scope(载荷缺失)时返回 null
|
||||
*/
|
||||
public TaskScopeStateEntity copyParsedScope(Long originalTaskId, Long newTaskId, String moduleType) {
|
||||
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(
|
||||
new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, originalTaskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.orderByAsc(TaskScopeStateEntity::getId));
|
||||
TaskScopeStateEntity source = scopes.stream()
|
||||
.filter(scope -> scope.getParsedPayloadJson() != null
|
||||
&& !scope.getParsedPayloadJson().isBlank())
|
||||
.min(Comparator.comparing(TaskScopeStateEntity::getId))
|
||||
.orElse(null);
|
||||
if (source == null) {
|
||||
log.warn("[task-retry] 原任务缺少解析 scope,无法克隆 moduleType={} taskId={}",
|
||||
moduleType, originalTaskId);
|
||||
return null;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
TaskScopeStateEntity copy = new TaskScopeStateEntity();
|
||||
copy.setTaskId(newTaskId);
|
||||
copy.setModuleType(moduleType);
|
||||
copy.setScopeKey(source.getScopeKey());
|
||||
copy.setScopeHash(source.getScopeHash());
|
||||
copy.setParsedPayloadJson(source.getParsedPayloadJson());
|
||||
copy.setStateJson(INITIAL_SCOPE_STATE_JSON);
|
||||
copy.setChunkTotal(0);
|
||||
copy.setReceivedChunkCount(0);
|
||||
copy.setCompleted(0);
|
||||
copy.setCreatedAt(now);
|
||||
copy.setUpdatedAt(now);
|
||||
taskScopeStateMapper.insert(copy);
|
||||
return copy;
|
||||
}
|
||||
|
||||
/**
|
||||
* 复制 resultJson 但摘掉 {@code ownerInstanceId}:
|
||||
* 那是解析时那台机器的快照,重试任务由谁认领还不确定;留着会让另一台的归属校验误拒。
|
||||
* 解析失败直接抛出——带着坏 JSON 克隆出去只会在执行侧炸得更远。
|
||||
*/
|
||||
public String resultJsonWithoutOwner(String resultJson) {
|
||||
if (resultJson == null || resultJson.isBlank()) {
|
||||
throw new IllegalStateException("任务缺少 resultJson,无法克隆解析载荷");
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(resultJson);
|
||||
if (!root.isObject()) {
|
||||
throw new IllegalStateException("resultJson 不是对象");
|
||||
}
|
||||
ObjectNode copy = ((ObjectNode) root).deepCopy();
|
||||
copy.remove("ownerInstanceId");
|
||||
copy.remove("ownerInstanceReason");
|
||||
return objectMapper.writeValueAsString(copy);
|
||||
} catch (IllegalStateException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("解析 resultJson 失败: " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* 失败任务是否值得自动重试的判据。
|
||||
*
|
||||
* <p>用户口径(2026-10-22):失败任务自动重试 3 次,但 LLM 密钥/代理 IP 欠费、服务宕机等
|
||||
* 不可抗力不重试——这些原因不修外部条件时重跑多少次都是同一个结果,还会白烧代理与 LLM 额度。
|
||||
*
|
||||
* <p>另外把"账号/登录不可用"也归入不重试:aiprice 未登录且自动登录失败,重跑只会再失败一次。
|
||||
* 这一条不在用户列举的三类里,但属于同一类"重跑不会改变结果"的确定性失败,故单独打标签,
|
||||
* 在汇总日志里可见,运维可据标签调整。
|
||||
*/
|
||||
public final class TaskRetryEligibility {
|
||||
|
||||
public enum Category {
|
||||
/** LLM 密钥失效/欠费、代理 IP 套餐欠费:充值或换 key 之前重跑无意义。 */
|
||||
BALANCE("欠费或余额不足"),
|
||||
/** 上游服务不可用/宕机/连接被拒:等服务恢复。 */
|
||||
SERVICE_DOWN("服务不可用"),
|
||||
/** 账号未登录且自动登录失败:等用户重新登录。 */
|
||||
CREDENTIAL("账号或登录不可用");
|
||||
|
||||
private final String label;
|
||||
|
||||
Category(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public String label() {
|
||||
return label;
|
||||
}
|
||||
}
|
||||
|
||||
/** 余额/套餐类:覆盖 jikip 代理与 LLM 中转两种欠费文案。 */
|
||||
private static final List<String> BALANCE_MARKERS = List.of(
|
||||
"余额不足", "欠费", "套餐不存在", "套餐已过期", "套餐过期", "未开通",
|
||||
"insufficient_user_quota", "insufficient_quota", "quota exceeded",
|
||||
"预扣费额度");
|
||||
|
||||
/** LLM 密钥本身失效(区别于欠费)。 */
|
||||
private static final List<String> KEY_MARKERS = List.of(
|
||||
"密钥无效", "密钥失效", "无效的密钥", "密钥不可用",
|
||||
"invalid api key", "invalid_api_key", "api key 无效", "apikey 无效");
|
||||
|
||||
private static final List<String> SERVICE_MARKERS = List.of(
|
||||
"服务不可用", "服务宕机", "service unavailable",
|
||||
"connection refused", "连接被拒绝",
|
||||
"502 bad gateway", "503 service", "504 gateway", "upstream connect error",
|
||||
"对象存储不可达", "对象存储出口不可达", "存储出口不可达",
|
||||
"数据库连接失败", "数据库不可用");
|
||||
|
||||
private static final List<String> CREDENTIAL_MARKERS = List.of(
|
||||
"aiprice 未登录", "未登录且自动登录失败", "自动登录失败",
|
||||
"会话失效且重新登录失败");
|
||||
|
||||
private TaskRetryEligibility() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @return null=可重试;非 null=不可抗力分类
|
||||
*/
|
||||
public static Category forceMajeureCategory(String errorMessage) {
|
||||
if (errorMessage == null || errorMessage.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String text = errorMessage.toLowerCase(Locale.ROOT);
|
||||
if (containsAny(text, BALANCE_MARKERS) || containsAny(text, KEY_MARKERS)) {
|
||||
return Category.BALANCE;
|
||||
}
|
||||
if (containsAny(text, SERVICE_MARKERS)) {
|
||||
return Category.SERVICE_DOWN;
|
||||
}
|
||||
if (containsAny(text, CREDENTIAL_MARKERS)) {
|
||||
return Category.CREDENTIAL;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 汇总日志用的短原因;可重试时返回空串。 */
|
||||
public static String skipReason(String errorMessage) {
|
||||
Category category = forceMajeureCategory(errorMessage);
|
||||
return category == null ? "" : category.label();
|
||||
}
|
||||
|
||||
private static boolean containsAny(String text, List<String> markers) {
|
||||
for (String marker : markers) {
|
||||
if (text.contains(marker)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.task.spi;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
|
||||
/**
|
||||
* 失败任务自动重试的模块侧扩展点。
|
||||
*
|
||||
* <p>为什么是"克隆"而不是新建一条只有 request_json 的空任务:采集/相似ASIN/外观专利的执行输入
|
||||
* 都在解析阶段落成了模块侧状态(采集明细行、解析载荷指针、scope state),只有 request_json
|
||||
* 的续跑任务在客户端看来是一张空表 —— V129 上线后线上 19 条续跑任务全部以
|
||||
* 「长时间未收到心跳 / 长任务长期未被领取 / 结果为空」失败,根因就在这里。
|
||||
*
|
||||
* <p>各模块实现本接口,把原任务的解析输入与载荷原样克隆到新任务,再交给既有的
|
||||
* 兜底拉取 / 内部认领链路执行。task 侧不 import 业务模块(G5),只按 moduleType 建索引。
|
||||
*/
|
||||
public interface ClientTaskRetrySpi {
|
||||
|
||||
/** 本实现负责的 moduleType(与 biz_file_task.module_type 一致)。 */
|
||||
String moduleType();
|
||||
|
||||
/**
|
||||
* 把失败任务克隆为一条新的 PENDING 重试任务。
|
||||
*
|
||||
* @return 新任务 id;返回 null 表示当前任务缺少可复用的解析输入(调用方按跳过处理)
|
||||
*/
|
||||
Long cloneForRetry(FileTaskEntity original);
|
||||
}
|
||||
@@ -12,4 +12,13 @@
|
||||
#{row.keyword}, #{row.statusValue}, #{row.extraJson}, #{row.createdAt})
|
||||
</foreach>
|
||||
</insert>
|
||||
|
||||
<insert id="copyItemsForRetry">
|
||||
INSERT INTO biz_collect_data_item
|
||||
(task_id, row_index, source_file_key, source_filename, keyword, status_value, extra_json, created_at)
|
||||
SELECT #{targetTaskId}, row_index, source_file_key, source_filename, keyword, status_value, extra_json,
|
||||
#{createdAt}
|
||||
FROM biz_collect_data_item
|
||||
WHERE task_id = #{sourceTaskId}
|
||||
</insert>
|
||||
</mapper>
|
||||
|
||||
@@ -66,7 +66,9 @@ class MigratedSqlEquivalenceTest {
|
||||
java.util.Set.of("AdminUserMapper#countLiveWorkFor",
|
||||
// V157 加 token 两列时新写的累加语句;注解时代没有原文可对撞,等价性这支只能跳过它。
|
||||
// 与 increment 分两条是口径要求(计次先于响应、token 只在响应后有读数),不是随手加的第二条 SQL。
|
||||
"UserSecretUsageMapper#addTokens");
|
||||
"UserSecretUsageMapper#addTokens",
|
||||
// 失败任务重试:INSERT ... SELECT 复制采集明细行,非注解迁移件。
|
||||
"CollectDataItemMapper#copyItemsForRetry");
|
||||
|
||||
/** 账本 5 列:接口 / 方法 / @Select 种类 / 是否含常量插值 / 注解原文(Java 反转义后那条串)。 */
|
||||
private static final int LEDGER_COLUMNS = 5;
|
||||
|
||||
@@ -70,7 +70,9 @@ class SqlLayeringConventionTest {
|
||||
// 它同样是「新写」而非「从注解搬进 XML」,所以两处一起改:本格的总数 +
|
||||
// MigratedSqlEquivalenceTest.NOT_MIGRATED 的点名。与 increment 分两条是口径要求——
|
||||
// 计次发生在发请求之前(含重试),token 只有响应回来才有读数,合成一条 upsert 就会漏计超时的那次。
|
||||
private static final int XML_STATEMENT_TOTAL = 98;
|
||||
// 98 → 99:失败任务重试新增 {@code CollectDataItemMapper#copyItemsForRetry}
|
||||
// (INSERT ... SELECT 复制明细行),同样登记进 NOT_MIGRATED。
|
||||
private static final int XML_STATEMENT_TOTAL = 99;
|
||||
|
||||
/**
|
||||
* wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数:
|
||||
|
||||
+124
-59
@@ -5,15 +5,13 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
|
||||
import ch.qos.logback.core.read.ListAppender;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskRetrySpi;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
@@ -26,28 +24,25 @@ import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 客户端中断任务的自动续跑:保留失败记录 + 重新排队 PENDING 续跑任务。
|
||||
*
|
||||
* <p>背景:2026-09-18 任务 28587 因客户端被更新脚本重启而中断后一直挂 RUNNING;
|
||||
* 中断上报把任务标 FAILED 之后,还需要有人把长任务放回队列,否则一遇客户端更新就整个白跑。
|
||||
* 失败任务的自动重试:克隆重试任务、代数封顶、不可抗力跳过。
|
||||
*/
|
||||
class TaskResumeServiceTest {
|
||||
|
||||
private static ClientTaskPullSpi spi(String moduleType) {
|
||||
return new ClientTaskPullSpi() {
|
||||
private static ClientTaskRetrySpi retrySpi(String moduleType, Long childTaskId) {
|
||||
return new ClientTaskRetrySpi() {
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return moduleType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> buildQueuePayload(FileTaskEntity task) {
|
||||
return Map.of("type", "x-run", "data", Map.of("taskId", task.getId()));
|
||||
public Long cloneForRetry(FileTaskEntity original) {
|
||||
return childTaskId;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static TaskResumeService service(FileTaskMapper mapper, List<ClientTaskPullSpi> handlers,
|
||||
private static TaskResumeService service(FileTaskMapper mapper, List<ClientTaskRetrySpi> handlers,
|
||||
boolean enabled, int maxAttempt) {
|
||||
TaskResumeService service = new TaskResumeService(mapper, handlers);
|
||||
ReflectionTestUtils.setField(service, "enabled", enabled);
|
||||
@@ -57,14 +52,14 @@ class TaskResumeServiceTest {
|
||||
return service;
|
||||
}
|
||||
|
||||
private static FileTaskEntity interruptedTask(Long id, String moduleType) {
|
||||
private static FileTaskEntity failedTask(Long id, String moduleType, String errorMessage) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setModuleType(moduleType);
|
||||
task.setUserId(977L);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报(上次任务类型: similar-asin-run)");
|
||||
task.setErrorMessage(errorMessage);
|
||||
task.setRequestJson("{\"items\":[]}");
|
||||
task.setResumeAttempt(0);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
@@ -72,46 +67,82 @@ class TaskResumeServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void 中断任务会被重新排队为待执行的续跑任务() {
|
||||
void 失败任务会被克隆重试() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
when(spi.cloneForRetry(any())).thenReturn(9001L);
|
||||
TaskResumeService service = service(mapper, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.scannedTaskCount);
|
||||
assertEquals(1, stats.resumedTaskCount);
|
||||
ArgumentCaptor<FileTaskEntity> captor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(mapper).insert(captor.capture());
|
||||
FileTaskEntity resumed = captor.getValue();
|
||||
assertEquals("PENDING", resumed.getStatus(), "续跑任务必须是待领取状态,交给客户端兜底拉取");
|
||||
assertEquals(28587L, resumed.getResumeOfTaskId(), "必须回指原任务,用于追溯与幂等");
|
||||
assertEquals(1, resumed.getResumeAttempt());
|
||||
assertEquals(977L, resumed.getUserId());
|
||||
assertEquals("SIMILAR_ASIN", resumed.getModuleType());
|
||||
assertEquals("{\"items\":[]}", resumed.getRequestJson(), "原任务参数必须整体带走");
|
||||
assertTrue(resumed.getTaskNo().startsWith("SIMILAR_ASIN-"), "任务号沿用模块前缀");
|
||||
assertEquals(List.of(9001L), stats.retriedTaskIds);
|
||||
verify(spi).cloneForRetry(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 已有续跑任务时跳过避免重复排队() {
|
||||
void 欠费或服务宕机等不可抗力不重试() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||
when(mapper.selectCount(any())).thenReturn(1L, 1L);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(28587L, "SIMILAR_ASIN", "代理提取失败:余额不足"),
|
||||
failedTask(28588L, "APPEARANCE_PATENT", "上游服务不可用,请稍后再试")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
TaskResumeService service = service(mapper, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(2, stats.scannedTaskCount);
|
||||
assertEquals(2, stats.forceMajeureSkippedCount);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 已有重试任务时跳过避免重复克隆() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败")));
|
||||
// 第一次是"不支持重试的中断任务数",第二次是"是否已有重试子任务"
|
||||
when(mapper.selectCount(any())).thenReturn(0L, 1L);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
TaskResumeService service = service(mapper, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
verify(mapper, never()).insert(any(FileTaskEntity.class));
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 重试代数到上限后跳过() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
FileTaskEntity task = failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败");
|
||||
task.setResumeAttempt(3);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
TaskResumeService service = service(mapper, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 开关关闭时不扫描() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), false, 3);
|
||||
TaskResumeService service = service(mapper, List.of(retrySpi("SIMILAR_ASIN", 1L)), false, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
@@ -120,7 +151,7 @@ class TaskResumeServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有注册模块兜底实现时不扫描() {
|
||||
void 没有注册克隆实现时不扫描() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
TaskResumeService service = service(mapper, List.of(), true, 3);
|
||||
|
||||
@@ -131,58 +162,57 @@ class TaskResumeServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void 插入失败只计入跳过不抛出() {
|
||||
void 克隆失败只计入失败数不抛出() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
doThrow(new RuntimeException("库挂了")).when(mapper).insert(any(FileTaskEntity.class));
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
doThrow(new RuntimeException("库挂了")).when(spi).cloneForRetry(any());
|
||||
TaskResumeService service = service(mapper, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
assertEquals(1, stats.cloneFailedCount);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 无候选时统计不支持续跑的中断任务数() {
|
||||
void 无候选时统计不支持重试的中断任务数() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of());
|
||||
when(mapper.selectCount(any())).thenReturn(3L);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
TaskResumeService service = service(mapper, List.of(retrySpi("SIMILAR_ASIN", 1L)), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
assertEquals(3, stats.unsupportedTaskCount, "不支持续跑的模块也要可见,供运维人工重跑");
|
||||
assertEquals(3, stats.unsupportedTaskCount, "不支持重试的模块也要可见,供运维人工重跑");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有归属用户的中断任务跳过() {
|
||||
void 没有归属用户的任务跳过() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
FileTaskEntity orphan = interruptedTask(28587L, "SIMILAR_ASIN");
|
||||
FileTaskEntity orphan = failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败");
|
||||
orphan.setUserId(null);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(orphan));
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
TaskResumeService service = service(mapper, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
verify(mapper, never()).insert(any(FileTaskEntity.class));
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 开关关着时,统计必须说"本轮没跑",不能长得像"跑了但没活"。
|
||||
*
|
||||
* <p>线上 2026-09-28 实测:{@code resume(s=0 r=0 k=0 u=0)} 在两台连续 48 轮全零,
|
||||
* 而 {@code aiimage.task-resume.enabled} 的真实默认是
|
||||
* {@code ${AIIMAGE_TASK_RESUME_ENABLED:${AIIMAGE_CLIENT_TASK_PULL_ENABLED:false}}}(生产两个键都没设)
|
||||
* —— 也就是说 V129 整套续跑从来没执行过,但从巡检日志上和"恰好没有中断任务"完全无法区分。
|
||||
*/
|
||||
/** 开关关着时,统计必须说"本轮没跑",不能长得像"跑了但没活"。 */
|
||||
@Test
|
||||
void 开关关闭时统计标记本轮未执行() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), false, 3);
|
||||
TaskResumeService service = service(mapper, List.of(retrySpi("SIMILAR_ASIN", 1L)), false, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
@@ -196,7 +226,7 @@ class TaskResumeServiceTest {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of());
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
TaskResumeService service = service(mapper, List.of(retrySpi("SIMILAR_ASIN", 1L)), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
@@ -204,7 +234,7 @@ class TaskResumeServiceTest {
|
||||
assertEquals(0, stats.scannedTaskCount);
|
||||
}
|
||||
|
||||
/** 没有注册兜底实现等于跑不了:同样要标记未执行,不能报成"跑了没活"。 */
|
||||
/** 没有注册克隆实现等于跑不了:同样要标记未执行,不能报成"跑了没活"。 */
|
||||
@Test
|
||||
void 无模块注册时也算本轮未执行() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
@@ -215,8 +245,43 @@ class TaskResumeServiceTest {
|
||||
assertFalse(stats.ran);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dryRun只列候选不克隆() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
TaskResumeService service = service(mapper, List.of(spi), false, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.retryFailedTasks(36, 200, true, null);
|
||||
|
||||
assertTrue(stats.dryRun);
|
||||
assertEquals(List.of(28587L), stats.candidateTaskIds);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 回填不受定时开关限制() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("SIMILAR_ASIN");
|
||||
when(spi.cloneForRetry(any())).thenReturn(9001L);
|
||||
TaskResumeService service = service(mapper, List.of(spi), false, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.retryFailedTasks(36, 200, false, null);
|
||||
|
||||
assertEquals(1, stats.resumedTaskCount);
|
||||
assertEquals(List.of(9001L), stats.retriedTaskIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* 开机必须把续跑的生效配置打出来。
|
||||
* 开机必须把重试的生效配置打出来。
|
||||
*
|
||||
* <p>续跑是"功能写完了、测试全绿、线上默认关"的典型:光看代码里的
|
||||
* {@code @Value(":true")} 会以为默认开着,实际 yml 把默认压成了 false。
|
||||
@@ -230,7 +295,7 @@ class TaskResumeServiceTest {
|
||||
logger.addAppender(appender);
|
||||
try {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), false, 3);
|
||||
TaskResumeService service = service(mapper, List.of(retrySpi("SIMILAR_ASIN", 1L)), false, 3);
|
||||
service.logEffectiveConfig();
|
||||
|
||||
String line = appender.list.stream()
|
||||
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class TaskRetryCloneSupportTest {
|
||||
|
||||
private static TaskRetryCloneSupport support(FileResultMapper fileResultMapper,
|
||||
TaskScopeStateMapper scopeMapper) {
|
||||
return new TaskRetryCloneSupport(fileResultMapper, scopeMapper, new ObjectMapper());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 克隆任务头保留输入与代数并清空结果计数() {
|
||||
FileTaskMapperHolder holder = new FileTaskMapperHolder();
|
||||
FileTaskEntity original = holder.original();
|
||||
TaskRetryCloneSupport support = support(mock(FileResultMapper.class), mock(TaskScopeStateMapper.class));
|
||||
|
||||
FileTaskEntity child = support.buildChild(original, "{\"parsedPayloadRef\":\"p\"}");
|
||||
|
||||
assertEquals("PENDING", child.getStatus());
|
||||
assertEquals("SIMILAR_ASIN", child.getModuleType());
|
||||
assertEquals(original.getRequestJson(), child.getRequestJson());
|
||||
assertEquals("{\"parsedPayloadRef\":\"p\"}", child.getResultJson());
|
||||
assertEquals(977L, child.getUserId());
|
||||
assertEquals(original.getDeviceId(), child.getDeviceId());
|
||||
assertEquals(28587L, child.getResumeOfTaskId());
|
||||
assertEquals(1, child.getResumeAttempt());
|
||||
assertEquals(0, child.getSuccessFileCount());
|
||||
assertEquals(0, child.getFailedFileCount());
|
||||
assertEquals(original.getCreatedAt(), child.getCreatedAt(), "继承原创建时间才能立刻被兜底拉取看见");
|
||||
assertTrue(child.getTaskNo().startsWith("SIMILAR_ASIN-"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultJson摘掉owner快照() throws Exception {
|
||||
TaskRetryCloneSupport support = support(mock(FileResultMapper.class), mock(TaskScopeStateMapper.class));
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
String result = support.resultJsonWithoutOwner(
|
||||
"{\"parsedPayloadRef\":\"rustfs:x\",\"ownerInstanceId\":\"server-110\"}");
|
||||
|
||||
assertEquals("rustfs:x", mapper.readTree(result).path("parsedPayloadRef").asText());
|
||||
assertTrue(mapper.readTree(result).path("ownerInstanceId").isMissingNode());
|
||||
assertThrows(IllegalStateException.class, () -> support.resultJsonWithoutOwner("not-json"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 复制源结果行不带上次的结果文件() {
|
||||
FileResultMapper mapper = mock(FileResultMapper.class);
|
||||
FileResultEntity source = new FileResultEntity();
|
||||
source.setTaskId(28587L);
|
||||
source.setModuleType("SIMILAR_ASIN");
|
||||
source.setSourceFilename("a.xlsx");
|
||||
source.setSourceFileUrl("u");
|
||||
source.setRowCount(10);
|
||||
source.setResultFilename("result.xlsx");
|
||||
source.setResultFileUrl("https://old");
|
||||
source.setUserId(977L);
|
||||
when(mapper.selectOne(any())).thenReturn(source);
|
||||
TaskRetryCloneSupport support = support(mapper, mock(TaskScopeStateMapper.class));
|
||||
|
||||
support.copySourceResult(28587L, 9001L, "SIMILAR_ASIN");
|
||||
|
||||
ArgumentCaptor<FileResultEntity> captor = ArgumentCaptor.forClass(FileResultEntity.class);
|
||||
verify(mapper).insert(captor.capture());
|
||||
FileResultEntity copy = captor.getValue();
|
||||
assertEquals(9001L, copy.getTaskId());
|
||||
assertEquals("a.xlsx", copy.getSourceFilename());
|
||||
assertEquals(10, copy.getRowCount());
|
||||
assertEquals(1, copy.getSuccess());
|
||||
assertNull(copy.getResultFilename(), "上次的结果文件不能带进新任务");
|
||||
assertNull(copy.getResultFileUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 复制解析scope并重置执行状态() {
|
||||
TaskScopeStateMapper mapper = mock(TaskScopeStateMapper.class);
|
||||
TaskScopeStateEntity empty = new TaskScopeStateEntity();
|
||||
empty.setId(1L);
|
||||
TaskScopeStateEntity parsed = new TaskScopeStateEntity();
|
||||
parsed.setId(2L);
|
||||
parsed.setTaskId(28587L);
|
||||
parsed.setModuleType("SIMILAR_ASIN");
|
||||
parsed.setScopeKey("scope");
|
||||
parsed.setScopeHash("hash");
|
||||
parsed.setParsedPayloadJson("\"rustfs:x\"");
|
||||
parsed.setStateJson("{\"phase\":\"DONE\"}");
|
||||
parsed.setChunkTotal(9);
|
||||
parsed.setCompleted(1);
|
||||
parsed.setLlmStatus("DONE");
|
||||
when(mapper.selectList(any())).thenReturn(List.of(empty, parsed));
|
||||
TaskRetryCloneSupport support = support(mock(FileResultMapper.class), mapper);
|
||||
|
||||
TaskScopeStateEntity copy = support.copyParsedScope(28587L, 9001L, "SIMILAR_ASIN");
|
||||
|
||||
assertEquals(9001L, copy.getTaskId());
|
||||
assertEquals("scope", copy.getScopeKey());
|
||||
assertEquals("\"rustfs:x\"", copy.getParsedPayloadJson());
|
||||
assertEquals("{\"phase\":\"PARSED\"}", copy.getStateJson());
|
||||
assertEquals(0, copy.getChunkTotal());
|
||||
assertEquals(0, copy.getCompleted());
|
||||
assertNull(copy.getLlmStatus());
|
||||
verify(mapper).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有解析scope时返回空() {
|
||||
TaskScopeStateMapper mapper = mock(TaskScopeStateMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of());
|
||||
TaskRetryCloneSupport support = support(mock(FileResultMapper.class), mapper);
|
||||
|
||||
assertNull(support.copyParsedScope(28587L, 9001L, "APPEARANCE_PATENT"));
|
||||
}
|
||||
|
||||
/** 仅用于把建单样板集中在一处,避免每个用例重复写字段。 */
|
||||
private static final class FileTaskMapperHolder {
|
||||
FileTaskEntity original() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(28587L);
|
||||
task.setTaskNo("SIMILAR_ASIN-1");
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus("FAILED");
|
||||
task.setSourceFileCount(1);
|
||||
task.setSuccessFileCount(3);
|
||||
task.setFailedFileCount(1);
|
||||
task.setRequestJson("{\"files\":[]}");
|
||||
task.setResultJson("{\"parsedPayloadRef\":\"p\"}");
|
||||
task.setCreatedBy("user:977");
|
||||
task.setUserId(977L);
|
||||
task.setDeviceId("dev-1");
|
||||
task.setResumeAttempt(0);
|
||||
task.setCreatedAt(LocalDateTime.now().minusHours(2));
|
||||
return task;
|
||||
}
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
|
||||
class TaskRetryEligibilityTest {
|
||||
|
||||
@Test
|
||||
void 代理或LLM欠费不重试() {
|
||||
assertEquals(TaskRetryEligibility.Category.BALANCE,
|
||||
TaskRetryEligibility.forceMajeureCategory("代理提取失败:余额不足"));
|
||||
assertEquals(TaskRetryEligibility.Category.BALANCE,
|
||||
TaskRetryEligibility.forceMajeureCategory("套餐不存在,请续费"));
|
||||
assertEquals(TaskRetryEligibility.Category.BALANCE,
|
||||
TaskRetryEligibility.forceMajeureCategory("insufficient_user_quota"));
|
||||
assertEquals(TaskRetryEligibility.Category.BALANCE,
|
||||
TaskRetryEligibility.forceMajeureCategory("Invalid API Key"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 服务不可用不重试() {
|
||||
assertEquals(TaskRetryEligibility.Category.SERVICE_DOWN,
|
||||
TaskRetryEligibility.forceMajeureCategory("503 Service Unavailable"));
|
||||
assertEquals(TaskRetryEligibility.Category.SERVICE_DOWN,
|
||||
TaskRetryEligibility.forceMajeureCategory("对象存储出口不可达"));
|
||||
assertEquals(TaskRetryEligibility.Category.SERVICE_DOWN,
|
||||
TaskRetryEligibility.forceMajeureCategory("Connection refused"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 自动登录失败不重试() {
|
||||
assertEquals(TaskRetryEligibility.Category.CREDENTIAL,
|
||||
TaskRetryEligibility.forceMajeureCategory(
|
||||
"相似ASIN检测结果为空;客户端报告:aiprice 未登录且自动登录失败,货源查询无法采集"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 心跳超时和回传失败仍可重试() {
|
||||
assertNull(TaskRetryEligibility.forceMajeureCategory("任务心跳超时,已自动失败"));
|
||||
assertNull(TaskRetryEligibility.forceMajeureCategory(
|
||||
"中间分批回传失败,终止本次采集以避免服务端数据残缺"));
|
||||
assertNull(TaskRetryEligibility.forceMajeureCategory("任务长期未被领取,已自动失败"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 空原因按可重试处理() {
|
||||
assertNull(TaskRetryEligibility.forceMajeureCategory(null));
|
||||
assertNull(TaskRetryEligibility.forceMajeureCategory(" "));
|
||||
}
|
||||
}
|
||||
@@ -266,13 +266,14 @@ class SourceScanTest {
|
||||
// ---------------------------------------------------------------- N6 moduleSources
|
||||
|
||||
@Test
|
||||
void moduleSources_appearancepatent返回44支含拆分后的support件() {
|
||||
void moduleSources_appearancepatent返回45支含拆分后的support件() {
|
||||
List<Path> files = SourceScan.moduleSources("modules/appearancepatent");
|
||||
|
||||
assertThat(files)
|
||||
.as("真实集成:对真仓目录求值。44 支=1 controller+1 client+1 service 主件+3 spi/cache+"
|
||||
+ "21 service/support+16 model(dto7/vo9),与 find 实测同号(3A 拆薄第十三笔再落 5 支)")
|
||||
.hasSize(44);
|
||||
.as("真实集成:对真仓目录求值。45 支=1 controller+1 client+1 service 主件+4 spi/cache+"
|
||||
+ "21 service/support+16 model(dto7/vo9),与 find 实测同号"
|
||||
+ "(3A 拆薄第十三笔再落 5 支;失败任务重试再落 1 支 RetrySpi)")
|
||||
.hasSize(45);
|
||||
assertThat(files.stream().map(p -> p.toString().replace('\\', '/')).toList())
|
||||
.as("support 目录里的既有件必须在面里——聚合入口存在的理由就是拆分后的新文件要自动被看到")
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentRowNormalizer.java"));
|
||||
|
||||
Reference in New Issue
Block a user