fix(断点续传): 外观/结果作业/去重三处中断语义补齐,重启不再静默丢行
- 外观专利行标改两态(in-flight 15min / processed 24h):只有 CAS 写回 payload 成功 才转「已处理」,异常、覆盖率不足、CAS 打满等未落库出口一律撤销标记 —— 修「发 LLM 请求前就盖章,进程被杀的那一批永不再补检」 - 结果文件作业开机自回收:按 owner_instance_id 只收本机名下 RUNNING 且心跳超 90s 的行; 卡死阈值 30 → 5 分钟,重启接回从最长半小时压到 5 分钟 - 外观结果表缺检行显式写「未取得检测结果」并判失败,不再留空靠 0.5 覆盖率阈值蒙过 - 去重进度改 NodeSharedStore 跨节点共享,runId 用已落库的 task_no:轮询打到另一节点或 发版重启后仍能读到真实进度、已完成文件仍可下载;共享态缺失时按 task_no 从库重建; 逐文件心跳防 StaleTaskRepair 把长任务误判死;开机把本机残留 RUNNING 判失败并给提示
This commit is contained in:
+77
-14
@@ -19,6 +19,13 @@ public class AppearancePatentTaskCacheService {
|
||||
|
||||
private static final long TTL_HOURS = 24;
|
||||
|
||||
/**
|
||||
* 处理中标记 TTL。取 15 分钟:合批后单批实测约 30–120 秒(合批前 10 行要 220 秒),
|
||||
* 留 7 倍以上余量;同时它必须远小于原来的 24 小时——进程在批中被杀时,这批行只能等这个
|
||||
* TTL 自然过期才能重新成为候选,TTL 有多长,发版/崩溃造成的空缺窗口就有多长。
|
||||
*/
|
||||
private static final Duration IN_FLIGHT_TTL = Duration.ofMinutes(15);
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@@ -133,36 +140,92 @@ public class AppearancePatentTaskCacheService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 行级检测去重:Redis SETNX,防止 submitResult 与 stale 恢复双入口并发处理同一批。
|
||||
* 返回 true 表示首次标记(应执行检测),false 表示已处理过(跳过)。
|
||||
* 申领一行的检测权(两态标记之一:处理中)。返回 true 表示本次拿到,false 表示
|
||||
* 该行已完成或正被另一个入口(submitResult / stale 恢复)处理。
|
||||
*
|
||||
* <p>处理中只活 15 分钟:原实现在发 LLM 请求前就打 24 小时标记且没有撤销路径,
|
||||
* 进程被杀或载荷落库失败时,那批行既没结果又被判定「已处理」,续跑时静默跳过。
|
||||
*/
|
||||
public boolean markRowProcessed(Long taskId, String rowKey) {
|
||||
public boolean beginRowInspection(Long taskId, String rowKey) {
|
||||
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
if (Boolean.TRUE.equals(stringRedisTemplate.hasKey(processedRowKey(taskId, rowKey)))) {
|
||||
return false;
|
||||
}
|
||||
Boolean first = stringRedisTemplate.opsForValue().setIfAbsent(
|
||||
processedRowKey(taskId, rowKey), "1", Duration.ofHours(TTL_HOURS));
|
||||
return Boolean.TRUE.equals(first);
|
||||
inFlightRowKey(taskId, rowKey), "1", IN_FLIGHT_TTL);
|
||||
return first == null || first;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] mark row processed degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
|
||||
log.warn("[appearance-patent-cache] begin row inspection degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
|
||||
// Redis 降级时放行,由持久化层的 hasResolvedLlmFields 判据兜底去重。
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isRowProcessed(Long taskId, String rowKey) {
|
||||
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
|
||||
return false;
|
||||
/** 发请求前续期处理中标记:一批最慢可能跑到分钟级,避免中途被另一个入口抢走重复计费。 */
|
||||
public void touchRowInspection(Long taskId, List<String> rowKeys) {
|
||||
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
return Boolean.TRUE.equals(stringRedisTemplate.hasKey(processedRowKey(taskId, rowKey)));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] check row processed degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
|
||||
return false;
|
||||
for (String rowKey : rowKeys) {
|
||||
if (rowKey == null || rowKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.expire(inFlightRowKey(taskId, rowKey), IN_FLIGHT_TTL);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] touch row inspection degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 结果已落库:打 24 小时完成标记并释放处理中名额。只在 CAS 更新成功后调用。 */
|
||||
public void completeRowInspection(Long taskId, List<String> rowKeys) {
|
||||
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (String rowKey : rowKeys) {
|
||||
if (rowKey == null || rowKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.opsForValue().set(processedRowKey(taskId, rowKey), "1", Duration.ofHours(TTL_HOURS));
|
||||
stringRedisTemplate.delete(inFlightRowKey(taskId, rowKey));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] complete row inspection degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 撤销处理中标记(本批没拿到可用结果时调用):让下一轮立刻能重新申领,不等 15 分钟 TTL。
|
||||
* TTL 只用来兜住「进程在批中被杀」。
|
||||
*/
|
||||
public void releaseRowInspection(Long taskId, List<String> rowKeys) {
|
||||
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (String rowKey : rowKeys) {
|
||||
if (rowKey == null || rowKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.delete(inFlightRowKey(taskId, rowKey));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] release row inspection degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String inFlightRowKey(Long taskId, String rowKey) {
|
||||
return "appearance-patent:task:inflight-rows:" + taskId + ":" + rowKey;
|
||||
}
|
||||
|
||||
private String processedRowsKey(Long taskId) {
|
||||
return "appearance-patent:task:processed-rows:" + taskId;
|
||||
}
|
||||
|
||||
+19
-3
@@ -933,8 +933,11 @@ public class AppearancePatentTaskService {
|
||||
for (int i = 0; i < submitLimit; i += batchSize) {
|
||||
List<AppearancePatentResultRowDto> batchRows = candidates.subList(i, Math.min(i + batchSize, submitLimit));
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
List<String> batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
taskCacheService.touchRowInspection(task.getId(), batchRowKeys);
|
||||
long startedAt = System.currentTimeMillis();
|
||||
List<AppearancePatentResultRowDto> llmRows;
|
||||
boolean batchAnswered = true;
|
||||
try {
|
||||
llmRows = SecretUsageContext.call(task.getUserId(), UserSecretModule.APPEARANCE_PATENT.key(),
|
||||
() -> llmClient.inspectRows(batchRows, prompt, apiKey));
|
||||
@@ -943,8 +946,22 @@ public class AppearancePatentTaskService {
|
||||
log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} err={}",
|
||||
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message);
|
||||
llmRows = llmClient.markRowsFailed(batchRows, message);
|
||||
batchAnswered = false;
|
||||
}
|
||||
boolean rowsCompleted = false;
|
||||
try {
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
|
||||
// 完成标记只能在载荷真正落库之后打——这是「发版重启会不会丢行」的分界线
|
||||
if (batchAnswered) {
|
||||
taskCacheService.completeRowInspection(task.getId(), batchRowKeys);
|
||||
rowsCompleted = true;
|
||||
}
|
||||
} finally {
|
||||
// 本批结束时处理中名额必须让出来:要么已由完成标记接管,要么撤销给下一轮重试
|
||||
if (!rowsCompleted) {
|
||||
taskCacheService.releaseRowInspection(task.getId(), batchRowKeys);
|
||||
}
|
||||
}
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
|
||||
log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}",
|
||||
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal,
|
||||
System.currentTimeMillis() - startedAt);
|
||||
@@ -1007,10 +1024,9 @@ public class AppearancePatentTaskService {
|
||||
if (key.isBlank() || hasResolvedLlmFields(row)) {
|
||||
continue;
|
||||
}
|
||||
if (taskCacheService.isRowProcessed(chunk.getTaskId(), key)) {
|
||||
if (!taskCacheService.beginRowInspection(chunk.getTaskId(), key)) {
|
||||
continue;
|
||||
}
|
||||
taskCacheService.markRowProcessed(chunk.getTaskId(), key);
|
||||
candidates.add(row);
|
||||
}
|
||||
}
|
||||
|
||||
+29
-15
@@ -42,6 +42,17 @@ public final class AppearancePatentSheetBuilder {
|
||||
private AppearancePatentSheetBuilder() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 没拿到判定的行写进格子里的显式文案。
|
||||
*
|
||||
* <p>交付只要求 LLM 覆盖率过半({@code llm-coverage-min-ratio}),所以少数缺失行照样出文件。
|
||||
* 历史上这些格子留空,而"空白"和"没风险"在表格里长得一模一样——用户把漏检当成了安全,
|
||||
* 这正是最贵的那种静默失败。
|
||||
*/
|
||||
public static final String NO_RESULT_TEXT = "未取得检测结果";
|
||||
|
||||
private static final String NO_RESULT_REASON = NO_RESULT_TEXT + ":该行没有返回判定,重新运行任务可补检";
|
||||
|
||||
public static void buildResultSheet(Workbook workbook, List<AppearancePatentParsedRowVo> rows,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
buildResultSheet(workbook, rows, resultMap, null);
|
||||
@@ -87,10 +98,19 @@ public final class AppearancePatentSheetBuilder {
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
|
||||
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
|
||||
String titleRiskCell = resultRow == null
|
||||
? NO_RESULT_TEXT
|
||||
: firstNonBlank(userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()), NO_RESULT_TEXT);
|
||||
String appearanceRiskCell = resultRow == null
|
||||
? NO_RESULT_TEXT
|
||||
: firstNonBlank(userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()), NO_RESULT_TEXT);
|
||||
String conclusionCell = resultRow == null
|
||||
? NO_RESULT_TEXT
|
||||
: firstNonBlank(userFacingConclusion(resultRow), NO_RESULT_TEXT);
|
||||
row.createCell(col++).setCellValue(titleRiskCell);
|
||||
row.createCell(col++).setCellValue(appearanceRiskCell);
|
||||
row.createCell(col++).setCellValue(conclusionCell);
|
||||
row.createCell(col).setCellValue(resolveResultStatus(conclusionCell));
|
||||
}
|
||||
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap, finder);
|
||||
}
|
||||
@@ -121,9 +141,11 @@ public final class AppearancePatentSheetBuilder {
|
||||
AppearancePatentResultRowDto resultRow = finder.apply(parsedRow);
|
||||
Row row = sheet.createRow(rowIndex++);
|
||||
row.createCell(0).setCellValue(asin);
|
||||
row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), ""));
|
||||
row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), ""));
|
||||
row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), ""));
|
||||
// 只有"整行没结果"才代写原因;有结果的行原本就没原因可写(无风险),不能污染成缺失说明
|
||||
String missingReason = resultRow == null ? NO_RESULT_REASON : "";
|
||||
row.createCell(1).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getAppearanceReason(), ""));
|
||||
row.createCell(2).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getPatentReason(), ""));
|
||||
row.createCell(3).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getTitleReason(), ""));
|
||||
rowIndex++;
|
||||
}
|
||||
}
|
||||
@@ -233,14 +255,6 @@ public final class AppearancePatentSheetBuilder {
|
||||
return firstNonBlank(row.getConclusion(), "");
|
||||
}
|
||||
|
||||
public static String userFacingStatus(AppearancePatentResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
String conclusion = userFacingConclusion(row);
|
||||
return resolveResultStatus(conclusion);
|
||||
}
|
||||
|
||||
/**
|
||||
* 状态列口径:商标维度与外观维度都给出有效判定才算「成功」。
|
||||
* 结论列只有在两个维度都成功时才会产出「侵权」/「无侵权」(见 AppearancePatentLlmClient#buildConclusion),
|
||||
|
||||
+7
-1
@@ -66,7 +66,13 @@ public class DedupeRunController {
|
||||
}
|
||||
|
||||
@GetMapping("/run/{runId}/progress")
|
||||
@Operation(summary = "查询去重任务进度", description = "轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。")
|
||||
@Operation(summary = "查询去重任务进度", description = """
|
||||
轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。
|
||||
|
||||
runId 就是提交时落库的任务编号(DEDUPE-xxx),任何实例都能应答:
|
||||
优先读共享进度快照,快照缺失或落后时按任务编号从任务表与逐文件结果行重建,
|
||||
因此发版重启或轮询打到另一节点都不会再把已完成的文件报成「任务不存在」。
|
||||
""")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeRunProgressVo.class)))
|
||||
})
|
||||
|
||||
+242
-41
@@ -4,8 +4,12 @@ import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
|
||||
@@ -18,17 +22,20 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.util.WorkbookUtil;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
@@ -47,7 +54,6 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class DedupeRunService {
|
||||
|
||||
@@ -61,17 +67,28 @@ public class DedupeRunService {
|
||||
/** 单次请求最多允许的源文件数,避免每个文件形成大量排队对象。 */
|
||||
private static final int MAX_INPUT_FILES = 200;
|
||||
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
|
||||
/** 去重进度快照的共享保留期:任务结束后仍可查(与历史列表衔接),并留出发版重启的余量。 */
|
||||
private static final Duration RUN_STATE_TTL = Duration.ofHours(3);
|
||||
/** 开机回收时写回的中断原因:点明是服务重启,用户才知道要重新提交。 */
|
||||
private static final String BOOT_INTERRUPTED_MESSAGE = "服务重启,去重任务已中断,已完成的文件可在历史记录下载,请重新提交未处理文件";
|
||||
/** 开机回收一次最多处理多少条本机遗留任务(正常只有个位数)。 */
|
||||
private static final int MAX_BOOT_RECLAIM_ROWS = 100;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final StorageProperties storageProperties;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final DedupeTotalDataService dedupeTotalDataService;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
/** runId -> 进度快照(任务结束后保留 1 小时供轮询) */
|
||||
private final Map<String, DedupeRunProgressVo> runProgressMap = new ConcurrentHashMap<>();
|
||||
/** runId -> 完成时间戳,用于过期清理 */
|
||||
private final Map<String, Long> runCompletedAtMap = new ConcurrentHashMap<>();
|
||||
/**
|
||||
* runId → 进度快照。本地 Map 作快路径、Redis 作跨节点真源:
|
||||
* 生产是双实例且 nginx 亲和只覆盖常态,快照只在提交节点内存里时,
|
||||
* 轮询落到另一节点就会把「其实已经跑完」的任务报成「不存在或已过期」。
|
||||
*/
|
||||
private final NodeSharedStore<String, DedupeRunProgressVo> runProgressMap;
|
||||
/** runId → 完成时间戳,用于过期清理 */
|
||||
private final NodeSharedStore<String, Long> runCompletedAtMap;
|
||||
/** userId -> 运行中任务数,限制单用户并发任务 */
|
||||
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
|
||||
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
|
||||
@@ -80,6 +97,32 @@ public class DedupeRunService {
|
||||
private final ExecutorService fileExecutor = com.nanri.aiimage.common.util.ThreadPools
|
||||
.boundedFixed("dedupe-file-worker", MAX_PARALLEL_FILES);
|
||||
|
||||
public DedupeRunService(FileTaskMapper fileTaskMapper,
|
||||
FileResultMapper fileResultMapper,
|
||||
StorageProperties storageProperties,
|
||||
OssStorageService ossStorageService,
|
||||
DedupeTotalDataService dedupeTotalDataService,
|
||||
StringRedisTemplate stringRedisTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
InstanceMetadata instanceMetadata) {
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.fileResultMapper = fileResultMapper;
|
||||
this.storageProperties = storageProperties;
|
||||
this.ossStorageService = ossStorageService;
|
||||
this.dedupeTotalDataService = dedupeTotalDataService;
|
||||
this.instanceMetadata = instanceMetadata;
|
||||
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
|
||||
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
|
||||
// 完成时间只用于本节点保留期清理,不参与跨节点读取 → 不节流
|
||||
this.runCompletedAtMap = new NodeSharedStore<>("aiimage:dedupe:run:completed-at", RUN_STATE_TTL,
|
||||
Long.class, stringRedisTemplate, objectMapper, 0L);
|
||||
}
|
||||
|
||||
private String currentInstanceId() {
|
||||
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
|
||||
return instanceId == null || instanceId.isBlank() ? "unknown-instance" : instanceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。
|
||||
*/
|
||||
@@ -102,7 +145,6 @@ public class DedupeRunService {
|
||||
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
|
||||
}
|
||||
|
||||
String runId = IdUtil.fastSimpleUUID();
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType("DEDUPE");
|
||||
@@ -111,6 +153,8 @@ public class DedupeRunService {
|
||||
task.setSourceFileCount(request.getFiles().size());
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
// 归属实例落库:开机回收只把「本机自己名下」的 RUNNING 判死,双实例下才不会误杀对端在跑的任务
|
||||
task.setOwnerInstanceId(currentInstanceId());
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -121,6 +165,9 @@ public class DedupeRunService {
|
||||
decrementRunningCount(request.getUserId());
|
||||
throw ex;
|
||||
}
|
||||
// runId 直接用已落库且唯一的 task_no:内存快照丢失后(发版重启 / 轮询打到另一节点),
|
||||
// 凭 runId 就能走 uk_task_no 反查到任务行,重建出真实进度而不是报「任务不存在」。
|
||||
final String runId = task.getTaskNo();
|
||||
|
||||
DedupeRunProgressVo progress = new DedupeRunProgressVo();
|
||||
progress.setRunId(runId);
|
||||
@@ -130,7 +177,9 @@ public class DedupeRunService {
|
||||
progress.setSuccessCount(0);
|
||||
progress.setFailedCount(0);
|
||||
progress.setFinished(false);
|
||||
runProgressMap.put(runId, progress);
|
||||
// 首帧立即共享:客户端拿到 runId 后第一次轮询就可能落到另一节点,
|
||||
// 那一次如果读不到就会当场报「去重任务不存在或已过期」。
|
||||
runProgressMap.putNow(runId, snapshot(progress));
|
||||
|
||||
try {
|
||||
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
|
||||
@@ -146,23 +195,101 @@ public class DedupeRunService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询去重任务进度;runId 不存在或已过期时返回 status=not_found 的占位结果。
|
||||
* 查询去重任务进度。三级来源,全部落在"说真话"上:
|
||||
* 1. 共享进度快照(本节点或另一节点写入);
|
||||
* 2. 快照缺失/落后时按 runId(= task_no)反查 biz_file_task + 逐文件结果行重建;
|
||||
* 3. 库里也确实没有这条 run 才返回 status=not_found。
|
||||
*/
|
||||
public DedupeRunProgressVo getProgress(String runId) {
|
||||
DedupeRunProgressVo progress = runProgressMap.get(runId);
|
||||
if (progress == null) {
|
||||
DedupeRunProgressVo notFound = new DedupeRunProgressVo();
|
||||
notFound.setRunId(runId);
|
||||
notFound.setStatus("not_found");
|
||||
notFound.setTotal(0);
|
||||
notFound.setProcessedCount(0);
|
||||
notFound.setSuccessCount(0);
|
||||
notFound.setFailedCount(0);
|
||||
notFound.setFinished(false);
|
||||
notFound.setError("任务不存在或已过期");
|
||||
return notFound;
|
||||
if (progress != null) {
|
||||
if (snapshotIsTerminal(progress)) {
|
||||
return snapshot(progress);
|
||||
}
|
||||
// 快照还说自己 in flight:提交节点可能已经重启,库里早已是终态 —— 以库为准
|
||||
DedupeRunProgressVo rebuilt = progressFromDatabase(runId);
|
||||
if (rebuilt != null && rebuilt.isFinished()) {
|
||||
runProgressMap.remove(runId);
|
||||
return rebuilt;
|
||||
}
|
||||
return snapshot(progress);
|
||||
}
|
||||
return snapshot(progress);
|
||||
DedupeRunProgressVo rebuilt = progressFromDatabase(runId);
|
||||
if (rebuilt != null) {
|
||||
return rebuilt;
|
||||
}
|
||||
DedupeRunProgressVo notFound = new DedupeRunProgressVo();
|
||||
notFound.setRunId(runId);
|
||||
notFound.setStatus("not_found");
|
||||
notFound.setTotal(0);
|
||||
notFound.setProcessedCount(0);
|
||||
notFound.setSuccessCount(0);
|
||||
notFound.setFailedCount(0);
|
||||
notFound.setFinished(false);
|
||||
notFound.setError("任务不存在或已过期");
|
||||
return notFound;
|
||||
}
|
||||
|
||||
private boolean snapshotIsTerminal(DedupeRunProgressVo progress) {
|
||||
return progress.isFinished() || "success".equals(progress.getStatus()) || "failed".equals(progress.getStatus());
|
||||
}
|
||||
|
||||
/**
|
||||
* 用落库状态重建进度:任务行给总数与终态,biz_file_result 的逐文件行给已完成部分。
|
||||
* 结果行本来就是"每个文件一条"的完成集,所以重启后已上传成功的文件仍能拿到下载地址,
|
||||
* 用户不必因为进程被杀而重跑整批。
|
||||
*/
|
||||
private DedupeRunProgressVo progressFromDatabase(String runId) {
|
||||
FileTaskEntity task = fileTaskMapper.selectOne(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getTaskNo, runId)
|
||||
.eq(FileTaskEntity::getModuleType, "DEDUPE"));
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, "DEDUPE")
|
||||
.orderByAsc(FileResultEntity::getId));
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
List<DedupeResultItemVo> items = new ArrayList<>(rows.size());
|
||||
for (FileResultEntity row : rows) {
|
||||
if (row.getSuccess() != null && row.getSuccess() == 1) {
|
||||
successCount++;
|
||||
} else {
|
||||
failedCount++;
|
||||
}
|
||||
items.add(toResultItem(row, task));
|
||||
}
|
||||
|
||||
DedupeRunProgressVo vo = new DedupeRunProgressVo();
|
||||
vo.setRunId(runId);
|
||||
vo.setTotal(task.getSourceFileCount() == null ? rows.size() : task.getSourceFileCount());
|
||||
vo.setSuccessCount(successCount);
|
||||
vo.setFailedCount(failedCount);
|
||||
vo.setProcessedCount(successCount + failedCount);
|
||||
boolean succeeded = "SUCCESS".equals(task.getStatus());
|
||||
if (succeeded) {
|
||||
DedupeRunVo result = new DedupeRunVo();
|
||||
result.setTotal(vo.getTotal());
|
||||
result.setSuccessCount(successCount);
|
||||
result.setFailedCount(failedCount);
|
||||
result.setItems(items);
|
||||
vo.setResult(result);
|
||||
}
|
||||
if ("RUNNING".equals(task.getStatus()) || "PENDING".equals(task.getStatus())
|
||||
|| "SCHEDULED".equals(task.getStatus())) {
|
||||
vo.setStatus("running");
|
||||
vo.setFinished(false);
|
||||
return vo;
|
||||
}
|
||||
vo.setStatus(succeeded ? "success" : "failed");
|
||||
vo.setFinished(true);
|
||||
if (!succeeded) {
|
||||
vo.setError(task.getErrorMessage() == null || task.getErrorMessage().isBlank()
|
||||
? "去重任务已中断,请重新提交" : task.getErrorMessage());
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
|
||||
@@ -190,6 +317,10 @@ public class DedupeRunService {
|
||||
outcomeItems.add(item);
|
||||
}
|
||||
}
|
||||
// 锁外做 IO:进度快照共享出去(500ms 节流)+ 刷任务行心跳,
|
||||
// 让另一节点能报真实进度、也让 2 小时心跳判死不会误杀长任务
|
||||
runProgressMap.put(runId, snapshot(progress));
|
||||
touchRunHeartbeat(task.getId());
|
||||
}));
|
||||
}
|
||||
// 逐个等待:processFile 内部已把单文件失败记入 outcomeItems,不会往外抛;
|
||||
@@ -241,11 +372,78 @@ public class DedupeRunService {
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
} finally {
|
||||
runCompletedAtMap.put(runId, System.currentTimeMillis());
|
||||
// 终态必须绕过节流立刻共享:客户端下一次轮询可能落到另一节点
|
||||
runProgressMap.putNow(runId, snapshot(progress));
|
||||
runCompletedAtMap.putNow(runId, System.currentTimeMillis());
|
||||
decrementRunningCount(request.getUserId());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新任务行心跳。StaleTaskRepairService 按「updated_at 超过 2 小时未刷新」把 RUNNING 判死,
|
||||
* 而去重原本只在收尾时写一次库 —— 一批文件跑过 2 小时就会被误判成失败。
|
||||
*/
|
||||
void touchRunHeartbeat(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, taskId)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
} catch (Exception ex) {
|
||||
log.warn("dedupe run heartbeat failed taskId={} msg={}", taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 开机回收本机遗留的去重任务:进程刚起来,自己名下仍处于 RUNNING 的行不可能是活的,
|
||||
* 只会让前端对着「处理中」空转到轮询超时(最坏 2 小时才被心跳巡检判死)。
|
||||
* 只按 owner_instance_id 收敛,另一台实例正在跑的任务不受影响。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void reclaimInterruptedRunsOnBoot() {
|
||||
try {
|
||||
List<FileTaskEntity> orphans = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId, FileTaskEntity::getTaskNo)
|
||||
.eq(FileTaskEntity::getModuleType, "DEDUPE")
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.eq(FileTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||
.orderByAsc(FileTaskEntity::getId)
|
||||
.last("limit " + MAX_BOOT_RECLAIM_ROWS));
|
||||
if (orphans.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<Long> reclaimedIds = new ArrayList<>();
|
||||
for (FileTaskEntity orphan : orphans) {
|
||||
if (orphan == null || orphan.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, orphan.getId())
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, BOOT_INTERRUPTED_MESSAGE)
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated <= 0) {
|
||||
continue;
|
||||
}
|
||||
reclaimedIds.add(orphan.getId());
|
||||
if (orphan.getTaskNo() != null) {
|
||||
// 快照还留在 Redis 的话,下一轮会把已中断的任务继续报成"处理中"
|
||||
runProgressMap.remove(orphan.getTaskNo());
|
||||
}
|
||||
}
|
||||
if (!reclaimedIds.isEmpty()) {
|
||||
log.warn("dedupe boot reclaim failed interrupted runs owner={} count={} ids={}",
|
||||
currentInstanceId(), reclaimedIds.size(), reclaimedIds);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.error("dedupe boot reclaim failed owner={} err={}", currentInstanceId(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DedupeRunVo buildRunVo(DedupeRunRequest request, List<DedupeResultItemVo> items,
|
||||
DedupeRunProgressVo progress) {
|
||||
DedupeRunVo vo = new DedupeRunVo();
|
||||
@@ -409,26 +607,28 @@ public class DedupeRunService {
|
||||
// 任务级信息(状态/开始/结束时间)来自 biz_file_task,按 task_id 批量联查
|
||||
Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
|
||||
return entities.stream()
|
||||
.map(entity -> {
|
||||
DedupeResultItemVo vo = new DedupeResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getSourceFilename());
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||
vo.setError(entity.getErrorMessage());
|
||||
FileTaskEntity task = entity.getTaskId() == null ? null : taskMap.get(entity.getTaskId());
|
||||
vo.setTaskId(entity.getTaskId());
|
||||
vo.setTaskStatus(task == null ? null : task.getStatus());
|
||||
vo.setCreatedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||
vo.setStartedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||
return vo;
|
||||
})
|
||||
.map(entity -> toResultItem(entity, entity.getTaskId() == null ? null : taskMap.get(entity.getTaskId())))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/** 结果行 → 前端条目:历史列表与重启后的进度重建共用同一套口径,避免两处各写一遍。 */
|
||||
private DedupeResultItemVo toResultItem(FileResultEntity entity, FileTaskEntity task) {
|
||||
DedupeResultItemVo vo = new DedupeResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getSourceFilename());
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||
vo.setError(entity.getErrorMessage());
|
||||
vo.setTaskId(entity.getTaskId());
|
||||
vo.setTaskStatus(task == null ? null : task.getStatus());
|
||||
vo.setCreatedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||
vo.setStartedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 批量加载去重结果关联的任务信息(复用 appearancepatent 的列裁剪写法) */
|
||||
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
@@ -968,8 +1168,9 @@ public class DedupeRunService {
|
||||
|
||||
private void cleanupExpiredProgress() {
|
||||
long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS;
|
||||
runCompletedAtMap.forEach((runId, completedAt) -> {
|
||||
if (completedAt != null && completedAt < cutoff && runCompletedAtMap.remove(runId, completedAt)) {
|
||||
runCompletedAtMap.localEntriesSnapshot().forEach((runId, completedAt) -> {
|
||||
if (completedAt != null && completedAt < cutoff) {
|
||||
runCompletedAtMap.remove(runId);
|
||||
runProgressMap.remove(runId);
|
||||
}
|
||||
});
|
||||
|
||||
+62
@@ -428,6 +428,68 @@ public class TaskFileJobService {
|
||||
.collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a));
|
||||
}
|
||||
|
||||
/**
|
||||
* 开机自回收:把本机 owner 名下「RUNNING 且心跳已停」的作业退回 PENDING 并补发派发事件。
|
||||
*
|
||||
* <p>进程被杀(发版、崩溃、容器重建)时这些行不会自己退出 RUNNING,只能等 60 秒一轮的
|
||||
* stuck 扫描按时间阈值判死——阈值历史上是 30 分钟,等于每次发版都把在跑的结果文件作业
|
||||
* 冻结一整个阈值窗口,而这批行其实早已无人持有。进程刚起来时本机不可能还在跑它们,
|
||||
* 所以这一刻就能直接接回来,把等待从「阈值」压到「启动」。
|
||||
*
|
||||
* <p>两道保险:只取 owner 等值等于自己的行(绝不碰另一台实例在跑的),且必须已停止心跳
|
||||
* 一个完整周期以上(正常心跳是 60 秒一次,误抢会让在跑的作业被双份派发)。
|
||||
*
|
||||
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,每发一次版
|
||||
* 吃掉它那 5 次重试中的一次,反复发版会把正常任务判成"已达到最大重试次数"。
|
||||
*
|
||||
* <p>刻意不加事务,与 {@link #resetStuckRunningJobsDetailed} 同理:每行都带
|
||||
* status + updated_at 的 CAS 条件,逐行独立提交,锁即时释放。
|
||||
*/
|
||||
public int reclaimOwnedRunningJobs(String owner, long staleSeconds, int limit) {
|
||||
String normalizedOwner = owner == null ? "" : owner.trim();
|
||||
if (normalizedOwner.isBlank()) {
|
||||
return 0;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusSeconds(Math.max(60L, staleSeconds));
|
||||
int safeLimit = Math.max(1, Math.min(limit, 100));
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(TaskFileJobEntity::getOwner, normalizedOwner)
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, cutoff)
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
if (jobs.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
List<Long> resetIds = new ArrayList<>();
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
if (job == null || job.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "实例重启,作业已重新排队")
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
resetIds.add(job.getId());
|
||||
}
|
||||
}
|
||||
if (resetIds.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
Map<Long, TaskFileJobEntity> refreshedById = refreshJobsByIds(resetIds);
|
||||
for (Long jobId : resetIds) {
|
||||
publishDispatchEvent(refreshedById.get(jobId));
|
||||
}
|
||||
log.warn("[task-file-job] 开机自回收本机遗留作业 count={} owner={} staleSeconds={} limit={}",
|
||||
resetIds.size(), normalizedOwner, staleSeconds, safeLimit);
|
||||
return resetIds.size();
|
||||
}
|
||||
|
||||
public record StuckJobResetResult(int resetCount, List<TaskFileJobEntity> exhaustedJobs) {
|
||||
}
|
||||
|
||||
|
||||
+39
-1
@@ -12,6 +12,8 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -43,12 +45,22 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.batch-size:20}")
|
||||
private int batchSize;
|
||||
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
|
||||
/**
|
||||
* 卡死判定阈值。作业在跑期间每 60 秒 touch 一次 {@code updated_at},所以"超过 5 分钟没心跳"
|
||||
* 就等于持有者已经没了;取 30 分钟时一次实例被销毁重建(owner 变了、开机回收够不着)
|
||||
* 会把结果文件作业冻结半小时。队列积压导致的"认领后迟迟没执行"不会误伤:
|
||||
* 真正开跑前的 {@code activateRunningClaim} 用 updated_at 做 CAS,被复位的作业会直接放弃执行。
|
||||
*/
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:5}")
|
||||
private int stuckTimeoutMinutes;
|
||||
|
||||
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
|
||||
private long heartbeatIntervalMillis = 60000L;
|
||||
|
||||
/** 开机回收的"心跳已停"判据:默认 90 秒 ≈ 1.5 个心跳周期,容得下一次正常心跳延迟。 */
|
||||
@Value("${aiimage.result-file-job.boot-reclaim-stale-seconds:90}")
|
||||
private long bootReclaimStaleSeconds;
|
||||
|
||||
/** 多个结果文件 Job 并行运行时,心跳不能被单个 DB 慢调用串行阻塞。 */
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
|
||||
@@ -99,6 +111,32 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用就绪即回收本机遗留作业。
|
||||
*
|
||||
* <p>发版杀进程时正在跑的 job 会一直挂在 RUNNING,要等卡死扫描按 stuck-timeout-minutes
|
||||
* 判死才重新排队——那段时间里用户看到的是"任务卡住不动",而实际上没有任何进程在跑它。
|
||||
* 本机刚起来,这些 owner=自己的 RUNNING 行不可能还在本机执行,所以此刻就能直接接回来,
|
||||
* 不必等阈值;回收不消耗 retry_count(发版不是作业的错)。
|
||||
*
|
||||
* <p>只处理 owner 等于本机实例 id 的行,另一台实例在跑的作业不受影响。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void reclaimOwnedJobsOnBoot() {
|
||||
try {
|
||||
int reclaimed = taskFileJobService.reclaimOwnedRunningJobs(
|
||||
currentInstanceId(), bootReclaimStaleSeconds, batchSize);
|
||||
if (reclaimed > 0) {
|
||||
log.warn("[task-file-job] booted and reclaimed abandoned jobs of this instance reclaimed={} instanceId={} staleSeconds={}",
|
||||
reclaimed, currentInstanceId(), bootReclaimStaleSeconds);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// 回收只是加速恢复;失败不影响启动,兜底仍是 stuck 扫描。
|
||||
log.error("[task-file-job] boot reclaim failed, falling back to stuck scan instanceId={} timeoutMinutes={} err={}",
|
||||
currentInstanceId(), stuckTimeoutMinutes, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void process(TaskFileJobEntity job) {
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
|
||||
@@ -81,7 +81,7 @@ AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE=2
|
||||
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY=200
|
||||
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED=false
|
||||
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS=10000
|
||||
AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=30
|
||||
AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=5
|
||||
|
||||
AIIMAGE_ZINIAO_ENABLED=false
|
||||
AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router
|
||||
|
||||
@@ -275,7 +275,11 @@ aiimage:
|
||||
local-worker-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED:true}
|
||||
local-worker-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS:10000}
|
||||
stuck-scan-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_STUCK_SCAN_DELAY_MS:60000}
|
||||
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:30}
|
||||
# 作业运行期间每 60s touch 一次 updated_at,所以"5 分钟没心跳"就等于持有者已经没了;
|
||||
# 30 分钟是发版/容器重建后结果文件作业被白白冻结的时长。
|
||||
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:5}
|
||||
# 开机自回收的"已停止心跳"判据(≈1.5 个心跳周期),只回收 owner=本机 的 RUNNING 作业。
|
||||
boot-reclaim-stale-seconds: ${AIIMAGE_RESULT_FILE_JOB_BOOT_RECLAIM_STALE_SECONDS:90}
|
||||
heartbeat-interval-ms: ${AIIMAGE_RESULT_FILE_JOB_HEARTBEAT_INTERVAL_MS:60000}
|
||||
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
|
||||
# 客户端兜底拉取:页面没推送成功(只解析没点启动 / 推送前关页面 / 纯浏览器打开)、
|
||||
|
||||
+273
@@ -0,0 +1,273 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.service.ResultDownloadResolver;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 「行什么时候算检测完」的时序约束。
|
||||
*
|
||||
* <p>合批之后一次请求带走 5–10 行,一次中断就是整批白做。原实现在收集候选时(发请求**之前**)
|
||||
* 就把行标成已处理,所以发版重启留下的不是"稍后补检",而是"永不再检"。
|
||||
* 这里钉住两条:申领发生在收集阶段、完成标记只允许出现在载荷 CAS 落库成功之后。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AppearancePatentRowMarkTimingTest {
|
||||
|
||||
private static final Long TASK_ID = 29436L;
|
||||
private static final Long JOB_ID = 20788L;
|
||||
private static final String SCOPE_HASH = "scope-hash-1";
|
||||
private static final int CHUNK_INDEX = 1;
|
||||
private static final String POINTER = "ptr-chunk-1.json";
|
||||
private static final String UNRESOLVED_ROWS =
|
||||
"[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\"},"
|
||||
+ "{\"rowToken\":\"r2\",\"asin\":\"B0A0000002\",\"title\":\"COTTON BAG\",\"url\":\"http://img/2.jpg\",\"groupKey\":\"g2\"}]";
|
||||
private static final String RESOLVED_ROWS =
|
||||
"[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"},"
|
||||
+ "{\"rowToken\":\"r2\",\"asin\":\"B0A0000002\",\"title\":\"COTTON BAG\",\"url\":\"http://img/2.jpg\",\"groupKey\":\"g2\",\"titleRisk\":\"侵权\",\"appearanceRisk\":\"侵权\",\"conclusion\":\"侵权\"}]";
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private ResultDownloadResolver resultDownloadResolver;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private AppearancePatentLlmClient llmClient;
|
||||
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||
@Mock private AppearancePatentProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
@Mock private UserApiSecretService userApiSecretService;
|
||||
|
||||
@InjectMocks private AppearancePatentTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.getLlmBatchSize()).thenReturn(2);
|
||||
lenient().when(properties.getFlushPendingMinutes()).thenReturn(1);
|
||||
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||
lenient().when(userApiSecretService.findPlainValue(any(), anyString())).thenReturn("");
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER), anyString())).thenReturn(UNRESOLVED_ROWS);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk()));
|
||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk());
|
||||
}
|
||||
|
||||
/** 收集阶段只申领、不完成:这是「先干活后标记」和旧行为唯一看得见的区别。 */
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void collectionClaimsRowsWithoutCompletingThem() {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
|
||||
|
||||
assertEquals(2, candidates.size());
|
||||
verify(taskCacheService).beginRowInspection(TASK_ID, "r1");
|
||||
verify(taskCacheService).beginRowInspection(TASK_ID, "r2");
|
||||
verify(taskCacheService, never()).completeRowInspection(anyLong(), any());
|
||||
}
|
||||
|
||||
/** 已经被别的入口申领走的行不再进来,否则会双发 LLM、双扣次数。 */
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void rowsHeldByAnotherEntryAreSkipped() {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r1"))).thenReturn(true);
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r2"))).thenReturn(false);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
|
||||
|
||||
assertEquals(1, candidates.size());
|
||||
assertEquals("r1", candidates.get(0).getRowToken());
|
||||
}
|
||||
|
||||
/** 落库 CAS 成功之后才允许打完成标记——顺序错了就等于回到"发版丢行"的老 bug。 */
|
||||
@Test
|
||||
void completionMarkComesAfterPayloadCasUpdate() throws Exception {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
|
||||
stubMergeWrite(1);
|
||||
|
||||
submit();
|
||||
|
||||
InOrder order = inOrder(taskChunkMapper, taskCacheService);
|
||||
order.verify(taskChunkMapper).update(any(), any());
|
||||
order.verify(taskCacheService).completeRowInspection(eq(TASK_ID), eq(List.of("r1", "r2")));
|
||||
}
|
||||
|
||||
/**
|
||||
* LLM 整批失败:失败原因照样落库(用户看得见),但**不打完成标记**,而是撤销处理中标记——
|
||||
* assemble job 的 5 次重试几分钟内就跑完了,等 TTL 过期的话这批行在整个重试窗口里都申领不到。
|
||||
* 旧行为是"永远记着这次失败",再不会重检。
|
||||
*/
|
||||
@Test
|
||||
void failedLlmBatchLeavesRowsRetryable() throws Exception {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
when(llmClient.inspectRows(any(), any(), any())).thenThrow(new IllegalStateException("LLM 网关无应答"));
|
||||
when(llmClient.markRowsFailed(any(), anyString())).thenReturn(rows(UNRESOLVED_ROWS));
|
||||
stubMergeWrite(1);
|
||||
|
||||
submit();
|
||||
|
||||
verify(taskCacheService, never()).completeRowInspection(anyLong(), any());
|
||||
verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r1", "r2"));
|
||||
}
|
||||
|
||||
/** 成功批次不能撤销处理中标记:撤销是给"下一轮立刻重试"用的,成功行重跑就是重复扣次数。 */
|
||||
@Test
|
||||
void successfulBatchDoesNotReleaseInFlightMarks() throws Exception {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
|
||||
stubMergeWrite(1);
|
||||
|
||||
submit();
|
||||
|
||||
verify(taskCacheService, never()).releaseRowInspection(anyLong(), any());
|
||||
}
|
||||
|
||||
/** 结果拿到了但载荷写不进去(CAS 重试耗尽):同样不能算完成,否则结果是内存里的幻影。 */
|
||||
@Test
|
||||
void exhaustedCasDoesNotCompleteRows() throws Exception {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
|
||||
stubMergeWrite(0);
|
||||
|
||||
assertThrows(IllegalStateException.class, this::submit);
|
||||
|
||||
verify(taskCacheService, never()).completeRowInspection(anyLong(), any());
|
||||
verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r1", "r2"));
|
||||
}
|
||||
|
||||
/** 发送前续一次处理中标记,避免一批跑得比 TTL 长时被并发入口抢走。 */
|
||||
@Test
|
||||
void inFlightMarkIsRenewedRightBeforeSending() throws Exception {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
|
||||
stubMergeWrite(1);
|
||||
|
||||
submit();
|
||||
|
||||
ArgumentCaptor<List<String>> keys = ArgumentCaptor.forClass(List.class);
|
||||
InOrder order = inOrder(taskCacheService, llmClient);
|
||||
order.verify(taskCacheService).touchRowInspection(eq(TASK_ID), keys.capture());
|
||||
order.verify(llmClient).inspectRows(any(), any(), any());
|
||||
assertEquals(List.of("r1", "r2"), keys.getValue());
|
||||
}
|
||||
|
||||
// ===== 辅助 =====
|
||||
|
||||
private void submit() {
|
||||
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
|
||||
task(), new FileResultEntity(), job(), List.of(chunk()), Map.of());
|
||||
}
|
||||
|
||||
private void stubMergeWrite(int updatedRows) {
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
anyString(), anyLong(), anyString(), any(), anyString())).thenReturn("sibling-1");
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(updatedRows);
|
||||
lenient().when(taskChunkMapper.selectById(anyLong())).thenReturn(chunk());
|
||||
}
|
||||
|
||||
private List<AppearancePatentResultRowDto> rows(String json) throws Exception {
|
||||
return List.of(new ObjectMapper().readValue(json, AppearancePatentResultRowDto[].class));
|
||||
}
|
||||
|
||||
private FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
task.setUserId(7L);
|
||||
return task;
|
||||
}
|
||||
|
||||
private com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job() {
|
||||
com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job =
|
||||
new com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity();
|
||||
job.setId(JOB_ID);
|
||||
job.setTaskId(TASK_ID);
|
||||
return job;
|
||||
}
|
||||
|
||||
private TaskChunkEntity chunk() {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(11L);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(SCOPE_HASH);
|
||||
chunk.setChunkIndex(CHUNK_INDEX);
|
||||
chunk.setPayloadJson(POINTER);
|
||||
chunk.setPayloadHash("hash-old");
|
||||
return chunk;
|
||||
}
|
||||
}
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 外观专利行级检测标记的两态语义。
|
||||
*
|
||||
* <p>历史实现只有一个标记,且在**发 LLM 请求之前**就打({@code collectPendingLlmCandidates}),
|
||||
* TTL 24 小时、无撤销路径。于是进程被杀(发版、崩溃)或载荷落库失败时,那批行既没有结果、
|
||||
* 又被判定为「已处理」,续跑时直接跳过——线上表现为少数几行永远空白而任务报成功。
|
||||
* 现在拆成两态:**处理中**(短 TTL,谁在跑这行)与**已完成**(长 TTL,只在结果落库成功后打)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AppearancePatentTaskCacheServiceRowMarkTest {
|
||||
|
||||
private static final Long TASK_ID = 29436L;
|
||||
private static final String ROW_KEY = "r1";
|
||||
|
||||
@Mock private StringRedisTemplate redisTemplate;
|
||||
@Mock private ValueOperations<String, String> valueOperations;
|
||||
|
||||
private AppearancePatentTaskCacheService cache;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
cache = new AppearancePatentTaskCacheService(redisTemplate, new ObjectMapper());
|
||||
lenient().when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
}
|
||||
|
||||
/** 没任何标记时,第一次申领必须拿到这行。 */
|
||||
@Test
|
||||
void beginClaimsRowWhenNoMarkExists() {
|
||||
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(true);
|
||||
|
||||
assertTrue(cache.beginRowInspection(TASK_ID, ROW_KEY));
|
||||
}
|
||||
|
||||
/** 处理中标记还在(另一个入口正在跑这行)时不能重复申领,否则会双发 LLM、双扣次数。 */
|
||||
@Test
|
||||
void beginRejectsRowStillInFlight() {
|
||||
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(false);
|
||||
|
||||
assertFalse(cache.beginRowInspection(TASK_ID, ROW_KEY));
|
||||
}
|
||||
|
||||
/** 已完成的行连申领都不该发起:不能白占一个处理中名额。 */
|
||||
@Test
|
||||
void beginRejectsCompletedRowWithoutTouchingInFlightMark() {
|
||||
when(redisTemplate.hasKey(anyString())).thenReturn(true);
|
||||
|
||||
assertFalse(cache.beginRowInspection(TASK_ID, ROW_KEY));
|
||||
|
||||
verify(valueOperations, never()).setIfAbsent(anyString(), anyString(), any(Duration.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理中标记必须是分钟级:进程被杀后这些行要能重新成为候选。
|
||||
* 上限取 30 分钟——线上实测单批 10 行最慢 220 秒,再长就是拿丢行换并发安全。
|
||||
*/
|
||||
@Test
|
||||
void inFlightMarkExpiresWithinMinutes() {
|
||||
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(true);
|
||||
|
||||
cache.beginRowInspection(TASK_ID, ROW_KEY);
|
||||
|
||||
ArgumentCaptor<Duration> ttl = ArgumentCaptor.forClass(Duration.class);
|
||||
verify(valueOperations).setIfAbsent(anyString(), anyString(), ttl.capture());
|
||||
assertTrue(ttl.getValue().toSeconds() > 0, "处理中标记必须带 TTL,否则一次中断就永久丢行");
|
||||
assertTrue(ttl.getValue().toMinutes() <= 30, "处理中标记不得超过 30 分钟,实测单批最慢约 220 秒");
|
||||
}
|
||||
|
||||
/** 完成标记只在落库成功后打:长 TTL,并且要把处理中名额让出来。 */
|
||||
@Test
|
||||
void completeMarksLongTtlAndReleasesInFlight() {
|
||||
cache.completeRowInspection(TASK_ID, List.of(ROW_KEY));
|
||||
|
||||
ArgumentCaptor<String> completed = ArgumentCaptor.forClass(String.class);
|
||||
ArgumentCaptor<Duration> ttl = ArgumentCaptor.forClass(Duration.class);
|
||||
verify(valueOperations).set(completed.capture(), anyString(), ttl.capture());
|
||||
assertEquals(Duration.ofHours(24), ttl.getValue());
|
||||
ArgumentCaptor<String> deleted = ArgumentCaptor.forClass(String.class);
|
||||
verify(redisTemplate).delete(deleted.capture());
|
||||
assertNotEquals(completed.getValue(), deleted.getValue(), "清除的必须是处理中键,不能把刚打的完成标记删掉");
|
||||
}
|
||||
|
||||
/** 两态必须落在不同键上:同键的话「完成」会把短 TTL 覆盖成长 TTL,或反过来互相清除。 */
|
||||
@Test
|
||||
void inFlightAndCompletedMarksUseDifferentKeys() {
|
||||
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(true);
|
||||
|
||||
cache.beginRowInspection(TASK_ID, ROW_KEY);
|
||||
cache.completeRowInspection(TASK_ID, List.of(ROW_KEY));
|
||||
|
||||
ArgumentCaptor<String> inFlight = ArgumentCaptor.forClass(String.class);
|
||||
verify(valueOperations).setIfAbsent(inFlight.capture(), anyString(), any(Duration.class));
|
||||
ArgumentCaptor<String> completed = ArgumentCaptor.forClass(String.class);
|
||||
verify(valueOperations).set(completed.capture(), anyString(), any(Duration.class));
|
||||
assertNotEquals(inFlight.getValue(), completed.getValue());
|
||||
}
|
||||
|
||||
/** 发送前续期只延长处理中,绝不碰完成标记。 */
|
||||
@Test
|
||||
void touchRenewsOnlyInFlightMark() {
|
||||
cache.touchRowInspection(TASK_ID, List.of(ROW_KEY));
|
||||
|
||||
verify(redisTemplate).expire(anyString(), any(Duration.class));
|
||||
verify(valueOperations, never()).set(anyString(), anyString(), any(Duration.class));
|
||||
}
|
||||
|
||||
/** Redis 降级时必须放行:持久化层的 hasResolvedLlmFields 才是最终判据,不能因为 Redis 挂了就整单停摆。 */
|
||||
@Test
|
||||
void beginAllowsRowThroughWhenRedisFails() {
|
||||
when(redisTemplate.hasKey(anyString())).thenThrow(new IllegalStateException("redis down"));
|
||||
|
||||
assertTrue(cache.beginRowInspection(TASK_ID, ROW_KEY));
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测失败要立刻撤销处理中标记,而不是等 TTL 过期。
|
||||
*
|
||||
* <p>assemble job 的 5 次重试是在几分钟内跑完的,若失败后仍占着 15 分钟的处理中名额,
|
||||
* 这几行在重试窗口内永远申领不到——"能重检"就变成了"更快地彻底失败"。
|
||||
*/
|
||||
@Test
|
||||
void releaseDeletesInFlightMarkWithoutCompleting() {
|
||||
cache.releaseRowInspection(TASK_ID, List.of(ROW_KEY));
|
||||
|
||||
ArgumentCaptor<String> deleted = ArgumentCaptor.forClass(String.class);
|
||||
verify(redisTemplate).delete(deleted.capture());
|
||||
assertFalse(deleted.getValue().contains("processed-rows"), "撤销只能删处理中键,删完成标记会让已检测的行重跑并重复计费");
|
||||
verify(valueOperations, never()).set(anyString(), anyString(), any(Duration.class));
|
||||
}
|
||||
}
|
||||
+4
-3
@@ -209,8 +209,8 @@ class AppearancePatentTaskServiceDelegationTest {
|
||||
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
|
||||
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
|
||||
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
|
||||
assertEquals("未取得检测结果", data.getCell(9).getStringCellValue(), "无结果行判定列显式标注(空白与没风险无法区分)");
|
||||
assertEquals("失败", data.getCell(12).getStringCellValue(), "无结果行状态判失败");
|
||||
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌仍从解析行 values 回退");
|
||||
assertEquals("", data.getCell(5).getStringCellValue(), "无结果行价格留空(resolvePrice 仅读结果行,与现状一致)");
|
||||
}
|
||||
@@ -367,7 +367,8 @@ class AppearancePatentTaskServiceDelegationTest {
|
||||
row.setConclusion("");
|
||||
assertEquals("LLM http 503: This model is currently overloaded",
|
||||
AppearancePatentSheetBuilder.userFacingLlmCellValue(row, row.getTitleRisk()));
|
||||
assertEquals("失败", AppearancePatentSheetBuilder.userFacingStatus(row),
|
||||
assertEquals("失败", AppearancePatentSheetBuilder.resolveResultStatus(
|
||||
AppearancePatentSheetBuilder.userFacingConclusion(row)),
|
||||
"LLM 技术失败没得出判定 → 状态失败(旧口径按结论非空误判为成功)");
|
||||
}
|
||||
|
||||
|
||||
+72
-2
@@ -135,6 +135,12 @@ class AppearancePatentSheetBuilderTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 没有结果的行必须写明"没结果",不能留空。
|
||||
*
|
||||
* <p>覆盖率只要求过半(llm-coverage-min-ratio),所以少数缺失行照样交付;历史上这些格子是
|
||||
* 空字符串,用户在表格里看到的"空白"和"没风险"长得一模一样——静默漏判。
|
||||
*/
|
||||
@Test
|
||||
void test_sheet_builder_no_result_row_uses_parsed_fallback() throws Exception {
|
||||
try (Workbook wb = new XSSFWorkbook()) {
|
||||
@@ -145,8 +151,72 @@ class AppearancePatentSheetBuilderTest {
|
||||
assertEquals("title-a", data.getCell(6).getStringCellValue(), "无结果行回退解析行标题");
|
||||
assertEquals("http://img/a.jpg", data.getCell(7).getStringCellValue(), "回退解析行 URL");
|
||||
assertEquals("SKU-1", data.getCell(8).getStringCellValue(), "回退解析行 sku");
|
||||
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
|
||||
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
|
||||
assertEquals("未取得检测结果", data.getCell(9).getStringCellValue(), "无结果行标题维度显式标注");
|
||||
assertEquals("未取得检测结果", data.getCell(10).getStringCellValue(), "无结果行外观维度显式标注");
|
||||
assertEquals("未取得检测结果", data.getCell(11).getStringCellValue(), "无结果行结论显式标注");
|
||||
assertEquals("失败", data.getCell(12).getStringCellValue(), "无结果行状态判失败,不能留空");
|
||||
|
||||
Row reason = wb.getSheet("原因").getRow(1);
|
||||
assertEquals("B01A", reason.getCell(0).getStringCellValue());
|
||||
assertTrue(reason.getCell(1).getStringCellValue().contains("未取得检测结果"),
|
||||
"原因 sheet 也要说明这行为什么没有原因: " + reason.getCell(1).getStringCellValue());
|
||||
}
|
||||
}
|
||||
|
||||
/** 行存在但三个维度都没给出可用判定(客户端兜底成空串):同样要显式标注,且不能盖掉已有错误信息。 */
|
||||
@Test
|
||||
void test_sheet_builder_undeliverable_row_shows_missing_marker() throws Exception {
|
||||
try (Workbook wb = new XSSFWorkbook()) {
|
||||
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
|
||||
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
|
||||
r.setTitleRisk("");
|
||||
r.setAppearanceRisk("");
|
||||
r.setConclusion("");
|
||||
r.setError("LLM 网关无应答");
|
||||
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
|
||||
AppearancePatentSheetBuilder.rowKey(r), r));
|
||||
|
||||
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||
assertEquals("LLM 网关无应答", data.getCell(9).getStringCellValue(), "有错误信息时展示错误信息");
|
||||
assertEquals("失败", data.getCell(12).getStringCellValue(), "结论列没内容时状态仍判失败");
|
||||
}
|
||||
}
|
||||
|
||||
/** 结论被兜底成非技术性文案(如"外观识别异常")时也属于没结果,不能空着交付。 */
|
||||
@Test
|
||||
void test_sheet_builder_blank_verdict_row_is_marked() throws Exception {
|
||||
try (Workbook wb = new XSSFWorkbook()) {
|
||||
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
|
||||
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
|
||||
r.setTitleRisk("");
|
||||
r.setAppearanceRisk("");
|
||||
r.setConclusion("");
|
||||
r.setError("");
|
||||
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
|
||||
AppearancePatentSheetBuilder.rowKey(r), r));
|
||||
|
||||
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||
assertEquals("未取得检测结果", data.getCell(11).getStringCellValue(), "空白结论改写成显式标注");
|
||||
assertEquals("失败", data.getCell(12).getStringCellValue());
|
||||
}
|
||||
}
|
||||
|
||||
/** 有完整判定的行不受影响:显式标注只能出现在真的没结果的行上。 */
|
||||
@Test
|
||||
void test_sheet_builder_resolved_row_keeps_verdicts() throws Exception {
|
||||
try (Workbook wb = new XSSFWorkbook()) {
|
||||
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
|
||||
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
|
||||
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
|
||||
AppearancePatentSheetBuilder.rowKey(r), r));
|
||||
|
||||
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||
assertEquals("无风险", data.getCell(9).getStringCellValue());
|
||||
assertEquals("无风险", data.getCell(10).getStringCellValue());
|
||||
assertEquals("已侵权", data.getCell(11).getStringCellValue());
|
||||
assertEquals("成功", data.getCell(12).getStringCellValue());
|
||||
assertEquals("", wb.getSheet("原因").getRow(1).getCell(1).getStringCellValue(),
|
||||
"有结果的行不写缺失说明");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
package com.nanri.aiimage.modules.dedupe.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 去重任务的进度可见性(⑤)。
|
||||
*
|
||||
* <p>去重是服务端内存态任务:runId 是随机 UUID、进度只存在提交它的那个进程的 Map 里。
|
||||
* 生产是双实例 + 无粘滞保证,轮询落到另一实例就返回 not_found,前端直接报
|
||||
* 「去重任务不存在或已过期」—— 文件其实已经处理完并进了历史表。发版重启同理。
|
||||
*
|
||||
* <p>本测试锁定三件事:runId 必须能解析到落库的任务行;跨节点/重启后进度要能从
|
||||
* 共享状态或数据库重建;重建出来的状态必须是真话(含已完成文件的下载地址)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class DedupeRunProgressResumeTest {
|
||||
|
||||
private static final String PROGRESS_KEY_PREFIX = "aiimage:dedupe:run:progress:";
|
||||
private static final String RUN_TASK_NO = "DEDUPE-1234567890";
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private FileResultMapper fileResultMapper;
|
||||
@Mock
|
||||
private StorageProperties storageProperties;
|
||||
@Mock
|
||||
private OssStorageService ossStorageService;
|
||||
@Mock
|
||||
private DedupeTotalDataService dedupeTotalDataService;
|
||||
@Mock
|
||||
private StringRedisTemplate redisTemplate;
|
||||
@Mock
|
||||
private ValueOperations<String, String> valueOperations;
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
/** 模拟 Redis:两个 service 实例共用它就是"另一个节点"。 */
|
||||
private final Map<String, String> remoteState = new HashMap<>();
|
||||
|
||||
/** 脱离 Spring 时 MyBatis-Plus 没有 lambda 列缓存,Lambda 条件构造器要先初始化表信息。 */
|
||||
@org.junit.jupiter.api.BeforeAll
|
||||
static void initTableInfo() {
|
||||
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
|
||||
new org.apache.ibatis.builder.MapperBuilderAssistant(
|
||||
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
|
||||
FileTaskEntity.class);
|
||||
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
|
||||
new org.apache.ibatis.builder.MapperBuilderAssistant(
|
||||
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
|
||||
FileResultEntity.class);
|
||||
}
|
||||
|
||||
private DedupeRunService newService() {
|
||||
lenient().when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
lenient().when(valueOperations.get(anyString())).thenAnswer(invocation -> remoteState.get(invocation.getArgument(0)));
|
||||
lenient().doAnswer(invocation -> {
|
||||
remoteState.put(invocation.getArgument(0), invocation.getArgument(1));
|
||||
return null;
|
||||
}).when(valueOperations).set(anyString(), anyString(), any(Duration.class));
|
||||
lenient().when(redisTemplate.delete(anyString())).thenAnswer(invocation -> {
|
||||
remoteState.remove(invocation.getArgument(0, String.class));
|
||||
return true;
|
||||
});
|
||||
return new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
|
||||
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
|
||||
new InstanceMetadata("server-110"));
|
||||
}
|
||||
|
||||
private DedupeRunRequest request(int fileCount) {
|
||||
DedupeRunRequest request = new DedupeRunRequest();
|
||||
request.setUserId(9L);
|
||||
request.setSelectedColumns(List.of("id", "ASIN"));
|
||||
request.setKeepUnderscoreIds(true);
|
||||
List<DedupeSourceFileDto> files = new java.util.ArrayList<>();
|
||||
for (int i = 0; i < fileCount; i++) {
|
||||
DedupeSourceFileDto dto = new DedupeSourceFileDto();
|
||||
dto.setFileKey("key-" + i);
|
||||
dto.setOriginalFilename("file-" + i + ".xlsx");
|
||||
files.add(dto);
|
||||
}
|
||||
request.setFiles(files);
|
||||
return request;
|
||||
}
|
||||
|
||||
private FileTaskEntity task(String status, String errorMessage) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7L);
|
||||
task.setTaskNo(RUN_TASK_NO);
|
||||
task.setModuleType("DEDUPE");
|
||||
task.setStatus(status);
|
||||
task.setSourceFileCount(3);
|
||||
task.setErrorMessage(errorMessage);
|
||||
task.setCreatedAt(LocalDateTime.of(2026, 9, 27, 10, 0));
|
||||
return task;
|
||||
}
|
||||
|
||||
private FileResultEntity result(Long id, String sourceName, String objectKey, boolean success, String error) {
|
||||
FileResultEntity entity = new FileResultEntity();
|
||||
entity.setId(id);
|
||||
entity.setTaskId(7L);
|
||||
entity.setModuleType("DEDUPE");
|
||||
entity.setSourceFilename(sourceName);
|
||||
entity.setResultFilename(objectKey == null ? null : sourceName + "_cleaned.xlsx");
|
||||
entity.setResultFileUrl(objectKey);
|
||||
entity.setResultFileSize(objectKey == null ? null : 1024L);
|
||||
entity.setSuccess(success ? 1 : 0);
|
||||
entity.setErrorMessage(error);
|
||||
entity.setCreatedAt(LocalDateTime.of(2026, 9, 27, 10, 5));
|
||||
return entity;
|
||||
}
|
||||
|
||||
// ---------- runId 必须可解析 ----------
|
||||
|
||||
@Test
|
||||
@DisplayName("runId 用落库的 task_no,并把归属实例一起写进任务行")
|
||||
void runIdIsThePersistedTaskNoAndOwnerIsRecorded() {
|
||||
DedupeRunService service = newService();
|
||||
// 异步 worker 是否来得及读这个目录取决于线程时序,只做兜底、不参与断言
|
||||
lenient().when(storageProperties.getLocalTempDir()).thenReturn("D:/definitely-not-a-real-dedupe-dir/xx");
|
||||
|
||||
DedupeRunProgressVo submitted = service.submitRun(request(1));
|
||||
|
||||
ArgumentCaptor<FileTaskEntity> captor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(fileTaskMapper, atLeastOnce()).insert(captor.capture());
|
||||
FileTaskEntity persisted = captor.getValue();
|
||||
assertEquals(persisted.getTaskNo(), submitted.getRunId(),
|
||||
"runId 必须是已落库且唯一的 task_no,否则重启/换节点后无法解析到任务行");
|
||||
assertEquals("server-110", persisted.getOwnerInstanceId(),
|
||||
"归属实例要落库:开机回收只能收自己名下中断的任务,否则会把另一实例正在跑的任务判死");
|
||||
}
|
||||
|
||||
// ---------- 跨节点:共享状态 ----------
|
||||
|
||||
@Test
|
||||
@DisplayName("另一节点写入的进度快照要能直接读到,不再报 not_found")
|
||||
void progressWrittenByAnotherNodeIsReadable() throws Exception {
|
||||
DedupeRunProgressVo remote = new DedupeRunProgressVo();
|
||||
remote.setRunId(RUN_TASK_NO);
|
||||
remote.setStatus("success");
|
||||
remote.setTotal(3);
|
||||
remote.setProcessedCount(3);
|
||||
remote.setSuccessCount(2);
|
||||
remote.setFailedCount(1);
|
||||
remote.setFinished(true);
|
||||
remote.setResult(new com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo());
|
||||
remote.getResult().setItems(List.of());
|
||||
remoteState.put(PROGRESS_KEY_PREFIX + RUN_TASK_NO, objectMapper.writeValueAsString(remote));
|
||||
|
||||
DedupeRunProgressVo progress = newService().getProgress(RUN_TASK_NO);
|
||||
|
||||
assertEquals("success", progress.getStatus(), "本节点内存没有这条 run,但共享状态有 → 不能返回 not_found");
|
||||
assertTrue(progress.isFinished());
|
||||
assertEquals(2, progress.getSuccessCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("共享状态说 running 而库里已终态时,以库为准并清掉过期快照")
|
||||
void terminalDatabaseStateOvertakesStaleRunningSnapshot() throws Exception {
|
||||
DedupeRunProgressVo stale = new DedupeRunProgressVo();
|
||||
stale.setRunId(RUN_TASK_NO);
|
||||
stale.setStatus("running");
|
||||
stale.setTotal(3);
|
||||
stale.setProcessedCount(1);
|
||||
stale.setSuccessCount(1);
|
||||
stale.setFailedCount(0);
|
||||
stale.setFinished(false);
|
||||
remoteState.put(PROGRESS_KEY_PREFIX + RUN_TASK_NO, objectMapper.writeValueAsString(stale));
|
||||
when(fileTaskMapper.selectOne(any())).thenReturn(task("SUCCESS", null));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(
|
||||
result(1L, "a.xlsx", "dedupe/a.xlsx", true, null),
|
||||
result(2L, "b.xlsx", "dedupe/b.xlsx", true, null),
|
||||
result(3L, "c.xlsx", null, false, "表头缺少所选列")));
|
||||
when(ossStorageService.generateFreshDownloadUrl(anyString())).thenReturn("https://dl/a");
|
||||
|
||||
DedupeRunProgressVo progress = newService().getProgress(RUN_TASK_NO);
|
||||
|
||||
assertEquals("success", progress.getStatus(), "库已 SUCCESS,快照还 running 说明快照过期(提交节点已重启/被回收)");
|
||||
assertFalse(remoteState.containsKey(PROGRESS_KEY_PREFIX + RUN_TASK_NO),
|
||||
"过期快照要删掉,否则下一轮还是读到旧状态");
|
||||
}
|
||||
|
||||
// ---------- 重启/过期后:从库重建 ----------
|
||||
|
||||
@Test
|
||||
@DisplayName("进度快照丢失后用任务行与逐文件结果行重建,已完成文件仍可下载")
|
||||
void progressIsRebuiltFromDatabaseWhenSharedStateExpired() {
|
||||
DedupeRunService service = newService();
|
||||
when(fileTaskMapper.selectOne(any())).thenReturn(task("SUCCESS", null));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(
|
||||
result(1L, "a.xlsx", "dedupe/a.xlsx", true, null),
|
||||
result(2L, "b.xlsx", "dedupe/b.xlsx", true, null),
|
||||
result(3L, "c.xlsx", null, false, "表头缺少所选列")));
|
||||
when(ossStorageService.generateFreshDownloadUrl("dedupe/a.xlsx")).thenReturn("https://dl/a.xlsx");
|
||||
when(ossStorageService.generateFreshDownloadUrl("dedupe/b.xlsx")).thenReturn("https://dl/b.xlsx");
|
||||
|
||||
DedupeRunProgressVo progress = service.getProgress(RUN_TASK_NO);
|
||||
|
||||
assertEquals("success", progress.getStatus());
|
||||
assertEquals(RUN_TASK_NO, progress.getRunId());
|
||||
assertTrue(progress.isFinished());
|
||||
assertEquals(3, progress.getTotal());
|
||||
assertEquals(3, progress.getProcessedCount(), "processed 必须等于已落库结果行数");
|
||||
assertEquals(2, progress.getSuccessCount());
|
||||
assertEquals(1, progress.getFailedCount());
|
||||
assertNotNull(progress.getResult());
|
||||
assertEquals(3, progress.getResult().getItems().size(), "结果行就是交付物,重建后必须还能拿到 3 个文件的处置");
|
||||
assertEquals("https://dl/a.xlsx", progress.getResult().getItems().get(0).getDownloadUrl());
|
||||
assertFalse(progress.getResult().getItems().get(2).isSuccess());
|
||||
assertEquals("表头缺少所选列", progress.getResult().getItems().get(2).getError());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("库里仍在跑:按已落库结果行报真实进度,而不是 not_found")
|
||||
void runningTaskWithoutSharedStateReportsRealCounts() {
|
||||
DedupeRunService service = newService();
|
||||
when(fileTaskMapper.selectOne(any())).thenReturn(task("RUNNING", null));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(
|
||||
result(1L, "a.xlsx", "dedupe/a.xlsx", true, null)));
|
||||
|
||||
DedupeRunProgressVo progress = service.getProgress(RUN_TASK_NO);
|
||||
|
||||
assertEquals("running", progress.getStatus());
|
||||
assertFalse(progress.isFinished());
|
||||
assertEquals(1, progress.getProcessedCount());
|
||||
assertEquals(3, progress.getTotal());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("库里已失败:回写落库的失败原因")
|
||||
void failedTaskWithoutSharedStateReportsItsStoredError() {
|
||||
DedupeRunService service = newService();
|
||||
when(fileTaskMapper.selectOne(any())).thenReturn(task("FAILED", "上传文件不存在,请重新上传"));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(
|
||||
result(1L, "a.xlsx", null, false, "上传文件不存在,请重新上传")));
|
||||
|
||||
DedupeRunProgressVo progress = service.getProgress(RUN_TASK_NO);
|
||||
|
||||
assertEquals("failed", progress.getStatus());
|
||||
assertTrue(progress.isFinished());
|
||||
assertEquals("上传文件不存在,请重新上传", progress.getError());
|
||||
assertEquals(1, progress.getFailedCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("库里也没有这条 run 时,仍然如实返回 not_found")
|
||||
void unknownRunIdStillReportsNotFound() {
|
||||
DedupeRunService service = newService();
|
||||
when(fileTaskMapper.selectOne(any())).thenReturn(null);
|
||||
|
||||
DedupeRunProgressVo progress = service.getProgress("DEDUPE-999");
|
||||
|
||||
assertEquals("not_found", progress.getStatus());
|
||||
assertEquals("任务不存在或已过期", progress.getError());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("重建只查 DEDUPE 模块的行,避免同名 task_no 串到别的模块")
|
||||
void reconstructionIsScopedToDedupeModule() {
|
||||
DedupeRunService service = newService();
|
||||
when(fileTaskMapper.selectOne(any())).thenReturn(null);
|
||||
|
||||
service.getProgress(RUN_TASK_NO);
|
||||
|
||||
ArgumentCaptor<Wrapper<FileTaskEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||
verify(fileTaskMapper).selectOne(captor.capture());
|
||||
String sql = captor.getValue().getSqlSegment();
|
||||
assertTrue(sql.contains("module_type"), "按 runId 反查必须限定 module_type:" + sql);
|
||||
assertTrue(sql.contains("task_no"), "按已落库的 uk_task_no 唯一索引反查:" + sql);
|
||||
}
|
||||
|
||||
// ---------- 发版重启中断的在跑任务 ----------
|
||||
|
||||
@Test
|
||||
@DisplayName("开机把本机遗留的 RUNNING 去重任务判失败,不再空挂 2 小时")
|
||||
void bootReclaimFailsOwnInterruptedRuns() {
|
||||
DedupeRunService service = newService();
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task("RUNNING", null)));
|
||||
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
|
||||
|
||||
service.reclaimInterruptedRunsOnBoot();
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<FileTaskEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(fileTaskMapper).update(isNull(), captor.capture());
|
||||
LambdaUpdateWrapper<FileTaskEntity> update = captor.getValue();
|
||||
assertTrue(update.getSqlSet().contains("status="), "要写终态:" + update.getSqlSet());
|
||||
assertTrue(update.getSqlSet().contains("error_message="), "要写中断原因:" + update.getSqlSet());
|
||||
Map<String, Object> params = paramValues(update);
|
||||
assertTrue(params.containsValue("FAILED"), "本机刚起来,自己名下的 RUNNING 必然是中断残留:" + params);
|
||||
assertTrue(params.values().stream().anyMatch(v -> v instanceof String && ((String) v).contains("重启")),
|
||||
"原因要点明是服务重启,用户才知道要重新提交:" + params);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("开机回收只查本机归属的去重任务")
|
||||
void bootReclaimOnlyTargetsOwnDedupeRuns() {
|
||||
DedupeRunService service = newService();
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of());
|
||||
|
||||
service.reclaimInterruptedRunsOnBoot();
|
||||
|
||||
ArgumentCaptor<Wrapper<FileTaskEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||
verify(fileTaskMapper).selectList(captor.capture());
|
||||
String sql = captor.getValue().getSqlSegment();
|
||||
Map<String, Object> params = paramValues(captor.getValue());
|
||||
assertTrue(sql.contains("owner_instance_id"), "必须按归属实例过滤,否则会把另一实例正在跑的任务判死:" + sql);
|
||||
assertTrue(sql.contains("module_type"), "只管去重模块:" + sql);
|
||||
assertTrue(params.containsValue("server-110"), "查的是本机自己:" + params);
|
||||
verify(fileTaskMapper, org.mockito.Mockito.never()).update(isNull(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("开机回收挂在 ApplicationReadyEvent 上,且异常不能阻断启动")
|
||||
void bootReclaimIsAnnotatedAndCannotBreakStartup() {
|
||||
DedupeRunService service = newService();
|
||||
org.springframework.context.event.EventListener annotation = findListener();
|
||||
assertTrue(java.util.Arrays.asList(annotation.value()).contains(
|
||||
org.springframework.boot.context.event.ApplicationReadyEvent.class),
|
||||
"要在应用就绪后执行,太早查库会连不上数据源");
|
||||
|
||||
when(fileTaskMapper.selectList(any())).thenThrow(new RuntimeException("db down"));
|
||||
service.reclaimInterruptedRunsOnBoot();
|
||||
}
|
||||
|
||||
// ---------- 心跳:别让 2 小时判死误杀真活任务 ----------
|
||||
|
||||
@Test
|
||||
@DisplayName("每个文件处理完刷新任务行心跳,使巡检的 2 小时判死不误杀长任务")
|
||||
void heartbeatTouchesOnlyUpdatedAtOfTheTaskRow() {
|
||||
DedupeRunService service = newService();
|
||||
|
||||
service.touchRunHeartbeat(7L);
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<FileTaskEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(fileTaskMapper).update(isNull(), captor.capture());
|
||||
LambdaUpdateWrapper<FileTaskEntity> update = captor.getValue();
|
||||
assertTrue(update.getSqlSet().contains("updated_at"), "心跳就是刷 updated_at:" + update.getSqlSet());
|
||||
assertFalse(update.getSqlSet().contains("status="), "心跳不许改状态:" + update.getSqlSet());
|
||||
assertTrue(update.getSqlSegment().contains("id"), "按主键定位这一条任务:" + update.getSqlSegment());
|
||||
assertTrue(paramValues(update).containsValue(7L), "更新的是本任务:" + paramValues(update));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("taskId 缺失时心跳静默跳过(异步线程里没有任务 id 不能抛)")
|
||||
void heartbeatIgnoresMissingTaskId() {
|
||||
DedupeRunService service = newService();
|
||||
|
||||
service.touchRunHeartbeat(null);
|
||||
|
||||
verify(fileTaskMapper, org.mockito.Mockito.never()).update(isNull(), any());
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
private org.springframework.context.event.EventListener findListener() {
|
||||
for (java.lang.reflect.Method method : DedupeRunService.class.getDeclaredMethods()) {
|
||||
if ("reclaimInterruptedRunsOnBoot".equals(method.getName())) {
|
||||
org.springframework.context.event.EventListener annotation =
|
||||
method.getAnnotation(org.springframework.context.event.EventListener.class);
|
||||
assertNotNull(annotation, "开机回收必须由事件触发,不能靠 @PostConstruct 抢在数据源就绪前");
|
||||
return annotation;
|
||||
}
|
||||
}
|
||||
throw new AssertionError("缺少 reclaimInterruptedRunsOnBoot 方法");
|
||||
}
|
||||
|
||||
/** LambdaQueryWrapper/UpdateWrapper 的参数表要等 SQL 片段渲染后才会填充。 */
|
||||
private Map<String, Object> paramValues(Object wrapper) {
|
||||
if (wrapper instanceof com.baomidou.mybatisplus.core.conditions.AbstractWrapper<?, ?, ?> query) {
|
||||
if (query instanceof LambdaUpdateWrapper<?> update) {
|
||||
update.getSqlSet();
|
||||
}
|
||||
query.getCustomSqlSegment();
|
||||
return new HashMap<>(query.getParamNameValuePairs());
|
||||
}
|
||||
throw new IllegalStateException("不是 MyBatis-Plus wrapper: " + wrapper);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("共享状态写失败不影响提交(Redis 挂了退化为本地内存,与原行为一致)")
|
||||
void submitStillWorksWhenSharedStateWriteFails() {
|
||||
when(redisTemplate.opsForValue()).thenThrow(new RuntimeException("redis down"))
|
||||
.thenReturn(valueOperations);
|
||||
// 异步 worker 是否来得及读这个目录取决于线程时序,只做兜底、不参与断言
|
||||
lenient().when(storageProperties.getLocalTempDir()).thenReturn("D:/definitely-not-a-real-dedupe-dir/xx");
|
||||
DedupeRunService service = new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
|
||||
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
|
||||
new InstanceMetadata("server-110"));
|
||||
|
||||
DedupeRunProgressVo submitted = service.submitRun(request(1));
|
||||
|
||||
assertNotNull(submitted.getRunId());
|
||||
assertEquals("running", submitted.getStatus());
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -150,7 +150,8 @@ class DedupeRunServiceCleanRulesTest {
|
||||
writeInput(input, rows);
|
||||
|
||||
DedupeRunService service = new DedupeRunService(
|
||||
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService);
|
||||
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService,
|
||||
null, null, new com.nanri.aiimage.config.InstanceMetadata("test-instance"));
|
||||
when(dedupeTotalDataService.findExistingComparableValues(any())).thenReturn(Set.of());
|
||||
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
|
||||
.thenAnswer(invocation -> {
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 开机自回收:本机 owner 名下 RUNNING 且已停止心跳的作业,在进程起来时立刻退回 PENDING。
|
||||
*
|
||||
* <p>发版/崩溃杀掉的 job 会一直挂在 RUNNING,等 60 秒一轮的 stuck 扫描按时间判死才能复位
|
||||
* (阈值历史上是 30 分钟)。也就是说每次发版都会把在跑的结果文件作业冻结掉一整个阈值窗口,
|
||||
* 而这批 job 其实早已无人持有。开机回收把这个等待从「阈值」压到「进程起来那一刻」。
|
||||
*
|
||||
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,
|
||||
* 每发一次版扣掉它 5 次重试中的一次,反复发版会把正常任务判死。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TaskFileJobBootReclaimTest {
|
||||
|
||||
private static final String OWNER = "java-server";
|
||||
|
||||
@Mock private TaskFileJobMapper taskFileJobMapper;
|
||||
@Mock private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
private TaskFileJobService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
TaskFileJobEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity abandonedJob(Long id) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(id);
|
||||
job.setTaskId(1000L + id);
|
||||
job.setResultId(2000L + id);
|
||||
job.setModuleType("APPEARANCE_PATENT");
|
||||
job.setScopeKey("task:" + id + ":owner:" + OWNER);
|
||||
job.setOwner(OWNER);
|
||||
job.setJobType("ASSEMBLE_RESULT");
|
||||
job.setStatus("RUNNING");
|
||||
job.setRetryCount(1);
|
||||
job.setUpdatedAt(LocalDateTime.now().minusMinutes(3));
|
||||
return job;
|
||||
}
|
||||
|
||||
/** 心跳已停的自家 RUNNING 作业退回 PENDING 并补发派发事件,且不得消耗 retry_count。 */
|
||||
@Test
|
||||
void staleOwnedRunningJobsGoBackToPendingWithoutBurningRetry() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
|
||||
|
||||
int reclaimed = service.reclaimOwnedRunningJobs(OWNER, 90, 20);
|
||||
|
||||
assertEquals(2, reclaimed);
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updates =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskFileJobMapper, times(2)).update(isNull(), updates.capture());
|
||||
for (LambdaUpdateWrapper<TaskFileJobEntity> update : updates.getAllValues()) {
|
||||
String sqlSet = update.getSqlSet();
|
||||
assertTrue(sqlSet.contains("status="), "必须把状态改回 PENDING: " + sqlSet);
|
||||
assertFalse(sqlSet.contains("retry_count"), "开机回收不得消耗重试次数: " + sqlSet);
|
||||
}
|
||||
verify(applicationEventPublisher, times(2)).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
|
||||
/** 只回收「心跳已停」的:阈值必须是 now-staleSeconds,否则会抢另一个仍在跑同 owner 的进程的作业。 */
|
||||
@Test
|
||||
void onlyJobsStalerThanTheThresholdAreCandidates() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
LocalDateTime before = LocalDateTime.now();
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 20);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper).selectList(query.capture());
|
||||
String segment = query.getValue().getCustomSqlSegment();
|
||||
assertTrue(segment.contains("owner = "), "必须只回收本机 owner: " + segment);
|
||||
assertTrue(segment.contains("status"), "必须只回收 RUNNING: " + segment);
|
||||
assertTrue(segment.contains("updated_at"), "必须按心跳时间设阈值: " + segment);
|
||||
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
|
||||
.filter(v -> v instanceof LocalDateTime)
|
||||
.map(v -> (LocalDateTime) v)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
assertTrue(cutoff != null && !cutoff.isAfter(before.minusSeconds(90)),
|
||||
"阈值不得晚于 now-90s,否则会抢仍在心跳的作业: " + cutoff);
|
||||
}
|
||||
|
||||
/** 没有 owner 就没有"本机"可言:不查库、不改写,交给通用 stuck 扫描。 */
|
||||
@Test
|
||||
void blankOwnerReclaimsNothing() {
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(" ", 90, 20));
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(null, 90, 20));
|
||||
|
||||
verify(taskFileJobMapper, times(0)).selectList(any(LambdaQueryWrapper.class));
|
||||
verify(taskFileJobMapper, times(0)).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
/** CAS 没命中(已被其它路径复位)时不算回收成功,也不发派发事件。 */
|
||||
@Test
|
||||
void lostCasNeitherCountsNorDispatches() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(abandonedJob(3L)));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, 90, 20));
|
||||
|
||||
verify(applicationEventPublisher, times(0)).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
|
||||
/** 开机一次取太多会长时间持锁:limit 沿用既有夹紧。 */
|
||||
@Test
|
||||
void limitIsClamped() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 0);
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 5000);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper, times(2)).selectList(query.capture());
|
||||
List<String> segments = query.getAllValues().stream()
|
||||
.map(LambdaQueryWrapper::getCustomSqlSegment).toList();
|
||||
assertTrue(segments.get(0).contains("limit 1"), "limit=0 回退到 1: " + segments.get(0));
|
||||
assertTrue(segments.get(1).contains("limit 100"), "超大 limit 钳制到 100: " + segments.get(1));
|
||||
}
|
||||
|
||||
/** staleSeconds 给得太小时取不到任何作业:至少留出一个心跳周期的余量。 */
|
||||
@Test
|
||||
void staleThresholdNeverBelowHeartbeatInterval() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 1, 20);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper).selectList(query.capture());
|
||||
// 参数表要先把 SQL 段渲染出来才落值(MyBatis-Plus 惰性填充),否则读到空 map
|
||||
query.getValue().getCustomSqlSegment();
|
||||
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(v -> v instanceof LocalDateTime)
|
||||
.map(v -> (LocalDateTime) v)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
assertTrue(cutoff != null && cutoff.isBefore(LocalDateTime.now().minusSeconds(30)),
|
||||
"阈值过短会把仍在正常心跳的作业抢走: " + cutoff);
|
||||
}
|
||||
}
|
||||
+57
-1
@@ -30,6 +30,8 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -39,8 +41,11 @@ import java.nio.file.Paths;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -269,8 +274,13 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
Path source = Paths.get("src", "main", "java", "com", "nanri", "aiimage", "modules",
|
||||
"task", "service", "TaskResultFileJobWorker.java");
|
||||
String content = new String(java.nio.file.Files.readAllBytes(source), java.nio.charset.StandardCharsets.UTF_8);
|
||||
assertTrue(content.contains("stuck-timeout-minutes:30"), "@Value 默认卡死超时 30 分钟");
|
||||
assertTrue(content.contains("stuck-timeout-minutes:5"), "@Value 默认卡死超时 5 分钟");
|
||||
assertTrue(content.contains("batch-size:20"), "@Value 默认批次 20");
|
||||
Path yml = Paths.get("src", "main", "resources", "application.yml");
|
||||
String config = new String(Files.readAllBytes(yml), StandardCharsets.UTF_8);
|
||||
// 运行期真正生效的是 yml 的兜底值,只改 @Value 默认等于没改
|
||||
assertTrue(config.contains("STUCK_TIMEOUT_MINUTES:5}"), "application.yml 卡死超时 5 分钟: " + config);
|
||||
assertTrue(config.contains("BOOT_RECLAIM_STALE_SECONDS:90}"), "开机回收阈值可在 env 覆盖");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -288,6 +298,52 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
verify(taskFileJobService).markFailureFinalized(9L, "文件生成任务运行超时,已达到最大重试次数");
|
||||
}
|
||||
|
||||
/** 开机回收必须只针对"本机 instanceId 名下"的遗留作业,否则会抢另一台实例正在跑的 job。 */
|
||||
@Test
|
||||
void bootReclaimTargetsOwnAbandonedJobs() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
|
||||
setInt(worker, "batchSize", 20);
|
||||
setLong(worker, "bootReclaimStaleSeconds", 90L);
|
||||
|
||||
worker.reclaimOwnedJobsOnBoot();
|
||||
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 90L, 20);
|
||||
}
|
||||
|
||||
/** 回收只是加速恢复,DB 抽风时绝不能把应用启动一起带下水。 */
|
||||
@Test
|
||||
void bootReclaimFailureDoesNotBreakStartup() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
|
||||
when(taskFileJobService.reclaimOwnedRunningJobs(anyString(), anyLong(), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
worker.reclaimOwnedJobsOnBoot();
|
||||
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 0L, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* 必须挂在 ApplicationReadyEvent 上:早了(Bean 还没齐)会 NPE,晚了不如挂事件;
|
||||
* 挂在 ContextRefreshed 之类的事件上则会随上下文刷新重复执行。
|
||||
*/
|
||||
@Test
|
||||
void bootReclaimRunsOnApplicationReady() throws Exception {
|
||||
EventListener listener = TaskResultFileJobWorker.class
|
||||
.getMethod("reclaimOwnedJobsOnBoot")
|
||||
.getAnnotation(EventListener.class);
|
||||
assertNotNull(listener, "开机回收必须由事件驱动,否则没人调用");
|
||||
assertTrue(List.of(listener.value()).contains(ApplicationReadyEvent.class),
|
||||
"只在应用就绪后执行: " + List.of(listener.value()));
|
||||
}
|
||||
|
||||
private static void setLong(TaskResultFileJobWorker worker, String fieldName, long value) throws Exception {
|
||||
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.setLong(worker, value);
|
||||
}
|
||||
|
||||
private static void setInt(TaskResultFileJobWorker worker, String fieldName, int value) throws Exception {
|
||||
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
|
||||
Reference in New Issue
Block a user