fix(断点续传): 外观/结果作业/去重三处中断语义补齐,重启不再静默丢行
- 外观专利行标改两态(in-flight 15min / processed 24h):只有 CAS 写回 payload 成功 才转「已处理」,异常、覆盖率不足、CAS 打满等未落库出口一律撤销标记 —— 修「发 LLM 请求前就盖章,进程被杀的那一批永不再补检」 - 结果文件作业开机自回收:按 owner_instance_id 只收本机名下 RUNNING 且心跳超 90s 的行; 卡死阈值 30 → 5 分钟,重启接回从最长半小时压到 5 分钟 - 外观结果表缺检行显式写「未取得检测结果」并判失败,不再留空靠 0.5 覆盖率阈值蒙过 - 去重进度改 NodeSharedStore 跨节点共享,runId 用已落库的 task_no:轮询打到另一节点或 发版重启后仍能读到真实进度、已完成文件仍可下载;共享态缺失时按 task_no 从库重建; 逐文件心跳防 StaleTaskRepair 把长任务误判死;开机把本机残留 RUNNING 判失败并给提示
This commit is contained in:
+77
-14
@@ -19,6 +19,13 @@ public class AppearancePatentTaskCacheService {
|
||||
|
||||
private static final long TTL_HOURS = 24;
|
||||
|
||||
/**
|
||||
* 处理中标记 TTL。取 15 分钟:合批后单批实测约 30–120 秒(合批前 10 行要 220 秒),
|
||||
* 留 7 倍以上余量;同时它必须远小于原来的 24 小时——进程在批中被杀时,这批行只能等这个
|
||||
* TTL 自然过期才能重新成为候选,TTL 有多长,发版/崩溃造成的空缺窗口就有多长。
|
||||
*/
|
||||
private static final Duration IN_FLIGHT_TTL = Duration.ofMinutes(15);
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@@ -133,36 +140,92 @@ public class AppearancePatentTaskCacheService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 行级检测去重:Redis SETNX,防止 submitResult 与 stale 恢复双入口并发处理同一批。
|
||||
* 返回 true 表示首次标记(应执行检测),false 表示已处理过(跳过)。
|
||||
* 申领一行的检测权(两态标记之一:处理中)。返回 true 表示本次拿到,false 表示
|
||||
* 该行已完成或正被另一个入口(submitResult / stale 恢复)处理。
|
||||
*
|
||||
* <p>处理中只活 15 分钟:原实现在发 LLM 请求前就打 24 小时标记且没有撤销路径,
|
||||
* 进程被杀或载荷落库失败时,那批行既没结果又被判定「已处理」,续跑时静默跳过。
|
||||
*/
|
||||
public boolean markRowProcessed(Long taskId, String rowKey) {
|
||||
public boolean beginRowInspection(Long taskId, String rowKey) {
|
||||
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
if (Boolean.TRUE.equals(stringRedisTemplate.hasKey(processedRowKey(taskId, rowKey)))) {
|
||||
return false;
|
||||
}
|
||||
Boolean first = stringRedisTemplate.opsForValue().setIfAbsent(
|
||||
processedRowKey(taskId, rowKey), "1", Duration.ofHours(TTL_HOURS));
|
||||
return Boolean.TRUE.equals(first);
|
||||
inFlightRowKey(taskId, rowKey), "1", IN_FLIGHT_TTL);
|
||||
return first == null || first;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] mark row processed degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
|
||||
log.warn("[appearance-patent-cache] begin row inspection degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
|
||||
// Redis 降级时放行,由持久化层的 hasResolvedLlmFields 判据兜底去重。
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isRowProcessed(Long taskId, String rowKey) {
|
||||
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
|
||||
return false;
|
||||
/** 发请求前续期处理中标记:一批最慢可能跑到分钟级,避免中途被另一个入口抢走重复计费。 */
|
||||
public void touchRowInspection(Long taskId, List<String> rowKeys) {
|
||||
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
return Boolean.TRUE.equals(stringRedisTemplate.hasKey(processedRowKey(taskId, rowKey)));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] check row processed degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
|
||||
return false;
|
||||
for (String rowKey : rowKeys) {
|
||||
if (rowKey == null || rowKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.expire(inFlightRowKey(taskId, rowKey), IN_FLIGHT_TTL);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] touch row inspection degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 结果已落库:打 24 小时完成标记并释放处理中名额。只在 CAS 更新成功后调用。 */
|
||||
public void completeRowInspection(Long taskId, List<String> rowKeys) {
|
||||
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (String rowKey : rowKeys) {
|
||||
if (rowKey == null || rowKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.opsForValue().set(processedRowKey(taskId, rowKey), "1", Duration.ofHours(TTL_HOURS));
|
||||
stringRedisTemplate.delete(inFlightRowKey(taskId, rowKey));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] complete row inspection degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 撤销处理中标记(本批没拿到可用结果时调用):让下一轮立刻能重新申领,不等 15 分钟 TTL。
|
||||
* TTL 只用来兜住「进程在批中被杀」。
|
||||
*/
|
||||
public void releaseRowInspection(Long taskId, List<String> rowKeys) {
|
||||
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (String rowKey : rowKeys) {
|
||||
if (rowKey == null || rowKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.delete(inFlightRowKey(taskId, rowKey));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] release row inspection degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String inFlightRowKey(Long taskId, String rowKey) {
|
||||
return "appearance-patent:task:inflight-rows:" + taskId + ":" + rowKey;
|
||||
}
|
||||
|
||||
private String processedRowsKey(Long taskId) {
|
||||
return "appearance-patent:task:processed-rows:" + taskId;
|
||||
}
|
||||
|
||||
+19
-3
@@ -933,8 +933,11 @@ public class AppearancePatentTaskService {
|
||||
for (int i = 0; i < submitLimit; i += batchSize) {
|
||||
List<AppearancePatentResultRowDto> batchRows = candidates.subList(i, Math.min(i + batchSize, submitLimit));
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
List<String> batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
taskCacheService.touchRowInspection(task.getId(), batchRowKeys);
|
||||
long startedAt = System.currentTimeMillis();
|
||||
List<AppearancePatentResultRowDto> llmRows;
|
||||
boolean batchAnswered = true;
|
||||
try {
|
||||
llmRows = SecretUsageContext.call(task.getUserId(), UserSecretModule.APPEARANCE_PATENT.key(),
|
||||
() -> llmClient.inspectRows(batchRows, prompt, apiKey));
|
||||
@@ -943,8 +946,22 @@ public class AppearancePatentTaskService {
|
||||
log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} err={}",
|
||||
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message);
|
||||
llmRows = llmClient.markRowsFailed(batchRows, message);
|
||||
batchAnswered = false;
|
||||
}
|
||||
boolean rowsCompleted = false;
|
||||
try {
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
|
||||
// 完成标记只能在载荷真正落库之后打——这是「发版重启会不会丢行」的分界线
|
||||
if (batchAnswered) {
|
||||
taskCacheService.completeRowInspection(task.getId(), batchRowKeys);
|
||||
rowsCompleted = true;
|
||||
}
|
||||
} finally {
|
||||
// 本批结束时处理中名额必须让出来:要么已由完成标记接管,要么撤销给下一轮重试
|
||||
if (!rowsCompleted) {
|
||||
taskCacheService.releaseRowInspection(task.getId(), batchRowKeys);
|
||||
}
|
||||
}
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
|
||||
log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}",
|
||||
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal,
|
||||
System.currentTimeMillis() - startedAt);
|
||||
@@ -1007,10 +1024,9 @@ public class AppearancePatentTaskService {
|
||||
if (key.isBlank() || hasResolvedLlmFields(row)) {
|
||||
continue;
|
||||
}
|
||||
if (taskCacheService.isRowProcessed(chunk.getTaskId(), key)) {
|
||||
if (!taskCacheService.beginRowInspection(chunk.getTaskId(), key)) {
|
||||
continue;
|
||||
}
|
||||
taskCacheService.markRowProcessed(chunk.getTaskId(), key);
|
||||
candidates.add(row);
|
||||
}
|
||||
}
|
||||
|
||||
+29
-15
@@ -42,6 +42,17 @@ public final class AppearancePatentSheetBuilder {
|
||||
private AppearancePatentSheetBuilder() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 没拿到判定的行写进格子里的显式文案。
|
||||
*
|
||||
* <p>交付只要求 LLM 覆盖率过半({@code llm-coverage-min-ratio}),所以少数缺失行照样出文件。
|
||||
* 历史上这些格子留空,而"空白"和"没风险"在表格里长得一模一样——用户把漏检当成了安全,
|
||||
* 这正是最贵的那种静默失败。
|
||||
*/
|
||||
public static final String NO_RESULT_TEXT = "未取得检测结果";
|
||||
|
||||
private static final String NO_RESULT_REASON = NO_RESULT_TEXT + ":该行没有返回判定,重新运行任务可补检";
|
||||
|
||||
public static void buildResultSheet(Workbook workbook, List<AppearancePatentParsedRowVo> rows,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
buildResultSheet(workbook, rows, resultMap, null);
|
||||
@@ -87,10 +98,19 @@ public final class AppearancePatentSheetBuilder {
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
|
||||
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
|
||||
String titleRiskCell = resultRow == null
|
||||
? NO_RESULT_TEXT
|
||||
: firstNonBlank(userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()), NO_RESULT_TEXT);
|
||||
String appearanceRiskCell = resultRow == null
|
||||
? NO_RESULT_TEXT
|
||||
: firstNonBlank(userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()), NO_RESULT_TEXT);
|
||||
String conclusionCell = resultRow == null
|
||||
? NO_RESULT_TEXT
|
||||
: firstNonBlank(userFacingConclusion(resultRow), NO_RESULT_TEXT);
|
||||
row.createCell(col++).setCellValue(titleRiskCell);
|
||||
row.createCell(col++).setCellValue(appearanceRiskCell);
|
||||
row.createCell(col++).setCellValue(conclusionCell);
|
||||
row.createCell(col).setCellValue(resolveResultStatus(conclusionCell));
|
||||
}
|
||||
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap, finder);
|
||||
}
|
||||
@@ -121,9 +141,11 @@ public final class AppearancePatentSheetBuilder {
|
||||
AppearancePatentResultRowDto resultRow = finder.apply(parsedRow);
|
||||
Row row = sheet.createRow(rowIndex++);
|
||||
row.createCell(0).setCellValue(asin);
|
||||
row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), ""));
|
||||
row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), ""));
|
||||
row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), ""));
|
||||
// 只有"整行没结果"才代写原因;有结果的行原本就没原因可写(无风险),不能污染成缺失说明
|
||||
String missingReason = resultRow == null ? NO_RESULT_REASON : "";
|
||||
row.createCell(1).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getAppearanceReason(), ""));
|
||||
row.createCell(2).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getPatentReason(), ""));
|
||||
row.createCell(3).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getTitleReason(), ""));
|
||||
rowIndex++;
|
||||
}
|
||||
}
|
||||
@@ -233,14 +255,6 @@ public final class AppearancePatentSheetBuilder {
|
||||
return firstNonBlank(row.getConclusion(), "");
|
||||
}
|
||||
|
||||
public static String userFacingStatus(AppearancePatentResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
String conclusion = userFacingConclusion(row);
|
||||
return resolveResultStatus(conclusion);
|
||||
}
|
||||
|
||||
/**
|
||||
* 状态列口径:商标维度与外观维度都给出有效判定才算「成功」。
|
||||
* 结论列只有在两个维度都成功时才会产出「侵权」/「无侵权」(见 AppearancePatentLlmClient#buildConclusion),
|
||||
|
||||
+7
-1
@@ -66,7 +66,13 @@ public class DedupeRunController {
|
||||
}
|
||||
|
||||
@GetMapping("/run/{runId}/progress")
|
||||
@Operation(summary = "查询去重任务进度", description = "轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。")
|
||||
@Operation(summary = "查询去重任务进度", description = """
|
||||
轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。
|
||||
|
||||
runId 就是提交时落库的任务编号(DEDUPE-xxx),任何实例都能应答:
|
||||
优先读共享进度快照,快照缺失或落后时按任务编号从任务表与逐文件结果行重建,
|
||||
因此发版重启或轮询打到另一节点都不会再把已完成的文件报成「任务不存在」。
|
||||
""")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeRunProgressVo.class)))
|
||||
})
|
||||
|
||||
+242
-41
@@ -4,8 +4,12 @@ import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
|
||||
@@ -18,17 +22,20 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.util.WorkbookUtil;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
@@ -47,7 +54,6 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class DedupeRunService {
|
||||
|
||||
@@ -61,17 +67,28 @@ public class DedupeRunService {
|
||||
/** 单次请求最多允许的源文件数,避免每个文件形成大量排队对象。 */
|
||||
private static final int MAX_INPUT_FILES = 200;
|
||||
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
|
||||
/** 去重进度快照的共享保留期:任务结束后仍可查(与历史列表衔接),并留出发版重启的余量。 */
|
||||
private static final Duration RUN_STATE_TTL = Duration.ofHours(3);
|
||||
/** 开机回收时写回的中断原因:点明是服务重启,用户才知道要重新提交。 */
|
||||
private static final String BOOT_INTERRUPTED_MESSAGE = "服务重启,去重任务已中断,已完成的文件可在历史记录下载,请重新提交未处理文件";
|
||||
/** 开机回收一次最多处理多少条本机遗留任务(正常只有个位数)。 */
|
||||
private static final int MAX_BOOT_RECLAIM_ROWS = 100;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final StorageProperties storageProperties;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final DedupeTotalDataService dedupeTotalDataService;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
/** runId -> 进度快照(任务结束后保留 1 小时供轮询) */
|
||||
private final Map<String, DedupeRunProgressVo> runProgressMap = new ConcurrentHashMap<>();
|
||||
/** runId -> 完成时间戳,用于过期清理 */
|
||||
private final Map<String, Long> runCompletedAtMap = new ConcurrentHashMap<>();
|
||||
/**
|
||||
* runId → 进度快照。本地 Map 作快路径、Redis 作跨节点真源:
|
||||
* 生产是双实例且 nginx 亲和只覆盖常态,快照只在提交节点内存里时,
|
||||
* 轮询落到另一节点就会把「其实已经跑完」的任务报成「不存在或已过期」。
|
||||
*/
|
||||
private final NodeSharedStore<String, DedupeRunProgressVo> runProgressMap;
|
||||
/** runId → 完成时间戳,用于过期清理 */
|
||||
private final NodeSharedStore<String, Long> runCompletedAtMap;
|
||||
/** userId -> 运行中任务数,限制单用户并发任务 */
|
||||
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
|
||||
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
|
||||
@@ -80,6 +97,32 @@ public class DedupeRunService {
|
||||
private final ExecutorService fileExecutor = com.nanri.aiimage.common.util.ThreadPools
|
||||
.boundedFixed("dedupe-file-worker", MAX_PARALLEL_FILES);
|
||||
|
||||
public DedupeRunService(FileTaskMapper fileTaskMapper,
|
||||
FileResultMapper fileResultMapper,
|
||||
StorageProperties storageProperties,
|
||||
OssStorageService ossStorageService,
|
||||
DedupeTotalDataService dedupeTotalDataService,
|
||||
StringRedisTemplate stringRedisTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
InstanceMetadata instanceMetadata) {
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.fileResultMapper = fileResultMapper;
|
||||
this.storageProperties = storageProperties;
|
||||
this.ossStorageService = ossStorageService;
|
||||
this.dedupeTotalDataService = dedupeTotalDataService;
|
||||
this.instanceMetadata = instanceMetadata;
|
||||
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
|
||||
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
|
||||
// 完成时间只用于本节点保留期清理,不参与跨节点读取 → 不节流
|
||||
this.runCompletedAtMap = new NodeSharedStore<>("aiimage:dedupe:run:completed-at", RUN_STATE_TTL,
|
||||
Long.class, stringRedisTemplate, objectMapper, 0L);
|
||||
}
|
||||
|
||||
private String currentInstanceId() {
|
||||
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
|
||||
return instanceId == null || instanceId.isBlank() ? "unknown-instance" : instanceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。
|
||||
*/
|
||||
@@ -102,7 +145,6 @@ public class DedupeRunService {
|
||||
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
|
||||
}
|
||||
|
||||
String runId = IdUtil.fastSimpleUUID();
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType("DEDUPE");
|
||||
@@ -111,6 +153,8 @@ public class DedupeRunService {
|
||||
task.setSourceFileCount(request.getFiles().size());
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
// 归属实例落库:开机回收只把「本机自己名下」的 RUNNING 判死,双实例下才不会误杀对端在跑的任务
|
||||
task.setOwnerInstanceId(currentInstanceId());
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -121,6 +165,9 @@ public class DedupeRunService {
|
||||
decrementRunningCount(request.getUserId());
|
||||
throw ex;
|
||||
}
|
||||
// runId 直接用已落库且唯一的 task_no:内存快照丢失后(发版重启 / 轮询打到另一节点),
|
||||
// 凭 runId 就能走 uk_task_no 反查到任务行,重建出真实进度而不是报「任务不存在」。
|
||||
final String runId = task.getTaskNo();
|
||||
|
||||
DedupeRunProgressVo progress = new DedupeRunProgressVo();
|
||||
progress.setRunId(runId);
|
||||
@@ -130,7 +177,9 @@ public class DedupeRunService {
|
||||
progress.setSuccessCount(0);
|
||||
progress.setFailedCount(0);
|
||||
progress.setFinished(false);
|
||||
runProgressMap.put(runId, progress);
|
||||
// 首帧立即共享:客户端拿到 runId 后第一次轮询就可能落到另一节点,
|
||||
// 那一次如果读不到就会当场报「去重任务不存在或已过期」。
|
||||
runProgressMap.putNow(runId, snapshot(progress));
|
||||
|
||||
try {
|
||||
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
|
||||
@@ -146,23 +195,101 @@ public class DedupeRunService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询去重任务进度;runId 不存在或已过期时返回 status=not_found 的占位结果。
|
||||
* 查询去重任务进度。三级来源,全部落在"说真话"上:
|
||||
* 1. 共享进度快照(本节点或另一节点写入);
|
||||
* 2. 快照缺失/落后时按 runId(= task_no)反查 biz_file_task + 逐文件结果行重建;
|
||||
* 3. 库里也确实没有这条 run 才返回 status=not_found。
|
||||
*/
|
||||
public DedupeRunProgressVo getProgress(String runId) {
|
||||
DedupeRunProgressVo progress = runProgressMap.get(runId);
|
||||
if (progress == null) {
|
||||
DedupeRunProgressVo notFound = new DedupeRunProgressVo();
|
||||
notFound.setRunId(runId);
|
||||
notFound.setStatus("not_found");
|
||||
notFound.setTotal(0);
|
||||
notFound.setProcessedCount(0);
|
||||
notFound.setSuccessCount(0);
|
||||
notFound.setFailedCount(0);
|
||||
notFound.setFinished(false);
|
||||
notFound.setError("任务不存在或已过期");
|
||||
return notFound;
|
||||
if (progress != null) {
|
||||
if (snapshotIsTerminal(progress)) {
|
||||
return snapshot(progress);
|
||||
}
|
||||
// 快照还说自己 in flight:提交节点可能已经重启,库里早已是终态 —— 以库为准
|
||||
DedupeRunProgressVo rebuilt = progressFromDatabase(runId);
|
||||
if (rebuilt != null && rebuilt.isFinished()) {
|
||||
runProgressMap.remove(runId);
|
||||
return rebuilt;
|
||||
}
|
||||
return snapshot(progress);
|
||||
}
|
||||
return snapshot(progress);
|
||||
DedupeRunProgressVo rebuilt = progressFromDatabase(runId);
|
||||
if (rebuilt != null) {
|
||||
return rebuilt;
|
||||
}
|
||||
DedupeRunProgressVo notFound = new DedupeRunProgressVo();
|
||||
notFound.setRunId(runId);
|
||||
notFound.setStatus("not_found");
|
||||
notFound.setTotal(0);
|
||||
notFound.setProcessedCount(0);
|
||||
notFound.setSuccessCount(0);
|
||||
notFound.setFailedCount(0);
|
||||
notFound.setFinished(false);
|
||||
notFound.setError("任务不存在或已过期");
|
||||
return notFound;
|
||||
}
|
||||
|
||||
private boolean snapshotIsTerminal(DedupeRunProgressVo progress) {
|
||||
return progress.isFinished() || "success".equals(progress.getStatus()) || "failed".equals(progress.getStatus());
|
||||
}
|
||||
|
||||
/**
|
||||
* 用落库状态重建进度:任务行给总数与终态,biz_file_result 的逐文件行给已完成部分。
|
||||
* 结果行本来就是"每个文件一条"的完成集,所以重启后已上传成功的文件仍能拿到下载地址,
|
||||
* 用户不必因为进程被杀而重跑整批。
|
||||
*/
|
||||
private DedupeRunProgressVo progressFromDatabase(String runId) {
|
||||
FileTaskEntity task = fileTaskMapper.selectOne(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getTaskNo, runId)
|
||||
.eq(FileTaskEntity::getModuleType, "DEDUPE"));
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, "DEDUPE")
|
||||
.orderByAsc(FileResultEntity::getId));
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
List<DedupeResultItemVo> items = new ArrayList<>(rows.size());
|
||||
for (FileResultEntity row : rows) {
|
||||
if (row.getSuccess() != null && row.getSuccess() == 1) {
|
||||
successCount++;
|
||||
} else {
|
||||
failedCount++;
|
||||
}
|
||||
items.add(toResultItem(row, task));
|
||||
}
|
||||
|
||||
DedupeRunProgressVo vo = new DedupeRunProgressVo();
|
||||
vo.setRunId(runId);
|
||||
vo.setTotal(task.getSourceFileCount() == null ? rows.size() : task.getSourceFileCount());
|
||||
vo.setSuccessCount(successCount);
|
||||
vo.setFailedCount(failedCount);
|
||||
vo.setProcessedCount(successCount + failedCount);
|
||||
boolean succeeded = "SUCCESS".equals(task.getStatus());
|
||||
if (succeeded) {
|
||||
DedupeRunVo result = new DedupeRunVo();
|
||||
result.setTotal(vo.getTotal());
|
||||
result.setSuccessCount(successCount);
|
||||
result.setFailedCount(failedCount);
|
||||
result.setItems(items);
|
||||
vo.setResult(result);
|
||||
}
|
||||
if ("RUNNING".equals(task.getStatus()) || "PENDING".equals(task.getStatus())
|
||||
|| "SCHEDULED".equals(task.getStatus())) {
|
||||
vo.setStatus("running");
|
||||
vo.setFinished(false);
|
||||
return vo;
|
||||
}
|
||||
vo.setStatus(succeeded ? "success" : "failed");
|
||||
vo.setFinished(true);
|
||||
if (!succeeded) {
|
||||
vo.setError(task.getErrorMessage() == null || task.getErrorMessage().isBlank()
|
||||
? "去重任务已中断,请重新提交" : task.getErrorMessage());
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
|
||||
@@ -190,6 +317,10 @@ public class DedupeRunService {
|
||||
outcomeItems.add(item);
|
||||
}
|
||||
}
|
||||
// 锁外做 IO:进度快照共享出去(500ms 节流)+ 刷任务行心跳,
|
||||
// 让另一节点能报真实进度、也让 2 小时心跳判死不会误杀长任务
|
||||
runProgressMap.put(runId, snapshot(progress));
|
||||
touchRunHeartbeat(task.getId());
|
||||
}));
|
||||
}
|
||||
// 逐个等待:processFile 内部已把单文件失败记入 outcomeItems,不会往外抛;
|
||||
@@ -241,11 +372,78 @@ public class DedupeRunService {
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
} finally {
|
||||
runCompletedAtMap.put(runId, System.currentTimeMillis());
|
||||
// 终态必须绕过节流立刻共享:客户端下一次轮询可能落到另一节点
|
||||
runProgressMap.putNow(runId, snapshot(progress));
|
||||
runCompletedAtMap.putNow(runId, System.currentTimeMillis());
|
||||
decrementRunningCount(request.getUserId());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新任务行心跳。StaleTaskRepairService 按「updated_at 超过 2 小时未刷新」把 RUNNING 判死,
|
||||
* 而去重原本只在收尾时写一次库 —— 一批文件跑过 2 小时就会被误判成失败。
|
||||
*/
|
||||
void touchRunHeartbeat(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, taskId)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
} catch (Exception ex) {
|
||||
log.warn("dedupe run heartbeat failed taskId={} msg={}", taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 开机回收本机遗留的去重任务:进程刚起来,自己名下仍处于 RUNNING 的行不可能是活的,
|
||||
* 只会让前端对着「处理中」空转到轮询超时(最坏 2 小时才被心跳巡检判死)。
|
||||
* 只按 owner_instance_id 收敛,另一台实例正在跑的任务不受影响。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void reclaimInterruptedRunsOnBoot() {
|
||||
try {
|
||||
List<FileTaskEntity> orphans = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId, FileTaskEntity::getTaskNo)
|
||||
.eq(FileTaskEntity::getModuleType, "DEDUPE")
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.eq(FileTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||
.orderByAsc(FileTaskEntity::getId)
|
||||
.last("limit " + MAX_BOOT_RECLAIM_ROWS));
|
||||
if (orphans.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<Long> reclaimedIds = new ArrayList<>();
|
||||
for (FileTaskEntity orphan : orphans) {
|
||||
if (orphan == null || orphan.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, orphan.getId())
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, BOOT_INTERRUPTED_MESSAGE)
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated <= 0) {
|
||||
continue;
|
||||
}
|
||||
reclaimedIds.add(orphan.getId());
|
||||
if (orphan.getTaskNo() != null) {
|
||||
// 快照还留在 Redis 的话,下一轮会把已中断的任务继续报成"处理中"
|
||||
runProgressMap.remove(orphan.getTaskNo());
|
||||
}
|
||||
}
|
||||
if (!reclaimedIds.isEmpty()) {
|
||||
log.warn("dedupe boot reclaim failed interrupted runs owner={} count={} ids={}",
|
||||
currentInstanceId(), reclaimedIds.size(), reclaimedIds);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.error("dedupe boot reclaim failed owner={} err={}", currentInstanceId(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DedupeRunVo buildRunVo(DedupeRunRequest request, List<DedupeResultItemVo> items,
|
||||
DedupeRunProgressVo progress) {
|
||||
DedupeRunVo vo = new DedupeRunVo();
|
||||
@@ -409,26 +607,28 @@ public class DedupeRunService {
|
||||
// 任务级信息(状态/开始/结束时间)来自 biz_file_task,按 task_id 批量联查
|
||||
Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
|
||||
return entities.stream()
|
||||
.map(entity -> {
|
||||
DedupeResultItemVo vo = new DedupeResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getSourceFilename());
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||
vo.setError(entity.getErrorMessage());
|
||||
FileTaskEntity task = entity.getTaskId() == null ? null : taskMap.get(entity.getTaskId());
|
||||
vo.setTaskId(entity.getTaskId());
|
||||
vo.setTaskStatus(task == null ? null : task.getStatus());
|
||||
vo.setCreatedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||
vo.setStartedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||
return vo;
|
||||
})
|
||||
.map(entity -> toResultItem(entity, entity.getTaskId() == null ? null : taskMap.get(entity.getTaskId())))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/** 结果行 → 前端条目:历史列表与重启后的进度重建共用同一套口径,避免两处各写一遍。 */
|
||||
private DedupeResultItemVo toResultItem(FileResultEntity entity, FileTaskEntity task) {
|
||||
DedupeResultItemVo vo = new DedupeResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getSourceFilename());
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||
vo.setError(entity.getErrorMessage());
|
||||
vo.setTaskId(entity.getTaskId());
|
||||
vo.setTaskStatus(task == null ? null : task.getStatus());
|
||||
vo.setCreatedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||
vo.setStartedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
|
||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 批量加载去重结果关联的任务信息(复用 appearancepatent 的列裁剪写法) */
|
||||
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
@@ -968,8 +1168,9 @@ public class DedupeRunService {
|
||||
|
||||
private void cleanupExpiredProgress() {
|
||||
long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS;
|
||||
runCompletedAtMap.forEach((runId, completedAt) -> {
|
||||
if (completedAt != null && completedAt < cutoff && runCompletedAtMap.remove(runId, completedAt)) {
|
||||
runCompletedAtMap.localEntriesSnapshot().forEach((runId, completedAt) -> {
|
||||
if (completedAt != null && completedAt < cutoff) {
|
||||
runCompletedAtMap.remove(runId);
|
||||
runProgressMap.remove(runId);
|
||||
}
|
||||
});
|
||||
|
||||
+62
@@ -428,6 +428,68 @@ public class TaskFileJobService {
|
||||
.collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a));
|
||||
}
|
||||
|
||||
/**
|
||||
* 开机自回收:把本机 owner 名下「RUNNING 且心跳已停」的作业退回 PENDING 并补发派发事件。
|
||||
*
|
||||
* <p>进程被杀(发版、崩溃、容器重建)时这些行不会自己退出 RUNNING,只能等 60 秒一轮的
|
||||
* stuck 扫描按时间阈值判死——阈值历史上是 30 分钟,等于每次发版都把在跑的结果文件作业
|
||||
* 冻结一整个阈值窗口,而这批行其实早已无人持有。进程刚起来时本机不可能还在跑它们,
|
||||
* 所以这一刻就能直接接回来,把等待从「阈值」压到「启动」。
|
||||
*
|
||||
* <p>两道保险:只取 owner 等值等于自己的行(绝不碰另一台实例在跑的),且必须已停止心跳
|
||||
* 一个完整周期以上(正常心跳是 60 秒一次,误抢会让在跑的作业被双份派发)。
|
||||
*
|
||||
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,每发一次版
|
||||
* 吃掉它那 5 次重试中的一次,反复发版会把正常任务判成"已达到最大重试次数"。
|
||||
*
|
||||
* <p>刻意不加事务,与 {@link #resetStuckRunningJobsDetailed} 同理:每行都带
|
||||
* status + updated_at 的 CAS 条件,逐行独立提交,锁即时释放。
|
||||
*/
|
||||
public int reclaimOwnedRunningJobs(String owner, long staleSeconds, int limit) {
|
||||
String normalizedOwner = owner == null ? "" : owner.trim();
|
||||
if (normalizedOwner.isBlank()) {
|
||||
return 0;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusSeconds(Math.max(60L, staleSeconds));
|
||||
int safeLimit = Math.max(1, Math.min(limit, 100));
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(TaskFileJobEntity::getOwner, normalizedOwner)
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, cutoff)
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
if (jobs.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
List<Long> resetIds = new ArrayList<>();
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
if (job == null || job.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "实例重启,作业已重新排队")
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
resetIds.add(job.getId());
|
||||
}
|
||||
}
|
||||
if (resetIds.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
Map<Long, TaskFileJobEntity> refreshedById = refreshJobsByIds(resetIds);
|
||||
for (Long jobId : resetIds) {
|
||||
publishDispatchEvent(refreshedById.get(jobId));
|
||||
}
|
||||
log.warn("[task-file-job] 开机自回收本机遗留作业 count={} owner={} staleSeconds={} limit={}",
|
||||
resetIds.size(), normalizedOwner, staleSeconds, safeLimit);
|
||||
return resetIds.size();
|
||||
}
|
||||
|
||||
public record StuckJobResetResult(int resetCount, List<TaskFileJobEntity> exhaustedJobs) {
|
||||
}
|
||||
|
||||
|
||||
+39
-1
@@ -12,6 +12,8 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -43,12 +45,22 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.batch-size:20}")
|
||||
private int batchSize;
|
||||
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
|
||||
/**
|
||||
* 卡死判定阈值。作业在跑期间每 60 秒 touch 一次 {@code updated_at},所以"超过 5 分钟没心跳"
|
||||
* 就等于持有者已经没了;取 30 分钟时一次实例被销毁重建(owner 变了、开机回收够不着)
|
||||
* 会把结果文件作业冻结半小时。队列积压导致的"认领后迟迟没执行"不会误伤:
|
||||
* 真正开跑前的 {@code activateRunningClaim} 用 updated_at 做 CAS,被复位的作业会直接放弃执行。
|
||||
*/
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:5}")
|
||||
private int stuckTimeoutMinutes;
|
||||
|
||||
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
|
||||
private long heartbeatIntervalMillis = 60000L;
|
||||
|
||||
/** 开机回收的"心跳已停"判据:默认 90 秒 ≈ 1.5 个心跳周期,容得下一次正常心跳延迟。 */
|
||||
@Value("${aiimage.result-file-job.boot-reclaim-stale-seconds:90}")
|
||||
private long bootReclaimStaleSeconds;
|
||||
|
||||
/** 多个结果文件 Job 并行运行时,心跳不能被单个 DB 慢调用串行阻塞。 */
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
|
||||
@@ -99,6 +111,32 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用就绪即回收本机遗留作业。
|
||||
*
|
||||
* <p>发版杀进程时正在跑的 job 会一直挂在 RUNNING,要等卡死扫描按 stuck-timeout-minutes
|
||||
* 判死才重新排队——那段时间里用户看到的是"任务卡住不动",而实际上没有任何进程在跑它。
|
||||
* 本机刚起来,这些 owner=自己的 RUNNING 行不可能还在本机执行,所以此刻就能直接接回来,
|
||||
* 不必等阈值;回收不消耗 retry_count(发版不是作业的错)。
|
||||
*
|
||||
* <p>只处理 owner 等于本机实例 id 的行,另一台实例在跑的作业不受影响。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void reclaimOwnedJobsOnBoot() {
|
||||
try {
|
||||
int reclaimed = taskFileJobService.reclaimOwnedRunningJobs(
|
||||
currentInstanceId(), bootReclaimStaleSeconds, batchSize);
|
||||
if (reclaimed > 0) {
|
||||
log.warn("[task-file-job] booted and reclaimed abandoned jobs of this instance reclaimed={} instanceId={} staleSeconds={}",
|
||||
reclaimed, currentInstanceId(), bootReclaimStaleSeconds);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// 回收只是加速恢复;失败不影响启动,兜底仍是 stuck 扫描。
|
||||
log.error("[task-file-job] boot reclaim failed, falling back to stuck scan instanceId={} timeoutMinutes={} err={}",
|
||||
currentInstanceId(), stuckTimeoutMinutes, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void process(TaskFileJobEntity job) {
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user