fix(断点续传): 外观/结果作业/去重三处中断语义补齐,重启不再静默丢行

- 外观专利行标改两态(in-flight 15min / processed 24h):只有 CAS 写回 payload 成功
  才转「已处理」,异常、覆盖率不足、CAS 打满等未落库出口一律撤销标记 —— 修「发 LLM
  请求前就盖章,进程被杀的那一批永不再补检」
- 结果文件作业开机自回收:按 owner_instance_id 只收本机名下 RUNNING 且心跳超 90s 的行;
  卡死阈值 30 → 5 分钟,重启接回从最长半小时压到 5 分钟
- 外观结果表缺检行显式写「未取得检测结果」并判失败,不再留空靠 0.5 覆盖率阈值蒙过
- 去重进度改 NodeSharedStore 跨节点共享,runId 用已落库的 task_no:轮询打到另一节点或
  发版重启后仍能读到真实进度、已完成文件仍可下载;共享态缺失时按 task_no 从库重建;
  逐文件心跳防 StaleTaskRepair 把长任务误判死;开机把本机残留 RUNNING 判失败并给提示
This commit is contained in:
2026-09-27 22:26:09 +08:00
parent ed7dd9381b
commit a58dc8a680
17 changed files with 1662 additions and 84 deletions
@@ -66,7 +66,13 @@ public class DedupeRunController {
}
@GetMapping("/run/{runId}/progress")
@Operation(summary = "查询去重任务进度", description = "轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。")
@Operation(summary = "查询去重任务进度", description = """
轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。
runId 就是提交时落库的任务编号(DEDUPE-xxx),任何实例都能应答:
优先读共享进度快照,快照缺失或落后时按任务编号从任务表与逐文件结果行重建,
因此发版重启或轮询打到另一节点都不会再把已完成的文件报成「任务不存在」。
""")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeRunProgressVo.class)))
})
@@ -4,8 +4,12 @@ import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.NodeSharedStore;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
@@ -18,17 +22,20 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.data.redis.core.StringRedisTemplate;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashSet;
@@ -47,7 +54,6 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
@Slf4j
public class DedupeRunService {
@@ -61,17 +67,28 @@ public class DedupeRunService {
/** 单次请求最多允许的源文件数,避免每个文件形成大量排队对象。 */
private static final int MAX_INPUT_FILES = 200;
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
/** 去重进度快照的共享保留期:任务结束后仍可查(与历史列表衔接),并留出发版重启的余量。 */
private static final Duration RUN_STATE_TTL = Duration.ofHours(3);
/** 开机回收时写回的中断原因:点明是服务重启,用户才知道要重新提交。 */
private static final String BOOT_INTERRUPTED_MESSAGE = "服务重启,去重任务已中断,已完成的文件可在历史记录下载,请重新提交未处理文件";
/** 开机回收一次最多处理多少条本机遗留任务(正常只有个位数)。 */
private static final int MAX_BOOT_RECLAIM_ROWS = 100;
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties;
private final OssStorageService ossStorageService;
private final DedupeTotalDataService dedupeTotalDataService;
private final InstanceMetadata instanceMetadata;
/** runId -> 进度快照(任务结束后保留 1 小时供轮询) */
private final Map<String, DedupeRunProgressVo> runProgressMap = new ConcurrentHashMap<>();
/** runId -> 完成时间戳,用于过期清理 */
private final Map<String, Long> runCompletedAtMap = new ConcurrentHashMap<>();
/**
* runId → 进度快照。本地 Map 作快路径、Redis 作跨节点真源:
* 生产是双实例且 nginx 亲和只覆盖常态,快照只在提交节点内存里时,
* 轮询落到另一节点就会把「其实已经跑完」的任务报成「不存在或已过期」。
*/
private final NodeSharedStore<String, DedupeRunProgressVo> runProgressMap;
/** runId → 完成时间戳,用于过期清理 */
private final NodeSharedStore<String, Long> runCompletedAtMap;
/** userId -> 运行中任务数,限制单用户并发任务 */
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
@@ -80,6 +97,32 @@ public class DedupeRunService {
private final ExecutorService fileExecutor = com.nanri.aiimage.common.util.ThreadPools
.boundedFixed("dedupe-file-worker", MAX_PARALLEL_FILES);
public DedupeRunService(FileTaskMapper fileTaskMapper,
FileResultMapper fileResultMapper,
StorageProperties storageProperties,
OssStorageService ossStorageService,
DedupeTotalDataService dedupeTotalDataService,
StringRedisTemplate stringRedisTemplate,
ObjectMapper objectMapper,
InstanceMetadata instanceMetadata) {
this.fileTaskMapper = fileTaskMapper;
this.fileResultMapper = fileResultMapper;
this.storageProperties = storageProperties;
this.ossStorageService = ossStorageService;
this.dedupeTotalDataService = dedupeTotalDataService;
this.instanceMetadata = instanceMetadata;
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
// 完成时间只用于本节点保留期清理,不参与跨节点读取 → 不节流
this.runCompletedAtMap = new NodeSharedStore<>("aiimage:dedupe:run:completed-at", RUN_STATE_TTL,
Long.class, stringRedisTemplate, objectMapper, 0L);
}
private String currentInstanceId() {
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
return instanceId == null || instanceId.isBlank() ? "unknown-instance" : instanceId;
}
/**
* 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。
*/
@@ -102,7 +145,6 @@ public class DedupeRunService {
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
}
String runId = IdUtil.fastSimpleUUID();
FileTaskEntity task = new FileTaskEntity();
task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType("DEDUPE");
@@ -111,6 +153,8 @@ public class DedupeRunService {
task.setSourceFileCount(request.getFiles().size());
task.setCreatedBy("user:" + request.getUserId());
task.setUserId(request.getUserId());
// 归属实例落库:开机回收只把「本机自己名下」的 RUNNING 判死,双实例下才不会误杀对端在跑的任务
task.setOwnerInstanceId(currentInstanceId());
task.setRequestJson(JSONUtil.toJsonStr(request));
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
@@ -121,6 +165,9 @@ public class DedupeRunService {
decrementRunningCount(request.getUserId());
throw ex;
}
// runId 直接用已落库且唯一的 task_no:内存快照丢失后(发版重启 / 轮询打到另一节点),
// 凭 runId 就能走 uk_task_no 反查到任务行,重建出真实进度而不是报「任务不存在」。
final String runId = task.getTaskNo();
DedupeRunProgressVo progress = new DedupeRunProgressVo();
progress.setRunId(runId);
@@ -130,7 +177,9 @@ public class DedupeRunService {
progress.setSuccessCount(0);
progress.setFailedCount(0);
progress.setFinished(false);
runProgressMap.put(runId, progress);
// 首帧立即共享:客户端拿到 runId 后第一次轮询就可能落到另一节点,
// 那一次如果读不到就会当场报「去重任务不存在或已过期」。
runProgressMap.putNow(runId, snapshot(progress));
try {
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
@@ -146,23 +195,101 @@ public class DedupeRunService {
}
/**
* 查询去重任务进度;runId 不存在或已过期时返回 status=not_found 的占位结果。
* 查询去重任务进度。三级来源,全部落在"说真话"上:
* 1. 共享进度快照(本节点或另一节点写入);
* 2. 快照缺失/落后时按 runId(= task_no)反查 biz_file_task + 逐文件结果行重建;
* 3. 库里也确实没有这条 run 才返回 status=not_found。
*/
public DedupeRunProgressVo getProgress(String runId) {
DedupeRunProgressVo progress = runProgressMap.get(runId);
if (progress == null) {
DedupeRunProgressVo notFound = new DedupeRunProgressVo();
notFound.setRunId(runId);
notFound.setStatus("not_found");
notFound.setTotal(0);
notFound.setProcessedCount(0);
notFound.setSuccessCount(0);
notFound.setFailedCount(0);
notFound.setFinished(false);
notFound.setError("任务不存在或已过期");
return notFound;
if (progress != null) {
if (snapshotIsTerminal(progress)) {
return snapshot(progress);
}
// 快照还说自己 in flight:提交节点可能已经重启,库里早已是终态 —— 以库为准
DedupeRunProgressVo rebuilt = progressFromDatabase(runId);
if (rebuilt != null && rebuilt.isFinished()) {
runProgressMap.remove(runId);
return rebuilt;
}
return snapshot(progress);
}
return snapshot(progress);
DedupeRunProgressVo rebuilt = progressFromDatabase(runId);
if (rebuilt != null) {
return rebuilt;
}
DedupeRunProgressVo notFound = new DedupeRunProgressVo();
notFound.setRunId(runId);
notFound.setStatus("not_found");
notFound.setTotal(0);
notFound.setProcessedCount(0);
notFound.setSuccessCount(0);
notFound.setFailedCount(0);
notFound.setFinished(false);
notFound.setError("任务不存在或已过期");
return notFound;
}
private boolean snapshotIsTerminal(DedupeRunProgressVo progress) {
return progress.isFinished() || "success".equals(progress.getStatus()) || "failed".equals(progress.getStatus());
}
/**
* 用落库状态重建进度:任务行给总数与终态,biz_file_result 的逐文件行给已完成部分。
* 结果行本来就是"每个文件一条"的完成集,所以重启后已上传成功的文件仍能拿到下载地址,
* 用户不必因为进程被杀而重跑整批。
*/
private DedupeRunProgressVo progressFromDatabase(String runId) {
FileTaskEntity task = fileTaskMapper.selectOne(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getTaskNo, runId)
.eq(FileTaskEntity::getModuleType, "DEDUPE"));
if (task == null) {
return null;
}
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, "DEDUPE")
.orderByAsc(FileResultEntity::getId));
int successCount = 0;
int failedCount = 0;
List<DedupeResultItemVo> items = new ArrayList<>(rows.size());
for (FileResultEntity row : rows) {
if (row.getSuccess() != null && row.getSuccess() == 1) {
successCount++;
} else {
failedCount++;
}
items.add(toResultItem(row, task));
}
DedupeRunProgressVo vo = new DedupeRunProgressVo();
vo.setRunId(runId);
vo.setTotal(task.getSourceFileCount() == null ? rows.size() : task.getSourceFileCount());
vo.setSuccessCount(successCount);
vo.setFailedCount(failedCount);
vo.setProcessedCount(successCount + failedCount);
boolean succeeded = "SUCCESS".equals(task.getStatus());
if (succeeded) {
DedupeRunVo result = new DedupeRunVo();
result.setTotal(vo.getTotal());
result.setSuccessCount(successCount);
result.setFailedCount(failedCount);
result.setItems(items);
vo.setResult(result);
}
if ("RUNNING".equals(task.getStatus()) || "PENDING".equals(task.getStatus())
|| "SCHEDULED".equals(task.getStatus())) {
vo.setStatus("running");
vo.setFinished(false);
return vo;
}
vo.setStatus(succeeded ? "success" : "failed");
vo.setFinished(true);
if (!succeeded) {
vo.setError(task.getErrorMessage() == null || task.getErrorMessage().isBlank()
? "去重任务已中断,请重新提交" : task.getErrorMessage());
}
return vo;
}
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
@@ -190,6 +317,10 @@ public class DedupeRunService {
outcomeItems.add(item);
}
}
// 锁外做 IO:进度快照共享出去(500ms 节流)+ 刷任务行心跳,
// 让另一节点能报真实进度、也让 2 小时心跳判死不会误杀长任务
runProgressMap.put(runId, snapshot(progress));
touchRunHeartbeat(task.getId());
}));
}
// 逐个等待:processFile 内部已把单文件失败记入 outcomeItems,不会往外抛;
@@ -241,11 +372,78 @@ public class DedupeRunService {
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
} finally {
runCompletedAtMap.put(runId, System.currentTimeMillis());
// 终态必须绕过节流立刻共享:客户端下一次轮询可能落到另一节点
runProgressMap.putNow(runId, snapshot(progress));
runCompletedAtMap.putNow(runId, System.currentTimeMillis());
decrementRunningCount(request.getUserId());
}
}
/**
* 刷新任务行心跳。StaleTaskRepairService 按「updated_at 超过 2 小时未刷新」把 RUNNING 判死,
* 而去重原本只在收尾时写一次库 —— 一批文件跑过 2 小时就会被误判成失败。
*/
void touchRunHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, taskId)
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
} catch (Exception ex) {
log.warn("dedupe run heartbeat failed taskId={} msg={}", taskId, ex.getMessage());
}
}
/**
* 开机回收本机遗留的去重任务:进程刚起来,自己名下仍处于 RUNNING 的行不可能是活的,
* 只会让前端对着「处理中」空转到轮询超时(最坏 2 小时才被心跳巡检判死)。
* 只按 owner_instance_id 收敛,另一台实例正在跑的任务不受影响。
*/
@EventListener(ApplicationReadyEvent.class)
public void reclaimInterruptedRunsOnBoot() {
try {
List<FileTaskEntity> orphans = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId, FileTaskEntity::getTaskNo)
.eq(FileTaskEntity::getModuleType, "DEDUPE")
.eq(FileTaskEntity::getStatus, "RUNNING")
.eq(FileTaskEntity::getOwnerInstanceId, currentInstanceId())
.orderByAsc(FileTaskEntity::getId)
.last("limit " + MAX_BOOT_RECLAIM_ROWS));
if (orphans.isEmpty()) {
return;
}
List<Long> reclaimedIds = new ArrayList<>();
for (FileTaskEntity orphan : orphans) {
if (orphan == null || orphan.getId() == null) {
continue;
}
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, orphan.getId())
.eq(FileTaskEntity::getStatus, "RUNNING")
.set(FileTaskEntity::getStatus, "FAILED")
.set(FileTaskEntity::getErrorMessage, BOOT_INTERRUPTED_MESSAGE)
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now())
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
if (updated <= 0) {
continue;
}
reclaimedIds.add(orphan.getId());
if (orphan.getTaskNo() != null) {
// 快照还留在 Redis 的话,下一轮会把已中断的任务继续报成"处理中"
runProgressMap.remove(orphan.getTaskNo());
}
}
if (!reclaimedIds.isEmpty()) {
log.warn("dedupe boot reclaim failed interrupted runs owner={} count={} ids={}",
currentInstanceId(), reclaimedIds.size(), reclaimedIds);
}
} catch (Exception ex) {
log.error("dedupe boot reclaim failed owner={} err={}", currentInstanceId(), ex.getMessage(), ex);
}
}
private DedupeRunVo buildRunVo(DedupeRunRequest request, List<DedupeResultItemVo> items,
DedupeRunProgressVo progress) {
DedupeRunVo vo = new DedupeRunVo();
@@ -409,26 +607,28 @@ public class DedupeRunService {
// 任务级信息(状态/开始/结束时间)来自 biz_file_task,按 task_id 批量联查
Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
return entities.stream()
.map(entity -> {
DedupeResultItemVo vo = new DedupeResultItemVo();
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
FileTaskEntity task = entity.getTaskId() == null ? null : taskMap.get(entity.getTaskId());
vo.setTaskId(entity.getTaskId());
vo.setTaskStatus(task == null ? null : task.getStatus());
vo.setCreatedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
})
.map(entity -> toResultItem(entity, entity.getTaskId() == null ? null : taskMap.get(entity.getTaskId())))
.toList();
}
/** 结果行 → 前端条目:历史列表与重启后的进度重建共用同一套口径,避免两处各写一遍。 */
private DedupeResultItemVo toResultItem(FileResultEntity entity, FileTaskEntity task) {
DedupeResultItemVo vo = new DedupeResultItemVo();
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
vo.setTaskId(entity.getTaskId());
vo.setTaskStatus(task == null ? null : task.getStatus());
vo.setCreatedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? entity.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
}
/** 批量加载去重结果关联的任务信息(复用 appearancepatent 的列裁剪写法) */
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
@@ -968,8 +1168,9 @@ public class DedupeRunService {
private void cleanupExpiredProgress() {
long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS;
runCompletedAtMap.forEach((runId, completedAt) -> {
if (completedAt != null && completedAt < cutoff && runCompletedAtMap.remove(runId, completedAt)) {
runCompletedAtMap.localEntriesSnapshot().forEach((runId, completedAt) -> {
if (completedAt != null && completedAt < cutoff) {
runCompletedAtMap.remove(runId);
runProgressMap.remove(runId);
}
});