fix(断点续传): 外观/结果作业/去重三处中断语义补齐,重启不再静默丢行

- 外观专利行标改两态(in-flight 15min / processed 24h):只有 CAS 写回 payload 成功
  才转「已处理」,异常、覆盖率不足、CAS 打满等未落库出口一律撤销标记 —— 修「发 LLM
  请求前就盖章,进程被杀的那一批永不再补检」
- 结果文件作业开机自回收:按 owner_instance_id 只收本机名下 RUNNING 且心跳超 90s 的行;
  卡死阈值 30 → 5 分钟,重启接回从最长半小时压到 5 分钟
- 外观结果表缺检行显式写「未取得检测结果」并判失败,不再留空靠 0.5 覆盖率阈值蒙过
- 去重进度改 NodeSharedStore 跨节点共享,runId 用已落库的 task_no:轮询打到另一节点或
  发版重启后仍能读到真实进度、已完成文件仍可下载;共享态缺失时按 task_no 从库重建;
  逐文件心跳防 StaleTaskRepair 把长任务误判死;开机把本机残留 RUNNING 判失败并给提示
This commit is contained in:
2026-09-27 22:26:09 +08:00
parent ed7dd9381b
commit a58dc8a680
17 changed files with 1662 additions and 84 deletions
@@ -19,6 +19,13 @@ public class AppearancePatentTaskCacheService {
private static final long TTL_HOURS = 24;
/**
* 处理中标记 TTL。取 15 分钟:合批后单批实测约 30–120 秒(合批前 10 行要 220 秒),
* 留 7 倍以上余量;同时它必须远小于原来的 24 小时——进程在批中被杀时,这批行只能等这个
* TTL 自然过期才能重新成为候选,TTL 有多长,发版/崩溃造成的空缺窗口就有多长。
*/
private static final Duration IN_FLIGHT_TTL = Duration.ofMinutes(15);
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
@@ -133,36 +140,92 @@ public class AppearancePatentTaskCacheService {
}
/**
* 行级检测去重:Redis SETNX,防止 submitResult 与 stale 恢复双入口并发处理同一批。
* 返回 true 表示首次标记(应执行检测),false 表示已处理过(跳过)。
* 申领一行的检测权(两态标记之一:处理中)。返回 true 表示本次拿到,false 表示
* 该行已完成或正被另一个入口(submitResult / stale 恢复)处理。
*
* <p>处理中只活 15 分钟:原实现在发 LLM 请求前就打 24 小时标记且没有撤销路径,
* 进程被杀或载荷落库失败时,那批行既没结果又被判定「已处理」,续跑时静默跳过。
*/
public boolean markRowProcessed(Long taskId, String rowKey) {
public boolean beginRowInspection(Long taskId, String rowKey) {
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
return false;
}
try {
if (Boolean.TRUE.equals(stringRedisTemplate.hasKey(processedRowKey(taskId, rowKey)))) {
return false;
}
Boolean first = stringRedisTemplate.opsForValue().setIfAbsent(
processedRowKey(taskId, rowKey), "1", Duration.ofHours(TTL_HOURS));
return Boolean.TRUE.equals(first);
inFlightRowKey(taskId, rowKey), "1", IN_FLIGHT_TTL);
return first == null || first;
} catch (Exception ex) {
log.warn("[appearance-patent-cache] mark row processed degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
log.warn("[appearance-patent-cache] begin row inspection degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
// Redis 降级时放行,由持久化层的 hasResolvedLlmFields 判据兜底去重。
return true;
}
}
public boolean isRowProcessed(Long taskId, String rowKey) {
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
return false;
/** 发请求前续期处理中标记:一批最慢可能跑到分钟级,避免中途被另一个入口抢走重复计费。 */
public void touchRowInspection(Long taskId, List<String> rowKeys) {
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
return;
}
try {
return Boolean.TRUE.equals(stringRedisTemplate.hasKey(processedRowKey(taskId, rowKey)));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] check row processed degraded taskId={} rowKey={} msg={}", taskId, rowKey, ex.getMessage());
return false;
for (String rowKey : rowKeys) {
if (rowKey == null || rowKey.isBlank()) {
continue;
}
try {
stringRedisTemplate.expire(inFlightRowKey(taskId, rowKey), IN_FLIGHT_TTL);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] touch row inspection degraded taskId={} rowKey={} msg={}",
taskId, rowKey, ex.getMessage());
}
}
}
/** 结果已落库:打 24 小时完成标记并释放处理中名额。只在 CAS 更新成功后调用。 */
public void completeRowInspection(Long taskId, List<String> rowKeys) {
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
return;
}
for (String rowKey : rowKeys) {
if (rowKey == null || rowKey.isBlank()) {
continue;
}
try {
stringRedisTemplate.opsForValue().set(processedRowKey(taskId, rowKey), "1", Duration.ofHours(TTL_HOURS));
stringRedisTemplate.delete(inFlightRowKey(taskId, rowKey));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] complete row inspection degraded taskId={} rowKey={} msg={}",
taskId, rowKey, ex.getMessage());
}
}
}
/**
* 撤销处理中标记(本批没拿到可用结果时调用):让下一轮立刻能重新申领,不等 15 分钟 TTL。
* TTL 只用来兜住「进程在批中被杀」。
*/
public void releaseRowInspection(Long taskId, List<String> rowKeys) {
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
return;
}
for (String rowKey : rowKeys) {
if (rowKey == null || rowKey.isBlank()) {
continue;
}
try {
stringRedisTemplate.delete(inFlightRowKey(taskId, rowKey));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] release row inspection degraded taskId={} rowKey={} msg={}",
taskId, rowKey, ex.getMessage());
}
}
}
private String inFlightRowKey(Long taskId, String rowKey) {
return "appearance-patent:task:inflight-rows:" + taskId + ":" + rowKey;
}
private String processedRowsKey(Long taskId) {
return "appearance-patent:task:processed-rows:" + taskId;
}
@@ -933,8 +933,11 @@ public class AppearancePatentTaskService {
for (int i = 0; i < submitLimit; i += batchSize) {
List<AppearancePatentResultRowDto> batchRows = candidates.subList(i, Math.min(i + batchSize, submitLimit));
taskFileJobService.touchRunning(job.getId());
List<String> batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
taskCacheService.touchRowInspection(task.getId(), batchRowKeys);
long startedAt = System.currentTimeMillis();
List<AppearancePatentResultRowDto> llmRows;
boolean batchAnswered = true;
try {
llmRows = SecretUsageContext.call(task.getUserId(), UserSecretModule.APPEARANCE_PATENT.key(),
() -> llmClient.inspectRows(batchRows, prompt, apiKey));
@@ -943,8 +946,22 @@ public class AppearancePatentTaskService {
log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} err={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message);
llmRows = llmClient.markRowsFailed(batchRows, message);
batchAnswered = false;
}
boolean rowsCompleted = false;
try {
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
// 完成标记只能在载荷真正落库之后打——这是「发版重启会不会丢行」的分界线
if (batchAnswered) {
taskCacheService.completeRowInspection(task.getId(), batchRowKeys);
rowsCompleted = true;
}
} finally {
// 本批结束时处理中名额必须让出来:要么已由完成标记接管,要么撤销给下一轮重试
if (!rowsCompleted) {
taskCacheService.releaseRowInspection(task.getId(), batchRowKeys);
}
}
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal,
System.currentTimeMillis() - startedAt);
@@ -1007,10 +1024,9 @@ public class AppearancePatentTaskService {
if (key.isBlank() || hasResolvedLlmFields(row)) {
continue;
}
if (taskCacheService.isRowProcessed(chunk.getTaskId(), key)) {
if (!taskCacheService.beginRowInspection(chunk.getTaskId(), key)) {
continue;
}
taskCacheService.markRowProcessed(chunk.getTaskId(), key);
candidates.add(row);
}
}
@@ -42,6 +42,17 @@ public final class AppearancePatentSheetBuilder {
private AppearancePatentSheetBuilder() {
}
/**
* 没拿到判定的行写进格子里的显式文案。
*
* <p>交付只要求 LLM 覆盖率过半({@code llm-coverage-min-ratio}),所以少数缺失行照样出文件。
* 历史上这些格子留空,而"空白"和"没风险"在表格里长得一模一样——用户把漏检当成了安全,
* 这正是最贵的那种静默失败。
*/
public static final String NO_RESULT_TEXT = "未取得检测结果";
private static final String NO_RESULT_REASON = NO_RESULT_TEXT + ":该行没有返回判定,重新运行任务可补检";
public static void buildResultSheet(Workbook workbook, List<AppearancePatentParsedRowVo> rows,
Map<String, AppearancePatentResultRowDto> resultMap) {
buildResultSheet(workbook, rows, resultMap, null);
@@ -87,10 +98,19 @@ public final class AppearancePatentSheetBuilder {
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
String titleRiskCell = resultRow == null
? NO_RESULT_TEXT
: firstNonBlank(userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()), NO_RESULT_TEXT);
String appearanceRiskCell = resultRow == null
? NO_RESULT_TEXT
: firstNonBlank(userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()), NO_RESULT_TEXT);
String conclusionCell = resultRow == null
? NO_RESULT_TEXT
: firstNonBlank(userFacingConclusion(resultRow), NO_RESULT_TEXT);
row.createCell(col++).setCellValue(titleRiskCell);
row.createCell(col++).setCellValue(appearanceRiskCell);
row.createCell(col++).setCellValue(conclusionCell);
row.createCell(col).setCellValue(resolveResultStatus(conclusionCell));
}
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap, finder);
}
@@ -121,9 +141,11 @@ public final class AppearancePatentSheetBuilder {
AppearancePatentResultRowDto resultRow = finder.apply(parsedRow);
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(asin);
row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), ""));
row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), ""));
row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), ""));
// 只有"整行没结果"才代写原因;有结果的行原本就没原因可写(无风险),不能污染成缺失说明
String missingReason = resultRow == null ? NO_RESULT_REASON : "";
row.createCell(1).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getAppearanceReason(), ""));
row.createCell(2).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getPatentReason(), ""));
row.createCell(3).setCellValue(resultRow == null ? missingReason : firstNonBlank(resultRow.getTitleReason(), ""));
rowIndex++;
}
}
@@ -233,14 +255,6 @@ public final class AppearancePatentSheetBuilder {
return firstNonBlank(row.getConclusion(), "");
}
public static String userFacingStatus(AppearancePatentResultRowDto row) {
if (row == null) {
return "";
}
String conclusion = userFacingConclusion(row);
return resolveResultStatus(conclusion);
}
/**
* 状态列口径:商标维度与外观维度都给出有效判定才算「成功」。
* 结论列只有在两个维度都成功时才会产出「侵权」/「无侵权」(见 AppearancePatentLlmClient#buildConclusion),