fix(外观专利): 完成标记只打在真正落库的行上

网关少回一行、或回来的 rowToken 在分片里匹配不到时,整批结果落库不完整,
但调用方仍把全部发送行标成已检测:没落库的那行带着"已检测"的空壳 24 小时内
不再补检,用户扣了次数却在结果里永久没有结论。合并写入改为回报真正写进去的
行键,只对这些行打完成标记,其余撤销交下一轮重发(受轮次上限约束)。
This commit is contained in:
2026-09-28 03:09:49 +08:00
parent eaafd5022a
commit 10dec56e8e
3 changed files with 82 additions and 32 deletions
@@ -950,18 +950,27 @@ public class AppearancePatentTaskService {
llmRows = llmClient.markRowsFailed(batchRows, message); llmRows = llmClient.markRowsFailed(batchRows, message);
batchAnswered = false; batchAnswered = false;
} }
boolean rowsCompleted = false; List<String> keysToRetry = new ArrayList<>(batchRowKeys);
try { try {
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows); Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
// 完成标记只能在载荷真正落库之后打——这是「发版重启会不会丢行」的分界线 // 完成标记只能打在真正落库的行上——这是「发版重启会不会丢行」的分界线
if (batchAnswered) { if (batchAnswered) {
taskCacheService.completeRowInspection(task.getId(), batchRowKeys); List<String> landed = batchRowKeys.stream().filter(landedKeys::contains).toList();
rowsCompleted = true; if (!landed.isEmpty()) {
taskCacheService.completeRowInspection(task.getId(), landed);
}
keysToRetry.removeAll(landed);
} }
} finally { } finally {
// 本批结束时处理中名额必须让出来:要么已由完成标记接管,要么撤销给下一轮重试 // 本批结束时处理中名额必须让出来:没落库的行撤销后下一轮马上能重新申领,不等 TTL
if (!rowsCompleted) { if (!keysToRetry.isEmpty()) {
taskCacheService.releaseRowInspection(task.getId(), batchRowKeys); taskCacheService.releaseRowInspection(task.getId(), keysToRetry);
if (batchAnswered) {
// 结果拿到了却落不进分片(少回、rowToken 被改写、分片已清):
// 打成已完成就是"扣了次数永久没结论",宁可重发一轮。
log.warn("[appearance-patent] 本批结果未全部落库,撤销已检测标记交由下一轮重发 taskId={} jobId={} batch={}/{} sent={} retrying={}",
task.getId(), job.getId(), batchIndex, batchTotal, batchRowKeys.size(), keysToRetry);
}
} }
} }
log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}", log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}",
@@ -1018,12 +1027,12 @@ public class AppearancePatentTaskService {
return retryable; return retryable;
} }
private void mergeLlmRowsIntoChunks(FileTaskEntity task, private Set<String> mergeLlmRowsIntoChunks(FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId, Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
List<AppearancePatentResultRowDto> llmRows, List<AppearancePatentResultRowDto> llmRows,
List<AppearancePatentResultRowDto> batchRows) { List<AppearancePatentResultRowDto> batchRows) {
if (llmRows == null || llmRows.isEmpty()) { if (llmRows == null || llmRows.isEmpty()) {
return; return Set.of();
} }
Map<String, AppearancePatentResultRowDto> deduped = new LinkedHashMap<>(); Map<String, AppearancePatentResultRowDto> deduped = new LinkedHashMap<>();
for (int i = 0; i < llmRows.size(); i++) { for (int i = 0; i < llmRows.size(); i++) {
@@ -1039,7 +1048,7 @@ public class AppearancePatentTaskService {
deduped.put(key, row); deduped.put(key, row);
} }
} }
mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId); return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId);
} }
/** /**
@@ -1361,9 +1370,11 @@ public class AppearancePatentTaskService {
} }
private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List<AppearancePatentResultRowDto> rows) { /** 返回真正写进这一片载荷的行键:一片也没写成时返回空集,调用方据此决定哪些行还能重发。 */
private List<String> mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex,
List<AppearancePatentResultRowDto> rows) {
if (rows == null || rows.isEmpty()) { if (rows == null || rows.isEmpty()) {
return; return List.of();
} }
String conflictDetail = "未发生冲突"; String conflictDetail = "未发生冲突";
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) { for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
@@ -1374,7 +1385,7 @@ public class AppearancePatentTaskService {
.eq(TaskChunkEntity::getChunkIndex, chunkIndex) .eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1")); .last("limit 1"));
if (chunk == null) { if (chunk == null) {
return; return List.of();
} }
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk); Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
for (AppearancePatentResultRowDto row : rows) { for (AppearancePatentResultRowDto row : rows) {
@@ -1405,7 +1416,7 @@ public class AppearancePatentTaskService {
log.debug("[appearance-patent] chunk payload replaced taskId={} scopeHash={} chunk={} oldPayload={} newPayload={} attempt={}", log.debug("[appearance-patent] chunk payload replaced taskId={} scopeHash={} chunk={} oldPayload={} newPayload={} attempt={}",
taskId, scopeHash, chunkIndex, oldPayload, storedPayload, attempt); taskId, scopeHash, chunkIndex, oldPayload, storedPayload, attempt);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload); transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
return; return rows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
} }
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位) // CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位)
String currentHash = currentPayloadHash(chunk.getId()); String currentHash = currentPayloadHash(chunk.getId());
@@ -1859,23 +1870,24 @@ public class AppearancePatentTaskService {
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成"); saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成");
} }
private void mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task, private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows, List<AppearancePatentResultRowDto> llmRows,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) { Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
mergeLlmRowsIntoSubmittedChunks(task, llmRows, allRowsByBaseId, null, null); return mergeLlmRowsIntoSubmittedChunks(task, llmRows, allRowsByBaseId, null, null);
} }
private void mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task, /** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
List<AppearancePatentResultRowDto> llmRows, private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId, List<AppearancePatentResultRowDto> llmRows,
String fallbackScopeHash, Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
Integer fallbackChunkIndex) { String fallbackScopeHash,
Integer fallbackChunkIndex) {
if (task == null || llmRows == null || llmRows.isEmpty()) { if (task == null || llmRows == null || llmRows.isEmpty()) {
return; return Set.of();
} }
List<TaskChunkEntity> chunks = loadSubmittedChunks(task.getId()); List<TaskChunkEntity> chunks = loadSubmittedChunks(task.getId());
if (chunks.isEmpty()) { if (chunks.isEmpty()) {
return; return Set.of();
} }
Map<String, Map<String, AppearancePatentResultRowDto>> rowsByChunk = new LinkedHashMap<>(); Map<String, Map<String, AppearancePatentResultRowDto>> rowsByChunk = new LinkedHashMap<>();
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>(); Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
@@ -1913,13 +1925,16 @@ public class AppearancePatentTaskService {
} }
} }
} }
Set<String> landedKeys = new LinkedHashSet<>();
for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : mergeRowsByChunk.entrySet()) { for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : mergeRowsByChunk.entrySet()) {
TaskChunkEntity chunk = chunkByKey.get(entry.getKey()); TaskChunkEntity chunk = chunkByKey.get(entry.getKey());
if (chunk == null || entry.getValue().isEmpty()) { if (chunk == null || entry.getValue().isEmpty()) {
continue; continue;
} }
mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), new ArrayList<>(entry.getValue().values())); landedKeys.addAll(mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(),
new ArrayList<>(entry.getValue().values())));
} }
return landedKeys;
} }
private String chunkStorageKey(String scopeHash, Integer chunkIndex) { private String chunkStorageKey(String scopeHash, Integer chunkIndex) {
@@ -79,6 +79,9 @@ class AppearancePatentRowMarkTimingTest {
private static final String RESOLVED_ROWS = private static final String RESOLVED_ROWS =
"[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"}," "[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"},"
+ "{\"rowToken\":\"r2\",\"asin\":\"B0A0000002\",\"title\":\"COTTON BAG\",\"url\":\"http://img/2.jpg\",\"groupKey\":\"g2\",\"titleRisk\":\"侵权\",\"appearanceRisk\":\"侵权\",\"conclusion\":\"侵权\"}]"; + "{\"rowToken\":\"r2\",\"asin\":\"B0A0000002\",\"title\":\"COTTON BAG\",\"url\":\"http://img/2.jpg\",\"groupKey\":\"g2\",\"titleRisk\":\"侵权\",\"appearanceRisk\":\"侵权\",\"conclusion\":\"侵权\"}]";
/** 网关的真实故障形态之一:发 2 行只回 1 行。 */
private static final String PARTIAL_RESOLVED_ROWS =
"[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"}]";
@Mock private LocalFileStorageService localFileStorageService; @Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService; @Mock private OssStorageService ossStorageService;
@@ -211,6 +214,37 @@ class AppearancePatentRowMarkTimingTest {
verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r1", "r2")); verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r1", "r2"));
} }
/**
* 发 2 行只回 1 行:落库成功的那行算完成,没回来的那行必须撤销交下一轮重发。
*
* <p>此前整批一起打完成标记,少回的行就带着"已检测"的空壳永久消失——
* 次数扣了、Excel 里那一行没有结论,正是"扣次数没出结果"的成因之一。
*/
@Test
void rowsMissingFromLlmAnswerStayRetryable() throws Exception {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(PARTIAL_RESOLVED_ROWS));
stubMergeWrite(1);
submit();
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("r1"));
verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r2"));
}
/** 回来的行一个都落不进分片(rowToken 被网关改写):整批都不能算完成。 */
@Test
void rowsThatMatchNoChunkRowStayRetryable() throws Exception {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(
"[{\"rowToken\":\"r9\",\"asin\":\"B0A0000009\",\"title\":\"UNKNOWN\",\"url\":\"http://img/9.jpg\"}]"));
submit();
verify(taskCacheService, never()).completeRowInspection(anyLong(), any());
verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r1", "r2"));
}
/** 发送前续一次处理中标记,避免一批跑得比 TTL 长时被并发入口抢走。 */ /** 发送前续一次处理中标记,避免一批跑得比 TTL 长时被并发入口抢走。 */
@Test @Test
void inFlightMarkIsRenewedRightBeforeSending() throws Exception { void inFlightMarkIsRenewedRightBeforeSending() throws Exception {
+3 -2
View File
@@ -121,9 +121,10 @@ main 代码零调用者)。
该文件被 git 跟踪,含生产库口令与 OSS secret。这与"result_json 存用户 apiKey"是两件事。 该文件被 git 跟踪,含生产库口令与 OSS secret。这与"result_json 存用户 apiKey"是两件事。
处置是**轮换 + 改走 env + 清历史**,三步都需要人执行(轮换会影响所有读这些值的服务,需同窗口发布)。 处置是**轮换 + 改走 env + 清历史**,三步都需要人执行(轮换会影响所有读这些值的服务,需同窗口发布)。
## 三、遗留:本地已上线,Gitea 远端还差 2 个提交 ## 三、遗留:本地已上线,Gitea 远端还差 6 个提交
`git push` 今夜没落库(远端 `master` 仍是 `56913563`,本地 HEAD `6a82697a`)。 `git push` 今夜没落库(远端 `master` 仍是 `56913563`,本地 HEAD 为本轮"未落库不判完成"那一笔,
其前依次是 `8536ddb3`、`6a82697a`、`15384358`、`2d33f478`、`eaafd502`)。
用 `GIT_CURL_VERBOSE` 定位到的真实原因不是网络慢,而是**认证弹窗在等人**: 用 `GIT_CURL_VERBOSE` 定位到的真实原因不是网络慢,而是**认证弹窗在等人**:
``` ```