From 9b0188f5ad4cb92f758952f7cc161a72b15e2794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 28 Sep 2026 00:42:22 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E5=A4=96=E8=A7=82=E4=B8=93=E5=88=A9):=20?= =?UTF-8?q?=E8=A1=8C=E9=87=8D=E5=8F=91=E5=8A=A0=E8=BD=AE=E6=AC=A1=E4=B8=8A?= =?UTF-8?q?=E9=99=90=EF=BC=8C=E8=B6=85=E9=99=90=E5=B0=B1=E5=9C=B0=E5=88=A4?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E5=B9=B6=E5=86=99=E6=98=8E=E5=8E=9F=E5=9B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 两态行标让失败行下一轮能重新申领(否则发版/中断就永久丢行),代价是轮次层此前 完全没有闸:上游持续劣化时同一批行会一轮轮重发,每轮都过 SecretUsageContext.call 扣用户的 LLM 次数;而 assemble job 走 deferRunning 不累加 retry_count, MAX_RETRY_COUNT 那道闸永远到不了,HTTP 层的 llmRetryTimes=3 也只管单轮内部。 新增 aiimage.appearance-patent.llm-row-max-rounds(默认 3,<=0 关闭)。轮次落在独立 Redis 键上(处理中键会被撤销、完成键只在成功时打,只有这张表能回答"这行被重发过几次"), 首轮设 TTL 后续不自增 TTL;Redis 故障时读成 0,宁可不闸也不把在跑的检测判死。 顺带修 countUninspectedRows:Python 心跳超时恢复路径原先用 collectPendingLlmCandidates 数待检行数,那一步会把行申领走,紧接着真正发起检测时就一行也拿不到,整单只能等 处理中标记 TTL 过期。 --- .../config/AppearancePatentProperties.java | 10 +++ .../AppearancePatentTaskCacheService.java | 44 +++++++++++ .../service/AppearancePatentTaskService.java | 73 ++++++++++++++++++- .../src/main/resources/application.yml | 3 + .../AppearancePatentRowMarkTimingTest.java | 56 +++++++++++++- ...ancePatentTaskCacheServiceRowMarkTest.java | 42 +++++++++++ 6 files changed, 225 insertions(+), 3 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java index 14b0fbee7..e38daff89 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java @@ -46,6 +46,16 @@ public class AppearancePatentProperties { * 每行每个 LLM 请求的重试次数(含首次) */ private int llmRetryTimes = 3; + /** + * 单行最多被「轮次」发起多少次(一轮 = 一次批量派发;每轮内部另有 llmRetryTimes 次 HTTP 重试)。 + * <=0 关闭本闸。 + * + *

行检测失败会撤销处理中标记让下一轮重新申领(否则发版/中断就永久丢行),代价是轮次层此前 + * 完全没有上限:上游持续劣化时同一批行会一轮轮重发,每轮都过 SecretUsageContext.call 扣用户的 + * LLM 次数,而 assemble job 的 deferRunning 不累加 retryCount,MAX_RETRY_COUNT 那道闸永远到不了。 + * 默认 3 轮:与 HTTP 层同量级,够躲过单次抖动,又不至于把一个坏掉的网关陪跑到 24 小时。 + */ + private int llmRowMaxRounds = 3; private int staleTimeoutMinutes = 30; private String staleFinalizeCron = "0 */2 * * * *"; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java index 12a910b51..549d70da1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java @@ -226,6 +226,50 @@ public class AppearancePatentTaskCacheService { return "appearance-patent:task:inflight-rows:" + taskId + ":" + rowKey; } + /** + * 记一次行级检测轮次(发起 LLM 调用之前调)。轮次键单独一条:处理中键会被撤销、 + * 完成键只在成功时打,只有这张表能回答"这行已经被重发过几次"。 + */ + public void countRowInspectionRound(Long taskId, List rowKeys) { + if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) { + return; + } + for (String rowKey : rowKeys) { + if (rowKey == null || rowKey.isBlank()) { + continue; + } + String key = rowRoundsKey(taskId, rowKey); + try { + Long round = stringRedisTemplate.opsForValue().increment(key); + if (round != null && round == 1L) { + stringRedisTemplate.expire(key, Duration.ofHours(TTL_HOURS)); + } + } catch (Exception ex) { + log.warn("[appearance-patent-cache] count row inspection round degraded taskId={} rowKey={} msg={}", + taskId, rowKey, ex.getMessage()); + } + } + } + + /** 只读:该行已被发起过多少轮。Redis 降级时返回 0,判死的闸不能建立在缓存可用性上。 */ + public long rowInspectionRounds(Long taskId, String rowKey) { + if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) { + return 0L; + } + try { + String raw = stringRedisTemplate.opsForValue().get(rowRoundsKey(taskId, rowKey)); + return raw == null || raw.isBlank() ? 0L : Long.parseLong(raw); + } catch (Exception ex) { + log.warn("[appearance-patent-cache] read row inspection rounds degraded taskId={} rowKey={} msg={}", + taskId, rowKey, ex.getMessage()); + return 0L; + } + } + + private String rowRoundsKey(Long taskId, String rowKey) { + return "appearance-patent:task:row-rounds:" + taskId + ":" + rowKey; + } + private String processedRowsKey(Long taskId) { return "appearance-patent:task:processed-rows:" + taskId; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index c995c50bf..b616051fb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -907,7 +907,8 @@ public class AppearancePatentTaskService { String prompt = readAiPrompt(task); String apiKey = readApiKey(task); int batchSize = Math.max(1, properties.getLlmBatchSize()); - List candidates = collectPendingLlmCandidates(chunks); + List candidates = + terminalizeRoundExhaustedRows(task, allRowsByBaseId, collectPendingLlmCandidates(chunks)); boolean flushRemainder = isResultSubmissionComplete(task.getId()); if (!flushRemainder && !candidates.isEmpty()) { LocalDateTime jobUpdatedAt = job.getUpdatedAt(); @@ -935,6 +936,7 @@ public class AppearancePatentTaskService { taskFileJobService.touchRunning(job.getId()); List batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList(); taskCacheService.touchRowInspection(task.getId(), batchRowKeys); + taskCacheService.countRowInspectionRound(task.getId(), batchRowKeys); long startedAt = System.currentTimeMillis(); List llmRows; boolean batchAnswered = true; @@ -969,6 +971,53 @@ public class AppearancePatentTaskService { } } + /** + * 轮次已到顶的行就地判失败并打成「已完成」,从此退出候选。 + * + *

失败行会被撤销处理中标记、下一轮重新申领——这是「不丢行」的另一半代价。没有这道闸的话, + * 上游劣化窗口内同一批行会一轮轮重发,每轮都过 {@link SecretUsageContext} 扣用户的 LLM 次数, + * 而 assemble job 走的是 deferRunning(不累加 retryCount),MAX_RETRY_COUNT 那条永不触发。 + */ + private List terminalizeRoundExhaustedRows( + FileTaskEntity task, + Map> allRowsByBaseId, + List candidates) { + int maxRounds = properties.getLlmRowMaxRounds(); + if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) { + return candidates == null ? List.of() : candidates; + } + List retryable = new ArrayList<>(candidates.size()); + List exhausted = new ArrayList<>(); + for (AppearancePatentResultRowDto row : candidates) { + String key = rowKey(row); + if (!key.isBlank() && taskCacheService.rowInspectionRounds(task.getId(), key) >= maxRounds) { + exhausted.add(row); + } else { + retryable.add(row); + } + } + if (exhausted.isEmpty()) { + return retryable; + } + String reason = "重发轮次已达上限 " + maxRounds + " 轮仍未取到结论,请稍后重新发起检测"; + log.warn("[appearance-patent] llm round cap reached taskId={} rows={} maxRounds={}", + task.getId(), exhausted.size(), maxRounds); + List exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList(); + boolean rowsTerminated = false; + try { + mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason), exhausted); + rowsTerminated = true; + } finally { + // 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试 + if (rowsTerminated) { + taskCacheService.completeRowInspection(task.getId(), exhaustedKeys); + } else { + taskCacheService.releaseRowInspection(task.getId(), exhaustedKeys); + } + } + return retryable; + } + private void mergeLlmRowsIntoChunks(FileTaskEntity task, Map> allRowsByBaseId, List llmRows, @@ -1033,6 +1082,26 @@ public class AppearancePatentTaskService { return candidates; } + /** + * 只计数、不申领。用 {@link #collectPendingLlmCandidates} 计数会把行标成处理中, + * 紧接着真正发起检测的那一步就一行也申领不到,只能等处理中标记的 TTL 过期。 + */ + private int countUninspectedRows(List chunks) { + if (chunks == null || chunks.isEmpty()) { + return 0; + } + int total = 0; + for (TaskChunkEntity chunk : chunks) { + Map persistedRows = readChunkRows(chunk); + for (AppearancePatentResultRowDto row : pickGroupRepresentativesForLlm(persistedRows.values())) { + if (!rowKey(row).isBlank() && !hasResolvedLlmFields(row)) { + total++; + } + } + } + return total; + } + private int countLlmWorkUnits(List chunks, int batchSize) { if (chunks == null || chunks.isEmpty()) { return 0; @@ -1157,7 +1226,7 @@ public class AppearancePatentTaskService { taskId); return false; } - int pendingRows = collectPendingLlmCandidates(loadSubmittedChunks(taskId)).size(); + int pendingRows = countUninspectedRows(loadSubmittedChunks(taskId)); log.warn("[appearance-patent] Python 回传心跳超时,已封口上传并继续检测/文件收尾 taskId={} pendingRows={} activeAssembleJobs={} forcedScopes={}", taskId, pendingRows, activeAssembleJobs, forcedScopes); } else { diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 15cd2a9cc..a9c09dfa3 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -337,6 +337,9 @@ aiimage: # 正常任务实测约 0.91,默认 0.5 只拦"904 行只检 1 行"这类塌陷。 llm-coverage-min-ratio: ${AIIMAGE_APPEARANCE_PATENT_LLM_COVERAGE_MIN_RATIO:0.5} llm-retry-times: ${AIIMAGE_APPEARANCE_PATENT_LLM_RETRY_TIMES:3} + # 单行最多被重发几轮(<=0 关闭)。失败行会被释放、下一轮重新申领,没有这道闸时 + # 上游劣化窗口内会无限重发并每轮扣用户 LLM 次数——deferRunning 不累加 retryCount,任务层那道永不触发。 + llm-row-max-rounds: ${AIIMAGE_APPEARANCE_PATENT_LLM_ROW_MAX_ROUNDS:3} flush-pending-minutes: ${AIIMAGE_APPEARANCE_PATENT_FLUSH_PENDING_MINUTES:1} stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:30} stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentRowMarkTimingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentRowMarkTimingTest.java index 68eaf465e..b4cb3474c 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentRowMarkTimingTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentRowMarkTimingTest.java @@ -49,6 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.lenient; @@ -61,7 +62,8 @@ import static org.mockito.Mockito.when; * *

合批之后一次请求带走 5–10 行,一次中断就是整批白做。原实现在收集候选时(发请求**之前**) * 就把行标成已处理,所以发版重启留下的不是"稍后补检",而是"永不再检"。 - * 这里钉住两条:申领发生在收集阶段、完成标记只允许出现在载荷 CAS 落库成功之后。 + * 这里钉住三条:申领发生在收集阶段、完成标记只允许出现在载荷 CAS 落库成功之后、 + * 重发轮次有上限(超限的行就地判失败并写明原因,不再白扣用户次数)。 */ @ExtendWith(MockitoExtension.class) class AppearancePatentRowMarkTimingTest { @@ -225,6 +227,58 @@ class AppearancePatentRowMarkTimingTest { assertEquals(List.of("r1", "r2"), keys.getValue()); } + /** + * 重发轮次到顶的行必须就地判失败、写明原因,并且**不再发 LLM**。 + * + *

两态标记让失败行下一轮重新成为候选,而 assemble job 的 deferRunning 不碰 retryCount, + * 所以轮次层此前没有任何闸:上游劣化窗口内这批行会一轮轮重发,每轮都走 + * SecretUsageContext.call 扣用户的次数,且看不到"为什么还在跑"。 + */ + @Test + void rowsOverRoundCapAreTerminatedWithoutAnotherLlmCall() throws Exception { + when(properties.getLlmRowMaxRounds()).thenReturn(2); + when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true); + when(taskCacheService.rowInspectionRounds(eq(TASK_ID), anyString())).thenReturn(2L); + when(llmClient.markRowsFailed(any(), anyString())).thenReturn(rows(UNRESOLVED_ROWS)); + stubMergeWrite(1); + + submit(); + + verify(llmClient, never()).inspectRows(any(), any(), any()); + verify(llmClient).markRowsFailed(any(), contains("轮次")); + verify(taskCacheService).completeRowInspection(eq(TASK_ID), eq(List.of("r1", "r2"))); + verify(taskCacheService, never()).releaseRowInspection(anyLong(), any()); + } + + /** 轮次没到顶的行照旧发出去,但必须先记一轮——不计数就等于上限永远到不了。 */ + @Test + void sentBatchCountsOneRoundBeforeCallingLlm() throws Exception { + when(properties.getLlmRowMaxRounds()).thenReturn(2); + when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true); + when(taskCacheService.rowInspectionRounds(eq(TASK_ID), anyString())).thenReturn(1L); + when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS)); + stubMergeWrite(1); + + submit(); + + InOrder order = inOrder(taskCacheService, llmClient); + order.verify(taskCacheService).countRowInspectionRound(TASK_ID, List.of("r1", "r2")); + order.verify(llmClient).inspectRows(any(), any(), any()); + } + + /** + * 只是「数还有多少行待检」的入口不能顺手把行申领走: + * stale 恢复路径是先数一遍再发起检测,数的时候申领掉的话紧接着那一步就一行也拿不到, + * 整单只能等处理中标记 TTL 过期。 + */ + @Test + void countingPendingRowsDoesNotClaimThem() { + Integer pending = ReflectionTestUtils.invokeMethod(service, "countUninspectedRows", List.of(chunk())); + + assertEquals(2, pending == null ? 0 : pending); + verify(taskCacheService, never()).beginRowInspection(anyLong(), anyString()); + } + // ===== 辅助 ===== private void submit() { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheServiceRowMarkTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheServiceRowMarkTest.java index 89ab82958..d0019738b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheServiceRowMarkTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheServiceRowMarkTest.java @@ -152,4 +152,46 @@ class AppearancePatentTaskCacheServiceRowMarkTest { assertFalse(deleted.getValue().contains("processed-rows"), "撤销只能删处理中键,删完成标记会让已检测的行重跑并重复计费"); verify(valueOperations, never()).set(anyString(), anyString(), any(Duration.class)); } + + /** 每发起一轮就自增一次:这是重发上限唯一的数据来源。 */ + @Test + void roundCountIncrementsPerRound() { + when(valueOperations.increment(anyString())).thenReturn(2L); + + cache.countRowInspectionRound(TASK_ID, List.of(ROW_KEY)); + + ArgumentCaptor key = ArgumentCaptor.forClass(String.class); + verify(valueOperations).increment(key.capture()); + assertTrue(key.getValue().contains("row-rounds"), "轮次必须落在独立键上,不能和处理中/完成标记共用"); + } + + /** 轮次键要跟着行的生命周期过期:INCR 出来的键默认永不过期,会一直堆在 Redis 里。 */ + @Test + void firstRoundPutsTtlOnRoundKey() { + when(valueOperations.increment(anyString())).thenReturn(1L); + + cache.countRowInspectionRound(TASK_ID, List.of(ROW_KEY)); + + ArgumentCaptor key = ArgumentCaptor.forClass(String.class); + verify(redisTemplate).expire(key.capture(), any(Duration.class)); + assertTrue(key.getValue().contains("row-rounds")); + } + + /** 续轮不能再设 TTL:每轮一次 expire 是纯浪费,首轮设过就一直有效。 */ + @Test + void laterRoundDoesNotRefreshTtl() { + when(valueOperations.increment(anyString())).thenReturn(2L); + + cache.countRowInspectionRound(TASK_ID, List.of(ROW_KEY)); + + verify(redisTemplate, never()).expire(anyString(), any(Duration.class)); + } + + /** 只读轮次:Redis 降级时读成 0,宁可不闸也不能因为缓存故障把还在跑的检测判死。 */ + @Test + void roundReadDegradesToZero() { + when(valueOperations.get(anyString())).thenThrow(new IllegalStateException("redis down")); + + assertEquals(0L, cache.rowInspectionRounds(TASK_ID, ROW_KEY)); + } }