fix(外观专利): 行重发加轮次上限,超限就地判失败并写明原因

两态行标让失败行下一轮能重新申领(否则发版/中断就永久丢行),代价是轮次层此前
完全没有闸:上游持续劣化时同一批行会一轮轮重发,每轮都过 SecretUsageContext.call
扣用户的 LLM 次数;而 assemble job 走 deferRunning 不累加 retry_count,
MAX_RETRY_COUNT 那道闸永远到不了,HTTP 层的 llmRetryTimes=3 也只管单轮内部。

新增 aiimage.appearance-patent.llm-row-max-rounds(默认 3,<=0 关闭)。轮次落在独立
Redis 键上(处理中键会被撤销、完成键只在成功时打,只有这张表能回答"这行被重发过几次"),
首轮设 TTL 后续不自增 TTL;Redis 故障时读成 0,宁可不闸也不把在跑的检测判死。

顺带修 countUninspectedRows:Python 心跳超时恢复路径原先用 collectPendingLlmCandidates
数待检行数,那一步会把行申领走,紧接着真正发起检测时就一行也拿不到,整单只能等
处理中标记 TTL 过期。
This commit is contained in:
2026-09-28 00:42:22 +08:00
parent a58dc8a680
commit 9b0188f5ad
6 changed files with 225 additions and 3 deletions
@@ -46,6 +46,16 @@ public class AppearancePatentProperties {
* 每行每个 LLM 请求的重试次数(含首次)
*/
private int llmRetryTimes = 3;
/**
* 单行最多被「轮次」发起多少次(一轮 = 一次批量派发;每轮内部另有 llmRetryTimes 次 HTTP 重试)。
* <=0 关闭本闸。
*
* <p>行检测失败会撤销处理中标记让下一轮重新申领(否则发版/中断就永久丢行),代价是轮次层此前
* 完全没有上限:上游持续劣化时同一批行会一轮轮重发,每轮都过 SecretUsageContext.call 扣用户的
* LLM 次数,而 assemble job 的 deferRunning 不累加 retryCount,MAX_RETRY_COUNT 那道闸永远到不了。
* 默认 3 轮:与 HTTP 层同量级,够躲过单次抖动,又不至于把一个坏掉的网关陪跑到 24 小时。
*/
private int llmRowMaxRounds = 3;
private int staleTimeoutMinutes = 30;
private String staleFinalizeCron = "0 */2 * * * *";
@@ -226,6 +226,50 @@ public class AppearancePatentTaskCacheService {
return "appearance-patent:task:inflight-rows:" + taskId + ":" + rowKey;
}
/**
* 记一次行级检测轮次(发起 LLM 调用之前调)。轮次键单独一条:处理中键会被撤销、
* 完成键只在成功时打,只有这张表能回答"这行已经被重发过几次"。
*/
public void countRowInspectionRound(Long taskId, List<String> rowKeys) {
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
return;
}
for (String rowKey : rowKeys) {
if (rowKey == null || rowKey.isBlank()) {
continue;
}
String key = rowRoundsKey(taskId, rowKey);
try {
Long round = stringRedisTemplate.opsForValue().increment(key);
if (round != null && round == 1L) {
stringRedisTemplate.expire(key, Duration.ofHours(TTL_HOURS));
}
} catch (Exception ex) {
log.warn("[appearance-patent-cache] count row inspection round degraded taskId={} rowKey={} msg={}",
taskId, rowKey, ex.getMessage());
}
}
}
/** 只读:该行已被发起过多少轮。Redis 降级时返回 0,判死的闸不能建立在缓存可用性上。 */
public long rowInspectionRounds(Long taskId, String rowKey) {
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
return 0L;
}
try {
String raw = stringRedisTemplate.opsForValue().get(rowRoundsKey(taskId, rowKey));
return raw == null || raw.isBlank() ? 0L : Long.parseLong(raw);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] read row inspection rounds degraded taskId={} rowKey={} msg={}",
taskId, rowKey, ex.getMessage());
return 0L;
}
}
private String rowRoundsKey(Long taskId, String rowKey) {
return "appearance-patent:task:row-rounds:" + taskId + ":" + rowKey;
}
private String processedRowsKey(Long taskId) {
return "appearance-patent:task:processed-rows:" + taskId;
}
@@ -907,7 +907,8 @@ public class AppearancePatentTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
int batchSize = Math.max(1, properties.getLlmBatchSize());
List<AppearancePatentResultRowDto> candidates = collectPendingLlmCandidates(chunks);
List<AppearancePatentResultRowDto> candidates =
terminalizeRoundExhaustedRows(task, allRowsByBaseId, collectPendingLlmCandidates(chunks));
boolean flushRemainder = isResultSubmissionComplete(task.getId());
if (!flushRemainder && !candidates.isEmpty()) {
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
@@ -935,6 +936,7 @@ public class AppearancePatentTaskService {
taskFileJobService.touchRunning(job.getId());
List<String> batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
taskCacheService.touchRowInspection(task.getId(), batchRowKeys);
taskCacheService.countRowInspectionRound(task.getId(), batchRowKeys);
long startedAt = System.currentTimeMillis();
List<AppearancePatentResultRowDto> llmRows;
boolean batchAnswered = true;
@@ -969,6 +971,53 @@ public class AppearancePatentTaskService {
}
}
/**
* 轮次已到顶的行就地判失败并打成「已完成」,从此退出候选。
*
* <p>失败行会被撤销处理中标记、下一轮重新申领——这是「不丢行」的另一半代价。没有这道闸的话,
* 上游劣化窗口内同一批行会一轮轮重发,每轮都过 {@link SecretUsageContext} 扣用户的 LLM 次数,
* 而 assemble job 走的是 deferRunning(不累加 retryCount),MAX_RETRY_COUNT 那条永不触发。
*/
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
List<AppearancePatentResultRowDto> candidates) {
int maxRounds = properties.getLlmRowMaxRounds();
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
return candidates == null ? List.of() : candidates;
}
List<AppearancePatentResultRowDto> retryable = new ArrayList<>(candidates.size());
List<AppearancePatentResultRowDto> exhausted = new ArrayList<>();
for (AppearancePatentResultRowDto row : candidates) {
String key = rowKey(row);
if (!key.isBlank() && taskCacheService.rowInspectionRounds(task.getId(), key) >= maxRounds) {
exhausted.add(row);
} else {
retryable.add(row);
}
}
if (exhausted.isEmpty()) {
return retryable;
}
String reason = "重发轮次已达上限 " + maxRounds + " 轮仍未取到结论,请稍后重新发起检测";
log.warn("[appearance-patent] llm round cap reached taskId={} rows={} maxRounds={}",
task.getId(), exhausted.size(), maxRounds);
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
boolean rowsTerminated = false;
try {
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason), exhausted);
rowsTerminated = true;
} finally {
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
if (rowsTerminated) {
taskCacheService.completeRowInspection(task.getId(), exhaustedKeys);
} else {
taskCacheService.releaseRowInspection(task.getId(), exhaustedKeys);
}
}
return retryable;
}
private void mergeLlmRowsIntoChunks(FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
List<AppearancePatentResultRowDto> llmRows,
@@ -1033,6 +1082,26 @@ public class AppearancePatentTaskService {
return candidates;
}
/**
* 只计数、不申领。用 {@link #collectPendingLlmCandidates} 计数会把行标成处理中,
* 紧接着真正发起检测的那一步就一行也申领不到,只能等处理中标记的 TTL 过期。
*/
private int countUninspectedRows(List<TaskChunkEntity> chunks) {
if (chunks == null || chunks.isEmpty()) {
return 0;
}
int total = 0;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
for (AppearancePatentResultRowDto row : pickGroupRepresentativesForLlm(persistedRows.values())) {
if (!rowKey(row).isBlank() && !hasResolvedLlmFields(row)) {
total++;
}
}
}
return total;
}
private int countLlmWorkUnits(List<TaskChunkEntity> chunks, int batchSize) {
if (chunks == null || chunks.isEmpty()) {
return 0;
@@ -1157,7 +1226,7 @@ public class AppearancePatentTaskService {
taskId);
return false;
}
int pendingRows = collectPendingLlmCandidates(loadSubmittedChunks(taskId)).size();
int pendingRows = countUninspectedRows(loadSubmittedChunks(taskId));
log.warn("[appearance-patent] Python 回传心跳超时,已封口上传并继续检测/文件收尾 taskId={} pendingRows={} activeAssembleJobs={} forcedScopes={}",
taskId, pendingRows, activeAssembleJobs, forcedScopes);
} else {
@@ -337,6 +337,9 @@ aiimage:
# 正常任务实测约 0.91,默认 0.5 只拦"904 行只检 1 行"这类塌陷。
llm-coverage-min-ratio: ${AIIMAGE_APPEARANCE_PATENT_LLM_COVERAGE_MIN_RATIO:0.5}
llm-retry-times: ${AIIMAGE_APPEARANCE_PATENT_LLM_RETRY_TIMES:3}
# 单行最多被重发几轮(<=0 关闭)。失败行会被释放、下一轮重新申领,没有这道闸时
# 上游劣化窗口内会无限重发并每轮扣用户 LLM 次数——deferRunning 不累加 retryCount,任务层那道永不触发。
llm-row-max-rounds: ${AIIMAGE_APPEARANCE_PATENT_LLM_ROW_MAX_ROUNDS:3}
flush-pending-minutes: ${AIIMAGE_APPEARANCE_PATENT_FLUSH_PENDING_MINUTES:1}
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:30}
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
@@ -49,6 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.lenient;
@@ -61,7 +62,8 @@ import static org.mockito.Mockito.when;
*
* <p>合批之后一次请求带走 5–10 行,一次中断就是整批白做。原实现在收集候选时(发请求**之前**)
* 就把行标成已处理,所以发版重启留下的不是"稍后补检",而是"永不再检"。
* 这里钉住两条:申领发生在收集阶段、完成标记只允许出现在载荷 CAS 落库成功之后。
* 这里钉住三条:申领发生在收集阶段、完成标记只允许出现在载荷 CAS 落库成功之后、
* 重发轮次有上限(超限的行就地判失败并写明原因,不再白扣用户次数)。
*/
@ExtendWith(MockitoExtension.class)
class AppearancePatentRowMarkTimingTest {
@@ -225,6 +227,58 @@ class AppearancePatentRowMarkTimingTest {
assertEquals(List.of("r1", "r2"), keys.getValue());
}
/**
* 重发轮次到顶的行必须就地判失败、写明原因,并且**不再发 LLM**。
*
* <p>两态标记让失败行下一轮重新成为候选,而 assemble job 的 deferRunning 不碰 retryCount,
* 所以轮次层此前没有任何闸:上游劣化窗口内这批行会一轮轮重发,每轮都走
* SecretUsageContext.call 扣用户的次数,且看不到"为什么还在跑"。
*/
@Test
void rowsOverRoundCapAreTerminatedWithoutAnotherLlmCall() throws Exception {
when(properties.getLlmRowMaxRounds()).thenReturn(2);
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(taskCacheService.rowInspectionRounds(eq(TASK_ID), anyString())).thenReturn(2L);
when(llmClient.markRowsFailed(any(), anyString())).thenReturn(rows(UNRESOLVED_ROWS));
stubMergeWrite(1);
submit();
verify(llmClient, never()).inspectRows(any(), any(), any());
verify(llmClient).markRowsFailed(any(), contains("轮次"));
verify(taskCacheService).completeRowInspection(eq(TASK_ID), eq(List.of("r1", "r2")));
verify(taskCacheService, never()).releaseRowInspection(anyLong(), any());
}
/** 轮次没到顶的行照旧发出去,但必须先记一轮——不计数就等于上限永远到不了。 */
@Test
void sentBatchCountsOneRoundBeforeCallingLlm() throws Exception {
when(properties.getLlmRowMaxRounds()).thenReturn(2);
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(taskCacheService.rowInspectionRounds(eq(TASK_ID), anyString())).thenReturn(1L);
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
stubMergeWrite(1);
submit();
InOrder order = inOrder(taskCacheService, llmClient);
order.verify(taskCacheService).countRowInspectionRound(TASK_ID, List.of("r1", "r2"));
order.verify(llmClient).inspectRows(any(), any(), any());
}
/**
* 只是「数还有多少行待检」的入口不能顺手把行申领走:
* stale 恢复路径是先数一遍再发起检测,数的时候申领掉的话紧接着那一步就一行也拿不到,
* 整单只能等处理中标记 TTL 过期。
*/
@Test
void countingPendingRowsDoesNotClaimThem() {
Integer pending = ReflectionTestUtils.invokeMethod(service, "countUninspectedRows", List.of(chunk()));
assertEquals(2, pending == null ? 0 : pending);
verify(taskCacheService, never()).beginRowInspection(anyLong(), anyString());
}
// ===== 辅助 =====
private void submit() {
@@ -152,4 +152,46 @@ class AppearancePatentTaskCacheServiceRowMarkTest {
assertFalse(deleted.getValue().contains("processed-rows"), "撤销只能删处理中键,删完成标记会让已检测的行重跑并重复计费");
verify(valueOperations, never()).set(anyString(), anyString(), any(Duration.class));
}
/** 每发起一轮就自增一次:这是重发上限唯一的数据来源。 */
@Test
void roundCountIncrementsPerRound() {
when(valueOperations.increment(anyString())).thenReturn(2L);
cache.countRowInspectionRound(TASK_ID, List.of(ROW_KEY));
ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
verify(valueOperations).increment(key.capture());
assertTrue(key.getValue().contains("row-rounds"), "轮次必须落在独立键上,不能和处理中/完成标记共用");
}
/** 轮次键要跟着行的生命周期过期:INCR 出来的键默认永不过期,会一直堆在 Redis 里。 */
@Test
void firstRoundPutsTtlOnRoundKey() {
when(valueOperations.increment(anyString())).thenReturn(1L);
cache.countRowInspectionRound(TASK_ID, List.of(ROW_KEY));
ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
verify(redisTemplate).expire(key.capture(), any(Duration.class));
assertTrue(key.getValue().contains("row-rounds"));
}
/** 续轮不能再设 TTL:每轮一次 expire 是纯浪费,首轮设过就一直有效。 */
@Test
void laterRoundDoesNotRefreshTtl() {
when(valueOperations.increment(anyString())).thenReturn(2L);
cache.countRowInspectionRound(TASK_ID, List.of(ROW_KEY));
verify(redisTemplate, never()).expire(anyString(), any(Duration.class));
}
/** 只读轮次:Redis 降级时读成 0,宁可不闸也不能因为缓存故障把还在跑的检测判死。 */
@Test
void roundReadDegradesToZero() {
when(valueOperations.get(anyString())).thenThrow(new IllegalStateException("redis down"));
assertEquals(0L, cache.rowInspectionRounds(TASK_ID, ROW_KEY));
}
}