fix(外观专利): 行重发加轮次上限,超限就地判失败并写明原因
两态行标让失败行下一轮能重新申领(否则发版/中断就永久丢行),代价是轮次层此前 完全没有闸:上游持续劣化时同一批行会一轮轮重发,每轮都过 SecretUsageContext.call 扣用户的 LLM 次数;而 assemble job 走 deferRunning 不累加 retry_count, MAX_RETRY_COUNT 那道闸永远到不了,HTTP 层的 llmRetryTimes=3 也只管单轮内部。 新增 aiimage.appearance-patent.llm-row-max-rounds(默认 3,<=0 关闭)。轮次落在独立 Redis 键上(处理中键会被撤销、完成键只在成功时打,只有这张表能回答"这行被重发过几次"), 首轮设 TTL 后续不自增 TTL;Redis 故障时读成 0,宁可不闸也不把在跑的检测判死。 顺带修 countUninspectedRows:Python 心跳超时恢复路径原先用 collectPendingLlmCandidates 数待检行数,那一步会把行申领走,紧接着真正发起检测时就一行也拿不到,整单只能等 处理中标记 TTL 过期。
This commit is contained in:
@@ -46,6 +46,16 @@ public class AppearancePatentProperties {
|
||||
* 每行每个 LLM 请求的重试次数(含首次)
|
||||
*/
|
||||
private int llmRetryTimes = 3;
|
||||
/**
|
||||
* 单行最多被「轮次」发起多少次(一轮 = 一次批量派发;每轮内部另有 llmRetryTimes 次 HTTP 重试)。
|
||||
* <=0 关闭本闸。
|
||||
*
|
||||
* <p>行检测失败会撤销处理中标记让下一轮重新申领(否则发版/中断就永久丢行),代价是轮次层此前
|
||||
* 完全没有上限:上游持续劣化时同一批行会一轮轮重发,每轮都过 SecretUsageContext.call 扣用户的
|
||||
* LLM 次数,而 assemble job 的 deferRunning 不累加 retryCount,MAX_RETRY_COUNT 那道闸永远到不了。
|
||||
* 默认 3 轮:与 HTTP 层同量级,够躲过单次抖动,又不至于把一个坏掉的网关陪跑到 24 小时。
|
||||
*/
|
||||
private int llmRowMaxRounds = 3;
|
||||
private int staleTimeoutMinutes = 30;
|
||||
private String staleFinalizeCron = "0 */2 * * * *";
|
||||
|
||||
|
||||
+44
@@ -226,6 +226,50 @@ public class AppearancePatentTaskCacheService {
|
||||
return "appearance-patent:task:inflight-rows:" + taskId + ":" + rowKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* 记一次行级检测轮次(发起 LLM 调用之前调)。轮次键单独一条:处理中键会被撤销、
|
||||
* 完成键只在成功时打,只有这张表能回答"这行已经被重发过几次"。
|
||||
*/
|
||||
public void countRowInspectionRound(Long taskId, List<String> rowKeys) {
|
||||
if (taskId == null || taskId <= 0 || rowKeys == null || rowKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (String rowKey : rowKeys) {
|
||||
if (rowKey == null || rowKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
String key = rowRoundsKey(taskId, rowKey);
|
||||
try {
|
||||
Long round = stringRedisTemplate.opsForValue().increment(key);
|
||||
if (round != null && round == 1L) {
|
||||
stringRedisTemplate.expire(key, Duration.ofHours(TTL_HOURS));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] count row inspection round degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 只读:该行已被发起过多少轮。Redis 降级时返回 0,判死的闸不能建立在缓存可用性上。 */
|
||||
public long rowInspectionRounds(Long taskId, String rowKey) {
|
||||
if (taskId == null || taskId <= 0 || rowKey == null || rowKey.isBlank()) {
|
||||
return 0L;
|
||||
}
|
||||
try {
|
||||
String raw = stringRedisTemplate.opsForValue().get(rowRoundsKey(taskId, rowKey));
|
||||
return raw == null || raw.isBlank() ? 0L : Long.parseLong(raw);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent-cache] read row inspection rounds degraded taskId={} rowKey={} msg={}",
|
||||
taskId, rowKey, ex.getMessage());
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
|
||||
private String rowRoundsKey(Long taskId, String rowKey) {
|
||||
return "appearance-patent:task:row-rounds:" + taskId + ":" + rowKey;
|
||||
}
|
||||
|
||||
private String processedRowsKey(Long taskId) {
|
||||
return "appearance-patent:task:processed-rows:" + taskId;
|
||||
}
|
||||
|
||||
+71
-2
@@ -907,7 +907,8 @@ public class AppearancePatentTaskService {
|
||||
String prompt = readAiPrompt(task);
|
||||
String apiKey = readApiKey(task);
|
||||
int batchSize = Math.max(1, properties.getLlmBatchSize());
|
||||
List<AppearancePatentResultRowDto> candidates = collectPendingLlmCandidates(chunks);
|
||||
List<AppearancePatentResultRowDto> candidates =
|
||||
terminalizeRoundExhaustedRows(task, allRowsByBaseId, collectPendingLlmCandidates(chunks));
|
||||
boolean flushRemainder = isResultSubmissionComplete(task.getId());
|
||||
if (!flushRemainder && !candidates.isEmpty()) {
|
||||
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
|
||||
@@ -935,6 +936,7 @@ public class AppearancePatentTaskService {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
List<String> batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
taskCacheService.touchRowInspection(task.getId(), batchRowKeys);
|
||||
taskCacheService.countRowInspectionRound(task.getId(), batchRowKeys);
|
||||
long startedAt = System.currentTimeMillis();
|
||||
List<AppearancePatentResultRowDto> llmRows;
|
||||
boolean batchAnswered = true;
|
||||
@@ -969,6 +971,53 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 轮次已到顶的行就地判失败并打成「已完成」,从此退出候选。
|
||||
*
|
||||
* <p>失败行会被撤销处理中标记、下一轮重新申领——这是「不丢行」的另一半代价。没有这道闸的话,
|
||||
* 上游劣化窗口内同一批行会一轮轮重发,每轮都过 {@link SecretUsageContext} 扣用户的 LLM 次数,
|
||||
* 而 assemble job 走的是 deferRunning(不累加 retryCount),MAX_RETRY_COUNT 那条永不触发。
|
||||
*/
|
||||
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
|
||||
FileTaskEntity task,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
List<AppearancePatentResultRowDto> candidates) {
|
||||
int maxRounds = properties.getLlmRowMaxRounds();
|
||||
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
|
||||
return candidates == null ? List.of() : candidates;
|
||||
}
|
||||
List<AppearancePatentResultRowDto> retryable = new ArrayList<>(candidates.size());
|
||||
List<AppearancePatentResultRowDto> exhausted = new ArrayList<>();
|
||||
for (AppearancePatentResultRowDto row : candidates) {
|
||||
String key = rowKey(row);
|
||||
if (!key.isBlank() && taskCacheService.rowInspectionRounds(task.getId(), key) >= maxRounds) {
|
||||
exhausted.add(row);
|
||||
} else {
|
||||
retryable.add(row);
|
||||
}
|
||||
}
|
||||
if (exhausted.isEmpty()) {
|
||||
return retryable;
|
||||
}
|
||||
String reason = "重发轮次已达上限 " + maxRounds + " 轮仍未取到结论,请稍后重新发起检测";
|
||||
log.warn("[appearance-patent] llm round cap reached taskId={} rows={} maxRounds={}",
|
||||
task.getId(), exhausted.size(), maxRounds);
|
||||
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
boolean rowsTerminated = false;
|
||||
try {
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason), exhausted);
|
||||
rowsTerminated = true;
|
||||
} finally {
|
||||
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
|
||||
if (rowsTerminated) {
|
||||
taskCacheService.completeRowInspection(task.getId(), exhaustedKeys);
|
||||
} else {
|
||||
taskCacheService.releaseRowInspection(task.getId(), exhaustedKeys);
|
||||
}
|
||||
}
|
||||
return retryable;
|
||||
}
|
||||
|
||||
private void mergeLlmRowsIntoChunks(FileTaskEntity task,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
@@ -1033,6 +1082,26 @@ public class AppearancePatentTaskService {
|
||||
return candidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* 只计数、不申领。用 {@link #collectPendingLlmCandidates} 计数会把行标成处理中,
|
||||
* 紧接着真正发起检测的那一步就一行也申领不到,只能等处理中标记的 TTL 过期。
|
||||
*/
|
||||
private int countUninspectedRows(List<TaskChunkEntity> chunks) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int total = 0;
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
|
||||
for (AppearancePatentResultRowDto row : pickGroupRepresentativesForLlm(persistedRows.values())) {
|
||||
if (!rowKey(row).isBlank() && !hasResolvedLlmFields(row)) {
|
||||
total++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private int countLlmWorkUnits(List<TaskChunkEntity> chunks, int batchSize) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return 0;
|
||||
@@ -1157,7 +1226,7 @@ public class AppearancePatentTaskService {
|
||||
taskId);
|
||||
return false;
|
||||
}
|
||||
int pendingRows = collectPendingLlmCandidates(loadSubmittedChunks(taskId)).size();
|
||||
int pendingRows = countUninspectedRows(loadSubmittedChunks(taskId));
|
||||
log.warn("[appearance-patent] Python 回传心跳超时,已封口上传并继续检测/文件收尾 taskId={} pendingRows={} activeAssembleJobs={} forcedScopes={}",
|
||||
taskId, pendingRows, activeAssembleJobs, forcedScopes);
|
||||
} else {
|
||||
|
||||
@@ -337,6 +337,9 @@ aiimage:
|
||||
# 正常任务实测约 0.91,默认 0.5 只拦"904 行只检 1 行"这类塌陷。
|
||||
llm-coverage-min-ratio: ${AIIMAGE_APPEARANCE_PATENT_LLM_COVERAGE_MIN_RATIO:0.5}
|
||||
llm-retry-times: ${AIIMAGE_APPEARANCE_PATENT_LLM_RETRY_TIMES:3}
|
||||
# 单行最多被重发几轮(<=0 关闭)。失败行会被释放、下一轮重新申领,没有这道闸时
|
||||
# 上游劣化窗口内会无限重发并每轮扣用户 LLM 次数——deferRunning 不累加 retryCount,任务层那道永不触发。
|
||||
llm-row-max-rounds: ${AIIMAGE_APPEARANCE_PATENT_LLM_ROW_MAX_ROUNDS:3}
|
||||
flush-pending-minutes: ${AIIMAGE_APPEARANCE_PATENT_FLUSH_PENDING_MINUTES:1}
|
||||
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:30}
|
||||
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
|
||||
|
||||
Reference in New Issue
Block a user