perf(外观专利): LLM 批循环按 payload_hash 复用分片行索引,不再每批重读对象存储

一次 submitLlmBatches 有 N 批,每批落库前都要知道「哪些行属于哪个分片」,此前这份索引
是每批现做:把该任务全部分片的载荷重读重解析一遍,读的量是 (批数+1)×分片数,全在 LLM
请求的串行路径上。改成收集候选那一遍就把行键记进一次性的索引,后续批次只对 payload_hash
变过的分片重读;载荷哈希与载荷内容在所有写入点都是同一条 UPDATE 落的,可作新鲜度锚。

落库路径一行未改:mergeChunkPayload 仍自己 selectOne + 重读最新载荷再做 CAS,索引只用于
「行属于哪一片」的匹配,不作为合并基底。

测试 AppearancePatentChunkRowIndexTest 三例,逐个做过变异验证:去掉哈希比较只咬住
「变过必须重读」,命中缓存返回空集只咬住「复用后行仍要匹配得上分片」;
全量 3413 例通过。
This commit is contained in:
2026-10-02 10:35:23 +08:00
parent 87a306458c
commit b728bb3f4d
3 changed files with 321 additions and 21 deletions
@@ -913,8 +913,9 @@ public class AppearancePatentTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
int batchSize = Math.max(1, properties.getLlmBatchSize());
List<AppearancePatentResultRowDto> candidates =
terminalizeRoundExhaustedRows(task, allRowsByBaseId, collectPendingLlmCandidates(chunks));
ChunkRowKeyIndex rowKeyIndex = new ChunkRowKeyIndex();
List<AppearancePatentResultRowDto> candidates = terminalizeRoundExhaustedRows(
task, allRowsByBaseId, collectPendingLlmCandidates(chunks, rowKeyIndex), rowKeyIndex);
boolean flushRemainder = isResultSubmissionComplete(task.getId());
if (!flushRemainder && !candidates.isEmpty()) {
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
@@ -967,7 +968,7 @@ public class AppearancePatentTaskService {
}
List<String> keysToRetry = new ArrayList<>(batchRowKeys);
try {
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows, rowKeyIndex);
// 完成标记只能打在真正落库的行上——这是「发版重启会不会丢行」的分界线
if (batchAnswered) {
List<String> landed = batchRowKeys.stream().filter(landedKeys::contains).toList();
@@ -1007,7 +1008,8 @@ public class AppearancePatentTaskService {
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
List<AppearancePatentResultRowDto> candidates) {
List<AppearancePatentResultRowDto> candidates,
ChunkRowKeyIndex rowKeyIndex) {
int maxRounds = properties.getLlmRowMaxRounds();
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
return candidates == null ? List.of() : candidates;
@@ -1031,7 +1033,8 @@ public class AppearancePatentTaskService {
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
boolean rowsTerminated = false;
try {
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason), exhausted);
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason),
exhausted, rowKeyIndex);
rowsTerminated = true;
} finally {
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
@@ -1047,7 +1050,8 @@ public class AppearancePatentTaskService {
private Set<String> mergeLlmRowsIntoChunks(FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
List<AppearancePatentResultRowDto> llmRows,
List<AppearancePatentResultRowDto> batchRows) {
List<AppearancePatentResultRowDto> batchRows,
ChunkRowKeyIndex rowKeyIndex) {
if (llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
@@ -1065,7 +1069,8 @@ public class AppearancePatentTaskService {
deduped.put(key, row);
}
}
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId);
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId,
null, null, rowKeyIndex);
}
/**
@@ -1084,7 +1089,8 @@ public class AppearancePatentTaskService {
}
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks) {
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
ChunkRowKeyIndex rowKeyIndex) {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
@@ -1094,6 +1100,9 @@ public class AppearancePatentTaskService {
if (persistedRows.isEmpty()) {
continue;
}
// 这一遍已经把所有分片的载荷读进来了,顺手把「这片里有哪些行」记进索引:
// 下面每批落库前都要这份索引,不记的话就得每批把全部分片重读重解析一遍
rememberChunkRowKeys(rowKeyIndex, chunk, persistedRows.keySet());
for (AppearancePatentResultRowDto row : pickGroupRepresentativesForLlm(persistedRows.values())) {
String key = rowKey(row);
if (key.isBlank() || hasResolvedLlmFields(row)) {
@@ -1908,18 +1917,13 @@ public class AppearancePatentTaskService {
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成");
}
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
return mergeLlmRowsIntoSubmittedChunks(task, llmRows, allRowsByBaseId, null, null);
}
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
String fallbackScopeHash,
Integer fallbackChunkIndex) {
Integer fallbackChunkIndex,
ChunkRowKeyIndex rowKeyIndex) {
if (task == null || llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
@@ -1927,11 +1931,11 @@ public class AppearancePatentTaskService {
if (chunks.isEmpty()) {
return Set.of();
}
Map<String, Map<String, AppearancePatentResultRowDto>> rowsByChunk = new LinkedHashMap<>();
Map<String, Set<String>> rowKeysByChunk = new LinkedHashMap<>();
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
String chunkKey = chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
rowsByChunk.put(chunkKey, readChunkRows(chunk));
rowKeysByChunk.put(chunkKey, chunkRowKeySet(rowKeyIndex, chunk));
chunkByKey.put(chunkKey, chunk);
}
Map<String, Map<String, AppearancePatentResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
@@ -1942,8 +1946,8 @@ public class AppearancePatentTaskService {
continue;
}
boolean matched = false;
for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : rowsByChunk.entrySet()) {
if (entry.getValue().containsKey(rowKey)) {
for (Map.Entry<String, Set<String>> entry : rowKeysByChunk.entrySet()) {
if (entry.getValue().contains(rowKey)) {
mergeRowsByChunk.computeIfAbsent(entry.getKey(), ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
@@ -1979,6 +1983,37 @@ public class AppearancePatentTaskService {
return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex);
}
/**
* 一次 {@link #submitLlmBatches} 内共享的「哪个分片里有哪些行」索引,按 payload_hash 判新鲜。
*
* <p>只存 rowKey 不存整行,索引本身不会把载荷钉在内存里。作用域必须是单次调用(局部变量,随栈丢弃):
* 本服务是单例、多个任务同时在跑批,做成字段就是多线程共享一个裸 LinkedHashMap,而且分片键永不过期、只增不减。
* 数据正确性反过来不依赖局部性——收集候选那一遍每次都会重读并重记全部分片。
*/
static final class ChunkRowKeyIndex {
private final Map<String, Snapshot> snapshots = new LinkedHashMap<>();
}
/** 把刚解析出来的分片行键记进索引(自己拷一份,不握住整行载荷),返回索引里那份。 */
private Set<String> rememberChunkRowKeys(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk, Set<String> rowKeys) {
Set<String> copied = new LinkedHashSet<>(rowKeys);
rowKeyIndex.snapshots.put(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()),
new Snapshot(chunk.getPayloadHash(), copied));
return copied;
}
/** 分片当前载荷里的全部行键;载荷哈希与索引里那次解析一致时直接复用,不重读对象存储。 */
private Set<String> chunkRowKeySet(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk) {
Snapshot cached = rowKeyIndex.snapshots.get(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()));
if (cached != null && Objects.equals(cached.payloadHash, chunk.getPayloadHash())) {
return cached.rowKeys;
}
return rememberChunkRowKeys(rowKeyIndex, chunk, readChunkRows(chunk).keySet());
}
private record Snapshot(String payloadHash, Set<String> rowKeys) {
}
private boolean isResultSubmissionComplete(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;