perf(外观专利): LLM 批循环按 payload_hash 复用分片行索引,不再每批重读对象存储
一次 submitLlmBatches 有 N 批,每批落库前都要知道「哪些行属于哪个分片」,此前这份索引 是每批现做:把该任务全部分片的载荷重读重解析一遍,读的量是 (批数+1)×分片数,全在 LLM 请求的串行路径上。改成收集候选那一遍就把行键记进一次性的索引,后续批次只对 payload_hash 变过的分片重读;载荷哈希与载荷内容在所有写入点都是同一条 UPDATE 落的,可作新鲜度锚。 落库路径一行未改:mergeChunkPayload 仍自己 selectOne + 重读最新载荷再做 CAS,索引只用于 「行属于哪一片」的匹配,不作为合并基底。 测试 AppearancePatentChunkRowIndexTest 三例,逐个做过变异验证:去掉哈希比较只咬住 「变过必须重读」,命中缓存返回空集只咬住「复用后行仍要匹配得上分片」; 全量 3413 例通过。
This commit is contained in:
+54
-19
@@ -913,8 +913,9 @@ public class AppearancePatentTaskService {
|
||||
String prompt = readAiPrompt(task);
|
||||
String apiKey = readApiKey(task);
|
||||
int batchSize = Math.max(1, properties.getLlmBatchSize());
|
||||
List<AppearancePatentResultRowDto> candidates =
|
||||
terminalizeRoundExhaustedRows(task, allRowsByBaseId, collectPendingLlmCandidates(chunks));
|
||||
ChunkRowKeyIndex rowKeyIndex = new ChunkRowKeyIndex();
|
||||
List<AppearancePatentResultRowDto> candidates = terminalizeRoundExhaustedRows(
|
||||
task, allRowsByBaseId, collectPendingLlmCandidates(chunks, rowKeyIndex), rowKeyIndex);
|
||||
boolean flushRemainder = isResultSubmissionComplete(task.getId());
|
||||
if (!flushRemainder && !candidates.isEmpty()) {
|
||||
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
|
||||
@@ -967,7 +968,7 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
List<String> keysToRetry = new ArrayList<>(batchRowKeys);
|
||||
try {
|
||||
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
|
||||
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows, rowKeyIndex);
|
||||
// 完成标记只能打在真正落库的行上——这是「发版重启会不会丢行」的分界线
|
||||
if (batchAnswered) {
|
||||
List<String> landed = batchRowKeys.stream().filter(landedKeys::contains).toList();
|
||||
@@ -1007,7 +1008,8 @@ public class AppearancePatentTaskService {
|
||||
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
|
||||
FileTaskEntity task,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
List<AppearancePatentResultRowDto> candidates) {
|
||||
List<AppearancePatentResultRowDto> candidates,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
int maxRounds = properties.getLlmRowMaxRounds();
|
||||
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
|
||||
return candidates == null ? List.of() : candidates;
|
||||
@@ -1031,7 +1033,8 @@ public class AppearancePatentTaskService {
|
||||
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
boolean rowsTerminated = false;
|
||||
try {
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason), exhausted);
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason),
|
||||
exhausted, rowKeyIndex);
|
||||
rowsTerminated = true;
|
||||
} finally {
|
||||
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
|
||||
@@ -1047,7 +1050,8 @@ public class AppearancePatentTaskService {
|
||||
private Set<String> mergeLlmRowsIntoChunks(FileTaskEntity task,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
List<AppearancePatentResultRowDto> batchRows) {
|
||||
List<AppearancePatentResultRowDto> batchRows,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
if (llmRows == null || llmRows.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
@@ -1065,7 +1069,8 @@ public class AppearancePatentTaskService {
|
||||
deduped.put(key, row);
|
||||
}
|
||||
}
|
||||
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId);
|
||||
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId,
|
||||
null, null, rowKeyIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1084,7 +1089,8 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
|
||||
|
||||
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks) {
|
||||
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
@@ -1094,6 +1100,9 @@ public class AppearancePatentTaskService {
|
||||
if (persistedRows.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// 这一遍已经把所有分片的载荷读进来了,顺手把「这片里有哪些行」记进索引:
|
||||
// 下面每批落库前都要这份索引,不记的话就得每批把全部分片重读重解析一遍
|
||||
rememberChunkRowKeys(rowKeyIndex, chunk, persistedRows.keySet());
|
||||
for (AppearancePatentResultRowDto row : pickGroupRepresentativesForLlm(persistedRows.values())) {
|
||||
String key = rowKey(row);
|
||||
if (key.isBlank() || hasResolvedLlmFields(row)) {
|
||||
@@ -1908,18 +1917,13 @@ public class AppearancePatentTaskService {
|
||||
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成");
|
||||
}
|
||||
|
||||
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
|
||||
return mergeLlmRowsIntoSubmittedChunks(task, llmRows, allRowsByBaseId, null, null);
|
||||
}
|
||||
|
||||
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
|
||||
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
String fallbackScopeHash,
|
||||
Integer fallbackChunkIndex) {
|
||||
Integer fallbackChunkIndex,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
if (task == null || llmRows == null || llmRows.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
@@ -1927,11 +1931,11 @@ public class AppearancePatentTaskService {
|
||||
if (chunks.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Map<String, Map<String, AppearancePatentResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
Map<String, Set<String>> rowKeysByChunk = new LinkedHashMap<>();
|
||||
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
String chunkKey = chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
|
||||
rowsByChunk.put(chunkKey, readChunkRows(chunk));
|
||||
rowKeysByChunk.put(chunkKey, chunkRowKeySet(rowKeyIndex, chunk));
|
||||
chunkByKey.put(chunkKey, chunk);
|
||||
}
|
||||
Map<String, Map<String, AppearancePatentResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
|
||||
@@ -1942,8 +1946,8 @@ public class AppearancePatentTaskService {
|
||||
continue;
|
||||
}
|
||||
boolean matched = false;
|
||||
for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : rowsByChunk.entrySet()) {
|
||||
if (entry.getValue().containsKey(rowKey)) {
|
||||
for (Map.Entry<String, Set<String>> entry : rowKeysByChunk.entrySet()) {
|
||||
if (entry.getValue().contains(rowKey)) {
|
||||
mergeRowsByChunk.computeIfAbsent(entry.getKey(), ignored -> new LinkedHashMap<>())
|
||||
.put(rowKey, expandedRow);
|
||||
matched = true;
|
||||
@@ -1979,6 +1983,37 @@ public class AppearancePatentTaskService {
|
||||
return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次 {@link #submitLlmBatches} 内共享的「哪个分片里有哪些行」索引,按 payload_hash 判新鲜。
|
||||
*
|
||||
* <p>只存 rowKey 不存整行,索引本身不会把载荷钉在内存里。作用域必须是单次调用(局部变量,随栈丢弃):
|
||||
* 本服务是单例、多个任务同时在跑批,做成字段就是多线程共享一个裸 LinkedHashMap,而且分片键永不过期、只增不减。
|
||||
* 数据正确性反过来不依赖局部性——收集候选那一遍每次都会重读并重记全部分片。
|
||||
*/
|
||||
static final class ChunkRowKeyIndex {
|
||||
private final Map<String, Snapshot> snapshots = new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
/** 把刚解析出来的分片行键记进索引(自己拷一份,不握住整行载荷),返回索引里那份。 */
|
||||
private Set<String> rememberChunkRowKeys(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk, Set<String> rowKeys) {
|
||||
Set<String> copied = new LinkedHashSet<>(rowKeys);
|
||||
rowKeyIndex.snapshots.put(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()),
|
||||
new Snapshot(chunk.getPayloadHash(), copied));
|
||||
return copied;
|
||||
}
|
||||
|
||||
/** 分片当前载荷里的全部行键;载荷哈希与索引里那次解析一致时直接复用,不重读对象存储。 */
|
||||
private Set<String> chunkRowKeySet(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk) {
|
||||
Snapshot cached = rowKeyIndex.snapshots.get(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()));
|
||||
if (cached != null && Objects.equals(cached.payloadHash, chunk.getPayloadHash())) {
|
||||
return cached.rowKeys;
|
||||
}
|
||||
return rememberChunkRowKeys(rowKeyIndex, chunk, readChunkRows(chunk).keySet());
|
||||
}
|
||||
|
||||
private record Snapshot(String payloadHash, Set<String> rowKeys) {
|
||||
}
|
||||
|
||||
private boolean isResultSubmissionComplete(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return false;
|
||||
|
||||
Reference in New Issue
Block a user