perf(货源): 分片载荷判空与建行图共用一次 GET,merge 与候选收集各省一遍对象存储读取

This commit is contained in:
2026-10-02 13:13:13 +08:00
parent 8812a3b64a
commit a83f7f51bd
3 changed files with 291 additions and 26 deletions
@@ -157,8 +157,10 @@ public class SimilarAsinPipelineSupport {
if (task == null || llmRows == null || llmRows.isEmpty()) {
return;
}
List<TaskChunkEntity> chunks = loadSubmittedChunks(task.getId());
if (chunks.isEmpty()) {
// 判空与建行图共用同一次载荷读取:改前 loadSubmittedChunks 逐片 GET 只为回答"这片有没有行",
// 紧接着下面又逐片 GET 一次。
List<SubmittedChunk> submitted = readSubmittedChunks(loadSubmittedChunks(task.getId()));
if (submitted.isEmpty()) {
return;
}
// P2-11:把当前 batch 命中的图片 url 异步丢入预热队列。
@@ -180,9 +182,10 @@ public class SimilarAsinPipelineSupport {
}
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
for (SubmittedChunk entry : submitted) {
TaskChunkEntity chunk = entry.chunk();
String chunkKey = SimilarAsinChunkMergeSupport.chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
rowsByChunk.put(chunkKey, chunkPayloadSupport.readChunkRows(chunk));
rowsByChunk.put(chunkKey, entry.rows());
chunkByKey.put(chunkKey, chunk);
}
// Task 11:合并前按稳定 rowKey 去重,消除重复行逐行 expand/分配/写回 的 O(n²) 热点。
@@ -396,10 +399,39 @@ public class SimilarAsinPipelineSupport {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
return chunks.stream()
.filter(Objects::nonNull)
.filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty())
.toList();
return chunks;
}
/**
* 一次作业轮次内共享的分片载荷读取结果:{@code chunk} 是库里的分片行,{@code rows} 是它当前载荷里的行。
* 只在单次调用里传递,不做字段级缓存。
*/
public record SubmittedChunk(TaskChunkEntity chunk, Map<String, SimilarAsinResultRowDto> rows) {
}
/**
* 逐片读一次载荷、跳过空片,顺序与入参一致。
* <p>
* 「这片有没有行」和「这片里有哪些行」原本是两次对象存储 GET + 整片 JSON 解析(分片数 C 就是 2C 次),
* 这里合并成一次;写回路径 {@link #mergeChunkPayload} 仍自己重读最新载荷再 CAS,不参与复用。
*/
public List<SubmittedChunk> readSubmittedChunks(List<TaskChunkEntity> chunks) {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
List<SubmittedChunk> submitted = new ArrayList<>(chunks.size());
for (TaskChunkEntity chunk : chunks) {
if (chunk == null) {
continue;
}
Map<String, SimilarAsinResultRowDto> rows = chunkPayloadSupport.readChunkRows(chunk);
if (rows.isEmpty()) {
continue;
}
submitted.add(new SubmittedChunk(chunk, rows));
}
return submitted;
}
@@ -827,11 +859,9 @@ public class SimilarAsinPipelineSupport {
.last("limit 1"));
chunks = chunk == null ? List.of() : List.of(chunk);
}
chunks = chunks == null ? List.of() : chunks.stream()
.filter(Objects::nonNull)
.filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty())
.toList();
if (chunks.isEmpty()) {
// 分片载荷只读这一次:改前这里逐片 GET 只为过滤空片,下面 collectPendingLlmCandidates 又逐片 GET 一次。
List<SubmittedChunk> submitted = readSubmittedChunks(chunks);
if (submitted.isEmpty()) {
return;
}
FileResultEntity result = host.findOrCreateResultRecordForAssembly(task, payloadSupport.allRowCount(task));
@@ -844,7 +874,7 @@ public class SimilarAsinPipelineSupport {
return;
}
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId);
boolean pendingLlm = submitLlmBatchesForSubmittedChunks(task, result, job, submitted, allRowsByBaseId);
if (pendingLlm) {
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(task.getId());
@@ -914,19 +944,16 @@ public class SimilarAsinPipelineSupport {
}
public List<LlmCandidate> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
public List<LlmCandidate> collectPendingLlmCandidates(List<SubmittedChunk> submitted,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (chunks == null || chunks.isEmpty()) {
if (submitted == null || submitted.isEmpty()) {
return List.of();
}
List<LlmCandidate> candidates = new ArrayList<>();
for (TaskChunkEntity chunk : chunks) {
Map<String, SimilarAsinResultRowDto> persistedRows = chunkPayloadSupport.readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
for (SubmittedChunk entry : submitted) {
TaskChunkEntity chunk = entry.chunk();
List<SimilarAsinResultRowDto> unresolvedRows =
enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()), allRowsByBaseId);
enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(entry.rows().values()), allRowsByBaseId);
for (SimilarAsinResultRowDto row : unresolvedRows) {
candidates.add(new LlmCandidate(chunk.getScopeHash(), chunk.getChunkIndex(), row));
}
@@ -1012,7 +1039,16 @@ public class SimilarAsinPipelineSupport {
TaskFileJobEntity job,
List<TaskChunkEntity> chunks,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (chunks == null || chunks.isEmpty()) {
return submitLlmBatchesForSubmittedChunks(task, result, job, readSubmittedChunks(chunks), allRowsByBaseId);
}
public boolean submitLlmBatchesForSubmittedChunks(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
List<SubmittedChunk> submitted,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (submitted == null || submitted.isEmpty()) {
return countPendingLlmStates(task.getId()) > 0;
}
String prompt = payloadSupport.readAiPrompt(task);
@@ -1027,7 +1063,7 @@ public class SimilarAsinPipelineSupport {
task.getId(), batchSize);
batchSize = 1;
}
List<LlmCandidate> candidates = collectPendingLlmCandidates(chunks, allRowsByBaseId);
List<LlmCandidate> candidates = collectPendingLlmCandidates(submitted, allRowsByBaseId);
if (candidates.isEmpty()) {
return countPendingLlmStates(task.getId()) > 0;
}