perf(货源): 分片载荷判空与建行图共用一次 GET,merge 与候选收集各省一遍对象存储读取
This commit is contained in:
+60
-24
@@ -157,8 +157,10 @@ public class SimilarAsinPipelineSupport {
|
||||
if (task == null || llmRows == null || llmRows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<TaskChunkEntity> chunks = loadSubmittedChunks(task.getId());
|
||||
if (chunks.isEmpty()) {
|
||||
// 判空与建行图共用同一次载荷读取:改前 loadSubmittedChunks 逐片 GET 只为回答"这片有没有行",
|
||||
// 紧接着下面又逐片 GET 一次。
|
||||
List<SubmittedChunk> submitted = readSubmittedChunks(loadSubmittedChunks(task.getId()));
|
||||
if (submitted.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// P2-11:把当前 batch 命中的图片 url 异步丢入预热队列。
|
||||
@@ -180,9 +182,10 @@ public class SimilarAsinPipelineSupport {
|
||||
}
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
for (SubmittedChunk entry : submitted) {
|
||||
TaskChunkEntity chunk = entry.chunk();
|
||||
String chunkKey = SimilarAsinChunkMergeSupport.chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
|
||||
rowsByChunk.put(chunkKey, chunkPayloadSupport.readChunkRows(chunk));
|
||||
rowsByChunk.put(chunkKey, entry.rows());
|
||||
chunkByKey.put(chunkKey, chunk);
|
||||
}
|
||||
// Task 11:合并前按稳定 rowKey 去重,消除重复行逐行 expand/分配/写回 的 O(n²) 热点。
|
||||
@@ -396,10 +399,39 @@ public class SimilarAsinPipelineSupport {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return chunks.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty())
|
||||
.toList();
|
||||
return chunks;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 一次作业轮次内共享的分片载荷读取结果:{@code chunk} 是库里的分片行,{@code rows} 是它当前载荷里的行。
|
||||
* 只在单次调用里传递,不做字段级缓存。
|
||||
*/
|
||||
public record SubmittedChunk(TaskChunkEntity chunk, Map<String, SimilarAsinResultRowDto> rows) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐片读一次载荷、跳过空片,顺序与入参一致。
|
||||
* <p>
|
||||
* 「这片有没有行」和「这片里有哪些行」原本是两次对象存储 GET + 整片 JSON 解析(分片数 C 就是 2C 次),
|
||||
* 这里合并成一次;写回路径 {@link #mergeChunkPayload} 仍自己重读最新载荷再 CAS,不参与复用。
|
||||
*/
|
||||
public List<SubmittedChunk> readSubmittedChunks(List<TaskChunkEntity> chunks) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<SubmittedChunk> submitted = new ArrayList<>(chunks.size());
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
if (chunk == null) {
|
||||
continue;
|
||||
}
|
||||
Map<String, SimilarAsinResultRowDto> rows = chunkPayloadSupport.readChunkRows(chunk);
|
||||
if (rows.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
submitted.add(new SubmittedChunk(chunk, rows));
|
||||
}
|
||||
return submitted;
|
||||
}
|
||||
|
||||
|
||||
@@ -827,11 +859,9 @@ public class SimilarAsinPipelineSupport {
|
||||
.last("limit 1"));
|
||||
chunks = chunk == null ? List.of() : List.of(chunk);
|
||||
}
|
||||
chunks = chunks == null ? List.of() : chunks.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty())
|
||||
.toList();
|
||||
if (chunks.isEmpty()) {
|
||||
// 分片载荷只读这一次:改前这里逐片 GET 只为过滤空片,下面 collectPendingLlmCandidates 又逐片 GET 一次。
|
||||
List<SubmittedChunk> submitted = readSubmittedChunks(chunks);
|
||||
if (submitted.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
FileResultEntity result = host.findOrCreateResultRecordForAssembly(task, payloadSupport.allRowCount(task));
|
||||
@@ -844,7 +874,7 @@ public class SimilarAsinPipelineSupport {
|
||||
return;
|
||||
}
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
||||
boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId);
|
||||
boolean pendingLlm = submitLlmBatchesForSubmittedChunks(task, result, job, submitted, allRowsByBaseId);
|
||||
if (pendingLlm) {
|
||||
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
|
||||
touchJavaSideTaskActivity(task.getId());
|
||||
@@ -914,19 +944,16 @@ public class SimilarAsinPipelineSupport {
|
||||
}
|
||||
|
||||
|
||||
public List<LlmCandidate> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
|
||||
public List<LlmCandidate> collectPendingLlmCandidates(List<SubmittedChunk> submitted,
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
if (submitted == null || submitted.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<LlmCandidate> candidates = new ArrayList<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
Map<String, SimilarAsinResultRowDto> persistedRows = chunkPayloadSupport.readChunkRows(chunk);
|
||||
if (persistedRows.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
for (SubmittedChunk entry : submitted) {
|
||||
TaskChunkEntity chunk = entry.chunk();
|
||||
List<SimilarAsinResultRowDto> unresolvedRows =
|
||||
enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()), allRowsByBaseId);
|
||||
enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(entry.rows().values()), allRowsByBaseId);
|
||||
for (SimilarAsinResultRowDto row : unresolvedRows) {
|
||||
candidates.add(new LlmCandidate(chunk.getScopeHash(), chunk.getChunkIndex(), row));
|
||||
}
|
||||
@@ -1012,7 +1039,16 @@ public class SimilarAsinPipelineSupport {
|
||||
TaskFileJobEntity job,
|
||||
List<TaskChunkEntity> chunks,
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return submitLlmBatchesForSubmittedChunks(task, result, job, readSubmittedChunks(chunks), allRowsByBaseId);
|
||||
}
|
||||
|
||||
|
||||
public boolean submitLlmBatchesForSubmittedChunks(FileTaskEntity task,
|
||||
FileResultEntity result,
|
||||
TaskFileJobEntity job,
|
||||
List<SubmittedChunk> submitted,
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
|
||||
if (submitted == null || submitted.isEmpty()) {
|
||||
return countPendingLlmStates(task.getId()) > 0;
|
||||
}
|
||||
String prompt = payloadSupport.readAiPrompt(task);
|
||||
@@ -1027,7 +1063,7 @@ public class SimilarAsinPipelineSupport {
|
||||
task.getId(), batchSize);
|
||||
batchSize = 1;
|
||||
}
|
||||
List<LlmCandidate> candidates = collectPendingLlmCandidates(chunks, allRowsByBaseId);
|
||||
List<LlmCandidate> candidates = collectPendingLlmCandidates(submitted, allRowsByBaseId);
|
||||
if (candidates.isEmpty()) {
|
||||
return countPendingLlmStates(task.getId()) > 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user