super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-10: chunk 结果建立 rowKey 批量索引，消除跨 chunk 线性扫描

Browse Source 
mergeCozeRowsIntoChunk 先 indexRowsByChunkKey 建立 rowKey→chunkKey
索引，assignCozeRowsToChunks 按 O(1) 查找分配行归属，保留原有
命中/fallback/orphan 语义与顺序稳定性，每 chunk 只读一次 payload。
This commit is contained in:
黄自达
2026-08-29 16:07:45 +08:00
parent 17a3292c78
commit 1ccf4f74ef
2 changed files with 450 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+75 -29
View File
@@ -1509,6 +1509,74 @@ public class SimilarAsinTaskService {
return all;
}
/**
* Task 10：把 chunk 结果行建立成 rowKey → chunkKey 的批量索引。
* 供 assignCozeRowsToChunks 使用，把跨 chunk 线性扫描降为 O(1) 查找。
* 同一 rowKey 出现在多个 chunk 时保留第一个（putIfAbsent），行为确定。
*/
Map<String, String> indexRowsByChunkKey(Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk) {
Map<String, String> index = new java.util.HashMap<>();
if (rowsByChunk == null || rowsByChunk.isEmpty()) {
return index;
}
for (Map.Entry<String, Map<String, SimilarAsinResultRowDto>> entry : rowsByChunk.entrySet()) {
if (entry.getValue() == null) {
continue;
}
for (String rowKey : entry.getValue().keySet()) {
if (rowKey == null || rowKey.isBlank()) {
continue;
}
index.putIfAbsent(rowKey, entry.getKey());
}
}
return index;
}
/**
* Task 10：基于 rowKey 索引为 coze 回传行分配归属 chunk。
* 命中索引 → 归属该 chunk；未命中且有有效 fallback（chunkScopeHash + chunkIndex）
* 且 fallback chunk 存在 → 归属 fallback；否则进 orphan 列表。
* 与原实现逐 chunk 线性扫描语义完全一致，但每个行查找降为 O(1)。
*/
Map<String, Map<String, SimilarAsinResultRowDto>> assignCozeRowsToChunks(
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk,
List<SimilarAsinResultRowDto> cozeRows,
Map<String, String> rowKeyIndex,
String chunkScopeHash,
Integer chunkIndex,
List<SimilarAsinResultRowDto> orphans) {
Map<String, Map<String, SimilarAsinResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
if (cozeRows == null || cozeRows.isEmpty() || orphans == null) {
return mergeRowsByChunk;
}
String fallbackKey = chunkScopeHash != null && !chunkScopeHash.isBlank() && chunkIndex != null
? chunkStorageKey(chunkScopeHash, chunkIndex) : null;
boolean fallbackValid = fallbackKey != null && rowsByChunk != null && rowsByChunk.containsKey(fallbackKey);
for (SimilarAsinResultRowDto expandedRow : cozeRows) {
if (expandedRow == null) {
continue;
}
String rowKey = rowKey(expandedRow);
if (rowKey.isBlank()) {
continue;
}
String chunkKey = rowKeyIndex == null ? null : rowKeyIndex.get(rowKey);
if (chunkKey != null) {
mergeRowsByChunk.computeIfAbsent(chunkKey, ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
} else if (fallbackValid) {
mergeRowsByChunk.computeIfAbsent(fallbackKey, ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
} else {
orphans.add(expandedRow);
log.error("[similar-asin] coze row has no submitted chunk rowKey={} asin={} country={}",
rowKey, expandedRow.getAsin(), expandedRow.getCountry());
}
}
return mergeRowsByChunk;
}
private void applyCozeToPersistedChunks(FileTaskEntity task, Runnable progressHook) {
if (task == null || task.getId() == null) {
return;
@@ -3781,37 +3849,15 @@ public class SimilarAsinTaskService {
rowsByChunk.put(chunkKey, readChunkRows(chunk));
chunkByKey.put(chunkKey, chunk);
}
Map<String, Map<String, SimilarAsinResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
List<SimilarAsinResultRowDto> orphanRows = new ArrayList<>();
List<SimilarAsinResultRowDto> expandedAll = new ArrayList<>();
for (SimilarAsinResultRowDto resultRow : cozeRows) {
for (SimilarAsinResultRowDto expandedRow : expandRows(List.of(resultRow), allRowsByBaseId)) {
String rowKey = rowKey(expandedRow);
if (rowKey.isBlank()) {
continue;
}
boolean matched = false;
for (Map.Entry<String, Map<String, SimilarAsinResultRowDto>> entry : rowsByChunk.entrySet()) {
if (entry.getValue().containsKey(rowKey)) {
mergeRowsByChunk.computeIfAbsent(entry.getKey(), ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
}
}
if (!matched && chunkScopeHash != null && !chunkScopeHash.isBlank() && chunkIndex != null) {
String fallbackKey = chunkStorageKey(chunkScopeHash, chunkIndex);
if (chunkByKey.containsKey(fallbackKey)) {
mergeRowsByChunk.computeIfAbsent(fallbackKey, ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
}
}
if (!matched) {
orphanRows.add(expandedRow);
log.error("[similar-asin] coze row has no submitted chunk taskId={} rowKey={} asin={} country={}",
task.getId(), rowKey, expandedRow.getAsin(), expandedRow.getCountry());
}
}
expandedAll.addAll(expandRows(List.of(resultRow), allRowsByBaseId));
}
// Task 10：先建 rowKey → chunkKey 批量索引，把逐 chunk 线性扫描降为 O(1) 查找。
Map<String, String> rowKeyIndex = indexRowsByChunkKey(rowsByChunk);
List<SimilarAsinResultRowDto> orphanRows = new ArrayList<>();
Map<String, Map<String, SimilarAsinResultRowDto>> mergeRowsByChunk =
assignCozeRowsToChunks(rowsByChunk, expandedAll, rowKeyIndex, chunkScopeHash, chunkIndex, orphanRows);
if (!orphanRows.isEmpty()) {
persistOrphanCozeRows(task.getId(), orphanRows);
}