super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-3: 统一解析载荷全量行恢复逻辑，新旧结构均可恢复

Browse Source 
新增 SimilarAsinTaskService.resolveAllRows(payload)：优先 items（新规范结构）、
其次 allItems（旧结构）、最后 groups 展开（最旧结构），并收敛 rowCount/
loadAllRowsByBaseId/hydrateParsedPayloadRows/两条 assemble 路径的重复 fallback 链。
旧 payload（含 allItems 或仅 groups）仍可恢复全量行，行数、顺序不丢失。
9 个测试覆盖默认/批量/幂等/空/单行/超限/新旧 JSON 反序列化/组展开顺序。
This commit is contained in:
黄自达
2026-08-29 14:36:15 +08:00
parent 8c84ff3394
commit 9f8bdecc9e
3 changed files with 247 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+33 -44
View File
@@ -938,20 +938,29 @@ public class SimilarAsinTaskService {
if (payload == null) {
return 0;
}
if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) {
return payload.getAllItems().size();
return resolveAllRows(payload).size();
}
/**
* 从解析载荷统一恢复全量行：优先 items（新规范结构），其次 allItems（旧结构），
* 最后 groups 展开（最旧结构）。新旧结构均可恢复完整行集合，供 rowCount、
* Coze 候选加载与结果文件组装复用。
*/
static List<SimilarAsinParsedRowVo> resolveAllRows(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return List.of();
}
if (payload.getItems() != null && !payload.getItems().isEmpty()) {
return payload.getItems().size();
List<SimilarAsinParsedRowVo> rows = payload.getItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getAllItems();
}
if (payload.getGroups() != null && !payload.getGroups().isEmpty()) {
return payload.getGroups().stream()
.map(SimilarAsinParsedGroupVo::getItems)
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
rows = payload.getGroups().stream()
.filter(Objects::nonNull)
.mapToInt(List::size)
.sum();
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
.toList();
}
return 0;
return rows == null ? List.of() : rows;
}
private PersistSubmittedChunkResult persistSubmittedChunk(PreparedSubmittedChunk prepared) {
@@ -1399,17 +1408,7 @@ public class SimilarAsinTaskService {
private Map<String, List<SimilarAsinParsedRowVo>> loadAllRowsByBaseId(FileTaskEntity task) {
try {
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
List<SimilarAsinParsedRowVo> rows = payload.getAllItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getItems();
}
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
rows = payload.getGroups().stream()
.filter(Objects::nonNull)
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
.toList();
}
return groupRowsByBaseId(rows);
return groupRowsByBaseId(resolveAllRows(payload));
} catch (Exception ex) {
log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage());
return new LinkedHashMap<>();
@@ -4275,8 +4274,9 @@ public class SimilarAsinTaskService {
*/
private void assembleResultWorkbookBounded(FileTaskEntity task, FileResultEntity result) {
SimilarAsinParsedPayloadDto parsed = readParsedPayload(task);
List<SimilarAsinParsedRowVo> parsedRows = resolveAllRows(parsed);
List<SourceRows> sourceRows = splitRowsBySourceFile(
parsed, parsed.getAllItems(), result.getSourceFilename());
parsed, parsedRows, result.getSourceFilename());
if (sourceRows.isEmpty()) {
throw new BusinessException("Similar ASIN result rows are empty");
}
@@ -4372,7 +4372,7 @@ public class SimilarAsinTaskService {
.mapToInt(SourceResultWorkbook::conformPropagated)
.sum();
log.info("[similar-asin] bounded assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={} propagated={} sources={} costMs={}",
task.getId(), parsed.getAllItems().size(), persistedResultRows, resultRows,
task.getId(), parsedRows.size(), persistedResultRows, resultRows,
resolvedRows, conformPropagated, sourceRows.size(),
System.currentTimeMillis() - assembleStart);
if (resultRows == 0) {
@@ -4408,7 +4408,7 @@ public class SimilarAsinTaskService {
result.setResultFileUrl(objectKey);
result.setResultFileSize(uploadFile.length());
result.setResultContentType(contentType);
result.setRowCount(parsed.getAllItems().size());
result.setRowCount(parsedRows.size());
} finally {
taskImageCache.clear();
for (File workbookFile : workbookFiles) {
@@ -4428,15 +4428,16 @@ public class SimilarAsinTaskService {
return;
}
SimilarAsinParsedPayloadDto parsed = readParsedPayload(task);
Map<String, SimilarAsinResultRowDto> resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size());
List<SimilarAsinParsedRowVo> parsedRows = resolveAllRows(parsed);
Map<String, SimilarAsinResultRowDto> resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsedRows.size());
int persistedResultRows = resultMap.size();
resultMap.entrySet().removeIf(entry -> !isExportableResultRow(entry.getValue()));
long resolvedRows = parsed.getAllItems().stream()
long resolvedRows = parsedRows.stream()
.filter(row -> findResultRow(row, resultMap) != null)
.count();
log.info("[similar-asin] assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={}",
task.getId(), parsed.getAllItems().size(), persistedResultRows, resultMap.size(), resolvedRows);
if (!parsed.getAllItems().isEmpty() && resultMap.isEmpty()) {
task.getId(), parsedRows.size(), persistedResultRows, resultMap.size(), resolvedRows);
if (!parsedRows.isEmpty() && resultMap.isEmpty()) {
// P1-4：检查是否有 chunk-read-failed 标记，把 chunk index 列表附在错误信息里。
String chunkReadFailureSummary = collectChunkReadFailureSummary(task.getId());
if (!chunkReadFailureSummary.isBlank()) {
@@ -4448,7 +4449,7 @@ public class SimilarAsinTaskService {
// 命中"不符合"（包含匹配），则组内所有行的"是否符合类目"统一为标准值"不符合"。
// 单行组跳过。仅修改 isConform 列，不影响其他列。
int conformPropagated = CozeGroupResultPropagator.propagateByGroup(
parsed.getAllItems(),
parsedRows,
SimilarAsinParsedRowVo::getDisplayId,
row -> findResultRow(row, resultMap),
SimilarAsinResultRowDto::getIsConform,
@@ -4464,7 +4465,7 @@ public class SimilarAsinTaskService {
if (!outputDir.exists() && !outputDir.mkdirs()) {
throw new BusinessException("创建结果目录失败");
}
List<SourceRows> sourceRows = splitRowsBySourceFile(parsed, parsed.getAllItems(), result.getSourceFilename());
List<SourceRows> sourceRows = splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename());
List<SourceResultWorkbook> workbooks = new ArrayList<>();
List<File> workbookFiles = new ArrayList<>(sourceRows.size());
File zip = null;
@@ -4564,7 +4565,7 @@ public class SimilarAsinTaskService {
result.setResultFileUrl(objectKey);
result.setResultFileSize(uploadFile.length());
result.setResultContentType(contentType);
result.setRowCount(parsed.getAllItems().size());
result.setRowCount(resolveAllRows(parsed).size());
} finally {
for (File workbookFile : workbookFiles) {
if (workbookFile.exists() && !workbookFile.delete()) {
@@ -5923,19 +5924,7 @@ public class SimilarAsinTaskService {
if (payload == null) {
return new SimilarAsinParsedPayloadDto();
}
List<SimilarAsinParsedRowVo> rows = payload.getAllItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getItems();
}
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
rows = payload.getGroups().stream()
.filter(Objects::nonNull)
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
.toList();
}
if (rows == null) {
rows = List.of();
}
List<SimilarAsinParsedRowVo> rows = resolveAllRows(payload);
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
payload.setAllItems(rows);
}