From 9f8bdecc9ea42d9e7ea874a016206d12cbc83596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 29 Aug 2026 14:36:15 +0800 Subject: [PATCH] =?UTF-8?q?task-3:=20=E7=BB=9F=E4=B8=80=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E8=BD=BD=E8=8D=B7=E5=85=A8=E9=87=8F=E8=A1=8C=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E6=96=B0=E6=97=A7=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=9D=87=E5=8F=AF=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 SimilarAsinTaskService.resolveAllRows(payload)：优先 items（新规范结构）、 其次 allItems（旧结构）、最后 groups 展开（最旧结构），并收敛 rowCount/ loadAllRowsByBaseId/hydrateParsedPayloadRows/两条 assemble 路径的重复 fallback 链。 旧 payload（含 allItems 或仅 groups）仍可恢复全量行，行数、顺序不丢失。 9 个测试覆盖默认/批量/幂等/空/单行/超限/新旧 JSON 反序列化/组展开顺序。 --- .../service/SimilarAsinTaskService.java | 77 +++---- ...ilarAsinTaskServiceResolveAllRowsTest.java | 210 ++++++++++++++++++ progress.json | 8 +- 3 files changed, 247 insertions(+), 48 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 8c40b9e3..2f39da25 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -938,20 +938,29 @@ public class SimilarAsinTaskService { if (payload == null) { return 0; } - if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) { - return payload.getAllItems().size(); + return resolveAllRows(payload).size(); + } + + /** + * 从解析载荷统一恢复全量行：优先 items（新规范结构），其次 allItems（旧结构）， + * 最后 groups 展开（最旧结构）。新旧结构均可恢复完整行集合，供 rowCount、 + * Coze 候选加载与结果文件组装复用。 + */ + static List resolveAllRows(SimilarAsinParsedPayloadDto payload) { + if (payload == null) { + return List.of(); } - if (payload.getItems() != null && !payload.getItems().isEmpty()) { - return payload.getItems().size(); + List rows = payload.getItems(); + if (rows == null || rows.isEmpty()) { + rows = payload.getAllItems(); } - if (payload.getGroups() != null && !payload.getGroups().isEmpty()) { - return payload.getGroups().stream() - .map(SimilarAsinParsedGroupVo::getItems) + if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { + rows = payload.getGroups().stream() .filter(Objects::nonNull) - .mapToInt(List::size) - .sum(); + .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) + .toList(); } - return 0; + return rows == null ? List.of() : rows; } private PersistSubmittedChunkResult persistSubmittedChunk(PreparedSubmittedChunk prepared) { @@ -1399,17 +1408,7 @@ public class SimilarAsinTaskService { private Map> loadAllRowsByBaseId(FileTaskEntity task) { try { SimilarAsinParsedPayloadDto payload = readParsedPayload(task); - List rows = payload.getAllItems(); - if (rows == null || rows.isEmpty()) { - rows = payload.getItems(); - } - if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { - rows = payload.getGroups().stream() - .filter(Objects::nonNull) - .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) - .toList(); - } - return groupRowsByBaseId(rows); + return groupRowsByBaseId(resolveAllRows(payload)); } catch (Exception ex) { log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage()); return new LinkedHashMap<>(); @@ -4275,8 +4274,9 @@ public class SimilarAsinTaskService { */ private void assembleResultWorkbookBounded(FileTaskEntity task, FileResultEntity result) { SimilarAsinParsedPayloadDto parsed = readParsedPayload(task); + List parsedRows = resolveAllRows(parsed); List sourceRows = splitRowsBySourceFile( - parsed, parsed.getAllItems(), result.getSourceFilename()); + parsed, parsedRows, result.getSourceFilename()); if (sourceRows.isEmpty()) { throw new BusinessException("Similar ASIN result rows are empty"); } @@ -4372,7 +4372,7 @@ public class SimilarAsinTaskService { .mapToInt(SourceResultWorkbook::conformPropagated) .sum(); log.info("[similar-asin] bounded assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={} propagated={} sources={} costMs={}", - task.getId(), parsed.getAllItems().size(), persistedResultRows, resultRows, + task.getId(), parsedRows.size(), persistedResultRows, resultRows, resolvedRows, conformPropagated, sourceRows.size(), System.currentTimeMillis() - assembleStart); if (resultRows == 0) { @@ -4408,7 +4408,7 @@ public class SimilarAsinTaskService { result.setResultFileUrl(objectKey); result.setResultFileSize(uploadFile.length()); result.setResultContentType(contentType); - result.setRowCount(parsed.getAllItems().size()); + result.setRowCount(parsedRows.size()); } finally { taskImageCache.clear(); for (File workbookFile : workbookFiles) { @@ -4428,15 +4428,16 @@ public class SimilarAsinTaskService { return; } SimilarAsinParsedPayloadDto parsed = readParsedPayload(task); - Map resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size()); + List parsedRows = resolveAllRows(parsed); + Map resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsedRows.size()); int persistedResultRows = resultMap.size(); resultMap.entrySet().removeIf(entry -> !isExportableResultRow(entry.getValue())); - long resolvedRows = parsed.getAllItems().stream() + long resolvedRows = parsedRows.stream() .filter(row -> findResultRow(row, resultMap) != null) .count(); log.info("[similar-asin] assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={}", - task.getId(), parsed.getAllItems().size(), persistedResultRows, resultMap.size(), resolvedRows); - if (!parsed.getAllItems().isEmpty() && resultMap.isEmpty()) { + task.getId(), parsedRows.size(), persistedResultRows, resultMap.size(), resolvedRows); + if (!parsedRows.isEmpty() && resultMap.isEmpty()) { // P1-4：检查是否有 chunk-read-failed 标记，把 chunk index 列表附在错误信息里。 String chunkReadFailureSummary = collectChunkReadFailureSummary(task.getId()); if (!chunkReadFailureSummary.isBlank()) { @@ -4448,7 +4449,7 @@ public class SimilarAsinTaskService { // 命中"不符合"（包含匹配），则组内所有行的"是否符合类目"统一为标准值"不符合"。 // 单行组跳过。仅修改 isConform 列，不影响其他列。 int conformPropagated = CozeGroupResultPropagator.propagateByGroup( - parsed.getAllItems(), + parsedRows, SimilarAsinParsedRowVo::getDisplayId, row -> findResultRow(row, resultMap), SimilarAsinResultRowDto::getIsConform, @@ -4464,7 +4465,7 @@ public class SimilarAsinTaskService { if (!outputDir.exists() && !outputDir.mkdirs()) { throw new BusinessException("创建结果目录失败"); } - List sourceRows = splitRowsBySourceFile(parsed, parsed.getAllItems(), result.getSourceFilename()); + List sourceRows = splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename()); List workbooks = new ArrayList<>(); List workbookFiles = new ArrayList<>(sourceRows.size()); File zip = null; @@ -4564,7 +4565,7 @@ public class SimilarAsinTaskService { result.setResultFileUrl(objectKey); result.setResultFileSize(uploadFile.length()); result.setResultContentType(contentType); - result.setRowCount(parsed.getAllItems().size()); + result.setRowCount(resolveAllRows(parsed).size()); } finally { for (File workbookFile : workbookFiles) { if (workbookFile.exists() && !workbookFile.delete()) { @@ -5923,19 +5924,7 @@ public class SimilarAsinTaskService { if (payload == null) { return new SimilarAsinParsedPayloadDto(); } - List rows = payload.getAllItems(); - if (rows == null || rows.isEmpty()) { - rows = payload.getItems(); - } - if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { - rows = payload.getGroups().stream() - .filter(Objects::nonNull) - .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) - .toList(); - } - if (rows == null) { - rows = List.of(); - } + List rows = resolveAllRows(payload); if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) { payload.setAllItems(rows); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java new file mode 100644 index 00000000..6c7a00f6 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceResolveAllRowsTest.java @@ -0,0 +1,210 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Task 3：保留旧 payload 读取兼容逻辑，验证新旧结构均可恢复全量行。 + * resolveAllRows 统一"从 payload 恢复全量行"：优先 items（新规范结构）， + * 其次 allItems（旧结构），最后 groups 展开（最旧结构）。 + */ +class SimilarAsinTaskServiceResolveAllRowsTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static SimilarAsinParsedRowVo row(String fileKey, int index) { + SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo(); + row.setSourceFileKey(fileKey); + row.setSourceFilename("base.xlsx"); + row.setRowIndex(index); + row.setSourceId(String.valueOf(index)); + row.setDisplayId(String.valueOf(index)); + row.setRowToken(fileKey + "::row::" + index); + row.setAsin("B0CJ8SNXXV"); + row.setCountry("英国"); + return row; + } + + private static List rows(String fileKey, int count) { + List result = new ArrayList<>(); + for (int i = 1; i <= count; i++) { + result.add(row(fileKey, i)); + } + return result; + } + + private static SimilarAsinParsedGroupVo group(List items) { + SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo(); + group.setSourceFileKey("uploads/20260829/base.xlsx"); + group.setSourceFilename("base.xlsx"); + group.setGroupKey("uploads/20260829/base.xlsx::1@1"); + group.setBaseId("1"); + group.setDisplayId("1"); + group.setItemCount(items.size()); + group.setItems(new ArrayList<>(items)); + return group; + } + + @Test + void test_task_003_payload_normal_default_path() { + // 新结构：items 有值 → 恢复全量行 + SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); + List items = rows("uploads/20260829/base.xlsx", 100); + payload.setItems(items); + List restored = SimilarAsinTaskService.resolveAllRows(payload); + assertEquals(100, restored.size()); + assertEquals("uploads/20260829/base.xlsx::row::1", restored.get(0).getRowToken()); + } + + @Test + void test_task_003_payload_normal_multiple_items() { + // 新结构 1000 行：不丢失且顺序稳定 + SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); + List items = rows("uploads/20260829/multi.xlsx", 1000); + payload.setItems(items); + List restored = SimilarAsinTaskService.resolveAllRows(payload); + assertEquals(1000, restored.size()); + for (int i = 0; i < restored.size(); i++) { + assertEquals(i + 1, restored.get(i).getRowIndex()); + } + // 旧结构 1000 行：allItems 全量恢复 + SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto(); + legacy.setAllItems(items); + List restoredLegacy = SimilarAsinTaskService.resolveAllRows(legacy); + assertEquals(1000, restoredLegacy.size()); + assertEquals("uploads/20260829/multi.xlsx::row::1", restoredLegacy.get(0).getRowToken()); + } + + @Test + void test_task_003_payload_normal_repeated_operation_is_idempotent() { + // 重复调用返回相同行集合（不修改 payload 本身） + SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); + List items = rows("uploads/20260829/idem.xlsx", 50); + payload.setItems(items); + List first = SimilarAsinTaskService.resolveAllRows(payload); + List second = SimilarAsinTaskService.resolveAllRows(payload); + assertEquals(first.size(), second.size()); + for (int i = 0; i < first.size(); i++) { + assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken()); + } + // payload 未被修改：items 仍 50 行 + assertEquals(50, payload.getItems().size()); + } + + @Test + void test_task_003_payload_boundary_empty_input() { + // 空 payload：返回空列表而非 null，不创建无效资源 + SimilarAsinParsedPayloadDto empty = new SimilarAsinParsedPayloadDto(); + List restored = SimilarAsinTaskService.resolveAllRows(empty); + assertNotNull(restored); + assertEquals(0, restored.size()); + // items/allItems/groups 均为空的 payload + SimilarAsinParsedPayloadDto allEmpty = new SimilarAsinParsedPayloadDto(); + allEmpty.setItems(List.of()); + allEmpty.setAllItems(List.of()); + allEmpty.setGroups(List.of()); + assertEquals(0, SimilarAsinTaskService.resolveAllRows(allEmpty).size()); + } + + @Test + void test_task_003_payload_boundary_single_item() { + // 单行新结构 + SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); + payload.setItems(rows("uploads/20260829/single.xlsx", 1)); + assertEquals(1, SimilarAsinTaskService.resolveAllRows(payload).size()); + // 单行旧结构（仅 groups） + SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto(); + legacy.setGroups(List.of(group(rows("uploads/20260829/single.xlsx", 1)))); + assertEquals(1, SimilarAsinTaskService.resolveAllRows(legacy).size()); + } + + @Test + void test_task_003_payload_boundary_limit_and_overflow() { + // 旧结构 allItems 5000 行全量恢复，不丢失 + SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto(); + legacy.setAllItems(rows("uploads/20260829/max.xlsx", 5000)); + assertEquals(5000, SimilarAsinTaskService.resolveAllRows(legacy).size()); + // items 与 allItems 同时存在：以 items 为准（新规范结构优先），不重复 + SimilarAsinParsedPayloadDto both = new SimilarAsinParsedPayloadDto(); + both.setItems(rows("uploads/20260829/both.xlsx", 10)); + both.setAllItems(rows("uploads/20260829/both.xlsx", 20)); + List restored = SimilarAsinTaskService.resolveAllRows(both); + assertEquals(10, restored.size()); + // groups 也同时存在：仍以 items 为准 + both.setGroups(List.of(group(rows("uploads/20260829/both.xlsx", 30)))); + assertEquals(10, SimilarAsinTaskService.resolveAllRows(both).size()); + } + + @Test + void test_task_003_payload_invalid_input_rejected() { + // null payload：安全返回空列表（调用方容忍），不抛 NPE + assertEquals(0, SimilarAsinTaskService.resolveAllRows(null).size()); + // groups 中含 null 元素：跳过不抛异常 + SimilarAsinParsedPayloadDto messy = new SimilarAsinParsedPayloadDto(); + List groups = new ArrayList<>(); + groups.add(null); + groups.add(group(rows("uploads/20260829/messy.xlsx", 5))); + messy.setGroups(groups); + assertEquals(5, SimilarAsinTaskService.resolveAllRows(messy).size()); + // group.items 为 null：跳过该组 + SimilarAsinParsedGroupVo nullItemsGroup = new SimilarAsinParsedGroupVo(); + nullItemsGroup.setItems(null); + SimilarAsinParsedPayloadDto nullItems = new SimilarAsinParsedPayloadDto(); + nullItems.setGroups(List.of(nullItemsGroup, group(rows("uploads/20260829/n2.xlsx", 3)))); + assertEquals(3, SimilarAsinTaskService.resolveAllRows(nullItems).size()); + } + + @Test + void test_task_003_payload_dependency_failure_releases_resources() throws Exception { + // 旧格式 JSON（只有 allItems）反序列化 → hydrate 后 resolveAllRows 恢复全量行 + String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false," + + "\"sourceFiles\":[],\"headers\":[],\"groups\":[]," + + "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0AAA00001\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":1}," + + "{\"rowToken\":\"t2\",\"asin\":\"B0AAA00002\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":2}]}"; + SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class); + // 未 hydrate 时：allItems 恢复（items 为空走 allItems） + List fromLegacy = SimilarAsinTaskService.resolveAllRows(payload); + assertEquals(2, fromLegacy.size()); + assertEquals("t1", fromLegacy.get(0).getRowToken()); + assertEquals("B0AAA00002", fromLegacy.get(1).getAsin()); + // 新格式 JSON（只有 items）反序列化 → 直接恢复 + String newJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false," + + "\"sourceFiles\":[],\"headers\":[]," + + "\"items\":[{\"rowToken\":\"n1\",\"asin\":\"B0NEW00001\",\"sourceFileKey\":\"uploads/20260829/b.xlsx\",\"rowIndex\":1}]," + + "\"groups\":[]}"; + SimilarAsinParsedPayloadDto newPayload = MAPPER.readValue(newJson, SimilarAsinParsedPayloadDto.class); + List fromNew = SimilarAsinTaskService.resolveAllRows(newPayload); + assertEquals(1, fromNew.size()); + assertEquals("n1", fromNew.get(0).getRowToken()); + // 两种格式行数之和互不影响，恢复结果稳定 + assertTrue(fromLegacy.size() == 2 && fromNew.size() == 1); + } + + @Test + void test_task_003_payload_normal_groups_expansion_preserves_order() { + // 最旧结构：仅 groups 嵌套行，展开后顺序稳定（按组、组内原序） + SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto(); + List g1 = rows("uploads/20260829/g.xlsx", 2); + List g2 = rows("uploads/20260829/g.xlsx", 3); + // 组内行号各自独立从 1 开始（真实解析语义），第二组用不同 fileKey 区分来源 + List g2b = rows("uploads/20260829/g2.xlsx", 3); + legacy.setGroups(List.of(group(g1), group(g2b))); + List restored = SimilarAsinTaskService.resolveAllRows(legacy); + assertEquals(5, restored.size()); + assertEquals("uploads/20260829/g.xlsx::row::1", restored.get(0).getRowToken()); + assertEquals("uploads/20260829/g.xlsx::row::2", restored.get(1).getRowToken()); + assertEquals("uploads/20260829/g2.xlsx::row::1", restored.get(2).getRowToken()); + assertEquals("uploads/20260829/g2.xlsx::row::2", restored.get(3).getRowToken()); + assertEquals("uploads/20260829/g2.xlsx::row::3", restored.get(4).getRowToken()); + } +} diff --git a/progress.json b/progress.json index acbdb684..79d1877c 100644 --- a/progress.json +++ b/progress.json @@ -1,10 +1,10 @@ { "version": 1, "total_tasks": 100, - "completed": 2, - "rounds": 2, + "completed": 3, + "rounds": 3, "started_at": "2026-08-29T14:10:48+08:00", - "updated_at": "2026-08-29T14:35:00+08:00", + "updated_at": "2026-08-29T14:45:00+08:00", "tasks": [ { "id": 1, @@ -25,7 +25,7 @@ "title": "保留旧 payload 读取兼容逻辑，并验证新旧结构均可恢复全量行", "module": "similarasin", "dependency": "2", - "status": "pending" + "status": "done" }, { "id": 4,