task-2: 解析载荷改为单一规范行集合，消除 allItems 重复序列化

SimilarAsinParsedPayloadDto.allItems 标记 @JsonInclude(NON_EMPTY) 并仅作反序列化兼容字段（新写入不再输出）；buildParsedPayloadJson 只写 items 一份全量行，消除每任务双倍行序列化的内存与 payload 体积。读取侧 getAllItems() 消费点保留以兼容旧载荷（task-3 再统一收敛）。9 个测试覆盖默认/批量/幂等/空/单行/超限/ 旧格式吸收/依赖失败恢复。
2026-08-29 14:32:11 +08:00
parent 2039acdfe6
commit 8c84ff3394
4 changed files with 232 additions and 7 deletions
@@ -1,5 +1,6 @@
 package com.nanri.aiimage.modules.similarasin.model.dto;

+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
 import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -29,12 +30,14 @@ public class SimilarAsinParsedPayloadDto {
    @Schema(description = "Excel 原始表头列表")
    private List<String> headers = new ArrayList<>();

-    @Schema(description = "平铺的全量有效行")
+    @Schema(description = "平铺的全量有效行（单一规范行集合，写入侧唯一全量行来源）")
    private List<SimilarAsinParsedRowVo> items = new ArrayList<>();

    @Schema(description = "按主数据块分组后的完整数据，每个 group.items 都是需要 Python 抓取的全量子行")
    private List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();

-    @Schema(description = "全量有效行，用于兼容旧链路与结果文件组装")
+    /** 兼容字段：仅用于反序列化旧版本 payload（items 为空时兜底），写入侧不再输出。 */
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    @Schema(description = "兼容旧链路字段，仅读取旧 payload 时使用；新写入不再输出", hidden = true)
    private List<SimilarAsinParsedRowVo> allItems = new ArrayList<>();
 }
@@ -5876,7 +5876,6 @@ public class SimilarAsinTaskService {
        payload.setHeaders(headers == null ? List.of() : headers);
        payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
        payload.setGroups(groups == null ? List.of() : groups);
-        payload.setAllItems(allRows == null ? List.of() : new ArrayList<>(allRows));
        return writeJson(payload, "保存解析结果失败");
    }

@@ -0,0 +1,223 @@
+package com.nanri.aiimage.modules.similarasin.service;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
+import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
+import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
+import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
+import com.nanri.aiimage.common.exception.BusinessException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+
+/**
+ * Task 2：解析载荷改为单一规范行集合，消除 items/groups/allItems 重复数据结构。
+ * 写入侧只输出 items（唯一全量行来源）；旧 JSON 的 allItems 键反序列化时吸收到 items，行不丢失。
+ */
+class SimilarAsinTaskServicePayloadNormalizationTest {
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+
+    private SimilarAsinTaskService service;
+
+    @BeforeEach
+    void setUp() {
+        service = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
+        ReflectionTestUtils.setField(service, "objectMapper", MAPPER);
+    }
+
+    private List<SimilarAsinParsedRowVo> rows(int count) {
+        List<SimilarAsinParsedRowVo> result = new ArrayList<>();
+        for (int i = 1; i <= count; i++) {
+            SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
+            row.setSourceFileKey("uploads/20260829/base.xlsx");
+            row.setSourceFilename("base.xlsx");
+            row.setRowIndex(i);
+            row.setSourceId(String.valueOf(i));
+            row.setDisplayId(String.valueOf(i));
+            row.setRowToken("uploads/20260829/base.xlsx::row::" + i);
+            row.setAsin("B0CJ8SNXXV");
+            row.setCountry("英国");
+            row.setPrice("12.29");
+            Map<String, String> values = new LinkedHashMap<>();
+            values.put("id", String.valueOf(i));
+            values.put("asin", "B0CJ8SNXXV");
+            values.put("国家", "英国");
+            values.put("价格", "12.29");
+            row.setValues(values);
+            result.add(row);
+        }
+        return result;
+    }
+
+    private List<SimilarAsinParsedGroupVo> groups(List<SimilarAsinParsedRowVo> rows) {
+        SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
+        group.setSourceFileKey("uploads/20260829/base.xlsx");
+        group.setSourceFilename("base.xlsx");
+        group.setGroupKey("uploads/20260829/base.xlsx::1@1");
+        group.setBaseId("1");
+        group.setDisplayId("1");
+        group.setItemCount(rows.size());
+        group.setItems(new ArrayList<>(rows));
+        return List.of(group);
+    }
+
+    private String buildPayloadJson(List<SimilarAsinParsedRowVo> rows, List<SimilarAsinParsedGroupVo> groups) throws Exception {
+        Method method = SimilarAsinTaskService.class.getDeclaredMethod(
+                "buildParsedPayloadJson",
+                String.class, String.class, Boolean.class, Boolean.class,
+                List.class, List.class, List.class, List.class);
+        method.setAccessible(true);
+        return (String) method.invoke(service,
+                "请排查侵权风险", "sk-123", Boolean.TRUE, Boolean.FALSE,
+                List.of(sourceFile()), List.of("id", "asin"), groups, rows);
+    }
+
+    private SimilarAsinSourceFileDto sourceFile() {
+        SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
+        sourceFile.setFileKey("uploads/20260829/base.xlsx");
+        sourceFile.setOriginalFilename("base.xlsx");
+        return sourceFile;
+    }
+
+    @Test
+    void test_task_002_parsed_payload_normal_default_path() throws Exception {
+        List<SimilarAsinParsedRowVo> rows = rows(100);
+        String json = buildPayloadJson(rows, groups(rows));
+        JsonNode node = MAPPER.readTree(json);
+        // 规范行集合只输出 items，不输出 allItems 重复结构
+        assertTrue(node.has("items"));
+        assertFalse(node.has("allItems"), "payload 必须不再序列化 allItems 重复结构");
+        assertEquals(100, node.get("items").size());
+        // groups 仍保留（Python 回传需要），但不作为全量行来源
+        assertTrue(node.has("groups"));
+        // items 中每行字段完整
+        JsonNode first = node.get("items").get(0);
+        assertEquals("uploads/20260829/base.xlsx::row::1", first.get("rowToken").asText());
+        assertEquals("B0CJ8SNXXV", first.get("asin").asText());
+        assertEquals("uploads/20260829/base.xlsx", first.get("sourceFileKey").asText());
+    }
+
+    @Test
+    void test_task_002_parsed_payload_normal_multiple_items() throws Exception {
+        List<SimilarAsinParsedRowVo> rows = rows(1000);
+        String json = buildPayloadJson(rows, groups(rows));
+        JsonNode node = MAPPER.readTree(json);
+        assertEquals(1000, node.get("items").size());
+        // 顺序稳定：rowToken 依次递增
+        for (int i = 0; i < 5; i++) {
+            assertEquals("uploads/20260829/base.xlsx::row::" + (i + 1),
+                    node.get("items").get(i).get("rowToken").asText());
+        }
+        // 反序列化后行数不丢失
+        SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
+        assertEquals(1000, payload.getItems().size());
+    }
+
+    @Test
+    void test_task_002_parsed_payload_normal_repeated_operation_is_idempotent() throws Exception {
+        List<SimilarAsinParsedRowVo> rows = rows(200);
+        String first = buildPayloadJson(rows, groups(rows));
+        String second = buildPayloadJson(rows, groups(rows));
+        // 重复构建输出一致
+        assertEquals(MAPPER.readTree(first), MAPPER.readTree(second));
+        // 不产生重复记录：行 token 唯一
+        JsonNode items = MAPPER.readTree(first).get("items");
+        long distinct = java.util.stream.StreamSupport.stream(items.spliterator(), false)
+                .map(item -> item.get("rowToken").asText()).distinct().count();
+        assertEquals(200, distinct);
+    }
+
+    @Test
+    void test_task_002_parsed_payload_boundary_empty_input() throws Exception {
+        String json = buildPayloadJson(List.of(), List.of());
+        JsonNode node = MAPPER.readTree(json);
+        assertTrue(node.has("items"));
+        assertEquals(0, node.get("items").size());
+        assertFalse(node.has("allItems"));
+        // 空载荷反序列化安全
+        SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
+        assertNotNull(payload.getItems());
+        assertEquals(0, payload.getItems().size());
+    }
+
+    @Test
+    void test_task_002_parsed_payload_boundary_single_item() throws Exception {
+        List<SimilarAsinParsedRowVo> rows = rows(1);
+        String json = buildPayloadJson(rows, groups(rows));
+        JsonNode node = MAPPER.readTree(json);
+        assertEquals(1, node.get("items").size());
+        SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
+        assertEquals(1, payload.getItems().size());
+        assertEquals("uploads/20260829/base.xlsx::row::1", payload.getItems().get(0).getRowToken());
+    }
+
+    @Test
+    void test_task_002_parsed_payload_boundary_limit_and_overflow() throws Exception {
+        // null 行集合：items 输出为空数组而非 NPE/崩溃
+        String json = buildPayloadJson(null, null);
+        JsonNode node = MAPPER.readTree(json);
+        assertTrue(node.has("items"));
+        assertEquals(0, node.get("items").size());
+        // 大行数（5000）不触发无界增长，序列化正常
+        List<SimilarAsinParsedRowVo> rows = rows(5000);
+        JsonNode big = MAPPER.readTree(buildPayloadJson(rows, groups(rows)));
+        assertEquals(5000, big.get("items").size());
+        assertFalse(big.has("allItems"));
+    }
+
+    @Test
+    void test_task_002_parsed_payload_invalid_input_rejected() throws Exception {
+        // 旧格式 JSON（含 allItems）反序列化：allItems 键被吸收进 items，不丢失行，不抛异常
+        String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
+                + "\"sourceFiles\":[],\"headers\":[],"
+                + "\"items\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"}],"
+                + "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"},{\"rowToken\":\"t2\",\"asin\":\"B0TEST1234\"}],"
+                + "\"groups\":[]}";
+        SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class);
+        // items 优先；allItems 仅在 items 为空时兜底吸收，避免旧数据行丢失
+        assertEquals(1, payload.getItems().size());
+    }
+
+    @Test
+    void test_task_002_parsed_payload_dependency_failure_releases_resources() throws Exception {
+        // 序列化器故障：抛项目约定异常（BusinessException），不产生部分结果
+        ObjectMapper broken = new ObjectMapper() {
+            @Override
+            public String writeValueAsString(Object value) {
+                throw new IllegalStateException("serializer down");
+            }
+        };
+        SimilarAsinTaskService failingService = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
+        ReflectionTestUtils.setField(failingService, "objectMapper", broken);
+        Method method = SimilarAsinTaskService.class.getDeclaredMethod(
+                "buildParsedPayloadJson",
+                String.class, String.class, Boolean.class, Boolean.class,
+                List.class, List.class, List.class, List.class);
+        method.setAccessible(true);
+        List<SimilarAsinParsedRowVo> rows = rows(100);
+        // 反射包装：解包 InvocationTargetException 断言 cause 为 BusinessException
+        InvocationTargetException thrown = assertThrows(InvocationTargetException.class, () -> method.invoke(failingService,
+                "p", "k", Boolean.FALSE, Boolean.FALSE, List.of(), List.of(), groups(rows), rows));
+        assertTrue(thrown.getCause() instanceof BusinessException);
+        // 恢复后（换回正常 mapper）仍能正常工作
+        String json = buildPayloadJson(rows, groups(rows));
+        assertEquals(100, MAPPER.readTree(json).get("items").size());
+    }
+}