diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java index 1ab7e4fbb..533730fdc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java @@ -190,13 +190,22 @@ public class AppearancePatentExcelParser { values)); } + /** + * 有效表头列数:只统计「非空表头」的最大列索引 + 1。表头行被整行格式污染时(整行被实例化成 + * 上万空单元格),若按最大单元格列号展开,每行 values 都会被填充上万条占位空值,解析结果与 + * 载荷膨胀上千倍(同源事故见 SimilarAsinExcelParser,2026-09-26)。 + */ private int lastColumnCount(Map rowMap) { if (rowMap == null || rowMap.isEmpty()) { return 0; } int lastColumnCount = 0; - for (Integer columnIndex : rowMap.keySet()) { - if (columnIndex != null && columnIndex >= lastColumnCount) { + for (Map.Entry entry : rowMap.entrySet()) { + Integer columnIndex = entry.getKey(); + if (columnIndex == null || columnIndex < 0 || normalize(entry.getValue()).isBlank()) { + continue; + } + if (columnIndex >= lastColumnCount) { lastColumnCount = columnIndex + 1; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index 6640c69d0..b9ed424db 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -1896,12 +1896,22 @@ public class CollectDataService { if (headerMap == null || headerMap.isEmpty()) { throw new BusinessException("Excel 表头为空"); } + // 列范围只按「非空表头」计算:表头行被整行格式污染时(整行被实例化成上万空单元格), + // 若按最大单元格列号展开,每行 extra 都会被填充上万条占位空值,解析结果与载荷膨胀上千倍 + // (同源事故见 SimilarAsinExcelParser)。尾部空列不再生成占位列。 int lastCol = -1; - for (Integer col : headerMap.keySet()) { - if (col != null && col > lastCol) { + for (Map.Entry entry : headerMap.entrySet()) { + Integer col = entry.getKey(); + if (col == null || col < 0 || normalize(entry.getValue()).isBlank()) { + continue; + } + if (col > lastCol) { lastCol = col; } } + if (lastCol < 0) { + throw new BusinessException("Excel 表头为空"); + } for (int i = 0; i <= lastCol; i++) { String value = normalize(headerMap.getOrDefault(i, "")); headers.add(value.isBlank() ? "列" + (i + 1) : value); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParser.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParser.java index f1d272a53..4bc94623a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParser.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParser.java @@ -120,12 +120,25 @@ public class SimilarAsinExcelParser { // EasyExcel 回调给出 列号 → 表头文本,与 POI Row 遍历等价(缺列一般为 null/空串) Map map = new LinkedHashMap<>(); List headerNames = new ArrayList<>(); + // 列范围只按「非空表头」计算:表头行被整行格式污染时(如 A1:XFD1 整行被实例化成 + // 16384 个空单元格,只有前几列有值),若按最大单元格列号展开,headers 会撑到 16384 列, + // 每行数据都被填充上万条占位空值,payload 膨胀上千倍(2026-09-26 生产事故:142KB 的 + // xlsx 生成 2.3GB 载荷,建任务 500)。尾部空列不再生成占位列。 int lastCol = -1; for (Map.Entry e : headerMapRaw.entrySet()) { - if (e.getKey() != null && e.getKey() > lastCol) { + if (e.getKey() == null || e.getKey() < 0) { + continue; + } + if (normalize(e.getValue()).isBlank()) { + continue; + } + if (e.getKey() > lastCol) { lastCol = e.getKey(); } } + if (lastCol < 0) { + throw new BusinessException("Excel 表头为空"); + } for (int i = 0; i <= lastCol; i++) { String val = normalize(headerMapRaw.getOrDefault(i, "")); headerNames.add(val.isBlank() ? "列" + (i + 1) : val); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParserTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParserTest.java index 5851a567e..106e37608 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParserTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParserTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import java.io.File; import java.io.FileOutputStream; import java.util.List; +import java.util.Objects; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -164,4 +165,21 @@ class AppearancePatentExcelParserTest { assertEquals("asin", parsed.headers().get(3), "重复表头保留"); assertEquals("B01A", parsed.rows().get(0).asin(), "重复列取首个匹配列"); } + + @Test + void test_parser_ignores_trailing_blank_header_columns_from_row_formatting() throws Exception { + // 2026-09-26 生产事故样本:表头行被整行格式污染成 16384 个单元格(只有前 4 列有值), + // 列范围必须收缩到最后一个非空表头,否则每行 values 被填充 1.6 万个占位空值、载荷膨胀上千倍 + File file = new File(Objects.requireNonNull( + getClass().getResource("/excel/header-row-format-pollution.xlsx")).toURI()); + + AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file); + + assertEquals(4, parsed.headers().size(), "尾部空列不应扩展表头范围"); + assertEquals(2075, parsed.rows().size()); + AppearancePatentExcelParser.AppearanceExcelRow first = parsed.rows().get(0); + assertEquals(4, first.values().size(), "行内值不应包含表头范围外的占位空列"); + assertEquals("B0G2XX6GD5", first.asin()); + assertEquals("英国", first.country()); + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataHeaderPollutionTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataHeaderPollutionTest.java new file mode 100644 index 000000000..cfd1b0163 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataHeaderPollutionTest.java @@ -0,0 +1,56 @@ +package com.nanri.aiimage.modules.collectdata.service; + +import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSourceFileDto; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.File; +import java.net.URL; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * 集采解析回归:表头行被整行格式污染时(行被实例化为 Excel 最大列数 16384 个空单元格, + * 只有前 4 列有值),列范围必须收缩到最后一个非空表头,否则每行 extra 都被填充 1.6 万个 + * 占位空值,解析结果与载荷膨胀上千倍。样本来自 2026-09-26 生产事故(同源修复见 + * SimilarAsinExcelParser / AppearancePatentExcelParser)。 + */ +@ExtendWith(MockitoExtension.class) +class CollectDataHeaderPollutionTest { + + @InjectMocks + private CollectDataService service; + + @Test + void 表头行整行格式污染时行内不应出现上万占位列() throws Exception { + URL url = getClass().getResource("/excel/header-row-format-pollution-keyword.xlsx"); + assertNotNull(url, "测试样本缺失"); + File file = new File(url.toURI()); + CollectDataSourceFileDto source = new CollectDataSourceFileDto(); + source.setFileKey("pollution-key"); + source.setOriginalFilename("header-row-format-pollution-keyword.xlsx"); + + Object parsed = ReflectionTestUtils.invokeMethod(service, "parseWorkbook", file, source); + assertNotNull(parsed, "parseWorkbook 应返回结果"); + + Integer totalRows = (Integer) ReflectionTestUtils.invokeMethod(parsed, "totalRows"); + List rows = (List) ReflectionTestUtils.invokeMethod(parsed, "rows"); + assertNotNull(rows); + System.out.println("[测试] totalRows=" + totalRows + " rows=" + rows.size()); + + assertEquals(2075, totalRows, "数据行数应与样本一致"); + Map extra = (Map) ReflectionTestUtils.invokeMethod(rows.get(0), "extra"); + assertNotNull(extra); + // 样本表头 关键词/asin/国家/价格:关键词列为 keyword 不进 extra,其余 3 列进 extra + assertEquals(3, extra.size(), "行内 extra 不应包含表头范围外的占位空列"); + assertFalse(extra.containsKey("列5"), "尾部空列不应生成占位列"); + assertFalse(extra.containsKey("列16384"), "尾部空列不应生成占位列"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParserTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParserTest.java index b5c7b73b8..f1b86f307 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParserTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinExcelParserTest.java @@ -11,7 +11,9 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; +import java.net.URL; import java.nio.file.Files; +import java.util.Objects; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -196,4 +198,23 @@ class SimilarAsinExcelParserTest { assertEquals(1, first.rows().size()); assertEquals("B01ABC", first.rows().get(0).asin()); } + + @Test + void test_parse_ignores_trailing_blank_header_columns_from_row_formatting() throws Exception { + // 2026-09-26 生产事故样本:表头行被整行格式污染成 16384 个单元格(只有前 4 列有值), + // 列范围必须收缩到最后一个非空表头,否则每行 values 被填充 1.6 万个占位空值、载荷膨胀上千倍 + URL url = getClass().getResource("/excel/header-row-format-pollution.xlsx"); + assertNotNull(url, "测试样本缺失"); + File file = new File(Objects.requireNonNull(url).toURI()); + + SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file); + + assertEquals(4, parsed.headers().size(), "尾部空列不应扩展表头范围"); + assertEquals(2075, parsed.rows().size(), "数据行数应与样本一致"); + SimilarAsinExcelParser.SimilarAsinExcelRow first = parsed.rows().get(0); + assertEquals(4, first.values().size(), "行内值不应包含表头范围外的占位空列"); + assertEquals("B0G2XX6GD5", first.asin()); + assertEquals("英国", first.country()); + assertEquals("89.34", first.price()); + } } diff --git a/backend-java/src/test/resources/excel/header-row-format-pollution-keyword.xlsx b/backend-java/src/test/resources/excel/header-row-format-pollution-keyword.xlsx new file mode 100644 index 000000000..9f7cad21c Binary files /dev/null and b/backend-java/src/test/resources/excel/header-row-format-pollution-keyword.xlsx differ diff --git a/backend-java/src/test/resources/excel/header-row-format-pollution.xlsx b/backend-java/src/test/resources/excel/header-row-format-pollution.xlsx new file mode 100644 index 000000000..bb726eee3 Binary files /dev/null and b/backend-java/src/test/resources/excel/header-row-format-pollution.xlsx differ