From 3500feb921a914af538de2162f406821533439f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Tue, 20 Oct 2026 20:49:19 +0800 Subject: [PATCH] =?UTF-8?q?fix(brand-check):=20=E5=87=BA=E8=A1=A8=E6=95=B4?= =?UTF-8?q?=E8=AF=BB=E6=BA=90=E6=96=87=E4=BB=B6=E5=B1=95=E5=BC=80=E5=85=A8?= =?UTF-8?q?=E9=87=8F=E8=A1=8C=E2=80=94=E2=80=94=E5=90=8C=E5=93=81=E7=89=8C?= =?UTF-8?q?=E4=B8=8D=E5=90=8C=20ASIN=20=E4=B8=8D=E5=86=8D=E8=A2=AB?= =?UTF-8?q?=E5=88=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 任务 2735 复盘:品牌检测解析阶段按品牌删行后,同一份行集既当执行方查询清单 又当出表数据源,904 行源文件结果只剩 173 行。现解析器加 dedupeRowsByBrand 开关,payload 仍逐品牌一行(执行方按行提品牌,不重烧代理),组装时用真实源 文件整读展开全量行;不符合/未检测品牌的挪 sheet 语义不变。 新增 BrandTaskServiceResultRowsTest 4 例,Abort 夹具换真实 Excel 并钉双 ASIN 保行。 --- .../brand/service/BrandTaskService.java | 45 ++-- .../service/BrandTaskAbortServiceTest.java | 30 ++- .../BrandTaskServiceResultRowsTest.java | 219 ++++++++++++++++++ 3 files changed, 278 insertions(+), 16 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskServiceResultRowsTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java index 1f3642bb4..5e152fe69 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java @@ -770,7 +770,10 @@ public class BrandTaskService { File sourceLocalFile = resolveSourceFile(sourceFile); String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile); File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename)); - writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate, undetectedBrands); + // 出表整读源文件展开全量行:缓存 rows 是按品牌去重的查询集(执行方靠它逐品牌只查一次), + // 同品牌不同 ASIN 是不同商品,结果文件删不得。 + List> outputRows = parseBrandFile(sourceLocalFile, false).rows(); + writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate, undetectedBrands, outputRows); outputEntries.add(new OutputEntry(sourceLocalFile, originalFilename, outputFile, originalFilename)); } if (outputEntries.isEmpty()) { @@ -1033,7 +1036,16 @@ public class BrandTaskService { return blankToDefault(sourceFile.getOriginalFilename(), inputFile.getName()); } - private ParsedBrandFile parseBrandFile(File inputFile) { + ParsedBrandFile parseBrandFile(File inputFile) { + return parseBrandFile(inputFile, true); + } + + /** + * @param dedupeRowsByBrand 真=回传执行方的 payload 形态:每品牌只留首行,保证逐品牌只查一次 + * (执行方从 rows 提品牌清单,放开会按行重烧按次计费的代理); + * 假=出表形态:同品牌不同 ASIN 是不同商品,结果文件必须每行都在。 + */ + ParsedBrandFile parseBrandFile(File inputFile, boolean dedupeRowsByBrand) { // 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。 // 此前 WorkbookFactory.create 把整表读成 DOM,品牌源文件(几十万行)会整表驻留堆内存。 // 解析结果(columns/rows/uniqueBrands/行号)与错误文案不变,行号沿用 sheet 绝对 0 基行号 +1。 @@ -1041,7 +1053,7 @@ public class BrandTaskService { log.warn("[brand] 源文件不是 Excel(疑似 CSV/文本),拒绝解析 file={}", inputFile.getName()); throw new BusinessException("读取 Excel 失败"); } - BrandFileSheetContext context = new BrandFileSheetContext(); + BrandFileSheetContext context = new BrandFileSheetContext(dedupeRowsByBrand); try { ExcelStreamReader.readFirstSheet(inputFile, context); } catch (BusinessException ex) { @@ -1072,15 +1084,20 @@ public class BrandTaskService { } } - /** 品牌源文件流式解析上下文:表头解析 + 按品牌去重累积结果行(内存不随整表 DOM 放大)。 */ + /** 品牌源文件流式解析上下文:表头解析 + 结果行累积(内存不随整表 DOM 放大)。 */ private final class BrandFileSheetContext implements ExcelStreamReader.SheetRowHandler { + private final boolean dedupeRowsByBrand; private final List> rows = new ArrayList<>(); private final Set uniqueBrands = new LinkedHashSet<>(); private List columns; private Map headerIndexes; private String sheetName = ""; + BrandFileSheetContext(boolean dedupeRowsByBrand) { + this.dedupeRowsByBrand = dedupeRowsByBrand; + } + @Override public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { this.sheetName = sheetName == null ? "" : sheetName; @@ -1136,7 +1153,8 @@ public class BrandTaskService { rows.add(rowData); return; } - if (uniqueBrands.add(brand)) { + boolean firstSeen = uniqueBrands.add(brand); + if (firstSeen || !dedupeRowsByBrand) { rowData.put("__rowIndex", rowIndex + 1); rows.add(rowData); } @@ -1188,11 +1206,12 @@ public class BrandTaskService { .replaceAll("\\s+", " "); } - private void writeBrandWorkbook(File outputFile, - String strategy, - BrandParsedFileCacheDto cachedFile, - BrandFileAggregateCacheDto resultFile, - List undetectedBrands) throws IOException { + void writeBrandWorkbook(File outputFile, + String strategy, + BrandParsedFileCacheDto cachedFile, + BrandFileAggregateCacheDto resultFile, + List undetectedBrands, + List> outputRows) throws IOException { String actualStrategy = normalizeStrategy(strategy); SXSSFWorkbook workbook = new SXSSFWorkbook(200); workbook.setCompressTempFiles(true); @@ -1210,7 +1229,7 @@ public class BrandTaskService { // 避免用户把「没查过」的行误当成「已通过检测」上架 Set undetectedBrandSet = normalizeBrandSet(undetectedBrands); List> undetectedRows = new ArrayList<>(); - List> sourceRows = cachedFile.getRows() == null ? List.of() : cachedFile.getRows(); + List> sourceRows = outputRows == null ? List.of() : outputRows; int writeRowIndex = 1; for (Map rowData : sourceRows) { String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), "")); @@ -1713,8 +1732,8 @@ public class BrandTaskService { brandTaskProgressCacheService.markFailed(taskId, message); } - private record ParsedBrandFile(String sheetName, List columns, List> rows, - List uniqueBrands) { + record ParsedBrandFile(String sheetName, List columns, List> rows, + List uniqueBrands) { } private record OutputEntry(File sourceFile, String zipSourceFilename, File resultFile, String zipResultFilename) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java index 4a6121c6b..140ffed30 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java @@ -27,7 +27,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import java.io.File; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -171,7 +170,13 @@ class BrandTaskAbortServiceTest { private void assertPartialAssembly(String initialStatus) throws Exception { File sourceFile = tempDir.resolve("source.xlsx").toFile(); - Files.write(sourceFile.toPath(), new byte[]{1, 2, 3}); + writeSourceWorkbook(sourceFile, new String[][]{ + {"品牌", "ASIN"}, + {"A", "B0A1"}, + {"A", "B0A2"}, + {"B", "B0B1"}, + {"C", "B0C1"}, + }); String fileUrl = sourceFile.getAbsolutePath(); when(brandCrawlTaskMapper.selectById(2309L)).thenReturn(task(initialStatus, fileUrl)); TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class); @@ -222,7 +227,26 @@ class BrandTaskAbortServiceTest { assertNotNull(undetected, "部分结果应包含「未检测品牌」sheet"); assertEquals("B", undetected.getRow(1).getCell(0).getStringCellValue()); assertEquals("C", undetected.getRow(2).getCell(0).getStringCellValue()); - assertEquals(1, workbook.getSheet("Sheet1").getLastRowNum(), "主 sheet 只剩表头 + 已检测品牌 A 一行"); + org.apache.poi.ss.usermodel.Sheet main = workbook.getSheet("Sheet1"); + assertEquals(2, main.getLastRowNum(), + "主 sheet = 表头 + 已检测品牌 A 的全部行:同品牌两个 ASIN 都必须在(2735 复盘)"); + assertEquals("B0A1", main.getRow(1).getCell(1).getStringCellValue()); + assertEquals("B0A2", main.getRow(2).getCell(1).getStringCellValue()); + } + + private void writeSourceWorkbook(File file, String[][] data) throws Exception { + try (org.apache.poi.xssf.usermodel.XSSFWorkbook workbook = new org.apache.poi.xssf.usermodel.XSSFWorkbook()) { + org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet("Sheet1"); + for (int r = 0; r < data.length; r++) { + org.apache.poi.ss.usermodel.Row row = sheet.createRow(r); + for (int c = 0; c < data[r].length; c++) { + row.createCell(c).setCellValue(data[r][c]); + } + } + try (java.io.FileOutputStream out = new java.io.FileOutputStream(file)) { + workbook.write(out); + } + } } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskServiceResultRowsTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskServiceResultRowsTest.java new file mode 100644 index 000000000..c2979e3fd --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskServiceResultRowsTest.java @@ -0,0 +1,219 @@ +package com.nanri.aiimage.modules.brand.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.BrandProgressProperties; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; +import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandInvalidBrandDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto; +import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +/** + * 品牌检测结果出表的行数契约(任务 2735 复盘):查询按品牌去重(同品牌只查一次 WIPO), + * 但同品牌不同 ASIN 是不同商品行,结果文件一行都不能少。 + */ +@ExtendWith(MockitoExtension.class) +class BrandTaskServiceResultRowsTest { + + @Mock private BrandCrawlTaskMapper brandCrawlTaskMapper; + @Mock private OssStorageService ossStorageService; + @Mock private StorageProperties storageProperties; + @Mock private BrandProgressProperties brandProgressProperties; + @Mock private BrandTaskProgressCacheService brandTaskProgressCacheService; + @Mock private BrandTaskStorageService brandTaskStorageService; + @Mock private DistributedJobLockService distributedJobLockService; + @Mock private TaskDistributedLockService taskDistributedLockService; + @Mock private TaskFileJobService taskFileJobService; + @Mock private TaskProgressSnapshotService taskProgressSnapshotService; + + @TempDir + Path tempDir; + + @BeforeAll + static void initializeTableInfo() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + BrandCrawlTaskEntity.class); + } + + private BrandTaskService service() { + return new BrandTaskService(brandCrawlTaskMapper, ossStorageService, storageProperties, + brandProgressProperties, brandTaskProgressCacheService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), + brandTaskStorageService, + distributedJobLockService, taskDistributedLockService, new ObjectMapper(), + taskFileJobService, taskProgressSnapshotService, + mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class)); + } + + /** 4 数据行:AA 两个不同 ASIN(同品牌)、BB、CC 各一行 */ + private File sourceWorkbook() throws Exception { + File file = tempDir.resolve("brand-source.xlsx").toFile(); + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("Sheet1"); + String[][] data = { + {"品牌", "ASIH", "状态", "时间"}, + {"Brand: AA", "B001", "", ""}, + {"Brand: AA", "B002", "", ""}, + {"Brand: BB", "B003", "", ""}, + {"Brand: CC", "B004", "", ""}, + }; + for (int r = 0; r < data.length; r++) { + Row row = sheet.createRow(r); + for (int c = 0; c < data[r].length; c++) { + row.createCell(c).setCellValue(data[r][c]); + } + } + try (FileOutputStream out = new FileOutputStream(file)) { + workbook.write(out); + } + } + return file; + } + + private BrandParsedFileCacheDto cachedFile(BrandTaskService.ParsedBrandFile deduped) { + BrandParsedFileCacheDto dto = new BrandParsedFileCacheDto(); + dto.setFileUrl("https://oss.example/brand-source.xlsx"); + dto.setSheetName(deduped.sheetName()); + dto.setColumns(deduped.columns()); + dto.setRows(deduped.rows()); + return dto; + } + + private BrandFileAggregateCacheDto aggregateWithInvalid(String brand) { + BrandFileAggregateCacheDto dto = new BrandFileAggregateCacheDto(); + dto.setFileUrl("https://oss.example/brand-source.xlsx"); + BrandInvalidBrandDto invalid = new BrandInvalidBrandDto(); + invalid.setBrand(brand); + invalid.setCountry("英国"); + invalid.setStatus("已注册"); + dto.setInvalidBrands(new ArrayList<>(List.of(invalid))); + return dto; + } + + private List sheetValues(Sheet sheet, int column) { + List values = new ArrayList<>(); + for (int r = 1; r <= sheet.getLastRowNum(); r++) { + Row row = sheet.getRow(r); + if (row != null && row.getCell(column) != null) { + values.add(row.getCell(column).getStringCellValue()); + } + } + return values; + } + + @Test + void payloadParseDedupesRowsButExpansionParseKeepsEveryRow() throws Exception { + File file = sourceWorkbook(); + BrandTaskService service = service(); + + BrandTaskService.ParsedBrandFile forPayload = service.parseBrandFile(file); + assertEquals(3, forPayload.rows().size(), "回传执行方的行集必须每品牌一行,防按行重查烧代理"); + assertEquals(3, forPayload.uniqueBrands().size()); + + BrandTaskService.ParsedBrandFile forOutput = service.parseBrandFile(file, false); + assertEquals(4, forOutput.rows().size(), "出表行集要保留同品牌不同 ASIN 的全部行"); + assertEquals(3, forOutput.uniqueBrands().size(), "查询集仍按品牌去重,与出表模式一致"); + } + + @Test + void resultWorkbookKeepsEveryAsinRowAndOnlyRemovesJudgedBrands() throws Exception { + File file = sourceWorkbook(); + BrandTaskService service = service(); + BrandTaskService.ParsedBrandFile deduped = service.parseBrandFile(file); + BrandTaskService.ParsedBrandFile expanded = service.parseBrandFile(file, false); + + File output = tempDir.resolve("brand-result.xlsx").toFile(); + service.writeBrandWorkbook(output, "Simple", cachedFile(deduped), aggregateWithInvalid("Brand: CC"), + List.of(), expanded.rows()); + + try (Workbook workbook = WorkbookFactory.create(output)) { + Sheet main = workbook.getSheet("Sheet1"); + assertNotNull(main); + List asins = sheetValues(main, 1); + assertEquals(List.of("B001", "B002", "B003"), asins, + "同品牌不同 ASIN 的行都要出;仅不符合品牌 CC 的行移出主表"); + Sheet invalid = workbook.getSheet("不符合品牌"); + assertNotNull(invalid); + assertEquals(List.of("Brand: CC"), sheetValues(invalid, 0)); + assertNull(workbook.getSheet("未检测品牌"), "成功收尾没有未检测 sheet"); + } + } + + @Test + void partialResultMovesAllUndetectedRowsToTheirOwnSheet() throws Exception { + File file = sourceWorkbook(); + BrandTaskService service = service(); + BrandTaskService.ParsedBrandFile deduped = service.parseBrandFile(file); + BrandTaskService.ParsedBrandFile expanded = service.parseBrandFile(file, false); + + File output = tempDir.resolve("brand-partial.xlsx").toFile(); + service.writeBrandWorkbook(output, "Simple", cachedFile(deduped), aggregateWithInvalid("Brand: CC"), + List.of("Brand: BB"), expanded.rows()); + + try (Workbook workbook = WorkbookFactory.create(output)) { + Sheet main = workbook.getSheet("Sheet1"); + assertEquals(List.of("B001", "B002"), sheetValues(main, 1)); + Sheet undetected = workbook.getSheet("未检测品牌"); + assertNotNull(undetected, "未检测品牌的行要整行挪进独立 sheet"); + assertEquals(List.of("B003"), sheetValues(undetected, 1)); + } + } + + @Test + void blankBrandRowsAreAlwaysKeptInBothModes() throws Exception { + File file = tempDir.resolve("brand-blank.xlsx").toFile(); + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("Sheet1"); + String[][] data = { + {"品牌", "ASIH", "状态", "时间"}, + {"", "B001", "", ""}, + {"", "B002", "", ""}, + }; + for (int r = 0; r < data.length; r++) { + Row row = sheet.createRow(r); + for (int c = 0; c < data[r].length; c++) { + row.createCell(c).setCellValue(data[r][c]); + } + } + try (FileOutputStream out = new FileOutputStream(file)) { + workbook.write(out); + } + } + BrandTaskService service = service(); + assertEquals(2, service.parseBrandFile(file).rows().size(), "空品牌行不参与品牌去重,两种模式都保留"); + assertEquals(2, service.parseBrandFile(file, false).rows().size()); + assertEquals(0, service.parseBrandFile(file).uniqueBrands().size()); + } +}