fix(brand-check): 出表整读源文件展开全量行——同品牌不同 ASIN 不再被删

任务 2735 复盘:品牌检测解析阶段按品牌删行后,同一份行集既当执行方查询清单
又当出表数据源,904 行源文件结果只剩 173 行。现解析器加 dedupeRowsByBrand
开关,payload 仍逐品牌一行(执行方按行提品牌,不重烧代理),组装时用真实源
文件整读展开全量行;不符合/未检测品牌的挪 sheet 语义不变。
新增 BrandTaskServiceResultRowsTest 4 例,Abort 夹具换真实 Excel 并钉双 ASIN 保行。
This commit is contained in:
2026-10-20 20:49:19 +08:00
parent e71a1077af
commit 3500feb921
3 changed files with 278 additions and 16 deletions
@@ -770,7 +770,10 @@ public class BrandTaskService {
File sourceLocalFile = resolveSourceFile(sourceFile);
String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile);
File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename));
writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate, undetectedBrands);
// 出表整读源文件展开全量行:缓存 rows 是按品牌去重的查询集(执行方靠它逐品牌只查一次),
// 同品牌不同 ASIN 是不同商品,结果文件删不得。
List<Map<String, Object>> outputRows = parseBrandFile(sourceLocalFile, false).rows();
writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate, undetectedBrands, outputRows);
outputEntries.add(new OutputEntry(sourceLocalFile, originalFilename, outputFile, originalFilename));
}
if (outputEntries.isEmpty()) {
@@ -1033,7 +1036,16 @@ public class BrandTaskService {
return blankToDefault(sourceFile.getOriginalFilename(), inputFile.getName());
}
private ParsedBrandFile parseBrandFile(File inputFile) {
ParsedBrandFile parseBrandFile(File inputFile) {
return parseBrandFile(inputFile, true);
}
/**
* @param dedupeRowsByBrand 真=回传执行方的 payload 形态:每品牌只留首行,保证逐品牌只查一次
* (执行方从 rows 提品牌清单,放开会按行重烧按次计费的代理);
* 假=出表形态:同品牌不同 ASIN 是不同商品,结果文件必须每行都在。
*/
ParsedBrandFile parseBrandFile(File inputFile, boolean dedupeRowsByBrand) {
// 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。
// 此前 WorkbookFactory.create 把整表读成 DOM,品牌源文件(几十万行)会整表驻留堆内存。
// 解析结果(columns/rows/uniqueBrands/行号)与错误文案不变,行号沿用 sheet 绝对 0 基行号 +1。
@@ -1041,7 +1053,7 @@ public class BrandTaskService {
log.warn("[brand] 源文件不是 Excel(疑似 CSV/文本),拒绝解析 file={}", inputFile.getName());
throw new BusinessException("读取 Excel 失败");
}
BrandFileSheetContext context = new BrandFileSheetContext();
BrandFileSheetContext context = new BrandFileSheetContext(dedupeRowsByBrand);
try {
ExcelStreamReader.readFirstSheet(inputFile, context);
} catch (BusinessException ex) {
@@ -1072,15 +1084,20 @@ public class BrandTaskService {
}
}
/** 品牌源文件流式解析上下文:表头解析 + 按品牌去重累积结果行(内存不随整表 DOM 放大)。 */
/** 品牌源文件流式解析上下文:表头解析 + 结果行累积(内存不随整表 DOM 放大)。 */
private final class BrandFileSheetContext implements ExcelStreamReader.SheetRowHandler {
private final boolean dedupeRowsByBrand;
private final List<Map<String, Object>> rows = new ArrayList<>();
private final Set<String> uniqueBrands = new LinkedHashSet<>();
private List<String> columns;
private Map<String, Integer> headerIndexes;
private String sheetName = "";
BrandFileSheetContext(boolean dedupeRowsByBrand) {
this.dedupeRowsByBrand = dedupeRowsByBrand;
}
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
this.sheetName = sheetName == null ? "" : sheetName;
@@ -1136,7 +1153,8 @@ public class BrandTaskService {
rows.add(rowData);
return;
}
if (uniqueBrands.add(brand)) {
boolean firstSeen = uniqueBrands.add(brand);
if (firstSeen || !dedupeRowsByBrand) {
rowData.put("__rowIndex", rowIndex + 1);
rows.add(rowData);
}
@@ -1188,11 +1206,12 @@ public class BrandTaskService {
.replaceAll("\\s+", " ");
}
private void writeBrandWorkbook(File outputFile,
String strategy,
BrandParsedFileCacheDto cachedFile,
BrandFileAggregateCacheDto resultFile,
List<String> undetectedBrands) throws IOException {
void writeBrandWorkbook(File outputFile,
String strategy,
BrandParsedFileCacheDto cachedFile,
BrandFileAggregateCacheDto resultFile,
List<String> undetectedBrands,
List<Map<String, Object>> outputRows) throws IOException {
String actualStrategy = normalizeStrategy(strategy);
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
@@ -1210,7 +1229,7 @@ public class BrandTaskService {
// 避免用户把「没查过」的行误当成「已通过检测」上架
Set<String> undetectedBrandSet = normalizeBrandSet(undetectedBrands);
List<Map<String, Object>> undetectedRows = new ArrayList<>();
List<Map<String, Object>> sourceRows = cachedFile.getRows() == null ? List.of() : cachedFile.getRows();
List<Map<String, Object>> sourceRows = outputRows == null ? List.of() : outputRows;
int writeRowIndex = 1;
for (Map<String, Object> rowData : sourceRows) {
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), ""));
@@ -1713,8 +1732,8 @@ public class BrandTaskService {
brandTaskProgressCacheService.markFailed(taskId, message);
}
private record ParsedBrandFile(String sheetName, List<String> columns, List<Map<String, Object>> rows,
List<String> uniqueBrands) {
record ParsedBrandFile(String sheetName, List<String> columns, List<Map<String, Object>> rows,
List<String> uniqueBrands) {
}
private record OutputEntry(File sourceFile, String zipSourceFilename, File resultFile, String zipResultFilename) {
@@ -27,7 +27,6 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -171,7 +170,13 @@ class BrandTaskAbortServiceTest {
private void assertPartialAssembly(String initialStatus) throws Exception {
File sourceFile = tempDir.resolve("source.xlsx").toFile();
Files.write(sourceFile.toPath(), new byte[]{1, 2, 3});
writeSourceWorkbook(sourceFile, new String[][]{
{"品牌", "ASIN"},
{"A", "B0A1"},
{"A", "B0A2"},
{"B", "B0B1"},
{"C", "B0C1"},
});
String fileUrl = sourceFile.getAbsolutePath();
when(brandCrawlTaskMapper.selectById(2309L)).thenReturn(task(initialStatus, fileUrl));
TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class);
@@ -222,7 +227,26 @@ class BrandTaskAbortServiceTest {
assertNotNull(undetected, "部分结果应包含「未检测品牌」sheet");
assertEquals("B", undetected.getRow(1).getCell(0).getStringCellValue());
assertEquals("C", undetected.getRow(2).getCell(0).getStringCellValue());
assertEquals(1, workbook.getSheet("Sheet1").getLastRowNum(), "主 sheet 只剩表头 + 已检测品牌 A 一行");
org.apache.poi.ss.usermodel.Sheet main = workbook.getSheet("Sheet1");
assertEquals(2, main.getLastRowNum(),
"主 sheet = 表头 + 已检测品牌 A 的全部行:同品牌两个 ASIN 都必须在(2735 复盘)");
assertEquals("B0A1", main.getRow(1).getCell(1).getStringCellValue());
assertEquals("B0A2", main.getRow(2).getCell(1).getStringCellValue());
}
private void writeSourceWorkbook(File file, String[][] data) throws Exception {
try (org.apache.poi.xssf.usermodel.XSSFWorkbook workbook = new org.apache.poi.xssf.usermodel.XSSFWorkbook()) {
org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet("Sheet1");
for (int r = 0; r < data.length; r++) {
org.apache.poi.ss.usermodel.Row row = sheet.createRow(r);
for (int c = 0; c < data[r].length; c++) {
row.createCell(c).setCellValue(data[r][c]);
}
}
try (java.io.FileOutputStream out = new java.io.FileOutputStream(file)) {
workbook.write(out);
}
}
}
@Test
@@ -0,0 +1,219 @@
package com.nanri.aiimage.modules.brand.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandInvalidBrandDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.Mockito.mock;
/**
* 品牌检测结果出表的行数契约(任务 2735 复盘):查询按品牌去重(同品牌只查一次 WIPO),
* 但同品牌不同 ASIN 是不同商品行,结果文件一行都不能少。
*/
@ExtendWith(MockitoExtension.class)
class BrandTaskServiceResultRowsTest {
@Mock private BrandCrawlTaskMapper brandCrawlTaskMapper;
@Mock private OssStorageService ossStorageService;
@Mock private StorageProperties storageProperties;
@Mock private BrandProgressProperties brandProgressProperties;
@Mock private BrandTaskProgressCacheService brandTaskProgressCacheService;
@Mock private BrandTaskStorageService brandTaskStorageService;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@TempDir
Path tempDir;
@BeforeAll
static void initializeTableInfo() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
BrandCrawlTaskEntity.class);
}
private BrandTaskService service() {
return new BrandTaskService(brandCrawlTaskMapper, ossStorageService, storageProperties,
brandProgressProperties, brandTaskProgressCacheService,
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
brandTaskStorageService,
distributedJobLockService, taskDistributedLockService, new ObjectMapper(),
taskFileJobService, taskProgressSnapshotService,
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
}
/** 4 数据行:AA 两个不同 ASIN(同品牌)、BB、CC 各一行 */
private File sourceWorkbook() throws Exception {
File file = tempDir.resolve("brand-source.xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
Sheet sheet = workbook.createSheet("Sheet1");
String[][] data = {
{"品牌", "ASIH", "状态", "时间"},
{"Brand: AA", "B001", "", ""},
{"Brand: AA", "B002", "", ""},
{"Brand: BB", "B003", "", ""},
{"Brand: CC", "B004", "", ""},
};
for (int r = 0; r < data.length; r++) {
Row row = sheet.createRow(r);
for (int c = 0; c < data[r].length; c++) {
row.createCell(c).setCellValue(data[r][c]);
}
}
try (FileOutputStream out = new FileOutputStream(file)) {
workbook.write(out);
}
}
return file;
}
private BrandParsedFileCacheDto cachedFile(BrandTaskService.ParsedBrandFile deduped) {
BrandParsedFileCacheDto dto = new BrandParsedFileCacheDto();
dto.setFileUrl("https://oss.example/brand-source.xlsx");
dto.setSheetName(deduped.sheetName());
dto.setColumns(deduped.columns());
dto.setRows(deduped.rows());
return dto;
}
private BrandFileAggregateCacheDto aggregateWithInvalid(String brand) {
BrandFileAggregateCacheDto dto = new BrandFileAggregateCacheDto();
dto.setFileUrl("https://oss.example/brand-source.xlsx");
BrandInvalidBrandDto invalid = new BrandInvalidBrandDto();
invalid.setBrand(brand);
invalid.setCountry("英国");
invalid.setStatus("已注册");
dto.setInvalidBrands(new ArrayList<>(List.of(invalid)));
return dto;
}
private List<String> sheetValues(Sheet sheet, int column) {
List<String> values = new ArrayList<>();
for (int r = 1; r <= sheet.getLastRowNum(); r++) {
Row row = sheet.getRow(r);
if (row != null && row.getCell(column) != null) {
values.add(row.getCell(column).getStringCellValue());
}
}
return values;
}
@Test
void payloadParseDedupesRowsButExpansionParseKeepsEveryRow() throws Exception {
File file = sourceWorkbook();
BrandTaskService service = service();
BrandTaskService.ParsedBrandFile forPayload = service.parseBrandFile(file);
assertEquals(3, forPayload.rows().size(), "回传执行方的行集必须每品牌一行,防按行重查烧代理");
assertEquals(3, forPayload.uniqueBrands().size());
BrandTaskService.ParsedBrandFile forOutput = service.parseBrandFile(file, false);
assertEquals(4, forOutput.rows().size(), "出表行集要保留同品牌不同 ASIN 的全部行");
assertEquals(3, forOutput.uniqueBrands().size(), "查询集仍按品牌去重,与出表模式一致");
}
@Test
void resultWorkbookKeepsEveryAsinRowAndOnlyRemovesJudgedBrands() throws Exception {
File file = sourceWorkbook();
BrandTaskService service = service();
BrandTaskService.ParsedBrandFile deduped = service.parseBrandFile(file);
BrandTaskService.ParsedBrandFile expanded = service.parseBrandFile(file, false);
File output = tempDir.resolve("brand-result.xlsx").toFile();
service.writeBrandWorkbook(output, "Simple", cachedFile(deduped), aggregateWithInvalid("Brand: CC"),
List.of(), expanded.rows());
try (Workbook workbook = WorkbookFactory.create(output)) {
Sheet main = workbook.getSheet("Sheet1");
assertNotNull(main);
List<String> asins = sheetValues(main, 1);
assertEquals(List.of("B001", "B002", "B003"), asins,
"同品牌不同 ASIN 的行都要出;仅不符合品牌 CC 的行移出主表");
Sheet invalid = workbook.getSheet("不符合品牌");
assertNotNull(invalid);
assertEquals(List.of("Brand: CC"), sheetValues(invalid, 0));
assertNull(workbook.getSheet("未检测品牌"), "成功收尾没有未检测 sheet");
}
}
@Test
void partialResultMovesAllUndetectedRowsToTheirOwnSheet() throws Exception {
File file = sourceWorkbook();
BrandTaskService service = service();
BrandTaskService.ParsedBrandFile deduped = service.parseBrandFile(file);
BrandTaskService.ParsedBrandFile expanded = service.parseBrandFile(file, false);
File output = tempDir.resolve("brand-partial.xlsx").toFile();
service.writeBrandWorkbook(output, "Simple", cachedFile(deduped), aggregateWithInvalid("Brand: CC"),
List.of("Brand: BB"), expanded.rows());
try (Workbook workbook = WorkbookFactory.create(output)) {
Sheet main = workbook.getSheet("Sheet1");
assertEquals(List.of("B001", "B002"), sheetValues(main, 1));
Sheet undetected = workbook.getSheet("未检测品牌");
assertNotNull(undetected, "未检测品牌的行要整行挪进独立 sheet");
assertEquals(List.of("B003"), sheetValues(undetected, 1));
}
}
@Test
void blankBrandRowsAreAlwaysKeptInBothModes() throws Exception {
File file = tempDir.resolve("brand-blank.xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
Sheet sheet = workbook.createSheet("Sheet1");
String[][] data = {
{"品牌", "ASIH", "状态", "时间"},
{"", "B001", "", ""},
{"", "B002", "", ""},
};
for (int r = 0; r < data.length; r++) {
Row row = sheet.createRow(r);
for (int c = 0; c < data[r].length; c++) {
row.createCell(c).setCellValue(data[r][c]);
}
}
try (FileOutputStream out = new FileOutputStream(file)) {
workbook.write(out);
}
}
BrandTaskService service = service();
assertEquals(2, service.parseBrandFile(file).rows().size(), "空品牌行不参与品牌去重,两种模式都保留");
assertEquals(2, service.parseBrandFile(file, false).rows().size());
assertEquals(0, service.parseBrandFile(file).uniqueBrands().size());
}
}