fix(去重): 主链接行位于其子链接行之后时未被丢弃 + 导出表头与数据行错列
线上任务 28422(文件 新数据变体完.xlsx)结果中 16 个主ID同时保留了主链接与 子链接,违反所选规则(keepIntegerIds=false / keepUnderscoreIds=true / keepIntegerMainIdsWhenNoSubIds=true)。 根因:旧实现把整数主链接行暂存,依赖「后到的同主ID子链接行」把它丢弃。 当顺序为「子链接在前、主链接在后」时,子行到来时暂存区尚空,无人记录该主ID 已有子链接,主链接一路存活到 flush。改用 IdRuleRowPicker 先收集、收尾统一按 「该主ID是否出现过子链接行」判定,与源文件顺序无关;补 subRows/mainRows/ droppedMainRows 排查日志。 同时修一处潜在错列:表头按 selectedColumns 原顺序写、数据行按 orderedSelectedColumns(id/ASIN/国家/价格/品牌 提前)写,两者不同序时整体 错列;本任务所选列恰为导出优先级顺序故未暴露。 实测:用线上源文件重算,输出 7273 → 7257 行(正好少掉那 16 个主/子并存的主ID)。
This commit is contained in:
+211
@@ -0,0 +1,211 @@
|
||||
package com.nanri.aiimage.modules.dedupe.service;
|
||||
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeRunService.DedupeCandidateRow;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 去重清洗的保留规则与导出列顺序回归测试。
|
||||
*
|
||||
* <p>覆盖两类线上缺陷:主链接行排在其子链接行之后时未被丢弃;导出表头按所选列原顺序写、
|
||||
* 数据行按重排后顺序写导致整体错列。</p>
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class DedupeRunServiceCleanRulesTest {
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private FileResultMapper fileResultMapper;
|
||||
@Mock
|
||||
private StorageProperties storageProperties;
|
||||
@Mock
|
||||
private OssStorageService ossStorageService;
|
||||
@Mock
|
||||
private DedupeTotalDataService dedupeTotalDataService;
|
||||
|
||||
private static final String[] HEADERS = {"id", "ASIN", "国家", "价格", "发货类型", "品牌", "状态", "创建时间"};
|
||||
|
||||
/**
|
||||
* 同一主 ID 的主/子链接不得并存,且与源文件中的先后顺序无关:
|
||||
* 1 为主链接在前(旧实现正确),33/96 为子链接在前(旧实现漏删主链接)。
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("子链接在前时,主链接行也必须被丢弃")
|
||||
void dropsMainRowPlacedAfterItsSubRows(@TempDir Path tempDir) throws Exception {
|
||||
List<String[]> rows = new ArrayList<>();
|
||||
rows.add(new String[]{"1", "B0AAAAAA01", "英国", "10.00", "FBM", "品牌A", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"1_2", "B0AAAAAA02", "英国", "11.00", "FBM", "品牌A", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"1_3", "B0AAAAAA03", "英国", "12.00", "FBM", "品牌A", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"2", "B0BBBBBB01", "英国", "20.00", "FBM", "品牌B", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"33_1", "B0CCCCCC01", "英国", "30.00", "FBM", "品牌C", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"33", "B0CCCCCC02", "英国", "31.00", "FBM", "品牌C", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"96_1", "B0DDDDDD01", "英国", "40.00", "FBM", "品牌D", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"96_2", "B0DDDDDD02", "英国", "41.00", "FBM", "品牌D", "在售", "2026-09-01"});
|
||||
rows.add(new String[]{"96", "B0DDDDDD03", "英国", "42.00", "FBM", "品牌D", "在售", "2026-09-01"});
|
||||
// FBA 行始终被过滤
|
||||
rows.add(new String[]{"3", "B0EEEEEE01", "英国", "50.00", "FBA", "品牌E", "在售", "2026-09-01"});
|
||||
|
||||
File output = runClean(tempDir, rows, List.of("id", "ASIN", "国家", "价格", "品牌"));
|
||||
|
||||
List<String> ids = readOutputColumn(output, 0);
|
||||
assertEquals(List.of("1_2", "1_3", "2", "33_1", "96_1", "96_2"), ids,
|
||||
"同一主 ID 只能保留子链接;无子链接的主 ID 才保留主链接行");
|
||||
}
|
||||
|
||||
/** 所选列顺序与导出优先级不同时,表头必须按重排后的顺序写,否则表头与数据整体错列。 */
|
||||
@Test
|
||||
@DisplayName("所选列顺序与导出优先级不同时,表头与数据行必须同序")
|
||||
void writesHeaderInSameOrderAsData(@TempDir Path tempDir) throws Exception {
|
||||
List<String[]> rows = new ArrayList<>();
|
||||
rows.add(new String[]{"7", "B0FFFFFF01", "英国", "9.99", "FBM", "品牌F", "在售", "2026-09-02"});
|
||||
|
||||
File output = runClean(tempDir, rows,
|
||||
List.of("状态", "id", "ASIN", "品牌", "国家", "价格"));
|
||||
|
||||
List<String> header = readOutputRow(output, 0);
|
||||
assertEquals(List.of("id", "ASIN", "国家", "价格", "品牌", "状态"), header);
|
||||
assertEquals(List.of("7", "B0FFFFFF01", "英国", "9.99", "品牌F", "在售"), readOutputRow(output, 1));
|
||||
}
|
||||
|
||||
/** 纯规则单测:保留主链接开关打开时,整数行不再受子链接影响。 */
|
||||
@Test
|
||||
@DisplayName("开启保留主链接时,整数行不受子链接影响")
|
||||
void keepsIntegerRowWhenMainIdRuleEnabled() {
|
||||
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(true, true, true);
|
||||
picker.select("13", () -> row("B0GGGGGG01"));
|
||||
picker.select("13_1", () -> row("B0GGGGGG02"));
|
||||
|
||||
assertEquals(List.of("B0GGGGGG01", "B0GGGGGG02"), asins(picker.resolve()));
|
||||
}
|
||||
|
||||
/** 纯规则单测:主链接在前、子链接在后时丢弃主链接(旧实现已正确,防回退)。 */
|
||||
@Test
|
||||
@DisplayName("主链接在前、子链接在后时丢弃主链接")
|
||||
void dropsMainRowPlacedBeforeSubRows() {
|
||||
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true);
|
||||
picker.select("5", () -> row("B0HHHHHH01"));
|
||||
picker.select("5_2", () -> row("B0HHHHHH02"));
|
||||
|
||||
assertEquals(List.of("B0HHHHHH02"), asins(picker.resolve()));
|
||||
}
|
||||
|
||||
/** 纯规则单测:子链接在前、主链接在后时同样丢弃主链接。 */
|
||||
@Test
|
||||
@DisplayName("子链接在前、主链接在后时丢弃主链接")
|
||||
void dropsMainRowPlacedAfterSubRows() {
|
||||
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true);
|
||||
picker.select("33_1", () -> row("B0IIIIII01"));
|
||||
picker.select("33", () -> row("B0IIIIII02"));
|
||||
|
||||
assertEquals(List.of("B0IIIIII01"), asins(picker.resolve()));
|
||||
}
|
||||
|
||||
/** 纯规则单测:id 为空的行不保留,且不影响其他行。 */
|
||||
@Test
|
||||
@DisplayName("id 为空的行不保留")
|
||||
void skipsRowsWithBlankId() {
|
||||
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true);
|
||||
picker.select("", () -> row("B0JJJJJJ01"));
|
||||
picker.select(null, () -> row("B0JJJJJJ02"));
|
||||
picker.select("4", () -> row("B0JJJJJJ03"));
|
||||
|
||||
assertEquals(List.of("B0JJJJJJ03"), asins(picker.resolve()));
|
||||
}
|
||||
|
||||
private static DedupeCandidateRow row(String asin) {
|
||||
return new DedupeCandidateRow(List.of(asin), asin);
|
||||
}
|
||||
|
||||
private static List<String> asins(List<DedupeCandidateRow> rows) {
|
||||
return rows.stream().map(DedupeCandidateRow::asinValue).toList();
|
||||
}
|
||||
|
||||
private File runClean(Path tempDir, List<String[]> rows, List<String> selectedColumns) throws Exception {
|
||||
File input = tempDir.resolve("input.xlsx").toFile();
|
||||
File output = tempDir.resolve("output.xlsx").toFile();
|
||||
writeInput(input, rows);
|
||||
|
||||
DedupeRunService service = new DedupeRunService(
|
||||
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService);
|
||||
when(dedupeTotalDataService.findExistingComparableValues(any())).thenReturn(Set.of());
|
||||
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
|
||||
.thenAnswer(invocation -> {
|
||||
String value = invocation.getArgument(0);
|
||||
return value == null ? "" : value.trim().toUpperCase();
|
||||
});
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "cleanExcelByStreamRules",
|
||||
input, output, selectedColumns, false, true, true);
|
||||
return output;
|
||||
}
|
||||
|
||||
private void writeInput(File file, List<String[]> rows) throws Exception {
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
Sheet sheet = workbook.createSheet("variant_collection");
|
||||
Row headerRow = sheet.createRow(0);
|
||||
for (int i = 0; i < HEADERS.length; i++) {
|
||||
headerRow.createCell(i).setCellValue(HEADERS[i]);
|
||||
}
|
||||
for (int r = 0; r < rows.size(); r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
String[] values = rows.get(r);
|
||||
for (int c = 0; c < values.length; c++) {
|
||||
row.createCell(c).setCellValue(values[c]);
|
||||
}
|
||||
}
|
||||
try (java.io.FileOutputStream fos = new java.io.FileOutputStream(file)) {
|
||||
workbook.write(fos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> readOutputRow(File file, int rowIndex) throws Exception {
|
||||
try (FileInputStream fis = new FileInputStream(file); XSSFWorkbook workbook = new XSSFWorkbook(fis)) {
|
||||
Row row = workbook.getSheetAt(0).getRow(rowIndex);
|
||||
List<String> values = new ArrayList<>();
|
||||
for (int c = 0; c < row.getLastCellNum(); c++) {
|
||||
values.add(row.getCell(c) == null ? "" : row.getCell(c).getStringCellValue());
|
||||
}
|
||||
return values;
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> readOutputColumn(File file, int columnIndex) throws Exception {
|
||||
try (FileInputStream fis = new FileInputStream(file); XSSFWorkbook workbook = new XSSFWorkbook(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
List<String> values = new ArrayList<>();
|
||||
for (int r = 1; r <= sheet.getLastRowNum(); r++) {
|
||||
Row row = sheet.getRow(r);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
values.add(row.getCell(columnIndex) == null ? "" : row.getCell(columnIndex).getStringCellValue());
|
||||
}
|
||||
return values;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user