From ab28b168ec1f63772afe922d82a956f9eb14c51d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 16 Sep 2026 16:33:04 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E5=8E=BB=E9=87=8D):=20=E4=B8=BB=E9=93=BE?= =?UTF-8?q?=E6=8E=A5=E8=A1=8C=E4=BD=8D=E4=BA=8E=E5=85=B6=E5=AD=90=E9=93=BE?= =?UTF-8?q?=E6=8E=A5=E8=A1=8C=E4=B9=8B=E5=90=8E=E6=97=B6=E6=9C=AA=E8=A2=AB?= =?UTF-8?q?=E4=B8=A2=E5=BC=83=20+=20=E5=AF=BC=E5=87=BA=E8=A1=A8=E5=A4=B4?= =?UTF-8?q?=E4=B8=8E=E6=95=B0=E6=8D=AE=E8=A1=8C=E9=94=99=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 线上任务 28422（文件 新数据变体完.xlsx）结果中 16 个主ID同时保留了主链接与 子链接，违反所选规则（keepIntegerIds=false / keepUnderscoreIds=true / keepIntegerMainIdsWhenNoSubIds=true）。 根因：旧实现把整数主链接行暂存，依赖「后到的同主ID子链接行」把它丢弃。 当顺序为「子链接在前、主链接在后」时，子行到来时暂存区尚空，无人记录该主ID 已有子链接，主链接一路存活到 flush。改用 IdRuleRowPicker 先收集、收尾统一按 「该主ID是否出现过子链接行」判定，与源文件顺序无关；补 subRows/mainRows/ droppedMainRows 排查日志。 同时修一处潜在错列：表头按 selectedColumns 原顺序写、数据行按 orderedSelectedColumns（id/ASIN/国家/价格/品牌 提前）写，两者不同序时整体 错列；本任务所选列恰为导出优先级顺序故未暴露。 实测：用线上源文件重算，输出 7273 → 7257 行（正好少掉那 16 个主/子并存的主ID）。 --- .../dedupe/service/DedupeRunService.java | 155 +++++++------ .../DedupeRunServiceCleanRulesTest.java | 211 ++++++++++++++++++ 2 files changed, 293 insertions(+), 73 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceCleanRulesTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 9bf9474d..326e79d2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -42,6 +42,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -487,7 +488,7 @@ public class DedupeRunService { readResult.scannedRows = new AtomicInteger(0); readResult.filteredFbaRows = new AtomicInteger(0); - PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup(); + IdRuleRowPicker rowPicker = new IdRuleRowPicker(keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds); // 表头列索引缓存：由 onHeader 填充，数据行为空时使用 final Map[] headerIndexCache = new Map[]{Map.of()}; @@ -526,23 +527,16 @@ public class DedupeRunService { } Integer idColumnIndex = headerIndex.get("id"); if (idColumnIndex == null) { - readResult.rows.add(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null)); + rowPicker.addAlways(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null)); return; } - appendRowByIdRule( + rowPicker.select( normalizeCellText(cellText(rowMap, idColumnIndex)), - rowMap, - headerIndex, - orderedSelectedColumns, - keepIntegerIds, - keepUnderscoreIds, - keepIntegerMainIdsWhenNoSubIds, - pendingMainIdGroup, - readResult.rows + () -> buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null) ); } }); - pendingMainIdGroup.flush(readResult.rows); + readResult.rows = rowPicker.resolve(); long readNs = elapsedNs(readStartNs); Set candidateAsinValues = new HashSet<>(); @@ -560,8 +554,9 @@ public class DedupeRunService { try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) { org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName)); Row outputHeaderRow = outputSheet.createRow(0); - for (int i = 0; i < selectedColumns.size(); i++) { - outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i)); + // 表头必须与数据行同序：数据按 orderedSelectedColumns 取值，表头写 selectedColumns 会整体错列 + for (int i = 0; i < orderedSelectedColumns.size(); i++) { + outputHeaderRow.createCell(i).setCellValue(orderedSelectedColumns.get(i)); } Set writtenAsinValues = new HashSet<>(); @@ -659,34 +654,80 @@ public class DedupeRunService { return new DedupeCandidateRow(selectedValues, asinValue); } - private void appendRowByIdRule(String idValue, Map rowMap, - Map headerIndex, List selectedColumns, - boolean keepIntegerIds, boolean keepUnderscoreIds, - boolean keepIntegerMainIdsWhenNoSubIds, - PendingMainIdGroup pendingMainIdGroup, - List rows) { - if (idValue == null || idValue.isBlank()) { - return; + /** + * 按 ID 保留规则挑选输出行。 + * + *

主链接行是否保留，只取决于「该主 ID 是否出现过子链接行」，与两类行在源文件中的 + * 先后顺序无关。旧实现用「暂存主链接行 + 后到的同主 ID 子链接行把它丢弃」的方式， + * 一旦顺序是「子链接在前、主链接在后」（子行到来时暂存区尚空，无人记录该主 ID 已有 + * 子链接），主链接就会一路存活到收尾，导致同一主 ID 的主/子链接同时出现在结果里。

+ */ + static final class IdRuleRowPicker { + + private final boolean keepIntegerIds; + private final boolean keepUnderscoreIds; + private final boolean keepIntegerMainIdsWhenNoSubIds; + /** 出现过子链接行的主 ID 集合，与源文件顺序无关 */ + private final Set mainIdsWithSubRows = new HashSet<>(); + private final List pickedRows = new ArrayList<>(); + private int subRowCount = 0; + private int mainRowCount = 0; + + IdRuleRowPicker(boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerMainIdsWhenNoSubIds) { + this.keepIntegerIds = keepIntegerIds; + this.keepUnderscoreIds = keepUnderscoreIds; + this.keepIntegerMainIdsWhenNoSubIds = keepIntegerMainIdsWhenNoSubIds; } - String mainId = extractMainId(idValue); - if (pendingMainIdGroup.hasDifferentMainId(mainId)) { - pendingMainIdGroup.flush(rows); + + /** ID 不参与保留规则判定的行（如无 id 列或 id 为空）直接保留。 */ + void addAlways(DedupeCandidateRow row) { + pickedRows.add(new PickedRow(row, null)); } - if (isUnderscoreId(idValue)) { - pendingMainIdGroup.discardIfSameMainId(mainId); - if (keepUnderscoreIds) { - rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null)); - } - return; - } - if (isIntegerId(idValue)) { - if (keepIntegerIds) { - rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null)); + + /** 按 ID 形态与保留规则挑选；rowSupplier 仅在确定保留时才求值。 */ + void select(String idValue, Supplier rowSupplier) { + if (idValue == null || idValue.isBlank()) { return; } - if (keepIntegerMainIdsWhenNoSubIds) { - pendingMainIdGroup.add(mainId, buildCandidateRow(rowMap, headerIndex, selectedColumns, null)); + String mainId = extractMainId(idValue); + if (isUnderscoreId(idValue)) { + subRowCount++; + mainIdsWithSubRows.add(mainId); + if (keepUnderscoreIds) { + pickedRows.add(new PickedRow(rowSupplier.get(), null)); + } + return; } + if (isIntegerId(idValue)) { + if (keepIntegerIds) { + pickedRows.add(new PickedRow(rowSupplier.get(), null)); + return; + } + if (keepIntegerMainIdsWhenNoSubIds) { + mainRowCount++; + pickedRows.add(new PickedRow(rowSupplier.get(), mainId)); + } + } + } + + /** 收尾统一判定：带条件的主链接行仅在该主 ID 没有子链接行时保留。 */ + List resolve() { + List resolved = new ArrayList<>(pickedRows.size()); + int droppedMainRowCount = 0; + for (PickedRow picked : pickedRows) { + if (picked.conditionalMainId() != null && mainIdsWithSubRows.contains(picked.conditionalMainId())) { + droppedMainRowCount++; + continue; + } + resolved.add(picked.row()); + } + log.info("dedupe id rules subRows={} mainRows={} mainIdsWithSubRows={} droppedMainRows={} keptRows={}", + subRowCount, mainRowCount, mainIdsWithSubRows.size(), droppedMainRowCount, resolved.size()); + return resolved; + } + + /** conditionalMainId 非空表示该行是主链接行，需等收尾时看同主 ID 有无子链接行 */ + private record PickedRow(DedupeCandidateRow row, String conditionalMainId) { } } @@ -730,7 +771,7 @@ public class DedupeRunService { return String.join("/", parts); } - private String extractMainId(String text) { + static String extractMainId(String text) { if (text == null || text.isBlank()) { return ""; } @@ -744,7 +785,7 @@ public class DedupeRunService { return ""; } - private boolean isIntegerId(String text) { + static boolean isIntegerId(String text) { if (text == null || text.isEmpty()) { return false; } @@ -756,7 +797,7 @@ public class DedupeRunService { return true; } - private boolean isUnderscoreId(String text) { + static boolean isUnderscoreId(String text) { if (text == null || text.length() < 3) { return false; } @@ -952,38 +993,6 @@ public class DedupeRunService { private AtomicInteger filteredFbaRows; } - private record DedupeCandidateRow(List selectedValues, String asinValue) { - } - - private static final class PendingMainIdGroup { - private String mainId; - private final List rows = new ArrayList<>(); - - private void add(String nextMainId, DedupeCandidateRow row) { - if (hasDifferentMainId(nextMainId)) { - rows.clear(); - } - mainId = nextMainId; - rows.add(row); - } - - private boolean hasDifferentMainId(String nextMainId) { - return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId)); - } - - private void discardIfSameMainId(String nextMainId) { - if (mainId != null && mainId.equals(nextMainId)) { - rows.clear(); - mainId = null; - } - } - - private void flush(List outputRows) { - if (!rows.isEmpty()) { - outputRows.addAll(rows); - rows.clear(); - } - mainId = null; - } + record DedupeCandidateRow(List selectedValues, String asinValue) { } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceCleanRulesTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceCleanRulesTest.java new file mode 100644 index 00000000..e4396932 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceCleanRulesTest.java @@ -0,0 +1,211 @@ +package com.nanri.aiimage.modules.dedupe.service; + +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.dedupe.service.DedupeRunService.DedupeCandidateRow; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +/** + * 去重清洗的保留规则与导出列顺序回归测试。 + * + *

覆盖两类线上缺陷：主链接行排在其子链接行之后时未被丢弃；导出表头按所选列原顺序写、 + * 数据行按重排后顺序写导致整体错列。

+ */ +@ExtendWith(MockitoExtension.class) +class DedupeRunServiceCleanRulesTest { + + @Mock + private FileTaskMapper fileTaskMapper; + @Mock + private FileResultMapper fileResultMapper; + @Mock + private StorageProperties storageProperties; + @Mock + private OssStorageService ossStorageService; + @Mock + private DedupeTotalDataService dedupeTotalDataService; + + private static final String[] HEADERS = {"id", "ASIN", "国家", "价格", "发货类型", "品牌", "状态", "创建时间"}; + + /** + * 同一主 ID 的主/子链接不得并存，且与源文件中的先后顺序无关： + * 1 为主链接在前（旧实现正确），33/96 为子链接在前（旧实现漏删主链接）。 + */ + @Test + @DisplayName("子链接在前时，主链接行也必须被丢弃") + void dropsMainRowPlacedAfterItsSubRows(@TempDir Path tempDir) throws Exception { + List rows = new ArrayList<>(); + rows.add(new String[]{"1", "B0AAAAAA01", "英国", "10.00", "FBM", "品牌A", "在售", "2026-09-01"}); + rows.add(new String[]{"1_2", "B0AAAAAA02", "英国", "11.00", "FBM", "品牌A", "在售", "2026-09-01"}); + rows.add(new String[]{"1_3", "B0AAAAAA03", "英国", "12.00", "FBM", "品牌A", "在售", "2026-09-01"}); + rows.add(new String[]{"2", "B0BBBBBB01", "英国", "20.00", "FBM", "品牌B", "在售", "2026-09-01"}); + rows.add(new String[]{"33_1", "B0CCCCCC01", "英国", "30.00", "FBM", "品牌C", "在售", "2026-09-01"}); + rows.add(new String[]{"33", "B0CCCCCC02", "英国", "31.00", "FBM", "品牌C", "在售", "2026-09-01"}); + rows.add(new String[]{"96_1", "B0DDDDDD01", "英国", "40.00", "FBM", "品牌D", "在售", "2026-09-01"}); + rows.add(new String[]{"96_2", "B0DDDDDD02", "英国", "41.00", "FBM", "品牌D", "在售", "2026-09-01"}); + rows.add(new String[]{"96", "B0DDDDDD03", "英国", "42.00", "FBM", "品牌D", "在售", "2026-09-01"}); + // FBA 行始终被过滤 + rows.add(new String[]{"3", "B0EEEEEE01", "英国", "50.00", "FBA", "品牌E", "在售", "2026-09-01"}); + + File output = runClean(tempDir, rows, List.of("id", "ASIN", "国家", "价格", "品牌")); + + List ids = readOutputColumn(output, 0); + assertEquals(List.of("1_2", "1_3", "2", "33_1", "96_1", "96_2"), ids, + "同一主 ID 只能保留子链接；无子链接的主 ID 才保留主链接行"); + } + + /** 所选列顺序与导出优先级不同时，表头必须按重排后的顺序写，否则表头与数据整体错列。 */ + @Test + @DisplayName("所选列顺序与导出优先级不同时，表头与数据行必须同序") + void writesHeaderInSameOrderAsData(@TempDir Path tempDir) throws Exception { + List rows = new ArrayList<>(); + rows.add(new String[]{"7", "B0FFFFFF01", "英国", "9.99", "FBM", "品牌F", "在售", "2026-09-02"}); + + File output = runClean(tempDir, rows, + List.of("状态", "id", "ASIN", "品牌", "国家", "价格")); + + List header = readOutputRow(output, 0); + assertEquals(List.of("id", "ASIN", "国家", "价格", "品牌", "状态"), header); + assertEquals(List.of("7", "B0FFFFFF01", "英国", "9.99", "品牌F", "在售"), readOutputRow(output, 1)); + } + + /** 纯规则单测：保留主链接开关打开时，整数行不再受子链接影响。 */ + @Test + @DisplayName("开启保留主链接时，整数行不受子链接影响") + void keepsIntegerRowWhenMainIdRuleEnabled() { + DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(true, true, true); + picker.select("13", () -> row("B0GGGGGG01")); + picker.select("13_1", () -> row("B0GGGGGG02")); + + assertEquals(List.of("B0GGGGGG01", "B0GGGGGG02"), asins(picker.resolve())); + } + + /** 纯规则单测：主链接在前、子链接在后时丢弃主链接（旧实现已正确，防回退）。 */ + @Test + @DisplayName("主链接在前、子链接在后时丢弃主链接") + void dropsMainRowPlacedBeforeSubRows() { + DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true); + picker.select("5", () -> row("B0HHHHHH01")); + picker.select("5_2", () -> row("B0HHHHHH02")); + + assertEquals(List.of("B0HHHHHH02"), asins(picker.resolve())); + } + + /** 纯规则单测：子链接在前、主链接在后时同样丢弃主链接。 */ + @Test + @DisplayName("子链接在前、主链接在后时丢弃主链接") + void dropsMainRowPlacedAfterSubRows() { + DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true); + picker.select("33_1", () -> row("B0IIIIII01")); + picker.select("33", () -> row("B0IIIIII02")); + + assertEquals(List.of("B0IIIIII01"), asins(picker.resolve())); + } + + /** 纯规则单测：id 为空的行不保留，且不影响其他行。 */ + @Test + @DisplayName("id 为空的行不保留") + void skipsRowsWithBlankId() { + DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true); + picker.select("", () -> row("B0JJJJJJ01")); + picker.select(null, () -> row("B0JJJJJJ02")); + picker.select("4", () -> row("B0JJJJJJ03")); + + assertEquals(List.of("B0JJJJJJ03"), asins(picker.resolve())); + } + + private static DedupeCandidateRow row(String asin) { + return new DedupeCandidateRow(List.of(asin), asin); + } + + private static List asins(List rows) { + return rows.stream().map(DedupeCandidateRow::asinValue).toList(); + } + + private File runClean(Path tempDir, List rows, List selectedColumns) throws Exception { + File input = tempDir.resolve("input.xlsx").toFile(); + File output = tempDir.resolve("output.xlsx").toFile(); + writeInput(input, rows); + + DedupeRunService service = new DedupeRunService( + fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService); + when(dedupeTotalDataService.findExistingComparableValues(any())).thenReturn(Set.of()); + when(dedupeTotalDataService.normalizeComparableValueOrBlank(any())) + .thenAnswer(invocation -> { + String value = invocation.getArgument(0); + return value == null ? "" : value.trim().toUpperCase(); + }); + + ReflectionTestUtils.invokeMethod(service, "cleanExcelByStreamRules", + input, output, selectedColumns, false, true, true); + return output; + } + + private void writeInput(File file, List rows) throws Exception { + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("variant_collection"); + Row headerRow = sheet.createRow(0); + for (int i = 0; i < HEADERS.length; i++) { + headerRow.createCell(i).setCellValue(HEADERS[i]); + } + for (int r = 0; r < rows.size(); r++) { + Row row = sheet.createRow(r + 1); + String[] values = rows.get(r); + for (int c = 0; c < values.length; c++) { + row.createCell(c).setCellValue(values[c]); + } + } + try (java.io.FileOutputStream fos = new java.io.FileOutputStream(file)) { + workbook.write(fos); + } + } + } + + private List readOutputRow(File file, int rowIndex) throws Exception { + try (FileInputStream fis = new FileInputStream(file); XSSFWorkbook workbook = new XSSFWorkbook(fis)) { + Row row = workbook.getSheetAt(0).getRow(rowIndex); + List values = new ArrayList<>(); + for (int c = 0; c < row.getLastCellNum(); c++) { + values.add(row.getCell(c) == null ? "" : row.getCell(c).getStringCellValue()); + } + return values; + } + } + + private List readOutputColumn(File file, int columnIndex) throws Exception { + try (FileInputStream fis = new FileInputStream(file); XSSFWorkbook workbook = new XSSFWorkbook(fis)) { + Sheet sheet = workbook.getSheetAt(0); + List values = new ArrayList<>(); + for (int r = 1; r <= sheet.getLastRowNum(); r++) { + Row row = sheet.getRow(r); + if (row == null) { + continue; + } + values.add(row.getCell(columnIndex) == null ? "" : row.getCell(columnIndex).getStringCellValue()); + } + return values; + } + } +}