super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(去重): 主链接行位于其子链接行之后时未被丢弃 + 导出表头与数据行错列

Browse Source 
线上任务 28422（文件 新数据变体完.xlsx）结果中 16 个主ID同时保留了主链接与
子链接，违反所选规则（keepIntegerIds=false / keepUnderscoreIds=true /
keepIntegerMainIdsWhenNoSubIds=true）。

根因：旧实现把整数主链接行暂存，依赖「后到的同主ID子链接行」把它丢弃。
当顺序为「子链接在前、主链接在后」时，子行到来时暂存区尚空，无人记录该主ID
已有子链接，主链接一路存活到 flush。改用 IdRuleRowPicker 先收集、收尾统一按
「该主ID是否出现过子链接行」判定，与源文件顺序无关；补 subRows/mainRows/
droppedMainRows 排查日志。

同时修一处潜在错列：表头按 selectedColumns 原顺序写、数据行按
orderedSelectedColumns（id/ASIN/国家/价格/品牌 提前）写，两者不同序时整体
错列；本任务所选列恰为导出优先级顺序故未暴露。

实测：用线上源文件重算，输出 7273 → 7257 行（正好少掉那 16 个主/子并存的主ID）。
This commit is contained in:
黄自达
2026-09-16 16:33:04 +08:00
parent 367b4b7553
commit ab28b168ec
2 changed files with 293 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
+82 -73
View File
@@ -42,6 +42,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -487,7 +488,7 @@ public class DedupeRunService {
readResult.scannedRows = new AtomicInteger(0);
readResult.filteredFbaRows = new AtomicInteger(0);
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
IdRuleRowPicker rowPicker = new IdRuleRowPicker(keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds);
// 表头列索引缓存：由 onHeader 填充，数据行为空时使用
final Map<String, Integer>[] headerIndexCache = new Map[]{Map.of()};
@@ -526,23 +527,16 @@ public class DedupeRunService {
}
Integer idColumnIndex = headerIndex.get("id");
if (idColumnIndex == null) {
readResult.rows.add(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
rowPicker.addAlways(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
return;
}
appendRowByIdRule(
rowPicker.select(
normalizeCellText(cellText(rowMap, idColumnIndex)),
rowMap,
headerIndex,
orderedSelectedColumns,
keepIntegerIds,
keepUnderscoreIds,
keepIntegerMainIdsWhenNoSubIds,
pendingMainIdGroup,
readResult.rows
() -> buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null)
);
}
});
pendingMainIdGroup.flush(readResult.rows);
readResult.rows = rowPicker.resolve();
long readNs = elapsedNs(readStartNs);
Set<String> candidateAsinValues = new HashSet<>();
@@ -560,8 +554,9 @@ public class DedupeRunService {
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName));
Row outputHeaderRow = outputSheet.createRow(0);
for (int i = 0; i < selectedColumns.size(); i++) {
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
// 表头必须与数据行同序：数据按 orderedSelectedColumns 取值，表头写 selectedColumns 会整体错列
for (int i = 0; i < orderedSelectedColumns.size(); i++) {
outputHeaderRow.createCell(i).setCellValue(orderedSelectedColumns.get(i));
}
Set<String> writtenAsinValues = new HashSet<>();
@@ -659,34 +654,80 @@ public class DedupeRunService {
return new DedupeCandidateRow(selectedValues, asinValue);
}
private void appendRowByIdRule(String idValue, Map<Integer, String> rowMap,
Map<String, Integer> headerIndex, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds,
PendingMainIdGroup pendingMainIdGroup,
List<DedupeCandidateRow> rows) {
if (idValue == null || idValue.isBlank()) {
return;
/**
* 按 ID 保留规则挑选输出行。
*
* <p>主链接行是否保留，只取决于「该主 ID 是否出现过子链接行」，与两类行在源文件中的
* 先后顺序无关。旧实现用「暂存主链接行 + 后到的同主 ID 子链接行把它丢弃」的方式，
* 一旦顺序是「子链接在前、主链接在后」（子行到来时暂存区尚空，无人记录该主 ID 已有
* 子链接），主链接就会一路存活到收尾，导致同一主 ID 的主/子链接同时出现在结果里。</p>
*/
static final class IdRuleRowPicker {
private final boolean keepIntegerIds;
private final boolean keepUnderscoreIds;
private final boolean keepIntegerMainIdsWhenNoSubIds;
/** 出现过子链接行的主 ID 集合，与源文件顺序无关 */
private final Set<String> mainIdsWithSubRows = new HashSet<>();
private final List<PickedRow> pickedRows = new ArrayList<>();
private int subRowCount = 0;
private int mainRowCount = 0;
IdRuleRowPicker(boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerMainIdsWhenNoSubIds) {
this.keepIntegerIds = keepIntegerIds;
this.keepUnderscoreIds = keepUnderscoreIds;
this.keepIntegerMainIdsWhenNoSubIds = keepIntegerMainIdsWhenNoSubIds;
}
String mainId = extractMainId(idValue);
if (pendingMainIdGroup.hasDifferentMainId(mainId)) {
pendingMainIdGroup.flush(rows);
/** ID 不参与保留规则判定的行（如无 id 列或 id 为空）直接保留。 */
void addAlways(DedupeCandidateRow row) {
pickedRows.add(new PickedRow(row, null));
}
if (isUnderscoreId(idValue)) {
pendingMainIdGroup.discardIfSameMainId(mainId);
if (keepUnderscoreIds) {
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
}
return;
}
if (isIntegerId(idValue)) {
if (keepIntegerIds) {
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
/** 按 ID 形态与保留规则挑选；rowSupplier 仅在确定保留时才求值。 */
void select(String idValue, Supplier<DedupeCandidateRow> rowSupplier) {
if (idValue == null || idValue.isBlank()) {
return;
}
if (keepIntegerMainIdsWhenNoSubIds) {
pendingMainIdGroup.add(mainId, buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
String mainId = extractMainId(idValue);
if (isUnderscoreId(idValue)) {
subRowCount++;
mainIdsWithSubRows.add(mainId);
if (keepUnderscoreIds) {
pickedRows.add(new PickedRow(rowSupplier.get(), null));
}
return;
}
if (isIntegerId(idValue)) {
if (keepIntegerIds) {
pickedRows.add(new PickedRow(rowSupplier.get(), null));
return;
}
if (keepIntegerMainIdsWhenNoSubIds) {
mainRowCount++;
pickedRows.add(new PickedRow(rowSupplier.get(), mainId));
}
}
}
/** 收尾统一判定：带条件的主链接行仅在该主 ID 没有子链接行时保留。 */
List<DedupeCandidateRow> resolve() {
List<DedupeCandidateRow> resolved = new ArrayList<>(pickedRows.size());
int droppedMainRowCount = 0;
for (PickedRow picked : pickedRows) {
if (picked.conditionalMainId() != null && mainIdsWithSubRows.contains(picked.conditionalMainId())) {
droppedMainRowCount++;
continue;
}
resolved.add(picked.row());
}
log.info("dedupe id rules subRows={} mainRows={} mainIdsWithSubRows={} droppedMainRows={} keptRows={}",
subRowCount, mainRowCount, mainIdsWithSubRows.size(), droppedMainRowCount, resolved.size());
return resolved;
}
/** conditionalMainId 非空表示该行是主链接行，需等收尾时看同主 ID 有无子链接行 */
private record PickedRow(DedupeCandidateRow row, String conditionalMainId) {
}
}
@@ -730,7 +771,7 @@ public class DedupeRunService {
return String.join("/", parts);
}
private String extractMainId(String text) {
static String extractMainId(String text) {
if (text == null || text.isBlank()) {
return "";
}
@@ -744,7 +785,7 @@ public class DedupeRunService {
return "";
}
private boolean isIntegerId(String text) {
static boolean isIntegerId(String text) {
if (text == null || text.isEmpty()) {
return false;
}
@@ -756,7 +797,7 @@ public class DedupeRunService {
return true;
}
private boolean isUnderscoreId(String text) {
static boolean isUnderscoreId(String text) {
if (text == null || text.length() < 3) {
return false;
}
@@ -952,38 +993,6 @@ public class DedupeRunService {
private AtomicInteger filteredFbaRows;
}
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
}
private static final class PendingMainIdGroup {
private String mainId;
private final List<DedupeCandidateRow> rows = new ArrayList<>();
private void add(String nextMainId, DedupeCandidateRow row) {
if (hasDifferentMainId(nextMainId)) {
rows.clear();
}
mainId = nextMainId;
rows.add(row);
}
private boolean hasDifferentMainId(String nextMainId) {
return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId));
}
private void discardIfSameMainId(String nextMainId) {
if (mainId != null && mainId.equals(nextMainId)) {
rows.clear();
mainId = null;
}
}
private void flush(List<DedupeCandidateRow> outputRows) {
if (!rows.isEmpty()) {
outputRows.addAll(rows);
rows.clear();
}
mainId = null;
}
record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
}
}
backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceCleanRulesTest.java
+211
View File
@@ -0,0 +1,211 @@
package com.nanri.aiimage.modules.dedupe.service;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.dedupe.service.DedupeRunService.DedupeCandidateRow;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
/**
* 去重清洗的保留规则与导出列顺序回归测试。
*
* <p>覆盖两类线上缺陷：主链接行排在其子链接行之后时未被丢弃；导出表头按所选列原顺序写、
* 数据行按重排后顺序写导致整体错列。</p>
*/
@ExtendWith(MockitoExtension.class)
class DedupeRunServiceCleanRulesTest {
@Mock
private FileTaskMapper fileTaskMapper;
@Mock
private FileResultMapper fileResultMapper;
@Mock
private StorageProperties storageProperties;
@Mock
private OssStorageService ossStorageService;
@Mock
private DedupeTotalDataService dedupeTotalDataService;
private static final String[] HEADERS = {"id", "ASIN", "国家", "价格", "发货类型", "品牌", "状态", "创建时间"};
/**
* 同一主 ID 的主/子链接不得并存，且与源文件中的先后顺序无关：
* 1 为主链接在前（旧实现正确），33/96 为子链接在前（旧实现漏删主链接）。
*/
@Test
@DisplayName("子链接在前时，主链接行也必须被丢弃")
void dropsMainRowPlacedAfterItsSubRows(@TempDir Path tempDir) throws Exception {
List<String[]> rows = new ArrayList<>();
rows.add(new String[]{"1", "B0AAAAAA01", "英国", "10.00", "FBM", "品牌A", "在售", "2026-09-01"});
rows.add(new String[]{"1_2", "B0AAAAAA02", "英国", "11.00", "FBM", "品牌A", "在售", "2026-09-01"});
rows.add(new String[]{"1_3", "B0AAAAAA03", "英国", "12.00", "FBM", "品牌A", "在售", "2026-09-01"});
rows.add(new String[]{"2", "B0BBBBBB01", "英国", "20.00", "FBM", "品牌B", "在售", "2026-09-01"});
rows.add(new String[]{"33_1", "B0CCCCCC01", "英国", "30.00", "FBM", "品牌C", "在售", "2026-09-01"});
rows.add(new String[]{"33", "B0CCCCCC02", "英国", "31.00", "FBM", "品牌C", "在售", "2026-09-01"});
rows.add(new String[]{"96_1", "B0DDDDDD01", "英国", "40.00", "FBM", "品牌D", "在售", "2026-09-01"});
rows.add(new String[]{"96_2", "B0DDDDDD02", "英国", "41.00", "FBM", "品牌D", "在售", "2026-09-01"});
rows.add(new String[]{"96", "B0DDDDDD03", "英国", "42.00", "FBM", "品牌D", "在售", "2026-09-01"});
// FBA 行始终被过滤
rows.add(new String[]{"3", "B0EEEEEE01", "英国", "50.00", "FBA", "品牌E", "在售", "2026-09-01"});
File output = runClean(tempDir, rows, List.of("id", "ASIN", "国家", "价格", "品牌"));
List<String> ids = readOutputColumn(output, 0);
assertEquals(List.of("1_2", "1_3", "2", "33_1", "96_1", "96_2"), ids,
"同一主 ID 只能保留子链接；无子链接的主 ID 才保留主链接行");
}
/** 所选列顺序与导出优先级不同时，表头必须按重排后的顺序写，否则表头与数据整体错列。 */
@Test
@DisplayName("所选列顺序与导出优先级不同时，表头与数据行必须同序")
void writesHeaderInSameOrderAsData(@TempDir Path tempDir) throws Exception {
List<String[]> rows = new ArrayList<>();
rows.add(new String[]{"7", "B0FFFFFF01", "英国", "9.99", "FBM", "品牌F", "在售", "2026-09-02"});
File output = runClean(tempDir, rows,
List.of("状态", "id", "ASIN", "品牌", "国家", "价格"));
List<String> header = readOutputRow(output, 0);
assertEquals(List.of("id", "ASIN", "国家", "价格", "品牌", "状态"), header);
assertEquals(List.of("7", "B0FFFFFF01", "英国", "9.99", "品牌F", "在售"), readOutputRow(output, 1));
}
/** 纯规则单测：保留主链接开关打开时，整数行不再受子链接影响。 */
@Test
@DisplayName("开启保留主链接时，整数行不受子链接影响")
void keepsIntegerRowWhenMainIdRuleEnabled() {
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(true, true, true);
picker.select("13", () -> row("B0GGGGGG01"));
picker.select("13_1", () -> row("B0GGGGGG02"));
assertEquals(List.of("B0GGGGGG01", "B0GGGGGG02"), asins(picker.resolve()));
}
/** 纯规则单测：主链接在前、子链接在后时丢弃主链接（旧实现已正确，防回退）。 */
@Test
@DisplayName("主链接在前、子链接在后时丢弃主链接")
void dropsMainRowPlacedBeforeSubRows() {
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true);
picker.select("5", () -> row("B0HHHHHH01"));
picker.select("5_2", () -> row("B0HHHHHH02"));
assertEquals(List.of("B0HHHHHH02"), asins(picker.resolve()));
}
/** 纯规则单测：子链接在前、主链接在后时同样丢弃主链接。 */
@Test
@DisplayName("子链接在前、主链接在后时丢弃主链接")
void dropsMainRowPlacedAfterSubRows() {
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true);
picker.select("33_1", () -> row("B0IIIIII01"));
picker.select("33", () -> row("B0IIIIII02"));
assertEquals(List.of("B0IIIIII01"), asins(picker.resolve()));
}
/** 纯规则单测：id 为空的行不保留，且不影响其他行。 */
@Test
@DisplayName("id 为空的行不保留")
void skipsRowsWithBlankId() {
DedupeRunService.IdRuleRowPicker picker = new DedupeRunService.IdRuleRowPicker(false, true, true);
picker.select("", () -> row("B0JJJJJJ01"));
picker.select(null, () -> row("B0JJJJJJ02"));
picker.select("4", () -> row("B0JJJJJJ03"));
assertEquals(List.of("B0JJJJJJ03"), asins(picker.resolve()));
}
private static DedupeCandidateRow row(String asin) {
return new DedupeCandidateRow(List.of(asin), asin);
}
private static List<String> asins(List<DedupeCandidateRow> rows) {
return rows.stream().map(DedupeCandidateRow::asinValue).toList();
}
private File runClean(Path tempDir, List<String[]> rows, List<String> selectedColumns) throws Exception {
File input = tempDir.resolve("input.xlsx").toFile();
File output = tempDir.resolve("output.xlsx").toFile();
writeInput(input, rows);
DedupeRunService service = new DedupeRunService(
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService);
when(dedupeTotalDataService.findExistingComparableValues(any())).thenReturn(Set.of());
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
.thenAnswer(invocation -> {
String value = invocation.getArgument(0);
return value == null ? "" : value.trim().toUpperCase();
});
ReflectionTestUtils.invokeMethod(service, "cleanExcelByStreamRules",
input, output, selectedColumns, false, true, true);
return output;
}
private void writeInput(File file, List<String[]> rows) throws Exception {
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
Sheet sheet = workbook.createSheet("variant_collection");
Row headerRow = sheet.createRow(0);
for (int i = 0; i < HEADERS.length; i++) {
headerRow.createCell(i).setCellValue(HEADERS[i]);
}
for (int r = 0; r < rows.size(); r++) {
Row row = sheet.createRow(r + 1);
String[] values = rows.get(r);
for (int c = 0; c < values.length; c++) {
row.createCell(c).setCellValue(values[c]);
}
}
try (java.io.FileOutputStream fos = new java.io.FileOutputStream(file)) {
workbook.write(fos);
}
}
}
private List<String> readOutputRow(File file, int rowIndex) throws Exception {
try (FileInputStream fis = new FileInputStream(file); XSSFWorkbook workbook = new XSSFWorkbook(fis)) {
Row row = workbook.getSheetAt(0).getRow(rowIndex);
List<String> values = new ArrayList<>();
for (int c = 0; c < row.getLastCellNum(); c++) {
values.add(row.getCell(c) == null ? "" : row.getCell(c).getStringCellValue());
}
return values;
}
}
private List<String> readOutputColumn(File file, int columnIndex) throws Exception {
try (FileInputStream fis = new FileInputStream(file); XSSFWorkbook workbook = new XSSFWorkbook(fis)) {
Sheet sheet = workbook.getSheetAt(0);
List<String> values = new ArrayList<>();
for (int r = 1; r <= sheet.getLastRowNum(); r++) {
Row row = sheet.getRow(r);
if (row == null) {
continue;
}
values.add(row.getCell(columnIndex) == null ? "" : row.getCell(columnIndex).getStringCellValue());
}
return values;
}
}
}