fix(去重): 主链接行位于其子链接行之后时未被丢弃 + 导出表头与数据行错列

线上任务 28422（文件新数据变体完.xlsx）结果中 16 个主ID同时保留了主链接与子链接，违反所选规则（keepIntegerIds=false / keepUnderscoreIds=true / keepIntegerMainIdsWhenNoSubIds=true）。根因：旧实现把整数主链接行暂存，依赖「后到的同主ID子链接行」把它丢弃。当顺序为「子链接在前、主链接在后」时，子行到来时暂存区尚空，无人记录该主ID 已有子链接，主链接一路存活到 flush。改用 IdRuleRowPicker 先收集、收尾统一按「该主ID是否出现过子链接行」判定，与源文件顺序无关；补 subRows/mainRows/ droppedMainRows 排查日志。同时修一处潜在错列：表头按 selectedColumns 原顺序写、数据行按 orderedSelectedColumns（id/ASIN/国家/价格/品牌提前）写，两者不同序时整体错列；本任务所选列恰为导出优先级顺序故未暴露。实测：用线上源文件重算，输出 7273 → 7257 行（正好少掉那 16 个主/子并存的主ID）。
2026-09-16 16:33:04 +08:00
parent 367b4b7553
commit ab28b168ec
2 changed files with 293 additions and 73 deletions
@@ -42,6 +42,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;

@@ -487,7 +488,7 @@ public class DedupeRunService {
        readResult.scannedRows = new AtomicInteger(0);
        readResult.filteredFbaRows = new AtomicInteger(0);

-        PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
+        IdRuleRowPicker rowPicker = new IdRuleRowPicker(keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds);
        // 表头列索引缓存：由 onHeader 填充，数据行为空时使用
        final Map<String, Integer>[] headerIndexCache = new Map[]{Map.of()};

@@ -526,23 +527,16 @@ public class DedupeRunService {
                }
                Integer idColumnIndex = headerIndex.get("id");
                if (idColumnIndex == null) {
-                    readResult.rows.add(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
+                    rowPicker.addAlways(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
                    return;
                }
-                appendRowByIdRule(
+                rowPicker.select(
                        normalizeCellText(cellText(rowMap, idColumnIndex)),
-                        rowMap,
-                        headerIndex,
-                        orderedSelectedColumns,
-                        keepIntegerIds,
-                        keepUnderscoreIds,
-                        keepIntegerMainIdsWhenNoSubIds,
-                        pendingMainIdGroup,
-                        readResult.rows
+                        () -> buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null)
                );
            }
        });
-        pendingMainIdGroup.flush(readResult.rows);
+        readResult.rows = rowPicker.resolve();
        long readNs = elapsedNs(readStartNs);

        Set<String> candidateAsinValues = new HashSet<>();
@@ -560,8 +554,9 @@ public class DedupeRunService {
        try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
            org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName));
            Row outputHeaderRow = outputSheet.createRow(0);
-            for (int i = 0; i < selectedColumns.size(); i++) {
-                outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
+            // 表头必须与数据行同序：数据按 orderedSelectedColumns 取值，表头写 selectedColumns 会整体错列
+            for (int i = 0; i < orderedSelectedColumns.size(); i++) {
+                outputHeaderRow.createCell(i).setCellValue(orderedSelectedColumns.get(i));
            }

            Set<String> writtenAsinValues = new HashSet<>();
@@ -659,34 +654,80 @@ public class DedupeRunService {
        return new DedupeCandidateRow(selectedValues, asinValue);
    }

-    private void appendRowByIdRule(String idValue, Map<Integer, String> rowMap,
-                                   Map<String, Integer> headerIndex, List<String> selectedColumns,
-                                   boolean keepIntegerIds, boolean keepUnderscoreIds,
-                                   boolean keepIntegerMainIdsWhenNoSubIds,
-                                   PendingMainIdGroup pendingMainIdGroup,
-                                   List<DedupeCandidateRow> rows) {
-        if (idValue == null || idValue.isBlank()) {
-            return;
+    /**
+     * 按 ID 保留规则挑选输出行。
+     *
+     * <p>主链接行是否保留，只取决于「该主 ID 是否出现过子链接行」，与两类行在源文件中的
+     * 先后顺序无关。旧实现用「暂存主链接行 + 后到的同主 ID 子链接行把它丢弃」的方式，
+     * 一旦顺序是「子链接在前、主链接在后」（子行到来时暂存区尚空，无人记录该主 ID 已有
+     * 子链接），主链接就会一路存活到收尾，导致同一主 ID 的主/子链接同时出现在结果里。</p>
+     */
+    static final class IdRuleRowPicker {
+
+        private final boolean keepIntegerIds;
+        private final boolean keepUnderscoreIds;
+        private final boolean keepIntegerMainIdsWhenNoSubIds;
+        /** 出现过子链接行的主 ID 集合，与源文件顺序无关 */
+        private final Set<String> mainIdsWithSubRows = new HashSet<>();
+        private final List<PickedRow> pickedRows = new ArrayList<>();
+        private int subRowCount = 0;
+        private int mainRowCount = 0;
+
+        IdRuleRowPicker(boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerMainIdsWhenNoSubIds) {
+            this.keepIntegerIds = keepIntegerIds;
+            this.keepUnderscoreIds = keepUnderscoreIds;
+            this.keepIntegerMainIdsWhenNoSubIds = keepIntegerMainIdsWhenNoSubIds;
        }
-        String mainId = extractMainId(idValue);
-        if (pendingMainIdGroup.hasDifferentMainId(mainId)) {
-            pendingMainIdGroup.flush(rows);
+
+        /** ID 不参与保留规则判定的行（如无 id 列或 id 为空）直接保留。 */
+        void addAlways(DedupeCandidateRow row) {
+            pickedRows.add(new PickedRow(row, null));
        }
-        if (isUnderscoreId(idValue)) {
-            pendingMainIdGroup.discardIfSameMainId(mainId);
-            if (keepUnderscoreIds) {
-                rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
-            }
-            return;
-        }
-        if (isIntegerId(idValue)) {
-            if (keepIntegerIds) {
-                rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
+
+        /** 按 ID 形态与保留规则挑选；rowSupplier 仅在确定保留时才求值。 */
+        void select(String idValue, Supplier<DedupeCandidateRow> rowSupplier) {
+            if (idValue == null || idValue.isBlank()) {
                return;
            }
-            if (keepIntegerMainIdsWhenNoSubIds) {
-                pendingMainIdGroup.add(mainId, buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
+            String mainId = extractMainId(idValue);
+            if (isUnderscoreId(idValue)) {
+                subRowCount++;
+                mainIdsWithSubRows.add(mainId);
+                if (keepUnderscoreIds) {
+                    pickedRows.add(new PickedRow(rowSupplier.get(), null));
+                }
+                return;
            }
+            if (isIntegerId(idValue)) {
+                if (keepIntegerIds) {
+                    pickedRows.add(new PickedRow(rowSupplier.get(), null));
+                    return;
+                }
+                if (keepIntegerMainIdsWhenNoSubIds) {
+                    mainRowCount++;
+                    pickedRows.add(new PickedRow(rowSupplier.get(), mainId));
+                }
+            }
+        }
+
+        /** 收尾统一判定：带条件的主链接行仅在该主 ID 没有子链接行时保留。 */
+        List<DedupeCandidateRow> resolve() {
+            List<DedupeCandidateRow> resolved = new ArrayList<>(pickedRows.size());
+            int droppedMainRowCount = 0;
+            for (PickedRow picked : pickedRows) {
+                if (picked.conditionalMainId() != null && mainIdsWithSubRows.contains(picked.conditionalMainId())) {
+                    droppedMainRowCount++;
+                    continue;
+                }
+                resolved.add(picked.row());
+            }
+            log.info("dedupe id rules subRows={} mainRows={} mainIdsWithSubRows={} droppedMainRows={} keptRows={}",
+                    subRowCount, mainRowCount, mainIdsWithSubRows.size(), droppedMainRowCount, resolved.size());
+            return resolved;
+        }
+
+        /** conditionalMainId 非空表示该行是主链接行，需等收尾时看同主 ID 有无子链接行 */
+        private record PickedRow(DedupeCandidateRow row, String conditionalMainId) {
        }
    }

@@ -730,7 +771,7 @@ public class DedupeRunService {
        return String.join("/", parts);
    }

-    private String extractMainId(String text) {
+    static String extractMainId(String text) {
        if (text == null || text.isBlank()) {
            return "";
        }
@@ -744,7 +785,7 @@ public class DedupeRunService {
        return "";
    }

-    private boolean isIntegerId(String text) {
+    static boolean isIntegerId(String text) {
        if (text == null || text.isEmpty()) {
            return false;
        }
@@ -756,7 +797,7 @@ public class DedupeRunService {
        return true;
    }

-    private boolean isUnderscoreId(String text) {
+    static boolean isUnderscoreId(String text) {
        if (text == null || text.length() < 3) {
            return false;
        }
@@ -952,38 +993,6 @@ public class DedupeRunService {
        private AtomicInteger filteredFbaRows;
    }

-    private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
-    }
-
-    private static final class PendingMainIdGroup {
-        private String mainId;
-        private final List<DedupeCandidateRow> rows = new ArrayList<>();
-
-        private void add(String nextMainId, DedupeCandidateRow row) {
-            if (hasDifferentMainId(nextMainId)) {
-                rows.clear();
-            }
-            mainId = nextMainId;
-            rows.add(row);
-        }
-
-        private boolean hasDifferentMainId(String nextMainId) {
-            return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId));
-        }
-
-        private void discardIfSameMainId(String nextMainId) {
-            if (mainId != null && mainId.equals(nextMainId)) {
-                rows.clear();
-                mainId = null;
-            }
-        }
-
-        private void flush(List<DedupeCandidateRow> outputRows) {
-            if (!rows.isEmpty()) {
-                outputRows.addAll(rows);
-                rows.clear();
-            }
-            mainId = null;
-        }
+    record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
    }
 }