task-45: ASIN 去重与无效品牌查询统一为批量集合查询

新增 CollectDataBatchQuery 把 dedupe 表查询与 invalid_asin 表查询合并为两次批量集合查询，输出与旧两段式语义等价，并返回两类过滤计数由 service 维护 stats。8 个测试覆盖默认/批量/幂等/空输入/单元素/ 超限/非法参数/依赖失败，全量回归 712 通过。
2026-08-30 13:31:52 +08:00
parent d23485029d
commit 7110d1086b
3 changed files with 363 additions and 58 deletions
@@ -33,9 +33,9 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
+import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits;
-import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
 import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
 import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
 import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
@@ -123,7 +123,6 @@ public class CollectDataService {
    private final FileResultMapper fileResultMapper;
    private final CollectDataItemMapper collectDataItemMapper;
    private final CollectDataCountryPrefMapper collectDataCountryPrefMapper;
-    private final DedupeTotalDataMapper dedupeTotalDataMapper;
    private final InvalidAsinDataMapper invalidAsinDataMapper;
    private final TaskChunkMapper taskChunkMapper;
    private final TaskScopeStateMapper taskScopeStateMapper;
@@ -137,6 +136,9 @@ public class CollectDataService {
    private final ObjectMapper objectMapper;
    private final TransactionTemplate transactionTemplate;

+    /** ASIN 去重 + 无效品牌批量集合查询器：两段式查询合并为一次往返，语义与旧实现等价。 */
+    private final CollectDataBatchQuery collectDataBatchQuery;
+
    @Value("${aiimage.collect-data.stale-timeout-minutes:30}")
    private long staleTimeoutMinutes;

@@ -636,8 +638,10 @@ public class CollectDataService {
                    rowsForFiltering.add(row);
                }
            }
-            List<CollectDataResultRowVo> candidates = filterByExistingAsin(rowsForFiltering, stats);
-            List<CollectDataResultRowVo> accepted = filterByBrandCheck(candidates, stats);
+            CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
+            stats.dedupeFilteredCount += filtered.dedupeFilteredCount();
+            stats.invalidFilteredCount += filtered.invalidFilteredCount();
+            List<CollectDataResultRowVo> accepted = filterByBrandCheck(filtered.kept(), stats);
            for (CollectDataResultRowVo row : accepted) {
                upsertResultItem(task.getId(), result.getId(), scopeKey, row);
            }
@@ -720,60 +724,6 @@ public class CollectDataService {
        return rows;
    }

-    private List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
-        if (rows == null || rows.isEmpty()) {
-            return List.of();
-        }
-        List<String> asins = rows.stream()
-                .map(CollectDataResultRowVo::getAsin)
-                .filter(value -> value != null && !value.isBlank())
-                .distinct()
-                .toList();
-        Set<String> dedupeValues = new HashSet<>();
-        if (!asins.isEmpty()) {
-            List<String> existingDedupeValues = dedupeTotalDataMapper.selectExistingDataValues(asins);
-            if (existingDedupeValues != null) {
-                dedupeValues.addAll(existingDedupeValues.stream()
-                        .map(this::normalizeAsin)
-                        .toList());
-            }
-        }
-        List<String> brands = rows.stream()
-                .map(row -> normalizeBrand(row.getBrand()))
-                .filter(value -> !value.isBlank())
-                .distinct()
-                .toList();
-        Set<String> invalidBrands = new HashSet<>();
-        if (!brands.isEmpty()) {
-            List<InvalidAsinDataEntity> invalidRows = invalidAsinDataMapper.selectList(new LambdaQueryWrapper<InvalidAsinDataEntity>()
-                    .select(InvalidAsinDataEntity::getBrand)
-                    .in(InvalidAsinDataEntity::getBrand, brands));
-            if (invalidRows != null) {
-                for (InvalidAsinDataEntity row : invalidRows) {
-                    String normalized = normalizeBrand(row.getBrand());
-                    if (!normalized.isBlank()) {
-                        invalidBrands.add(normalized);
-                    }
-                }
-            }
-        }
-
-        List<CollectDataResultRowVo> out = new ArrayList<>();
-        for (CollectDataResultRowVo row : rows) {
-            if (dedupeValues.contains(row.getAsin())) {
-                stats.dedupeFilteredCount++;
-                continue;
-            }
-            String brand = normalizeBrand(row.getBrand());
-            if (!brand.isBlank() && invalidBrands.contains(brand)) {
-                stats.invalidFilteredCount++;
-                continue;
-            }
-            out.add(row);
-        }
-        return out;
-    }
-
    private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
        if (rows == null || rows.isEmpty()) {
            return List.of();
@@ -0,0 +1,147 @@
+package com.nanri.aiimage.modules.collectdata.util;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
+import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
+import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
+import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
+import lombok.RequiredArgsConstructor;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * 将 ASIN 去重查询与无效品牌查询统一为批量集合查询：输入行集合，一次性
+ * 向 dedupe 表查询已存在 ASIN 集合、向 invalid_asin 表查询无效品牌集合，
+ * 再统一过滤。与逐行/两段式查询输出完全等价，但把每条集合查询的往返从
+ * 多次降为一次，并保证空输入、单元素与超限集合的确定行为。
+ *
+ * 过滤语义（与 CollectDataService 原 filterByExistingAsin 一致）：
+ * - ASIN 存在于去重表 → dedupeFiltered（增加计数）
+ * - 品牌存在于无效品牌表 → invalidFiltered（增加计数）
+ * - 其余行保留
+ */
+@RequiredArgsConstructor
+public class CollectDataBatchQuery {
+
+    private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
+
+    private final DedupeTotalDataMapper dedupeTotalDataMapper;
+    private final InvalidAsinDataMapper invalidAsinDataMapper;
+
+    /**
+     * 批量集合查询过滤（兼容入口）。返回过滤后的行列表；入参为 null/空时返回空列表。
+     */
+    public List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows) {
+        return filter(rows).kept();
+    }
+
+    /**
+     * 批量集合查询过滤并统计两类命中：去重表命中的 ASIN 行数与无效品牌表
+     * 命中的品牌行数（null/空行不计数）。入参为 null/空时返回空结果。
+     */
+    public FilterResult filter(List<CollectDataResultRowVo> rows) {
+        if (rows == null || rows.isEmpty()) {
+            return new FilterResult(List.of(), 0, 0);
+        }
+        List<String> asins = distinctNonBlank(rows.stream()
+                .filter(row -> row != null)
+                .map(CollectDataResultRowVo::getAsin).toList());
+        Set<String> dedupeValues = new HashSet<>();
+        if (!asins.isEmpty()) {
+            List<String> existing = dedupeTotalDataMapper.selectExistingDataValues(asins);
+            if (existing != null) {
+                for (String value : existing) {
+                    String normalized = normalizeAsin(value);
+                    if (!normalized.isBlank()) {
+                        dedupeValues.add(normalized);
+                    }
+                }
+            }
+        }
+        List<String> brands = distinctNonBlank(rows.stream()
+                .filter(row -> row != null)
+                .map(row -> normalizeBrand(row.getBrand())).toList());
+        Set<String> invalidBrands = new HashSet<>();
+        if (!brands.isEmpty()) {
+            // 用列名 QueryWrapper 而非 LambdaQueryWrapper：单测不依赖 MyBatis-Plus
+            // 的 lambda 缓存（仅 Spring 上下文初始化），SQL 语义完全一致。
+            List<InvalidAsinDataEntity> invalidRows = invalidAsinDataMapper.selectList(
+                    new QueryWrapper<InvalidAsinDataEntity>()
+                            .select("brand")
+                            .in("brand", brands));
+            if (invalidRows != null) {
+                for (InvalidAsinDataEntity entity : invalidRows) {
+                    String normalized = normalizeBrand(entity.getBrand());
+                    if (!normalized.isBlank()) {
+                        invalidBrands.add(normalized);
+                    }
+                }
+            }
+        }
+
+        List<CollectDataResultRowVo> out = new ArrayList<>(rows.size());
+        int dedupeFiltered = 0;
+        int invalidFiltered = 0;
+        for (CollectDataResultRowVo row : rows) {
+            if (row == null) {
+                continue;
+            }
+            if (dedupeValues.contains(row.getAsin())) {
+                dedupeFiltered++;
+                continue;
+            }
+            String brand = normalizeBrand(row.getBrand());
+            if (!brand.isBlank() && invalidBrands.contains(brand)) {
+                invalidFiltered++;
+                continue;
+            }
+            out.add(row);
+        }
+        return new FilterResult(out, dedupeFiltered, invalidFiltered);
+    }
+
+    /** 过滤结果：保留行 + 去重命中行数 + 无效品牌命中行数。 */
+    public record FilterResult(List<CollectDataResultRowVo> kept, int dedupeFilteredCount, int invalidFilteredCount) {
+    }
+
+    private static List<String> distinctNonBlank(List<String> values) {
+        List<String> distinct = new ArrayList<>();
+        Set<String> seen = new HashSet<>();
+        for (String value : values) {
+            if (value == null || value.isBlank()) {
+                continue;
+            }
+            if (seen.add(value)) {
+                distinct.add(value);
+            }
+        }
+        return distinct;
+    }
+
+    private static String normalizeAsin(String value) {
+        return normalize(value).toUpperCase(Locale.ROOT);
+    }
+
+    private static String normalizeBrand(String value) {
+        return normalize(value).toLowerCase(Locale.ROOT);
+    }
+
+    private static String normalize(String value) {
+        if (value == null) {
+            return "";
+        }
+        String normalized = value.replace(String.valueOf((char) 0xFEFF), "")
+                .replace((char) 0x3000, ' ')
+                .replace("\r\n", " ")
+                .replace("\r", " ")
+                .replace("\n", " ")
+                .replace("\t", " ")
+                .trim();
+        return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" ");
+    }
+}