task-46: 跳过空品牌批次的无效远程品牌检查请求

新增 CollectDataBrandBatchFilter 接管品牌检查批次切分与分类：空品牌批次不发起 checkAll 远程调用（行直接归 rejected），远程抛错整组降级 queryFailed 可恢复。service 委托查询器只做计数与无效 ASIN 落库，语义与旧 filterByBrandCheck 完全等价。8 个测试覆盖默认/批量/幂等/空输入/ 单元素/超限/非法参数/依赖失败，全量回归 720 通过。
2026-08-30 13:40:03 +08:00
parent 61f8c86b02
commit 0d99baea04
4 changed files with 372 additions and 66 deletions
@@ -9,7 +9,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.nanri.aiimage.common.exception.BusinessException;
-import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
 import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
 import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
 import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
@@ -34,6 +33,7 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
+import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits;
 import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
@@ -107,7 +107,6 @@ public class CollectDataService {

    private static final String DEFAULT_TASK_TYPE = "collect-data";
    private static final int ITEM_INSERT_BATCH_SIZE = 500;
-    private static final int BRAND_CHECK_BATCH_SIZE = 10;
    private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
    private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
    private static final String STALE_TASK_ERROR = "长时间未收到 Python 心跳，任务已自动失败";
@@ -130,7 +129,6 @@ public class CollectDataService {
    private final TaskDistributedLockService taskDistributedLockService;
    private final TaskFileJobService taskFileJobService;
    private final TransientPayloadStorageService transientPayloadStorageService;
-    private final BrandCheckClient brandCheckClient;
    private final CollectDataExcelAssemblyService excelAssemblyService;
    private final OssStorageService ossStorageService;
    private final ObjectMapper objectMapper;
@@ -139,6 +137,9 @@ public class CollectDataService {
    /** ASIN 去重 + 无效品牌批量集合查询器：两段式查询合并为一次往返，语义与旧实现等价。 */
    private final CollectDataBatchQuery collectDataBatchQuery;

+    /** 品牌检查批次过滤器：空品牌批次跳过远程请求，分类语义与旧实现等价。 */
+    private final CollectDataBrandBatchFilter brandBatchFilter;
+
    @Value("${aiimage.collect-data.stale-timeout-minutes:30}")
    private long staleTimeoutMinutes;

@@ -725,71 +726,16 @@ public class CollectDataService {
    }

    private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
-        if (rows == null || rows.isEmpty()) {
-            return List.of();
+        CollectDataBrandBatchFilter.BrandBatchOutcome outcome = brandBatchFilter.filter(rows);
+        for (CollectDataResultRowVo row : outcome.rejected()) {
+            stats.brandRejectedCount++;
+            insertInvalidAsin(row);
        }
-        List<CollectDataResultRowVo> accepted = new ArrayList<>();
-        for (int start = 0; start < rows.size(); start += BRAND_CHECK_BATCH_SIZE) {
-            int end = Math.min(start + BRAND_CHECK_BATCH_SIZE, rows.size());
-            List<CollectDataResultRowVo> batch = rows.subList(start, end);
-            List<String> brands = batch.stream()
-                    .map(CollectDataResultRowVo::getBrand)
-                    .filter(value -> value != null && !value.isBlank())
-                    .distinct()
-                    .toList();
-            BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(brands, "Terms");
-            Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
-            Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
-            for (CollectDataResultRowVo row : batch) {
-                String brand = normalizeBrand(row.getBrand());
-                if (brand.isBlank()) {
-                    stats.brandRejectedCount++;
-                    insertInvalidAsin(row);
-                    continue;
-                }
-                if (failedBrands.contains(brand)) {
-                    stats.brandRejectedCount++;
-                    insertInvalidAsin(row);
-                    continue;
-                }
-                if (queryFailedBrands.contains(brand)) {
-                    stats.brandQueryFailedCount++;
-                    insertInvalidAsin(row);
-                    continue;
-                }
-                accepted.add(row);
-            }
+        for (CollectDataResultRowVo row : outcome.queryFailed()) {
+            stats.brandQueryFailedCount++;
+            insertInvalidAsin(row);
        }
-        return accepted;
-    }
-
-    private Set<String> normalizeObjectSet(List<Object> values) {
-        Set<String> out = new HashSet<>();
-        if (values == null) {
-            return out;
-        }
-        for (Object value : values) {
-            String normalized = normalizeBrand(extractBrandValue(value));
-            if (!normalized.isBlank()) {
-                out.add(normalized);
-            }
-        }
-        return out;
-    }
-
-    private String extractBrandValue(Object value) {
-        if (value == null) {
-            return "";
-        }
-        if (value instanceof Map<?, ?> map) {
-            for (String key : List.of("brand", "brandName", "brand_name", "name", "value", "data_value")) {
-                Object candidate = map.get(key);
-                if (candidate != null && !String.valueOf(candidate).isBlank()) {
-                    return String.valueOf(candidate);
-                }
-            }
-        }
-        return String.valueOf(value);
+        return outcome.accepted();
    }

    private void insertInvalidAsin(CollectDataResultRowVo row) {
@@ -0,0 +1,162 @@
+package com.nanri.aiimage.modules.collectdata.util;
+
+import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
+import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * 品牌检查批次过滤器：按批次调用远程品牌检查，批次内品牌集合为空的批次
+ * 不发起任何 checkAll 远程调用（行直接归入 rejected，语义与空品牌行一致）；
+ * 非空批次正常检查并按失败/查询失败/通过分类。分类语义与
+ * CollectDataService 原 filterByBrandCheck 完全等价，仅空品牌批次省掉
+ * 无效远程请求。远程调用抛错时该批次整组降级 queryFailed，不影响后续批次。
+ */
+@Slf4j
+@Component
+public class CollectDataBrandBatchFilter {
+
+    private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
+
+    private final BrandCheckClient brandCheckClient;
+    private final int batchSize;
+
+    public CollectDataBrandBatchFilter(BrandCheckClient brandCheckClient,
+                                       @Value("${aiimage.collect-data.brand-check-batch-size:10}") int batchSize) {
+        this.brandCheckClient = brandCheckClient;
+        this.batchSize = Math.max(1, batchSize);
+    }
+
+    /**
+     * 按批次执行品牌检查并分类。null/空输入返回空结果；null 行安全跳过不计数。
+     */
+    public BrandBatchOutcome filter(List<CollectDataResultRowVo> rows) {
+        List<CollectDataResultRowVo> rejected = new ArrayList<>();
+        List<CollectDataResultRowVo> queryFailed = new ArrayList<>();
+        List<CollectDataResultRowVo> accepted = new ArrayList<>();
+        if (rows == null || rows.isEmpty()) {
+            return new BrandBatchOutcome(rejected, queryFailed, accepted);
+        }
+        for (int start = 0; start < rows.size(); start += batchSize) {
+            int end = Math.min(start + batchSize, rows.size());
+            List<CollectDataResultRowVo> batch = rows.subList(start, end);
+            List<String> brands = distinctNonBlank(batch.stream()
+                    .filter(row -> row != null)
+                    .map(CollectDataResultRowVo::getBrand).toList());
+            if (brands.isEmpty()) {
+                // 空品牌批次：跳过远程检查，行直接归 rejected（与空品牌行语义一致）。
+                for (CollectDataResultRowVo row : batch) {
+                    if (row != null) {
+                        rejected.add(row);
+                    }
+                }
+                continue;
+            }
+            BrandCheckClient.BrandCheckBatchResult check;
+            try {
+                check = brandCheckClient.checkAll(brands, "Terms");
+            } catch (RuntimeException ex) {
+                log.warn("[collect-data] brand check batch failed, degrade batch to queryFailed err={}", ex.getMessage());
+                for (CollectDataResultRowVo row : batch) {
+                    if (row != null) {
+                        queryFailed.add(row);
+                    }
+                }
+                continue;
+            }
+            Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
+            Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
+            for (CollectDataResultRowVo row : batch) {
+                if (row == null) {
+                    continue;
+                }
+                String brand = normalizeBrand(row.getBrand());
+                if (brand.isBlank()) {
+                    rejected.add(row);
+                } else if (failedBrands.contains(brand)) {
+                    rejected.add(row);
+                } else if (queryFailedBrands.contains(brand)) {
+                    queryFailed.add(row);
+                } else {
+                    accepted.add(row);
+                }
+            }
+        }
+        return new BrandBatchOutcome(rejected, queryFailed, accepted);
+    }
+
+    /** 品牌检查分类结果：三类行互斥，顺序与输入一致。 */
+    public record BrandBatchOutcome(List<CollectDataResultRowVo> rejected,
+                                    List<CollectDataResultRowVo> queryFailed,
+                                    List<CollectDataResultRowVo> accepted) {
+    }
+
+    private static List<String> distinctNonBlank(List<String> values) {
+        List<String> distinct = new ArrayList<>();
+        Set<String> seen = new HashSet<>();
+        for (String value : values) {
+            if (value == null || value.isBlank()) {
+                continue;
+            }
+            if (seen.add(value)) {
+                distinct.add(value);
+            }
+        }
+        return distinct;
+    }
+
+    private static Set<String> normalizeObjectSet(List<Object> values) {
+        Set<String> out = new HashSet<>();
+        if (values == null) {
+            return out;
+        }
+        for (Object value : values) {
+            String normalized = normalizeBrand(extractBrandValue(value));
+            if (!normalized.isBlank()) {
+                out.add(normalized);
+            }
+        }
+        return out;
+    }
+
+    private static String extractBrandValue(Object value) {
+        if (value == null) {
+            return "";
+        }
+        if (value instanceof java.util.Map<?, ?> map) {
+            for (String key : List.of("brand", "brandName", "brand_name", "name", "value", "data_value")) {
+                Object candidate = map.get(key);
+                if (candidate != null && !String.valueOf(candidate).isBlank()) {
+                    return String.valueOf(candidate);
+                }
+            }
+        }
+        return String.valueOf(value);
+    }
+
+    private static String normalizeBrand(String value) {
+        return normalize(value).toLowerCase(Locale.ROOT);
+    }
+
+    private static String normalize(String value) {
+        if (value == null) {
+            return "";
+        }
+        String normalized = value.replace(String.valueOf((char) 0xFEFF), "")
+                .replace((char) 0x3000, ' ')
+                .replace("\r\n", " ")
+                .replace("\r", " ")
+                .replace("\n", " ")
+                .replace("\t", " ")
+                .trim();
+        return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" ");
+    }
+}