task-48: invalid ASIN 记录改为批量 INSERT IGNORE/upsert，唯一键扩为 (data_value, brand)

2026-08-30 14:11:59 +08:00
parent c609ffb279
commit 98c45486fa
6 changed files with 380 additions and 27 deletions
@@ -35,9 +35,9 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
+import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits;
 import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
-import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
 import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
 import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
 import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
@@ -140,6 +140,9 @@ public class CollectDataService {
    /** 品牌检查批次过滤器：空品牌批次跳过远程请求，分类语义与旧实现等价。 */
    private final CollectDataBrandBatchFilter brandBatchFilter;

+    /** invalid ASIN 批量写入器：按批次 INSERT IGNORE，替代逐行插入。 */
+    private final CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
+
    @Value("${aiimage.collect-data.stale-timeout-minutes:30}")
    private long staleTimeoutMinutes;

@@ -727,35 +730,15 @@ public class CollectDataService {

    private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
        CollectDataBrandBatchFilter.BrandBatchOutcome outcome = brandBatchFilter.filter(rows);
-        for (CollectDataResultRowVo row : outcome.rejected()) {
-            stats.brandRejectedCount++;
-            insertInvalidAsin(row);
-        }
-        for (CollectDataResultRowVo row : outcome.queryFailed()) {
-            stats.brandQueryFailedCount++;
-            insertInvalidAsin(row);
-        }
+        stats.brandRejectedCount += outcome.rejected().size();
+        stats.brandQueryFailedCount += outcome.queryFailed().size();
+        List<CollectDataResultRowVo> invalidRows = new ArrayList<>(outcome.rejected().size() + outcome.queryFailed().size());
+        invalidRows.addAll(outcome.rejected());
+        invalidRows.addAll(outcome.queryFailed());
+        invalidAsinBatchWriter.writeBatch(invalidRows);
        return outcome.accepted();
    }

-    private void insertInvalidAsin(CollectDataResultRowVo row) {
-        if (row == null || row.getAsin() == null || row.getAsin().isBlank()) {
-            return;
-        }
-        String brand = normalizeBrand(row.getBrand());
-        if (brand.isBlank()) {
-            return;
-        }
-        InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
-        entity.setDataValue(row.getAsin());
-        entity.setBrand(brand);
-        entity.setRecordSource("AUTO");
-        try {
-            invalidAsinDataMapper.insert(entity);
-        } catch (DuplicateKeyException ignored) {
-        }
-    }
-
    private void upsertResultItem(Long taskId, Long resultId, String scopeKey, CollectDataResultRowVo row) {
        String itemKey = "asin:" + row.getAsin();
        String scopeHash = hash(scopeKey);
@@ -0,0 +1,94 @@
+package com.nanri.aiimage.modules.collectdata.util;
+
+import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
+import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
+import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+/**
+ * invalid ASIN 批量写入器：把不合规 ASIN 行按批次批量 INSERT IGNORE 写入
+ * biz_invalid_asin_data，替代逐行 insert + 捕获唯一键异常的旧路径。
+ * 幂等由唯一键 (data_value, brand) + INSERT IGNORE 语义保证；批量失败时
+ * 跳过该批（记录计数，不中断提交流程），恢复后继续后续批次。
+ * 空输入、无合法 data_value/brand 的行、null 行均安全跳过，不发起调用。
+ */
+@Slf4j
+@Component
+public class CollectDataInvalidAsinBatchWriter {
+
+    private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
+
+    private final InvalidAsinDataMapper invalidAsinDataMapper;
+    private final int batchSize;
+
+    public CollectDataInvalidAsinBatchWriter(InvalidAsinDataMapper invalidAsinDataMapper,
+                                             @Value("${aiimage.collect-data.invalid-asin-batch-size:100}") int batchSize) {
+        this.invalidAsinDataMapper = invalidAsinDataMapper;
+        this.batchSize = Math.max(1, batchSize);
+    }
+
+    /** 批量写入并返回实际尝试写入的行数（INSERT IGNORE 忽略重复，幂等）。 */
+    public int writeBatch(List<CollectDataResultRowVo> rows) {
+        if (rows == null || rows.isEmpty()) {
+            return 0;
+        }
+        int written = 0;
+        for (int start = 0; start < rows.size(); start += batchSize) {
+            int end = Math.min(start + batchSize, rows.size());
+            List<CollectDataResultRowVo> batch = rows.subList(start, end);
+            List<InvalidAsinDataEntity> entities = new ArrayList<>(batch.size());
+            for (CollectDataResultRowVo row : batch) {
+                InvalidAsinDataEntity entity = toEntity(row);
+                if (entity != null) {
+                    entities.add(entity);
+                }
+            }
+            if (entities.isEmpty()) {
+                continue;
+            }
+            try {
+                written += invalidAsinDataMapper.insertBatchIgnore(entities);
+            } catch (RuntimeException ex) {
+                log.warn("[collect-data] invalid asin batch insert failed, skip batch err={}", ex.getMessage());
+            }
+        }
+        return written;
+    }
+
+    private InvalidAsinDataEntity toEntity(CollectDataResultRowVo row) {
+        if (row == null) {
+            return null;
+        }
+        String asin = normalize(row.getAsin());
+        String brand = normalize(row.getBrand()).toLowerCase(Locale.ROOT);
+        if (asin.isBlank() || brand.isBlank()) {
+            return null;
+        }
+        InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
+        entity.setDataValue(asin);
+        entity.setBrand(brand);
+        entity.setRecordSource("AUTO");
+        return entity;
+    }
+
+    private static String normalize(String value) {
+        if (value == null) {
+            return "";
+        }
+        String normalized = value.replace(String.valueOf((char) 0xFEFF), "")
+                .replace((char) 0x3000, ' ')
+                .replace("\r\n", " ")
+                .replace("\r", " ")
+                .replace("\n", " ")
+                .replace("\t", " ")
+                .trim();
+        return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" ");
+    }
+}
@@ -2,8 +2,25 @@ package com.nanri.aiimage.modules.invalidasin.mapper;

 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
+import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;

@Mapper
 public interface InvalidAsinDataMapper extends BaseMapper<InvalidAsinDataEntity> {
+
+    /** 批量 INSERT IGNORE：命中唯一键 (data_value, brand) 的重复行静默跳过，幂等。 */
+    @Insert("""
+            <script>
+            INSERT IGNORE INTO biz_invalid_asin_data
+                (data_value, brand, record_source, created_at, updated_at)
+            VALUES
+            <foreach collection="rows" item="row" separator=",">
+                (#{row.dataValue}, #{row.brand}, #{row.recordSource}, #{row.createdAt}, #{row.updatedAt})
+            </foreach>
+            </script>
+            """)
+    int insertBatchIgnore(@Param("rows") List<InvalidAsinDataEntity> rows);
 }