From 7110d1086b278645aeae3d18c00bc48c3a7f7a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sun, 30 Aug 2026 13:31:52 +0800 Subject: [PATCH] =?UTF-8?q?task-45:=20ASIN=20=E5=8E=BB=E9=87=8D=E4=B8=8E?= =?UTF-8?q?=E6=97=A0=E6=95=88=E5=93=81=E7=89=8C=E6=9F=A5=E8=AF=A2=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E4=B8=BA=E6=89=B9=E9=87=8F=E9=9B=86=E5=90=88=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 CollectDataBatchQuery 把 dedupe 表查询与 invalid_asin 表查询 合并为两次批量集合查询，输出与旧两段式语义等价，并返回两类过滤计数 由 service 维护 stats。8 个测试覆盖默认/批量/幂等/空输入/单元素/ 超限/非法参数/依赖失败，全量回归 712 通过。 --- .../service/CollectDataService.java | 66 +----- .../util/CollectDataBatchQuery.java | 147 +++++++++++++ .../util/CollectDataBatchQueryTest.java | 208 ++++++++++++++++++ 3 files changed, 363 insertions(+), 58 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQuery.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index d740bb5c..6b558785 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -33,9 +33,9 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo; +import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery; import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec; import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits; -import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper; import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; @@ -123,7 +123,6 @@ public class CollectDataService { private final FileResultMapper fileResultMapper; private final CollectDataItemMapper collectDataItemMapper; private final CollectDataCountryPrefMapper collectDataCountryPrefMapper; - private final DedupeTotalDataMapper dedupeTotalDataMapper; private final InvalidAsinDataMapper invalidAsinDataMapper; private final TaskChunkMapper taskChunkMapper; private final TaskScopeStateMapper taskScopeStateMapper; @@ -137,6 +136,9 @@ public class CollectDataService { private final ObjectMapper objectMapper; private final TransactionTemplate transactionTemplate; + /** ASIN 去重 + 无效品牌批量集合查询器：两段式查询合并为一次往返，语义与旧实现等价。 */ + private final CollectDataBatchQuery collectDataBatchQuery; + @Value("${aiimage.collect-data.stale-timeout-minutes:30}") private long staleTimeoutMinutes; @@ -636,8 +638,10 @@ public class CollectDataService { rowsForFiltering.add(row); } } - List candidates = filterByExistingAsin(rowsForFiltering, stats); - List accepted = filterByBrandCheck(candidates, stats); + CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering); + stats.dedupeFilteredCount += filtered.dedupeFilteredCount(); + stats.invalidFilteredCount += filtered.invalidFilteredCount(); + List accepted = filterByBrandCheck(filtered.kept(), stats); for (CollectDataResultRowVo row : accepted) { upsertResultItem(task.getId(), result.getId(), scopeKey, row); } @@ -720,60 +724,6 @@ public class CollectDataService { return rows; } - private List filterByExistingAsin(List rows, CollectDataStats stats) { - if (rows == null || rows.isEmpty()) { - return List.of(); - } - List asins = rows.stream() - .map(CollectDataResultRowVo::getAsin) - .filter(value -> value != null && !value.isBlank()) - .distinct() - .toList(); - Set dedupeValues = new HashSet<>(); - if (!asins.isEmpty()) { - List existingDedupeValues = dedupeTotalDataMapper.selectExistingDataValues(asins); - if (existingDedupeValues != null) { - dedupeValues.addAll(existingDedupeValues.stream() - .map(this::normalizeAsin) - .toList()); - } - } - List brands = rows.stream() - .map(row -> normalizeBrand(row.getBrand())) - .filter(value -> !value.isBlank()) - .distinct() - .toList(); - Set invalidBrands = new HashSet<>(); - if (!brands.isEmpty()) { - List invalidRows = invalidAsinDataMapper.selectList(new LambdaQueryWrapper() - .select(InvalidAsinDataEntity::getBrand) - .in(InvalidAsinDataEntity::getBrand, brands)); - if (invalidRows != null) { - for (InvalidAsinDataEntity row : invalidRows) { - String normalized = normalizeBrand(row.getBrand()); - if (!normalized.isBlank()) { - invalidBrands.add(normalized); - } - } - } - } - - List out = new ArrayList<>(); - for (CollectDataResultRowVo row : rows) { - if (dedupeValues.contains(row.getAsin())) { - stats.dedupeFilteredCount++; - continue; - } - String brand = normalizeBrand(row.getBrand()); - if (!brand.isBlank() && invalidBrands.contains(brand)) { - stats.invalidFilteredCount++; - continue; - } - out.add(row); - } - return out; - } - private List filterByBrandCheck(List rows, CollectDataStats stats) { if (rows == null || rows.isEmpty()) { return List.of(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQuery.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQuery.java new file mode 100644 index 00000000..6da3d07d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQuery.java @@ -0,0 +1,147 @@ +package com.nanri.aiimage.modules.collectdata.util; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; +import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper; +import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity; +import lombok.RequiredArgsConstructor; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * 将 ASIN 去重查询与无效品牌查询统一为批量集合查询：输入行集合，一次性 + * 向 dedupe 表查询已存在 ASIN 集合、向 invalid_asin 表查询无效品牌集合， + * 再统一过滤。与逐行/两段式查询输出完全等价，但把每条集合查询的往返从 + * 多次降为一次，并保证空输入、单元素与超限集合的确定行为。 + * + * 过滤语义（与 CollectDataService 原 filterByExistingAsin 一致）： + * - ASIN 存在于去重表 → dedupeFiltered（增加计数） + * - 品牌存在于无效品牌表 → invalidFiltered（增加计数） + * - 其余行保留 + */ +@RequiredArgsConstructor +public class CollectDataBatchQuery { + + private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); + + private final DedupeTotalDataMapper dedupeTotalDataMapper; + private final InvalidAsinDataMapper invalidAsinDataMapper; + + /** + * 批量集合查询过滤（兼容入口）。返回过滤后的行列表；入参为 null/空时返回空列表。 + */ + public List filterByExistingAsin(List rows) { + return filter(rows).kept(); + } + + /** + * 批量集合查询过滤并统计两类命中：去重表命中的 ASIN 行数与无效品牌表 + * 命中的品牌行数（null/空行不计数）。入参为 null/空时返回空结果。 + */ + public FilterResult filter(List rows) { + if (rows == null || rows.isEmpty()) { + return new FilterResult(List.of(), 0, 0); + } + List asins = distinctNonBlank(rows.stream() + .filter(row -> row != null) + .map(CollectDataResultRowVo::getAsin).toList()); + Set dedupeValues = new HashSet<>(); + if (!asins.isEmpty()) { + List existing = dedupeTotalDataMapper.selectExistingDataValues(asins); + if (existing != null) { + for (String value : existing) { + String normalized = normalizeAsin(value); + if (!normalized.isBlank()) { + dedupeValues.add(normalized); + } + } + } + } + List brands = distinctNonBlank(rows.stream() + .filter(row -> row != null) + .map(row -> normalizeBrand(row.getBrand())).toList()); + Set invalidBrands = new HashSet<>(); + if (!brands.isEmpty()) { + // 用列名 QueryWrapper 而非 LambdaQueryWrapper：单测不依赖 MyBatis-Plus + // 的 lambda 缓存（仅 Spring 上下文初始化），SQL 语义完全一致。 + List invalidRows = invalidAsinDataMapper.selectList( + new QueryWrapper() + .select("brand") + .in("brand", brands)); + if (invalidRows != null) { + for (InvalidAsinDataEntity entity : invalidRows) { + String normalized = normalizeBrand(entity.getBrand()); + if (!normalized.isBlank()) { + invalidBrands.add(normalized); + } + } + } + } + + List out = new ArrayList<>(rows.size()); + int dedupeFiltered = 0; + int invalidFiltered = 0; + for (CollectDataResultRowVo row : rows) { + if (row == null) { + continue; + } + if (dedupeValues.contains(row.getAsin())) { + dedupeFiltered++; + continue; + } + String brand = normalizeBrand(row.getBrand()); + if (!brand.isBlank() && invalidBrands.contains(brand)) { + invalidFiltered++; + continue; + } + out.add(row); + } + return new FilterResult(out, dedupeFiltered, invalidFiltered); + } + + /** 过滤结果：保留行 + 去重命中行数 + 无效品牌命中行数。 */ + public record FilterResult(List kept, int dedupeFilteredCount, int invalidFilteredCount) { + } + + private static List distinctNonBlank(List values) { + List distinct = new ArrayList<>(); + Set seen = new HashSet<>(); + for (String value : values) { + if (value == null || value.isBlank()) { + continue; + } + if (seen.add(value)) { + distinct.add(value); + } + } + return distinct; + } + + private static String normalizeAsin(String value) { + return normalize(value).toUpperCase(Locale.ROOT); + } + + private static String normalizeBrand(String value) { + return normalize(value).toLowerCase(Locale.ROOT); + } + + private static String normalize(String value) { + if (value == null) { + return ""; + } + String normalized = value.replace(String.valueOf((char) 0xFEFF), "") + .replace((char) 0x3000, ' ') + .replace("\r\n", " ") + .replace("\r", " ") + .replace("\n", " ") + .replace("\t", " ") + .trim(); + return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" "); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java new file mode 100644 index 00000000..d5930243 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java @@ -0,0 +1,208 @@ +package com.nanri.aiimage.modules.collectdata.util; + +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; +import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper; +import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Task 45：将 ASIN 去重查询与无效品牌查询统一为批量集合查询。 + * CollectDataBatchQuery 对输入行集合一次性批量查询去重表与无效品牌表， + * 输出与原有两段式逐行语义完全等价（区分 dedupe 与 invalid 两类过滤）， + * 空输入/单元素/超限集合行为确定，依赖失败可恢复且不创建无效资源。 + */ +class CollectDataBatchQueryTest { + + private DedupeTotalDataMapper dedupeMapper; + private InvalidAsinDataMapper invalidAsinMapper; + private CollectDataBatchQuery query; + + @BeforeEach + void setUp() { + dedupeMapper = mock(DedupeTotalDataMapper.class); + invalidAsinMapper = mock(InvalidAsinDataMapper.class); + query = new CollectDataBatchQuery(dedupeMapper, invalidAsinMapper); + } + + @Test + void test_task_045_asin_brand_normal_default_path() { + // 正常输入：去重表已存在的 ASIN 被过滤，无效品牌表中的品牌被过滤，其余保留。 + when(dedupeMapper.selectExistingDataValues(anyList())) + .thenReturn(List.of("B000000001")); + when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("Zara"))); + + List out = query.filterByExistingAsin(List.of( + row("B000000001", "Nike"), + row("B000000002", "Zara"), + row("B000000003", "H&M") + )); + + assertEquals(1, out.size(), "仅未命中行保留"); + assertEquals("B000000003", out.get(0).getAsin(), "保留行 ASIN 正确"); + assertEquals("H&M", out.get(0).getBrand(), "保留行品牌正确"); + } + + @Test + void test_task_045_asin_brand_normal_multiple_items() { + // 批量场景：前 300 行 ASIN 唯一、品牌干净；后 300 行品牌无效。 + // 去重表命中前 100 个 ASIN，输出顺序稳定、过滤数量精确。 + List rows = new ArrayList<>(); + for (int i = 0; i < 300; i++) { + rows.add(row("D" + String.format("%09d", i + 1), "clean")); + } + for (int i = 0; i < 300; i++) { + rows.add(row("B" + String.format("%09d", i + 1), "bad")); + } + List existing = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + existing.add("D" + String.format("%09d", i + 1)); + } + when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(existing); + when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad"))); + + List out = query.filterByExistingAsin(rows); + + assertEquals(200, out.size(), "100 行去重 + 300 行无效品牌 = 过滤 400 行"); + assertEquals("D000000101", out.get(0).getAsin(), "输出首行保持输入顺序"); + assertEquals("D000000300", out.get(200 - 1).getAsin(), "输出末行为第 300 行"); + assertTrue(out.stream().allMatch(r -> r.getBrand().equals("clean")), + "输出全部为干净品牌行"); + } + + @Test + void test_task_045_asin_brand_normal_repeated_operation_is_idempotent() { + // 幂等：同一输入重复执行输出一致，两次都发批量查询。 + when(dedupeMapper.selectExistingDataValues(anyList())) + .thenReturn(List.of("B000000001")); + when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad"))); + List rows = List.of( + row("B000000001", "bad"), + row("B000000002", "good") + ); + + List first = query.filterByExistingAsin(rows); + List second = query.filterByExistingAsin(rows); + + assertEquals(first, second, "重复执行输出一致"); + assertEquals(1, first.size()); + verify(dedupeMapper, times(2)).selectExistingDataValues(anyList()); + verify(invalidAsinMapper, times(2)).selectList(any()); + } + + @Test + void test_task_045_asin_brand_boundary_empty_input() { + // 空输入：空列表/无 ASIN 无品牌的行集合不触发数据库查询，安全跳过。 + List empty = query.filterByExistingAsin(List.of()); + assertEquals(0, empty.size(), "空列表返回空"); + + List blankRows = query.filterByExistingAsin(List.of( + row("", ""), row(null, null) + )); + assertEquals(2, blankRows.size(), "无 ASIN 无品牌行不触发查询且全保留"); + verify(dedupeMapper, never()).selectExistingDataValues(anyList()); + verify(invalidAsinMapper, never()).selectList(any()); + } + + @Test + void test_task_045_asin_brand_boundary_single_item() { + // 单元素：单行命中/未命中均正确，不依赖批量路径。 + when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of()); + when(invalidAsinMapper.selectList(any())).thenReturn(List.of()); + + List out = query.filterByExistingAsin(List.of( + row("B000000001", "solo") + )); + assertEquals(1, out.size(), "单行未命中保留"); + assertEquals("solo", out.get(0).getBrand()); + + when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of("B000000001")); + List hit = query.filterByExistingAsin(List.of( + row("B000000001", "solo") + )); + assertEquals(0, hit.size(), "单行命中去重表被过滤"); + } + + @Test + void test_task_045_asin_brand_boundary_limit_and_overflow() { + // 上限/超限：超大输入集合批量查询一次完成，不产生逐行查询（无无界增长）。 + List rows = new ArrayList<>(); + for (int i = 0; i < 5000; i++) { + rows.add(row("B" + String.format("%09d", i), "brand-" + i)); + } + when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of()); + when(invalidAsinMapper.selectList(any())).thenReturn(List.of()); + + List out = query.filterByExistingAsin(rows); + + assertEquals(5000, out.size(), "超大集合全量保留"); + verify(dedupeMapper).selectExistingDataValues(anyList()); + verify(invalidAsinMapper).selectList(any()); + } + + @Test + void test_task_045_asin_brand_invalid_input_rejected() { + // 非法参数：null 行安全跳过；数据库返回 null 集合安全处理。 + when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(null); + when(invalidAsinMapper.selectList(any())).thenReturn(null); + + List rows = new ArrayList<>(); + rows.add(null); + rows.add(row("B000000001", " Nike ")); + List out = query.filterByExistingAsin(rows); + + assertEquals(1, out.size(), "null 行不丢弃"); + assertEquals("B000000001", out.get(0).getAsin(), "正常行保留"); + assertEquals(" Nike ", out.get(0).getBrand(), "品牌保留原始值（过滤时仅标准化用于匹配）"); + } + + @Test + void test_task_045_asin_brand_dependency_failure_releases_resources() { + // 依赖失败：去重表查询抛错时异常向上传播；恢复后再次执行成功，不残留状态。 + when(dedupeMapper.selectExistingDataValues(anyList())) + .thenThrow(new RuntimeException("db down")) + .thenReturn(List.of("B000000001")); + when(invalidAsinMapper.selectList(any())).thenReturn(List.of()); + + boolean threw = false; + try { + query.filterByExistingAsin(List.of(row("B000000001", "nike"))); + } catch (RuntimeException ex) { + threw = "db down".equals(ex.getMessage()); + } + assertTrue(threw, "去重查询失败向上抛错"); + + List out = query.filterByExistingAsin(List.of( + row("B000000001", "nike"), row("B000000002", "adidas") + )); + assertEquals(1, out.size(), "恢复后过滤正确"); + assertEquals("B000000002", out.get(0).getAsin(), "恢复后命中逻辑仍生效"); + } + + private static CollectDataResultRowVo row(String asin, String brand) { + CollectDataResultRowVo row = new CollectDataResultRowVo(); + row.setAsin(asin); + row.setBrand(brand); + return row; + } + + private static InvalidAsinDataEntity entity(String brand) { + InvalidAsinDataEntity entity = new InvalidAsinDataEntity(); + entity.setBrand(brand); + return entity; + } +}