feat(需求): 撞款重复检查改查数据库——①采集明细先落库再更新文件(V111 biz_shop_data_crawl_item行级明细,归档时countryResults展开,同店同日幂等替换) ②撞款扫描换源直接查明细表(不再解析OSS xlsx,修复旧文件表头差异致4家店静默丢弃) ③存量Excel数据自动迁移(日报快照row_payload优先+OSS宽松表头回退:ASIN码/小写兼容) ④移除84家店铺全量空态注入(重复检查基数=已采集店,超管口径8家) ⑤新增明细聚合/宽松解析单测6个

2026-09-07 02:38:39 +08:00
parent 678605e9fb
commit 5f8c469b8e
24 changed files with 638 additions and 160 deletions
@@ -136,6 +136,7 @@ class ShopDataCrawlChunkUpsertTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -172,6 +172,7 @@ class ShopDataCrawlCleanupTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));
        ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
@@ -145,6 +145,7 @@ class ShopDataCrawlDailyFileIncrementalTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -151,6 +151,7 @@ class ShopDataCrawlDailyFileJobSplitTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -151,6 +151,7 @@ class ShopDataCrawlDailyFileLockTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -133,6 +133,7 @@ class ShopDataCrawlLightweightProgressTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -132,6 +132,7 @@ class ShopDataCrawlOwnerColumnTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -145,6 +145,7 @@ class ShopDataCrawlProgressQueryTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -132,6 +132,7 @@ class ShopDataCrawlRowDedupKeyTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -133,6 +133,7 @@ class ShopDataCrawlScopeCounterTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -134,6 +134,7 @@ class ShopDataCrawlScopeMergeTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -128,6 +128,7 @@ class ShopDataCrawlTaskServiceChunkTest {
                transientPayloadStorageService,
                instanceMetadata,
                dailyFileService,
+                mock(ShopDataCrawlItemStoreService.class),
                null,
                mock(TaskProgressLightAssembler.class));

@@ -0,0 +1,52 @@
+package com.nanri.aiimage.modules.shopduplicatecheck.service;
+
+import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ShopLatestItemRowDto;
+import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow;
+import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * 撞款明细数据源（biz_shop_data_crawl_item）→ ShopParsed 转换：
+ * 按店分组、空 ASIN 行过滤、国家去重大写、分组标签注入。
+ */
+class ShopDataDuplicateCheckScanServiceTest {
+
+    @Test
+    void buildParsedShops_groupsByShopAndNormalizesCountry() {
+        List<ShopLatestItemRowDto> rows = List.of(
+                new ShopLatestItemRowDto("店A", "uk", "B0001", "2026-09-01 10:00", "9.90", "品牌甲"),
+                new ShopLatestItemRowDto("店A", "DE", "B0001", "2026-09-01 11:00", "9.90", "品牌甲"),
+                new ShopLatestItemRowDto("店B", "FR", "B0002", "2026-09-02 09:00", "19.9", "品牌乙"),
+                new ShopLatestItemRowDto("店B", "FR", "", "2026-09-02 09:01", "1.00", "空ASIN行"),
+                new ShopLatestItemRowDto(null, "UK", "B0003", "", "", ""));
+        List<ShopParsed> parsed = ShopDataDuplicateCheckScanService.buildParsedShops(
+                rows, Map.of("店b", "华东组"));
+
+        assertEquals(3, parsed.size(), "按店分组（含未命名兜底）");
+        ShopParsed shopA = parsed.stream().filter(s -> s.shopName().equals("店A")).findFirst().orElseThrow();
+        assertEquals(List.of("UK", "DE"), shopA.countryCodes(), "国家码去重大写");
+        assertEquals(2, shopA.rows().size());
+        RawRow ukRow = shopA.rows().get(0);
+        assertEquals("B0001", ukRow.asin());
+        assertEquals("UK", ukRow.country());
+
+        ShopParsed shopB = parsed.stream().filter(s -> s.shopName().equals("店B")).findFirst().orElseThrow();
+        assertEquals(1, shopB.rows().size(), "空 ASIN 行过滤");
+        assertEquals("华东组", shopB.groupName(), "分组标签注入（shopKey 大小写不敏感）");
+
+        ShopParsed unnamed = parsed.stream().filter(s -> s.shopName().equals("未命名")).findFirst().orElseThrow();
+        assertEquals(1, unnamed.rows().size(), "无店铺名兜底未命名");
+        assertEquals("", unnamed.rows().get(0).brand(), "缺省字段空串兜底");
+    }
+
+    @Test
+    void buildParsedShops_emptyInputReturnsEmpty() {
+        assertTrue(ShopDataDuplicateCheckScanService.buildParsedShops(List.of(), Map.of()).isEmpty());
+    }
+}
@@ -0,0 +1,78 @@
+package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
+
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * 宽松表头解析（兼容旧版文件）：ASIN 列大小写/空白/「ASIN码」类变体均可识别，
+ * 是存量 Excel 数据迁移入库的前提（旧版 4 家店因表头差异被整店丢弃的根因）。
+ */
+class DuplicateCheckWorkbookParserTest {
+
+    private static byte[] workbookWith(boolean stream, String sheetName, String[] headers, String[] values) throws Exception {
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            if (stream) {
+                SXSSFWorkbook workbook = new SXSSFWorkbook();
+                write(workbook, sheetName, headers, values);
+                workbook.write(out);
+                workbook.dispose();
+            } else {
+                XSSFWorkbook workbook = new XSSFWorkbook();
+                write(workbook, sheetName, headers, values);
+                workbook.write(out);
+                workbook.close();
+            }
+            return out.toByteArray();
+        }
+    }
+
+    private static void write(org.apache.poi.ss.usermodel.Workbook workbook, String sheetName,
+                              String[] headers, String[] values) {
+        org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet(sheetName);
+        org.apache.poi.ss.usermodel.Row header = sheet.createRow(0);
+        for (int i = 0; i < headers.length; i++) {
+            header.createCell(i).setCellValue(headers[i]);
+        }
+        org.apache.poi.ss.usermodel.Row row = sheet.createRow(1);
+        for (int i = 0; i < values.length; i++) {
+            row.createCell(i).setCellValue(values[i]);
+        }
+    }
+
+    @Test
+    void parsesLegacyAsinCodeHeader() throws Exception {
+        byte[] bytes = workbookWith(false, "英国",
+                new String[]{"ASIN码", "日期", "价格", "品牌"},
+                new String[]{"B0OLD001", "2026年8月15日 上午10:59", "£34.46", "DOLGABA"});
+        var rows = DuplicateCheckWorkbookParser.parse(bytes);
+        assertEquals(1, rows.size(), "「ASIN码」表头变体可识别");
+        assertEquals("B0OLD001", rows.get(0).asin());
+        assertEquals("UK", rows.get(0).country(), "中文 sheet 名映射 UK");
+        assertEquals("2026年8月15日 上午10:59", rows.get(0).date());
+        assertEquals("DOLGABA", rows.get(0).brand());
+    }
+
+    @Test
+    void parsesLowercaseAsinHeader() throws Exception {
+        byte[] bytes = workbookWith(true, "asin",
+                new String[]{"asin", "date", "price", "brand"},
+                new String[]{"B0LOW001", "2026-08-15 10:59", "9.99", "X"});
+        var rows = DuplicateCheckWorkbookParser.parse(bytes);
+        assertEquals(1, rows.size(), "小写 asin 表头可识别");
+        assertEquals("B0LOW001", rows.get(0).asin());
+        assertEquals(rows.get(0).country(), "", "未知 sheet 名不丢行（国家为空）");
+    }
+
+    @Test
+    void skipsSheetWithoutAsinColumn() throws Exception {
+        byte[] bytes = workbookWith(false, "德国",
+                new String[]{"日期", "价格"},
+                new String[]{"2026-08-15", "9.99"});
+        assertEquals(0, DuplicateCheckWorkbookParser.parse(bytes).size(), "无 ASIN 列的 sheet 跳过");
+    }
+}