task-21: 建立店铺抓取性能基线夹具（1k/5k 行、多国家、图片成功/失败）

ShopDataCrawlPerfFixture 提供单店铺多国家抓取结果的确定性生成器： - generateItems：1k/5k 行、1-5 国家、图片成功/失败（前 N 行无缩略图）场景； - samplePayload：行数/chunk 数/payload 字节/图片成功与失败行数采样。全量测试 516 通过。
2026-08-29 18:24:10 +08:00
parent 3d5fd42bfa
commit 81da7a182e
2 changed files with 350 additions and 0 deletions
@@ -0,0 +1,154 @@
+package com.nanri.aiimage.modules.shopdatacrawl.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
+import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
+import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 店铺抓取性能基线夹具：单店铺 1k/5k 行、多国家、图片成功/失败场景的确定性生成器。
+ * 同一输入必然产生相同输出（幂等）；失败场景通过 failedThumbRows 指定前 N 行为无缩略图行。
+ * 上限约束：单次最多 MAX_ROWS 行，防止基线夹具本身造成无界内存增长。
+ */
+@Slf4j
+public class ShopDataCrawlPerfFixture {
+
+    public static final int MAX_ROWS = 5000;
+    public static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
+    private static final String THUMB_HOST = "https://thumb.example/";
+
+    private final ObjectMapper objectMapper;
+
+    public ShopDataCrawlPerfFixture(ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
+    }
+
+    /**
+     * 生成单店铺 countryCount 个国家的 rowCount 行抓取结果。
+     * failedThumbRows 指定前 N 行无商品图片（图片失败场景），其余行带确定性占位图 URL。
+     * 图片统计以行内 commodityImage 实际内容为准（非空=成功，空=失败）。
+     */
+    public List<ShopDataCrawlResultItemVo> generateItems(String shopName, int rowCount, int countryCount,
+                                                         boolean withImages, int failedThumbRows) {
+        if (shopName == null || shopName.isBlank()) {
+            throw new IllegalArgumentException("shopName 不能为空");
+        }
+        if (rowCount < 0 || rowCount > MAX_ROWS) {
+            throw new IllegalArgumentException("rowCount 必须在 [0, " + MAX_ROWS + "] 范围内，实际 " + rowCount);
+        }
+        if (countryCount < 1 || countryCount > COUNTRIES.size()) {
+            throw new IllegalArgumentException("countryCount 必须在 [1, " + COUNTRIES.size() + "] 范围内，实际 " + countryCount);
+        }
+        if (failedThumbRows < 0 || failedThumbRows > rowCount) {
+            throw new IllegalArgumentException("failedThumbRows 必须在 [0, " + rowCount + "] 范围内，实际 " + failedThumbRows);
+        }
+        if (rowCount == 0) {
+            return new ArrayList<>();
+        }
+        List<String> countries = COUNTRIES.subList(0, countryCount);
+        ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
+        item.setShopName(shopName);
+        item.setShopId(shopName + ":shop-id");
+        item.setPlatform("Amazon");
+        item.setSuccess(true);
+        item.setTaskStatus("SUCCESS");
+        item.setCountryCodes(countries);
+        List<ShopDataCrawlCountryResultDto> countryResults = new ArrayList<>();
+        for (String country : countries) {
+            ShopDataCrawlCountryResultDto countryResult = new ShopDataCrawlCountryResultDto();
+            countryResult.setCountry(country);
+            countryResult.setItems(new ArrayList<>());
+            countryResults.add(countryResult);
+        }
+        for (int i = 0; i < rowCount; i++) {
+            int rowIndex = i + 1;
+            String country = countries.get(i % countries.size());
+            long seed = (shopName.hashCode() * 31L + country.hashCode() * 31L + rowIndex) & 0x7fffffffL;
+            ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
+            row.setDate("2026-07-25");
+            row.setAsin(deterministicAsin(seed));
+            row.setBrand("Brand-" + (seed % 100));
+            row.setInventorySales(String.valueOf(1 + seed % 500));
+            row.setSalesRank("#" + (1 + seed % 5000));
+            row.setPageViews(String.valueOf(100 + seed % 9000));
+            row.setUnitsSold(String.valueOf(1 + seed % 200));
+            row.setPrice(String.format("%.2f", 1 + (seed % 9900) / 100.0));
+            row.setRecommendedOffer(String.format("%.2f", 0.5 + (seed % 9900) / 100.0));
+            if (withImages && i >= failedThumbRows) {
+                row.setCommodityImage(THUMB_HOST + row.getAsin() + ".jpg");
+            } else {
+                row.setCommodityImage("");
+            }
+            countryResults.get(i % countries.size()).getItems().add(row);
+        }
+        item.setCountryResults(countryResults);
+        return List.of(item);
+    }
+
+    /**
+     * 采样全量行 payload 大小、chunk 划分数与图片成功/失败行数。
+     * withImages 为预期图片模式的提示参数，图片统计始终以行内 commodityImage 实际内容为准。
+     * 序列化失败时向上抛出不产生部分结果。
+     */
+    public Metrics samplePayload(List<ShopDataCrawlResultItemVo> items, boolean withImages, int chunkSize) {
+        if (items == null) {
+            throw new IllegalArgumentException("items 不能为 null");
+        }
+        if (chunkSize <= 0) {
+            throw new IllegalArgumentException("chunkSize 必须为正数，实际 " + chunkSize);
+        }
+        int rowCount = 0;
+        int urlsWithImages = 0;
+        int failedThumbUrls = 0;
+        for (ShopDataCrawlResultItemVo item : items) {
+            if (item == null || item.getCountryResults() == null) {
+                continue;
+            }
+            for (ShopDataCrawlCountryResultDto country : item.getCountryResults()) {
+                if (country == null || country.getItems() == null) {
+                    continue;
+                }
+                for (ShopDataCrawlRowDto row : country.getItems()) {
+                    if (row == null) {
+                        continue;
+                    }
+                    rowCount++;
+                    if (row.getCommodityImage() == null || row.getCommodityImage().isBlank()) {
+                        failedThumbUrls++;
+                    } else {
+                        urlsWithImages++;
+                    }
+                }
+            }
+        }
+        int chunkCount = rowCount == 0 ? 0 : (rowCount + chunkSize - 1) / chunkSize;
+        if (rowCount == 0) {
+            return new Metrics(0, 0, 0, 0, 0);
+        }
+        try {
+            byte[] bytes = objectMapper.writeValueAsBytes(items);
+            return new Metrics(rowCount, chunkCount, bytes.length, urlsWithImages, failedThumbUrls);
+        } catch (Exception ex) {
+            throw new IllegalStateException("店铺抓取基线 payload 采样序列化失败", ex);
+        }
+    }
+
+    public record Metrics(int rowCount, int chunkCount, long payloadBytes,
+                          int urlsWithImages, int failedThumbUrls) {
+    }
+
+    private static String deterministicAsin(long seed) {
+        StringBuilder sb = new StringBuilder("B0");
+        long state = seed;
+        for (int i = 0; i < 8; i++) {
+            state = state * 6364136223846793005L + 1442695040888963407L;
+            int pick = (int) ((state >>> 33) % 36);
+            sb.append(pick < 10 ? (char) ('0' + pick) : (char) ('A' + pick - 10));
+        }
+        return sb.toString();
+    }
+}
@@ -0,0 +1,196 @@
+package com.nanri.aiimage.modules.shopdatacrawl.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
+import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
+import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Spy;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+
+/**
+ * Task 21：店铺抓取性能基线。
+ * ShopDataCrawlPerfFixture 建立单店铺 1k/5k 行、多国家、图片成功/失败场景的确定性生成器，
+ * 并提供 payload 大小、chunk 划分数、图片 URL 数采样；图片行 URL 固定为占位主机，
+ * 图片失败场景在 sampleImageStats 中以 urlsWithImages/failedThumbUrls 呈现。
+ */
+@ExtendWith(MockitoExtension.class)
+class ShopDataCrawlPerfFixtureBaselineTest {
+
+    @Spy private ObjectMapper objectMapper = new ObjectMapper();
+
+    @Test
+    void test_task_021_image_normal_default_path() {
+        // 正常输入：1000 行生成，5 个国家均有行，payload/chunk/图片统计完整。
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        List<ShopDataCrawlResultItemVo> items =
+                fixture.generateItems("Demo Shop", 1000, 5, true, 0);
+
+        assertEquals(1, items.size(), "单店铺一个结果项");
+        assertEquals(5, items.get(0).getCountryResults().size(), "5 个国家结果分片");
+        int totalRows = items.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum();
+        assertEquals(1000, totalRows, "行数不丢失");
+        assertTrue(items.get(0).getCountryResults().stream()
+                .allMatch(c -> "UK,DE,FR,ES,IT".contains(c.getCountry())), "国家代码合法");
+
+        ShopDataCrawlPerfFixture.Metrics metrics =
+                fixture.samplePayload(items, true, 200);
+        assertEquals(1000, metrics.rowCount());
+        assertEquals(5, metrics.chunkCount(), "1000 行 / 200 每 chunk = 5 个 chunk");
+        assertTrue(metrics.payloadBytes() > 0);
+        assertEquals(1000, metrics.urlsWithImages(), "图片模式全行带图");
+        assertEquals(0, metrics.failedThumbUrls());
+    }
+
+    @Test
+    void test_task_021_image_normal_multiple_items() {
+        // 批量场景：1k 与 5k 两种规模 + 图片成功/失败两种场景，结果不丢失、顺序稳定。
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        List<ShopDataCrawlResultItemVo> oneK =
+                fixture.generateItems("Shop-A", 1000, 5, true, 0);
+        List<ShopDataCrawlResultItemVo> fiveK =
+                fixture.generateItems("Shop-B", 5000, 5, true, 500);
+
+        int oneKTotal = oneK.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum();
+        int fiveKTotal = fiveK.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum();
+        assertEquals(1000, oneKTotal);
+        assertEquals(5000, fiveKTotal, "5k 行规模不丢失");
+
+        ShopDataCrawlPerfFixture.Metrics oneKImage = fixture.samplePayload(oneK, true, 200);
+        ShopDataCrawlPerfFixture.Metrics fiveKImage = fixture.samplePayload(fiveK, true, 200);
+        assertEquals(1000, oneKImage.urlsWithImages());
+        assertEquals(500, fiveKImage.failedThumbUrls(), "失败场景：500 行无缩略图");
+        assertEquals(4500, fiveKImage.urlsWithImages(), "5000 - 500 失败 = 4500 成功");
+        assertEquals(25, fiveKImage.chunkCount(), "5000 行 / 200 每 chunk = 25 个 chunk");
+        assertTrue(fiveKImage.payloadBytes() > oneKImage.payloadBytes(), "更大规模 payload 更大");
+    }
+
+    @Test
+    void test_task_021_image_normal_repeated_operation_is_idempotent() {
+        // 重复执行：同一店铺/行数两次生成结果一致（确定性），不产生重复行。
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        List<ShopDataCrawlResultItemVo> first =
+                fixture.generateItems("Idem-Shop", 1000, 5, true, 100);
+        List<ShopDataCrawlResultItemVo> second =
+                fixture.generateItems("Idem-Shop", 1000, 5, true, 100);
+
+        assertEquals(sampleAsins(first), sampleAsins(second), "两次生成 ASIN 顺序一致");
+        assertEquals(first.get(0).getCountryResults().size(), second.get(0).getCountryResults().size());
+        ShopDataCrawlPerfFixture.Metrics m1 = fixture.samplePayload(first, true, 200);
+        ShopDataCrawlPerfFixture.Metrics m2 = fixture.samplePayload(second, true, 200);
+        assertEquals(m1.payloadBytes(), m2.payloadBytes(), "payload 字节幂等");
+        assertEquals(m1.urlsWithImages(), m2.urlsWithImages());
+    }
+
+    @Test
+    void test_task_021_image_boundary_empty_input() {
+        // 空输入：0 行生成返回空结果项；采样返回全零指标。
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        List<ShopDataCrawlResultItemVo> items = fixture.generateItems("Empty-Shop", 0, 5, true, 0);
+        assertNotNull(items);
+        assertEquals(0, items.size());
+
+        ShopDataCrawlPerfFixture.Metrics metrics = fixture.samplePayload(items, true, 200);
+        assertEquals(0, metrics.rowCount());
+        assertEquals(0, metrics.chunkCount());
+        assertEquals(0, metrics.payloadBytes());
+        assertEquals(0, metrics.urlsWithImages());
+        assertEquals(0, metrics.failedThumbUrls());
+    }
+
+    @Test
+    void test_task_021_image_boundary_single_item() {
+        // 单元素：1 行生成不依赖批量路径；图片成功/失败两种单行场景均可采样。
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        List<ShopDataCrawlResultItemVo> single =
+                fixture.generateItems("Single-Shop", 1, 1, true, 0);
+        assertEquals(1, single.size());
+        assertEquals(1, single.get(0).getCountryResults().size());
+        assertEquals(1, single.get(0).getCountryResults().get(0).getItems().size());
+        ShopDataCrawlPerfFixture.Metrics m = fixture.samplePayload(single, true, 200);
+        assertEquals(1, m.rowCount());
+        assertEquals(1, m.urlsWithImages());
+
+        List<ShopDataCrawlResultItemVo> singleFail =
+                fixture.generateItems("Single-Shop", 1, 1, true, 1);
+        ShopDataCrawlPerfFixture.Metrics mf = fixture.samplePayload(singleFail, true, 200);
+        assertEquals(1, mf.failedThumbUrls(), "1 行失败场景：该行无缩略图");
+        assertEquals(0, mf.urlsWithImages());
+    }
+
+    @Test
+    void test_task_021_image_boundary_limit_and_overflow() {
+        // 上限/超限：超过 MAX_ROWS 拒绝；5000 行基线在预算内完成，payload 有界。
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        IllegalArgumentException overflow = assertThrows(IllegalArgumentException.class,
+                () -> fixture.generateItems("Over-Shop",
+                        ShopDataCrawlPerfFixture.MAX_ROWS + 1, 5, true, 0));
+        assertTrue(overflow.getMessage().contains("rowCount"), "超限消息应可识别");
+
+        List<ShopDataCrawlResultItemVo> max =
+                fixture.generateItems("Max-Shop", 5000, 5, true, 0);
+        ShopDataCrawlPerfFixture.Metrics metrics = fixture.samplePayload(max, true, 200);
+        assertEquals(5000, metrics.rowCount());
+        assertTrue(metrics.payloadBytes() < 64L * 1024 * 1024, "5000 行 payload 必须有界");
+    }
+
+    @Test
+    void test_task_021_image_invalid_input_rejected() {
+        // 非法参数：null 店铺、非法国家数、非法失败数、null 列表采样 → 明确异常。
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        IllegalArgumentException nullShop = assertThrows(IllegalArgumentException.class,
+                () -> fixture.generateItems(null, 100, 5, true, 0));
+        assertTrue(nullShop.getMessage().contains("shopName"));
+        assertThrows(IllegalArgumentException.class,
+                () -> fixture.generateItems("Shop", 100, 0, true, 0));
+        assertThrows(IllegalArgumentException.class,
+                () -> fixture.generateItems("Shop", 100, 6, true, 0));
+        assertThrows(IllegalArgumentException.class,
+                () -> fixture.generateItems("Shop", 100, 5, true, -1));
+        assertThrows(IllegalArgumentException.class,
+                () -> fixture.samplePayload(null, true, 200));
+        assertThrows(IllegalArgumentException.class,
+                () -> fixture.samplePayload(
+                        fixture.generateItems("Shop", 100, 5, true, 0), true, 0));
+    }
+
+    @Test
+    void test_task_021_image_dependency_failure_releases_resources() throws Exception {
+        // 依赖失败：序列化失败抛 IllegalStateException 且不产生部分结果；恢复后重试成功。
+        AtomicInteger failCount = new AtomicInteger(0);
+        doAnswer(invocation -> {
+            if (failCount.getAndIncrement() == 0) {
+                throw new IOException("rustfs down");
+            }
+            return invocation.callRealMethod();
+        }).when(objectMapper).writeValueAsBytes(any());
+
+        ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
+        List<ShopDataCrawlResultItemVo> items = fixture.generateItems("Fail-Shop", 1000, 5, true, 0);
+        assertThrows(IllegalStateException.class, () -> fixture.samplePayload(items, true, 200));
+
+        ShopDataCrawlPerfFixture.Metrics recovered = fixture.samplePayload(items, true, 200);
+        assertEquals(1000, recovered.rowCount(), "依赖恢复后重试成功");
+        assertEquals(1000, recovered.urlsWithImages(), "图片统计在失败后仍完整");
+    }
+
+    private static List<String> sampleAsins(List<ShopDataCrawlResultItemVo> items) {
+        return items.stream()
+                .flatMap(item -> item.getCountryResults().stream())
+                .flatMap(country -> country.getItems().stream())
+                .map(ShopDataCrawlRowDto::getAsin)
+                .toList();
+    }
+}