From 81da7a182e77b9f6a445d1ddfbc28c3c0577a065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 29 Aug 2026 18:24:10 +0800 Subject: [PATCH] =?UTF-8?q?task-21:=20=E5=BB=BA=E7=AB=8B=E5=BA=97=E9=93=BA?= =?UTF-8?q?=E6=8A=93=E5=8F=96=E6=80=A7=E8=83=BD=E5=9F=BA=E7=BA=BF=E5=A4=B9?= =?UTF-8?q?=E5=85=B7=EF=BC=881k/5k=20=E8=A1=8C=E3=80=81=E5=A4=9A=E5=9B=BD?= =?UTF-8?q?=E5=AE=B6=E3=80=81=E5=9B=BE=E7=89=87=E6=88=90=E5=8A=9F/?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShopDataCrawlPerfFixture 提供单店铺多国家抓取结果的确定性生成器： - generateItems：1k/5k 行、1-5 国家、图片成功/失败（前 N 行无缩略图）场景； - samplePayload：行数/chunk 数/payload 字节/图片成功与失败行数采样。 全量测试 516 通过。 --- .../util/ShopDataCrawlPerfFixture.java | 154 ++++++++++++++ .../ShopDataCrawlPerfFixtureBaselineTest.java | 196 ++++++++++++++++++ 2 files changed, 350 insertions(+) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixture.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixtureBaselineTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixture.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixture.java new file mode 100644 index 00000000..1285f675 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixture.java @@ -0,0 +1,154 @@ +package com.nanri.aiimage.modules.shopdatacrawl.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; + +/** + * 店铺抓取性能基线夹具：单店铺 1k/5k 行、多国家、图片成功/失败场景的确定性生成器。 + * 同一输入必然产生相同输出（幂等）；失败场景通过 failedThumbRows 指定前 N 行为无缩略图行。 + * 上限约束：单次最多 MAX_ROWS 行，防止基线夹具本身造成无界内存增长。 + */ +@Slf4j +public class ShopDataCrawlPerfFixture { + + public static final int MAX_ROWS = 5000; + public static final List COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT"); + private static final String THUMB_HOST = "https://thumb.example/"; + + private final ObjectMapper objectMapper; + + public ShopDataCrawlPerfFixture(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + /** + * 生成单店铺 countryCount 个国家的 rowCount 行抓取结果。 + * failedThumbRows 指定前 N 行无商品图片（图片失败场景），其余行带确定性占位图 URL。 + * 图片统计以行内 commodityImage 实际内容为准（非空=成功，空=失败）。 + */ + public List generateItems(String shopName, int rowCount, int countryCount, + boolean withImages, int failedThumbRows) { + if (shopName == null || shopName.isBlank()) { + throw new IllegalArgumentException("shopName 不能为空"); + } + if (rowCount < 0 || rowCount > MAX_ROWS) { + throw new IllegalArgumentException("rowCount 必须在 [0, " + MAX_ROWS + "] 范围内，实际 " + rowCount); + } + if (countryCount < 1 || countryCount > COUNTRIES.size()) { + throw new IllegalArgumentException("countryCount 必须在 [1, " + COUNTRIES.size() + "] 范围内，实际 " + countryCount); + } + if (failedThumbRows < 0 || failedThumbRows > rowCount) { + throw new IllegalArgumentException("failedThumbRows 必须在 [0, " + rowCount + "] 范围内，实际 " + failedThumbRows); + } + if (rowCount == 0) { + return new ArrayList<>(); + } + List countries = COUNTRIES.subList(0, countryCount); + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setShopName(shopName); + item.setShopId(shopName + ":shop-id"); + item.setPlatform("Amazon"); + item.setSuccess(true); + item.setTaskStatus("SUCCESS"); + item.setCountryCodes(countries); + List countryResults = new ArrayList<>(); + for (String country : countries) { + ShopDataCrawlCountryResultDto countryResult = new ShopDataCrawlCountryResultDto(); + countryResult.setCountry(country); + countryResult.setItems(new ArrayList<>()); + countryResults.add(countryResult); + } + for (int i = 0; i < rowCount; i++) { + int rowIndex = i + 1; + String country = countries.get(i % countries.size()); + long seed = (shopName.hashCode() * 31L + country.hashCode() * 31L + rowIndex) & 0x7fffffffL; + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate("2026-07-25"); + row.setAsin(deterministicAsin(seed)); + row.setBrand("Brand-" + (seed % 100)); + row.setInventorySales(String.valueOf(1 + seed % 500)); + row.setSalesRank("#" + (1 + seed % 5000)); + row.setPageViews(String.valueOf(100 + seed % 9000)); + row.setUnitsSold(String.valueOf(1 + seed % 200)); + row.setPrice(String.format("%.2f", 1 + (seed % 9900) / 100.0)); + row.setRecommendedOffer(String.format("%.2f", 0.5 + (seed % 9900) / 100.0)); + if (withImages && i >= failedThumbRows) { + row.setCommodityImage(THUMB_HOST + row.getAsin() + ".jpg"); + } else { + row.setCommodityImage(""); + } + countryResults.get(i % countries.size()).getItems().add(row); + } + item.setCountryResults(countryResults); + return List.of(item); + } + + /** + * 采样全量行 payload 大小、chunk 划分数与图片成功/失败行数。 + * withImages 为预期图片模式的提示参数，图片统计始终以行内 commodityImage 实际内容为准。 + * 序列化失败时向上抛出不产生部分结果。 + */ + public Metrics samplePayload(List items, boolean withImages, int chunkSize) { + if (items == null) { + throw new IllegalArgumentException("items 不能为 null"); + } + if (chunkSize <= 0) { + throw new IllegalArgumentException("chunkSize 必须为正数，实际 " + chunkSize); + } + int rowCount = 0; + int urlsWithImages = 0; + int failedThumbUrls = 0; + for (ShopDataCrawlResultItemVo item : items) { + if (item == null || item.getCountryResults() == null) { + continue; + } + for (ShopDataCrawlCountryResultDto country : item.getCountryResults()) { + if (country == null || country.getItems() == null) { + continue; + } + for (ShopDataCrawlRowDto row : country.getItems()) { + if (row == null) { + continue; + } + rowCount++; + if (row.getCommodityImage() == null || row.getCommodityImage().isBlank()) { + failedThumbUrls++; + } else { + urlsWithImages++; + } + } + } + } + int chunkCount = rowCount == 0 ? 0 : (rowCount + chunkSize - 1) / chunkSize; + if (rowCount == 0) { + return new Metrics(0, 0, 0, 0, 0); + } + try { + byte[] bytes = objectMapper.writeValueAsBytes(items); + return new Metrics(rowCount, chunkCount, bytes.length, urlsWithImages, failedThumbUrls); + } catch (Exception ex) { + throw new IllegalStateException("店铺抓取基线 payload 采样序列化失败", ex); + } + } + + public record Metrics(int rowCount, int chunkCount, long payloadBytes, + int urlsWithImages, int failedThumbUrls) { + } + + private static String deterministicAsin(long seed) { + StringBuilder sb = new StringBuilder("B0"); + long state = seed; + for (int i = 0; i < 8; i++) { + state = state * 6364136223846793005L + 1442695040888963407L; + int pick = (int) ((state >>> 33) % 36); + sb.append(pick < 10 ? (char) ('0' + pick) : (char) ('A' + pick - 10)); + } + return sb.toString(); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixtureBaselineTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixtureBaselineTest.java new file mode 100644 index 00000000..992e29c7 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPerfFixtureBaselineTest.java @@ -0,0 +1,196 @@ +package com.nanri.aiimage.modules.shopdatacrawl.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +/** + * Task 21：店铺抓取性能基线。 + * ShopDataCrawlPerfFixture 建立单店铺 1k/5k 行、多国家、图片成功/失败场景的确定性生成器， + * 并提供 payload 大小、chunk 划分数、图片 URL 数采样；图片行 URL 固定为占位主机， + * 图片失败场景在 sampleImageStats 中以 urlsWithImages/failedThumbUrls 呈现。 + */ +@ExtendWith(MockitoExtension.class) +class ShopDataCrawlPerfFixtureBaselineTest { + + @Spy private ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void test_task_021_image_normal_default_path() { + // 正常输入：1000 行生成，5 个国家均有行，payload/chunk/图片统计完整。 + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + List items = + fixture.generateItems("Demo Shop", 1000, 5, true, 0); + + assertEquals(1, items.size(), "单店铺一个结果项"); + assertEquals(5, items.get(0).getCountryResults().size(), "5 个国家结果分片"); + int totalRows = items.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum(); + assertEquals(1000, totalRows, "行数不丢失"); + assertTrue(items.get(0).getCountryResults().stream() + .allMatch(c -> "UK,DE,FR,ES,IT".contains(c.getCountry())), "国家代码合法"); + + ShopDataCrawlPerfFixture.Metrics metrics = + fixture.samplePayload(items, true, 200); + assertEquals(1000, metrics.rowCount()); + assertEquals(5, metrics.chunkCount(), "1000 行 / 200 每 chunk = 5 个 chunk"); + assertTrue(metrics.payloadBytes() > 0); + assertEquals(1000, metrics.urlsWithImages(), "图片模式全行带图"); + assertEquals(0, metrics.failedThumbUrls()); + } + + @Test + void test_task_021_image_normal_multiple_items() { + // 批量场景：1k 与 5k 两种规模 + 图片成功/失败两种场景，结果不丢失、顺序稳定。 + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + List oneK = + fixture.generateItems("Shop-A", 1000, 5, true, 0); + List fiveK = + fixture.generateItems("Shop-B", 5000, 5, true, 500); + + int oneKTotal = oneK.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum(); + int fiveKTotal = fiveK.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum(); + assertEquals(1000, oneKTotal); + assertEquals(5000, fiveKTotal, "5k 行规模不丢失"); + + ShopDataCrawlPerfFixture.Metrics oneKImage = fixture.samplePayload(oneK, true, 200); + ShopDataCrawlPerfFixture.Metrics fiveKImage = fixture.samplePayload(fiveK, true, 200); + assertEquals(1000, oneKImage.urlsWithImages()); + assertEquals(500, fiveKImage.failedThumbUrls(), "失败场景：500 行无缩略图"); + assertEquals(4500, fiveKImage.urlsWithImages(), "5000 - 500 失败 = 4500 成功"); + assertEquals(25, fiveKImage.chunkCount(), "5000 行 / 200 每 chunk = 25 个 chunk"); + assertTrue(fiveKImage.payloadBytes() > oneKImage.payloadBytes(), "更大规模 payload 更大"); + } + + @Test + void test_task_021_image_normal_repeated_operation_is_idempotent() { + // 重复执行：同一店铺/行数两次生成结果一致（确定性），不产生重复行。 + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + List first = + fixture.generateItems("Idem-Shop", 1000, 5, true, 100); + List second = + fixture.generateItems("Idem-Shop", 1000, 5, true, 100); + + assertEquals(sampleAsins(first), sampleAsins(second), "两次生成 ASIN 顺序一致"); + assertEquals(first.get(0).getCountryResults().size(), second.get(0).getCountryResults().size()); + ShopDataCrawlPerfFixture.Metrics m1 = fixture.samplePayload(first, true, 200); + ShopDataCrawlPerfFixture.Metrics m2 = fixture.samplePayload(second, true, 200); + assertEquals(m1.payloadBytes(), m2.payloadBytes(), "payload 字节幂等"); + assertEquals(m1.urlsWithImages(), m2.urlsWithImages()); + } + + @Test + void test_task_021_image_boundary_empty_input() { + // 空输入：0 行生成返回空结果项；采样返回全零指标。 + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + List items = fixture.generateItems("Empty-Shop", 0, 5, true, 0); + assertNotNull(items); + assertEquals(0, items.size()); + + ShopDataCrawlPerfFixture.Metrics metrics = fixture.samplePayload(items, true, 200); + assertEquals(0, metrics.rowCount()); + assertEquals(0, metrics.chunkCount()); + assertEquals(0, metrics.payloadBytes()); + assertEquals(0, metrics.urlsWithImages()); + assertEquals(0, metrics.failedThumbUrls()); + } + + @Test + void test_task_021_image_boundary_single_item() { + // 单元素：1 行生成不依赖批量路径；图片成功/失败两种单行场景均可采样。 + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + List single = + fixture.generateItems("Single-Shop", 1, 1, true, 0); + assertEquals(1, single.size()); + assertEquals(1, single.get(0).getCountryResults().size()); + assertEquals(1, single.get(0).getCountryResults().get(0).getItems().size()); + ShopDataCrawlPerfFixture.Metrics m = fixture.samplePayload(single, true, 200); + assertEquals(1, m.rowCount()); + assertEquals(1, m.urlsWithImages()); + + List singleFail = + fixture.generateItems("Single-Shop", 1, 1, true, 1); + ShopDataCrawlPerfFixture.Metrics mf = fixture.samplePayload(singleFail, true, 200); + assertEquals(1, mf.failedThumbUrls(), "1 行失败场景：该行无缩略图"); + assertEquals(0, mf.urlsWithImages()); + } + + @Test + void test_task_021_image_boundary_limit_and_overflow() { + // 上限/超限：超过 MAX_ROWS 拒绝；5000 行基线在预算内完成，payload 有界。 + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + IllegalArgumentException overflow = assertThrows(IllegalArgumentException.class, + () -> fixture.generateItems("Over-Shop", + ShopDataCrawlPerfFixture.MAX_ROWS + 1, 5, true, 0)); + assertTrue(overflow.getMessage().contains("rowCount"), "超限消息应可识别"); + + List max = + fixture.generateItems("Max-Shop", 5000, 5, true, 0); + ShopDataCrawlPerfFixture.Metrics metrics = fixture.samplePayload(max, true, 200); + assertEquals(5000, metrics.rowCount()); + assertTrue(metrics.payloadBytes() < 64L * 1024 * 1024, "5000 行 payload 必须有界"); + } + + @Test + void test_task_021_image_invalid_input_rejected() { + // 非法参数：null 店铺、非法国家数、非法失败数、null 列表采样 → 明确异常。 + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + IllegalArgumentException nullShop = assertThrows(IllegalArgumentException.class, + () -> fixture.generateItems(null, 100, 5, true, 0)); + assertTrue(nullShop.getMessage().contains("shopName")); + assertThrows(IllegalArgumentException.class, + () -> fixture.generateItems("Shop", 100, 0, true, 0)); + assertThrows(IllegalArgumentException.class, + () -> fixture.generateItems("Shop", 100, 6, true, 0)); + assertThrows(IllegalArgumentException.class, + () -> fixture.generateItems("Shop", 100, 5, true, -1)); + assertThrows(IllegalArgumentException.class, + () -> fixture.samplePayload(null, true, 200)); + assertThrows(IllegalArgumentException.class, + () -> fixture.samplePayload( + fixture.generateItems("Shop", 100, 5, true, 0), true, 0)); + } + + @Test + void test_task_021_image_dependency_failure_releases_resources() throws Exception { + // 依赖失败：序列化失败抛 IllegalStateException 且不产生部分结果；恢复后重试成功。 + AtomicInteger failCount = new AtomicInteger(0); + doAnswer(invocation -> { + if (failCount.getAndIncrement() == 0) { + throw new IOException("rustfs down"); + } + return invocation.callRealMethod(); + }).when(objectMapper).writeValueAsBytes(any()); + + ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper); + List items = fixture.generateItems("Fail-Shop", 1000, 5, true, 0); + assertThrows(IllegalStateException.class, () -> fixture.samplePayload(items, true, 200)); + + ShopDataCrawlPerfFixture.Metrics recovered = fixture.samplePayload(items, true, 200); + assertEquals(1000, recovered.rowCount(), "依赖恢复后重试成功"); + assertEquals(1000, recovered.urlsWithImages(), "图片统计在失败后仍完整"); + } + + private static List sampleAsins(List items) { + return items.stream() + .flatMap(item -> item.getCountryResults().stream()) + .flatMap(country -> country.getItems().stream()) + .map(ShopDataCrawlRowDto::getAsin) + .toList(); + } +}