task-21: 建立店铺抓取性能基线夹具（1k/5k 行、多国家、图片成功/失败）
ShopDataCrawlPerfFixture 提供单店铺多国家抓取结果的确定性生成器： - generateItems：1k/5k 行、1-5 国家、图片成功/失败（前 N 行无缩略图）场景； - samplePayload：行数/chunk 数/payload 字节/图片成功与失败行数采样。 全量测试 516 通过。
This commit is contained in:
+196
@@ -0,0 +1,196 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
|
||||
/**
|
||||
* Task 21：店铺抓取性能基线。
|
||||
* ShopDataCrawlPerfFixture 建立单店铺 1k/5k 行、多国家、图片成功/失败场景的确定性生成器，
|
||||
* 并提供 payload 大小、chunk 划分数、图片 URL 数采样；图片行 URL 固定为占位主机，
|
||||
* 图片失败场景在 sampleImageStats 中以 urlsWithImages/failedThumbUrls 呈现。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ShopDataCrawlPerfFixtureBaselineTest {
|
||||
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void test_task_021_image_normal_default_path() {
|
||||
// 正常输入：1000 行生成，5 个国家均有行，payload/chunk/图片统计完整。
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
List<ShopDataCrawlResultItemVo> items =
|
||||
fixture.generateItems("Demo Shop", 1000, 5, true, 0);
|
||||
|
||||
assertEquals(1, items.size(), "单店铺一个结果项");
|
||||
assertEquals(5, items.get(0).getCountryResults().size(), "5 个国家结果分片");
|
||||
int totalRows = items.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum();
|
||||
assertEquals(1000, totalRows, "行数不丢失");
|
||||
assertTrue(items.get(0).getCountryResults().stream()
|
||||
.allMatch(c -> "UK,DE,FR,ES,IT".contains(c.getCountry())), "国家代码合法");
|
||||
|
||||
ShopDataCrawlPerfFixture.Metrics metrics =
|
||||
fixture.samplePayload(items, true, 200);
|
||||
assertEquals(1000, metrics.rowCount());
|
||||
assertEquals(5, metrics.chunkCount(), "1000 行 / 200 每 chunk = 5 个 chunk");
|
||||
assertTrue(metrics.payloadBytes() > 0);
|
||||
assertEquals(1000, metrics.urlsWithImages(), "图片模式全行带图");
|
||||
assertEquals(0, metrics.failedThumbUrls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_021_image_normal_multiple_items() {
|
||||
// 批量场景：1k 与 5k 两种规模 + 图片成功/失败两种场景，结果不丢失、顺序稳定。
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
List<ShopDataCrawlResultItemVo> oneK =
|
||||
fixture.generateItems("Shop-A", 1000, 5, true, 0);
|
||||
List<ShopDataCrawlResultItemVo> fiveK =
|
||||
fixture.generateItems("Shop-B", 5000, 5, true, 500);
|
||||
|
||||
int oneKTotal = oneK.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum();
|
||||
int fiveKTotal = fiveK.get(0).getCountryResults().stream().mapToInt(c -> c.getItems().size()).sum();
|
||||
assertEquals(1000, oneKTotal);
|
||||
assertEquals(5000, fiveKTotal, "5k 行规模不丢失");
|
||||
|
||||
ShopDataCrawlPerfFixture.Metrics oneKImage = fixture.samplePayload(oneK, true, 200);
|
||||
ShopDataCrawlPerfFixture.Metrics fiveKImage = fixture.samplePayload(fiveK, true, 200);
|
||||
assertEquals(1000, oneKImage.urlsWithImages());
|
||||
assertEquals(500, fiveKImage.failedThumbUrls(), "失败场景：500 行无缩略图");
|
||||
assertEquals(4500, fiveKImage.urlsWithImages(), "5000 - 500 失败 = 4500 成功");
|
||||
assertEquals(25, fiveKImage.chunkCount(), "5000 行 / 200 每 chunk = 25 个 chunk");
|
||||
assertTrue(fiveKImage.payloadBytes() > oneKImage.payloadBytes(), "更大规模 payload 更大");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_021_image_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行：同一店铺/行数两次生成结果一致（确定性），不产生重复行。
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
List<ShopDataCrawlResultItemVo> first =
|
||||
fixture.generateItems("Idem-Shop", 1000, 5, true, 100);
|
||||
List<ShopDataCrawlResultItemVo> second =
|
||||
fixture.generateItems("Idem-Shop", 1000, 5, true, 100);
|
||||
|
||||
assertEquals(sampleAsins(first), sampleAsins(second), "两次生成 ASIN 顺序一致");
|
||||
assertEquals(first.get(0).getCountryResults().size(), second.get(0).getCountryResults().size());
|
||||
ShopDataCrawlPerfFixture.Metrics m1 = fixture.samplePayload(first, true, 200);
|
||||
ShopDataCrawlPerfFixture.Metrics m2 = fixture.samplePayload(second, true, 200);
|
||||
assertEquals(m1.payloadBytes(), m2.payloadBytes(), "payload 字节幂等");
|
||||
assertEquals(m1.urlsWithImages(), m2.urlsWithImages());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_021_image_boundary_empty_input() {
|
||||
// 空输入：0 行生成返回空结果项；采样返回全零指标。
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
List<ShopDataCrawlResultItemVo> items = fixture.generateItems("Empty-Shop", 0, 5, true, 0);
|
||||
assertNotNull(items);
|
||||
assertEquals(0, items.size());
|
||||
|
||||
ShopDataCrawlPerfFixture.Metrics metrics = fixture.samplePayload(items, true, 200);
|
||||
assertEquals(0, metrics.rowCount());
|
||||
assertEquals(0, metrics.chunkCount());
|
||||
assertEquals(0, metrics.payloadBytes());
|
||||
assertEquals(0, metrics.urlsWithImages());
|
||||
assertEquals(0, metrics.failedThumbUrls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_021_image_boundary_single_item() {
|
||||
// 单元素：1 行生成不依赖批量路径；图片成功/失败两种单行场景均可采样。
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
List<ShopDataCrawlResultItemVo> single =
|
||||
fixture.generateItems("Single-Shop", 1, 1, true, 0);
|
||||
assertEquals(1, single.size());
|
||||
assertEquals(1, single.get(0).getCountryResults().size());
|
||||
assertEquals(1, single.get(0).getCountryResults().get(0).getItems().size());
|
||||
ShopDataCrawlPerfFixture.Metrics m = fixture.samplePayload(single, true, 200);
|
||||
assertEquals(1, m.rowCount());
|
||||
assertEquals(1, m.urlsWithImages());
|
||||
|
||||
List<ShopDataCrawlResultItemVo> singleFail =
|
||||
fixture.generateItems("Single-Shop", 1, 1, true, 1);
|
||||
ShopDataCrawlPerfFixture.Metrics mf = fixture.samplePayload(singleFail, true, 200);
|
||||
assertEquals(1, mf.failedThumbUrls(), "1 行失败场景：该行无缩略图");
|
||||
assertEquals(0, mf.urlsWithImages());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_021_image_boundary_limit_and_overflow() {
|
||||
// 上限/超限：超过 MAX_ROWS 拒绝；5000 行基线在预算内完成，payload 有界。
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
IllegalArgumentException overflow = assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.generateItems("Over-Shop",
|
||||
ShopDataCrawlPerfFixture.MAX_ROWS + 1, 5, true, 0));
|
||||
assertTrue(overflow.getMessage().contains("rowCount"), "超限消息应可识别");
|
||||
|
||||
List<ShopDataCrawlResultItemVo> max =
|
||||
fixture.generateItems("Max-Shop", 5000, 5, true, 0);
|
||||
ShopDataCrawlPerfFixture.Metrics metrics = fixture.samplePayload(max, true, 200);
|
||||
assertEquals(5000, metrics.rowCount());
|
||||
assertTrue(metrics.payloadBytes() < 64L * 1024 * 1024, "5000 行 payload 必须有界");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_021_image_invalid_input_rejected() {
|
||||
// 非法参数：null 店铺、非法国家数、非法失败数、null 列表采样 → 明确异常。
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
IllegalArgumentException nullShop = assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.generateItems(null, 100, 5, true, 0));
|
||||
assertTrue(nullShop.getMessage().contains("shopName"));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.generateItems("Shop", 100, 0, true, 0));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.generateItems("Shop", 100, 6, true, 0));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.generateItems("Shop", 100, 5, true, -1));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.samplePayload(null, true, 200));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.samplePayload(
|
||||
fixture.generateItems("Shop", 100, 5, true, 0), true, 0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_021_image_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败：序列化失败抛 IllegalStateException 且不产生部分结果；恢复后重试成功。
|
||||
AtomicInteger failCount = new AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (failCount.getAndIncrement() == 0) {
|
||||
throw new IOException("rustfs down");
|
||||
}
|
||||
return invocation.callRealMethod();
|
||||
}).when(objectMapper).writeValueAsBytes(any());
|
||||
|
||||
ShopDataCrawlPerfFixture fixture = new ShopDataCrawlPerfFixture(objectMapper);
|
||||
List<ShopDataCrawlResultItemVo> items = fixture.generateItems("Fail-Shop", 1000, 5, true, 0);
|
||||
assertThrows(IllegalStateException.class, () -> fixture.samplePayload(items, true, 200));
|
||||
|
||||
ShopDataCrawlPerfFixture.Metrics recovered = fixture.samplePayload(items, true, 200);
|
||||
assertEquals(1000, recovered.rowCount(), "依赖恢复后重试成功");
|
||||
assertEquals(1000, recovered.urlsWithImages(), "图片统计在失败后仍完整");
|
||||
}
|
||||
|
||||
private static List<String> sampleAsins(List<ShopDataCrawlResultItemVo> items) {
|
||||
return items.stream()
|
||||
.flatMap(item -> item.getCountryResults().stream())
|
||||
.flatMap(country -> country.getItems().stream())
|
||||
.map(ShopDataCrawlRowDto::getAsin)
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user