task-24: 店铺图片预取增加任务级数量、字节和超时上限
ShopDataCrawlPrefetchBudget 对预取 URL 施加三重预算：数量截断（去重后按序取前 N）、 字节估算门（urlCount×avgBytes 超预算拒绝）、超时钳制（200ms/URL 估算钳到任务上限）； Excel 组装在 prefetch 前应用预算，未预取 URL 由 embed 兜底直接下载。 全量测试 540 通过。
This commit is contained in:
+201
@@ -0,0 +1,201 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.util;
|
||||
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 24：为店铺图片预取增加任务级数量、字节和超时上限。
|
||||
* ShopDataCrawlPrefetchBudget 对预取 URL 集合施加三重预算：
|
||||
* - maxUrls：数量上限，超出按顺序截断（去重后）；
|
||||
* - maxBytes：字节估算门，预计总量超过预算时拒绝预取；
|
||||
* - maxTimeoutSeconds：预取超时上限，按 url 数估算的 deadline 被钳制到该上限。
|
||||
* ShopDataCrawlExcelAssemblyService 在 prefetch 前应用预算，截断后的列表进入预取，
|
||||
* 未预取的 URL 由 embed 阶段按原有兜底链路直接下载。
|
||||
*/
|
||||
class ShopDataCrawlPrefetchBudgetTest {
|
||||
@TempDir Path tempDir;
|
||||
|
||||
private static final String URL = "https://thumb.example/";
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_normal_default_path() {
|
||||
// 正常输入：100 个 URL 在预算内全部通过，顺序保留、重复去重。
|
||||
ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(200, 64L * 1024 * 1024, 120);
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
urls.add(URL + i + ".jpg");
|
||||
}
|
||||
urls.add(urls.get(0));
|
||||
urls.add(urls.get(50));
|
||||
|
||||
List<String> bounded = budget.boundedUrls(urls);
|
||||
assertEquals(100, bounded.size(), "重复 URL 去重后全量通过");
|
||||
assertEquals(urls.get(0), bounded.get(0), "顺序保留");
|
||||
assertFalse(budget.wouldExceedBytes(bounded, 100_000), "100×100KB=10MB 未超 64MB");
|
||||
assertEquals(20_000, budget.timeoutMillisFor(100), "100×200ms=20s 在 [15s,120s] 内");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_normal_multiple_items() {
|
||||
// 批量场景：5000 个 URL 超数量上限 → 截断为前 1000 个，头部保留、尾部丢弃。
|
||||
ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(1000, 64L * 1024 * 1024, 120);
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 5000; i++) {
|
||||
urls.add(URL + i + ".jpg");
|
||||
}
|
||||
List<String> bounded = budget.boundedUrls(urls);
|
||||
assertEquals(1000, bounded.size(), "截断到数量上限");
|
||||
assertEquals(URL + "0.jpg", bounded.get(0), "头部 URL 保留");
|
||||
assertEquals(URL + "999.jpg", bounded.get(999), "边界 URL 保留");
|
||||
assertFalse(bounded.contains(URL + "1000.jpg"), "尾部 URL 丢弃");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行：同一输入两次截断结果一致，不产生重复项。
|
||||
ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(50, 64L * 1024 * 1024, 120);
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
urls.add(URL + (i % 20) + ".jpg");
|
||||
}
|
||||
List<String> first = budget.boundedUrls(urls);
|
||||
List<String> second = budget.boundedUrls(urls);
|
||||
assertEquals(first, second, "重复截断幂等");
|
||||
assertEquals(20, first.size(), "20 个唯一 URL 全量通过");
|
||||
assertEquals(first, budget.boundedUrls(urls), "多次调用输出稳定");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_boundary_empty_input() {
|
||||
// 空输入：空/空白集合安全返回空列表；字节估算为 0。
|
||||
ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 120);
|
||||
assertEquals(0, budget.boundedUrls(List.of()).size());
|
||||
assertEquals(0, budget.boundedUrls(null).size(), "null 输入安全跳过");
|
||||
assertFalse(budget.wouldExceedBytes(List.of(), 5 * 1024 * 1024), "空列表不超预算");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_boundary_single_item() {
|
||||
// 单元素：1 个 URL 直接通过；超时估算取下限 15s。
|
||||
ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 120);
|
||||
List<String> single = budget.boundedUrls(List.of(URL + "only.jpg"));
|
||||
assertEquals(1, single.size());
|
||||
assertEquals(URL + "only.jpg", single.get(0));
|
||||
assertEquals(15_000, budget.timeoutMillisFor(1), "单 URL 预算 200ms，取下限 15s");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_boundary_limit_and_overflow() throws Exception {
|
||||
// 上限/超限：超数量截断；字节估算超限拒绝；超时钳制到任务上限；
|
||||
// Excel 组装实际只预取预算内的 URL，其余走 embed 兜底。
|
||||
ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(5, 64L * 1024 * 1024, 60);
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
urls.add(URL + i + ".jpg");
|
||||
}
|
||||
assertEquals(5, budget.boundedUrls(urls).size(), "超数量上限截断");
|
||||
assertFalse(budget.wouldExceedBytes(urls, 5 * 1024 * 1024), "10×5MB=50MB<64MB 不应超限");
|
||||
assertTrue(budget.wouldExceedBytes(
|
||||
java.util.stream.IntStream.range(0, 2000).mapToObj(i -> URL + i + ".jpg").toList(),
|
||||
5 * 1024 * 1024), "2000×5MB=10GB 超过 64MB");
|
||||
assertEquals(60_000, budget.timeoutMillisFor(1000), "1000×200ms=200s 被钳制到任务上限 60s");
|
||||
|
||||
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||
when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer(
|
||||
invocation -> new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024, 5);
|
||||
|
||||
List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||
row.setDate("2026-07-25");
|
||||
row.setAsin("B0" + String.format("%08d", i));
|
||||
row.setCommodityImage(URL + i + ".jpg");
|
||||
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
|
||||
country.setCountry("UK");
|
||||
country.setItems(List.of(row));
|
||||
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
|
||||
item.setSuccess(true);
|
||||
item.setCountryResults(List.of(country));
|
||||
items.add(item);
|
||||
}
|
||||
File output = tempDir.resolve("budget.xlsx").toFile();
|
||||
service.writeWorkbook(output, items);
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
ArgumentCaptor<Collection> captor = ArgumentCaptor.forClass(Collection.class);
|
||||
verify(imageEmbedder).prefetch(captor.capture(), any());
|
||||
assertEquals(5, captor.getValue().size(), "预取仅收到预算内 URL");
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
assertEquals(10, workbook.getSheet("英国").getLastRowNum(), "全部 10 行写入，未预取 URL 兜底成功");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_invalid_input_rejected() {
|
||||
// 非法参数：非正数量/字节/超时上限 → 明确异常与字段名。
|
||||
IllegalArgumentException urls = assertThrows(IllegalArgumentException.class,
|
||||
() -> ShopDataCrawlPrefetchBudget.of(0, 64L * 1024 * 1024, 120));
|
||||
assertTrue(urls.getMessage().contains("maxUrls"));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> ShopDataCrawlPrefetchBudget.of(-1, 64L * 1024 * 1024, 120));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> ShopDataCrawlPrefetchBudget.of(100, 0, 120));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> ShopDataCrawlPrefetchBudget.of(100, -5, 120));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_024_image_prefetch_dependency_failure_releases_resources() {
|
||||
// 依赖失败：URL 集合含 null/空白条目（失败源）时安全跳过，结果仍有界且稳定。
|
||||
ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 120);
|
||||
List<String> dirty = new ArrayList<>();
|
||||
dirty.add(URL + "a.jpg");
|
||||
dirty.add(null);
|
||||
dirty.add(" ");
|
||||
dirty.add(URL + "b.jpg");
|
||||
dirty.add(null);
|
||||
List<String> bounded = budget.boundedUrls(dirty);
|
||||
assertEquals(2, bounded.size(), "null/空白条目跳过，不产生非法 URL");
|
||||
assertEquals(URL + "a.jpg", bounded.get(0));
|
||||
assertEquals(URL + "b.jpg", bounded.get(1));
|
||||
assertEquals(bounded, budget.boundedUrls(dirty), "失败输入后再次调用仍稳定");
|
||||
assertEquals(2, budget.boundedUrls(dirty).size(), "失败输入不残留状态");
|
||||
}
|
||||
|
||||
private static byte[] jpegBytes() throws Exception {
|
||||
BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB);
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", output);
|
||||
return output.toByteArray();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user