From 6d488515cab12327ec222adc938ae226d6c16d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 29 Aug 2026 18:29:25 +0800 Subject: [PATCH] =?UTF-8?q?task-22:=20=E5=BA=97=E9=93=BA=20Excel=20?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E7=BC=93=E5=AD=98=E6=9B=BF=E6=8D=A2=E4=B8=BA?= =?UTF-8?q?=E6=9C=89=E7=95=8C=E5=AD=97=E8=8A=82=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BoundedImageCache（字节预算+条目上限，FIFO 淘汰，超预算单图拒绝）替换 ShopDataCrawlExcelAssemblyService 中的无界 ConcurrentHashMap；单图超预算时 embed 走原有直接下载兜底。全量测试 524 通过。 --- .../ShopDataCrawlExcelAssemblyService.java | 26 ++- .../shopdatacrawl/util/BoundedImageCache.java | 91 ++++++++ .../ShopDataCrawlBoundedImageCacheTest.java | 204 ++++++++++++++++++ 3 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/BoundedImageCache.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlBoundedImageCacheTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java index f1ceea88..bf8d9bc5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java @@ -4,6 +4,7 @@ import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -28,7 +29,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; @Service @Slf4j @@ -44,8 +44,22 @@ public class ShopDataCrawlExcelAssemblyService { private static final int BRAND_COLUMN = HEADERS.size() - 1; private static final int IMAGE_COLUMN_WIDTH = 18 * 256; private static final float IMAGE_ROW_HEIGHT_POINTS = 80f; + /** 图片缓存默认上限：64MB 字节预算 / 2000 条目，超过按 FIFO 淘汰，保证组装期内存有界。 */ + private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024; + private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000; private final SimilarAsinImageEmbedder imageEmbedder; + private long imageCacheMaxBytes = DEFAULT_IMAGE_CACHE_MAX_BYTES; + private int imageCacheMaxEntries = DEFAULT_IMAGE_CACHE_MAX_ENTRIES; + + public ShopDataCrawlExcelAssemblyService(SimilarAsinImageEmbedder imageEmbedder, long imageCacheMaxBytes) { + this.imageEmbedder = imageEmbedder; + this.imageCacheMaxBytes = imageCacheMaxBytes; + } + + private BoundedImageCache newImageCache() { + return new BoundedImageCache(imageCacheMaxBytes, imageCacheMaxEntries); + } public void writeWorkbook(File outputXlsx, List items) { try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream(); @@ -53,7 +67,7 @@ public class ShopDataCrawlExcelAssemblyService { FileOutputStream output = new FileOutputStream(outputXlsx)) { validateTemplate(workbook); Map> rowsByCountry = rowsByCountry(items); - Map imageCache = new ConcurrentHashMap<>(); + BoundedImageCache imageCache = newImageCache(); imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache); Map pictureIndexes = new LinkedHashMap<>(); for (int i = 0; i < COUNTRIES.size(); i++) { @@ -82,7 +96,7 @@ public class ShopDataCrawlExcelAssemblyService { FileOutputStream output = new FileOutputStream(outputXlsx)) { validateTemplate(workbook); Map> rowsByCountry = rowsByCountry(items); - Map imageCache = new ConcurrentHashMap<>(); + BoundedImageCache imageCache = newImageCache(); imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache); Map pictureIndexes = new LinkedHashMap<>(); for (int i = 0; i < COUNTRIES.size(); i++) { @@ -133,7 +147,7 @@ public class ShopDataCrawlExcelAssemblyService { private void writeSheet(XSSFWorkbook workbook, Sheet sheet, List rows, - Map imageCache, + BoundedImageCache imageCache, Map pictureIndexes) { Row header = sheet.getRow(0); Row styleRow = sheet.getRow(1); @@ -175,7 +189,7 @@ public class ShopDataCrawlExcelAssemblyService { Row row, ShopDataCrawlRowDto value, CellStyle[] styles, - Map imageCache, + BoundedImageCache imageCache, Map pictureIndexes) { String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(), value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()}; @@ -218,7 +232,7 @@ public class ShopDataCrawlExcelAssemblyService { Sheet sheet, Row row, String imageUrl, - Map imageCache, + BoundedImageCache imageCache, Map pictureIndexes) { String normalizedUrl = imageUrl.trim(); try { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/BoundedImageCache.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/BoundedImageCache.java new file mode 100644 index 00000000..2eb30f29 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/BoundedImageCache.java @@ -0,0 +1,91 @@ +package com.nanri.aiimage.modules.shopdatacrawl.util; + +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import lombok.extern.slf4j.Slf4j; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Task 22：店铺 Excel 图片的有界字节缓存。 + * 以总字节预算 + 条目上限约束图片缓存，超过预算按 FIFO（插入序）淘汰最旧条目； + * 单图超过预算时拒绝缓存并计数，embed 阶段对该 url 走原有直接下载兜底。 + * 与 {@link SimilarAsinImageEmbedder#prefetch(java.util.Collection, Map)} 的 + * Map 入参兼容，保证组装期缓存内存峰值有界。 + */ +@Slf4j +public class BoundedImageCache extends LinkedHashMap { + + private final long maxBytes; + private final int maxEntries; + private long sizeBytes; + private long evictionCount; + private long rejectedCount; + + public BoundedImageCache(long maxBytes, int maxEntries) { + super(16, 0.75f, false); + if (maxBytes <= 0) { + throw new IllegalArgumentException("maxBytes 必须为正数，实际 " + maxBytes); + } + if (maxEntries <= 0) { + throw new IllegalArgumentException("maxEntries 必须为正数，实际 " + maxEntries); + } + this.maxBytes = maxBytes; + this.maxEntries = maxEntries; + } + + public long sizeBytes() { + return sizeBytes; + } + + public long evictionCount() { + return evictionCount; + } + + public long rejectedCount() { + return rejectedCount; + } + + @Override + public SimilarAsinImageEmbedder.ResizedImage put(String key, SimilarAsinImageEmbedder.ResizedImage value) { + if (key == null || key.isBlank()) { + throw new IllegalArgumentException("image key 不能为空"); + } + if (value == null) { + throw new IllegalArgumentException("image value 不能为 null"); + } + if (value.bytes() == null || value.bytes().length > maxBytes) { + rejectedCount++; + log.debug("[shop-data-crawl][image-cache] reject oversized image bytes={} maxBytes={}", + value.bytes() == null ? 0 : value.bytes().length, maxBytes); + return null; + } + if (containsKey(key)) { + sizeBytes -= super.get(key).bytes().length; + } + while (!isEmpty() && (sizeBytes + value.bytes().length > maxBytes || size() >= maxEntries)) { + evictEldest(); + } + sizeBytes += value.bytes().length; + return super.put(key, value); + } + + @Override + public SimilarAsinImageEmbedder.ResizedImage putIfAbsent(String key, SimilarAsinImageEmbedder.ResizedImage value) { + if (key == null) { + return null; + } + SimilarAsinImageEmbedder.ResizedImage existing = super.get(key); + if (existing != null) { + return existing; + } + return put(key, value); + } + + private void evictEldest() { + Map.Entry eldest = entrySet().iterator().next(); + sizeBytes -= eldest.getValue().bytes().length; + super.remove(eldest.getKey()); + evictionCount++; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlBoundedImageCacheTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlBoundedImageCacheTest.java new file mode 100644 index 00000000..b60e7644 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlBoundedImageCacheTest.java @@ -0,0 +1,204 @@ +package com.nanri.aiimage.modules.shopdatacrawl.util; + +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Task 22：将店铺 Excel 图片缓存替换为有界字节缓存。 + * BoundedImageCache 以总字节预算 + 条目上限约束图片缓存，超过预算按 FIFO 淘汰最旧条目， + * 单图超过预算时拒绝缓存（embed 阶段走原有直接下载兜底），保证内存峰值有界。 + * ShopDataCrawlExcelAssemblyService 的 workbook 组装改用该缓存，语义与无界缓存一致。 + */ +class ShopDataCrawlBoundedImageCacheTest { + @TempDir Path tempDir; + + private static final String URL_A = "https://thumb.example/A.jpg"; + private static final String URL_B = "https://thumb.example/B.jpg"; + private static final String URL_C = "https://thumb.example/C.jpg"; + + @Test + void test_task_022_image_cache_excel_normal_default_path() { + // 正常输入：预算充足时全部放入，get 命中，字节合计正确，无淘汰。 + BoundedImageCache cache = new BoundedImageCache(10_000, 100); + SimilarAsinImageEmbedder.ResizedImage imageA = resizedImage(100); + SimilarAsinImageEmbedder.ResizedImage imageB = resizedImage(200); + cache.put(URL_A, imageA); + cache.put(URL_B, imageB); + + assertEquals(imageA, cache.get(URL_A), "A 命中"); + assertEquals(imageB, cache.get(URL_B), "B 命中"); + assertEquals(300, cache.sizeBytes(), "字节合计正确"); + assertEquals(2, cache.size()); + assertEquals(0, cache.evictionCount(), "预算充足不淘汰"); + assertEquals(0, cache.rejectedCount(), "无拒绝"); + } + + @Test + void test_task_022_image_cache_excel_normal_multiple_items() { + // 批量场景：大量图片超过字节预算 → 最旧条目被淘汰，最新条目可命中，总字节有界。 + BoundedImageCache cache = new BoundedImageCache(1_000, 100); + List images = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + SimilarAsinImageEmbedder.ResizedImage image = resizedImage(100); + images.add(image); + cache.put(URL_A + i, image); + } + assertTrue(cache.sizeBytes() <= 1_000, "总字节不得超过预算，实际=" + cache.sizeBytes()); + assertTrue(cache.size() <= 10, "1_000 预算 / 100 每图最多容纳 10 张"); + assertTrue(cache.evictionCount() >= 40, "超出部分被淘汰"); + assertNull(cache.get(URL_A + "0"), "最旧条目已淘汰"); + assertNotNull(cache.get(URL_A + "49"), "最新条目仍可命中"); + } + + @Test + void test_task_022_image_cache_excel_normal_repeated_operation_is_idempotent() { + // 重复执行：同一 URL 重复 put 替换不增加字节；putIfAbsent 不覆盖已有值。 + BoundedImageCache cache = new BoundedImageCache(1_000, 100); + SimilarAsinImageEmbedder.ResizedImage first = resizedImage(100); + SimilarAsinImageEmbedder.ResizedImage second = resizedImage(100); + cache.put(URL_A, first); + cache.put(URL_A, second); + assertEquals(100, cache.sizeBytes(), "替换不重复计入字节"); + assertEquals(1, cache.size()); + + SimilarAsinImageEmbedder.ResizedImage replaced = cache.putIfAbsent(URL_A, resizedImage(200)); + assertEquals(second, replaced, "putIfAbsent 返回已有值"); + assertEquals(100, cache.sizeBytes(), "putIfAbsent 不替换字节"); + assertEquals(0, cache.evictionCount(), "重复操作不触发淘汰"); + } + + @Test + void test_task_022_image_cache_excel_boundary_empty_input() { + // 空输入：空缓存 get 返回 null；null/空集合安全。 + BoundedImageCache cache = new BoundedImageCache(1_000, 100); + assertNull(cache.get(URL_A)); + assertEquals(0, cache.size()); + assertEquals(0, cache.sizeBytes()); + assertNull(cache.putIfAbsent(null, resizedImage(10)), "null key 不放入"); + assertEquals(0, cache.size()); + } + + @Test + void test_task_022_image_cache_excel_boundary_single_item() { + // 单元素边界：单图恰好等于预算可放入；剩余预算恰够时刚好放下。 + BoundedImageCache exact = new BoundedImageCache(100, 100); + SimilarAsinImageEmbedder.ResizedImage image = resizedImage(100); + exact.put(URL_A, image); + assertEquals(image, exact.get(URL_A), "单图等于预算可缓存"); + assertEquals(100, exact.sizeBytes()); + + BoundedImageCache tight = new BoundedImageCache(150, 100); + tight.put(URL_A, resizedImage(100)); + tight.put(URL_B, resizedImage(50)); + assertEquals(2, tight.size(), "剩余预算恰够时恰好放下"); + assertEquals(0, tight.evictionCount()); + } + + @Test + void test_task_022_image_cache_excel_boundary_limit_and_overflow() { + // 上限/超限：单图超过预算被拒绝缓存且不占用预算；总字节永不超过预算。 + BoundedImageCache cache = new BoundedImageCache(100, 100); + SimilarAsinImageEmbedder.ResizedImage huge = resizedImage(200); + cache.put(URL_A, huge); + assertNull(cache.get(URL_A), "单图超预算拒绝缓存"); + assertEquals(0, cache.sizeBytes(), "被拒绝的图不占用预算"); + assertEquals(1, cache.rejectedCount()); + + cache.put(URL_B, resizedImage(60)); + cache.put(URL_C, resizedImage(60)); + assertEquals(1, cache.size(), "第二次放入触发淘汰，仅保留最新"); + assertNull(cache.get(URL_B)); + assertNotNull(cache.get(URL_C)); + assertEquals(60, cache.sizeBytes(), "总字节不超预算"); + assertTrue(cache.evictionCount() >= 1); + } + + @Test + void test_task_022_image_cache_excel_invalid_input_rejected() { + // 非法参数：非正预算/非正条目上限 → 拒绝；null 值拒绝。 + assertThrows(IllegalArgumentException.class, () -> new BoundedImageCache(0, 100)); + assertThrows(IllegalArgumentException.class, () -> new BoundedImageCache(-1, 100)); + assertThrows(IllegalArgumentException.class, () -> new BoundedImageCache(1_000, 0)); + BoundedImageCache cache = new BoundedImageCache(1_000, 100); + assertThrows(IllegalArgumentException.class, () -> cache.put(null, resizedImage(10))); + assertThrows(IllegalArgumentException.class, () -> cache.put("", resizedImage(10))); + assertThrows(IllegalArgumentException.class, () -> cache.put(URL_A, null)); + assertEquals(0, cache.size(), "非法输入不产生缓存条目"); + } + + @Test + void test_task_022_image_cache_excel_dependency_failure_releases_resources() throws Exception { + // 依赖失败：图片下载失败不占缓存预算，缓存继续可用； + // Excel 组装用有界缓存后仍产出完整 workbook，缓存字节有界。 + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + String failUrl = "https://thumb.example/fail.jpg"; + when(imageEmbedder.fetchAndResizeForCache(failUrl)).thenReturn(null); + String okUrl = "https://thumb.example/ok.jpg"; + when(imageEmbedder.fetchAndResizeForCache(okUrl)) + .thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2)); + + long smallBudget = 200; + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, smallBudget); + List items = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate("2026-07-25"); + row.setAsin("B0" + String.format("%08d", i)); + row.setCommodityImage(i % 2 == 0 ? okUrl : failUrl); + ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto(); + country.setCountry("UK"); + country.setItems(List.of(row)); + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setSuccess(true); + item.setCountryResults(List.of(country)); + items.add(item); + } + + File output = tempDir.resolve("bounded.xlsx").toFile(); + service.writeWorkbook(output, items); + + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(10, workbook.getSheet("英国").getLastRowNum(), "全部 10 行写入"); + assertEquals(1, workbook.getAllPictures().size(), "失败图片不嵌入，成功图片去重嵌入"); + assertEquals("https://thumb.example/fail.jpg", + workbook.getSheet("英国").getRow(2).getCell(2).getStringCellValue(), + "失败图片兜底为 URL 文本"); + } + } + + private static SimilarAsinImageEmbedder.ResizedImage resizedImage(int bytes) { + return new SimilarAsinImageEmbedder.ResizedImage(new byte[bytes], 2, 2); + } + + private static byte[] jpegBytes() throws Exception { + BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ImageIO.write(image, "jpg", output); + return output.toByteArray(); + } +}