super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-22: 店铺 Excel 图片缓存替换为有界字节缓存

Browse Source 
BoundedImageCache（字节预算+条目上限，FIFO 淘汰，超预算单图拒绝）替换
ShopDataCrawlExcelAssemblyService 中的无界 ConcurrentHashMap；单图超预算时
embed 走原有直接下载兜底。全量测试 524 通过。
This commit is contained in:
黄自达
2026-08-29 18:29:25 +08:00
parent 064e13c132
commit 6d488515ca
3 changed files with 315 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java
+20 -6
View File
@@ -4,6 +4,7 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -28,7 +29,6 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@Slf4j
@@ -44,8 +44,22 @@ public class ShopDataCrawlExcelAssemblyService {
private static final int BRAND_COLUMN = HEADERS.size() - 1;
private static final int IMAGE_COLUMN_WIDTH = 18 * 256;
private static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
/** 图片缓存默认上限：64MB 字节预算 / 2000 条目，超过按 FIFO 淘汰，保证组装期内存有界。 */
private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024;
private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000;
private final SimilarAsinImageEmbedder imageEmbedder;
private long imageCacheMaxBytes = DEFAULT_IMAGE_CACHE_MAX_BYTES;
private int imageCacheMaxEntries = DEFAULT_IMAGE_CACHE_MAX_ENTRIES;
public ShopDataCrawlExcelAssemblyService(SimilarAsinImageEmbedder imageEmbedder, long imageCacheMaxBytes) {
this.imageEmbedder = imageEmbedder;
this.imageCacheMaxBytes = imageCacheMaxBytes;
}
private BoundedImageCache newImageCache() {
return new BoundedImageCache(imageCacheMaxBytes, imageCacheMaxEntries);
}
public void writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
@@ -53,7 +67,7 @@ public class ShopDataCrawlExcelAssemblyService {
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache = new ConcurrentHashMap<>();
BoundedImageCache imageCache = newImageCache();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
@@ -82,7 +96,7 @@ public class ShopDataCrawlExcelAssemblyService {
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache = new ConcurrentHashMap<>();
BoundedImageCache imageCache = newImageCache();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
@@ -133,7 +147,7 @@ public class ShopDataCrawlExcelAssemblyService {
private void writeSheet(XSSFWorkbook workbook,
Sheet sheet,
List<ShopDataCrawlRowDto> rows,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) {
Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1);
@@ -175,7 +189,7 @@ public class ShopDataCrawlExcelAssemblyService {
Row row,
ShopDataCrawlRowDto value,
CellStyle[] styles,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) {
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
@@ -218,7 +232,7 @@ public class ShopDataCrawlExcelAssemblyService {
Sheet sheet,
Row row,
String imageUrl,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) {
String normalizedUrl = imageUrl.trim();
try {
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/BoundedImageCache.java
+91
View File
@@ -0,0 +1,91 @@
package com.nanri.aiimage.modules.shopdatacrawl.util;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.extern.slf4j.Slf4j;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* Task 22：店铺 Excel 图片的有界字节缓存。
* 以总字节预算 + 条目上限约束图片缓存，超过预算按 FIFO（插入序）淘汰最旧条目；
* 单图超过预算时拒绝缓存并计数，embed 阶段对该 url 走原有直接下载兜底。
* 与 {@link SimilarAsinImageEmbedder#prefetch(java.util.Collection, Map)} 的
* Map 入参兼容，保证组装期缓存内存峰值有界。
*/
@Slf4j
public class BoundedImageCache extends LinkedHashMap<String, SimilarAsinImageEmbedder.ResizedImage> {
private final long maxBytes;
private final int maxEntries;
private long sizeBytes;
private long evictionCount;
private long rejectedCount;
public BoundedImageCache(long maxBytes, int maxEntries) {
super(16, 0.75f, false);
if (maxBytes <= 0) {
throw new IllegalArgumentException("maxBytes 必须为正数，实际 " + maxBytes);
}
if (maxEntries <= 0) {
throw new IllegalArgumentException("maxEntries 必须为正数，实际 " + maxEntries);
}
this.maxBytes = maxBytes;
this.maxEntries = maxEntries;
}
public long sizeBytes() {
return sizeBytes;
}
public long evictionCount() {
return evictionCount;
}
public long rejectedCount() {
return rejectedCount;
}
@Override
public SimilarAsinImageEmbedder.ResizedImage put(String key, SimilarAsinImageEmbedder.ResizedImage value) {
if (key == null || key.isBlank()) {
throw new IllegalArgumentException("image key 不能为空");
}
if (value == null) {
throw new IllegalArgumentException("image value 不能为 null");
}
if (value.bytes() == null || value.bytes().length > maxBytes) {
rejectedCount++;
log.debug("[shop-data-crawl][image-cache] reject oversized image bytes={} maxBytes={}",
value.bytes() == null ? 0 : value.bytes().length, maxBytes);
return null;
}
if (containsKey(key)) {
sizeBytes -= super.get(key).bytes().length;
}
while (!isEmpty() && (sizeBytes + value.bytes().length > maxBytes || size() >= maxEntries)) {
evictEldest();
}
sizeBytes += value.bytes().length;
return super.put(key, value);
}
@Override
public SimilarAsinImageEmbedder.ResizedImage putIfAbsent(String key, SimilarAsinImageEmbedder.ResizedImage value) {
if (key == null) {
return null;
}
SimilarAsinImageEmbedder.ResizedImage existing = super.get(key);
if (existing != null) {
return existing;
}
return put(key, value);
}
private void evictEldest() {
Map.Entry<String, SimilarAsinImageEmbedder.ResizedImage> eldest = entrySet().iterator().next();
sizeBytes -= eldest.getValue().bytes().length;
super.remove(eldest.getKey());
evictionCount++;
}
}
backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlBoundedImageCacheTest.java
+204
View File
@@ -0,0 +1,204 @@
package com.nanri.aiimage.modules.shopdatacrawl.util;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* Task 22：将店铺 Excel 图片缓存替换为有界字节缓存。
* BoundedImageCache 以总字节预算 + 条目上限约束图片缓存，超过预算按 FIFO 淘汰最旧条目，
* 单图超过预算时拒绝缓存（embed 阶段走原有直接下载兜底），保证内存峰值有界。
* ShopDataCrawlExcelAssemblyService 的 workbook 组装改用该缓存，语义与无界缓存一致。
*/
class ShopDataCrawlBoundedImageCacheTest {
@TempDir Path tempDir;
private static final String URL_A = "https://thumb.example/A.jpg";
private static final String URL_B = "https://thumb.example/B.jpg";
private static final String URL_C = "https://thumb.example/C.jpg";
@Test
void test_task_022_image_cache_excel_normal_default_path() {
// 正常输入：预算充足时全部放入，get 命中，字节合计正确，无淘汰。
BoundedImageCache cache = new BoundedImageCache(10_000, 100);
SimilarAsinImageEmbedder.ResizedImage imageA = resizedImage(100);
SimilarAsinImageEmbedder.ResizedImage imageB = resizedImage(200);
cache.put(URL_A, imageA);
cache.put(URL_B, imageB);
assertEquals(imageA, cache.get(URL_A), "A 命中");
assertEquals(imageB, cache.get(URL_B), "B 命中");
assertEquals(300, cache.sizeBytes(), "字节合计正确");
assertEquals(2, cache.size());
assertEquals(0, cache.evictionCount(), "预算充足不淘汰");
assertEquals(0, cache.rejectedCount(), "无拒绝");
}
@Test
void test_task_022_image_cache_excel_normal_multiple_items() {
// 批量场景：大量图片超过字节预算 → 最旧条目被淘汰，最新条目可命中，总字节有界。
BoundedImageCache cache = new BoundedImageCache(1_000, 100);
List<SimilarAsinImageEmbedder.ResizedImage> images = new ArrayList<>();
for (int i = 0; i < 50; i++) {
SimilarAsinImageEmbedder.ResizedImage image = resizedImage(100);
images.add(image);
cache.put(URL_A + i, image);
}
assertTrue(cache.sizeBytes() <= 1_000, "总字节不得超过预算，实际=" + cache.sizeBytes());
assertTrue(cache.size() <= 10, "1_000 预算 / 100 每图最多容纳 10 张");
assertTrue(cache.evictionCount() >= 40, "超出部分被淘汰");
assertNull(cache.get(URL_A + "0"), "最旧条目已淘汰");
assertNotNull(cache.get(URL_A + "49"), "最新条目仍可命中");
}
@Test
void test_task_022_image_cache_excel_normal_repeated_operation_is_idempotent() {
// 重复执行：同一 URL 重复 put 替换不增加字节；putIfAbsent 不覆盖已有值。
BoundedImageCache cache = new BoundedImageCache(1_000, 100);
SimilarAsinImageEmbedder.ResizedImage first = resizedImage(100);
SimilarAsinImageEmbedder.ResizedImage second = resizedImage(100);
cache.put(URL_A, first);
cache.put(URL_A, second);
assertEquals(100, cache.sizeBytes(), "替换不重复计入字节");
assertEquals(1, cache.size());
SimilarAsinImageEmbedder.ResizedImage replaced = cache.putIfAbsent(URL_A, resizedImage(200));
assertEquals(second, replaced, "putIfAbsent 返回已有值");
assertEquals(100, cache.sizeBytes(), "putIfAbsent 不替换字节");
assertEquals(0, cache.evictionCount(), "重复操作不触发淘汰");
}
@Test
void test_task_022_image_cache_excel_boundary_empty_input() {
// 空输入：空缓存 get 返回 null；null/空集合安全。
BoundedImageCache cache = new BoundedImageCache(1_000, 100);
assertNull(cache.get(URL_A));
assertEquals(0, cache.size());
assertEquals(0, cache.sizeBytes());
assertNull(cache.putIfAbsent(null, resizedImage(10)), "null key 不放入");
assertEquals(0, cache.size());
}
@Test
void test_task_022_image_cache_excel_boundary_single_item() {
// 单元素边界：单图恰好等于预算可放入；剩余预算恰够时刚好放下。
BoundedImageCache exact = new BoundedImageCache(100, 100);
SimilarAsinImageEmbedder.ResizedImage image = resizedImage(100);
exact.put(URL_A, image);
assertEquals(image, exact.get(URL_A), "单图等于预算可缓存");
assertEquals(100, exact.sizeBytes());
BoundedImageCache tight = new BoundedImageCache(150, 100);
tight.put(URL_A, resizedImage(100));
tight.put(URL_B, resizedImage(50));
assertEquals(2, tight.size(), "剩余预算恰够时恰好放下");
assertEquals(0, tight.evictionCount());
}
@Test
void test_task_022_image_cache_excel_boundary_limit_and_overflow() {
// 上限/超限：单图超过预算被拒绝缓存且不占用预算；总字节永不超过预算。
BoundedImageCache cache = new BoundedImageCache(100, 100);
SimilarAsinImageEmbedder.ResizedImage huge = resizedImage(200);
cache.put(URL_A, huge);
assertNull(cache.get(URL_A), "单图超预算拒绝缓存");
assertEquals(0, cache.sizeBytes(), "被拒绝的图不占用预算");
assertEquals(1, cache.rejectedCount());
cache.put(URL_B, resizedImage(60));
cache.put(URL_C, resizedImage(60));
assertEquals(1, cache.size(), "第二次放入触发淘汰，仅保留最新");
assertNull(cache.get(URL_B));
assertNotNull(cache.get(URL_C));
assertEquals(60, cache.sizeBytes(), "总字节不超预算");
assertTrue(cache.evictionCount() >= 1);
}
@Test
void test_task_022_image_cache_excel_invalid_input_rejected() {
// 非法参数：非正预算/非正条目上限 → 拒绝；null 值拒绝。
assertThrows(IllegalArgumentException.class, () -> new BoundedImageCache(0, 100));
assertThrows(IllegalArgumentException.class, () -> new BoundedImageCache(-1, 100));
assertThrows(IllegalArgumentException.class, () -> new BoundedImageCache(1_000, 0));
BoundedImageCache cache = new BoundedImageCache(1_000, 100);
assertThrows(IllegalArgumentException.class, () -> cache.put(null, resizedImage(10)));
assertThrows(IllegalArgumentException.class, () -> cache.put("", resizedImage(10)));
assertThrows(IllegalArgumentException.class, () -> cache.put(URL_A, null));
assertEquals(0, cache.size(), "非法输入不产生缓存条目");
}
@Test
void test_task_022_image_cache_excel_dependency_failure_releases_resources() throws Exception {
// 依赖失败：图片下载失败不占缓存预算，缓存继续可用；
// Excel 组装用有界缓存后仍产出完整 workbook，缓存字节有界。
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
String failUrl = "https://thumb.example/fail.jpg";
when(imageEmbedder.fetchAndResizeForCache(failUrl)).thenReturn(null);
String okUrl = "https://thumb.example/ok.jpg";
when(imageEmbedder.fetchAndResizeForCache(okUrl))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
long smallBudget = 200;
ShopDataCrawlExcelAssemblyService service =
new ShopDataCrawlExcelAssemblyService(imageEmbedder, smallBudget);
List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
for (int i = 0; i < 10; i++) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate("2026-07-25");
row.setAsin("B0" + String.format("%08d", i));
row.setCommodityImage(i % 2 == 0 ? okUrl : failUrl);
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
country.setCountry("UK");
country.setItems(List.of(row));
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
item.setSuccess(true);
item.setCountryResults(List.of(country));
items.add(item);
}
File output = tempDir.resolve("bounded.xlsx").toFile();
service.writeWorkbook(output, items);
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals(10, workbook.getSheet("英国").getLastRowNum(), "全部 10 行写入");
assertEquals(1, workbook.getAllPictures().size(), "失败图片不嵌入，成功图片去重嵌入");
assertEquals("https://thumb.example/fail.jpg",
workbook.getSheet("英国").getRow(2).getCell(2).getStringCellValue(),
"失败图片兜底为 URL 文本");
}
}
private static SimilarAsinImageEmbedder.ResizedImage resizedImage(int bytes) {
return new SimilarAsinImageEmbedder.ResizedImage(new byte[bytes], 2, 2);
}
private static byte[] jpegBytes() throws Exception {
BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB);
ByteArrayOutputStream output = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", output);
return output.toByteArray();
}
}