super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-22: 店铺 Excel 图片缓存替换为有界字节缓存

Browse Source 
BoundedImageCache（字节预算+条目上限，FIFO 淘汰，超预算单图拒绝）替换
ShopDataCrawlExcelAssemblyService 中的无界 ConcurrentHashMap；单图超预算时
embed 走原有直接下载兜底。全量测试 524 通过。
This commit is contained in:
黄自达
2026-08-29 18:29:25 +08:00
parent 064e13c132
commit 6d488515ca
3 changed files with 315 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java
+20 -6
View File
@@ -4,6 +4,7 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -28,7 +29,6 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@Slf4j
@@ -44,8 +44,22 @@ public class ShopDataCrawlExcelAssemblyService {
private static final int BRAND_COLUMN = HEADERS.size() - 1;
private static final int IMAGE_COLUMN_WIDTH = 18 * 256;
private static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
/** 图片缓存默认上限：64MB 字节预算 / 2000 条目，超过按 FIFO 淘汰，保证组装期内存有界。 */
private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024;
private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000;
private final SimilarAsinImageEmbedder imageEmbedder;
private long imageCacheMaxBytes = DEFAULT_IMAGE_CACHE_MAX_BYTES;
private int imageCacheMaxEntries = DEFAULT_IMAGE_CACHE_MAX_ENTRIES;
public ShopDataCrawlExcelAssemblyService(SimilarAsinImageEmbedder imageEmbedder, long imageCacheMaxBytes) {
this.imageEmbedder = imageEmbedder;
this.imageCacheMaxBytes = imageCacheMaxBytes;
}
private BoundedImageCache newImageCache() {
return new BoundedImageCache(imageCacheMaxBytes, imageCacheMaxEntries);
}
public void writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
@@ -53,7 +67,7 @@ public class ShopDataCrawlExcelAssemblyService {
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache = new ConcurrentHashMap<>();
BoundedImageCache imageCache = newImageCache();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
@@ -82,7 +96,7 @@ public class ShopDataCrawlExcelAssemblyService {
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache = new ConcurrentHashMap<>();
BoundedImageCache imageCache = newImageCache();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
@@ -133,7 +147,7 @@ public class ShopDataCrawlExcelAssemblyService {
private void writeSheet(XSSFWorkbook workbook,
Sheet sheet,
List<ShopDataCrawlRowDto> rows,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) {
Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1);
@@ -175,7 +189,7 @@ public class ShopDataCrawlExcelAssemblyService {
Row row,
ShopDataCrawlRowDto value,
CellStyle[] styles,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) {
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
@@ -218,7 +232,7 @@ public class ShopDataCrawlExcelAssemblyService {
Sheet sheet,
Row row,
String imageUrl,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) {
String normalizedUrl = imageUrl.trim();
try {
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/BoundedImageCache.java
+91
View File
@@ -0,0 +1,91 @@
package com.nanri.aiimage.modules.shopdatacrawl.util;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.extern.slf4j.Slf4j;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* Task 22：店铺 Excel 图片的有界字节缓存。
* 以总字节预算 + 条目上限约束图片缓存，超过预算按 FIFO（插入序）淘汰最旧条目；
* 单图超过预算时拒绝缓存并计数，embed 阶段对该 url 走原有直接下载兜底。
* 与 {@link SimilarAsinImageEmbedder#prefetch(java.util.Collection, Map)} 的
* Map 入参兼容，保证组装期缓存内存峰值有界。
*/
@Slf4j
public class BoundedImageCache extends LinkedHashMap<String, SimilarAsinImageEmbedder.ResizedImage> {
private final long maxBytes;
private final int maxEntries;
private long sizeBytes;
private long evictionCount;
private long rejectedCount;
public BoundedImageCache(long maxBytes, int maxEntries) {
super(16, 0.75f, false);
if (maxBytes <= 0) {
throw new IllegalArgumentException("maxBytes 必须为正数，实际 " + maxBytes);
}
if (maxEntries <= 0) {
throw new IllegalArgumentException("maxEntries 必须为正数，实际 " + maxEntries);
}
this.maxBytes = maxBytes;
this.maxEntries = maxEntries;
}
public long sizeBytes() {
return sizeBytes;
}
public long evictionCount() {
return evictionCount;
}
public long rejectedCount() {
return rejectedCount;
}
@Override
public SimilarAsinImageEmbedder.ResizedImage put(String key, SimilarAsinImageEmbedder.ResizedImage value) {
if (key == null || key.isBlank()) {
throw new IllegalArgumentException("image key 不能为空");
}
if (value == null) {
throw new IllegalArgumentException("image value 不能为 null");
}
if (value.bytes() == null || value.bytes().length > maxBytes) {
rejectedCount++;
log.debug("[shop-data-crawl][image-cache] reject oversized image bytes={} maxBytes={}",
value.bytes() == null ? 0 : value.bytes().length, maxBytes);
return null;
}
if (containsKey(key)) {
sizeBytes -= super.get(key).bytes().length;
}
while (!isEmpty() && (sizeBytes + value.bytes().length > maxBytes || size() >= maxEntries)) {
evictEldest();
}
sizeBytes += value.bytes().length;
return super.put(key, value);
}
@Override
public SimilarAsinImageEmbedder.ResizedImage putIfAbsent(String key, SimilarAsinImageEmbedder.ResizedImage value) {
if (key == null) {
return null;
}
SimilarAsinImageEmbedder.ResizedImage existing = super.get(key);
if (existing != null) {
return existing;
}
return put(key, value);
}
private void evictEldest() {
Map.Entry<String, SimilarAsinImageEmbedder.ResizedImage> eldest = entrySet().iterator().next();
sizeBytes -= eldest.getValue().bytes().length;
super.remove(eldest.getKey());
evictionCount++;
}
}