super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-24: 店铺图片预取增加任务级数量、字节和超时上限

Browse Source 
ShopDataCrawlPrefetchBudget 对预取 URL 施加三重预算：数量截断（去重后按序取前 N）、
字节估算门（urlCount×avgBytes 超预算拒绝）、超时钳制（200ms/URL 估算钳到任务上限）；
Excel 组装在 prefetch 前应用预算，未预取 URL 由 embed 兜底直接下载。
全量测试 540 通过。
This commit is contained in:
黄自达
2026-08-29 18:38:37 +08:00
parent a9cbd18246
commit 7eccbc016a
3 changed files with 321 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java
+23 -2
View File
@@ -5,6 +5,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache;
import com.nanri.aiimage.modules.shopdatacrawl.util.ShopDataCrawlPrefetchBudget;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -47,20 +48,40 @@ public class ShopDataCrawlExcelAssemblyService {
/** 图片缓存默认上限：64MB 字节预算 / 2000 条目，超过按 FIFO 淘汰，保证组装期内存有界。 */
private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024;
private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000;
/** 预取默认上限：单任务最多预取 2000 个唯一 URL；其余 URL 由 embed 阶段兜底直接下载。 */
private static final int DEFAULT_PREFETCH_MAX_URLS = 2000;
/** 预取超时默认上限（秒），按 200ms/URL 估算的 deadline 被钳制到该值。 */
private static final long DEFAULT_PREFETCH_TIMEOUT_SECONDS = 120L;
private final SimilarAsinImageEmbedder imageEmbedder;
private long imageCacheMaxBytes = DEFAULT_IMAGE_CACHE_MAX_BYTES;
private int imageCacheMaxEntries = DEFAULT_IMAGE_CACHE_MAX_ENTRIES;
private int prefetchMaxUrls = DEFAULT_PREFETCH_MAX_URLS;
public ShopDataCrawlExcelAssemblyService(SimilarAsinImageEmbedder imageEmbedder, long imageCacheMaxBytes) {
this.imageEmbedder = imageEmbedder;
this.imageCacheMaxBytes = imageCacheMaxBytes;
}
public ShopDataCrawlExcelAssemblyService(SimilarAsinImageEmbedder imageEmbedder, long imageCacheMaxBytes,
int prefetchMaxUrls) {
this.imageEmbedder = imageEmbedder;
this.imageCacheMaxBytes = imageCacheMaxBytes;
this.prefetchMaxUrls = prefetchMaxUrls;
}
private BoundedImageCache newImageCache() {
return new BoundedImageCache(imageCacheMaxBytes, imageCacheMaxEntries);
}
private ShopDataCrawlPrefetchBudget prefetchBudget() {
return ShopDataCrawlPrefetchBudget.of(prefetchMaxUrls, imageCacheMaxBytes, DEFAULT_PREFETCH_TIMEOUT_SECONDS);
}
private void prefetchImages(Map<String, List<ShopDataCrawlRowDto>> rowsByCountry, BoundedImageCache imageCache) {
imageEmbedder.prefetch(prefetchBudget().boundedUrls(imageUrls(rowsByCountry)), imageCache);
}
public void writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
XSSFWorkbook workbook = new XSSFWorkbook(input);
@@ -68,7 +89,7 @@ public class ShopDataCrawlExcelAssemblyService {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
BoundedImageCache imageCache = newImageCache();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
prefetchImages(rowsByCountry, imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
writeSheet(workbook, workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)), imageCache, pictureIndexes);
@@ -97,7 +118,7 @@ public class ShopDataCrawlExcelAssemblyService {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
BoundedImageCache imageCache = newImageCache();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
prefetchImages(rowsByCountry, imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
List<ShopDataCrawlRowDto> rows = rowsByCountry.get(COUNTRIES.get(i));
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/util/ShopDataCrawlPrefetchBudget.java
+97
View File
@@ -0,0 +1,97 @@
package com.nanri.aiimage.modules.shopdatacrawl.util;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
/**
* Task 24：店铺图片预取的任务级预算。
* 对预取 URL 集合施加三重上限：
* - maxUrls：数量上限，去重后按顺序截断；
* - maxBytes：字节估算门，预计总量（urlCount × avgBytesPerUrl）超过预算时拒绝预取；
* - maxTimeoutSeconds：预取超时上限，按 url 数估算的 deadline（200ms/url）被钳制到该上限。
* 被截断/拒绝的 URL 不进入预取，由 embed 阶段按原有兜底链路直接下载。
*/
public class ShopDataCrawlPrefetchBudget {
/** 预取 deadline 估算：每 URL 200ms，与 SimilarAsinImageEmbedder 的全局 deadline 计算一致。 */
private static final long MILLIS_PER_URL = 200L;
private static final long MIN_TIMEOUT_MILLIS = 15_000L;
private static final long MAX_TIMEOUT_MILLIS = 120_000L;
private final int maxUrls;
private final long maxBytes;
private final long maxTimeoutMillis;
private ShopDataCrawlPrefetchBudget(int maxUrls, long maxBytes, long maxTimeoutSeconds) {
if (maxUrls <= 0) {
throw new IllegalArgumentException("maxUrls 必须为正数，实际 " + maxUrls);
}
if (maxBytes <= 0) {
throw new IllegalArgumentException("maxBytes 必须为正数，实际 " + maxBytes);
}
if (maxTimeoutSeconds <= 0) {
throw new IllegalArgumentException("maxTimeoutSeconds 必须为正数，实际 " + maxTimeoutSeconds);
}
this.maxUrls = maxUrls;
this.maxBytes = maxBytes;
this.maxTimeoutMillis = maxTimeoutSeconds * 1000L;
}
public static ShopDataCrawlPrefetchBudget of(int maxUrls, long maxBytes, long maxTimeoutSeconds) {
return new ShopDataCrawlPrefetchBudget(maxUrls, maxBytes, maxTimeoutSeconds);
}
/** 去重（保持顺序）并按数量上限截断；null/空白条目跳过。null 集合返回空列表。 */
public List<String> boundedUrls(List<String> urls) {
if (urls == null || urls.isEmpty()) {
return new ArrayList<>();
}
LinkedHashSet<String> distinct = new LinkedHashSet<>();
for (String url : urls) {
if (url == null) {
continue;
}
String trimmed = url.trim();
if (!trimmed.isEmpty()) {
distinct.add(trimmed);
}
}
List<String> result = new ArrayList<>(Math.min(distinct.size(), maxUrls));
int added = 0;
for (String url : distinct) {
if (added >= maxUrls) {
break;
}
result.add(url);
added++;
}
return result;
}
/** 字节估算门：urlCount × avgBytesPerUrl > maxBytes 时拒绝预取。 */
public boolean wouldExceedBytes(List<String> urls, long avgBytesPerUrl) {
if (urls == null || urls.isEmpty()) {
return false;
}
if (avgBytesPerUrl <= 0) {
return false;
}
return (long) urls.size() * avgBytesPerUrl > maxBytes;
}
/** 预取超时上限：每 URL 200ms，clamp 到 [MIN, maxTimeoutMillis]，再钳到全局 [15s,120s]。 */
public long timeoutMillisFor(int urlCount) {
long estimated = Math.min(MAX_TIMEOUT_MILLIS,
Math.max(MIN_TIMEOUT_MILLIS, Math.max(1, urlCount) * MILLIS_PER_URL));
return Math.min(estimated, maxTimeoutMillis);
}
public int maxUrls() {
return maxUrls;
}
public long maxBytes() {
return maxBytes;
}
}