fix(上架): 图片预取并发按 IO 收口 + 缺图就地补图,修线上 14591 个图片单元格静默留空
线上日志证据:4 核机器上 `image pool initialized requested=2 effective=2 cpuBoundLimit=2`, 8 个下载 worker 被 CPU 核数压到 2;一次组装 total=4045 行只完成 1968 行就撞上 1800s 预算, 尾部 URL 全部预取不到,落单元格时直接留空且用户无感知。 三层修复: 1. 下载池与缩放池解耦——下载是 IO 等待,默认 8、上限只挡配置写错(64);缩放仍按 CPU 收口。 OkHttp 连接池随之放大,否则默认 5 条空闲连接会让 3 个 worker 反复重做 TLS 握手。 2. 预取未覆盖到的 URL 在写单元格时就地补下载(单张落盘、不驻堆),共用一个 300s 窗口; 窗口从「预取收工、开始写单元格」起算且首个 workbook 独占——在建 spool 时 arm 会被 1800s 预取整个吃满,等于没有兜底。已预取失败的 URL 记入失败集跳过,避免每个死源站 再烧掉一次 下载超时×重试 的预算。 3. 交付不再静默:缺行/缺图写入 result.notice(状态仍为 SUCCESS,不碰 task.errorMessage, 否则会被 completeLlmFileJob 的失败判据读成整单失败)。 imagePrefetchBudgetSeconds 在生产配置里无人读取,改名 imageEmbedFallbackBudgetSeconds 并 接上实际用途。
This commit is contained in:
@@ -41,9 +41,10 @@ public class SimilarAsinProperties {
|
||||
private int llmFlushPendingMinutes = 1;
|
||||
|
||||
/**
|
||||
* 图片下载、解码和缩放共享该池;4 核生产机默认 2,避免图片任务占满整机 CPU。
|
||||
* 图片下载并发。下载是 IO 等待,不再按 CPU 核数收口(resize 的 CPU 槽位由
|
||||
* SimilarAsinImageEmbedder 单独限流),所以 4 核生产机也可以开到 8。
|
||||
*/
|
||||
private int imageDownloadPoolSize = 2;
|
||||
private int imageDownloadPoolSize = 8;
|
||||
|
||||
/**
|
||||
* 单张图片下载超时(秒)。
|
||||
@@ -53,7 +54,7 @@ public class SimilarAsinProperties {
|
||||
*/
|
||||
private int imageDownloadTimeoutSeconds = 5;
|
||||
|
||||
/** 单个结果文件整批图片预取预算,耗尽后缺图单元格降级为 URL。 */
|
||||
/** 单个结果文件整批图片预取预算;耗尽后剩余的 URL 交给 embed 阶段按补图预算兜底。 */
|
||||
private int imagePrefetchTimeoutSeconds = 1800;
|
||||
|
||||
/** 多源结果文件组装的单任务硬上限;运行期间由文件任务 heartbeat 保活。 */
|
||||
@@ -139,11 +140,14 @@ public class SimilarAsinProperties {
|
||||
private int imageCacheTouchFlushThreshold = 1000;
|
||||
|
||||
/**
|
||||
* Task 16:图片预取短预算(秒)。assemble 阶段预取在预算内 best-effort
|
||||
* 尽力完成,预算耗尽即取消在途任务并回退 URL,避免整批预取拖垮结果组装。
|
||||
* 该值仅作用于 assemble 阶段预取;后台预热仍使用 imagePrefetchTimeoutSeconds。
|
||||
* embed(写单元格)阶段的缺图补下载预算(秒)。预取预算 imagePrefetchTimeoutSeconds
|
||||
* 耗尽后没落到临时目录的 URL,过去直接留空;现在写单元格时在这个时间窗内就地补下载
|
||||
* (单张落盘、不把字节留在堆上),窗耗尽才留空并计入交付缺失提示。
|
||||
*
|
||||
* <p>窗口从第一个 workbook 进入写单元格阶段起算,跨 workbook 共享同一个窗口,
|
||||
* 所以它直接叠加到结果组装总耗时上:默认 300s,远小于 result-file-timeout-minutes。
|
||||
*/
|
||||
private int imagePrefetchBudgetSeconds = 60;
|
||||
private int imageEmbedFallbackBudgetSeconds = 300;
|
||||
|
||||
/**
|
||||
* 货源查询直连 LLM 模式开关(默认 true:新任务与存量 PENDING 批次都走直连 LLM,
|
||||
|
||||
+37
-1
@@ -15,7 +15,7 @@ import java.util.regex.Pattern;
|
||||
* <ul>
|
||||
* <li>LLM 输出可用性判定:hasUsableLlmField / hasResolvedLlmFields / isTechnicalLlmFailure /
|
||||
* isSilentOutputFailure(技术性失败与"静默不输出"两类语义);</li>
|
||||
* <li>结果状态判定:resolveResultStatus(成功 / 失败 / ASIN未找到)、isAsinNotFound、isFailedStatus;</li>
|
||||
* <li>结果状态判定:resolveResultStatus(成功 / 失败 / ASIN未找到 / 未检测)、isAsinNotFound、isFailedStatus;</li>
|
||||
* <li>导出过滤与文本渲染:isExportableResultRow、userFacingLlmCellValue、
|
||||
* hasText / isSpreadsheetErrorValue / isBlankOrSpreadsheetError;</li>
|
||||
* <li>标识与文件名工具:baseId、normalizeDisplayId、safeFileStem。</li>
|
||||
@@ -142,6 +142,13 @@ public final class SimilarAsinResultTextSupport {
|
||||
if (isAsinNotFound(row.getError())) {
|
||||
return "ASIN未找到";
|
||||
}
|
||||
// 缺必填字段(图片 URL / ASIN / 标题缺失)的行:Python 端根本没采到货源数据,
|
||||
// 语义是"没检测"而不是"检测后失败"。状态列明确标注,避免整列留白让用户误以为
|
||||
// "跑了但没结果"(2026-09-20 任务 28551:790/1000 行空白,任务仍报成功)。
|
||||
// 必须排在 isAsinNotFound 之后:404 行的 reason 里带的也是同一句缺字段文案。
|
||||
if (isSilentOutputFailure(row)) {
|
||||
return "未检测";
|
||||
}
|
||||
String rawStatus = row.getStatus() == null ? "" : row.getStatus().trim();
|
||||
if (!rawStatus.isBlank()) {
|
||||
return isFailedStatus(rawStatus) ? "失败" : "成功";
|
||||
@@ -156,6 +163,35 @@ public final class SimilarAsinResultTextSupport {
|
||||
return hasText(row.getConclusion()) ? "成功" : "失败";
|
||||
}
|
||||
|
||||
/**
|
||||
* 结果行数不全时的用户可见提示;行数齐了返回 null。
|
||||
*
|
||||
* <p>行数不全的典型成因是客户端采不到货源(扩展不可用、商品页抓取失败),
|
||||
* 这些行在结果里显示"未检测"。任务状态按产品决定仍保持成功,但必须让用户
|
||||
* 看出"这批只跑了一部分"——否则 1000 行的文件里 790 行空白会被当成
|
||||
* "跑了但没结果"(2026-09-20 任务 28551)。
|
||||
*/
|
||||
public static String incompleteRowsMessage(long resolvedRows, int totalRows) {
|
||||
long undetectedRows = (long) totalRows - resolvedRows;
|
||||
if (undetectedRows <= 0) {
|
||||
return null;
|
||||
}
|
||||
return "仅完成 " + resolvedRows + "/" + totalRows
|
||||
+ " 行,其余 " + undetectedRows + " 行未采到数据(未检测)";
|
||||
}
|
||||
|
||||
/**
|
||||
* 图片单元格缺口提示,与 {@link #incompleteRowsMessage} 同一产品口径:任务状态保持成功,
|
||||
* 但交付缺图必须让用户看出来——过去预取预算耗尽后这些格子是静默留白的
|
||||
* (线上 14591 个单元格),用户只能自己打开文件逐行发现。
|
||||
*/
|
||||
public static String blankImageCellsMessage(long blankCells) {
|
||||
if (blankCells <= 0) {
|
||||
return null;
|
||||
}
|
||||
return "另有 " + blankCells + " 个图片单元格未能嵌入(图片源站响应过慢),数据列不受影响";
|
||||
}
|
||||
|
||||
public static boolean isExportableResultRow(SimilarAsinResultRowDto row) {
|
||||
if (row == null) {
|
||||
return false;
|
||||
|
||||
+78
-18
@@ -152,6 +152,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
? properties.getImageCacheMaxBytes()
|
||||
: BoundedImageCache.DEFAULT_MAX_BYTES;
|
||||
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
|
||||
SimilarAsinImageEmbedder.ImageSpool imageSpool = createTaskImageSpool(outputDir);
|
||||
try {
|
||||
int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
|
||||
long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
|
||||
@@ -185,7 +186,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
List.of("不符合"),
|
||||
"[similar-asin] taskId=" + task.getId()
|
||||
);
|
||||
writeResultWorkbook(xlsx, captured.rows(), sourceResultMap, taskImageCache);
|
||||
writeResultWorkbook(xlsx, captured.rows(), sourceResultMap, taskImageCache, imageSpool);
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
return new SourceResultWorkbook(
|
||||
xlsx, filename, captured.rows().size(), persistedRows,
|
||||
@@ -269,7 +270,9 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
result.setResultFileSize(uploadFile.length());
|
||||
result.setResultContentType(contentType);
|
||||
result.setRowCount(parsedRows.size());
|
||||
applyPartialDeliveryNotice(task, result, resolvedRows, parsedRows.size(), imageSpool);
|
||||
} finally {
|
||||
closeTaskImageSpool(imageSpool, task.getId());
|
||||
taskImageCache.clear();
|
||||
for (File workbookFile : workbookFiles) {
|
||||
if (workbookFile.exists() && !workbookFile.delete()) {
|
||||
@@ -337,6 +340,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
? properties.getImageCacheMaxBytes()
|
||||
: BoundedImageCache.DEFAULT_MAX_BYTES;
|
||||
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
|
||||
SimilarAsinImageEmbedder.ImageSpool imageSpool = createTaskImageSpool(outputDir);
|
||||
try {
|
||||
// 单源和多源统一提交真实 Future,确保超时取消能中断实际 workbook 线程。
|
||||
// 所有源文件共享同一个绝对 deadline,避免逐个 Future 各等待一轮完整超时。
|
||||
@@ -356,7 +360,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
workbookFiles.add(xlsx);
|
||||
futures.add(assembleExecutor.submit(() -> {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
|
||||
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache, imageSpool);
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
|
||||
}));
|
||||
@@ -426,7 +430,9 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
result.setResultFileSize(uploadFile.length());
|
||||
result.setResultContentType(contentType);
|
||||
result.setRowCount(SimilarAsinPayloadSupport.resolveAllRows(parsed).size());
|
||||
applyPartialDeliveryNotice(task, result, resolvedRows, parsedRows.size(), imageSpool);
|
||||
} finally {
|
||||
closeTaskImageSpool(imageSpool, task.getId());
|
||||
for (File workbookFile : workbookFiles) {
|
||||
if (workbookFile.exists() && !workbookFile.delete()) {
|
||||
log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
|
||||
@@ -747,18 +753,14 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
private void writeResultWorkbook(File xlsx,
|
||||
List<SimilarAsinParsedRowVo> rowsToWrite,
|
||||
Map<String, SimilarAsinResultRowDto> resultMap,
|
||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> taskImageCache) {
|
||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> taskImageCache,
|
||||
SimilarAsinImageEmbedder.ImageSpool imageSpool) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
// 单元格内嵌图片:写完 workbook 后 patch WPS DISPIMG 结构(WPS/钉钉/飞书 打开即为单元格图片,
|
||||
// 与 Excel 365 richData 不同,DISPIMG 是 WPS 原生协议;协议细节见 WpsCellImageWriter 类注释)。
|
||||
// imageSpool 由调用方按任务创建:多源文件的 URL 大量重叠,per-workbook 建 spool 会让
|
||||
// 同一批图片被重复下载 N 遍(4 源 × 4000 URL 直接把预取预算烧穿)。
|
||||
WpsCellImageWriter.Session cellImageSession = WpsCellImageWriter.createSession();
|
||||
SimilarAsinImageEmbedder.ImageSpool imageSpool;
|
||||
try {
|
||||
Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-");
|
||||
imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException("create Similar ASIN image spool failed", ex);
|
||||
}
|
||||
try {
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
|
||||
Sheet sheet = workbook.createSheet("相似asin检测");
|
||||
@@ -817,6 +819,10 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
}
|
||||
}
|
||||
|
||||
// 补图窗口从"预取收工、开始写单元格"这一刻起算,任务级只 arm 一次。
|
||||
// 在建 spool 时 arm 会被 1800s 的预取整个吃满,等写单元格时窗早已耗尽,等于没有兜底。
|
||||
imageSpool.allowFallbackDownload(properties.getImageEmbedFallbackBudgetSeconds());
|
||||
|
||||
int rowIndex = 1;
|
||||
for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.<SimilarAsinParsedRowVo>of() : rowsToWrite) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
@@ -838,8 +844,9 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
String isConform = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getIsConform());
|
||||
String reason = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getReason());
|
||||
String category = resultRow == null ? "" : SimilarAsinResultTextSupport.userFacingLlmCellValue(resultRow, resultRow.getCategory());
|
||||
// 状态列不再对"静默失败"行整格留白:缺必填字段的行由 resolveResultStatus
|
||||
// 输出"未检测",让用户一眼看出这行没跑成(28551:790 行空白像"没结果")。
|
||||
String status = resultRow == null
|
||||
|| (SimilarAsinResultTextSupport.isSilentOutputFailure(resultRow) && !SimilarAsinResultTextSupport.isAsinNotFound(resultRow.getError()))
|
||||
? ""
|
||||
: SimilarAsinResultTextSupport.resolveResultStatus(resultRow, isStock, similarity, isConform, reason, category);
|
||||
row.createCell(col++).setCellValue(isStock);
|
||||
@@ -879,13 +886,66 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("生成相似ASIN检测结果失败", ex);
|
||||
} finally {
|
||||
try {
|
||||
imageSpool.close();
|
||||
} catch (IOException ex) {
|
||||
log.warn("[similar-asin] delete image spool failed xlsx={} err={}",
|
||||
xlsx.getAbsolutePath(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 任务级图片临时目录。多源结果文件的图片 URL 大量重叠,per-workbook 建 spool 会把同一批
|
||||
* 图片重复下载 N 遍(4 源 × 4000 URL),把预取预算烧穿后尾部大片单元格静默留空——线上因此
|
||||
* 缺了 14591 个图片单元格。补下载预算在写单元格前才 arm,见 {@code writeResultWorkbook}。
|
||||
*/
|
||||
private SimilarAsinImageEmbedder.ImageSpool createTaskImageSpool(File outputDir) {
|
||||
try {
|
||||
return new SimilarAsinImageEmbedder.ImageSpool(
|
||||
Files.createTempDirectory(outputDir.toPath(), "similar-asin-images-"));
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException("create Similar ASIN image spool failed", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 结果只交付了一部分时,往 result(不是 task)上挂一句用户可见的说明。
|
||||
* 产品决定:任务状态仍保持成功,不翻 FAILED,但必须让人看出"这批只跑了一部分"——
|
||||
* 否则 1000 行的文件里 790 行空白会被当成"跑了但没结果"
|
||||
* (2026-09-20 任务 28551:790/1000 行未采到数据仍报成功)。
|
||||
* 只在没有既有失败原因时写入,避免覆盖掉更重要的错误信息。
|
||||
* 注意不要顺手把 task.errorMessage 也写上:completeLlmFileJob 的失败判据读的是
|
||||
* task 的 errorMessage,写了会把任务翻成 FAILED,与上面的产品决定相悖。
|
||||
*/
|
||||
private void applyPartialDeliveryNotice(FileTaskEntity task, FileResultEntity result,
|
||||
long resolvedRows, int parsedRowCount,
|
||||
SimilarAsinImageEmbedder.ImageSpool imageSpool) {
|
||||
String deliveryMessage = joinDeliveryMessages(
|
||||
SimilarAsinResultTextSupport.incompleteRowsMessage(resolvedRows, parsedRowCount),
|
||||
SimilarAsinResultTextSupport.blankImageCellsMessage(imageSpool.blankCells()));
|
||||
if (deliveryMessage == null) {
|
||||
return;
|
||||
}
|
||||
log.warn("[similar-asin] result delivery incomplete taskId={} resolved={}/{} blankImageCells={} fallbackDownloaded={} message={}",
|
||||
task.getId(), resolvedRows, parsedRowCount, imageSpool.blankCells(),
|
||||
imageSpool.fallbackDownloaded(), deliveryMessage);
|
||||
if (result.getErrorMessage() == null || result.getErrorMessage().isBlank()) {
|
||||
result.setErrorMessage(deliveryMessage);
|
||||
}
|
||||
}
|
||||
|
||||
/** 行缺口与图片缺口各自成句;两句都在时并成一句用户可见提示。 */
|
||||
private static String joinDeliveryMessages(String rowMessage, String imageMessage) {
|
||||
boolean hasRowMessage = rowMessage != null && !rowMessage.isBlank();
|
||||
boolean hasImageMessage = imageMessage != null && !imageMessage.isBlank();
|
||||
if (hasRowMessage && hasImageMessage) {
|
||||
return rowMessage + ";" + imageMessage;
|
||||
}
|
||||
return hasRowMessage ? rowMessage : (hasImageMessage ? imageMessage : null);
|
||||
}
|
||||
|
||||
private static void closeTaskImageSpool(SimilarAsinImageEmbedder.ImageSpool imageSpool, Long taskId) {
|
||||
try {
|
||||
log.info("[similar-asin] image spool closed taskId={} spooled={} fallbackDownloaded={} blankCells={}",
|
||||
taskId, imageSpool.size(), imageSpool.fallbackDownloaded(), imageSpool.blankCells());
|
||||
imageSpool.close();
|
||||
} catch (IOException ex) {
|
||||
log.warn("[similar-asin] delete image spool failed taskId={} err={}", taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+125
-38
@@ -4,6 +4,7 @@ import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.ConnectionPool;
|
||||
import okhttp3.Dns;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
@@ -64,6 +65,7 @@ import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* similar-asin 结果 xlsx 的图片嵌入器:下载、缩略图压缩、任务内缓存、POI 嵌入与单图失败兜底集中在一处。
|
||||
@@ -81,8 +83,13 @@ public class SimilarAsinImageEmbedder {
|
||||
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
|
||||
/** P2-10:retry 由 1 升到 2,配合 5s timeout 单图最坏耗时 ≈ 15s。 */
|
||||
static final int DOWNLOAD_MAX_RETRY = 2;
|
||||
/** 4 核生产机默认 2;构造时还会按 JVM 可见 CPU 数的一半做硬上限。 */
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 2;
|
||||
/**
|
||||
* 下载并发。下载是 IO 等待,不受 CPU 核数收口(CPU 收口只作用于 resize,见构造器);
|
||||
* 生产 4 核机器上按 2 跑时,单 workbook 4000 个 URL 烧满 1800s 预取预算仍剩 2000 个没落盘。
|
||||
*/
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
|
||||
/** 下载并发的绝对上限:只挡配置写错(环境变量放大到几千),不再按 CPU 核数收口。 */
|
||||
static final int MAX_DOWNLOAD_POOL_SIZE = 64;
|
||||
static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
|
||||
// 单元格固定尺寸,图片在其中等比缩放(不拉伸);resize 仍按长边 1280 px 控制堆体积。
|
||||
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
|
||||
@@ -126,9 +133,7 @@ public class SimilarAsinImageEmbedder {
|
||||
private static final String UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
||||
private static final String IMAGE_ACCEPT = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8";
|
||||
private static final String DOWNLOAD_ACCEPT = "*/*";
|
||||
private static final String ACCEPT_LANGUAGE = "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7";
|
||||
private static final String AMAZON_REFERER = "https://www.amazon.com/";
|
||||
|
||||
private final int downloadTimeoutSeconds;
|
||||
private final int downloadPoolSize;
|
||||
@@ -148,15 +153,14 @@ public class SimilarAsinImageEmbedder {
|
||||
: properties.getImagePrefetchTimeoutSeconds();
|
||||
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
|
||||
int requestedPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
|
||||
int cpuBoundPoolSize = cpuBoundPoolLimit();
|
||||
this.downloadPoolSize = Math.min(requestedPoolSize, cpuBoundPoolSize);
|
||||
if (requestedPoolSize > this.downloadPoolSize) {
|
||||
log.warn("[similar-asin][image] image pool clamped requested={} effective={} visibleProcessors={}",
|
||||
requestedPoolSize, this.downloadPoolSize, Runtime.getRuntime().availableProcessors());
|
||||
}
|
||||
this.imageProcessingSlots = new Semaphore(this.downloadPoolSize, true);
|
||||
log.info("[similar-asin][image] image pool initialized requested={} effective={} cpuBoundLimit={}",
|
||||
requestedPoolSize, this.downloadPoolSize, cpuBoundPoolSize);
|
||||
// 下载是 IO 等待、resize 才是 CPU 活,两者必须分开收口:过去 downloadPoolSize 同时当作
|
||||
// resize 槽位数并被 CPU 上限夹住,4 核生产机上整条预取只有 2 路并发,
|
||||
// 单 workbook 4000 个 URL 烧满 1800s 预算仍剩 2000 个没落盘(线上 14591 个图片单元格留空)。
|
||||
this.downloadPoolSize = Math.min(MAX_DOWNLOAD_POOL_SIZE, Math.max(1, requestedPoolSize));
|
||||
int resizeSlots = cpuBoundPoolLimit();
|
||||
this.imageProcessingSlots = new Semaphore(resizeSlots, true);
|
||||
log.info("[similar-asin][image] image pool initialized pool={} resizeSlots={} visibleProcessors={}",
|
||||
this.downloadPoolSize, resizeSlots, Runtime.getRuntime().availableProcessors());
|
||||
this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
|
||||
? rawPrefetchTimeout
|
||||
: DEFAULT_PREFETCH_TIMEOUT_SECONDS;
|
||||
@@ -170,6 +174,9 @@ public class SimilarAsinImageEmbedder {
|
||||
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||
.writeTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||
.callTimeout(Duration.ofSeconds(downloadTimeoutSeconds * 2L))
|
||||
// OkHttp 默认只留 5 条空闲连接;下载并发提到 8 之后,多出来的 worker 每次都要
|
||||
// 重做 TLS 握手(货源图集中在两三个 CDN 域名上),预取吞吐会被握手吃掉一截。
|
||||
.connectionPool(new ConnectionPool(this.downloadPoolSize, 5, TimeUnit.MINUTES))
|
||||
.retryOnConnectionFailure(true)
|
||||
.followRedirects(true)
|
||||
.followSslRedirects(true)
|
||||
@@ -498,6 +505,7 @@ public class SimilarAsinImageEmbedder {
|
||||
imageSpool.put(url, thumb);
|
||||
} catch (Exception ex) {
|
||||
failed.incrementAndGet();
|
||||
imageSpool.markPrefetchFailed(url);
|
||||
log.debug("[similar-asin][image] disk-prefetch-fail url={} err={}", url, errorSummary(ex));
|
||||
}
|
||||
return null;
|
||||
@@ -639,10 +647,23 @@ public class SimilarAsinImageEmbedder {
|
||||
imageId = cellImageSession.registerImageByPath(spooled.path(), spooled.width(), spooled.height());
|
||||
dim = new ImageDim(spooled.width(), spooled.height());
|
||||
} else if (imageSpool != null) {
|
||||
// 预取未命中说明该 URL 下载失败:不重试、不把 URL 文本写进单元格,留空并打日志。
|
||||
log.warn("[similar-asin][image] cell-image-miss url={} colIdx={} (prefetch did not spool), leave blank",
|
||||
trimmedUrl, colIdx);
|
||||
return null;
|
||||
// 预取没落盘 ≠ 图下不下来:预取是有预算的(预取超时 / 并发槽位),千行文件的尾部
|
||||
// 必然覆盖不完,原先在这里直接留空,线上因此静默缺了 14591 个图片单元格。
|
||||
// 现在按 spool 的补下载预算就地补一张、落盘后按路径注册(堆上只存当前这一张);
|
||||
// 预取已经试过并失败的不在重试范围内(见 ImageSpool#prefetchFailed)。
|
||||
boolean alreadyFailed = imageSpool.prefetchFailed(trimmedUrl);
|
||||
SpoolImage recovered = !alreadyFailed && imageSpool.fallbackAllowed()
|
||||
? spillOneImage(trimmedUrl, imageSpool)
|
||||
: null;
|
||||
if (recovered == null) {
|
||||
imageSpool.countedBlankCell();
|
||||
log.warn("[similar-asin][image] cell-image-miss url={} colIdx={} ({}), leave blank",
|
||||
trimmedUrl, colIdx, alreadyFailed ? "prefetch failed" : "prefetch did not spool");
|
||||
return null;
|
||||
}
|
||||
imageSpool.countedFallbackDownload();
|
||||
imageId = cellImageSession.registerImageByPath(recovered.path(), recovered.width(), recovered.height());
|
||||
dim = new ImageDim(recovered.width(), recovered.height());
|
||||
} else {
|
||||
ResizedImage thumb = downloadAndResize(trimmedUrl, taskImageCache);
|
||||
if (thumb == null) {
|
||||
@@ -660,6 +681,30 @@ public class SimilarAsinImageEmbedder {
|
||||
return dim;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写单元格阶段的单张补下载:与预取走同一条下载/缩放链路,结果立即落到任务临时目录,
|
||||
* 因此堆上最多驻留这一张的字节。任何失败(下载不了、超尺寸、预算耗尽被中断)都返回 null,
|
||||
* 由调用方留空并计数。
|
||||
*/
|
||||
private SpoolImage spillOneImage(String url, ImageSpool imageSpool) {
|
||||
try {
|
||||
ResizedImage thumb = readLocalCachedThumb(url);
|
||||
if (thumb == null) {
|
||||
thumb = fetchAndResizeDirectForTest(url);
|
||||
if (thumb == null) {
|
||||
thumb = fetchAndResizeDirect(url);
|
||||
}
|
||||
}
|
||||
if (thumb == null || thumb.bytes() == null || thumb.bytes().length == 0) {
|
||||
return null;
|
||||
}
|
||||
return imageSpool.put(url, thumb);
|
||||
} catch (Exception ex) {
|
||||
log.debug("[similar-asin][image] cell-image-fallback-fail url={} err={}", url, errorSummary(ex));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasLocalCachedThumb(String url) {
|
||||
Path path = localCachePath(url);
|
||||
return path != null && Files.isRegularFile(path);
|
||||
@@ -955,31 +1000,16 @@ public class SimilarAsinImageEmbedder {
|
||||
}
|
||||
|
||||
static Request buildImageRequest(String url) {
|
||||
Request.Builder builder = new Request.Builder()
|
||||
return new Request.Builder()
|
||||
.url(url)
|
||||
.header("User-Agent", UA)
|
||||
.header("Accept-Language", ACCEPT_LANGUAGE)
|
||||
.header("Accept", IMAGE_ACCEPT)
|
||||
.header("Cache-Control", "no-cache")
|
||||
.get();
|
||||
if (isCozeSignedImageUrl(url)) {
|
||||
// LLM/TOS signed image links behave like direct file downloads.
|
||||
// A foreign Referer can be rejected, so keep this close to a browser address-bar download.
|
||||
builder.header("Accept", DOWNLOAD_ACCEPT);
|
||||
} else {
|
||||
builder.header("Accept", IMAGE_ACCEPT)
|
||||
.header("Referer", AMAZON_REFERER);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
static boolean isCozeSignedImageUrl(String url) {
|
||||
try {
|
||||
URI uri = new URI(url);
|
||||
String host = uri.getHost();
|
||||
return host != null && host.toLowerCase(Locale.ROOT).endsWith("-bot-platform-tos-sign.coze.cn");
|
||||
} catch (URISyntaxException ex) {
|
||||
return false;
|
||||
}
|
||||
// 不带任何 Referer:货源图分散在自家 OSS、阿里图床、Amazon 三种站点,
|
||||
// 前两者对外来 Referer 会判防盗链 403,Amazon 无 Referer 也照常 200。
|
||||
.get()
|
||||
.build();
|
||||
}
|
||||
|
||||
static long elapsedMs(long startedNanos) {
|
||||
@@ -1233,6 +1263,16 @@ public class SimilarAsinImageEmbedder {
|
||||
|
||||
private final Path directory;
|
||||
private final ConcurrentMap<String, SpoolImage> images = new ConcurrentHashMap<>();
|
||||
/** embed 阶段就地补图的时间预算截止点;未 armed({@link Long#MIN_VALUE})表示不补。 */
|
||||
private final AtomicLong fallbackDeadlineNanos = new AtomicLong(Long.MIN_VALUE);
|
||||
private final AtomicInteger fallbackDownloaded = new AtomicInteger();
|
||||
private final AtomicInteger blankCells = new AtomicInteger();
|
||||
/**
|
||||
* 预取阶段"试过且失败"的 URL。embed 阶段的补图只救"预取没来得及跑"的 URL:
|
||||
* 单张失败要烧掉 download-timeout × (重试+1) ≈ 30s,让补图窗口去重撞死站,
|
||||
* 真正没跑过的 URL 就一个都救不回来了。
|
||||
*/
|
||||
private final Set<String> prefetchFailed = ConcurrentHashMap.newKeySet();
|
||||
private volatile boolean closed;
|
||||
|
||||
public ImageSpool(Path directory) throws IOException {
|
||||
@@ -1241,6 +1281,53 @@ public class SimilarAsinImageEmbedder {
|
||||
Files.createDirectories(this.directory);
|
||||
}
|
||||
|
||||
/**
|
||||
* 开启 embed 阶段的缺图补下载预算(秒)。预取只覆盖得完一部分 URL,剩下的过去一律留空;
|
||||
* 这里给写单元格阶段一个有限的补救窗口,窗口耗尽后回到留空并由 {@link #blankCells()} 计数。
|
||||
*
|
||||
* <p>spool 是任务级的、多个 workbook 并行写,所以第一个进来 arm 的 workbook 定下截止点,
|
||||
* 后来的不续期:否则每个 workbook 各要一个窗口,补图能烧掉的时间就没有上界了。
|
||||
*/
|
||||
public void allowFallbackDownload(long budgetSeconds) {
|
||||
if (budgetSeconds <= 0L) {
|
||||
return;
|
||||
}
|
||||
fallbackDeadlineNanos.compareAndSet(Long.MIN_VALUE,
|
||||
System.nanoTime() + TimeUnit.SECONDS.toNanos(budgetSeconds));
|
||||
}
|
||||
|
||||
boolean fallbackAllowed() {
|
||||
return System.nanoTime() < fallbackDeadlineNanos.get();
|
||||
}
|
||||
|
||||
/** 补下载成功的张数与最终留空的单元格数:交付完整性口径,供结果提示与日志使用。 */
|
||||
public int fallbackDownloaded() {
|
||||
return fallbackDownloaded.get();
|
||||
}
|
||||
|
||||
public int blankCells() {
|
||||
return blankCells.get();
|
||||
}
|
||||
|
||||
void countedFallbackDownload() {
|
||||
fallbackDownloaded.incrementAndGet();
|
||||
}
|
||||
|
||||
void countedBlankCell() {
|
||||
blankCells.incrementAndGet();
|
||||
}
|
||||
|
||||
/** 记下"预取试过这张但失败了",embed 阶段的补下载据此跳过,不把有限的窗口喂给死站。 */
|
||||
void markPrefetchFailed(String url) {
|
||||
if (url != null && !url.isBlank()) {
|
||||
prefetchFailed.add(url.trim());
|
||||
}
|
||||
}
|
||||
|
||||
boolean prefetchFailed(String url) {
|
||||
return url != null && prefetchFailed.contains(url.trim());
|
||||
}
|
||||
|
||||
/** Task 18:暴露临时目录,供生命周期校验与日志。 */
|
||||
public Path directory() {
|
||||
return directory;
|
||||
|
||||
@@ -238,6 +238,13 @@ aiimage:
|
||||
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:7}
|
||||
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
|
||||
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
|
||||
# 客户端升级包(MinIO client 桶 nanri-image/versions/):此前只增不删,2026-09-21 已堆到
|
||||
# 61 个对象 / 6.2GiB。按「保留最近 N 个」而不是按天过期——2026-09-08 那次给 client 桶根
|
||||
# 加全桶 30 天 ILM,误删了 3.0.24~3.0.37 共 9 个仍在下载的安装包。默认 dry-run 只记日志。
|
||||
software-version:
|
||||
retention-keep-count: ${AIIMAGE_SOFTWARE_VERSION_RETENTION_KEEP_COUNT:20}
|
||||
retention-dry-run: ${AIIMAGE_SOFTWARE_VERSION_RETENTION_DRY_RUN:true}
|
||||
retention-cron: ${AIIMAGE_SOFTWARE_VERSION_RETENTION_CRON:0 50 4 * * *}
|
||||
permission-schema-init:
|
||||
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
|
||||
task-pressure:
|
||||
@@ -290,8 +297,8 @@ aiimage:
|
||||
max-attempt: ${AIIMAGE_TASK_RESUME_MAX_ATTEMPT:3}
|
||||
window-minutes: ${AIIMAGE_TASK_RESUME_WINDOW_MINUTES:30}
|
||||
limit: ${AIIMAGE_TASK_RESUME_LIMIT:20}
|
||||
coze-task:
|
||||
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
||||
task-queue:
|
||||
max-concurrent: ${AIIMAGE_TASK_QUEUE_MAX_CONCURRENT:12}
|
||||
brand-check:
|
||||
base-url: ${AIIMAGE_BRAND_CHECK_BASE_URL:http://47.110.241.161:16890}
|
||||
path: ${AIIMAGE_BRAND_CHECK_PATH:/brand_check}
|
||||
@@ -318,8 +325,11 @@ aiimage:
|
||||
# 多任务并行时会成倍放大、打满用户自己的网关密钥配额(2026-09-14 随品牌一起下调)。
|
||||
llm-row-concurrency: ${AIIMAGE_APPEARANCE_PATENT_LLM_ROW_CONCURRENCY:5}
|
||||
max-parse-rows: ${AIIMAGE_APPEARANCE_PATENT_MAX_PARSE_ROWS:50000}
|
||||
# LLM 覆盖率下限:组装结果时拿到结论的行占比低于该值即判任务失败(<=0 关闭)。
|
||||
# 正常任务实测约 0.91,默认 0.5 只拦"904 行只检 1 行"这类塌陷。
|
||||
llm-coverage-min-ratio: ${AIIMAGE_APPEARANCE_PATENT_LLM_COVERAGE_MIN_RATIO:0.5}
|
||||
llm-retry-times: ${AIIMAGE_APPEARANCE_PATENT_LLM_RETRY_TIMES:3}
|
||||
flush-pending-minutes: ${AIIMAGE_APPEARANCE_PATENT_FLUSH_PENDING_MINUTES:${AIIMAGE_APPEARANCE_PATENT_COZE_FLUSH_PENDING_MINUTES:1}}
|
||||
flush-pending-minutes: ${AIIMAGE_APPEARANCE_PATENT_FLUSH_PENDING_MINUTES:1}
|
||||
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:30}
|
||||
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
|
||||
similar-asin:
|
||||
@@ -329,9 +339,10 @@ aiimage:
|
||||
db-job-touch-interval-millis: ${AIIMAGE_SIMILAR_ASIN_DB_JOB_TOUCH_INTERVAL_MILLIS:60000}
|
||||
llm-flush-pending-minutes: ${AIIMAGE_SIMILAR_ASIN_LLM_FLUSH_PENDING_MINUTES:1}
|
||||
no-result-upload-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180}
|
||||
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:2}
|
||||
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:8}
|
||||
image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:5}
|
||||
image-prefetch-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_PREFETCH_TIMEOUT_SECONDS:1800}
|
||||
image-embed-fallback-budget-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_EMBED_FALLBACK_BUDGET_SECONDS:300}
|
||||
result-file-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_RESULT_FILE_TIMEOUT_MINUTES:90}
|
||||
image-cache-max-bytes: ${AIIMAGE_SIMILAR_ASIN_IMAGE_CACHE_MAX_BYTES:268435456}
|
||||
image-local-cache-dir: ${AIIMAGE_SIMILAR_ASIN_IMAGE_LOCAL_CACHE_DIR:${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}/similar-asin-image-cache}
|
||||
|
||||
Reference in New Issue
Block a user