fix(上架): 图片预取并发按 IO 收口 + 缺图就地补图,修线上 14591 个图片单元格静默留空
线上日志证据:4 核机器上 `image pool initialized requested=2 effective=2 cpuBoundLimit=2`, 8 个下载 worker 被 CPU 核数压到 2;一次组装 total=4045 行只完成 1968 行就撞上 1800s 预算, 尾部 URL 全部预取不到,落单元格时直接留空且用户无感知。 三层修复: 1. 下载池与缩放池解耦——下载是 IO 等待,默认 8、上限只挡配置写错(64);缩放仍按 CPU 收口。 OkHttp 连接池随之放大,否则默认 5 条空闲连接会让 3 个 worker 反复重做 TLS 握手。 2. 预取未覆盖到的 URL 在写单元格时就地补下载(单张落盘、不驻堆),共用一个 300s 窗口; 窗口从「预取收工、开始写单元格」起算且首个 workbook 独占——在建 spool 时 arm 会被 1800s 预取整个吃满,等于没有兜底。已预取失败的 URL 记入失败集跳过,避免每个死源站 再烧掉一次 下载超时×重试 的预算。 3. 交付不再静默:缺行/缺图写入 result.notice(状态仍为 SUCCESS,不碰 task.errorMessage, 否则会被 completeLlmFileJob 的失败判据读成整单失败)。 imagePrefetchBudgetSeconds 在生产配置里无人读取,改名 imageEmbedFallbackBudgetSeconds 并 接上实际用途。
This commit is contained in:
+72
-13
@@ -67,18 +67,20 @@ class SimilarAsinImageEmbedderTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultsImageDownloadPoolToTwoAndCapsItByVisibleCpu() {
|
||||
assertEquals(2, new SimilarAsinProperties().getImageDownloadPoolSize());
|
||||
assertEquals(Math.min(2, SimilarAsinImageEmbedder.cpuBoundPoolLimit()), embedder.downloadPoolSize());
|
||||
void defaultsImageDownloadPoolToEightAndLeavesItUncappedByCpu() {
|
||||
assertEquals(8, new SimilarAsinProperties().getImageDownloadPoolSize());
|
||||
assertEquals(8, embedder.downloadPoolSize());
|
||||
}
|
||||
|
||||
@Test
|
||||
void clampsOversizedImagePoolConfigurationByVisibleCpu() {
|
||||
void oversizedImagePoolConfigurationIsCappedByIoCeilingNotCpu() {
|
||||
SimilarAsinProperties configured = new SimilarAsinProperties();
|
||||
configured.setImageDownloadPoolSize(Integer.MAX_VALUE);
|
||||
SimilarAsinImageEmbedder limited = new SimilarAsinImageEmbedder(configured, createOssStorageService());
|
||||
try {
|
||||
assertEquals(SimilarAsinImageEmbedder.cpuBoundPoolLimit(), limited.downloadPoolSize());
|
||||
// 下载是 IO 等待,只受绝对上限保护配置写错,不再被 CPU 核数夹住;CPU 收口只作用于 resize。
|
||||
assertEquals(SimilarAsinImageEmbedder.MAX_DOWNLOAD_POOL_SIZE, limited.downloadPoolSize());
|
||||
assertTrue(limited.downloadPoolSize() > 2, "下载并发不应再退化成 CPU 收口后的个位数");
|
||||
} finally {
|
||||
limited.shutdown();
|
||||
}
|
||||
@@ -240,6 +242,64 @@ class SimilarAsinImageEmbedderTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void armedFallbackWindowRecoversUrlPrefetchNeverReached() throws Exception {
|
||||
SimilarAsinImageEmbedder recovering = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, null), createOssStorageService());
|
||||
AtomicInteger networkCalls = new AtomicInteger();
|
||||
replaceHttpClient(recovering, failingClient(networkCalls));
|
||||
String url = "https://images.example.com/skipped-by-prefetch.jpg";
|
||||
recovering.registerPrefetchHandler(url, () -> { });
|
||||
recovering.registerPrefetchResult(url, new SimilarAsinImageEmbedder.ResizedImage(
|
||||
createJpegBytes(), 64, 48));
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-image-fallback-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
spool.allowFallbackDownload(30);
|
||||
var row = workbook.createSheet().createRow(1);
|
||||
|
||||
SimilarAsinImageEmbedder.ImageDim dim = recovering.embedAsExcelCellImage(
|
||||
1, 9, url, row, new HashMap<>(), WpsCellImageWriter.createSession(), spool);
|
||||
|
||||
assertNotNull(dim, "预取没跑到的 URL,补图窗口内应就地补上而不是留空");
|
||||
assertEquals(64, dim.width());
|
||||
assertEquals(1, spool.size(), "补到的图应落到任务级 spool,不留在堆上");
|
||||
assertEquals(1, spool.fallbackDownloaded());
|
||||
assertEquals(0, spool.blankCells());
|
||||
assertEquals(0, networkCalls.get(), "补图走测试钩子,不应打真网络");
|
||||
} finally {
|
||||
recovering.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void armedFallbackWindowSkipsUrlsPrefetchAlreadyFailed() throws Exception {
|
||||
SimilarAsinImageEmbedder deadEmbedder = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, null), createOssStorageService());
|
||||
AtomicInteger networkCalls = new AtomicInteger();
|
||||
replaceHttpClient(deadEmbedder, failingClient(networkCalls));
|
||||
String url = "https://images.example.com/dead.jpg";
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-image-fallback-skip-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
deadEmbedder.prefetchToDisk(List.of(url), spool);
|
||||
int callsAfterPrefetch = networkCalls.get();
|
||||
assertTrue(callsAfterPrefetch > 0, "预取阶段应已试过这张图并失败");
|
||||
spool.allowFallbackDownload(30);
|
||||
var row = workbook.createSheet().createRow(1);
|
||||
|
||||
assertNull(deadEmbedder.embedAsExcelCellImage(
|
||||
1, 9, url, row, new HashMap<>(), WpsCellImageWriter.createSession(), spool));
|
||||
|
||||
assertEquals(callsAfterPrefetch, networkCalls.get(),
|
||||
"补图窗口只救\"预取没跑到\"的 URL,重撞死站会把有限窗口吃光");
|
||||
assertEquals(1, spool.blankCells());
|
||||
assertEquals(0, spool.fallbackDownloaded());
|
||||
} finally {
|
||||
deadEmbedder.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void imageSpoolWritesThumbnailAndDeletesTaskDirectoryOnClose() throws Exception {
|
||||
Path directory = Files.createTempDirectory("similar-asin-image-spool-test-");
|
||||
@@ -421,7 +481,7 @@ class SimilarAsinImageEmbedderTest {
|
||||
assertEquals("GET", req.method());
|
||||
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||
assertTrue(req.header("Accept").contains("image/"), "Accept should prefer images");
|
||||
assertEquals("https://www.amazon.com/", req.header("Referer"));
|
||||
assertNull(req.header("Referer"), "图片下载不发 Referer:外来 Referer 会被阿里图床判防盗链 403");
|
||||
assertEquals("no-cache", req.header("Cache-Control"));
|
||||
assertNotNull(req.header("Accept-Language"));
|
||||
}
|
||||
@@ -439,23 +499,22 @@ class SimilarAsinImageEmbedderTest {
|
||||
|
||||
@Test
|
||||
void downloadCandidatesLeaveNonAmazonAndUnsignedUrlsUntouched() {
|
||||
String coze = "https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a";
|
||||
assertEquals(List.of(coze), SimilarAsinImageEmbedder.downloadCandidates(coze));
|
||||
String oss = "https://oss.aishufu.top/nanri-ai-images/supply_images/1f2e.jpg?x-signature=a";
|
||||
assertEquals(List.of(oss), SimilarAsinImageEmbedder.downloadCandidates(oss));
|
||||
assertEquals(List.of("https://cbu01.alicdn.com/O1CN01x.jpg"),
|
||||
SimilarAsinImageEmbedder.downloadCandidates("https://cbu01.alicdn.com/O1CN01x.jpg"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildImageRequestUsesDownloadHeadersForCozeSignedImages() {
|
||||
void buildImageRequestSendsSameHeadersForEveryHost() {
|
||||
var req = SimilarAsinImageEmbedder.buildImageRequest(
|
||||
"https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a");
|
||||
"https://cbu01.alicdn.com/img/ibank/O1CN01abc_1953737542.jpg");
|
||||
|
||||
assertEquals("GET", req.method());
|
||||
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||
assertEquals("*/*", req.header("Accept"));
|
||||
assertNull(req.header("Referer"), "Coze signed download links should not carry an Amazon referer");
|
||||
assertTrue(req.header("Accept").contains("image/"), "Accept should prefer images");
|
||||
assertNull(req.header("Referer"), "阿里图床按 Referer 白名单放行,带任何外来 Referer 都会 403");
|
||||
assertEquals("no-cache", req.header("Cache-Control"));
|
||||
assertTrue(SimilarAsinImageEmbedder.isCozeSignedImageUrl(req.url().toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user