super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(similarasin): LLM 输入图转存前压缩,规避网关拉图超时

Browse Source 
- 主图/拼图转存 MinIO supply_images 前统一走 SimilarAsinImageEmbedder.resizeImage
  (长边 1280 / JPEG q0.75 / 迭代降质至 ≤150KB),体积 1~5MB → 100~300KB
- 根因:LLM 网关按 URL 拉取 OSS 图片,原图过大偶发 context deadline exceeded
  (convert_request_failed),27117 等任务 1 行 LLM 500 即整任务卡死
- 压缩失败降级原字节不阻断行处理;压缩前后字节数 INFO 日志便于线上排查
- SimilarAsinImageEmbedder.resizeImage 由包私有改 public 供复用
This commit is contained in:
黄自达
2026-09-07 22:28:13 +08:00
parent 83e38d4278
commit abcfa5bef7
4 changed files with 38 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinLlmService.java
+30 -2
View File
@@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -157,6 +158,8 @@ public class SimilarAsinLlmService {
private final ProductCategoryService productCategoryService;
private final PuzzleImageMerger puzzleImageMerger;
private final OssStorageService ossStorageService;
/** 输入图转存前压缩复用（长边 1280 / JPEG q0.75 / ≤150KB），压小 OSS 图片让 LLM 网关拉取不再超时。 */
private final SimilarAsinImageEmbedder imageEmbedder;
private volatile ExecutorService rowExecutor;
private volatile HttpClient downloadHttpClient;
@@ -326,8 +329,9 @@ public class SimilarAsinLlmService {
if (merged == null) {
return null;
}
byte[] stored = compressForLlm("拼图", merged, "puzzle-merge:" + items.size() + "张");
try {
return uploadBytesToSupplyImages(merged);
return uploadBytesToSupplyImages(stored);
} catch (Exception ex) {
log.warn("[similar-asin][llm] puzzle upload failed items={} err={}", items.size(), failureMessage(ex));
return null;
@@ -517,11 +521,35 @@ public class SimilarAsinLlmService {
if (bytes == null) {
return null;
}
byte[] stored = compressForLlm("主图", bytes, url);
String objectKey = "supply_images/" + UUID.randomUUID() + ".jpg";
ossStorageService.uploadBytes(ossProperties.getBucket(), objectKey, bytes, "image/jpeg");
ossStorageService.uploadBytes(ossProperties.getBucket(), objectKey, stored, "image/jpeg");
return ossStorageService.getPublicUrl(objectKey);
}
/**
* 图片转存前统一压缩：网关（EMaster）按 URL 拉取 OSS 图片，原图过大（亚马逊主图常见 1~5MB）
* 或拼图网格超大会 context deadline exceeded（convert_request_failed）。
* 压缩失败（异常格式等）降级原字节，不阻断行处理。
*/
private byte[] compressForLlm(String kind, byte[] raw, String sourceUrl) {
if (raw == null || raw.length == 0) {
return raw;
}
try {
SimilarAsinImageEmbedder.ResizedImage resized = imageEmbedder.resizeImage(sourceUrl, raw);
if (resized != null && resized.bytes() != null && resized.bytes().length < raw.length) {
log.info("[similar-asin][llm] {} compress for llm url={} bytes={}->{}",
kind, abbreviate(sourceUrl, 120), raw.length, resized.bytes().length);
return resized.bytes();
}
} catch (Exception ex) {
log.warn("[similar-asin][llm] {} compress failed, fallback raw url={} err={}",
kind, abbreviate(sourceUrl, 120), failureMessage(ex));
}
return raw;
}
private byte[] downloadBytes(String url) {
String trimmed = url == null ? "" : url.trim();
if (trimmed.isBlank()) {
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
+2 -1
View File
@@ -1004,8 +1004,9 @@ public class SimilarAsinImageEmbedder {
* 等比缩放到长边 TARGET_LONG_EDGE_PX，JPEG q=0.75 输出；返回字节 + 实际像素，供调用方按图自适应单元格尺寸。
* 出口校验字节数 ≤ MAX_THUMB_SIZE_BYTES，超限按 MAX_THUMB_SIZE → 长边 → 质量的顺序迭代降级，
* 仍然超限才抛 ResizeOversizeException 触发文本兜底。
* 除 assemble 嵌入外，LLM 输入图转存前也复用本方法压缩（网关拉取 OSS 图超时的根因是原图过大）。
*/
ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
public ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
acquireImageProcessingSlot();
try {
ensureImageWorkNotInterrupted();
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinLlmLocalVerify.java
+3 -1
View File
@@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.mockito.Mockito;
import java.io.File;
@@ -99,7 +100,8 @@ public class SimilarAsinLlmLocalVerify {
}).when(categoryService).children(Mockito.any(), Mockito.anyLong(), Mockito.anyLong());
SimilarAsinLlmService service = new SimilarAsinLlmService(
client, props, ossProps, categoryService, merger, oss);
client, props, ossProps, categoryService, merger, oss,
new SimilarAsinImageEmbedder(props, oss));
long start = System.currentTimeMillis();
List<SimilarAsinResultRowDto> result = service.inspectRows(rows, null, apiKey, imgSwitch, categorySwitch);
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinLlmServiceTest.java
+3 -1
View File
@@ -9,6 +9,7 @@ import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
@@ -76,7 +77,8 @@ class SimilarAsinLlmServiceTest {
mock(OssProperties.class),
categoryService,
merger,
ossStorage);
ossStorage,
mock(SimilarAsinImageEmbedder.class));
svc.setDownloadHttpClientForTest(mockHttpClient());
return svc;
}