fix(货源): 主图转存对瞬时 TLS 重置做有界重试,失败行日志带上真实原因
生产取证:主机A 近 3 小时 252 条 `main download fail` 全是 m.media-amazon.com 的 "Remote host terminated the handshake",而这些 URL 事后重取 8 条里 7 条直接 200。 旧实现单发、握手被 RST 就 return null 并丢掉真实异常,导致整行白占一轮重发。 新增 aiimage.similar-asin.llm-image-download-retry-times(默认额外 2 次,置 0 退回旧行为), 只重试连接级异常,非 200 不重试;25 秒总预算防慢源放大尾延迟;重试耗尽抛 IOException 让行失败原因上行到日志。
This commit is contained in:
@@ -186,4 +186,13 @@ public class SimilarAsinProperties {
|
||||
|
||||
/** 拼接图/主图下载超时(秒),慢源图片较多时放大该值。 */
|
||||
private int llmImageDownloadTimeoutSeconds = 10;
|
||||
|
||||
/**
|
||||
* 主图转存的<b>额外</b>重试次数(0=不重试,与原行为一致)。
|
||||
* 依据 2026-09-28 生产取证:主机A 近 3 小时 252 条主图下载失败全部是
|
||||
* {@code m.media-amazon.com} 的 "Remote host terminated the handshake",
|
||||
* 而这些 URL 事后 curl 8 条里 7 条直接 200 —— 瞬时抖动,重试就能救回整行
|
||||
* (不重试则整行判失败并重发一轮流程)。设 0 可退回旧行为。
|
||||
*/
|
||||
private int llmImageDownloadRetryTimes = 2;
|
||||
}
|
||||
|
||||
+71
-21
@@ -16,6 +16,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
@@ -58,6 +59,9 @@ public class SimilarAsinLlmService {
|
||||
private static final String STATUS_NOT_EXISTS = "不存在";
|
||||
private static final String NO_MATCH = "无";
|
||||
|
||||
/** 单行主图下载(含重试)的耗时上限,超过就带着最后一次异常退出重试。 */
|
||||
private static final long MAIN_DOWNLOAD_BUDGET_MILLIS = 25_000L;
|
||||
|
||||
/**
|
||||
* 一级/二级类目合并匹配系统提示词(工作流 109409/1544733 两次调用合一):
|
||||
* 候选以「一级 → 其下二级」的两层清单一次给出,ID 字段仍不进输出契约
|
||||
@@ -318,7 +322,13 @@ public class SimilarAsinLlmService {
|
||||
row.setMainUrl("");
|
||||
return;
|
||||
}
|
||||
String stored = uploadExternalUrlToSupplyImages(primaryUrl);
|
||||
String stored;
|
||||
try {
|
||||
stored = uploadExternalUrlToSupplyImages(primaryUrl);
|
||||
} catch (IOException ex) {
|
||||
throw new IllegalStateException("主图转存失败: " + abbreviate(primaryUrl, 160)
|
||||
+ " 原因: " + failureMessage(ex), ex);
|
||||
}
|
||||
if (stored == null) {
|
||||
throw new IllegalStateException("主图转存失败: " + abbreviate(primaryUrl, 160));
|
||||
}
|
||||
@@ -499,8 +509,12 @@ public class SimilarAsinLlmService {
|
||||
return ossStorageService.getPublicUrl(objectKey);
|
||||
}
|
||||
|
||||
/** 下载外部图片并转存到 MinIO supply_images,失败返回 null。 */
|
||||
private String uploadExternalUrlToSupplyImages(String url) {
|
||||
/**
|
||||
* 下载外部图片并转存到 MinIO supply_images。
|
||||
* 返回 null="这张图确实取不到"(URL 为空或源站给了非 200);连接级失败重试耗尽后抛 IOException,
|
||||
* 让行失败原因带上真实错误(旧实现只回 null,日志里塌成一句"主图转存失败: <url>",看不出是可重试的抖动)。
|
||||
*/
|
||||
private String uploadExternalUrlToSupplyImages(String url) throws IOException {
|
||||
byte[] bytes = downloadBytes(url);
|
||||
if (bytes == null) {
|
||||
return null;
|
||||
@@ -534,30 +548,66 @@ public class SimilarAsinLlmService {
|
||||
return raw;
|
||||
}
|
||||
|
||||
private byte[] downloadBytes(String url) {
|
||||
/**
|
||||
* 主图下载:连接级异常按 {@code aiimage.similar-asin.llm-image-download-retry-times} 重试(默认额外 2 次)。
|
||||
* 只重试连接级失败——非 200 是源站的确定答复,重试只是白等超时。
|
||||
* 总耗时受 {@link #MAIN_DOWNLOAD_BUDGET_MILLIS} 约束:慢源 hang 住时不放大尾延迟
|
||||
* (同模块的 alicdn 拼图已被这条咬过,见 SimilarAsinProperties#imageDownloadTimeoutSeconds 注释)。
|
||||
*/
|
||||
private byte[] downloadBytes(String url) throws IOException {
|
||||
String trimmed = url == null ? "" : url.trim();
|
||||
if (trimmed.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
int timeoutSeconds = Math.max(1, properties.getLlmImageDownloadTimeoutSeconds());
|
||||
try {
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(trimmed))
|
||||
.timeout(Duration.ofSeconds(timeoutSeconds))
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
|
||||
// 刻意不发 Referer:阿里图床按 Referer 白名单防盗链,外链域名会被判 403
|
||||
// (实测 cbu01.alicdn.com 带外站 Referer 稳定 403、无 Referer 直接 200)。
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<byte[]> response = downloadHttpClient().send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() == 200 && response.body() != null && response.body().length > 0) {
|
||||
return response.body();
|
||||
int retryTimes = Math.max(0, properties.getLlmImageDownloadRetryTimes());
|
||||
long deadline = System.currentTimeMillis() + MAIN_DOWNLOAD_BUDGET_MILLIS;
|
||||
IOException last = null;
|
||||
for (int attempt = 0; attempt <= retryTimes; attempt++) {
|
||||
if (attempt > 0 && (System.currentTimeMillis() > deadline || !sleepBeforeRetry(attempt))) {
|
||||
break;
|
||||
}
|
||||
log.warn("[similar-asin][llm] main download http {} url={}", response.statusCode(), abbreviate(trimmed, 160));
|
||||
return null;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin][llm] main download fail url={} err={}", abbreviate(trimmed, 160), failureMessage(ex));
|
||||
return null;
|
||||
try {
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(trimmed))
|
||||
.timeout(Duration.ofSeconds(timeoutSeconds))
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
|
||||
// 刻意不发 Referer:阿里图床按 Referer 白名单防盗链,外链域名会被判 403
|
||||
// (实测 cbu01.alicdn.com 带外站 Referer 稳定 403、无 Referer 直接 200)。
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<byte[]> response = downloadHttpClient().send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||
if (response.statusCode() == 200 && response.body() != null && response.body().length > 0) {
|
||||
if (attempt > 0) {
|
||||
log.info("[similar-asin][llm] main download retry-success url={} attempt={}/{}",
|
||||
abbreviate(trimmed, 160), attempt + 1, retryTimes + 1);
|
||||
}
|
||||
return response.body();
|
||||
}
|
||||
log.warn("[similar-asin][llm] main download http {} url={}", response.statusCode(), abbreviate(trimmed, 160));
|
||||
return null;
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
return null;
|
||||
} catch (Exception ex) {
|
||||
last = new IOException("主图下载失败(" + failureMessage(ex) + ")", ex);
|
||||
log.warn("[similar-asin][llm] main download fail url={} attempt={}/{} err={}",
|
||||
abbreviate(trimmed, 160), attempt + 1, retryTimes + 1, failureMessage(ex));
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw last == null ? new IOException("主图下载失败: " + abbreviate(trimmed, 160)) : last;
|
||||
}
|
||||
|
||||
private static boolean sleepBeforeRetry(int attempt) {
|
||||
try {
|
||||
Thread.sleep(Math.min(1000L, 250L * attempt));
|
||||
return true;
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -377,6 +377,7 @@ aiimage:
|
||||
llm-retry-times: ${AIIMAGE_SIMILAR_ASIN_LLM_RETRY_TIMES:3}
|
||||
llm-row-concurrency: ${AIIMAGE_SIMILAR_ASIN_LLM_ROW_CONCURRENCY:5}
|
||||
llm-image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:10}
|
||||
llm-image-download-retry-times: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_DOWNLOAD_RETRY_TIMES:2}
|
||||
collect-data:
|
||||
stale-timeout-minutes: ${AIIMAGE_COLLECT_DATA_STALE_TIMEOUT_MINUTES:30}
|
||||
no-result-upload-timeout-minutes: ${AIIMAGE_COLLECT_DATA_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180}
|
||||
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
||||
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 主图下载重试语义。动因是 2026-09-28 生产取证:主机A 近 3 小时 252 条
|
||||
* "err=Remote host terminated the handshake" 全部来自 m.media-amazon.com,
|
||||
* 而这些 URL 事后能直接取到 —— 旧实现一次都不重试,整行因此被判失败并重发。
|
||||
*/
|
||||
class SimilarAsinMainImageDownloadRetryTest {
|
||||
|
||||
private static final String URL = "https://m.media-amazon.com/images/I/61Yh2j3VZmL._AC_SX679_.jpg";
|
||||
private static final String RESET = "Remote host terminated the handshake";
|
||||
private static final byte[] BODY = new byte[]{1, 2, 3, 4, 5, 6, 7, 8};
|
||||
|
||||
@AfterEach
|
||||
void clearInterruptFlag() {
|
||||
// 中断用例会把标志位留在测试线程上,不清会污染后续测试。
|
||||
Thread.interrupted();
|
||||
}
|
||||
|
||||
private SimilarAsinLlmService service(SimilarAsinProperties props, HttpClient client) {
|
||||
SimilarAsinLlmService svc = new SimilarAsinLlmService(
|
||||
mock(SimilarAsinLlmClient.class),
|
||||
props,
|
||||
mock(OssProperties.class),
|
||||
mock(ProductCategoryService.class),
|
||||
mock(PuzzleImageMerger.class),
|
||||
mock(OssStorageService.class),
|
||||
mock(SimilarAsinImageEmbedder.class));
|
||||
svc.setDownloadHttpClientForTest(client);
|
||||
return svc;
|
||||
}
|
||||
|
||||
private Object invokeDownload(SimilarAsinLlmService svc, String url) throws Throwable {
|
||||
Method m = SimilarAsinLlmService.class.getDeclaredMethod("downloadBytes", String.class);
|
||||
m.setAccessible(true);
|
||||
try {
|
||||
return m.invoke(svc, url);
|
||||
} catch (InvocationTargetException ex) {
|
||||
throw ex.getCause();
|
||||
}
|
||||
}
|
||||
|
||||
/** 前 {@code failTimes} 次连接级异常,之后返回 200。 */
|
||||
private HttpClient flakyClient(int failTimes, AtomicInteger calls) {
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
try {
|
||||
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
int n = calls.incrementAndGet();
|
||||
if (n <= failTimes) {
|
||||
throw new IOException(RESET);
|
||||
}
|
||||
return new StubResponse(200, BODY);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 抖动两次后第三次成功_不再丢整行() throws Throwable {
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
SimilarAsinLlmService svc = service(new SimilarAsinProperties(), flakyClient(2, calls));
|
||||
|
||||
byte[] bytes = (byte[]) invokeDownload(svc, URL);
|
||||
|
||||
assertArrayEquals(BODY, bytes);
|
||||
assertEquals(3, calls.get(), "默认额外重试 2 次 → 第三次应成功");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 重试耗尽后抛出带真实原因的IOException() throws Throwable {
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
SimilarAsinLlmService svc = service(new SimilarAsinProperties(), flakyClient(99, calls));
|
||||
|
||||
IOException ex = assertThrows(IOException.class, () -> invokeDownload(svc, URL));
|
||||
|
||||
assertTrue(ex.getMessage().contains(RESET), "要把真实异常带出来,不能再塌成一句「主图转存失败」");
|
||||
assertEquals(3, calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 源站明确回非200时不重试() throws Throwable {
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
try {
|
||||
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
calls.incrementAndGet();
|
||||
return new StubResponse(403, new byte[0]);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
|
||||
assertNull(invokeDownload(service(new SimilarAsinProperties(), client), URL));
|
||||
assertEquals(1, calls.get(), "403 是确定答复,重试只是白等超时");
|
||||
}
|
||||
|
||||
@Test
|
||||
void retryTimes置0退回单次尝试旧行为() throws Throwable {
|
||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
||||
props.setLlmImageDownloadRetryTimes(0);
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
|
||||
IOException ex = assertThrows(IOException.class,
|
||||
() -> invokeDownload(service(props, flakyClient(99, calls)), URL));
|
||||
|
||||
assertTrue(ex.getMessage().contains(RESET));
|
||||
assertEquals(1, calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 线程被中断时立刻停止重试() throws Throwable {
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
try {
|
||||
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
calls.incrementAndGet();
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException(RESET);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
|
||||
assertThrows(IOException.class, () -> invokeDownload(service(new SimilarAsinProperties(), client), URL));
|
||||
|
||||
assertEquals(1, calls.get(), "中断后不该再排下一次重试");
|
||||
assertTrue(Thread.currentThread().isInterrupted(), "中断标志必须回设,不能吞掉");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 空URL仍按取不到处理不抛异常() throws Throwable {
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
SimilarAsinLlmService svc = service(new SimilarAsinProperties(), client);
|
||||
|
||||
assertNull(invokeDownload(svc, " "));
|
||||
assertNull(invokeDownload(svc, null));
|
||||
verify(client, never()).send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class));
|
||||
}
|
||||
|
||||
private static final class StubResponse implements HttpResponse<byte[]> {
|
||||
private final int statusCode;
|
||||
private final byte[] body;
|
||||
|
||||
private StubResponse(int statusCode, byte[] body) {
|
||||
this.statusCode = statusCode;
|
||||
this.body = body;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int statusCode() {
|
||||
return statusCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpRequest request() {
|
||||
return Mockito.mock(HttpRequest.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.util.Optional<HttpResponse<byte[]>> previousResponse() {
|
||||
return java.util.Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.http.HttpHeaders headers() {
|
||||
return java.net.http.HttpHeaders.of(java.util.Map.of(), (a, b) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] body() {
|
||||
return body;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.URI uri() {
|
||||
return java.net.URI.create(URL);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpClient.Version version() {
|
||||
return HttpClient.Version.HTTP_1_1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.util.Optional<javax.net.ssl.SSLSession> sslSession() {
|
||||
return java.util.Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user