fix(货源): 主图转存对瞬时 TLS 重置做有界重试,失败行日志带上真实原因

生产取证:主机A 近 3 小时 252 条 `main download fail` 全是 m.media-amazon.com 的
"Remote host terminated the handshake",而这些 URL 事后重取 8 条里 7 条直接 200。
旧实现单发、握手被 RST 就 return null 并丢掉真实异常,导致整行白占一轮重发。

新增 aiimage.similar-asin.llm-image-download-retry-times(默认额外 2 次,置 0 退回旧行为),
只重试连接级异常,非 200 不重试;25 秒总预算防慢源放大尾延迟;重试耗尽抛 IOException
让行失败原因上行到日志。
This commit is contained in:
2026-09-28 11:52:11 +08:00
parent 9efb39b0a5
commit 3a39c36bde
5 changed files with 439 additions and 21 deletions
@@ -186,4 +186,13 @@ public class SimilarAsinProperties {
/** 拼接图/主图下载超时(秒),慢源图片较多时放大该值。 */
private int llmImageDownloadTimeoutSeconds = 10;
/**
* 主图转存的<b>额外</b>重试次数(0=不重试,与原行为一致)。
* 依据 2026-09-28 生产取证:主机A 近 3 小时 252 条主图下载失败全部是
* {@code m.media-amazon.com} 的 "Remote host terminated the handshake",
* 而这些 URL 事后 curl 8 条里 7 条直接 200 —— 瞬时抖动,重试就能救回整行
* (不重试则整行判失败并重发一轮流程)。设 0 可退回旧行为。
*/
private int llmImageDownloadRetryTimes = 2;
}
@@ -16,6 +16,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
@@ -58,6 +59,9 @@ public class SimilarAsinLlmService {
private static final String STATUS_NOT_EXISTS = "不存在";
private static final String NO_MATCH = "无";
/** 单行主图下载(含重试)的耗时上限,超过就带着最后一次异常退出重试。 */
private static final long MAIN_DOWNLOAD_BUDGET_MILLIS = 25_000L;
/**
* 一级/二级类目合并匹配系统提示词(工作流 109409/1544733 两次调用合一):
* 候选以「一级 → 其下二级」的两层清单一次给出,ID 字段仍不进输出契约
@@ -318,7 +322,13 @@ public class SimilarAsinLlmService {
row.setMainUrl("");
return;
}
String stored = uploadExternalUrlToSupplyImages(primaryUrl);
String stored;
try {
stored = uploadExternalUrlToSupplyImages(primaryUrl);
} catch (IOException ex) {
throw new IllegalStateException("主图转存失败: " + abbreviate(primaryUrl, 160)
+ " 原因: " + failureMessage(ex), ex);
}
if (stored == null) {
throw new IllegalStateException("主图转存失败: " + abbreviate(primaryUrl, 160));
}
@@ -499,8 +509,12 @@ public class SimilarAsinLlmService {
return ossStorageService.getPublicUrl(objectKey);
}
/** 下载外部图片并转存到 MinIO supply_images,失败返回 null。 */
private String uploadExternalUrlToSupplyImages(String url) {
/**
* 下载外部图片并转存到 MinIO supply_images。
* 返回 null="这张图确实取不到"(URL 为空或源站给了非 200);连接级失败重试耗尽后抛 IOException,
* 让行失败原因带上真实错误(旧实现只回 null,日志里塌成一句"主图转存失败: &lt;url&gt;",看不出是可重试的抖动)。
*/
private String uploadExternalUrlToSupplyImages(String url) throws IOException {
byte[] bytes = downloadBytes(url);
if (bytes == null) {
return null;
@@ -534,30 +548,66 @@ public class SimilarAsinLlmService {
return raw;
}
private byte[] downloadBytes(String url) {
/**
* 主图下载:连接级异常按 {@code aiimage.similar-asin.llm-image-download-retry-times} 重试(默认额外 2 次)。
* 只重试连接级失败——非 200 是源站的确定答复,重试只是白等超时。
* 总耗时受 {@link #MAIN_DOWNLOAD_BUDGET_MILLIS} 约束:慢源 hang 住时不放大尾延迟
* (同模块的 alicdn 拼图已被这条咬过,见 SimilarAsinProperties#imageDownloadTimeoutSeconds 注释)。
*/
private byte[] downloadBytes(String url) throws IOException {
String trimmed = url == null ? "" : url.trim();
if (trimmed.isBlank()) {
return null;
}
int timeoutSeconds = Math.max(1, properties.getLlmImageDownloadTimeoutSeconds());
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(trimmed))
.timeout(Duration.ofSeconds(timeoutSeconds))
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
// 刻意不发 Referer:阿里图床按 Referer 白名单防盗链,外链域名会被判 403
// (实测 cbu01.alicdn.com 带外站 Referer 稳定 403、无 Referer 直接 200)。
.GET()
.build();
HttpResponse<byte[]> response = downloadHttpClient().send(request, HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() == 200 && response.body() != null && response.body().length > 0) {
return response.body();
int retryTimes = Math.max(0, properties.getLlmImageDownloadRetryTimes());
long deadline = System.currentTimeMillis() + MAIN_DOWNLOAD_BUDGET_MILLIS;
IOException last = null;
for (int attempt = 0; attempt <= retryTimes; attempt++) {
if (attempt > 0 && (System.currentTimeMillis() > deadline || !sleepBeforeRetry(attempt))) {
break;
}
log.warn("[similar-asin][llm] main download http {} url={}", response.statusCode(), abbreviate(trimmed, 160));
return null;
} catch (Exception ex) {
log.warn("[similar-asin][llm] main download fail url={} err={}", abbreviate(trimmed, 160), failureMessage(ex));
return null;
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(trimmed))
.timeout(Duration.ofSeconds(timeoutSeconds))
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
// 刻意不发 Referer:阿里图床按 Referer 白名单防盗链,外链域名会被判 403
// (实测 cbu01.alicdn.com 带外站 Referer 稳定 403、无 Referer 直接 200)。
.GET()
.build();
HttpResponse<byte[]> response = downloadHttpClient().send(request, HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() == 200 && response.body() != null && response.body().length > 0) {
if (attempt > 0) {
log.info("[similar-asin][llm] main download retry-success url={} attempt={}/{}",
abbreviate(trimmed, 160), attempt + 1, retryTimes + 1);
}
return response.body();
}
log.warn("[similar-asin][llm] main download http {} url={}", response.statusCode(), abbreviate(trimmed, 160));
return null;
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
return null;
} catch (Exception ex) {
last = new IOException("主图下载失败(" + failureMessage(ex) + ")", ex);
log.warn("[similar-asin][llm] main download fail url={} attempt={}/{} err={}",
abbreviate(trimmed, 160), attempt + 1, retryTimes + 1, failureMessage(ex));
if (Thread.currentThread().isInterrupted()) {
break;
}
}
}
throw last == null ? new IOException("主图下载失败: " + abbreviate(trimmed, 160)) : last;
}
private static boolean sleepBeforeRetry(int attempt) {
try {
Thread.sleep(Math.min(1000L, 250L * attempt));
return true;
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
return false;
}
}
@@ -377,6 +377,7 @@ aiimage:
llm-retry-times: ${AIIMAGE_SIMILAR_ASIN_LLM_RETRY_TIMES:3}
llm-row-concurrency: ${AIIMAGE_SIMILAR_ASIN_LLM_ROW_CONCURRENCY:5}
llm-image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:10}
llm-image-download-retry-times: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_DOWNLOAD_RETRY_TIMES:2}
collect-data:
stale-timeout-minutes: ${AIIMAGE_COLLECT_DATA_STALE_TIMEOUT_MINUTES:30}
no-result-upload-timeout-minutes: ${AIIMAGE_COLLECT_DATA_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180}
@@ -0,0 +1,226 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 主图下载重试语义。动因是 2026-09-28 生产取证:主机A 近 3 小时 252 条
* "err=Remote host terminated the handshake" 全部来自 m.media-amazon.com,
* 而这些 URL 事后能直接取到 —— 旧实现一次都不重试,整行因此被判失败并重发。
*/
class SimilarAsinMainImageDownloadRetryTest {
private static final String URL = "https://m.media-amazon.com/images/I/61Yh2j3VZmL._AC_SX679_.jpg";
private static final String RESET = "Remote host terminated the handshake";
private static final byte[] BODY = new byte[]{1, 2, 3, 4, 5, 6, 7, 8};
@AfterEach
void clearInterruptFlag() {
// 中断用例会把标志位留在测试线程上,不清会污染后续测试。
Thread.interrupted();
}
private SimilarAsinLlmService service(SimilarAsinProperties props, HttpClient client) {
SimilarAsinLlmService svc = new SimilarAsinLlmService(
mock(SimilarAsinLlmClient.class),
props,
mock(OssProperties.class),
mock(ProductCategoryService.class),
mock(PuzzleImageMerger.class),
mock(OssStorageService.class),
mock(SimilarAsinImageEmbedder.class));
svc.setDownloadHttpClientForTest(client);
return svc;
}
private Object invokeDownload(SimilarAsinLlmService svc, String url) throws Throwable {
Method m = SimilarAsinLlmService.class.getDeclaredMethod("downloadBytes", String.class);
m.setAccessible(true);
try {
return m.invoke(svc, url);
} catch (InvocationTargetException ex) {
throw ex.getCause();
}
}
/** 前 {@code failTimes} 次连接级异常,之后返回 200。 */
private HttpClient flakyClient(int failTimes, AtomicInteger calls) {
HttpClient client = mock(HttpClient.class);
try {
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
.thenAnswer(invocation -> {
int n = calls.incrementAndGet();
if (n <= failTimes) {
throw new IOException(RESET);
}
return new StubResponse(200, BODY);
});
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
return client;
}
@Test
void 抖动两次后第三次成功_不再丢整行() throws Throwable {
AtomicInteger calls = new AtomicInteger();
SimilarAsinLlmService svc = service(new SimilarAsinProperties(), flakyClient(2, calls));
byte[] bytes = (byte[]) invokeDownload(svc, URL);
assertArrayEquals(BODY, bytes);
assertEquals(3, calls.get(), "默认额外重试 2 次 → 第三次应成功");
}
@Test
void 重试耗尽后抛出带真实原因的IOException() throws Throwable {
AtomicInteger calls = new AtomicInteger();
SimilarAsinLlmService svc = service(new SimilarAsinProperties(), flakyClient(99, calls));
IOException ex = assertThrows(IOException.class, () -> invokeDownload(svc, URL));
assertTrue(ex.getMessage().contains(RESET), "要把真实异常带出来,不能再塌成一句「主图转存失败」");
assertEquals(3, calls.get());
}
@Test
void 源站明确回非200时不重试() throws Throwable {
AtomicInteger calls = new AtomicInteger();
HttpClient client = mock(HttpClient.class);
try {
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
.thenAnswer(invocation -> {
calls.incrementAndGet();
return new StubResponse(403, new byte[0]);
});
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
assertNull(invokeDownload(service(new SimilarAsinProperties(), client), URL));
assertEquals(1, calls.get(), "403 是确定答复,重试只是白等超时");
}
@Test
void retryTimes置0退回单次尝试旧行为() throws Throwable {
SimilarAsinProperties props = new SimilarAsinProperties();
props.setLlmImageDownloadRetryTimes(0);
AtomicInteger calls = new AtomicInteger();
IOException ex = assertThrows(IOException.class,
() -> invokeDownload(service(props, flakyClient(99, calls)), URL));
assertTrue(ex.getMessage().contains(RESET));
assertEquals(1, calls.get());
}
@Test
void 线程被中断时立刻停止重试() throws Throwable {
AtomicInteger calls = new AtomicInteger();
HttpClient client = mock(HttpClient.class);
try {
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
.thenAnswer(invocation -> {
calls.incrementAndGet();
Thread.currentThread().interrupt();
throw new IOException(RESET);
});
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
assertThrows(IOException.class, () -> invokeDownload(service(new SimilarAsinProperties(), client), URL));
assertEquals(1, calls.get(), "中断后不该再排下一次重试");
assertTrue(Thread.currentThread().isInterrupted(), "中断标志必须回设,不能吞掉");
}
@Test
void 空URL仍按取不到处理不抛异常() throws Throwable {
HttpClient client = mock(HttpClient.class);
SimilarAsinLlmService svc = service(new SimilarAsinProperties(), client);
assertNull(invokeDownload(svc, " "));
assertNull(invokeDownload(svc, null));
verify(client, never()).send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class));
}
private static final class StubResponse implements HttpResponse<byte[]> {
private final int statusCode;
private final byte[] body;
private StubResponse(int statusCode, byte[] body) {
this.statusCode = statusCode;
this.body = body;
}
@Override
public int statusCode() {
return statusCode;
}
@Override
public HttpRequest request() {
return Mockito.mock(HttpRequest.class);
}
@Override
public java.util.Optional<HttpResponse<byte[]>> previousResponse() {
return java.util.Optional.empty();
}
@Override
public java.net.http.HttpHeaders headers() {
return java.net.http.HttpHeaders.of(java.util.Map.of(), (a, b) -> true);
}
@Override
public byte[] body() {
return body;
}
@Override
public java.net.URI uri() {
return java.net.URI.create(URL);
}
@Override
public HttpClient.Version version() {
return HttpClient.Version.HTTP_1_1;
}
@Override
public java.util.Optional<javax.net.ssl.SSLSession> sslSession() {
return java.util.Optional.empty();
}
}
}