fix(货源): 主图重试换备用域名绕开被钉死的坏 CDN 节点
根因不是"瞬时抖动":m.media-amazon.com 的多 CDN 池里 Akamai 系边缘没配这个 虚拟主机名,带该 SNI 一律 RST(同 IP 不带 SNI 或用 images-na.ssl-images-amazon.com 则正常),而 JVM 正向 DNS 缓存会把坏节点钉住一整批 → 秒级重试三次全打在它身上。 实测最近 20 条失败 URL 走备用域名 20/20 成功且 md5 与原域名逐字节相同。
This commit is contained in:
@@ -192,10 +192,22 @@ public class SimilarAsinProperties {
|
||||
* 依据 2026-09-28 生产取证:主机A 近 3 小时 252 条主图下载失败全部是
|
||||
* {@code m.media-amazon.com} 的 "Remote host terminated the handshake"(瞬时握手被 RST,
|
||||
* 事后 curl 8 条里 7 条直接 200),而旧实现一次都不重试、真实异常还被丢掉。
|
||||
* 发版后实测(同文档 §6.1):<b>恢复窗口是分钟级</b>,当前秒级 backoff 只救回 3/18=17% 段抖动,
|
||||
* 行失败率 14.0% vs 旧代码 14.6% 基本没动 —— 确定收益是行失败日志自带真实原因、
|
||||
* 以及非 200 与连接级失败被分开计数。抬到秒级以上前先测恢复窗口分布(会占住并发仅 5 的行 worker)。
|
||||
* 发版后实测(同文档 §6.1):<b>秒级 backoff 只救回 3/18=17%</b>,行失败率 14.0% vs 旧代码 14.6% 基本没动,
|
||||
* 确定收益是行失败日志自带真实原因、以及非 200 与连接级失败被分开计数。
|
||||
* 原因见下一条 §6.2:三次重试打在<b>同一个被钉住的坏 CDN 节点</b>上,所以等再久也不如换域名。
|
||||
* 设 0 可退回旧行为。
|
||||
*/
|
||||
private int llmImageDownloadRetryTimes = 2;
|
||||
|
||||
/**
|
||||
* 主图下载的<b>备用域名</b>映射,格式 {@code from=to},多组用 {@code ;} 分隔;置空即完全关闭。
|
||||
* 仅在连接级失败后的重试里生效(首试仍用原 URL)。
|
||||
* 依据 2026-09-28 主机A 取证:{@code m.media-amazon.com} 由多 CDN 池解析,
|
||||
* 其中 Akamai 系(184.27.207.231/245)与 Fastly(199.232.113.16)对
|
||||
* SNI=m.media-amazon.com <b>一律 RST</b>(同一 IP 不带 SNI 则 TLSv1.3 握手成功),
|
||||
* 而 JVM 正向 DNS 缓存会把选中的节点钉住一段时间(JDK 默认 30 秒,此处未实测)→ 整批 10 行一起挂,
|
||||
* 这正是上一条秒级 backoff 只救回 17% 的原因(三次都打在同一个坏节点上)。
|
||||
* 实测最近 20 条失败 URL 换 {@code images-na.ssl-images-amazon.com}:<b>20/20 成功且 md5 逐字节相同</b>。
|
||||
*/
|
||||
private String llmImageHostAliasMap = "m.media-amazon.com=images-na.ssl-images-amazon.com";
|
||||
}
|
||||
|
||||
+53
-5
@@ -549,7 +549,8 @@ public class SimilarAsinLlmService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 主图下载:连接级异常按 {@code aiimage.similar-asin.llm-image-download-retry-times} 重试(默认额外 2 次)。
|
||||
* 主图下载:连接级异常按 {@code aiimage.similar-asin.llm-image-download-retry-times} 重试(默认额外 2 次),
|
||||
* 且第 2 次起换到 {@code aiimage.similar-asin.llm-image-host-alias-map} 里的备用域名。
|
||||
* 只重试连接级失败——非 200 是源站的确定答复,重试只是白等超时。
|
||||
* 总耗时受 {@link #MAIN_DOWNLOAD_BUDGET_MILLIS} 约束:慢源 hang 住时不放大尾延迟
|
||||
* (同模块的 alicdn 拼图已被这条咬过,见 SimilarAsinProperties#imageDownloadTimeoutSeconds 注释)。
|
||||
@@ -564,12 +565,15 @@ public class SimilarAsinLlmService {
|
||||
long deadline = System.currentTimeMillis() + MAIN_DOWNLOAD_BUDGET_MILLIS;
|
||||
IOException last = null;
|
||||
for (int attempt = 0; attempt <= retryTimes; attempt++) {
|
||||
// 首试保持原域名;连接级失败后换备用域名 —— 坏的是解析到的那个 CDN 节点,
|
||||
// 同一个名字重试三次还是它,换个名字才等于换一批节点。
|
||||
String target = attempt == 0 ? trimmed : defaultToAlias(aliasedUrl(trimmed), trimmed);
|
||||
if (attempt > 0 && (System.currentTimeMillis() > deadline || !sleepBeforeRetry(attempt))) {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(trimmed))
|
||||
.uri(URI.create(target))
|
||||
.timeout(Duration.ofSeconds(timeoutSeconds))
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
|
||||
// 刻意不发 Referer:阿里图床按 Referer 白名单防盗链,外链域名会被判 403
|
||||
@@ -580,11 +584,11 @@ public class SimilarAsinLlmService {
|
||||
if (response.statusCode() == 200 && response.body() != null && response.body().length > 0) {
|
||||
if (attempt > 0) {
|
||||
log.info("[similar-asin][llm] main download retry-success url={} attempt={}/{}",
|
||||
abbreviate(trimmed, 160), attempt + 1, retryTimes + 1);
|
||||
abbreviate(target, 160), attempt + 1, retryTimes + 1);
|
||||
}
|
||||
return response.body();
|
||||
}
|
||||
log.warn("[similar-asin][llm] main download http {} url={}", response.statusCode(), abbreviate(trimmed, 160));
|
||||
log.warn("[similar-asin][llm] main download http {} url={}", response.statusCode(), abbreviate(target, 160));
|
||||
return null;
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
@@ -592,7 +596,7 @@ public class SimilarAsinLlmService {
|
||||
} catch (Exception ex) {
|
||||
last = new IOException("主图下载失败(" + failureMessage(ex) + ")", ex);
|
||||
log.warn("[similar-asin][llm] main download fail url={} attempt={}/{} err={}",
|
||||
abbreviate(trimmed, 160), attempt + 1, retryTimes + 1, failureMessage(ex));
|
||||
abbreviate(target, 160), attempt + 1, retryTimes + 1, failureMessage(ex));
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
break;
|
||||
}
|
||||
@@ -601,6 +605,50 @@ public class SimilarAsinLlmService {
|
||||
throw last == null ? new IOException("主图下载失败: " + abbreviate(trimmed, 160)) : last;
|
||||
}
|
||||
|
||||
private static String defaultToAlias(String alias, String fallback) {
|
||||
return alias == null ? fallback : alias;
|
||||
}
|
||||
|
||||
/** 按 {@code aiimage.similar-asin.llm-image-host-alias-map} 换 host,其余部分逐字符保留。 */
|
||||
private String aliasedUrl(String url) {
|
||||
String map = properties.getLlmImageHostAliasMap();
|
||||
if (map == null || map.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
URI uri;
|
||||
try {
|
||||
uri = URI.create(url);
|
||||
} catch (Exception ex) {
|
||||
return null;
|
||||
}
|
||||
String host = uri.getHost();
|
||||
if (host == null || host.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String text = uri.toString();
|
||||
int at = text.indexOf("//" + host);
|
||||
if (at < 0) {
|
||||
return null;
|
||||
}
|
||||
int end = at + 2 + host.length();
|
||||
char after = end < text.length() ? text.charAt(end) : '/';
|
||||
if (after != ':' && after != '/' && after != '?' && after != '#') {
|
||||
return null;
|
||||
}
|
||||
for (String pair : map.split(";")) {
|
||||
int eq = pair.indexOf('=');
|
||||
if (eq <= 0 || eq >= pair.length() - 1) {
|
||||
continue;
|
||||
}
|
||||
String from = pair.substring(0, eq).trim();
|
||||
String to = pair.substring(eq + 1).trim();
|
||||
if (from.equalsIgnoreCase(host) && !to.isEmpty()) {
|
||||
return text.substring(0, at + 2) + to + text.substring(end);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean sleepBeforeRetry(int attempt) {
|
||||
try {
|
||||
Thread.sleep(Math.min(1000L, 250L * attempt));
|
||||
|
||||
@@ -378,6 +378,7 @@ aiimage:
|
||||
llm-row-concurrency: ${AIIMAGE_SIMILAR_ASIN_LLM_ROW_CONCURRENCY:5}
|
||||
llm-image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:10}
|
||||
llm-image-download-retry-times: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_DOWNLOAD_RETRY_TIMES:2}
|
||||
llm-image-host-alias-map: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_HOST_ALIAS_MAP:m.media-amazon.com=images-na.ssl-images-amazon.com}
|
||||
collect-data:
|
||||
stale-timeout-minutes: ${AIIMAGE_COLLECT_DATA_STALE_TIMEOUT_MINUTES:30}
|
||||
no-result-upload-timeout-minutes: ${AIIMAGE_COLLECT_DATA_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180}
|
||||
|
||||
+65
@@ -174,6 +174,71 @@ class SimilarAsinMainImageDownloadRetryTest {
|
||||
verify(client, never()).send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 按调用顺序记录请求 URI;前 {@code failTimes} 次抛连接级异常,之后返回 200。
|
||||
*/
|
||||
private HttpClient recordingClient(int failTimes, List<String> uris) {
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
try {
|
||||
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
uris.add(invocation.getArgument(0, HttpRequest.class).uri().toString());
|
||||
if (uris.size() <= failTimes) {
|
||||
throw new IOException(RESET);
|
||||
}
|
||||
return new StubResponse(200, BODY);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 连接级失败后改用备用域名重试() throws Throwable {
|
||||
List<String> uris = new java.util.ArrayList<>();
|
||||
SimilarAsinLlmService svc = service(new SimilarAsinProperties(), recordingClient(1, uris));
|
||||
|
||||
assertArrayEquals(BODY, (byte[]) invokeDownload(svc, URL));
|
||||
|
||||
assertEquals(2, uris.size());
|
||||
assertTrue(uris.get(0).contains("m.media-amazon.com"), "首试必须用原域名,不改变正常路径行为: " + uris.get(0));
|
||||
assertTrue(uris.get(1).contains("images-na.ssl-images-amazon.com"), "第二次必须换到备用域名: " + uris.get(1));
|
||||
assertTrue(uris.get(1).endsWith("/images/I/61Yh2j3VZmL._AC_SX679_.jpg"), "只换 host,路径与尺寸后缀必须原样: " + uris.get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 备用域名映射置空时不换域名() throws Throwable {
|
||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
||||
props.setLlmImageHostAliasMap("");
|
||||
List<String> uris = new java.util.ArrayList<>();
|
||||
|
||||
assertArrayEquals(BODY, (byte[]) invokeDownload(service(props, recordingClient(2, uris)), URL));
|
||||
|
||||
assertEquals(3, uris.size());
|
||||
assertTrue(uris.stream().allMatch(u -> u.equals(URL)), "置空即完全退回旧行为: " + uris);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 映射外的域名不受影响() throws Throwable {
|
||||
List<String> uris = new java.util.ArrayList<>();
|
||||
String alicdn = "https://cbu01.alicdn.com/img/ibank/x.jpg";
|
||||
|
||||
assertArrayEquals(BODY, (byte[]) invokeDownload(service(new SimilarAsinProperties(), recordingClient(1, uris)), alicdn));
|
||||
|
||||
assertEquals(List.of(alicdn, alicdn), uris, "阿里图床没有备用域名,不该被动到");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 带端口与query的URL换域名后其余部分不变() throws Throwable {
|
||||
List<String> uris = new java.util.ArrayList<>();
|
||||
String url = "https://m.media-amazon.com:443/images/I/x.jpg?foo=bar%20baz";
|
||||
|
||||
assertArrayEquals(BODY, (byte[]) invokeDownload(service(new SimilarAsinProperties(), recordingClient(1, uris)), url));
|
||||
|
||||
assertEquals("https://images-na.ssl-images-amazon.com:443/images/I/x.jpg?foo=bar%20baz", uris.get(1));
|
||||
}
|
||||
|
||||
private static final class StubResponse implements HttpResponse<byte[]> {
|
||||
private final int statusCode;
|
||||
private final byte[] body;
|
||||
|
||||
Reference in New Issue
Block a user