feat(unreachable-cooldown): 连接层不可达客户端级冷却,四家判定下沉共用

BACKLOG B24(窗长 300s 已拍板):连续 2 次连接未建立即对该 host 进
冷却窗,窗内调用零探针快速失败(归因文案与调用内短路逐字一致);到期
后下一条真实调用兼作探针,失败重进、成功恢复。isConnectFailure 四处
副本(BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient/
TaskOwnerForwardService)下沉 common/retry 共用。三家客户端接入同一
冷却表(两个 LLM 客户端共用 ai.t8star.org 一格,16890 独立一格),
窗长可配(0=关闭退回调用内短路)。另修 CollectDataBrandBatchFilter
首批预算竞态(1ms 预算下 GC 停顿误切首批,照 BrandCheckClient 首轮
不掐契约补 start>0 守卫)。变异验证:冷却禁用 6 例判红;定向 47/47 绿。
This commit is contained in:
2026-10-23 20:58:32 +08:00
parent 6af19b51ba
commit 09dfb01f9c
22 changed files with 426 additions and 99 deletions
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
import com.nanri.aiimage.common.retry.UnreachableCooldown;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
@@ -17,9 +18,6 @@ import org.springframework.web.client.RestClient;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
@@ -52,6 +50,7 @@ public class SimilarAsinLlmClient {
private final ObjectMapper objectMapper;
private final ExternalCallMetricsRecorder externalCallMetrics;
private final UserSecretUsageService userSecretUsageService;
private final UnreachableCooldown unreachableCooldown;
private volatile RestClient sharedRestClient;
/** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */
@@ -60,11 +59,13 @@ public class SimilarAsinLlmClient {
public SimilarAsinLlmClient(SimilarAsinProperties properties,
ObjectMapper objectMapper,
ExternalCallMetricsRecorder externalCallMetrics,
UserSecretUsageService userSecretUsageService) {
UserSecretUsageService userSecretUsageService,
UnreachableCooldown unreachableCooldown) {
this.properties = properties;
this.objectMapper = objectMapper;
this.externalCallMetrics = externalCallMetrics;
this.userSecretUsageService = userSecretUsageService;
this.unreachableCooldown = unreachableCooldown;
}
/** 文本对话,json_object 输出。 */
@@ -85,16 +86,32 @@ public class SimilarAsinLlmClient {
String responseFormat) {
String resolvedKey = resolveApiKey(apiKey);
int attempts = Math.max(1, properties.getLlmRetryTimes());
long cooldownMillis = Math.max(0, properties.getLlmUnreachableCooldownMillis());
String host = properties.getLlmHost();
if (unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
// 冷却期快失败不发请求、不记密钥次数:结论与短路版一致(调用方拿不到 LLM 结论),
// 差别只是不再为"确认还是断的"各烧一轮连接探针(B24)
log.debug("[similar-asin][llm] 客户端级冷却中,快速失败 model={} window={}ms", model, cooldownMillis);
throw new IllegalStateException(REASON_LLM_UNREACHABLE + ";lastErr=冷却中(连续连接未建立,暂不打上游)");
}
Exception lastFailure = null;
int connectFailures = 0;
for (int attempt = 1; attempt <= attempts; attempt++) {
try {
return invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
String content = invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
unreachableCooldown.recordConnected(host);
return content;
} catch (Exception ex) {
lastFailure = ex;
if (isConnectFailure(ex)) {
if (UnreachableCooldown.isConnectFailure(ex)) {
connectFailures++;
if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE) {
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
if (entered) {
log.warn("[similar-asin][llm] 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}",
2, cooldownMillis, host);
}
if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE
|| unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
log.warn("[similar-asin][llm] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达,"
+ "停止重试 attempt={} model={} err={}",
LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex));
@@ -106,6 +123,7 @@ public class SimilarAsinLlmClient {
} else {
// 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数
connectFailures = 0;
unreachableCooldown.recordConnected(host);
}
if (attempt >= attempts) {
break;
@@ -123,20 +141,6 @@ public class SimilarAsinLlmClient {
: new IllegalStateException(lastFailure.getMessage(), lastFailure);
}
/**
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
*/
static boolean isConnectFailure(Throwable error) {
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|| t instanceof UnknownHostException) {
return true;
}
}
return false;
}
/** 只认任务/用户自己的 api_key(payload → 按 uid 的服务端密钥表已在 readApiKey 兜过);
* 为空即空、跳过 LLM,绝不回落平台公费密钥 —— 各用户的任务只能烧各自的密钥。 */
public String resolveApiKey(String apiKey) {