feat(unreachable-cooldown): 连接层不可达客户端级冷却,四家判定下沉共用
BACKLOG B24(窗长 300s 已拍板):连续 2 次连接未建立即对该 host 进 冷却窗,窗内调用零探针快速失败(归因文案与调用内短路逐字一致);到期 后下一条真实调用兼作探针,失败重进、成功恢复。isConnectFailure 四处 副本(BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient/ TaskOwnerForwardService)下沉 common/retry 共用。三家客户端接入同一 冷却表(两个 LLM 客户端共用 ai.t8star.org 一格,16890 独立一格), 窗长可配(0=关闭退回调用内短路)。另修 CollectDataBrandBatchFilter 首批预算竞态(1ms 预算下 GC 停顿误切首批,照 BrandCheckClient 首轮 不掐契约补 start>0 守卫)。变异验证:冷却禁用 6 例判红;定向 47/47 绿。
This commit is contained in:
+24
-15
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
|
||||
@@ -21,9 +22,6 @@ import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
@@ -158,6 +156,7 @@ public class AppearancePatentLlmClient {
|
||||
private final ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final BrandCheckClient brandCheckClient;
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
private final UnreachableCooldown unreachableCooldown;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
/** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */
|
||||
@@ -647,15 +646,30 @@ public class AppearancePatentLlmClient {
|
||||
String apiKey,
|
||||
String responseFormat) throws Exception {
|
||||
int attempts = Math.max(1, properties.getLlmRetryTimes());
|
||||
long cooldownMillis = Math.max(0, properties.getLlmUnreachableCooldownMillis());
|
||||
String host = properties.getLlmHost();
|
||||
if (unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
// 冷却期快失败不发请求、不记密钥次数(B24)
|
||||
log.debug("[appearance-patent] 客户端级冷却中,快速失败 model={} window={}ms", model, cooldownMillis);
|
||||
throw new IllegalStateException(REASON_LLM_UNREACHABLE + ";lastErr=冷却中(连续连接未建立,暂不打上游)");
|
||||
}
|
||||
Exception lastFailure = null;
|
||||
AtomicInteger connectFailures = new AtomicInteger();
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
return invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt);
|
||||
String content = invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
return content;
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
if (isConnectFailure(ex)) {
|
||||
if (connectFailures.incrementAndGet() >= LLM_CONNECT_FAILURE_EVIDENCE) {
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
|
||||
if (entered) {
|
||||
log.warn("[appearance-patent] 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}",
|
||||
LLM_CONNECT_FAILURE_EVIDENCE, cooldownMillis, host);
|
||||
}
|
||||
if (connectFailures.incrementAndGet() >= LLM_CONNECT_FAILURE_EVIDENCE
|
||||
|| unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
log.warn("[appearance-patent] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达,"
|
||||
+ "停止重试 attempt={} model={} err={}",
|
||||
LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex));
|
||||
@@ -667,6 +681,7 @@ public class AppearancePatentLlmClient {
|
||||
} else {
|
||||
// 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数
|
||||
connectFailures.set(0);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
}
|
||||
if (attempt >= attempts) {
|
||||
break;
|
||||
@@ -681,17 +696,11 @@ public class AppearancePatentLlmClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
|
||||
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
|
||||
* 连接层判定下沉 {@code UnreachableCooldown.isConnectFailure}(B24):判定口径与
|
||||
* BrandCheckClient/SimilarAsinLlmClient 共用一份,不再各留包内副本。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
return UnreachableCooldown.isConnectFailure(error);
|
||||
}
|
||||
|
||||
private String invokeChatOnce(String model,
|
||||
|
||||
+33
-8
@@ -77,16 +77,19 @@ public class BrandCheckClient {
|
||||
private final BrandCheckProperties properties;
|
||||
private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserApiSecretService userApiSecretService;
|
||||
private final com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown;
|
||||
/** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */
|
||||
private final ThreadLocal<AtomicBoolean> rejectedSubmission = new ThreadLocal<>();
|
||||
private volatile RestClient sharedRestClient;
|
||||
|
||||
public BrandCheckClient(BrandCheckProperties properties,
|
||||
com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics,
|
||||
UserApiSecretService userApiSecretService) {
|
||||
UserApiSecretService userApiSecretService,
|
||||
com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown) {
|
||||
this.properties = properties;
|
||||
this.externalCallMetrics = externalCallMetrics;
|
||||
this.userApiSecretService = userApiSecretService;
|
||||
this.unreachableCooldown = unreachableCooldown;
|
||||
}
|
||||
|
||||
// 有界队列 + 拒绝即降级:大批量(几千品牌)检查时队列洪峰不无界堆内存。
|
||||
@@ -267,6 +270,18 @@ public class BrandCheckClient {
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
long cooldownMillis = Math.max(0, properties.getUnreachableCooldownMillis());
|
||||
if (unreachableCooldown.isCoolingDown(properties.getBaseUrl(), cooldownMillis)) {
|
||||
// 客户端级冷却(B24):上一批已确认连接层不可达,本批连首轮探针都不发。
|
||||
// 结论与批内短路版逐字一致(每行仍判查询失败、reason 同文案),差别只是不再为
|
||||
// "确认还是断的"每批各烧一轮连接探针;到期后下一批的真实请求兼作探针。
|
||||
log.warn("[brand-check] SERVICE_UNREACHABLE 客户端级冷却中,整批不打上游 brands={} window={}ms",
|
||||
brands.size(), cooldownMillis);
|
||||
for (String brand : brands) {
|
||||
outcomes.put(brand, new BrandCheckOutcome(List.of(), List.of(brand), REASON_SERVICE_UNREACHABLE));
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
long firstAttemptDeadlineNanos = deadlineNanos == Long.MAX_VALUE
|
||||
? Long.MAX_VALUE
|
||||
: deadlineNanos + FIRST_ATTEMPT_QUEUE_GRACE_MILLIS * 1_000_000L;
|
||||
@@ -363,6 +378,7 @@ public class BrandCheckClient {
|
||||
lastFailure = ex;
|
||||
response = null;
|
||||
if (isConnectFailure(ex)) {
|
||||
recordConnectFailure(ex);
|
||||
if (connectFailures.incrementAndGet() >= CONNECT_FAILURE_EVIDENCE) {
|
||||
ladderCompleted = false;
|
||||
connectDown = true;
|
||||
@@ -374,6 +390,7 @@ public class BrandCheckClient {
|
||||
} else {
|
||||
// 连上了只是这次没答好(读超时/解析失败):连接层证据不作数
|
||||
connectFailures.set(0);
|
||||
unreachableCooldown.recordConnected(properties.getBaseUrl());
|
||||
}
|
||||
if (attempt < attempts) {
|
||||
long retryDelayMillis = retryDelayMillis(attempt);
|
||||
@@ -385,6 +402,7 @@ public class BrandCheckClient {
|
||||
}
|
||||
// 拿到响应=连接是通的,证据清零,别跨到下一次真断连时凑数
|
||||
connectFailures.set(0);
|
||||
unreachableCooldown.recordConnected(properties.getBaseUrl());
|
||||
if (response != null && nullToEmpty(response.getQueryFaildData()).isEmpty()) {
|
||||
// 本次查询完成且无失败项:命中侵权(faild_data)属有效结论,不重试。
|
||||
return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of(), "");
|
||||
@@ -554,14 +572,21 @@ public class BrandCheckClient {
|
||||
* <p>故意不含读超时与 5xx:那两种是"连上了但这次没答好",重试救得回来;把它们算进
|
||||
* 不可达证据,就会把一次上游抖动误判成服务在掉、把同批其余品牌的重试一起掐掉。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
/** 连接层失败记入客户端级冷却表:达到阈值的那方打一条进入冷却的 warn(B24)。 */
|
||||
private void recordConnectFailure(Exception ex) {
|
||||
long cooldownMillis = Math.max(0, properties.getUnreachableCooldownMillis());
|
||||
if (unreachableCooldown.recordConnectFailure(properties.getBaseUrl(), cooldownMillis)) {
|
||||
log.warn("[brand-check] SERVICE_UNREACHABLE 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}",
|
||||
CONNECT_FAILURE_EVIDENCE, cooldownMillis, properties.getBaseUrl());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接层判定下沉 {@code UnreachableCooldown.isConnectFailure}(B24):判定口径与
|
||||
* 两个 LLM 客户端共用一份,不再各留包内副本。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
return com.nanri.aiimage.common.retry.UnreachableCooldown.isConnectFailure(error);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+4
-1
@@ -118,7 +118,10 @@ public class CollectDataBrandBatchFilter {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (deadlineNanos != Long.MAX_VALUE && System.nanoTime() >= deadlineNanos) {
|
||||
// 首批不掐(与 BrandCheckClient 的 attempt>1 同契约):预算检查只拦后续批次。
|
||||
// 对首批也查的话,deadline 计算与检查之间一次 GC/调度停顿(毫秒级)就会把首批
|
||||
// 整组误切成 queryFailed——1ms 预算的测试已实测抖出过这个形状。
|
||||
if (start > 0 && deadlineNanos != Long.MAX_VALUE && System.nanoTime() >= deadlineNanos) {
|
||||
// 预算用尽:不再为后面的批次捶上游。已缓存的品牌仍按缓存结论归类(不花代价),
|
||||
// 没结论的行归 queryFailed 入补测队列——既不按 accepted 放行(可能正是侵权品牌),
|
||||
// 也不按 rejected 拉黑(没查过就不该写进无效品牌表,否则永久误杀)。
|
||||
|
||||
+25
-21
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.similarasin.client;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
|
||||
@@ -17,9 +18,6 @@ import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
@@ -52,6 +50,7 @@ public class SimilarAsinLlmClient {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
private final UnreachableCooldown unreachableCooldown;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
/** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */
|
||||
@@ -60,11 +59,13 @@ public class SimilarAsinLlmClient {
|
||||
public SimilarAsinLlmClient(SimilarAsinProperties properties,
|
||||
ObjectMapper objectMapper,
|
||||
ExternalCallMetricsRecorder externalCallMetrics,
|
||||
UserSecretUsageService userSecretUsageService) {
|
||||
UserSecretUsageService userSecretUsageService,
|
||||
UnreachableCooldown unreachableCooldown) {
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
this.externalCallMetrics = externalCallMetrics;
|
||||
this.userSecretUsageService = userSecretUsageService;
|
||||
this.unreachableCooldown = unreachableCooldown;
|
||||
}
|
||||
|
||||
/** 文本对话,json_object 输出。 */
|
||||
@@ -85,16 +86,32 @@ public class SimilarAsinLlmClient {
|
||||
String responseFormat) {
|
||||
String resolvedKey = resolveApiKey(apiKey);
|
||||
int attempts = Math.max(1, properties.getLlmRetryTimes());
|
||||
long cooldownMillis = Math.max(0, properties.getLlmUnreachableCooldownMillis());
|
||||
String host = properties.getLlmHost();
|
||||
if (unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
// 冷却期快失败不发请求、不记密钥次数:结论与短路版一致(调用方拿不到 LLM 结论),
|
||||
// 差别只是不再为"确认还是断的"各烧一轮连接探针(B24)
|
||||
log.debug("[similar-asin][llm] 客户端级冷却中,快速失败 model={} window={}ms", model, cooldownMillis);
|
||||
throw new IllegalStateException(REASON_LLM_UNREACHABLE + ";lastErr=冷却中(连续连接未建立,暂不打上游)");
|
||||
}
|
||||
Exception lastFailure = null;
|
||||
int connectFailures = 0;
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
return invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
|
||||
String content = invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
return content;
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
if (isConnectFailure(ex)) {
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
connectFailures++;
|
||||
if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE) {
|
||||
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
|
||||
if (entered) {
|
||||
log.warn("[similar-asin][llm] 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}",
|
||||
2, cooldownMillis, host);
|
||||
}
|
||||
if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE
|
||||
|| unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
log.warn("[similar-asin][llm] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达,"
|
||||
+ "停止重试 attempt={} model={} err={}",
|
||||
LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex));
|
||||
@@ -106,6 +123,7 @@ public class SimilarAsinLlmClient {
|
||||
} else {
|
||||
// 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数
|
||||
connectFailures = 0;
|
||||
unreachableCooldown.recordConnected(host);
|
||||
}
|
||||
if (attempt >= attempts) {
|
||||
break;
|
||||
@@ -123,20 +141,6 @@ public class SimilarAsinLlmClient {
|
||||
: new IllegalStateException(lastFailure.getMessage(), lastFailure);
|
||||
}
|
||||
|
||||
/**
|
||||
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
|
||||
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 只认任务/用户自己的 api_key(payload → 按 uid 的服务端密钥表已在 readApiKey 兜过);
|
||||
* 为空即空、跳过 LLM,绝不回落平台公费密钥 —— 各用户的任务只能烧各自的密钥。 */
|
||||
public String resolveApiKey(String apiKey) {
|
||||
|
||||
Reference in New Issue
Block a user