feat(unreachable-cooldown): 连接层不可达客户端级冷却,四家判定下沉共用
BACKLOG B24(窗长 300s 已拍板):连续 2 次连接未建立即对该 host 进 冷却窗,窗内调用零探针快速失败(归因文案与调用内短路逐字一致);到期 后下一条真实调用兼作探针,失败重进、成功恢复。isConnectFailure 四处 副本(BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient/ TaskOwnerForwardService)下沉 common/retry 共用。三家客户端接入同一 冷却表(两个 LLM 客户端共用 ai.t8star.org 一格,16890 独立一格), 窗长可配(0=关闭退回调用内短路)。另修 CollectDataBrandBatchFilter 首批预算竞态(1ms 预算下 GC 停顿误切首批,照 BrandCheckClient 首轮 不掐契约补 start>0 守卫)。变异验证:冷却禁用 6 例判红;定向 47/47 绿。
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
package com.nanri.aiimage.common.retry;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* 连接层不可达的客户端级冷却表(按上游 host 分键,Spring 单例)。
|
||||
*
|
||||
* <p>背景(BACKLOG B24,2026-10-23 拍板):三家同型客户端({@code BrandCheckClient}/
|
||||
* {@code AppearancePatentLlmClient}/{@code SimilarAsinLlmClient})的连接层短路计数
|
||||
* 生命周期都只到单次调用,网关长断时每个新调用/分块仍要各自真发连接探针才知道"还是断的"。
|
||||
* 本表把「连续 2 次连接未建立」的证据跨调用累计,达到阈值即对该 host 进入冷却窗
|
||||
* (默认 300s,同仓两处先例:{@code pt_worker/proxy_pool.py} 与 16890 服务侧的钥匙冷却),
|
||||
* 窗内调用方不发请求直接快速失败;到期后下一条真实调用兼作探针,失败即重进、成功即恢复。
|
||||
*
|
||||
* <p>调用方契约:{@link #isCoolingDown} 在发请求前问一次;连接层失败记
|
||||
* {@link #recordConnectFailure}(返回 true 的那方打一条进入冷却的 warn);
|
||||
* 连接成功或"连上了没答好"(读超时/5xx/解析失败)记 {@link #recordConnected} 清零证据。
|
||||
* 窗长 {@code <= 0} 表示关闭冷却,全部方法退化为无操作——调用方退回各自的调用内短路行为。
|
||||
*/
|
||||
@Component
|
||||
public class UnreachableCooldown {
|
||||
|
||||
/**
|
||||
* 进入冷却的证据数:与三家客户端各自的调用内短路阈值同数字——一次 refused 可能是抖动,
|
||||
* 两次同形状且中间没有任何一次成功,才是服务在掉。
|
||||
*/
|
||||
private static final int EVIDENCE_THRESHOLD = 2;
|
||||
|
||||
private final ConcurrentHashMap<String, State> states = new ConcurrentHashMap<>();
|
||||
|
||||
private static final class State {
|
||||
final AtomicInteger evidence = new AtomicInteger();
|
||||
final AtomicLong cooldownUntilMillis = new AtomicLong();
|
||||
}
|
||||
|
||||
/**
|
||||
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
|
||||
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
|
||||
* 原 BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient 各有一份同款,
|
||||
* B24 下沉至此,四家(含 TaskOwnerForwardService)共用同一判定。
|
||||
*/
|
||||
public static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 该 host 是否正处冷却窗内:true 时调用方不发请求,直接按既有不可达归因文案快速失败。 */
|
||||
public boolean isCoolingDown(String host, long windowMillis) {
|
||||
if (windowMillis <= 0) {
|
||||
return false;
|
||||
}
|
||||
State state = states.get(normalizeHost(host));
|
||||
return state != null && state.cooldownUntilMillis.get() > System.currentTimeMillis();
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接层失败证据 +1;累计到阈值即对该 host 开(或重开)冷却窗。
|
||||
*
|
||||
* @return true 仅当"本次调用让冷却窗从无到有"(并发下只会有一个调用方拿到 true),
|
||||
* 调用方据此打一条进入冷却的 warn,期中快失败不刷屏
|
||||
*/
|
||||
public boolean recordConnectFailure(String host, long windowMillis) {
|
||||
if (windowMillis <= 0) {
|
||||
return false;
|
||||
}
|
||||
State state = states.computeIfAbsent(normalizeHost(host), key -> new State());
|
||||
if (state.evidence.incrementAndGet() < EVIDENCE_THRESHOLD) {
|
||||
return false;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
long previous = state.cooldownUntilMillis.get();
|
||||
if (previous > now) {
|
||||
return false;
|
||||
}
|
||||
return state.cooldownUntilMillis.compareAndSet(previous, now + windowMillis);
|
||||
}
|
||||
|
||||
/** 连接成功或"连上了没答好":证据清零(已开的冷却窗不受影响,等它自然到期)。 */
|
||||
public void recordConnected(String host) {
|
||||
State state = states.get(normalizeHost(host));
|
||||
if (state != null) {
|
||||
state.evidence.set(0);
|
||||
}
|
||||
}
|
||||
|
||||
/** host 归一:两个 LLM 客户端配的同一网关必须落进同一格,别因尾斜杠/大小写裂成两份。 */
|
||||
static String normalizeHost(String host) {
|
||||
String normalized = host == null ? "" : host.trim().toLowerCase();
|
||||
return normalized.endsWith("/") ? normalized.substring(0, normalized.length() - 1) : normalized;
|
||||
}
|
||||
}
|
||||
+1
-13
@@ -19,7 +19,6 @@ import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.util.ContentCachingRequestWrapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.ConnectException;
|
||||
import java.util.Enumeration;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
@@ -89,7 +88,7 @@ public class TaskOwnerForwardService {
|
||||
.retrieve()
|
||||
.toEntity(byte[].class);
|
||||
} catch (ResourceAccessException accessError) {
|
||||
if (!isConnectFailure(accessError)) {
|
||||
if (!com.nanri.aiimage.common.retry.UnreachableCooldown.isConnectFailure(accessError)) {
|
||||
throw accessError;
|
||||
}
|
||||
lastError = accessError;
|
||||
@@ -103,17 +102,6 @@ public class TaskOwnerForwardService {
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
private static boolean isConnectFailure(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
if (cursor instanceof ConnectException) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void sleepQuietly(long millis) {
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
|
||||
Reference in New Issue
Block a user