From 09dfb01f9c593f5e73255175414dd813f7d7f577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Fri, 23 Oct 2026 20:58:32 +0800 Subject: [PATCH] =?UTF-8?q?feat(unreachable-cooldown):=20=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E5=B1=82=E4=B8=8D=E5=8F=AF=E8=BE=BE=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=E7=BA=A7=E5=86=B7=E5=8D=B4=EF=BC=8C=E5=9B=9B=E5=AE=B6?= =?UTF-8?q?=E5=88=A4=E5=AE=9A=E4=B8=8B=E6=B2=89=E5=85=B1=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BACKLOG B24(窗长 300s 已拍板):连续 2 次连接未建立即对该 host 进 冷却窗,窗内调用零探针快速失败(归因文案与调用内短路逐字一致);到期 后下一条真实调用兼作探针,失败重进、成功恢复。isConnectFailure 四处 副本(BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient/ TaskOwnerForwardService)下沉 common/retry 共用。三家客户端接入同一 冷却表(两个 LLM 客户端共用 ai.t8star.org 一格,16890 独立一格), 窗长可配(0=关闭退回调用内短路)。另修 CollectDataBrandBatchFilter 首批预算竞态(1ms 预算下 GC 停顿误切首批,照 BrandCheckClient 首轮 不掐契约补 start>0 守卫)。变异验证:冷却禁用 6 例判红;定向 47/47 绿。 --- .../common/retry/UnreachableCooldown.java | 103 ++++++++++++++++++ .../service/TaskOwnerForwardService.java | 14 +-- .../config/AppearancePatentProperties.java | 7 ++ .../aiimage/config/BrandCheckProperties.java | 7 ++ .../aiimage/config/SimilarAsinProperties.java | 7 ++ .../client/AppearancePatentLlmClient.java | 39 ++++--- .../brand/client/BrandCheckClient.java | 41 +++++-- .../util/CollectDataBrandBatchFilter.java | 5 +- .../client/SimilarAsinLlmClient.java | 46 ++++---- .../src/main/resources/application.yml | 9 ++ .../common/retry/UnreachableCooldownTest.java | 77 +++++++++++++ .../config/HttpClientConnectionReuseTest.java | 15 +-- .../ExternalCallMetricsRecorderTest.java | 19 ++-- .../AppearancePatentBrandBatchTest.java | 5 +- .../AppearancePatentLlmClientHttpTest.java | 7 +- ...earancePatentLlmClientImageDedupeTest.java | 5 +- .../client/AppearancePatentLlmClientTest.java | 15 +-- ...pearancePatentLlmClientTokenUsageTest.java | 5 +- .../brand/client/BrandCheckClientTest.java | 43 +++++++- .../BrandCheckClientTitleBatchTest.java | 3 +- .../client/SimilarAsinLlmClientHttpTest.java | 50 ++++++++- .../service/SimilarAsinLlmLocalVerify.java | 3 +- 22 files changed, 426 insertions(+), 99 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/common/retry/UnreachableCooldown.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/common/retry/UnreachableCooldownTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/retry/UnreachableCooldown.java b/backend-java/src/main/java/com/nanri/aiimage/common/retry/UnreachableCooldown.java new file mode 100644 index 000000000..c91bdfe6f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/retry/UnreachableCooldown.java @@ -0,0 +1,103 @@ +package com.nanri.aiimage.common.retry; + +import org.springframework.stereotype.Component; + +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.UnknownHostException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 连接层不可达的客户端级冷却表(按上游 host 分键,Spring 单例)。 + * + *

背景(BACKLOG B24,2026-10-23 拍板):三家同型客户端({@code BrandCheckClient}/ + * {@code AppearancePatentLlmClient}/{@code SimilarAsinLlmClient})的连接层短路计数 + * 生命周期都只到单次调用,网关长断时每个新调用/分块仍要各自真发连接探针才知道"还是断的"。 + * 本表把「连续 2 次连接未建立」的证据跨调用累计,达到阈值即对该 host 进入冷却窗 + * (默认 300s,同仓两处先例:{@code pt_worker/proxy_pool.py} 与 16890 服务侧的钥匙冷却), + * 窗内调用方不发请求直接快速失败;到期后下一条真实调用兼作探针,失败即重进、成功即恢复。 + * + *

调用方契约:{@link #isCoolingDown} 在发请求前问一次;连接层失败记 + * {@link #recordConnectFailure}(返回 true 的那方打一条进入冷却的 warn); + * 连接成功或"连上了没答好"(读超时/5xx/解析失败)记 {@link #recordConnected} 清零证据。 + * 窗长 {@code <= 0} 表示关闭冷却,全部方法退化为无操作——调用方退回各自的调用内短路行为。 + */ +@Component +public class UnreachableCooldown { + + /** + * 进入冷却的证据数:与三家客户端各自的调用内短路阈值同数字——一次 refused 可能是抖动, + * 两次同形状且中间没有任何一次成功,才是服务在掉。 + */ + private static final int EVIDENCE_THRESHOLD = 2; + + private final ConcurrentHashMap states = new ConcurrentHashMap<>(); + + private static final class State { + final AtomicInteger evidence = new AtomicInteger(); + final AtomicLong cooldownUntilMillis = new AtomicLong(); + } + + /** + * 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。 + * 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。 + * 原 BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient 各有一份同款, + * B24 下沉至此,四家(含 TaskOwnerForwardService)共用同一判定。 + */ + public static boolean isConnectFailure(Throwable error) { + for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) { + if (t instanceof ConnectException || t instanceof NoRouteToHostException + || t instanceof UnknownHostException) { + return true; + } + } + return false; + } + + /** 该 host 是否正处冷却窗内:true 时调用方不发请求,直接按既有不可达归因文案快速失败。 */ + public boolean isCoolingDown(String host, long windowMillis) { + if (windowMillis <= 0) { + return false; + } + State state = states.get(normalizeHost(host)); + return state != null && state.cooldownUntilMillis.get() > System.currentTimeMillis(); + } + + /** + * 连接层失败证据 +1;累计到阈值即对该 host 开(或重开)冷却窗。 + * + * @return true 仅当"本次调用让冷却窗从无到有"(并发下只会有一个调用方拿到 true), + * 调用方据此打一条进入冷却的 warn,期中快失败不刷屏 + */ + public boolean recordConnectFailure(String host, long windowMillis) { + if (windowMillis <= 0) { + return false; + } + State state = states.computeIfAbsent(normalizeHost(host), key -> new State()); + if (state.evidence.incrementAndGet() < EVIDENCE_THRESHOLD) { + return false; + } + long now = System.currentTimeMillis(); + long previous = state.cooldownUntilMillis.get(); + if (previous > now) { + return false; + } + return state.cooldownUntilMillis.compareAndSet(previous, now + windowMillis); + } + + /** 连接成功或"连上了没答好":证据清零(已开的冷却窗不受影响,等它自然到期)。 */ + public void recordConnected(String host) { + State state = states.get(normalizeHost(host)); + if (state != null) { + state.evidence.set(0); + } + } + + /** host 归一:两个 LLM 客户端配的同一网关必须落进同一格,别因尾斜杠/大小写裂成两份。 */ + static String normalizeHost(String host) { + String normalized = host == null ? "" : host.trim().toLowerCase(); + return normalized.endsWith("/") ? normalized.substring(0, normalized.length() - 1) : normalized; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/service/TaskOwnerForwardService.java b/backend-java/src/main/java/com/nanri/aiimage/common/service/TaskOwnerForwardService.java index e93bf9098..922de5cd9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/common/service/TaskOwnerForwardService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/service/TaskOwnerForwardService.java @@ -19,7 +19,6 @@ import org.springframework.web.client.RestClient; import org.springframework.web.util.ContentCachingRequestWrapper; import java.io.IOException; -import java.net.ConnectException; import java.util.Enumeration; import java.util.List; import java.util.Set; @@ -89,7 +88,7 @@ public class TaskOwnerForwardService { .retrieve() .toEntity(byte[].class); } catch (ResourceAccessException accessError) { - if (!isConnectFailure(accessError)) { + if (!com.nanri.aiimage.common.retry.UnreachableCooldown.isConnectFailure(accessError)) { throw accessError; } lastError = accessError; @@ -103,17 +102,6 @@ public class TaskOwnerForwardService { throw lastError; } - private static boolean isConnectFailure(Throwable error) { - Throwable cursor = error; - while (cursor != null) { - if (cursor instanceof ConnectException) { - return true; - } - cursor = cursor.getCause(); - } - return false; - } - private static void sleepQuietly(long millis) { try { Thread.sleep(millis); diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java index 8d334971c..f0acec971 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java @@ -46,6 +46,13 @@ public class AppearancePatentProperties { * 每行每个 LLM 请求的重试次数(含首次) */ private int llmRetryTimes = 3; + + /** + * 连接层不可达的客户端级冷却窗长(0=关闭,退回调用内短路)。连续 2 次连接未建立即进窗, + * 窗内调用不发请求直接快速失败;到期后下一条真实调用兼作探针。拍板口径见 BACKLOG B24 + * (300s 同仓两处先例:pt_worker 代理池与 16890 服务侧的钥匙冷却)。 + */ + private int llmUnreachableCooldownMillis = 300000; /** * 单行最多被「轮次」发起多少次(一轮 = 一次批量派发;每轮内部另有 llmRetryTimes 次 HTTP 重试)。 * <=0 关闭本闸。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java index 13d1c430d..b461cce3a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java @@ -34,4 +34,11 @@ public class BrandCheckProperties { private int totalTimeoutMillis = 90000; private int connectTimeoutMillis = 10000; private int readTimeoutMillis = 60000; + + /** + * 连接层不可达的客户端级冷却窗长(0=关闭,退回批内短路)。连续 2 次连接未建立即进窗, + * 窗内整批品牌不发请求直接按服务不可达判失败;到期后下一批的真实请求兼作探针。 + * 拍板口径见 BACKLOG B24(300s 同仓两处先例:pt_worker 代理池与 16890 服务侧的钥匙冷却)。 + */ + private int unreachableCooldownMillis = 300000; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java index 166cc53ab..2b5ef0e33 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java @@ -197,6 +197,13 @@ public class SimilarAsinProperties { private int llmFirstAttemptReadTimeoutMillis = 300000; private int llmRetryTimes = 3; + /** + * 连接层不可达的客户端级冷却窗长(0=关闭,退回调用内短路)。连续 2 次连接未建立即进窗, + * 窗内调用不发请求直接快速失败;到期后下一条真实调用兼作探针。拍板口径见 BACKLOG B24 + * (300s 同仓两处先例:pt_worker 代理池与 16890 服务侧的钥匙冷却)。 + */ + private int llmUnreachableCooldownMillis = 300000; + /** 批内行级并发上限:每行最多 2 次图片对比 + 1 次合规 + 3 次类目匹配。 */ private int llmRowConcurrency = 5; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java index 7d86ab10f..841713e03 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.retry.LlmRetryBackoff; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.HttpClientPool; import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder; @@ -21,9 +22,6 @@ import org.springframework.web.client.RestClient; import java.io.IOException; import java.io.InputStream; -import java.net.ConnectException; -import java.net.NoRouteToHostException; -import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; @@ -158,6 +156,7 @@ public class AppearancePatentLlmClient { private final ExternalCallMetricsRecorder externalCallMetrics; private final BrandCheckClient brandCheckClient; private final UserSecretUsageService userSecretUsageService; + private final UnreachableCooldown unreachableCooldown; private volatile RestClient sharedRestClient; /** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */ @@ -647,15 +646,30 @@ public class AppearancePatentLlmClient { String apiKey, String responseFormat) throws Exception { int attempts = Math.max(1, properties.getLlmRetryTimes()); + long cooldownMillis = Math.max(0, properties.getLlmUnreachableCooldownMillis()); + String host = properties.getLlmHost(); + if (unreachableCooldown.isCoolingDown(host, cooldownMillis)) { + // 冷却期快失败不发请求、不记密钥次数(B24) + log.debug("[appearance-patent] 客户端级冷却中,快速失败 model={} window={}ms", model, cooldownMillis); + throw new IllegalStateException(REASON_LLM_UNREACHABLE + ";lastErr=冷却中(连续连接未建立,暂不打上游)"); + } Exception lastFailure = null; AtomicInteger connectFailures = new AtomicInteger(); for (int attempt = 1; attempt <= attempts; attempt++) { try { - return invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt); + String content = invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt); + unreachableCooldown.recordConnected(host); + return content; } catch (Exception ex) { lastFailure = ex; - if (isConnectFailure(ex)) { - if (connectFailures.incrementAndGet() >= LLM_CONNECT_FAILURE_EVIDENCE) { + if (UnreachableCooldown.isConnectFailure(ex)) { + boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis); + if (entered) { + log.warn("[appearance-patent] 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}", + LLM_CONNECT_FAILURE_EVIDENCE, cooldownMillis, host); + } + if (connectFailures.incrementAndGet() >= LLM_CONNECT_FAILURE_EVIDENCE + || unreachableCooldown.isCoolingDown(host, cooldownMillis)) { log.warn("[appearance-patent] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达," + "停止重试 attempt={} model={} err={}", LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex)); @@ -667,6 +681,7 @@ public class AppearancePatentLlmClient { } else { // 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数 connectFailures.set(0); + unreachableCooldown.recordConnected(host); } if (attempt >= attempts) { break; @@ -681,17 +696,11 @@ public class AppearancePatentLlmClient { } /** - * 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。 - * 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。 + * 连接层判定下沉 {@code UnreachableCooldown.isConnectFailure}(B24):判定口径与 + * BrandCheckClient/SimilarAsinLlmClient 共用一份,不再各留包内副本。 */ static boolean isConnectFailure(Throwable error) { - for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) { - if (t instanceof ConnectException || t instanceof NoRouteToHostException - || t instanceof UnknownHostException) { - return true; - } - } - return false; + return UnreachableCooldown.isConnectFailure(error); } private String invokeChatOnce(String model, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java index 8377c6e8a..6f0ea00f8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java @@ -77,16 +77,19 @@ public class BrandCheckClient { private final BrandCheckProperties properties; private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics; private final UserApiSecretService userApiSecretService; + private final com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown; /** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */ private final ThreadLocal rejectedSubmission = new ThreadLocal<>(); private volatile RestClient sharedRestClient; public BrandCheckClient(BrandCheckProperties properties, com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics, - UserApiSecretService userApiSecretService) { + UserApiSecretService userApiSecretService, + com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown) { this.properties = properties; this.externalCallMetrics = externalCallMetrics; this.userApiSecretService = userApiSecretService; + this.unreachableCooldown = unreachableCooldown; } // 有界队列 + 拒绝即降级:大批量(几千品牌)检查时队列洪峰不无界堆内存。 @@ -267,6 +270,18 @@ public class BrandCheckClient { } return outcomes; } + long cooldownMillis = Math.max(0, properties.getUnreachableCooldownMillis()); + if (unreachableCooldown.isCoolingDown(properties.getBaseUrl(), cooldownMillis)) { + // 客户端级冷却(B24):上一批已确认连接层不可达,本批连首轮探针都不发。 + // 结论与批内短路版逐字一致(每行仍判查询失败、reason 同文案),差别只是不再为 + // "确认还是断的"每批各烧一轮连接探针;到期后下一批的真实请求兼作探针。 + log.warn("[brand-check] SERVICE_UNREACHABLE 客户端级冷却中,整批不打上游 brands={} window={}ms", + brands.size(), cooldownMillis); + for (String brand : brands) { + outcomes.put(brand, new BrandCheckOutcome(List.of(), List.of(brand), REASON_SERVICE_UNREACHABLE)); + } + return outcomes; + } long firstAttemptDeadlineNanos = deadlineNanos == Long.MAX_VALUE ? Long.MAX_VALUE : deadlineNanos + FIRST_ATTEMPT_QUEUE_GRACE_MILLIS * 1_000_000L; @@ -363,6 +378,7 @@ public class BrandCheckClient { lastFailure = ex; response = null; if (isConnectFailure(ex)) { + recordConnectFailure(ex); if (connectFailures.incrementAndGet() >= CONNECT_FAILURE_EVIDENCE) { ladderCompleted = false; connectDown = true; @@ -374,6 +390,7 @@ public class BrandCheckClient { } else { // 连上了只是这次没答好(读超时/解析失败):连接层证据不作数 connectFailures.set(0); + unreachableCooldown.recordConnected(properties.getBaseUrl()); } if (attempt < attempts) { long retryDelayMillis = retryDelayMillis(attempt); @@ -385,6 +402,7 @@ public class BrandCheckClient { } // 拿到响应=连接是通的,证据清零,别跨到下一次真断连时凑数 connectFailures.set(0); + unreachableCooldown.recordConnected(properties.getBaseUrl()); if (response != null && nullToEmpty(response.getQueryFaildData()).isEmpty()) { // 本次查询完成且无失败项:命中侵权(faild_data)属有效结论,不重试。 return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of(), ""); @@ -554,14 +572,21 @@ public class BrandCheckClient { *

故意不含读超时与 5xx:那两种是"连上了但这次没答好",重试救得回来;把它们算进 * 不可达证据,就会把一次上游抖动误判成服务在掉、把同批其余品牌的重试一起掐掉。 */ - static boolean isConnectFailure(Throwable error) { - for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) { - if (t instanceof ConnectException || t instanceof NoRouteToHostException - || t instanceof UnknownHostException) { - return true; - } + /** 连接层失败记入客户端级冷却表:达到阈值的那方打一条进入冷却的 warn(B24)。 */ + private void recordConnectFailure(Exception ex) { + long cooldownMillis = Math.max(0, properties.getUnreachableCooldownMillis()); + if (unreachableCooldown.recordConnectFailure(properties.getBaseUrl(), cooldownMillis)) { + log.warn("[brand-check] SERVICE_UNREACHABLE 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}", + CONNECT_FAILURE_EVIDENCE, cooldownMillis, properties.getBaseUrl()); } - return false; + } + + /** + * 连接层判定下沉 {@code UnreachableCooldown.isConnectFailure}(B24):判定口径与 + * 两个 LLM 客户端共用一份,不再各留包内副本。 + */ + static boolean isConnectFailure(Throwable error) { + return com.nanri.aiimage.common.retry.UnreachableCooldown.isConnectFailure(error); } /** diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java index b564b089e..f15976862 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java @@ -118,7 +118,10 @@ public class CollectDataBrandBatchFilter { } continue; } - if (deadlineNanos != Long.MAX_VALUE && System.nanoTime() >= deadlineNanos) { + // 首批不掐(与 BrandCheckClient 的 attempt>1 同契约):预算检查只拦后续批次。 + // 对首批也查的话,deadline 计算与检查之间一次 GC/调度停顿(毫秒级)就会把首批 + // 整组误切成 queryFailed——1ms 预算的测试已实测抖出过这个形状。 + if (start > 0 && deadlineNanos != Long.MAX_VALUE && System.nanoTime() >= deadlineNanos) { // 预算用尽:不再为后面的批次捶上游。已缓存的品牌仍按缓存结论归类(不花代价), // 没结论的行归 queryFailed 入补测队列——既不按 accepted 放行(可能正是侵权品牌), // 也不按 rejected 拉黑(没查过就不该写进无效品牌表,否则永久误杀)。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java index 9ed39be45..56981bb99 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.similarasin.client; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.retry.LlmRetryBackoff; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.HttpClientPool; import com.nanri.aiimage.config.SimilarAsinProperties; import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder; @@ -17,9 +18,6 @@ import org.springframework.web.client.RestClient; import java.io.IOException; import java.io.InputStream; -import java.net.ConnectException; -import java.net.NoRouteToHostException; -import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; @@ -52,6 +50,7 @@ public class SimilarAsinLlmClient { private final ObjectMapper objectMapper; private final ExternalCallMetricsRecorder externalCallMetrics; private final UserSecretUsageService userSecretUsageService; + private final UnreachableCooldown unreachableCooldown; private volatile RestClient sharedRestClient; /** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */ @@ -60,11 +59,13 @@ public class SimilarAsinLlmClient { public SimilarAsinLlmClient(SimilarAsinProperties properties, ObjectMapper objectMapper, ExternalCallMetricsRecorder externalCallMetrics, - UserSecretUsageService userSecretUsageService) { + UserSecretUsageService userSecretUsageService, + UnreachableCooldown unreachableCooldown) { this.properties = properties; this.objectMapper = objectMapper; this.externalCallMetrics = externalCallMetrics; this.userSecretUsageService = userSecretUsageService; + this.unreachableCooldown = unreachableCooldown; } /** 文本对话,json_object 输出。 */ @@ -85,16 +86,32 @@ public class SimilarAsinLlmClient { String responseFormat) { String resolvedKey = resolveApiKey(apiKey); int attempts = Math.max(1, properties.getLlmRetryTimes()); + long cooldownMillis = Math.max(0, properties.getLlmUnreachableCooldownMillis()); + String host = properties.getLlmHost(); + if (unreachableCooldown.isCoolingDown(host, cooldownMillis)) { + // 冷却期快失败不发请求、不记密钥次数:结论与短路版一致(调用方拿不到 LLM 结论), + // 差别只是不再为"确认还是断的"各烧一轮连接探针(B24) + log.debug("[similar-asin][llm] 客户端级冷却中,快速失败 model={} window={}ms", model, cooldownMillis); + throw new IllegalStateException(REASON_LLM_UNREACHABLE + ";lastErr=冷却中(连续连接未建立,暂不打上游)"); + } Exception lastFailure = null; int connectFailures = 0; for (int attempt = 1; attempt <= attempts; attempt++) { try { - return invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt); + String content = invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt); + unreachableCooldown.recordConnected(host); + return content; } catch (Exception ex) { lastFailure = ex; - if (isConnectFailure(ex)) { + if (UnreachableCooldown.isConnectFailure(ex)) { connectFailures++; - if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE) { + boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis); + if (entered) { + log.warn("[similar-asin][llm] 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}", + 2, cooldownMillis, host); + } + if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE + || unreachableCooldown.isCoolingDown(host, cooldownMillis)) { log.warn("[similar-asin][llm] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达," + "停止重试 attempt={} model={} err={}", LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex)); @@ -106,6 +123,7 @@ public class SimilarAsinLlmClient { } else { // 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数 connectFailures = 0; + unreachableCooldown.recordConnected(host); } if (attempt >= attempts) { break; @@ -123,20 +141,6 @@ public class SimilarAsinLlmClient { : new IllegalStateException(lastFailure.getMessage(), lastFailure); } - /** - * 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。 - * 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。 - */ - static boolean isConnectFailure(Throwable error) { - for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) { - if (t instanceof ConnectException || t instanceof NoRouteToHostException - || t instanceof UnknownHostException) { - return true; - } - } - return false; - } - /** 只认任务/用户自己的 api_key(payload → 按 uid 的服务端密钥表已在 readApiKey 兜过); * 为空即空、跳过 LLM,绝不回落平台公费密钥 —— 各用户的任务只能烧各自的密钥。 */ public String resolveApiKey(String apiKey) { diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index a02d0144d..d2f592039 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -378,6 +378,9 @@ aiimage: total-timeout-millis: ${AIIMAGE_BRAND_CHECK_TOTAL_TIMEOUT_MILLIS:90000} connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000} read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000} + # 连接层不可达的客户端级冷却窗长(B24,0=关闭):连续 2 次连接未建立即进窗, + # 窗内整批品牌不发请求直接按服务不可达判失败;到期后下一批的真实请求兼作探针。 + unreachable-cooldown-millis: ${AIIMAGE_BRAND_CHECK_UNREACHABLE_COOLDOWN_MILLIS:300000} appearance-patent: no-result-upload-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180} llm-host: ${AIIMAGE_APPEARANCE_PATENT_LLM_HOST:https://ai.t8star.org} @@ -402,6 +405,9 @@ aiimage: # 实测空行 20.7%~51.3%,全部放行且零告警,因为旧分母按解析行标题判、用户表不带标题列时分母为 0)。 llm-coverage-min-ratio: ${AIIMAGE_APPEARANCE_PATENT_LLM_COVERAGE_MIN_RATIO:0.95} llm-retry-times: ${AIIMAGE_APPEARANCE_PATENT_LLM_RETRY_TIMES:3} + # 连接层不可达的客户端级冷却窗长(B24,0=关闭):连续 2 次连接未建立即进窗, + # 窗内调用不发请求直接按服务不可达快速失败;到期后下一条真实调用兼作探针。 + llm-unreachable-cooldown-millis: ${AIIMAGE_APPEARANCE_PATENT_LLM_UNREACHABLE_COOLDOWN_MILLIS:300000} # 单行最多被重发几轮(<=0 关闭)。失败行会被释放、下一轮重新申领,没有这道闸时 # 上游劣化窗口内会无限重发并每轮扣用户 LLM 次数——deferRunning 不累加 retryCount,任务层那道永不触发。 llm-row-max-rounds: ${AIIMAGE_APPEARANCE_PATENT_LLM_ROW_MAX_ROUNDS:3} @@ -440,6 +446,9 @@ aiimage: llm-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_READ_TIMEOUT_MILLIS:600000} llm-first-attempt-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_FIRST_ATTEMPT_READ_TIMEOUT_MILLIS:300000} llm-retry-times: ${AIIMAGE_SIMILAR_ASIN_LLM_RETRY_TIMES:3} + # 连接层不可达的客户端级冷却窗长(B24,0=关闭):连续 2 次连接未建立即进窗, + # 窗内调用不发请求直接按服务不可达快速失败;到期后下一条真实调用兼作探针。 + llm-unreachable-cooldown-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_UNREACHABLE_COOLDOWN_MILLIS:300000} llm-row-concurrency: ${AIIMAGE_SIMILAR_ASIN_LLM_ROW_CONCURRENCY:5} llm-category-batch-size: ${AIIMAGE_SIMILAR_ASIN_LLM_CATEGORY_BATCH_SIZE:10} # 合规判定一次请求合并的行数:省往返不省判定,每件商品仍各自出 is_conform/reason/category。 diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/retry/UnreachableCooldownTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/retry/UnreachableCooldownTest.java new file mode 100644 index 000000000..272014e9e --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/common/retry/UnreachableCooldownTest.java @@ -0,0 +1,77 @@ +package com.nanri.aiimage.common.retry; + +import org.junit.jupiter.api.Test; + +import java.net.ConnectException; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 连接层不可达冷却表的口径(B24):阈值 2、证据可清零、窗长到点自然到期、 + * 到期后一次失败即重进、窗长 0 整体关闭、同 host 不同写法必须同格。 + */ +class UnreachableCooldownTest { + + private final UnreachableCooldown cooldown = new UnreachableCooldown(); + private static final long WINDOW = 60_000L; + + @Test + void twoConsecutiveFailuresEnterCooldownOneDoesNot() { + assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isFalse(); + assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isFalse(); + assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isFalse(); + // 第二次失败:冷却窗从无到有,返回 true 的那方打进入 warn;之后的失败不重复报 + assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isTrue(); + assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isTrue(); + assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isFalse(); + } + + @Test + void connectedResetsEvidenceSoTwoFreshFailuresAreRequiredAgain() { + cooldown.recordConnectFailure("https://gw.example.com", WINDOW); + cooldown.recordConnected("https://gw.example.com"); + assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isFalse(); + assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isFalse(); + } + + @Test + void windowExpiresNaturallyAndSingleFailureReenters() throws InterruptedException { + String host = "https://short.example.com"; + cooldown.recordConnectFailure(host, 50); + cooldown.recordConnectFailure(host, 50); + assertThat(cooldown.isCoolingDown(host, 50)).isTrue(); + + Thread.sleep(120); + assertThat(cooldown.isCoolingDown(host, 50)).isFalse(); + + // 历史证据不清零:到期后探针再失败,一次即重进(B24 拍板口径) + assertThat(cooldown.recordConnectFailure(host, 50)).isTrue(); + assertThat(cooldown.isCoolingDown(host, 50)).isTrue(); + } + + @Test + void zeroWindowDisablesCooldownEntirely() { + for (int i = 0; i < 5; i++) { + assertThat(cooldown.recordConnectFailure("https://gw.example.com", 0)).isFalse(); + } + assertThat(cooldown.isCoolingDown("https://gw.example.com", 0)).isFalse(); + } + + @Test + void sameHostDifferentSpellingSharesOneCell() { + cooldown.recordConnectFailure("https://GW.example.com", WINDOW); + // 尾斜杠/大小写不同写法落进同一格:这条是第 2 次失败,必须进冷却 + assertThat(cooldown.recordConnectFailure("https://gw.example.com/", WINDOW)).isTrue(); + assertThat(cooldown.isCoolingDown("https://Gw.Example.Com", WINDOW)).isTrue(); + } + + @Test + void isConnectFailureWalksCauseChainAndIgnoresOtherFailures() { + assertThat(UnreachableCooldown.isConnectFailure(new ConnectException("refused"))).isTrue(); + RuntimeException wrapped = new RuntimeException( + new IllegalStateException(new ConnectException("refused"))); + assertThat(UnreachableCooldown.isConnectFailure(wrapped)).isTrue(); + assertThat(UnreachableCooldown.isConnectFailure(new RuntimeException("read timeout"))).isFalse(); + assertThat(UnreachableCooldown.isConnectFailure(null)).isFalse(); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientConnectionReuseTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientConnectionReuseTest.java index 37969d8ca..a099c5cf7 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientConnectionReuseTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientConnectionReuseTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.config; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.modules.brand.client.BrandCheckClient; import com.nanri.aiimage.modules.brand.client.BrandCheckClient.BrandCheckBatchResult; import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient; @@ -96,7 +97,7 @@ class HttpClientConnectionReuseTest { void test_task_077_brand_normal_default_path() throws Exception { // 默认路径:品牌检查客户端通过共享池创建单例 RestClient, // 工厂为带 keep-alive 连接池的 JdkClientHttpRequestFactory。 - BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null); + BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()); BrandCheckBatchResult result = client.checkTitleText(" ", null, null); assertTrue(result.brands().isEmpty(), "空标题安全跳过,不创建无效资源"); @@ -107,8 +108,8 @@ class HttpClientConnectionReuseTest { void test_task_077_brand_normal_multiple_items() throws Exception { // 批量场景:LLM/品牌/紫鸟三个客户端各自持有独立 RestClient, // 但底层连接池共用同一 HttpClient 实例,不重复创建。 - SimilarAsinLlmClient llm = new SimilarAsinLlmClient(new SimilarAsinProperties(), new ObjectMapper(), null, null); - BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null); + SimilarAsinLlmClient llm = new SimilarAsinLlmClient(new SimilarAsinProperties(), new ObjectMapper(), null, null, new UnreachableCooldown()); + BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()); ZiniaoClientImpl ziniao = new ZiniaoClientImpl(new ZiniaoProperties(), new ObjectMapper()); HttpClient llmClient = clientOf(factoryOf(restClientOf(llm))); @@ -124,7 +125,7 @@ class HttpClientConnectionReuseTest { void test_task_077_brand_normal_repeated_operation_is_idempotent() throws Exception { // 幂等:同一客户端重复触发请求创建逻辑只产生一个 RestClient, // 重复调用返回同一实例,不重复创建客户端对象。 - BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null); + BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()); assertSame(restClientOf(brand), restClientOf(brand), "品牌客户端复用同一 RestClient"); ZiniaoClientImpl ziniao = new ZiniaoClientImpl(new ZiniaoProperties(), new ObjectMapper()); @@ -134,7 +135,7 @@ class HttpClientConnectionReuseTest { @Test void test_task_077_brand_boundary_empty_input() throws Exception { // 空输入:空品牌列表不发起任何 HTTP 请求、不创建客户端资源。 - BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null); + BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()); BrandCheckBatchResult result = brand.checkAll(List.of(), "Terms", null); assertTrue(result.brands().isEmpty()); @@ -145,7 +146,7 @@ class HttpClientConnectionReuseTest { @Test void test_task_077_brand_boundary_single_item() throws Exception { // 单元素:单客户端单请求走共享池,工厂带连接池,行为与批量一致。 - BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null); + BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()); assertPooled(factoryOf(restClientOf(brand))); } @@ -155,7 +156,7 @@ class HttpClientConnectionReuseTest { // 不随实例数量线性增长连接资源。 int instances = 8; for (int i = 0; i < instances; i++) { - restClientOf(new BrandCheckClient(new BrandCheckProperties(), null, null)); + restClientOf(new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown())); restClientOf(new ZiniaoClientImpl(new ZiniaoProperties(), new ObjectMapper())); } assertSame(HttpClientPool.sharedHttpClient(), HttpClientPool.sharedHttpClient(), diff --git a/backend-java/src/test/java/com/nanri/aiimage/metrics/ExternalCallMetricsRecorderTest.java b/backend-java/src/test/java/com/nanri/aiimage/metrics/ExternalCallMetricsRecorderTest.java index f51e37c33..4cf508812 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/metrics/ExternalCallMetricsRecorderTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/metrics/ExternalCallMetricsRecorderTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.metrics; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.BrandCheckProperties; import com.nanri.aiimage.config.SimilarAsinProperties; import com.nanri.aiimage.config.ZiniaoProperties; @@ -162,7 +163,7 @@ class ExternalCallMetricsRecorderTest { SimilarAsinProperties props = llmProps(); SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, new ExternalCallMetricsRecorder(registry), - mock(UserSecretUsageService.class)); + mock(UserSecretUsageService.class), new UnreachableCooldown()); String content = client.invokeChat("test-model", "system", "hello", "test-key"); @@ -181,7 +182,7 @@ class ExternalCallMetricsRecorderTest { SimilarAsinProperties props = llmProps(); SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, new ExternalCallMetricsRecorder(registry), - mock(UserSecretUsageService.class)); + mock(UserSecretUsageService.class), new UnreachableCooldown()); for (int i = 0; i < 3; i++) { client.invokeChat("test-model", "system", "prompt-" + i, "test-key"); @@ -192,7 +193,7 @@ class ExternalCallMetricsRecorderTest { "client", "llm", "result", "success"), "三次成功全部记录"); // 全部外部客户端(LLM / 品牌检查 / 紫鸟)在同一次批量中各自记录指标 - BrandCheckClient brand = new BrandCheckClient(brandProps(), new ExternalCallMetricsRecorder(registry), stubbedUserSecrets()); + BrandCheckClient brand = new BrandCheckClient(brandProps(), new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown()); brand.checkAll(List.of("Nintendo", "LEGO", "Sony"), "Terms", 7L); ZiniaoClientImpl ziniao = new ZiniaoClientImpl(ziniaoProps(), objectMapper, new ExternalCallMetricsRecorder(registry)); assertEquals(1001L, ziniao.getCompanyIdByApiKey("test-api-key")); @@ -210,7 +211,7 @@ class ExternalCallMetricsRecorderTest { @Test void test_task_078_payload_metrics_normal_repeated_operation_is_idempotent() throws Exception { BrandCheckProperties props = brandProps(); - BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets()); + BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown()); client.check("Nintendo", "Terms", 7L); client.check("Nintendo", "Terms", 7L); @@ -227,7 +228,7 @@ class ExternalCallMetricsRecorderTest { @Test void test_task_078_payload_metrics_boundary_empty_input() throws Exception { BrandCheckProperties props = brandProps(); - BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets()); + BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown()); BrandCheckClient.BrandCheckBatchResult result = client.checkAll(List.of(), "Terms", 7L); @@ -242,7 +243,7 @@ class ExternalCallMetricsRecorderTest { @Test void test_task_078_payload_metrics_boundary_single_item() throws Exception { BrandCheckProperties props = brandProps(); - BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets()); + BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown()); BrandCheckClient.BrandCheckResponse response = client.check("LEGO", "Terms", 7L); @@ -264,7 +265,7 @@ class ExternalCallMetricsRecorderTest { int index = i; pool.submit(() -> { SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, recorder, - mock(UserSecretUsageService.class)); + mock(UserSecretUsageService.class), new UnreachableCooldown()); try { client.invokeChat("test-model", "system", "prompt-" + index, "test-key"); } catch (Exception ignored) { @@ -297,7 +298,7 @@ class ExternalCallMetricsRecorderTest { @Test void test_task_078_payload_metrics_invalid_input_rejected() throws Exception { BrandCheckProperties props = brandProps(); - BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets()); + BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown()); BrandCheckClient.BrandCheckBatchResult result = client.checkAll(null, "Terms", 7L); @@ -315,7 +316,7 @@ class ExternalCallMetricsRecorderTest { SimilarAsinProperties props = llmProps(); SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, recorder, - mock(UserSecretUsageService.class)); + mock(UserSecretUsageService.class), new UnreachableCooldown()); // 第一次调用走 500 失败路径,第二次调用恢复成功:错误可恢复 llmFailNext.set(true); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentBrandBatchTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentBrandBatchTest.java index c2af37bb9..794d4b12e 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentBrandBatchTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentBrandBatchTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; import com.nanri.aiimage.modules.brand.client.BrandCheckClient; @@ -46,7 +47,7 @@ class AppearancePatentBrandBatchTest { null, brandCheckClient, mock(UserSecretUsageService.class) - ); + , new UnreachableCooldown()); @Test void mergedBatchAsksBrandCheckOnceAndAttributesPerRow() throws Exception { @@ -115,7 +116,7 @@ class AppearancePatentBrandBatchTest { properties.setLlmRetryTimes(1); properties.setLlmReadTimeoutMillis(5_000); AppearancePatentLlmClient inspectClient = new AppearancePatentLlmClient( - properties, new ObjectMapper(), null, brandCheckClient, mock(UserSecretUsageService.class)); + properties, new ObjectMapper(), null, brandCheckClient, mock(UserSecretUsageService.class), new UnreachableCooldown()); List rows = inspectClient.inspectRows( List.of( diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java index eda983869..0c2208c84 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.BrandCheckProperties; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; @@ -72,7 +73,7 @@ class AppearancePatentLlmClientHttpTest { null, localBrandCheckClient(), mock(UserSecretUsageService.class) - ); + , new UnreachableCooldown()); } /** @@ -85,7 +86,7 @@ class AppearancePatentLlmClientHttpTest { brand.setRetryTimes(1); brand.setRetryIntervalMillis(0); brand.setTotalTimeoutMillis(5000); - return new BrandCheckClient(brand, null, null); + return new BrandCheckClient(brand, null, null, new UnreachableCooldown()); } private void handleBrandCheck(HttpExchange exchange) throws IOException { @@ -290,7 +291,7 @@ class AppearancePatentLlmClientHttpTest { properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort()); customizer.accept(properties); return new AppearancePatentLlmClient(properties, objectMapper, null, - localBrandCheckClient(), mock(UserSecretUsageService.class)); + localBrandCheckClient(), mock(UserSecretUsageService.class), new UnreachableCooldown()); } /** 连接层故障短路:网关根本连不上时,重试阶梯必须在证据数处停下,行级拿到可读的真因。 */ diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientImageDedupeTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientImageDedupeTest.java index 7fc9304dc..a17ab46c6 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientImageDedupeTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientImageDedupeTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.BrandCheckProperties; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; @@ -68,7 +69,7 @@ class AppearancePatentLlmClientImageDedupeTest { properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort()); properties.setLlmRetryTimes(1); client = new AppearancePatentLlmClient(properties, objectMapper, null, localBrandCheckClient(), - mock(UserSecretUsageService.class)); + mock(UserSecretUsageService.class), new UnreachableCooldown()); } private BrandCheckClient localBrandCheckClient() { @@ -77,7 +78,7 @@ class AppearancePatentLlmClientImageDedupeTest { brand.setRetryTimes(1); brand.setRetryIntervalMillis(0); brand.setTotalTimeoutMillis(5000); - return new BrandCheckClient(brand, null, null); + return new BrandCheckClient(brand, null, null, new UnreachableCooldown()); } @AfterEach diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTest.java index f6a57fc11..e96d74c3e 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.BrandCheckProperties; import com.nanri.aiimage.modules.brand.client.BrandCheckClient; @@ -26,9 +27,9 @@ class AppearancePatentLlmClientTest { new AppearancePatentProperties(), new ObjectMapper(), null, - new BrandCheckClient(new BrandCheckProperties(), null, null), + new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()), mock(UserSecretUsageService.class) - ); + , new UnreachableCooldown()); @Test void markRowsFailedLeavesUserFacingResultFilledWithReviewMessage() { @@ -123,7 +124,7 @@ class AppearancePatentLlmClientTest { null, brandCheckClient, mock(UserSecretUsageService.class) - ); + , new UnreachableCooldown()); AppearancePatentResultRowDto row = new AppearancePatentResultRowDto(); row.setId("1"); String dataText = """ @@ -140,7 +141,7 @@ class AppearancePatentLlmClientTest { private static AppearancePatentLlmClient clientWith(BrandCheckClient brandCheckClient) { return new AppearancePatentLlmClient(new AppearancePatentProperties(), new ObjectMapper(), - null, brandCheckClient, mock(UserSecretUsageService.class)); + null, brandCheckClient, mock(UserSecretUsageService.class), new UnreachableCooldown()); } private static AppearancePatentResultRowDto newRow(String id) { @@ -204,7 +205,7 @@ class AppearancePatentLlmClientTest { null, brandCheckClient, mock(UserSecretUsageService.class) - ); + , new UnreachableCooldown()); AppearancePatentResultRowDto row = new AppearancePatentResultRowDto(); row.setId("1"); String dataText = """ @@ -228,7 +229,7 @@ class AppearancePatentLlmClientTest { null, brandCheckClient, mock(UserSecretUsageService.class) - ); + , new UnreachableCooldown()); AppearancePatentResultRowDto row = new AppearancePatentResultRowDto(); row.setId("1"); String dataText = """ @@ -251,7 +252,7 @@ class AppearancePatentLlmClientTest { null, brandCheckClient, mock(UserSecretUsageService.class) - ); + , new UnreachableCooldown()); AppearancePatentResultRowDto row = new AppearancePatentResultRowDto(); row.setId("1"); String dataText = """ diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTokenUsageTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTokenUsageTest.java index 1bab55f5f..56bca2fb5 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTokenUsageTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientTokenUsageTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.BrandCheckProperties; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; @@ -59,7 +60,7 @@ class AppearancePatentLlmClientTokenUsageTest { AppearancePatentProperties properties = new AppearancePatentProperties(); properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort()); properties.setLlmRetryTimes(1); - client = new AppearancePatentLlmClient(properties, objectMapper, null, localBrandCheckClient(), usageService); + client = new AppearancePatentLlmClient(properties, objectMapper, null, localBrandCheckClient(), usageService, new UnreachableCooldown()); } private BrandCheckClient localBrandCheckClient() { @@ -68,7 +69,7 @@ class AppearancePatentLlmClientTokenUsageTest { brand.setRetryTimes(1); brand.setRetryIntervalMillis(0); brand.setTotalTimeoutMillis(5000); - return new BrandCheckClient(brand, null, null); + return new BrandCheckClient(brand, null, null, new UnreachableCooldown()); } @AfterEach diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java index 5affc7add..1b6bd16ad 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.brand.client; import com.nanri.aiimage.config.BrandCheckProperties; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService; import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; @@ -34,7 +35,7 @@ class BrandCheckClientTest { @Test void splitTitleTextSupportsCommonSeparatorsAndQuotes() { - BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null); + BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()); List brands = client.splitTitleText("'阿凡达,任天堂'; Disney、LEGO\nSony"); @@ -43,13 +44,43 @@ class BrandCheckClientTest { @Test void splitTitleTextDeduplicatesBlankValues() { - BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null); + BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()); List brands = client.splitTitleText(" 任天堂, ,任天堂,Sony "); assertThat(brands).containsExactly("任天堂", "Sony"); } + @Test + void clientLevelCooldownFailsWholeBatchWithoutHittingUpstream() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + HttpServer server = startBrandCheckServer(exchange -> { + attempts.incrementAndGet(); + respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[]}"); + }); + try { + UnreachableCooldown cooldown = new UnreachableCooldown(); + BrandCheckProperties props = new BrandCheckProperties(); + props.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + props.setPath("/brand_check"); + props.setRetryTimes(3); + props.setRetryIntervalMillis(1); + BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets(), cooldown); + + cooldown.recordConnectFailure(props.getBaseUrl(), 300_000L); + cooldown.recordConnectFailure(props.getBaseUrl(), 300_000L); + + BrandCheckClient.BrandCheckBatchResult result = client.checkAll(List.of("Apple", "Nike"), "Terms", 7L); + + assertThat(attempts.get()).as("冷却期内一个请求都不该发").isZero(); + assertThat(result.hasQueryFailedData()).isTrue(); + assertThat(result.queryFaildData()).containsExactly("Apple", "Nike"); + assertThat(result.reason()).contains("品牌检测服务不可达(连接未建立,非本行数据问题)"); + } finally { + server.stop(0); + } + } + @Test void queryFailureRetriesUntilSuccess() throws Exception { AtomicInteger attempts = new AtomicInteger(); @@ -242,7 +273,7 @@ class BrandCheckClientTest { props.setRetryTimes(retryTimes); props.setRetryIntervalMillis(retryIntervalMillis); props.setTotalTimeoutMillis(totalTimeoutMillis); - return new BrandCheckClient(props, null, stubbedUserSecrets()); + return new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown()); } private BrandCheckClient newClient(HttpServer server, int retryTimes) { @@ -251,7 +282,7 @@ class BrandCheckClientTest { props.setPath("/brand_check"); props.setRetryTimes(retryTimes); props.setRetryIntervalMillis(1); - return new BrandCheckClient(props, null, stubbedUserSecrets()); + return new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown()); } /** 重试阶梯可调的构造:用于验证「耗时预算掐的是重试,不是首轮」。 */ @@ -262,7 +293,7 @@ class BrandCheckClientTest { props.setRetryTimes(retryTimes); props.setRetryIntervalMillis(retryIntervalMillis); props.setTotalTimeoutMillis(totalTimeoutMillis); - return new BrandCheckClient(props, null, stubbedUserSecrets()); + return new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown()); } /** 有真实 HTTP 断言的用例必须桩出非空代理,否则客户端零请求短路、阶梯与指标全测不到。 */ @@ -361,7 +392,7 @@ class BrandCheckClientTest { props.setRetryIntervalMillis(1); props.setReadTimeoutMillis(1_000); props.setTotalTimeoutMillis(0); - BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets()); + BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown()); ExecutorService pool = (ExecutorService) ReflectionTestUtils.getField(client, "checkExecutor"); CountDownLatch occupied = new CountDownLatch(3); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTitleBatchTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTitleBatchTest.java index 661abee15..dd7fa46d6 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTitleBatchTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTitleBatchTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.brand.client; import com.nanri.aiimage.config.BrandCheckProperties; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService; import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; @@ -137,7 +138,7 @@ class BrandCheckClientTitleBatchTest { props.setRetryIntervalMillis(1); UserApiSecretService secrets = mock(UserApiSecretService.class); when(secrets.findPlainValue(anyLong(), anyString())).thenReturn("http://user:pass@127.0.0.1:9"); - return new BrandCheckClient(props, null, secrets); + return new BrandCheckClient(props, null, secrets, new UnreachableCooldown()); } private HttpServer startBrandCheckServer(com.sun.net.httpserver.HttpHandler handler) throws IOException { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java index 60bbb8800..0130f593a 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.similarasin.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.nanri.aiimage.config.SimilarAsinProperties; import com.nanri.aiimage.modules.usersecret.service.UserSecretUsageService; import com.sun.net.httpserver.HttpExchange; @@ -121,10 +122,57 @@ class SimilarAsinLlmClientHttpTest { assertThat(requestCount.get()).isEqualTo(2); } + /** B24:首次调用确认不可达即进客户端级冷却,冷却期内的后续调用零探针快速失败。 */ + @Test + void cooldownMakesSubsequentCallsFailFastWithoutProbes() { + SimilarAsinLlmClient deadHostClient = buildClient(p -> p.setLlmHost("http://127.0.0.1:1")); + + assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key")) + .hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)"); + assertThat(lastCooldown.isCoolingDown(lastHost, lastWindow)).isTrue(); + + // 快速失败走的是入口冷却分支(";lastErr=冷却中"后缀只有该路径会拼),一次探针都不发 + assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key")) + .hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)") + .hasMessageContaining(";lastErr=冷却中"); + } + + /** B24:冷却到期后下一条真实调用兼作探针;探针仍连不上 ⇒ 立即重进冷却。 */ + @Test + void expiredCooldownProbesAgainAndReentersOnFailure() throws InterruptedException { + SimilarAsinLlmClient deadHostClient = buildClient(p -> { + p.setLlmHost("http://127.0.0.1:1"); + p.setLlmUnreachableCooldownMillis(200); + }); + assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key")) + .hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)"); + + Thread.sleep(400); + assertThat(lastCooldown.isCoolingDown(lastHost, 200)).isFalse(); + + // 探针真发:lastErr 带的是连接异常而不是"冷却中";失败即重进 + assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key")) + .hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)") + .hasMessageNotContaining("冷却中"); + assertThat(lastCooldown.isCoolingDown(lastHost, 200)).isTrue(); + + // 重进之后的调用回到零探针快速失败 + assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key")) + .hasMessageContaining(";lastErr=冷却中"); + } + private SimilarAsinLlmClient buildClient(java.util.function.Consumer customizer) { SimilarAsinProperties properties = new SimilarAsinProperties(); properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort()); customizer.accept(properties); - return new SimilarAsinLlmClient(properties, objectMapper, null, mock(UserSecretUsageService.class)); + lastCooldown = new UnreachableCooldown(); + lastHost = properties.getLlmHost(); + lastWindow = Math.max(0, properties.getLlmUnreachableCooldownMillis()); + return new SimilarAsinLlmClient(properties, objectMapper, null, mock(UserSecretUsageService.class), lastCooldown); } + + /** 最近一次 buildClient 的冷却表与 host/窗长:冷却态断言直接读同一实例,不靠时间猜。 */ + private UnreachableCooldown lastCooldown; + private String lastHost; + private long lastWindow; } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinLlmLocalVerify.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinLlmLocalVerify.java index 3b2702d2a..d2cf3847f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinLlmLocalVerify.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinLlmLocalVerify.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.similarasin.service; import com.fasterxml.jackson.databind.JsonNode; +import com.nanri.aiimage.common.retry.UnreachableCooldown; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.config.OssProperties; import com.nanri.aiimage.config.SimilarAsinProperties; @@ -61,7 +62,7 @@ public class SimilarAsinLlmLocalVerify { props.setLlmRowConcurrency(2); props.setLlmImageDownloadTimeoutSeconds(10); - SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null, null); + SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null, null, new UnreachableCooldown()); OssProperties ossProps = new OssProperties(); ossProps.setEndpoint("https://oss.aishufu.top"); ossProps.setPublicEndpoint("https://oss.aishufu.top");