feat(unreachable-cooldown): 连接层不可达客户端级冷却,四家判定下沉共用
BACKLOG B24(窗长 300s 已拍板):连续 2 次连接未建立即对该 host 进 冷却窗,窗内调用零探针快速失败(归因文案与调用内短路逐字一致);到期 后下一条真实调用兼作探针,失败重进、成功恢复。isConnectFailure 四处 副本(BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient/ TaskOwnerForwardService)下沉 common/retry 共用。三家客户端接入同一 冷却表(两个 LLM 客户端共用 ai.t8star.org 一格,16890 独立一格), 窗长可配(0=关闭退回调用内短路)。另修 CollectDataBrandBatchFilter 首批预算竞态(1ms 预算下 GC 停顿误切首批,照 BrandCheckClient 首轮 不掐契约补 start>0 守卫)。变异验证:冷却禁用 6 例判红;定向 47/47 绿。
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
package com.nanri.aiimage.common.retry;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* 连接层不可达的客户端级冷却表(按上游 host 分键,Spring 单例)。
|
||||
*
|
||||
* <p>背景(BACKLOG B24,2026-10-23 拍板):三家同型客户端({@code BrandCheckClient}/
|
||||
* {@code AppearancePatentLlmClient}/{@code SimilarAsinLlmClient})的连接层短路计数
|
||||
* 生命周期都只到单次调用,网关长断时每个新调用/分块仍要各自真发连接探针才知道"还是断的"。
|
||||
* 本表把「连续 2 次连接未建立」的证据跨调用累计,达到阈值即对该 host 进入冷却窗
|
||||
* (默认 300s,同仓两处先例:{@code pt_worker/proxy_pool.py} 与 16890 服务侧的钥匙冷却),
|
||||
* 窗内调用方不发请求直接快速失败;到期后下一条真实调用兼作探针,失败即重进、成功即恢复。
|
||||
*
|
||||
* <p>调用方契约:{@link #isCoolingDown} 在发请求前问一次;连接层失败记
|
||||
* {@link #recordConnectFailure}(返回 true 的那方打一条进入冷却的 warn);
|
||||
* 连接成功或"连上了没答好"(读超时/5xx/解析失败)记 {@link #recordConnected} 清零证据。
|
||||
* 窗长 {@code <= 0} 表示关闭冷却,全部方法退化为无操作——调用方退回各自的调用内短路行为。
|
||||
*/
|
||||
@Component
|
||||
public class UnreachableCooldown {
|
||||
|
||||
/**
|
||||
* 进入冷却的证据数:与三家客户端各自的调用内短路阈值同数字——一次 refused 可能是抖动,
|
||||
* 两次同形状且中间没有任何一次成功,才是服务在掉。
|
||||
*/
|
||||
private static final int EVIDENCE_THRESHOLD = 2;
|
||||
|
||||
private final ConcurrentHashMap<String, State> states = new ConcurrentHashMap<>();
|
||||
|
||||
private static final class State {
|
||||
final AtomicInteger evidence = new AtomicInteger();
|
||||
final AtomicLong cooldownUntilMillis = new AtomicLong();
|
||||
}
|
||||
|
||||
/**
|
||||
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
|
||||
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
|
||||
* 原 BrandCheckClient/AppearancePatentLlmClient/SimilarAsinLlmClient 各有一份同款,
|
||||
* B24 下沉至此,四家(含 TaskOwnerForwardService)共用同一判定。
|
||||
*/
|
||||
public static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 该 host 是否正处冷却窗内:true 时调用方不发请求,直接按既有不可达归因文案快速失败。 */
|
||||
public boolean isCoolingDown(String host, long windowMillis) {
|
||||
if (windowMillis <= 0) {
|
||||
return false;
|
||||
}
|
||||
State state = states.get(normalizeHost(host));
|
||||
return state != null && state.cooldownUntilMillis.get() > System.currentTimeMillis();
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接层失败证据 +1;累计到阈值即对该 host 开(或重开)冷却窗。
|
||||
*
|
||||
* @return true 仅当"本次调用让冷却窗从无到有"(并发下只会有一个调用方拿到 true),
|
||||
* 调用方据此打一条进入冷却的 warn,期中快失败不刷屏
|
||||
*/
|
||||
public boolean recordConnectFailure(String host, long windowMillis) {
|
||||
if (windowMillis <= 0) {
|
||||
return false;
|
||||
}
|
||||
State state = states.computeIfAbsent(normalizeHost(host), key -> new State());
|
||||
if (state.evidence.incrementAndGet() < EVIDENCE_THRESHOLD) {
|
||||
return false;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
long previous = state.cooldownUntilMillis.get();
|
||||
if (previous > now) {
|
||||
return false;
|
||||
}
|
||||
return state.cooldownUntilMillis.compareAndSet(previous, now + windowMillis);
|
||||
}
|
||||
|
||||
/** 连接成功或"连上了没答好":证据清零(已开的冷却窗不受影响,等它自然到期)。 */
|
||||
public void recordConnected(String host) {
|
||||
State state = states.get(normalizeHost(host));
|
||||
if (state != null) {
|
||||
state.evidence.set(0);
|
||||
}
|
||||
}
|
||||
|
||||
/** host 归一:两个 LLM 客户端配的同一网关必须落进同一格,别因尾斜杠/大小写裂成两份。 */
|
||||
static String normalizeHost(String host) {
|
||||
String normalized = host == null ? "" : host.trim().toLowerCase();
|
||||
return normalized.endsWith("/") ? normalized.substring(0, normalized.length() - 1) : normalized;
|
||||
}
|
||||
}
|
||||
+1
-13
@@ -19,7 +19,6 @@ import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.util.ContentCachingRequestWrapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.ConnectException;
|
||||
import java.util.Enumeration;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
@@ -89,7 +88,7 @@ public class TaskOwnerForwardService {
|
||||
.retrieve()
|
||||
.toEntity(byte[].class);
|
||||
} catch (ResourceAccessException accessError) {
|
||||
if (!isConnectFailure(accessError)) {
|
||||
if (!com.nanri.aiimage.common.retry.UnreachableCooldown.isConnectFailure(accessError)) {
|
||||
throw accessError;
|
||||
}
|
||||
lastError = accessError;
|
||||
@@ -103,17 +102,6 @@ public class TaskOwnerForwardService {
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
private static boolean isConnectFailure(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
if (cursor instanceof ConnectException) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void sleepQuietly(long millis) {
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
|
||||
@@ -46,6 +46,13 @@ public class AppearancePatentProperties {
|
||||
* 每行每个 LLM 请求的重试次数(含首次)
|
||||
*/
|
||||
private int llmRetryTimes = 3;
|
||||
|
||||
/**
|
||||
* 连接层不可达的客户端级冷却窗长(0=关闭,退回调用内短路)。连续 2 次连接未建立即进窗,
|
||||
* 窗内调用不发请求直接快速失败;到期后下一条真实调用兼作探针。拍板口径见 BACKLOG B24
|
||||
* (300s 同仓两处先例:pt_worker 代理池与 16890 服务侧的钥匙冷却)。
|
||||
*/
|
||||
private int llmUnreachableCooldownMillis = 300000;
|
||||
/**
|
||||
* 单行最多被「轮次」发起多少次(一轮 = 一次批量派发;每轮内部另有 llmRetryTimes 次 HTTP 重试)。
|
||||
* <=0 关闭本闸。
|
||||
|
||||
@@ -34,4 +34,11 @@ public class BrandCheckProperties {
|
||||
private int totalTimeoutMillis = 90000;
|
||||
private int connectTimeoutMillis = 10000;
|
||||
private int readTimeoutMillis = 60000;
|
||||
|
||||
/**
|
||||
* 连接层不可达的客户端级冷却窗长(0=关闭,退回批内短路)。连续 2 次连接未建立即进窗,
|
||||
* 窗内整批品牌不发请求直接按服务不可达判失败;到期后下一批的真实请求兼作探针。
|
||||
* 拍板口径见 BACKLOG B24(300s 同仓两处先例:pt_worker 代理池与 16890 服务侧的钥匙冷却)。
|
||||
*/
|
||||
private int unreachableCooldownMillis = 300000;
|
||||
}
|
||||
|
||||
@@ -197,6 +197,13 @@ public class SimilarAsinProperties {
|
||||
private int llmFirstAttemptReadTimeoutMillis = 300000;
|
||||
private int llmRetryTimes = 3;
|
||||
|
||||
/**
|
||||
* 连接层不可达的客户端级冷却窗长(0=关闭,退回调用内短路)。连续 2 次连接未建立即进窗,
|
||||
* 窗内调用不发请求直接快速失败;到期后下一条真实调用兼作探针。拍板口径见 BACKLOG B24
|
||||
* (300s 同仓两处先例:pt_worker 代理池与 16890 服务侧的钥匙冷却)。
|
||||
*/
|
||||
private int llmUnreachableCooldownMillis = 300000;
|
||||
|
||||
/** 批内行级并发上限:每行最多 2 次图片对比 + 1 次合规 + 3 次类目匹配。 */
|
||||
private int llmRowConcurrency = 5;
|
||||
|
||||
|
||||
+24
-15
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
|
||||
@@ -21,9 +22,6 @@ import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
@@ -158,6 +156,7 @@ public class AppearancePatentLlmClient {
|
||||
private final ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final BrandCheckClient brandCheckClient;
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
private final UnreachableCooldown unreachableCooldown;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
/** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */
|
||||
@@ -647,15 +646,30 @@ public class AppearancePatentLlmClient {
|
||||
String apiKey,
|
||||
String responseFormat) throws Exception {
|
||||
int attempts = Math.max(1, properties.getLlmRetryTimes());
|
||||
long cooldownMillis = Math.max(0, properties.getLlmUnreachableCooldownMillis());
|
||||
String host = properties.getLlmHost();
|
||||
if (unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
// 冷却期快失败不发请求、不记密钥次数(B24)
|
||||
log.debug("[appearance-patent] 客户端级冷却中,快速失败 model={} window={}ms", model, cooldownMillis);
|
||||
throw new IllegalStateException(REASON_LLM_UNREACHABLE + ";lastErr=冷却中(连续连接未建立,暂不打上游)");
|
||||
}
|
||||
Exception lastFailure = null;
|
||||
AtomicInteger connectFailures = new AtomicInteger();
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
return invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt);
|
||||
String content = invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
return content;
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
if (isConnectFailure(ex)) {
|
||||
if (connectFailures.incrementAndGet() >= LLM_CONNECT_FAILURE_EVIDENCE) {
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
|
||||
if (entered) {
|
||||
log.warn("[appearance-patent] 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}",
|
||||
LLM_CONNECT_FAILURE_EVIDENCE, cooldownMillis, host);
|
||||
}
|
||||
if (connectFailures.incrementAndGet() >= LLM_CONNECT_FAILURE_EVIDENCE
|
||||
|| unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
log.warn("[appearance-patent] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达,"
|
||||
+ "停止重试 attempt={} model={} err={}",
|
||||
LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex));
|
||||
@@ -667,6 +681,7 @@ public class AppearancePatentLlmClient {
|
||||
} else {
|
||||
// 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数
|
||||
connectFailures.set(0);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
}
|
||||
if (attempt >= attempts) {
|
||||
break;
|
||||
@@ -681,17 +696,11 @@ public class AppearancePatentLlmClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
|
||||
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
|
||||
* 连接层判定下沉 {@code UnreachableCooldown.isConnectFailure}(B24):判定口径与
|
||||
* BrandCheckClient/SimilarAsinLlmClient 共用一份,不再各留包内副本。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
return UnreachableCooldown.isConnectFailure(error);
|
||||
}
|
||||
|
||||
private String invokeChatOnce(String model,
|
||||
|
||||
+33
-8
@@ -77,16 +77,19 @@ public class BrandCheckClient {
|
||||
private final BrandCheckProperties properties;
|
||||
private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserApiSecretService userApiSecretService;
|
||||
private final com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown;
|
||||
/** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */
|
||||
private final ThreadLocal<AtomicBoolean> rejectedSubmission = new ThreadLocal<>();
|
||||
private volatile RestClient sharedRestClient;
|
||||
|
||||
public BrandCheckClient(BrandCheckProperties properties,
|
||||
com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics,
|
||||
UserApiSecretService userApiSecretService) {
|
||||
UserApiSecretService userApiSecretService,
|
||||
com.nanri.aiimage.common.retry.UnreachableCooldown unreachableCooldown) {
|
||||
this.properties = properties;
|
||||
this.externalCallMetrics = externalCallMetrics;
|
||||
this.userApiSecretService = userApiSecretService;
|
||||
this.unreachableCooldown = unreachableCooldown;
|
||||
}
|
||||
|
||||
// 有界队列 + 拒绝即降级:大批量(几千品牌)检查时队列洪峰不无界堆内存。
|
||||
@@ -267,6 +270,18 @@ public class BrandCheckClient {
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
long cooldownMillis = Math.max(0, properties.getUnreachableCooldownMillis());
|
||||
if (unreachableCooldown.isCoolingDown(properties.getBaseUrl(), cooldownMillis)) {
|
||||
// 客户端级冷却(B24):上一批已确认连接层不可达,本批连首轮探针都不发。
|
||||
// 结论与批内短路版逐字一致(每行仍判查询失败、reason 同文案),差别只是不再为
|
||||
// "确认还是断的"每批各烧一轮连接探针;到期后下一批的真实请求兼作探针。
|
||||
log.warn("[brand-check] SERVICE_UNREACHABLE 客户端级冷却中,整批不打上游 brands={} window={}ms",
|
||||
brands.size(), cooldownMillis);
|
||||
for (String brand : brands) {
|
||||
outcomes.put(brand, new BrandCheckOutcome(List.of(), List.of(brand), REASON_SERVICE_UNREACHABLE));
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
long firstAttemptDeadlineNanos = deadlineNanos == Long.MAX_VALUE
|
||||
? Long.MAX_VALUE
|
||||
: deadlineNanos + FIRST_ATTEMPT_QUEUE_GRACE_MILLIS * 1_000_000L;
|
||||
@@ -363,6 +378,7 @@ public class BrandCheckClient {
|
||||
lastFailure = ex;
|
||||
response = null;
|
||||
if (isConnectFailure(ex)) {
|
||||
recordConnectFailure(ex);
|
||||
if (connectFailures.incrementAndGet() >= CONNECT_FAILURE_EVIDENCE) {
|
||||
ladderCompleted = false;
|
||||
connectDown = true;
|
||||
@@ -374,6 +390,7 @@ public class BrandCheckClient {
|
||||
} else {
|
||||
// 连上了只是这次没答好(读超时/解析失败):连接层证据不作数
|
||||
connectFailures.set(0);
|
||||
unreachableCooldown.recordConnected(properties.getBaseUrl());
|
||||
}
|
||||
if (attempt < attempts) {
|
||||
long retryDelayMillis = retryDelayMillis(attempt);
|
||||
@@ -385,6 +402,7 @@ public class BrandCheckClient {
|
||||
}
|
||||
// 拿到响应=连接是通的,证据清零,别跨到下一次真断连时凑数
|
||||
connectFailures.set(0);
|
||||
unreachableCooldown.recordConnected(properties.getBaseUrl());
|
||||
if (response != null && nullToEmpty(response.getQueryFaildData()).isEmpty()) {
|
||||
// 本次查询完成且无失败项:命中侵权(faild_data)属有效结论,不重试。
|
||||
return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of(), "");
|
||||
@@ -554,14 +572,21 @@ public class BrandCheckClient {
|
||||
* <p>故意不含读超时与 5xx:那两种是"连上了但这次没答好",重试救得回来;把它们算进
|
||||
* 不可达证据,就会把一次上游抖动误判成服务在掉、把同批其余品牌的重试一起掐掉。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
/** 连接层失败记入客户端级冷却表:达到阈值的那方打一条进入冷却的 warn(B24)。 */
|
||||
private void recordConnectFailure(Exception ex) {
|
||||
long cooldownMillis = Math.max(0, properties.getUnreachableCooldownMillis());
|
||||
if (unreachableCooldown.recordConnectFailure(properties.getBaseUrl(), cooldownMillis)) {
|
||||
log.warn("[brand-check] SERVICE_UNREACHABLE 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}",
|
||||
CONNECT_FAILURE_EVIDENCE, cooldownMillis, properties.getBaseUrl());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接层判定下沉 {@code UnreachableCooldown.isConnectFailure}(B24):判定口径与
|
||||
* 两个 LLM 客户端共用一份,不再各留包内副本。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
return com.nanri.aiimage.common.retry.UnreachableCooldown.isConnectFailure(error);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+4
-1
@@ -118,7 +118,10 @@ public class CollectDataBrandBatchFilter {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (deadlineNanos != Long.MAX_VALUE && System.nanoTime() >= deadlineNanos) {
|
||||
// 首批不掐(与 BrandCheckClient 的 attempt>1 同契约):预算检查只拦后续批次。
|
||||
// 对首批也查的话,deadline 计算与检查之间一次 GC/调度停顿(毫秒级)就会把首批
|
||||
// 整组误切成 queryFailed——1ms 预算的测试已实测抖出过这个形状。
|
||||
if (start > 0 && deadlineNanos != Long.MAX_VALUE && System.nanoTime() >= deadlineNanos) {
|
||||
// 预算用尽:不再为后面的批次捶上游。已缓存的品牌仍按缓存结论归类(不花代价),
|
||||
// 没结论的行归 queryFailed 入补测队列——既不按 accepted 放行(可能正是侵权品牌),
|
||||
// 也不按 rejected 拉黑(没查过就不该写进无效品牌表,否则永久误杀)。
|
||||
|
||||
+25
-21
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.similarasin.client;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
|
||||
@@ -17,9 +18,6 @@ import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
@@ -52,6 +50,7 @@ public class SimilarAsinLlmClient {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
private final UnreachableCooldown unreachableCooldown;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
/** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */
|
||||
@@ -60,11 +59,13 @@ public class SimilarAsinLlmClient {
|
||||
public SimilarAsinLlmClient(SimilarAsinProperties properties,
|
||||
ObjectMapper objectMapper,
|
||||
ExternalCallMetricsRecorder externalCallMetrics,
|
||||
UserSecretUsageService userSecretUsageService) {
|
||||
UserSecretUsageService userSecretUsageService,
|
||||
UnreachableCooldown unreachableCooldown) {
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
this.externalCallMetrics = externalCallMetrics;
|
||||
this.userSecretUsageService = userSecretUsageService;
|
||||
this.unreachableCooldown = unreachableCooldown;
|
||||
}
|
||||
|
||||
/** 文本对话,json_object 输出。 */
|
||||
@@ -85,16 +86,32 @@ public class SimilarAsinLlmClient {
|
||||
String responseFormat) {
|
||||
String resolvedKey = resolveApiKey(apiKey);
|
||||
int attempts = Math.max(1, properties.getLlmRetryTimes());
|
||||
long cooldownMillis = Math.max(0, properties.getLlmUnreachableCooldownMillis());
|
||||
String host = properties.getLlmHost();
|
||||
if (unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
// 冷却期快失败不发请求、不记密钥次数:结论与短路版一致(调用方拿不到 LLM 结论),
|
||||
// 差别只是不再为"确认还是断的"各烧一轮连接探针(B24)
|
||||
log.debug("[similar-asin][llm] 客户端级冷却中,快速失败 model={} window={}ms", model, cooldownMillis);
|
||||
throw new IllegalStateException(REASON_LLM_UNREACHABLE + ";lastErr=冷却中(连续连接未建立,暂不打上游)");
|
||||
}
|
||||
Exception lastFailure = null;
|
||||
int connectFailures = 0;
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
return invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
|
||||
String content = invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
|
||||
unreachableCooldown.recordConnected(host);
|
||||
return content;
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
if (isConnectFailure(ex)) {
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
connectFailures++;
|
||||
if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE) {
|
||||
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
|
||||
if (entered) {
|
||||
log.warn("[similar-asin][llm] 连接层证据累计 {} 次,进入客户端级冷却 {}ms host={}",
|
||||
2, cooldownMillis, host);
|
||||
}
|
||||
if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE
|
||||
|| unreachableCooldown.isCoolingDown(host, cooldownMillis)) {
|
||||
log.warn("[similar-asin][llm] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达,"
|
||||
+ "停止重试 attempt={} model={} err={}",
|
||||
LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex));
|
||||
@@ -106,6 +123,7 @@ public class SimilarAsinLlmClient {
|
||||
} else {
|
||||
// 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数
|
||||
connectFailures = 0;
|
||||
unreachableCooldown.recordConnected(host);
|
||||
}
|
||||
if (attempt >= attempts) {
|
||||
break;
|
||||
@@ -123,20 +141,6 @@ public class SimilarAsinLlmClient {
|
||||
: new IllegalStateException(lastFailure.getMessage(), lastFailure);
|
||||
}
|
||||
|
||||
/**
|
||||
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
|
||||
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
|
||||
*/
|
||||
static boolean isConnectFailure(Throwable error) {
|
||||
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
|
||||
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|
||||
|| t instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 只认任务/用户自己的 api_key(payload → 按 uid 的服务端密钥表已在 readApiKey 兜过);
|
||||
* 为空即空、跳过 LLM,绝不回落平台公费密钥 —— 各用户的任务只能烧各自的密钥。 */
|
||||
public String resolveApiKey(String apiKey) {
|
||||
|
||||
@@ -378,6 +378,9 @@ aiimage:
|
||||
total-timeout-millis: ${AIIMAGE_BRAND_CHECK_TOTAL_TIMEOUT_MILLIS:90000}
|
||||
connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000}
|
||||
read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000}
|
||||
# 连接层不可达的客户端级冷却窗长(B24,0=关闭):连续 2 次连接未建立即进窗,
|
||||
# 窗内整批品牌不发请求直接按服务不可达判失败;到期后下一批的真实请求兼作探针。
|
||||
unreachable-cooldown-millis: ${AIIMAGE_BRAND_CHECK_UNREACHABLE_COOLDOWN_MILLIS:300000}
|
||||
appearance-patent:
|
||||
no-result-upload-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_NO_RESULT_UPLOAD_TIMEOUT_MINUTES:180}
|
||||
llm-host: ${AIIMAGE_APPEARANCE_PATENT_LLM_HOST:https://ai.t8star.org}
|
||||
@@ -402,6 +405,9 @@ aiimage:
|
||||
# 实测空行 20.7%~51.3%,全部放行且零告警,因为旧分母按解析行标题判、用户表不带标题列时分母为 0)。
|
||||
llm-coverage-min-ratio: ${AIIMAGE_APPEARANCE_PATENT_LLM_COVERAGE_MIN_RATIO:0.95}
|
||||
llm-retry-times: ${AIIMAGE_APPEARANCE_PATENT_LLM_RETRY_TIMES:3}
|
||||
# 连接层不可达的客户端级冷却窗长(B24,0=关闭):连续 2 次连接未建立即进窗,
|
||||
# 窗内调用不发请求直接按服务不可达快速失败;到期后下一条真实调用兼作探针。
|
||||
llm-unreachable-cooldown-millis: ${AIIMAGE_APPEARANCE_PATENT_LLM_UNREACHABLE_COOLDOWN_MILLIS:300000}
|
||||
# 单行最多被重发几轮(<=0 关闭)。失败行会被释放、下一轮重新申领,没有这道闸时
|
||||
# 上游劣化窗口内会无限重发并每轮扣用户 LLM 次数——deferRunning 不累加 retryCount,任务层那道永不触发。
|
||||
llm-row-max-rounds: ${AIIMAGE_APPEARANCE_PATENT_LLM_ROW_MAX_ROUNDS:3}
|
||||
@@ -440,6 +446,9 @@ aiimage:
|
||||
llm-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_READ_TIMEOUT_MILLIS:600000}
|
||||
llm-first-attempt-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_FIRST_ATTEMPT_READ_TIMEOUT_MILLIS:300000}
|
||||
llm-retry-times: ${AIIMAGE_SIMILAR_ASIN_LLM_RETRY_TIMES:3}
|
||||
# 连接层不可达的客户端级冷却窗长(B24,0=关闭):连续 2 次连接未建立即进窗,
|
||||
# 窗内调用不发请求直接按服务不可达快速失败;到期后下一条真实调用兼作探针。
|
||||
llm-unreachable-cooldown-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_UNREACHABLE_COOLDOWN_MILLIS:300000}
|
||||
llm-row-concurrency: ${AIIMAGE_SIMILAR_ASIN_LLM_ROW_CONCURRENCY:5}
|
||||
llm-category-batch-size: ${AIIMAGE_SIMILAR_ASIN_LLM_CATEGORY_BATCH_SIZE:10}
|
||||
# 合规判定一次请求合并的行数:省往返不省判定,每件商品仍各自出 is_conform/reason/category。
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.nanri.aiimage.common.retry;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.net.ConnectException;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* 连接层不可达冷却表的口径(B24):阈值 2、证据可清零、窗长到点自然到期、
|
||||
* 到期后一次失败即重进、窗长 0 整体关闭、同 host 不同写法必须同格。
|
||||
*/
|
||||
class UnreachableCooldownTest {
|
||||
|
||||
private final UnreachableCooldown cooldown = new UnreachableCooldown();
|
||||
private static final long WINDOW = 60_000L;
|
||||
|
||||
@Test
|
||||
void twoConsecutiveFailuresEnterCooldownOneDoesNot() {
|
||||
assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isFalse();
|
||||
assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isFalse();
|
||||
assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isFalse();
|
||||
// 第二次失败:冷却窗从无到有,返回 true 的那方打进入 warn;之后的失败不重复报
|
||||
assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isTrue();
|
||||
assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isTrue();
|
||||
assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void connectedResetsEvidenceSoTwoFreshFailuresAreRequiredAgain() {
|
||||
cooldown.recordConnectFailure("https://gw.example.com", WINDOW);
|
||||
cooldown.recordConnected("https://gw.example.com");
|
||||
assertThat(cooldown.recordConnectFailure("https://gw.example.com", WINDOW)).isFalse();
|
||||
assertThat(cooldown.isCoolingDown("https://gw.example.com", WINDOW)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void windowExpiresNaturallyAndSingleFailureReenters() throws InterruptedException {
|
||||
String host = "https://short.example.com";
|
||||
cooldown.recordConnectFailure(host, 50);
|
||||
cooldown.recordConnectFailure(host, 50);
|
||||
assertThat(cooldown.isCoolingDown(host, 50)).isTrue();
|
||||
|
||||
Thread.sleep(120);
|
||||
assertThat(cooldown.isCoolingDown(host, 50)).isFalse();
|
||||
|
||||
// 历史证据不清零:到期后探针再失败,一次即重进(B24 拍板口径)
|
||||
assertThat(cooldown.recordConnectFailure(host, 50)).isTrue();
|
||||
assertThat(cooldown.isCoolingDown(host, 50)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void zeroWindowDisablesCooldownEntirely() {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
assertThat(cooldown.recordConnectFailure("https://gw.example.com", 0)).isFalse();
|
||||
}
|
||||
assertThat(cooldown.isCoolingDown("https://gw.example.com", 0)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void sameHostDifferentSpellingSharesOneCell() {
|
||||
cooldown.recordConnectFailure("https://GW.example.com", WINDOW);
|
||||
// 尾斜杠/大小写不同写法落进同一格:这条是第 2 次失败,必须进冷却
|
||||
assertThat(cooldown.recordConnectFailure("https://gw.example.com/", WINDOW)).isTrue();
|
||||
assertThat(cooldown.isCoolingDown("https://Gw.Example.Com", WINDOW)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isConnectFailureWalksCauseChainAndIgnoresOtherFailures() {
|
||||
assertThat(UnreachableCooldown.isConnectFailure(new ConnectException("refused"))).isTrue();
|
||||
RuntimeException wrapped = new RuntimeException(
|
||||
new IllegalStateException(new ConnectException("refused")));
|
||||
assertThat(UnreachableCooldown.isConnectFailure(wrapped)).isTrue();
|
||||
assertThat(UnreachableCooldown.isConnectFailure(new RuntimeException("read timeout"))).isFalse();
|
||||
assertThat(UnreachableCooldown.isConnectFailure(null)).isFalse();
|
||||
}
|
||||
}
|
||||
+8
-7
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
|
||||
import com.nanri.aiimage.modules.brand.client.BrandCheckClient.BrandCheckBatchResult;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
||||
@@ -96,7 +97,7 @@ class HttpClientConnectionReuseTest {
|
||||
void test_task_077_brand_normal_default_path() throws Exception {
|
||||
// 默认路径:品牌检查客户端通过共享池创建单例 RestClient,
|
||||
// 工厂为带 keep-alive 连接池的 JdkClientHttpRequestFactory。
|
||||
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null);
|
||||
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown());
|
||||
BrandCheckBatchResult result = client.checkTitleText(" ", null, null);
|
||||
|
||||
assertTrue(result.brands().isEmpty(), "空标题安全跳过,不创建无效资源");
|
||||
@@ -107,8 +108,8 @@ class HttpClientConnectionReuseTest {
|
||||
void test_task_077_brand_normal_multiple_items() throws Exception {
|
||||
// 批量场景:LLM/品牌/紫鸟三个客户端各自持有独立 RestClient,
|
||||
// 但底层连接池共用同一 HttpClient 实例,不重复创建。
|
||||
SimilarAsinLlmClient llm = new SimilarAsinLlmClient(new SimilarAsinProperties(), new ObjectMapper(), null, null);
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null);
|
||||
SimilarAsinLlmClient llm = new SimilarAsinLlmClient(new SimilarAsinProperties(), new ObjectMapper(), null, null, new UnreachableCooldown());
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown());
|
||||
ZiniaoClientImpl ziniao = new ZiniaoClientImpl(new ZiniaoProperties(), new ObjectMapper());
|
||||
|
||||
HttpClient llmClient = clientOf(factoryOf(restClientOf(llm)));
|
||||
@@ -124,7 +125,7 @@ class HttpClientConnectionReuseTest {
|
||||
void test_task_077_brand_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 幂等:同一客户端重复触发请求创建逻辑只产生一个 RestClient,
|
||||
// 重复调用返回同一实例,不重复创建客户端对象。
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null);
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown());
|
||||
assertSame(restClientOf(brand), restClientOf(brand), "品牌客户端复用同一 RestClient");
|
||||
|
||||
ZiniaoClientImpl ziniao = new ZiniaoClientImpl(new ZiniaoProperties(), new ObjectMapper());
|
||||
@@ -134,7 +135,7 @@ class HttpClientConnectionReuseTest {
|
||||
@Test
|
||||
void test_task_077_brand_boundary_empty_input() throws Exception {
|
||||
// 空输入:空品牌列表不发起任何 HTTP 请求、不创建客户端资源。
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null);
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown());
|
||||
BrandCheckBatchResult result = brand.checkAll(List.of(), "Terms", null);
|
||||
|
||||
assertTrue(result.brands().isEmpty());
|
||||
@@ -145,7 +146,7 @@ class HttpClientConnectionReuseTest {
|
||||
@Test
|
||||
void test_task_077_brand_boundary_single_item() throws Exception {
|
||||
// 单元素:单客户端单请求走共享池,工厂带连接池,行为与批量一致。
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null);
|
||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown());
|
||||
assertPooled(factoryOf(restClientOf(brand)));
|
||||
}
|
||||
|
||||
@@ -155,7 +156,7 @@ class HttpClientConnectionReuseTest {
|
||||
// 不随实例数量线性增长连接资源。
|
||||
int instances = 8;
|
||||
for (int i = 0; i < instances; i++) {
|
||||
restClientOf(new BrandCheckClient(new BrandCheckProperties(), null, null));
|
||||
restClientOf(new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()));
|
||||
restClientOf(new ZiniaoClientImpl(new ZiniaoProperties(), new ObjectMapper()));
|
||||
}
|
||||
assertSame(HttpClientPool.sharedHttpClient(), HttpClientPool.sharedHttpClient(),
|
||||
|
||||
+10
-9
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.metrics;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.config.ZiniaoProperties;
|
||||
@@ -162,7 +163,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
SimilarAsinProperties props = llmProps();
|
||||
SimilarAsinLlmClient client =
|
||||
new SimilarAsinLlmClient(props, objectMapper, new ExternalCallMetricsRecorder(registry),
|
||||
mock(UserSecretUsageService.class));
|
||||
mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
|
||||
String content = client.invokeChat("test-model", "system", "hello", "test-key");
|
||||
|
||||
@@ -181,7 +182,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
SimilarAsinProperties props = llmProps();
|
||||
SimilarAsinLlmClient client =
|
||||
new SimilarAsinLlmClient(props, objectMapper, new ExternalCallMetricsRecorder(registry),
|
||||
mock(UserSecretUsageService.class));
|
||||
mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
client.invokeChat("test-model", "system", "prompt-" + i, "test-key");
|
||||
@@ -192,7 +193,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
"client", "llm", "result", "success"), "三次成功全部记录");
|
||||
|
||||
// 全部外部客户端(LLM / 品牌检查 / 紫鸟)在同一次批量中各自记录指标
|
||||
BrandCheckClient brand = new BrandCheckClient(brandProps(), new ExternalCallMetricsRecorder(registry), stubbedUserSecrets());
|
||||
BrandCheckClient brand = new BrandCheckClient(brandProps(), new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown());
|
||||
brand.checkAll(List.of("Nintendo", "LEGO", "Sony"), "Terms", 7L);
|
||||
ZiniaoClientImpl ziniao = new ZiniaoClientImpl(ziniaoProps(), objectMapper, new ExternalCallMetricsRecorder(registry));
|
||||
assertEquals(1001L, ziniao.getCompanyIdByApiKey("test-api-key"));
|
||||
@@ -210,7 +211,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
@Test
|
||||
void test_task_078_payload_metrics_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
BrandCheckProperties props = brandProps();
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets());
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown());
|
||||
|
||||
client.check("Nintendo", "Terms", 7L);
|
||||
client.check("Nintendo", "Terms", 7L);
|
||||
@@ -227,7 +228,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
@Test
|
||||
void test_task_078_payload_metrics_boundary_empty_input() throws Exception {
|
||||
BrandCheckProperties props = brandProps();
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets());
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown());
|
||||
|
||||
BrandCheckClient.BrandCheckBatchResult result = client.checkAll(List.of(), "Terms", 7L);
|
||||
|
||||
@@ -242,7 +243,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
@Test
|
||||
void test_task_078_payload_metrics_boundary_single_item() throws Exception {
|
||||
BrandCheckProperties props = brandProps();
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets());
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown());
|
||||
|
||||
BrandCheckClient.BrandCheckResponse response = client.check("LEGO", "Terms", 7L);
|
||||
|
||||
@@ -264,7 +265,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
int index = i;
|
||||
pool.submit(() -> {
|
||||
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, recorder,
|
||||
mock(UserSecretUsageService.class));
|
||||
mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
try {
|
||||
client.invokeChat("test-model", "system", "prompt-" + index, "test-key");
|
||||
} catch (Exception ignored) {
|
||||
@@ -297,7 +298,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
@Test
|
||||
void test_task_078_payload_metrics_invalid_input_rejected() throws Exception {
|
||||
BrandCheckProperties props = brandProps();
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets());
|
||||
BrandCheckClient client = new BrandCheckClient(props, new ExternalCallMetricsRecorder(registry), stubbedUserSecrets(), new UnreachableCooldown());
|
||||
|
||||
BrandCheckClient.BrandCheckBatchResult result = client.checkAll(null, "Terms", 7L);
|
||||
|
||||
@@ -315,7 +316,7 @@ class ExternalCallMetricsRecorderTest {
|
||||
SimilarAsinProperties props = llmProps();
|
||||
SimilarAsinLlmClient client =
|
||||
new SimilarAsinLlmClient(props, objectMapper, recorder,
|
||||
mock(UserSecretUsageService.class));
|
||||
mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
|
||||
// 第一次调用走 500 失败路径,第二次调用恢复成功:错误可恢复
|
||||
llmFailNext.set(true);
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
|
||||
@@ -46,7 +47,7 @@ class AppearancePatentBrandBatchTest {
|
||||
null,
|
||||
brandCheckClient,
|
||||
mock(UserSecretUsageService.class)
|
||||
);
|
||||
, new UnreachableCooldown());
|
||||
|
||||
@Test
|
||||
void mergedBatchAsksBrandCheckOnceAndAttributesPerRow() throws Exception {
|
||||
@@ -115,7 +116,7 @@ class AppearancePatentBrandBatchTest {
|
||||
properties.setLlmRetryTimes(1);
|
||||
properties.setLlmReadTimeoutMillis(5_000);
|
||||
AppearancePatentLlmClient inspectClient = new AppearancePatentLlmClient(
|
||||
properties, new ObjectMapper(), null, brandCheckClient, mock(UserSecretUsageService.class));
|
||||
properties, new ObjectMapper(), null, brandCheckClient, mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
|
||||
List<AppearancePatentResultRowDto> rows = inspectClient.inspectRows(
|
||||
List.of(
|
||||
|
||||
+4
-3
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
@@ -72,7 +73,7 @@ class AppearancePatentLlmClientHttpTest {
|
||||
null,
|
||||
localBrandCheckClient(),
|
||||
mock(UserSecretUsageService.class)
|
||||
);
|
||||
, new UnreachableCooldown());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -85,7 +86,7 @@ class AppearancePatentLlmClientHttpTest {
|
||||
brand.setRetryTimes(1);
|
||||
brand.setRetryIntervalMillis(0);
|
||||
brand.setTotalTimeoutMillis(5000);
|
||||
return new BrandCheckClient(brand, null, null);
|
||||
return new BrandCheckClient(brand, null, null, new UnreachableCooldown());
|
||||
}
|
||||
|
||||
private void handleBrandCheck(HttpExchange exchange) throws IOException {
|
||||
@@ -290,7 +291,7 @@ class AppearancePatentLlmClientHttpTest {
|
||||
properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
customizer.accept(properties);
|
||||
return new AppearancePatentLlmClient(properties, objectMapper, null,
|
||||
localBrandCheckClient(), mock(UserSecretUsageService.class));
|
||||
localBrandCheckClient(), mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
}
|
||||
|
||||
/** 连接层故障短路:网关根本连不上时,重试阶梯必须在证据数处停下,行级拿到可读的真因。 */
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
@@ -68,7 +69,7 @@ class AppearancePatentLlmClientImageDedupeTest {
|
||||
properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
properties.setLlmRetryTimes(1);
|
||||
client = new AppearancePatentLlmClient(properties, objectMapper, null, localBrandCheckClient(),
|
||||
mock(UserSecretUsageService.class));
|
||||
mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
}
|
||||
|
||||
private BrandCheckClient localBrandCheckClient() {
|
||||
@@ -77,7 +78,7 @@ class AppearancePatentLlmClientImageDedupeTest {
|
||||
brand.setRetryTimes(1);
|
||||
brand.setRetryIntervalMillis(0);
|
||||
brand.setTotalTimeoutMillis(5000);
|
||||
return new BrandCheckClient(brand, null, null);
|
||||
return new BrandCheckClient(brand, null, null, new UnreachableCooldown());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
|
||||
+8
-7
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
|
||||
@@ -26,9 +27,9 @@ class AppearancePatentLlmClientTest {
|
||||
new AppearancePatentProperties(),
|
||||
new ObjectMapper(),
|
||||
null,
|
||||
new BrandCheckClient(new BrandCheckProperties(), null, null),
|
||||
new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown()),
|
||||
mock(UserSecretUsageService.class)
|
||||
);
|
||||
, new UnreachableCooldown());
|
||||
|
||||
@Test
|
||||
void markRowsFailedLeavesUserFacingResultFilledWithReviewMessage() {
|
||||
@@ -123,7 +124,7 @@ class AppearancePatentLlmClientTest {
|
||||
null,
|
||||
brandCheckClient,
|
||||
mock(UserSecretUsageService.class)
|
||||
);
|
||||
, new UnreachableCooldown());
|
||||
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||
row.setId("1");
|
||||
String dataText = """
|
||||
@@ -140,7 +141,7 @@ class AppearancePatentLlmClientTest {
|
||||
|
||||
private static AppearancePatentLlmClient clientWith(BrandCheckClient brandCheckClient) {
|
||||
return new AppearancePatentLlmClient(new AppearancePatentProperties(), new ObjectMapper(),
|
||||
null, brandCheckClient, mock(UserSecretUsageService.class));
|
||||
null, brandCheckClient, mock(UserSecretUsageService.class), new UnreachableCooldown());
|
||||
}
|
||||
|
||||
private static AppearancePatentResultRowDto newRow(String id) {
|
||||
@@ -204,7 +205,7 @@ class AppearancePatentLlmClientTest {
|
||||
null,
|
||||
brandCheckClient,
|
||||
mock(UserSecretUsageService.class)
|
||||
);
|
||||
, new UnreachableCooldown());
|
||||
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||
row.setId("1");
|
||||
String dataText = """
|
||||
@@ -228,7 +229,7 @@ class AppearancePatentLlmClientTest {
|
||||
null,
|
||||
brandCheckClient,
|
||||
mock(UserSecretUsageService.class)
|
||||
);
|
||||
, new UnreachableCooldown());
|
||||
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||
row.setId("1");
|
||||
String dataText = """
|
||||
@@ -251,7 +252,7 @@ class AppearancePatentLlmClientTest {
|
||||
null,
|
||||
brandCheckClient,
|
||||
mock(UserSecretUsageService.class)
|
||||
);
|
||||
, new UnreachableCooldown());
|
||||
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||
row.setId("1");
|
||||
String dataText = """
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
@@ -59,7 +60,7 @@ class AppearancePatentLlmClientTokenUsageTest {
|
||||
AppearancePatentProperties properties = new AppearancePatentProperties();
|
||||
properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
properties.setLlmRetryTimes(1);
|
||||
client = new AppearancePatentLlmClient(properties, objectMapper, null, localBrandCheckClient(), usageService);
|
||||
client = new AppearancePatentLlmClient(properties, objectMapper, null, localBrandCheckClient(), usageService, new UnreachableCooldown());
|
||||
}
|
||||
|
||||
private BrandCheckClient localBrandCheckClient() {
|
||||
@@ -68,7 +69,7 @@ class AppearancePatentLlmClientTokenUsageTest {
|
||||
brand.setRetryTimes(1);
|
||||
brand.setRetryIntervalMillis(0);
|
||||
brand.setTotalTimeoutMillis(5000);
|
||||
return new BrandCheckClient(brand, null, null);
|
||||
return new BrandCheckClient(brand, null, null, new UnreachableCooldown());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
|
||||
+37
-6
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.brand.client;
|
||||
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -34,7 +35,7 @@ class BrandCheckClientTest {
|
||||
|
||||
@Test
|
||||
void splitTitleTextSupportsCommonSeparatorsAndQuotes() {
|
||||
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null);
|
||||
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown());
|
||||
|
||||
List<String> brands = client.splitTitleText("'阿凡达,任天堂'; Disney、LEGO\nSony");
|
||||
|
||||
@@ -43,13 +44,43 @@ class BrandCheckClientTest {
|
||||
|
||||
@Test
|
||||
void splitTitleTextDeduplicatesBlankValues() {
|
||||
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null);
|
||||
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties(), null, null, new UnreachableCooldown());
|
||||
|
||||
List<String> brands = client.splitTitleText(" 任天堂, ,任天堂,Sony ");
|
||||
|
||||
assertThat(brands).containsExactly("任天堂", "Sony");
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientLevelCooldownFailsWholeBatchWithoutHittingUpstream() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
HttpServer server = startBrandCheckServer(exchange -> {
|
||||
attempts.incrementAndGet();
|
||||
respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[]}");
|
||||
});
|
||||
try {
|
||||
UnreachableCooldown cooldown = new UnreachableCooldown();
|
||||
BrandCheckProperties props = new BrandCheckProperties();
|
||||
props.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
props.setPath("/brand_check");
|
||||
props.setRetryTimes(3);
|
||||
props.setRetryIntervalMillis(1);
|
||||
BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets(), cooldown);
|
||||
|
||||
cooldown.recordConnectFailure(props.getBaseUrl(), 300_000L);
|
||||
cooldown.recordConnectFailure(props.getBaseUrl(), 300_000L);
|
||||
|
||||
BrandCheckClient.BrandCheckBatchResult result = client.checkAll(List.of("Apple", "Nike"), "Terms", 7L);
|
||||
|
||||
assertThat(attempts.get()).as("冷却期内一个请求都不该发").isZero();
|
||||
assertThat(result.hasQueryFailedData()).isTrue();
|
||||
assertThat(result.queryFaildData()).containsExactly("Apple", "Nike");
|
||||
assertThat(result.reason()).contains("品牌检测服务不可达(连接未建立,非本行数据问题)");
|
||||
} finally {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryFailureRetriesUntilSuccess() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
@@ -242,7 +273,7 @@ class BrandCheckClientTest {
|
||||
props.setRetryTimes(retryTimes);
|
||||
props.setRetryIntervalMillis(retryIntervalMillis);
|
||||
props.setTotalTimeoutMillis(totalTimeoutMillis);
|
||||
return new BrandCheckClient(props, null, stubbedUserSecrets());
|
||||
return new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown());
|
||||
}
|
||||
|
||||
private BrandCheckClient newClient(HttpServer server, int retryTimes) {
|
||||
@@ -251,7 +282,7 @@ class BrandCheckClientTest {
|
||||
props.setPath("/brand_check");
|
||||
props.setRetryTimes(retryTimes);
|
||||
props.setRetryIntervalMillis(1);
|
||||
return new BrandCheckClient(props, null, stubbedUserSecrets());
|
||||
return new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown());
|
||||
}
|
||||
|
||||
/** 重试阶梯可调的构造:用于验证「耗时预算掐的是重试,不是首轮」。 */
|
||||
@@ -262,7 +293,7 @@ class BrandCheckClientTest {
|
||||
props.setRetryTimes(retryTimes);
|
||||
props.setRetryIntervalMillis(retryIntervalMillis);
|
||||
props.setTotalTimeoutMillis(totalTimeoutMillis);
|
||||
return new BrandCheckClient(props, null, stubbedUserSecrets());
|
||||
return new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown());
|
||||
}
|
||||
|
||||
/** 有真实 HTTP 断言的用例必须桩出非空代理,否则客户端零请求短路、阶梯与指标全测不到。 */
|
||||
@@ -361,7 +392,7 @@ class BrandCheckClientTest {
|
||||
props.setRetryIntervalMillis(1);
|
||||
props.setReadTimeoutMillis(1_000);
|
||||
props.setTotalTimeoutMillis(0);
|
||||
BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets());
|
||||
BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets(), new UnreachableCooldown());
|
||||
|
||||
ExecutorService pool = (ExecutorService) ReflectionTestUtils.getField(client, "checkExecutor");
|
||||
CountDownLatch occupied = new CountDownLatch(3);
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.brand.client;
|
||||
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -137,7 +138,7 @@ class BrandCheckClientTitleBatchTest {
|
||||
props.setRetryIntervalMillis(1);
|
||||
UserApiSecretService secrets = mock(UserApiSecretService.class);
|
||||
when(secrets.findPlainValue(anyLong(), anyString())).thenReturn("http://user:pass@127.0.0.1:9");
|
||||
return new BrandCheckClient(props, null, secrets);
|
||||
return new BrandCheckClient(props, null, secrets, new UnreachableCooldown());
|
||||
}
|
||||
|
||||
private HttpServer startBrandCheckServer(com.sun.net.httpserver.HttpHandler handler) throws IOException {
|
||||
|
||||
+49
-1
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.similarasin.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserSecretUsageService;
|
||||
import com.sun.net.httpserver.HttpExchange;
|
||||
@@ -121,10 +122,57 @@ class SimilarAsinLlmClientHttpTest {
|
||||
assertThat(requestCount.get()).isEqualTo(2);
|
||||
}
|
||||
|
||||
/** B24:首次调用确认不可达即进客户端级冷却,冷却期内的后续调用零探针快速失败。 */
|
||||
@Test
|
||||
void cooldownMakesSubsequentCallsFailFastWithoutProbes() {
|
||||
SimilarAsinLlmClient deadHostClient = buildClient(p -> p.setLlmHost("http://127.0.0.1:1"));
|
||||
|
||||
assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key"))
|
||||
.hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)");
|
||||
assertThat(lastCooldown.isCoolingDown(lastHost, lastWindow)).isTrue();
|
||||
|
||||
// 快速失败走的是入口冷却分支(";lastErr=冷却中"后缀只有该路径会拼),一次探针都不发
|
||||
assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key"))
|
||||
.hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)")
|
||||
.hasMessageContaining(";lastErr=冷却中");
|
||||
}
|
||||
|
||||
/** B24:冷却到期后下一条真实调用兼作探针;探针仍连不上 ⇒ 立即重进冷却。 */
|
||||
@Test
|
||||
void expiredCooldownProbesAgainAndReentersOnFailure() throws InterruptedException {
|
||||
SimilarAsinLlmClient deadHostClient = buildClient(p -> {
|
||||
p.setLlmHost("http://127.0.0.1:1");
|
||||
p.setLlmUnreachableCooldownMillis(200);
|
||||
});
|
||||
assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key"))
|
||||
.hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)");
|
||||
|
||||
Thread.sleep(400);
|
||||
assertThat(lastCooldown.isCoolingDown(lastHost, 200)).isFalse();
|
||||
|
||||
// 探针真发:lastErr 带的是连接异常而不是"冷却中";失败即重进
|
||||
assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key"))
|
||||
.hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)")
|
||||
.hasMessageNotContaining("冷却中");
|
||||
assertThat(lastCooldown.isCoolingDown(lastHost, 200)).isTrue();
|
||||
|
||||
// 重进之后的调用回到零探针快速失败
|
||||
assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key"))
|
||||
.hasMessageContaining(";lastErr=冷却中");
|
||||
}
|
||||
|
||||
private SimilarAsinLlmClient buildClient(java.util.function.Consumer<SimilarAsinProperties> customizer) {
|
||||
SimilarAsinProperties properties = new SimilarAsinProperties();
|
||||
properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
customizer.accept(properties);
|
||||
return new SimilarAsinLlmClient(properties, objectMapper, null, mock(UserSecretUsageService.class));
|
||||
lastCooldown = new UnreachableCooldown();
|
||||
lastHost = properties.getLlmHost();
|
||||
lastWindow = Math.max(0, properties.getLlmUnreachableCooldownMillis());
|
||||
return new SimilarAsinLlmClient(properties, objectMapper, null, mock(UserSecretUsageService.class), lastCooldown);
|
||||
}
|
||||
|
||||
/** 最近一次 buildClient 的冷却表与 host/窗长:冷却态断言直接读同一实例,不靠时间猜。 */
|
||||
private UnreachableCooldown lastCooldown;
|
||||
private String lastHost;
|
||||
private long lastWindow;
|
||||
}
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
@@ -61,7 +62,7 @@ public class SimilarAsinLlmLocalVerify {
|
||||
props.setLlmRowConcurrency(2);
|
||||
props.setLlmImageDownloadTimeoutSeconds(10);
|
||||
|
||||
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null, null);
|
||||
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null, null, new UnreachableCooldown());
|
||||
OssProperties ossProps = new OssProperties();
|
||||
ossProps.setEndpoint("https://oss.aishufu.top");
|
||||
ossProps.setPublicEndpoint("https://oss.aishufu.top");
|
||||
|
||||
Reference in New Issue
Block a user