fix(similar-asin): LLM 连接层故障短路,行级写明服务不可达真因

BACKLOG B23:网关不可达时 invokeChat 原本对任何异常跑满 3 次阶梯,
批量失败再逐行回退把白等乘上行数。现对齐 BrandCheckClient /
AppearancePatentLlmClient 同款口径:连续 2 次连接未建立(读超时与
5xx 不计入且清零证据)即停,抛「LLM 服务不可达(连接未建立,非本行
数据问题)」。变异验证 23.4s 判红→收口 7.7s 绿;后端整量 4709 例
0 红 0 错 BUILD SUCCESS。
This commit is contained in:
2026-10-23 19:47:46 +08:00
parent b65b709700
commit 6af19b51ba
2 changed files with 172 additions and 0 deletions
@@ -17,6 +17,9 @@ import org.springframework.web.client.RestClient;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
@@ -35,6 +38,15 @@ public class SimilarAsinLlmClient {
private static final MediaType APPLICATION_JSON_UTF8 = new MediaType("application", "json", StandardCharsets.UTF_8);
private static final int MAX_LLM_RESPONSE_BYTES = 16 * 1024 * 1024;
/**
* 连接层短路的证据数:同一次调用里连续这么多次「连接根本没建立」且中间没有一次成功,
* 就判定 LLM 网关不可达,立刻停掉重试阶梯。与 {@code AppearancePatentLlmClient}/
* {@code BrandCheckClient} 同口径(B19/B23)——连接层故障重试救不回来,只会把同一笔
* 失败摊满整段退避预算;本侧批量失败后还逐行回退,白等会再乘一个行数。
*/
private static final int LLM_CONNECT_FAILURE_EVIDENCE = 2;
/** 连接层判定不可达时交付的归因文案:说清是运维故障,不是这行数据的问题。 */
static final String REASON_LLM_UNREACHABLE = "LLM 服务不可达(连接未建立,非本行数据问题)";
private final SimilarAsinProperties properties;
private final ObjectMapper objectMapper;
@@ -74,11 +86,27 @@ public class SimilarAsinLlmClient {
String resolvedKey = resolveApiKey(apiKey);
int attempts = Math.max(1, properties.getLlmRetryTimes());
Exception lastFailure = null;
int connectFailures = 0;
for (int attempt = 1; attempt <= attempts; attempt++) {
try {
return invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
} catch (Exception ex) {
lastFailure = ex;
if (isConnectFailure(ex)) {
connectFailures++;
if (connectFailures >= LLM_CONNECT_FAILURE_EVIDENCE) {
log.warn("[similar-asin][llm] LLM_UNREACHABLE 连续 {} 次连接未建立,判定 LLM 服务不可达,"
+ "停止重试 attempt={} model={} err={}",
LLM_CONNECT_FAILURE_EVIDENCE, attempt, model, failureMessage(ex));
// 结论不变(这次调用拿不到 LLM 结论),差别只在耗时与归因:
// 调用方拿到的是"网关没连上",而不是一句裸的连接异常
throw new IllegalStateException(
REASON_LLM_UNREACHABLE + ";lastErr=" + failureMessage(ex), ex);
}
} else {
// 连上了只是这次没答好(读超时/解析失败/5xx):连接层证据不作数
connectFailures = 0;
}
if (attempt >= attempts) {
break;
}
@@ -95,6 +123,20 @@ public class SimilarAsinLlmClient {
: new IllegalStateException(lastFailure.getMessage(), lastFailure);
}
/**
* 只认连接层故障(顺着 cause 链找):连接被拒/无路由/主机名解不了。
* 读超时与 5xx 是"连上了没答好",故意不算——那类抖动重试是有意义的。
*/
static boolean isConnectFailure(Throwable error) {
for (Throwable t = error; t != null && t != t.getCause(); t = t.getCause()) {
if (t instanceof ConnectException || t instanceof NoRouteToHostException
|| t instanceof UnknownHostException) {
return true;
}
}
return false;
}
/** 只认任务/用户自己的 api_key(payload → 按 uid 的服务端密钥表已在 readApiKey 兜过);
* 为空即空、跳过 LLM,绝不回落平台公费密钥 —— 各用户的任务只能烧各自的密钥。 */
public String resolveApiKey(String apiKey) {
@@ -0,0 +1,130 @@
package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.usersecret.service.UserSecretUsageService;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
/**
* 连接层短路口径(B23):网关根本连不上时,重试阶梯必须在证据数处停下并给出可读归因;
* 连得上只是没答好(5xx/读超时)不得短路,重试阶梯照跑满。
*/
class SimilarAsinLlmClientHttpTest {
private HttpServer server;
private final ObjectMapper objectMapper = new ObjectMapper();
private final AtomicInteger requestCount = new AtomicInteger();
/** 请求前挂起时长:模拟"连上了但不应答"的读超时形状。 */
private volatile long responseDelayMillis = 0L;
private volatile boolean failWith502 = false;
@BeforeEach
void setUp() throws IOException {
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/v1/chat/completions", this::handleChat);
// 默认单线程派发:handler 睡眠模拟读超时时会吞掉超时后补发的下一次请求
server.setExecutor(Executors.newCachedThreadPool());
server.start();
}
@AfterEach
void tearDown() {
server.stop(0);
}
private void handleChat(HttpExchange exchange) throws IOException {
exchange.getRequestBody().readAllBytes();
requestCount.incrementAndGet();
long delay = responseDelayMillis;
if (delay > 0L) {
try {
Thread.sleep(delay);
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
}
String body = failWith502
? "{\"error\":{\"message\":\"bad gateway\"}}"
: "{\"choices\":[{\"message\":{\"content\":\"{\\\"ok\\\":true}\"}}]}";
respond(exchange, failWith502 ? 502 : 200, body);
}
private void respond(HttpExchange exchange, int status, String body) throws IOException {
byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
exchange.getResponseHeaders().set("Content-Type", "application/json; charset=utf-8");
exchange.sendResponseHeaders(status, bytes.length);
try (OutputStream os = exchange.getResponseBody()) {
os.write(bytes);
}
}
/** 连接层故障短路:网关根本连不上时,第 2 次连接失败即停,归因文案说清是运维故障。 */
@Test
void unreachableGatewayShortCircuitsLadderAndSurfacesReason() {
SimilarAsinLlmClient deadHostClient = buildClient(p -> {
p.setLlmHost("http://127.0.0.1:1");
p.setLlmRetryTimes(3);
});
long startedAt = System.nanoTime();
assertThatThrownBy(() -> deadHostClient.invokeChat("test-model", "s", "u", "test-key"))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("LLM 服务不可达(连接未建立,非本行数据问题)")
.hasMessageNotContaining("LLM http ");
long elapsedMs = (System.nanoTime() - startedAt) / 1_000_000;
// 短路发生在第 2 次连接失败:只付 1 段退避(2s±30%);本机回环拒绝连接实测 ~2.3s/次
// ⇒ 短路 ~7.7s。跑满 3 次阶梯要多付一段 10s±30%(≥7s)⇒ 实测 ~20s,12s 档两侧都稳
assertThat(elapsedMs).isLessThan(12_000L);
}
/** 反向对照:连得上只是 5xx(非连接层故障)不得触发短路,重试阶梯要跑满。 */
@Test
void httpFailureKeepsFullRetryLadder() {
failWith502 = true;
SimilarAsinLlmClient threeAttemptClient = buildClient(p -> p.setLlmRetryTimes(3));
assertThatThrownBy(() -> threeAttemptClient.invokeChat("test-model", "s", "u", "test-key"))
.hasMessageContaining("LLM http 502");
assertThat(requestCount.get()).isEqualTo(3);
}
/** 读超时是"连上了没答好",故意不算连接层证据:阶梯照跑满,归因文案不得冒充不可达。 */
@Test
void readTimeoutIsNotCountedAsConnectFailure() {
responseDelayMillis = 1_000L;
SimilarAsinLlmClient shortTimeoutClient = buildClient(p -> {
p.setLlmRetryTimes(2);
p.setLlmFirstAttemptReadTimeoutMillis(300);
p.setLlmReadTimeoutMillis(400);
});
assertThatThrownBy(() -> shortTimeoutClient.invokeChat("test-model", "s", "u", "test-key"))
.hasMessageNotContaining("LLM 服务不可达");
assertThat(requestCount.get()).isEqualTo(2);
}
private SimilarAsinLlmClient buildClient(java.util.function.Consumer<SimilarAsinProperties> customizer) {
SimilarAsinProperties properties = new SimilarAsinProperties();
properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort());
customizer.accept(properties);
return new SimilarAsinLlmClient(properties, objectMapper, null, mock(UserSecretUsageService.class));
}
}