diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmBalanceGuard.java b/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmBalanceGuard.java new file mode 100644 index 000000000..4fb9af4bc --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmBalanceGuard.java @@ -0,0 +1,50 @@ +package com.nanri.aiimage.common.retry; + +import java.util.Locale; + +/** + * 上游 LLM 账户余额/额度耗尽的识别与归因(与连接层不可达分账)。 + * + *

2026-10-27 现网实测:{@code ai.t8star.org} 返回 HTTP 403 + * {@code {"error":{"code":"insufficient_user_quota","message":"预扣费额度失败, 用户[151246]剩余额度: ฿0.029168…"}}}。 + * 这类失败连接已经建立、上游明确回了余额欠费:既不是本行数据问题,也不是"服务不可达"。 + * 不单独识别就会被通用重试阶梯连打三次,用户看到的仍是与真因无关的不可达/冷却文案。 + * + *

识别只认余额/额度专属字样(ASCII 锚点优先,中文签名在容器日志里变了编码也还能命中); + * 429 限流与 5xx 抖动不在此列,仍走原重试阶梯。 + */ +public final class LlmBalanceGuard { + + /** 行级/任务级统一归因文案:说清是上游账户欠费,不是这行数据的问题。 */ + public static final String REASON_LLM_BALANCE_DUE = "LLM 服务余额欠费(上游账户额度不足,非本行数据问题)"; + + private static final String[] MARKERS = { + "insufficient_user_quota", + "insufficient_quota", + "insufficient balance", + "预扣费额度失败", + "额度不足", + "余额不足", + "余额欠费", + }; + + private LlmBalanceGuard() { + } + + /** 顺着 cause 链检查异常消息是否命中余额/额度耗尽字样。 */ + public static boolean isQuotaExhausted(Throwable error) { + for (Throwable current = error; current != null && current != current.getCause(); current = current.getCause()) { + String message = current.getMessage(); + if (message == null || message.isBlank()) { + continue; + } + String lower = message.toLowerCase(Locale.ROOT); + for (String marker : MARKERS) { + if (lower.contains(marker.toLowerCase(Locale.ROOT))) { + return true; + } + } + } + return false; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java index 470dc4612..853878fdc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.client; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.LlmBalanceGuard; import com.nanri.aiimage.common.retry.LlmKeyGate; import com.nanri.aiimage.common.retry.LlmRetryBackoff; import com.nanri.aiimage.common.retry.UnreachableCooldown; @@ -691,6 +692,14 @@ public class AppearancePatentLlmClient { if (LlmKeyGate.isRateLimited(ex)) { LlmKeyGate.penalizeRateLimited(apiKey, 2_000L); } + if (LlmBalanceGuard.isQuotaExhausted(ex)) { + // 余额欠费不是连接层故障,重试救不回来:一次就停,按欠费归因,不落进冷却分支 + unreachableCooldown.recordConnected(host); + log.warn("[appearance-patent] LLM_BALANCE_DUE 上游账户余额不足,停止重试 attempt={} model={} err={}", + attempt, model, failureMessage(ex)); + throw new IllegalStateException( + LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + failureMessage(ex), ex); + } lastFailure = ex; if (UnreachableCooldown.isConnectFailure(ex)) { boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java index 15d8d69ab..47b0b199d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.client; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.LlmBalanceGuard; import com.nanri.aiimage.common.retry.LlmKeyGate; import com.nanri.aiimage.common.retry.LlmRetryBackoff; import com.nanri.aiimage.common.retry.UnreachableCooldown; @@ -119,6 +120,14 @@ public class SimilarAsinLlmClient { // 429 时把同 key 的下一次许可整体后移,避免所有调用一起再撞 LlmKeyGate.penalizeRateLimited(resolvedKey, 2_000L); } + if (LlmBalanceGuard.isQuotaExhausted(ex)) { + // 余额欠费不是连接层故障,重试救不回来:一次就停,按欠费归因,不落进冷却分支 + unreachableCooldown.recordConnected(host); + log.warn("[similar-asin][llm] LLM_BALANCE_DUE 上游账户余额不足,停止重试 attempt={} model={} err={}", + attempt, model, failureMessage(ex)); + throw new IllegalStateException( + LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + failureMessage(ex), ex); + } lastFailure = ex; if (UnreachableCooldown.isConnectFailure(ex)) { connectFailures++; diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/retry/LlmBalanceGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/retry/LlmBalanceGuardTest.java new file mode 100644 index 000000000..37b230a90 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/common/retry/LlmBalanceGuardTest.java @@ -0,0 +1,40 @@ +package com.nanri.aiimage.common.retry; + +import org.junit.jupiter.api.Test; + +import java.net.ConnectException; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 余额欠费识别口径(2026-10-27):只认上游余额/额度专属字样, + * 连接层故障、读超时、5xx 与 429 限流都不得被误判成欠费。 + */ +class LlmBalanceGuardTest { + + /** 现网原文(A/B 机 java-server 2026-10-27 实测),必须命中。 */ + @Test + void detectsObservedGatewayQuotaBody() { + IllegalStateException observed = new IllegalStateException( + "LLM http 403: {\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: ฿0.029168, " + + "需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}"); + + assertThat(LlmBalanceGuard.isQuotaExhausted(observed)).isTrue(); + } + + @Test + void detectsQuotaMarkersThroughCauseChain() { + IllegalStateException wrapped = new IllegalStateException("batch failed", + new RuntimeException("upstream says insufficient_quota")); + + assertThat(LlmBalanceGuard.isQuotaExhausted(wrapped)).isTrue(); + } + + @Test + void transientFailuresAreNotQuotaExhaustion() { + assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("LLM http 502: bad gateway"))).isFalse(); + assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("LLM empty response"))).isFalse(); + assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("429 rate limit exceeded"))).isFalse(); + assertThat(LlmBalanceGuard.isQuotaExhausted(new ConnectException("Connection refused"))).isFalse(); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java index 0c2208c84..8f7ba8dff 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClientHttpTest.java @@ -38,6 +38,10 @@ class AppearancePatentLlmClientHttpTest { private final AtomicInteger imageRequestCount = new AtomicInteger(); private volatile boolean failTextRequests = false; private volatile boolean failImageRequests = false; + /** 余额欠费形状(2026-10-27 现网 403):文本/图片两条腿一起拦,验证归因与"不重试"。 */ + private volatile boolean failWithQuota403 = false; + private static final String QUOTA_403_BODY = "{\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: " + + "฿0.029168, 需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}"; private volatile long firstTextRequestDelayMillis = 0L; /** 批量提取响应改写钩子:模拟漏行、串味、乱序三种模型失真形状。 */ private volatile boolean dropLastTitleItem = false; @@ -106,12 +110,20 @@ class AppearancePatentLlmClientHttpTest { if (sequence == 1 && firstTextRequestDelayMillis > 0L) { sleep(firstTextRequestDelayMillis); } + if (failWithQuota403) { + respond(exchange, 403, QUOTA_403_BODY); + return; + } if (failTextRequests) { respond(exchange, 502, "{\"error\":{\"message\":\"bad gateway\"}}"); return; } } else { imageRequestCount.incrementAndGet(); + if (failWithQuota403) { + respond(exchange, 403, QUOTA_403_BODY); + return; + } if (failImageRequests) { respond(exchange, 502, "{\"error\":{\"message\":\"bad gateway\"}}"); return; @@ -327,6 +339,27 @@ class AppearancePatentLlmClientHttpTest { assertThat(textRequestCount.get()).isEqualTo(3); } + /** + * 余额欠费(2026-10-27 现网 403 insufficient_user_quota):行级必须显示"余额欠费", + * 文本请求只发一次(重试救不回来),同批外观请求不再发出。 + */ + @Test + void quotaExhaustionIsAttributedAsBalanceDueAndNotRetried() { + failWithQuota403 = true; + AppearancePatentLlmClient threeAttemptClient = buildClient(p -> p.setLlmRetryTimes(3)); + + List result = threeAttemptClient.inspectRows( + List.of(row("11", "B011", "任意标题", "", "https://img.example.com/11.jpg")), null, "test-key"); + + assertThat(result).hasSize(1); + assertThat(result.get(0).getTitleReason()) + .contains("LLM 服务余额欠费") + .contains("insufficient_user_quota") + .doesNotContain("LLM 服务不可达"); + assertThat(textRequestCount.get()).isEqualTo(1); + assertThat(imageRequestCount.get()).isZero(); + } + private boolean messagesBodyContains(HttpExchange exchange, String text) { return capturedBodies.stream().anyMatch(b -> b.contains(text)); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java index 0130f593a..04ec97af3 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClientHttpTest.java @@ -33,6 +33,10 @@ class SimilarAsinLlmClientHttpTest { /** 请求前挂起时长:模拟"连上了但不应答"的读超时形状。 */ private volatile long responseDelayMillis = 0L; private volatile boolean failWith502 = false; + private volatile boolean failWithQuota403 = false; + + private static final String QUOTA_403_BODY = "{\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: " + + "฿0.029168, 需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}"; @BeforeEach void setUp() throws IOException { @@ -51,6 +55,10 @@ class SimilarAsinLlmClientHttpTest { private void handleChat(HttpExchange exchange) throws IOException { exchange.getRequestBody().readAllBytes(); requestCount.incrementAndGet(); + if (failWithQuota403) { + respond(exchange, 403, QUOTA_403_BODY); + return; + } long delay = responseDelayMillis; if (delay > 0L) { try { @@ -94,6 +102,26 @@ class SimilarAsinLlmClientHttpTest { assertThat(elapsedMs).isLessThan(12_000L); } + /** + * 余额欠费(2026-10-27 现网 403 insufficient_user_quota):按"余额欠费"归因、一次就停, + * 既不跑满重试阶梯,也不得落进连接层不可达/冷却分支。 + */ + @Test + void quotaExhaustionIsAttributedAsBalanceDueAndNotRetried() { + failWithQuota403 = true; + SimilarAsinLlmClient threeAttemptClient = buildClient(p -> p.setLlmRetryTimes(3)); + + assertThatThrownBy(() -> threeAttemptClient.invokeChat("test-model", "s", "u", "test-key")) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("LLM 服务余额欠费") + .hasMessageContaining("insufficient_user_quota") + .hasMessageNotContaining("LLM 服务不可达") + .hasMessageNotContaining("冷却中"); + + assertThat(requestCount.get()).isEqualTo(1); + assertThat(lastCooldown.isCoolingDown(lastHost, lastWindow)).isFalse(); + } + /** 反向对照:连得上只是 5xx(非连接层故障)不得触发短路,重试阶梯要跑满。 */ @Test void httpFailureKeepsFullRetryLadder() {