fix(llm): 余额耗尽按「余额欠费」归因且不再重试进冷却

- 新增 LlmBalanceGuard:识别 insufficient_user_quota 等上游余额/额度字样
- SimilarAsin/AppearancePatent 客户端命中后一次就停,不落进不可达/冷却分支
- 单测:识别口径 3 例 + 两客户端各 1 例(只发 1 次、文案含余额欠费)
This commit is contained in:
2026-10-27 10:04:03 +08:00
parent e0abd33ed4
commit f3445c21c8
6 changed files with 169 additions and 0 deletions
@@ -0,0 +1,50 @@
package com.nanri.aiimage.common.retry;
import java.util.Locale;
/**
* 上游 LLM 账户余额/额度耗尽的识别与归因(与连接层不可达分账)。
*
* <p>2026-10-27 现网实测:{@code ai.t8star.org} 返回 HTTP 403
* {@code {"error":{"code":"insufficient_user_quota","message":"预扣费额度失败, 用户[151246]剩余额度: ฿0.029168…"}}}。
* 这类失败连接已经建立、上游明确回了余额欠费:既不是本行数据问题,也不是"服务不可达"。
* 不单独识别就会被通用重试阶梯连打三次,用户看到的仍是与真因无关的不可达/冷却文案。
*
* <p>识别只认余额/额度专属字样(ASCII 锚点优先,中文签名在容器日志里变了编码也还能命中);
* 429 限流与 5xx 抖动不在此列,仍走原重试阶梯。
*/
public final class LlmBalanceGuard {
/** 行级/任务级统一归因文案:说清是上游账户欠费,不是这行数据的问题。 */
public static final String REASON_LLM_BALANCE_DUE = "LLM 服务余额欠费(上游账户额度不足,非本行数据问题)";
private static final String[] MARKERS = {
"insufficient_user_quota",
"insufficient_quota",
"insufficient balance",
"预扣费额度失败",
"额度不足",
"余额不足",
"余额欠费",
};
private LlmBalanceGuard() {
}
/** 顺着 cause 链检查异常消息是否命中余额/额度耗尽字样。 */
public static boolean isQuotaExhausted(Throwable error) {
for (Throwable current = error; current != null && current != current.getCause(); current = current.getCause()) {
String message = current.getMessage();
if (message == null || message.isBlank()) {
continue;
}
String lower = message.toLowerCase(Locale.ROOT);
for (String marker : MARKERS) {
if (lower.contains(marker.toLowerCase(Locale.ROOT))) {
return true;
}
}
}
return false;
}
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.retry.LlmBalanceGuard;
import com.nanri.aiimage.common.retry.LlmKeyGate;
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
import com.nanri.aiimage.common.retry.UnreachableCooldown;
@@ -691,6 +692,14 @@ public class AppearancePatentLlmClient {
if (LlmKeyGate.isRateLimited(ex)) {
LlmKeyGate.penalizeRateLimited(apiKey, 2_000L);
}
if (LlmBalanceGuard.isQuotaExhausted(ex)) {
// 余额欠费不是连接层故障,重试救不回来:一次就停,按欠费归因,不落进冷却分支
unreachableCooldown.recordConnected(host);
log.warn("[appearance-patent] LLM_BALANCE_DUE 上游账户余额不足,停止重试 attempt={} model={} err={}",
attempt, model, failureMessage(ex));
throw new IllegalStateException(
LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + failureMessage(ex), ex);
}
lastFailure = ex;
if (UnreachableCooldown.isConnectFailure(ex)) {
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.retry.LlmBalanceGuard;
import com.nanri.aiimage.common.retry.LlmKeyGate;
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
import com.nanri.aiimage.common.retry.UnreachableCooldown;
@@ -119,6 +120,14 @@ public class SimilarAsinLlmClient {
// 429 时把同 key 的下一次许可整体后移,避免所有调用一起再撞
LlmKeyGate.penalizeRateLimited(resolvedKey, 2_000L);
}
if (LlmBalanceGuard.isQuotaExhausted(ex)) {
// 余额欠费不是连接层故障,重试救不回来:一次就停,按欠费归因,不落进冷却分支
unreachableCooldown.recordConnected(host);
log.warn("[similar-asin][llm] LLM_BALANCE_DUE 上游账户余额不足,停止重试 attempt={} model={} err={}",
attempt, model, failureMessage(ex));
throw new IllegalStateException(
LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + failureMessage(ex), ex);
}
lastFailure = ex;
if (UnreachableCooldown.isConnectFailure(ex)) {
connectFailures++;
@@ -0,0 +1,40 @@
package com.nanri.aiimage.common.retry;
import org.junit.jupiter.api.Test;
import java.net.ConnectException;
import static org.assertj.core.api.Assertions.assertThat;
/**
* 余额欠费识别口径(2026-10-27):只认上游余额/额度专属字样,
* 连接层故障、读超时、5xx 与 429 限流都不得被误判成欠费。
*/
class LlmBalanceGuardTest {
/** 现网原文(A/B 机 java-server 2026-10-27 实测),必须命中。 */
@Test
void detectsObservedGatewayQuotaBody() {
IllegalStateException observed = new IllegalStateException(
"LLM http 403: {\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: ฿0.029168, "
+ "需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}");
assertThat(LlmBalanceGuard.isQuotaExhausted(observed)).isTrue();
}
@Test
void detectsQuotaMarkersThroughCauseChain() {
IllegalStateException wrapped = new IllegalStateException("batch failed",
new RuntimeException("upstream says insufficient_quota"));
assertThat(LlmBalanceGuard.isQuotaExhausted(wrapped)).isTrue();
}
@Test
void transientFailuresAreNotQuotaExhaustion() {
assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("LLM http 502: bad gateway"))).isFalse();
assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("LLM empty response"))).isFalse();
assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("429 rate limit exceeded"))).isFalse();
assertThat(LlmBalanceGuard.isQuotaExhausted(new ConnectException("Connection refused"))).isFalse();
}
}
@@ -38,6 +38,10 @@ class AppearancePatentLlmClientHttpTest {
private final AtomicInteger imageRequestCount = new AtomicInteger();
private volatile boolean failTextRequests = false;
private volatile boolean failImageRequests = false;
/** 余额欠费形状(2026-10-27 现网 403):文本/图片两条腿一起拦,验证归因与"不重试"。 */
private volatile boolean failWithQuota403 = false;
private static final String QUOTA_403_BODY = "{\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: "
+ "฿0.029168, 需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}";
private volatile long firstTextRequestDelayMillis = 0L;
/** 批量提取响应改写钩子:模拟漏行、串味、乱序三种模型失真形状。 */
private volatile boolean dropLastTitleItem = false;
@@ -106,12 +110,20 @@ class AppearancePatentLlmClientHttpTest {
if (sequence == 1 && firstTextRequestDelayMillis > 0L) {
sleep(firstTextRequestDelayMillis);
}
if (failWithQuota403) {
respond(exchange, 403, QUOTA_403_BODY);
return;
}
if (failTextRequests) {
respond(exchange, 502, "{\"error\":{\"message\":\"bad gateway\"}}");
return;
}
} else {
imageRequestCount.incrementAndGet();
if (failWithQuota403) {
respond(exchange, 403, QUOTA_403_BODY);
return;
}
if (failImageRequests) {
respond(exchange, 502, "{\"error\":{\"message\":\"bad gateway\"}}");
return;
@@ -327,6 +339,27 @@ class AppearancePatentLlmClientHttpTest {
assertThat(textRequestCount.get()).isEqualTo(3);
}
/**
* 余额欠费(2026-10-27 现网 403 insufficient_user_quota):行级必须显示"余额欠费",
* 文本请求只发一次(重试救不回来),同批外观请求不再发出。
*/
@Test
void quotaExhaustionIsAttributedAsBalanceDueAndNotRetried() {
failWithQuota403 = true;
AppearancePatentLlmClient threeAttemptClient = buildClient(p -> p.setLlmRetryTimes(3));
List<AppearancePatentResultRowDto> result = threeAttemptClient.inspectRows(
List.of(row("11", "B011", "任意标题", "", "https://img.example.com/11.jpg")), null, "test-key");
assertThat(result).hasSize(1);
assertThat(result.get(0).getTitleReason())
.contains("LLM 服务余额欠费")
.contains("insufficient_user_quota")
.doesNotContain("LLM 服务不可达");
assertThat(textRequestCount.get()).isEqualTo(1);
assertThat(imageRequestCount.get()).isZero();
}
private boolean messagesBodyContains(HttpExchange exchange, String text) {
return capturedBodies.stream().anyMatch(b -> b.contains(text));
}
@@ -33,6 +33,10 @@ class SimilarAsinLlmClientHttpTest {
/** 请求前挂起时长:模拟"连上了但不应答"的读超时形状。 */
private volatile long responseDelayMillis = 0L;
private volatile boolean failWith502 = false;
private volatile boolean failWithQuota403 = false;
private static final String QUOTA_403_BODY = "{\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: "
+ "฿0.029168, 需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}";
@BeforeEach
void setUp() throws IOException {
@@ -51,6 +55,10 @@ class SimilarAsinLlmClientHttpTest {
private void handleChat(HttpExchange exchange) throws IOException {
exchange.getRequestBody().readAllBytes();
requestCount.incrementAndGet();
if (failWithQuota403) {
respond(exchange, 403, QUOTA_403_BODY);
return;
}
long delay = responseDelayMillis;
if (delay > 0L) {
try {
@@ -94,6 +102,26 @@ class SimilarAsinLlmClientHttpTest {
assertThat(elapsedMs).isLessThan(12_000L);
}
/**
* 余额欠费(2026-10-27 现网 403 insufficient_user_quota):按"余额欠费"归因、一次就停,
* 既不跑满重试阶梯,也不得落进连接层不可达/冷却分支。
*/
@Test
void quotaExhaustionIsAttributedAsBalanceDueAndNotRetried() {
failWithQuota403 = true;
SimilarAsinLlmClient threeAttemptClient = buildClient(p -> p.setLlmRetryTimes(3));
assertThatThrownBy(() -> threeAttemptClient.invokeChat("test-model", "s", "u", "test-key"))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("LLM 服务余额欠费")
.hasMessageContaining("insufficient_user_quota")
.hasMessageNotContaining("LLM 服务不可达")
.hasMessageNotContaining("冷却中");
assertThat(requestCount.get()).isEqualTo(1);
assertThat(lastCooldown.isCoolingDown(lastHost, lastWindow)).isFalse();
}
/** 反向对照:连得上只是 5xx(非连接层故障)不得触发短路,重试阶梯要跑满。 */
@Test
void httpFailureKeepsFullRetryLadder() {