fix(llm): 余额耗尽按「余额欠费」归因且不再重试进冷却
- 新增 LlmBalanceGuard:识别 insufficient_user_quota 等上游余额/额度字样 - SimilarAsin/AppearancePatent 客户端命中后一次就停,不落进不可达/冷却分支 - 单测:识别口径 3 例 + 两客户端各 1 例(只发 1 次、文案含余额欠费)
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
package com.nanri.aiimage.common.retry;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* 上游 LLM 账户余额/额度耗尽的识别与归因(与连接层不可达分账)。
|
||||
*
|
||||
* <p>2026-10-27 现网实测:{@code ai.t8star.org} 返回 HTTP 403
|
||||
* {@code {"error":{"code":"insufficient_user_quota","message":"预扣费额度失败, 用户[151246]剩余额度: ฿0.029168…"}}}。
|
||||
* 这类失败连接已经建立、上游明确回了余额欠费:既不是本行数据问题,也不是"服务不可达"。
|
||||
* 不单独识别就会被通用重试阶梯连打三次,用户看到的仍是与真因无关的不可达/冷却文案。
|
||||
*
|
||||
* <p>识别只认余额/额度专属字样(ASCII 锚点优先,中文签名在容器日志里变了编码也还能命中);
|
||||
* 429 限流与 5xx 抖动不在此列,仍走原重试阶梯。
|
||||
*/
|
||||
public final class LlmBalanceGuard {
|
||||
|
||||
/** 行级/任务级统一归因文案:说清是上游账户欠费,不是这行数据的问题。 */
|
||||
public static final String REASON_LLM_BALANCE_DUE = "LLM 服务余额欠费(上游账户额度不足,非本行数据问题)";
|
||||
|
||||
private static final String[] MARKERS = {
|
||||
"insufficient_user_quota",
|
||||
"insufficient_quota",
|
||||
"insufficient balance",
|
||||
"预扣费额度失败",
|
||||
"额度不足",
|
||||
"余额不足",
|
||||
"余额欠费",
|
||||
};
|
||||
|
||||
private LlmBalanceGuard() {
|
||||
}
|
||||
|
||||
/** 顺着 cause 链检查异常消息是否命中余额/额度耗尽字样。 */
|
||||
public static boolean isQuotaExhausted(Throwable error) {
|
||||
for (Throwable current = error; current != null && current != current.getCause(); current = current.getCause()) {
|
||||
String message = current.getMessage();
|
||||
if (message == null || message.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
String lower = message.toLowerCase(Locale.ROOT);
|
||||
for (String marker : MARKERS) {
|
||||
if (lower.contains(marker.toLowerCase(Locale.ROOT))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+9
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmBalanceGuard;
|
||||
import com.nanri.aiimage.common.retry.LlmKeyGate;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
@@ -691,6 +692,14 @@ public class AppearancePatentLlmClient {
|
||||
if (LlmKeyGate.isRateLimited(ex)) {
|
||||
LlmKeyGate.penalizeRateLimited(apiKey, 2_000L);
|
||||
}
|
||||
if (LlmBalanceGuard.isQuotaExhausted(ex)) {
|
||||
// 余额欠费不是连接层故障,重试救不回来:一次就停,按欠费归因,不落进冷却分支
|
||||
unreachableCooldown.recordConnected(host);
|
||||
log.warn("[appearance-patent] LLM_BALANCE_DUE 上游账户余额不足,停止重试 attempt={} model={} err={}",
|
||||
attempt, model, failureMessage(ex));
|
||||
throw new IllegalStateException(
|
||||
LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + failureMessage(ex), ex);
|
||||
}
|
||||
lastFailure = ex;
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
boolean entered = unreachableCooldown.recordConnectFailure(host, cooldownMillis);
|
||||
|
||||
+9
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmBalanceGuard;
|
||||
import com.nanri.aiimage.common.retry.LlmKeyGate;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
@@ -119,6 +120,14 @@ public class SimilarAsinLlmClient {
|
||||
// 429 时把同 key 的下一次许可整体后移,避免所有调用一起再撞
|
||||
LlmKeyGate.penalizeRateLimited(resolvedKey, 2_000L);
|
||||
}
|
||||
if (LlmBalanceGuard.isQuotaExhausted(ex)) {
|
||||
// 余额欠费不是连接层故障,重试救不回来:一次就停,按欠费归因,不落进冷却分支
|
||||
unreachableCooldown.recordConnected(host);
|
||||
log.warn("[similar-asin][llm] LLM_BALANCE_DUE 上游账户余额不足,停止重试 attempt={} model={} err={}",
|
||||
attempt, model, failureMessage(ex));
|
||||
throw new IllegalStateException(
|
||||
LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + failureMessage(ex), ex);
|
||||
}
|
||||
lastFailure = ex;
|
||||
if (UnreachableCooldown.isConnectFailure(ex)) {
|
||||
connectFailures++;
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.nanri.aiimage.common.retry;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.net.ConnectException;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* 余额欠费识别口径(2026-10-27):只认上游余额/额度专属字样,
|
||||
* 连接层故障、读超时、5xx 与 429 限流都不得被误判成欠费。
|
||||
*/
|
||||
class LlmBalanceGuardTest {
|
||||
|
||||
/** 现网原文(A/B 机 java-server 2026-10-27 实测),必须命中。 */
|
||||
@Test
|
||||
void detectsObservedGatewayQuotaBody() {
|
||||
IllegalStateException observed = new IllegalStateException(
|
||||
"LLM http 403: {\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: ฿0.029168, "
|
||||
+ "需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}");
|
||||
|
||||
assertThat(LlmBalanceGuard.isQuotaExhausted(observed)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsQuotaMarkersThroughCauseChain() {
|
||||
IllegalStateException wrapped = new IllegalStateException("batch failed",
|
||||
new RuntimeException("upstream says insufficient_quota"));
|
||||
|
||||
assertThat(LlmBalanceGuard.isQuotaExhausted(wrapped)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void transientFailuresAreNotQuotaExhaustion() {
|
||||
assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("LLM http 502: bad gateway"))).isFalse();
|
||||
assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("LLM empty response"))).isFalse();
|
||||
assertThat(LlmBalanceGuard.isQuotaExhausted(new IllegalStateException("429 rate limit exceeded"))).isFalse();
|
||||
assertThat(LlmBalanceGuard.isQuotaExhausted(new ConnectException("Connection refused"))).isFalse();
|
||||
}
|
||||
}
|
||||
+33
@@ -38,6 +38,10 @@ class AppearancePatentLlmClientHttpTest {
|
||||
private final AtomicInteger imageRequestCount = new AtomicInteger();
|
||||
private volatile boolean failTextRequests = false;
|
||||
private volatile boolean failImageRequests = false;
|
||||
/** 余额欠费形状(2026-10-27 现网 403):文本/图片两条腿一起拦,验证归因与"不重试"。 */
|
||||
private volatile boolean failWithQuota403 = false;
|
||||
private static final String QUOTA_403_BODY = "{\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: "
|
||||
+ "฿0.029168, 需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}";
|
||||
private volatile long firstTextRequestDelayMillis = 0L;
|
||||
/** 批量提取响应改写钩子:模拟漏行、串味、乱序三种模型失真形状。 */
|
||||
private volatile boolean dropLastTitleItem = false;
|
||||
@@ -106,12 +110,20 @@ class AppearancePatentLlmClientHttpTest {
|
||||
if (sequence == 1 && firstTextRequestDelayMillis > 0L) {
|
||||
sleep(firstTextRequestDelayMillis);
|
||||
}
|
||||
if (failWithQuota403) {
|
||||
respond(exchange, 403, QUOTA_403_BODY);
|
||||
return;
|
||||
}
|
||||
if (failTextRequests) {
|
||||
respond(exchange, 502, "{\"error\":{\"message\":\"bad gateway\"}}");
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
imageRequestCount.incrementAndGet();
|
||||
if (failWithQuota403) {
|
||||
respond(exchange, 403, QUOTA_403_BODY);
|
||||
return;
|
||||
}
|
||||
if (failImageRequests) {
|
||||
respond(exchange, 502, "{\"error\":{\"message\":\"bad gateway\"}}");
|
||||
return;
|
||||
@@ -327,6 +339,27 @@ class AppearancePatentLlmClientHttpTest {
|
||||
assertThat(textRequestCount.get()).isEqualTo(3);
|
||||
}
|
||||
|
||||
/**
|
||||
* 余额欠费(2026-10-27 现网 403 insufficient_user_quota):行级必须显示"余额欠费",
|
||||
* 文本请求只发一次(重试救不回来),同批外观请求不再发出。
|
||||
*/
|
||||
@Test
|
||||
void quotaExhaustionIsAttributedAsBalanceDueAndNotRetried() {
|
||||
failWithQuota403 = true;
|
||||
AppearancePatentLlmClient threeAttemptClient = buildClient(p -> p.setLlmRetryTimes(3));
|
||||
|
||||
List<AppearancePatentResultRowDto> result = threeAttemptClient.inspectRows(
|
||||
List.of(row("11", "B011", "任意标题", "", "https://img.example.com/11.jpg")), null, "test-key");
|
||||
|
||||
assertThat(result).hasSize(1);
|
||||
assertThat(result.get(0).getTitleReason())
|
||||
.contains("LLM 服务余额欠费")
|
||||
.contains("insufficient_user_quota")
|
||||
.doesNotContain("LLM 服务不可达");
|
||||
assertThat(textRequestCount.get()).isEqualTo(1);
|
||||
assertThat(imageRequestCount.get()).isZero();
|
||||
}
|
||||
|
||||
private boolean messagesBodyContains(HttpExchange exchange, String text) {
|
||||
return capturedBodies.stream().anyMatch(b -> b.contains(text));
|
||||
}
|
||||
|
||||
+28
@@ -33,6 +33,10 @@ class SimilarAsinLlmClientHttpTest {
|
||||
/** 请求前挂起时长:模拟"连上了但不应答"的读超时形状。 */
|
||||
private volatile long responseDelayMillis = 0L;
|
||||
private volatile boolean failWith502 = false;
|
||||
private volatile boolean failWithQuota403 = false;
|
||||
|
||||
private static final String QUOTA_403_BODY = "{\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: "
|
||||
+ "฿0.029168, 需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}";
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws IOException {
|
||||
@@ -51,6 +55,10 @@ class SimilarAsinLlmClientHttpTest {
|
||||
private void handleChat(HttpExchange exchange) throws IOException {
|
||||
exchange.getRequestBody().readAllBytes();
|
||||
requestCount.incrementAndGet();
|
||||
if (failWithQuota403) {
|
||||
respond(exchange, 403, QUOTA_403_BODY);
|
||||
return;
|
||||
}
|
||||
long delay = responseDelayMillis;
|
||||
if (delay > 0L) {
|
||||
try {
|
||||
@@ -94,6 +102,26 @@ class SimilarAsinLlmClientHttpTest {
|
||||
assertThat(elapsedMs).isLessThan(12_000L);
|
||||
}
|
||||
|
||||
/**
|
||||
* 余额欠费(2026-10-27 现网 403 insufficient_user_quota):按"余额欠费"归因、一次就停,
|
||||
* 既不跑满重试阶梯,也不得落进连接层不可达/冷却分支。
|
||||
*/
|
||||
@Test
|
||||
void quotaExhaustionIsAttributedAsBalanceDueAndNotRetried() {
|
||||
failWithQuota403 = true;
|
||||
SimilarAsinLlmClient threeAttemptClient = buildClient(p -> p.setLlmRetryTimes(3));
|
||||
|
||||
assertThatThrownBy(() -> threeAttemptClient.invokeChat("test-model", "s", "u", "test-key"))
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("LLM 服务余额欠费")
|
||||
.hasMessageContaining("insufficient_user_quota")
|
||||
.hasMessageNotContaining("LLM 服务不可达")
|
||||
.hasMessageNotContaining("冷却中");
|
||||
|
||||
assertThat(requestCount.get()).isEqualTo(1);
|
||||
assertThat(lastCooldown.isCoolingDown(lastHost, lastWindow)).isFalse();
|
||||
}
|
||||
|
||||
/** 反向对照:连得上只是 5xx(非连接层故障)不得触发短路,重试阶梯要跑满。 */
|
||||
@Test
|
||||
void httpFailureKeepsFullRetryLadder() {
|
||||
|
||||
Reference in New Issue
Block a user