perf(LLM任务): 首查短超时快速失败 + 退避抖动 + 标题失败跳过外观识别

生产 24h 数据:外观专利任务 LLM 重试失败 93+82 次几乎全是超时,
终态 41 行降级为「外观识别异常」;成功调用 p99=34s、超 60s 仅 0.02%。
原策略每轮重试都挂满 90s(被全局 call-timeout 钳制),且 1.5s/3s
密集重试整批落在同一劣化窗口内。

- 第 1 次尝试改用 llm-first-attempt-read-timeout-millis(默认 60s)快速失败,
  重试走完整读超时,慢而成功的正常调用不被误杀
- 重试等待改 LlmRetryBackoff:2s/10s + ±30% 抖动,覆盖更长窗口并打散同批尖峰
- 外观专利标题识别失败时跳过外观请求(该行必走回退,外观结果本就会被丢弃)
This commit is contained in:
2026-09-14 00:28:40 +08:00
parent 864c22ffc7
commit db6869b77e
8 changed files with 280 additions and 45 deletions
@@ -0,0 +1,47 @@
package com.nanri.aiimage.common.retry;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/** 重试退避策略:档位、上下界与抖动范围(固定 random 消除随机性)。 */
class LlmRetryBackoffTest {
@Test
void firstRetryWaitsTwoSeconds() {
assertThat(LlmRetryBackoff.delayMillis(1, 0.5d)).isEqualTo(2_000L);
}
@Test
void secondRetryWaitsTenSeconds() {
assertThat(LlmRetryBackoff.delayMillis(2, 0.5d)).isEqualTo(10_000L);
}
@Test
void attemptsBeyondTableUseLastTier() {
assertThat(LlmRetryBackoff.delayMillis(3, 0.5d)).isEqualTo(10_000L);
assertThat(LlmRetryBackoff.delayMillis(9, 0.5d)).isEqualTo(10_000L);
}
@Test
void jitterStaysWithinThirtyPercent() {
assertThat(LlmRetryBackoff.delayMillis(1, 0d)).isEqualTo(1_400L);
assertThat(LlmRetryBackoff.delayMillis(1, 1d)).isEqualTo(2_600L);
assertThat(LlmRetryBackoff.delayMillis(2, 0d)).isEqualTo(7_000L);
assertThat(LlmRetryBackoff.delayMillis(2, 1d)).isEqualTo(13_000L);
}
@Test
void attemptIndexBelowOneIsClampedToFirstTier() {
assertThat(LlmRetryBackoff.delayMillis(0, 0.5d)).isEqualTo(2_000L);
assertThat(LlmRetryBackoff.delayMillis(-3, 0.5d)).isEqualTo(2_000L);
}
@Test
void randomDelayAlwaysPositiveAndBounded() {
for (int i = 0; i < 200; i++) {
long delay = LlmRetryBackoff.delayMillis(1);
assertThat(delay).isBetween(1_400L, 2_600L);
}
}
}
@@ -31,6 +31,10 @@ class AppearancePatentLlmClientHttpTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final Map<String, AtomicInteger> callCounts = new ConcurrentHashMap<>();
private final List<String> capturedBodies = new ArrayList<>();
/** 纯文本(标题提取)请求计数:用于验证重试次数与「跳过外观」行为。 */
private final AtomicInteger textRequestCount = new AtomicInteger();
private volatile boolean failTextRequests = false;
private volatile long firstTextRequestDelayMillis = 0L;
private AppearancePatentLlmClient client;
@@ -60,6 +64,16 @@ class AppearancePatentLlmClientHttpTest {
private void handleChat(HttpExchange exchange) throws IOException {
String body = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8);
capturedBodies.add(body);
if (!body.contains("\"type\":\"image_url\"")) {
int sequence = textRequestCount.incrementAndGet();
if (sequence == 1 && firstTextRequestDelayMillis > 0L) {
sleep(firstTextRequestDelayMillis);
}
if (failTextRequests) {
respond(exchange, 502, "{\"error\":{\"message\":\"bad gateway\"}}");
return;
}
}
try {
@SuppressWarnings("unchecked")
Map<String, Object> request = objectMapper.readValue(body, Map.class);
@@ -103,6 +117,32 @@ class AppearancePatentLlmClientHttpTest {
}
}
private void respond(HttpExchange exchange, int status, String body) throws IOException {
byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
exchange.getResponseHeaders().set("Content-Type", "application/json; charset=utf-8");
exchange.sendResponseHeaders(status, bytes.length);
try (OutputStream os = exchange.getResponseBody()) {
os.write(bytes);
}
}
private void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
}
}
/** 用自定义超时/重试参数构建客户端(setUp 里的 client 用属性默认值)。 */
private AppearancePatentLlmClient buildClient(java.util.function.Consumer<AppearancePatentProperties> customizer) {
AppearancePatentProperties properties = new AppearancePatentProperties();
properties.setLlmHost("http://127.0.0.1:" + server.getAddress().getPort());
customizer.accept(properties);
return new AppearancePatentLlmClient(properties, objectMapper, null,
new BrandCheckClient(new BrandCheckProperties(), null), mock(UserSecretUsageService.class));
}
private boolean messagesBodyContains(HttpExchange exchange, String text) {
return capturedBodies.stream().anyMatch(b -> b.contains(text));
}
@@ -194,4 +234,43 @@ class AppearancePatentLlmClientHttpTest {
assertThat(appearanceBody).contains("\"type\":\"json_object\"");
assertThat(appearanceBody).contains("产品描述:普通数据线");
}
/** 标题识别失败 → 本行必走回退(外观结果会被丢弃),外观请求不再发出:劣化窗口下少一轮无效重试。 */
@Test
void titleFailureSkipsAppearanceCallAndKeepsFallback() {
failTextRequests = true;
AppearancePatentLlmClient singleAttemptClient = buildClient(p -> {
p.setLlmRetryTimes(1);
p.setLlmFirstAttemptReadTimeoutMillis(5_000);
p.setLlmReadTimeoutMillis(5_000);
});
List<AppearancePatentResultRowDto> rows = singleAttemptClient.inspectRows(
List.of(row("7", "B007", "任意标题", "", "https://img.example.com/7.jpg")), null, "test-key");
assertThat(rows.get(0).getAppearanceRisk()).isEqualTo("外观识别异常");
assertThat(capturedBodies).noneMatch(b -> b.contains("\"type\":\"image_url\""));
assertThat(textRequestCount.get()).isEqualTo(1);
}
/** 首查用更短超时快速失败、重试走完整超时并成功(上游偶发单请求不响应的兜底路径)。 */
@Test
void firstAttemptTimeoutRetriesWithFullBudgetAndSucceeds() {
firstTextRequestDelayMillis = 1_200L;
AppearancePatentLlmClient twoAttemptClient = buildClient(p -> {
p.setLlmRetryTimes(2);
p.setLlmFirstAttemptReadTimeoutMillis(400);
p.setLlmReadTimeoutMillis(8_000);
});
long startedAt = System.nanoTime();
List<AppearancePatentResultRowDto> rows = twoAttemptClient.inspectRows(
List.of(row("8", "B008", "普通数据线", "", "https://img.example.com/8.jpg")), null, "test-key");
long elapsedMs = (System.nanoTime() - startedAt) / 1_000_000;
AppearancePatentResultRowDto result = rows.get(0);
assertThat(result.getAppearanceRisk()).isEqualTo("无侵权");
assertThat(textRequestCount.get()).isEqualTo(2);
assertThat(elapsedMs).isGreaterThanOrEqualTo(400L);
}
}