perf(LLM任务): 首查短超时快速失败 + 退避抖动 + 标题失败跳过外观识别
生产 24h 数据:外观专利任务 LLM 重试失败 93+82 次几乎全是超时, 终态 41 行降级为「外观识别异常」;成功调用 p99=34s、超 60s 仅 0.02%。 原策略每轮重试都挂满 90s(被全局 call-timeout 钳制),且 1.5s/3s 密集重试整批落在同一劣化窗口内。 - 第 1 次尝试改用 llm-first-attempt-read-timeout-millis(默认 60s)快速失败, 重试走完整读超时,慢而成功的正常调用不被误杀 - 重试等待改 LlmRetryBackoff:2s/10s + ±30% 抖动,覆盖更长窗口并打散同批尖峰 - 外观专利标题识别失败时跳过外观请求(该行必走回退,外观结果本就会被丢弃)
This commit is contained in:
+59
-28
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
|
||||
@@ -103,6 +104,8 @@ public class AppearancePatentLlmClient {
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
/** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */
|
||||
private volatile RestClient firstAttemptRestClient;
|
||||
private volatile ExecutorService rowExecutor;
|
||||
|
||||
/**
|
||||
@@ -192,15 +195,20 @@ public class AppearancePatentLlmClient {
|
||||
String appearanceStatus = "";
|
||||
String appearanceReason = "";
|
||||
boolean appearanceFailed = false;
|
||||
try {
|
||||
AppearanceJudgement judgement = invokeAppearance(row, prompt, apiKey);
|
||||
appearanceStatus = judgement.status();
|
||||
appearanceReason = judgement.reason();
|
||||
} catch (Exception ex) {
|
||||
appearanceFailed = true;
|
||||
appearanceReason = firstNonBlank(ex.getMessage(), "外观识别失败");
|
||||
log.warn("[appearance-patent] appearance llm failed asin={} title={} url={} err={}",
|
||||
row.getAsin(), abbreviate(row.getTitle(), 120), abbreviate(row.getUrl(), 120), failureMessage(ex));
|
||||
if (titleFailed) {
|
||||
// 标题识别失败时本行必走回退分支(外观结果会被丢弃),再发外观请求只会在劣化中的上游白等一轮重试
|
||||
log.warn("[appearance-patent] 标题识别失败,跳过外观识别(本行同样走回退)asin={}", row.getAsin());
|
||||
} else {
|
||||
try {
|
||||
AppearanceJudgement judgement = invokeAppearance(row, prompt, apiKey);
|
||||
appearanceStatus = judgement.status();
|
||||
appearanceReason = judgement.reason();
|
||||
} catch (Exception ex) {
|
||||
appearanceFailed = true;
|
||||
appearanceReason = firstNonBlank(ex.getMessage(), "外观识别失败");
|
||||
log.warn("[appearance-patent] appearance llm failed asin={} title={} url={} err={}",
|
||||
row.getAsin(), abbreviate(row.getTitle(), 120), abbreviate(row.getUrl(), 120), failureMessage(ex));
|
||||
}
|
||||
}
|
||||
if (titleFailed || appearanceFailed) {
|
||||
String titleReason = titleFailed ? titleError : firstNonBlank(rawTitle, MISSING_ROW_DATA);
|
||||
@@ -274,15 +282,16 @@ public class AppearancePatentLlmClient {
|
||||
Exception lastFailure = null;
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
return invokeChatOnce(model, system, userText, images, apiKey, responseFormat);
|
||||
return invokeChatOnce(model, system, userText, images, apiKey, responseFormat, attempt);
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
if (attempt >= attempts) {
|
||||
break;
|
||||
}
|
||||
log.warn("[appearance-patent] llm retryable failure attempt={} model={} err={}",
|
||||
attempt, model, failureMessage(ex));
|
||||
sleepBeforeRetry(attempt);
|
||||
long retryDelayMillis = LlmRetryBackoff.delayMillis(attempt);
|
||||
log.warn("[appearance-patent] llm retryable failure attempt={} model={} 下次重试等待={}ms err={}",
|
||||
attempt, model, retryDelayMillis, failureMessage(ex));
|
||||
sleepQuietly(retryDelayMillis);
|
||||
}
|
||||
}
|
||||
throw lastFailure == null ? new IllegalStateException("LLM call failed") : lastFailure;
|
||||
@@ -293,13 +302,14 @@ public class AppearancePatentLlmClient {
|
||||
String userText,
|
||||
List<String> images,
|
||||
String apiKey,
|
||||
String responseFormat) {
|
||||
String responseFormat,
|
||||
int attempt) {
|
||||
recordSecretUsage();
|
||||
Map<String, Object> body = buildChatBody(model, system, userText, images, responseFormat);
|
||||
log.info("[appearance-patent] llm request model={} url={} body={}",
|
||||
model, joinUrl(properties.getLlmHost(), "/v1/chat/completions"),
|
||||
log.info("[appearance-patent] llm request attempt={} model={} url={} body={}",
|
||||
attempt, model, joinUrl(properties.getLlmHost(), "/v1/chat/completions"),
|
||||
writeJson(maskChatBody(body)));
|
||||
RestClient.RequestBodySpec request = restClient().post()
|
||||
RestClient.RequestBodySpec request = restClient(attempt).post()
|
||||
.uri(joinUrl(properties.getLlmHost(), "/v1/chat/completions"))
|
||||
.headers(headers -> {
|
||||
headers.setBearerAuth(stripBearer(apiKey));
|
||||
@@ -748,24 +758,49 @@ public class AppearancePatentLlmClient {
|
||||
}
|
||||
}
|
||||
|
||||
private RestClient restClient() {
|
||||
/**
|
||||
* 按尝试次数取客户端:第 1 次用「首次尝试超时」(快速失败,单次挂死代价更小),
|
||||
* 重试用完整读超时,避免把慢但成功的正常调用误杀(实测成功调用 p99≈34s)。
|
||||
*/
|
||||
private RestClient restClient(int attempt) {
|
||||
if (attempt == 1 && firstAttemptTimeoutMillis() < Math.max(1, properties.getLlmReadTimeoutMillis())) {
|
||||
RestClient client = firstAttemptRestClient;
|
||||
if (client != null) {
|
||||
return client;
|
||||
}
|
||||
synchronized (this) {
|
||||
if (firstAttemptRestClient == null) {
|
||||
firstAttemptRestClient = buildRestClient(firstAttemptTimeoutMillis());
|
||||
}
|
||||
return firstAttemptRestClient;
|
||||
}
|
||||
}
|
||||
RestClient client = sharedRestClient;
|
||||
if (client != null) {
|
||||
return client;
|
||||
}
|
||||
synchronized (this) {
|
||||
if (sharedRestClient == null) {
|
||||
RestClient.Builder builder = RestClient.builder()
|
||||
.requestFactory(HttpClientPool.requestFactory(properties.getLlmReadTimeoutMillis()));
|
||||
if (externalCallMetrics != null) {
|
||||
builder.requestInterceptor(externalCallMetrics.interceptor("llm"));
|
||||
}
|
||||
sharedRestClient = builder.build();
|
||||
sharedRestClient = buildRestClient(properties.getLlmReadTimeoutMillis());
|
||||
}
|
||||
return sharedRestClient;
|
||||
}
|
||||
}
|
||||
|
||||
private int firstAttemptTimeoutMillis() {
|
||||
int configured = properties.getLlmFirstAttemptReadTimeoutMillis();
|
||||
return configured > 0 ? configured : Math.max(1, properties.getLlmReadTimeoutMillis());
|
||||
}
|
||||
|
||||
private RestClient buildRestClient(int readTimeoutMillis) {
|
||||
RestClient.Builder builder = RestClient.builder()
|
||||
.requestFactory(HttpClientPool.requestFactory(readTimeoutMillis));
|
||||
if (externalCallMetrics != null) {
|
||||
builder.requestInterceptor(externalCallMetrics.interceptor("llm"));
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private ExecutorService rowExecutor() {
|
||||
ExecutorService executor = rowExecutor;
|
||||
if (executor != null) {
|
||||
@@ -844,10 +879,6 @@ public class AppearancePatentLlmClient {
|
||||
return normalized.substring(0, Math.max(0, maxLength - 3)) + "...";
|
||||
}
|
||||
|
||||
private void sleepBeforeRetry(int attemptIndex) {
|
||||
sleepQuietly(Math.max(1, attemptIndex) * 1500L);
|
||||
}
|
||||
|
||||
/** 密钥调用计次:每次真实 HTTP 请求(含重试)计 1 次;无任务上下文(无人归属)时跳过。 */
|
||||
private void recordSecretUsage() {
|
||||
SecretUsageContext.Context context = SecretUsageContext.current();
|
||||
|
||||
+47
-17
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
|
||||
@@ -43,6 +44,8 @@ public class SimilarAsinLlmClient {
|
||||
private final UserSecretUsageService userSecretUsageService;
|
||||
|
||||
private volatile RestClient sharedRestClient;
|
||||
/** 首次尝试专用客户端(更短读超时,快速失败);配置未生效时与 sharedRestClient 相同。 */
|
||||
private volatile RestClient firstAttemptRestClient;
|
||||
|
||||
public SimilarAsinLlmClient(SimilarAsinProperties properties,
|
||||
ObjectMapper objectMapper,
|
||||
@@ -75,15 +78,16 @@ public class SimilarAsinLlmClient {
|
||||
Exception lastFailure = null;
|
||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
return invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat);
|
||||
return invokeChatOnce(model, system, userText, images, resolvedKey, responseFormat, attempt);
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
if (attempt >= attempts) {
|
||||
break;
|
||||
}
|
||||
log.warn("[similar-asin][llm] retryable failure attempt={} model={} err={}",
|
||||
attempt, model, failureMessage(ex));
|
||||
sleepBeforeRetry(attempt);
|
||||
long retryDelayMillis = LlmRetryBackoff.delayMillis(attempt);
|
||||
log.warn("[similar-asin][llm] retryable failure attempt={} model={} 下次重试等待={}ms err={}",
|
||||
attempt, model, retryDelayMillis, failureMessage(ex));
|
||||
sleepQuietly(retryDelayMillis);
|
||||
}
|
||||
}
|
||||
throw lastFailure == null
|
||||
@@ -110,13 +114,14 @@ public class SimilarAsinLlmClient {
|
||||
String userText,
|
||||
List<String> images,
|
||||
String apiKey,
|
||||
String responseFormat) {
|
||||
String responseFormat,
|
||||
int attempt) {
|
||||
recordSecretUsage();
|
||||
Map<String, Object> body = buildChatBody(model, system, userText, images, responseFormat);
|
||||
log.debug("[similar-asin][llm] request model={} url={} body={}",
|
||||
model, joinUrl(properties.getLlmHost(), "/v1/chat/completions"),
|
||||
log.debug("[similar-asin][llm] request attempt={} model={} url={} body={}",
|
||||
attempt, model, joinUrl(properties.getLlmHost(), "/v1/chat/completions"),
|
||||
writeJson(maskChatBody(body)));
|
||||
RestClient.RequestBodySpec request = restClient().post()
|
||||
RestClient.RequestBodySpec request = restClient(attempt).post()
|
||||
.uri(joinUrl(properties.getLlmHost(), "/v1/chat/completions"))
|
||||
.headers(headers -> {
|
||||
headers.setBearerAuth(stripBearer(apiKey));
|
||||
@@ -310,27 +315,52 @@ public class SimilarAsinLlmClient {
|
||||
userSecretUsageService.record(context.userId(), UserSecretModule.SIMILAR_ASIN.key(), 1);
|
||||
}
|
||||
|
||||
private RestClient restClient() {
|
||||
/**
|
||||
* 按尝试次数取客户端:第 1 次用「首次尝试超时」(快速失败,单次挂死代价更小),
|
||||
* 重试用完整读超时,避免把慢但成功的正常调用误杀(实测成功调用 p99≈34s)。
|
||||
*/
|
||||
private RestClient restClient(int attempt) {
|
||||
if (attempt == 1 && firstAttemptTimeoutMillis() < Math.max(1, properties.getLlmReadTimeoutMillis())) {
|
||||
RestClient client = firstAttemptRestClient;
|
||||
if (client != null) {
|
||||
return client;
|
||||
}
|
||||
synchronized (this) {
|
||||
if (firstAttemptRestClient == null) {
|
||||
firstAttemptRestClient = buildRestClient(firstAttemptTimeoutMillis());
|
||||
}
|
||||
return firstAttemptRestClient;
|
||||
}
|
||||
}
|
||||
RestClient client = sharedRestClient;
|
||||
if (client != null) {
|
||||
return client;
|
||||
}
|
||||
synchronized (this) {
|
||||
if (sharedRestClient == null) {
|
||||
RestClient.Builder builder = RestClient.builder()
|
||||
.requestFactory(HttpClientPool.requestFactory(properties.getLlmReadTimeoutMillis()));
|
||||
if (externalCallMetrics != null) {
|
||||
builder.requestInterceptor(externalCallMetrics.interceptor("llm"));
|
||||
}
|
||||
sharedRestClient = builder.build();
|
||||
sharedRestClient = buildRestClient(properties.getLlmReadTimeoutMillis());
|
||||
}
|
||||
return sharedRestClient;
|
||||
}
|
||||
}
|
||||
|
||||
private void sleepBeforeRetry(int attemptIndex) {
|
||||
private int firstAttemptTimeoutMillis() {
|
||||
int configured = properties.getLlmFirstAttemptReadTimeoutMillis();
|
||||
return configured > 0 ? configured : Math.max(1, properties.getLlmReadTimeoutMillis());
|
||||
}
|
||||
|
||||
private RestClient buildRestClient(int readTimeoutMillis) {
|
||||
RestClient.Builder builder = RestClient.builder()
|
||||
.requestFactory(HttpClientPool.requestFactory(readTimeoutMillis));
|
||||
if (externalCallMetrics != null) {
|
||||
builder.requestInterceptor(externalCallMetrics.interceptor("llm"));
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private void sleepQuietly(long delayMillis) {
|
||||
try {
|
||||
Thread.sleep(Math.max(1, attemptIndex) * 1500L);
|
||||
Thread.sleep(delayMillis);
|
||||
} catch (InterruptedException interruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("LLM retry interrupted", interruptedException);
|
||||
|
||||
Reference in New Issue
Block a user