fix(brand-check): 检查池排队纳入批预算,队列满不再拖死回传

共享 3 线程池的排队等待不受采集批 10s 预算约束;16890 超时叠高后
单次品牌检查排队 7–35 分钟,Tomcat 线程被占满,客户端 180s×5 重试
耗尽,采集任务以「中间分批回传失败」整单失败。

现在排队最多等「批预算 + 1s 首轮宽限」,未开跑的品牌取消并按查询
失败交补测;队列满立即降级,不再 CallerRuns 到请求线程。
This commit is contained in:
2026-10-21 10:19:30 +08:00
parent 4d70a0d3f8
commit 73838916ef
3 changed files with 222 additions and 19 deletions
@@ -22,9 +22,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -37,12 +39,29 @@ public class BrandCheckClient {
/** 品牌检查并发度:避免一次批量检查把第三方服务打满。 */
private static final int BRAND_CHECK_CONCURRENCY = 3;
/** 检查队列容量:有界防 OOM;满员时提交方立即降级,不再回落到调用线程执行。 */
private static final int BRAND_CHECK_QUEUE_CAPACITY = 1000;
/**
* 预算用尽后,尚未开始的检查最多再等这么久去拿首轮机会。
* 超过即取消并转补测:排队等待必须有上界,否则慢上游会把同步回传线程拖到分钟级。
*/
private static final long FIRST_ATTEMPT_QUEUE_GRACE_MILLIS = 1_000L;
private static final String REASON_QUEUE_TIMEOUT = "品牌检查排队超预算";
private static final String REASON_QUEUE_FULL = "品牌检查队列已满";
private static final String REASON_RESPONSE_TIMEOUT = "品牌检查响应超时";
private static final String REASON_INTERRUPTED = "品牌检查被中断";
private static final String REASON_UNEXPECTED_FAILURE = "品牌检查异常";
/** 用户没配代理时交回的原因文案;采集侧与补测队列都按它把行判查询失败。 */
public static final String NO_PROXY_REASON = "用户未配置代理";
private final BrandCheckProperties properties;
private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics;
private final UserApiSecretService userApiSecretService;
/** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */
private final ThreadLocal<AtomicBoolean> rejectedSubmission = new ThreadLocal<>();
private volatile RestClient sharedRestClient;
public BrandCheckClient(BrandCheckProperties properties,
@@ -53,14 +72,21 @@ public class BrandCheckClient {
this.userApiSecretService = userApiSecretService;
}
// 有界队列 + CallerRuns:大批量(几千品牌)检查时队列洪峰不无界堆内存,
// 队列打满时提交线程自己执行一个检查任务,上游天然限速。
// 有界队列 + 拒绝即降级:大批量(几千品牌)检查时队列洪峰不无界堆内存。
// 队列满时绝不能回退到 CallerRuns——那会让 Tomcat 回传/心跳线程替上游跑完整条重试阶梯,
// 正是 2026-10-20 分片回传被拖到 20 分钟的直接成因。拒绝策略只置位提交标记,由提交方
// 换成「查询失败」结果转补测;不抛异常,也不在调用线程执行。
private final ExecutorService checkExecutor = new java.util.concurrent.ThreadPoolExecutor(
BRAND_CHECK_CONCURRENCY, BRAND_CHECK_CONCURRENCY,
60L, java.util.concurrent.TimeUnit.SECONDS,
new java.util.concurrent.LinkedBlockingQueue<>(1000),
new java.util.concurrent.LinkedBlockingQueue<>(BRAND_CHECK_QUEUE_CAPACITY),
namedThreadFactory("brand-check"),
new java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy());
(task, executor) -> {
AtomicBoolean rejected = rejectedSubmission.get();
if (rejected != null) {
rejected.set(true);
}
});
@PreDestroy
void shutdownCheckExecutor() {
@@ -132,8 +158,9 @@ public class BrandCheckClient {
}
/**
* 调用方自带整批耗时预算(毫秒,{@code <= 0} 表示不限制):预算只掐「重试」,
* 每个品牌的首轮请求始终执行,所以上游只是慢而不是坏时不会因此丢结论。
* 调用方自带整批耗时预算(毫秒,{@code <= 0} 表示不限制):预算内等结果,用尽后不再重试;
* 尚未开始的品牌再给 1 秒排队宽限拿首轮机会,仍排不上队就按查询失败转补测。
* 排队时间也计入上界——否则慢上游会把同步回传线程拖到分钟级。
*
* <p>分片回传用它把一次 POST 的等待上界收到几秒:{@code total-timeout-millis} 那 90 秒
* 是给后台批量任务(外观合批、品牌补测调度)的预算,同步等响应的桌面客户端不该替它扛
@@ -223,17 +250,42 @@ public class BrandCheckClient {
}
return outcomes;
}
long firstAttemptDeadlineNanos = deadlineNanos == Long.MAX_VALUE
? Long.MAX_VALUE
: deadlineNanos + FIRST_ATTEMPT_QUEUE_GRACE_MILLIS * 1_000_000L;
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(brands.size());
List<AtomicBoolean> startedFlags = new ArrayList<>(brands.size());
// 一次批量一份判定:本批里只要有一个品牌跑满阶梯仍被判查询失败,其余品牌就不再
// 各自烧完整条重试阶梯(见 checkOneBrand 的短路分支)。
AtomicBoolean upstreamRejecting = new AtomicBoolean(false);
for (String brand : brands) {
futures.add(CompletableFuture.supplyAsync(
() -> checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId),
checkExecutor));
AtomicBoolean started = new AtomicBoolean(false);
AtomicBoolean rejected = new AtomicBoolean(false);
rejectedSubmission.set(rejected);
CompletableFuture<BrandCheckOutcome> future;
try {
future = CompletableFuture.supplyAsync(
() -> {
started.set(true);
return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId);
},
checkExecutor);
} finally {
rejectedSubmission.remove();
}
if (rejected.get()) {
// 队列满:立刻降级,不许在调用线程(Tomcat 回传/心跳线程)里同步跑检查。
future.cancel(false);
log.warn("[brand-check] 检查队列已满,按查询失败转补测 brand={} capacity={}",
brand, BRAND_CHECK_QUEUE_CAPACITY);
future = CompletableFuture.completedFuture(queryFailedOutcome(brand, REASON_QUEUE_FULL));
}
futures.add(future);
startedFlags.add(started);
}
for (int index = 0; index < brands.size(); index++) {
outcomes.put(brands.get(index), awaitOutcome(futures.get(index)));
outcomes.put(brands.get(index), awaitOutcome(brands.get(index), futures.get(index),
startedFlags.get(index), deadlineNanos, firstAttemptDeadlineNanos));
}
return outcomes;
}
@@ -347,16 +399,97 @@ public class BrandCheckClient {
}
}
private BrandCheckOutcome awaitOutcome(CompletableFuture<BrandCheckOutcome> future) {
try {
return future.join();
} catch (CompletionException ex) {
log.warn("[brand-check] unexpected batch failure err={}",
ex.getCause() == null ? ex.getMessage() : String.valueOf(ex.getCause().getMessage()));
return new BrandCheckOutcome(List.of(), List.of(), "");
/**
* 等一个品牌检查结果:预算内等完成;预算用尽后只给尚未开始的检查一个短暂的首轮宽限,
* 仍排不上队就取消并按查询失败转补测。已在执行的请求最多再等一次读超时。
*
* <p>原实现用 {@code join()} 无限等队列:上游 16890 卡死时 3 个检查线程被慢请求占满,
* 后续分片回传全排在 1000 深的队列里 join,实测一次 chunk 被拖到 1284 秒;客户端
* 180 秒超时后 5 次重试耗尽,任务被判「中间分批回传失败」,同批心跳也一起被拖住。
*/
private BrandCheckOutcome awaitOutcome(String brand,
CompletableFuture<BrandCheckOutcome> future,
AtomicBoolean started,
long deadlineNanos,
long firstAttemptDeadlineNanos) {
// 0=预算内等待 1=排队宽限 2=取消或放行 3=等读超时 4=不限预算(旧行为)
int stage = deadlineNanos == Long.MAX_VALUE ? 4 : 0;
while (true) {
try {
switch (stage) {
case 4:
return future.get();
case 0: {
long budgetWait = deadlineNanos - System.nanoTime();
if (budgetWait > 0L) {
return future.get(budgetWait, TimeUnit.NANOSECONDS);
}
stage = 1;
break;
}
case 1: {
if (future.isDone()) {
return future.get();
}
long graceWait = firstAttemptDeadlineNanos - System.nanoTime();
if (graceWait > 0L) {
return future.get(graceWait, TimeUnit.NANOSECONDS);
}
stage = 2;
break;
}
case 2: {
if (!started.get() && future.cancel(false)) {
log.warn("[brand-check] 排队超过预算+{}ms 宽限,取消未开始的检查并转补测 brand={}",
FIRST_ATTEMPT_QUEUE_GRACE_MILLIS, brand);
return queryFailedOutcome(brand, REASON_QUEUE_TIMEOUT);
}
stage = 3;
break;
}
case 3: {
long readWait = Math.max(1L, properties.getReadTimeoutMillis()) * 1_000_000L;
return future.get(readWait, TimeUnit.NANOSECONDS);
}
default:
return queryFailedOutcome(brand, REASON_UNEXPECTED_FAILURE);
}
} catch (Exception failure) {
if (failure instanceof TimeoutException) {
if (stage == 0) {
stage = 1;
continue;
}
if (stage == 1) {
stage = 2;
continue;
}
if (stage == 3) {
future.cancel(true);
log.warn("[brand-check] 检查已开始但超过读超时仍未返回,取消并转补测 brand={}",
brand);
return queryFailedOutcome(brand, REASON_RESPONSE_TIMEOUT);
}
}
if (failure instanceof InterruptedException) {
Thread.currentThread().interrupt();
future.cancel(true);
log.warn("[brand-check] 等待检查结果被中断,按查询失败转补测 brand={}", brand);
return queryFailedOutcome(brand, REASON_INTERRUPTED);
}
// 检查线程异常退出不能当成「品牌没问题」:旧实现返回空结论,调用方会把该品牌放行。
Throwable cause = failure instanceof ExecutionException && failure.getCause() != null
? failure.getCause() : failure;
log.warn("[brand-check] 批量检查异常,按查询失败转补测 brand={} err={}", brand, cause.getMessage());
return queryFailedOutcome(brand, REASON_UNEXPECTED_FAILURE);
}
}
}
private BrandCheckOutcome queryFailedOutcome(String brand, String reason) {
return new BrandCheckOutcome(List.of(), List.of(brand), reason);
}
/**
* 单个品牌的结论。
*