fix(brand-check): 检查池排队纳入批预算,队列满不再拖死回传
共享 3 线程池的排队等待不受采集批 10s 预算约束;16890 超时叠高后 单次品牌检查排队 7–35 分钟,Tomcat 线程被占满,客户端 180s×5 重试 耗尽,采集任务以「中间分批回传失败」整单失败。 现在排队最多等「批预算 + 1s 首轮宽限」,未开跑的品牌取消并按查询 失败交补测;队列满立即降级,不再 CallerRuns 到请求线程。
This commit is contained in:
@@ -27,7 +27,8 @@ public class BrandCheckProperties {
|
||||
* 一次批量检查的总耗时上限(毫秒):整批用尽后不再重试,剩余品牌按「查询失败」收尾。
|
||||
* 上限的意义不是省时间,而是给「分片回传」这类同步调用方一个时延上界——上游卡死时
|
||||
* 单品牌 10 次重试曾把一次回传拖到 103.5 秒(taskId 28599),客户端重试预算耗尽后
|
||||
* 中止了整个采集。首轮请求始终执行,故上游只是略慢时不会误降级。
|
||||
* 中止了整个采集。首轮请求在「预算 + 1 秒排队宽限」内执行;检查池被慢请求占满时
|
||||
* 不再无限等队列,剩余品牌按查询失败转补测,避免把同步回传线程拖到分钟级。
|
||||
* 设为 0 或负数表示不限制。
|
||||
*/
|
||||
private int totalTimeoutMillis = 90000;
|
||||
|
||||
+151
-18
@@ -22,9 +22,11 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
@@ -37,12 +39,29 @@ public class BrandCheckClient {
|
||||
/** 品牌检查并发度:避免一次批量检查把第三方服务打满。 */
|
||||
private static final int BRAND_CHECK_CONCURRENCY = 3;
|
||||
|
||||
/** 检查队列容量:有界防 OOM;满员时提交方立即降级,不再回落到调用线程执行。 */
|
||||
private static final int BRAND_CHECK_QUEUE_CAPACITY = 1000;
|
||||
|
||||
/**
|
||||
* 预算用尽后,尚未开始的检查最多再等这么久去拿首轮机会。
|
||||
* 超过即取消并转补测:排队等待必须有上界,否则慢上游会把同步回传线程拖到分钟级。
|
||||
*/
|
||||
private static final long FIRST_ATTEMPT_QUEUE_GRACE_MILLIS = 1_000L;
|
||||
|
||||
private static final String REASON_QUEUE_TIMEOUT = "品牌检查排队超预算";
|
||||
private static final String REASON_QUEUE_FULL = "品牌检查队列已满";
|
||||
private static final String REASON_RESPONSE_TIMEOUT = "品牌检查响应超时";
|
||||
private static final String REASON_INTERRUPTED = "品牌检查被中断";
|
||||
private static final String REASON_UNEXPECTED_FAILURE = "品牌检查异常";
|
||||
|
||||
/** 用户没配代理时交回的原因文案;采集侧与补测队列都按它把行判查询失败。 */
|
||||
public static final String NO_PROXY_REASON = "用户未配置代理";
|
||||
|
||||
private final BrandCheckProperties properties;
|
||||
private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserApiSecretService userApiSecretService;
|
||||
/** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */
|
||||
private final ThreadLocal<AtomicBoolean> rejectedSubmission = new ThreadLocal<>();
|
||||
private volatile RestClient sharedRestClient;
|
||||
|
||||
public BrandCheckClient(BrandCheckProperties properties,
|
||||
@@ -53,14 +72,21 @@ public class BrandCheckClient {
|
||||
this.userApiSecretService = userApiSecretService;
|
||||
}
|
||||
|
||||
// 有界队列 + CallerRuns:大批量(几千品牌)检查时队列洪峰不无界堆内存,
|
||||
// 队列打满时提交线程自己执行一个检查任务,上游天然限速。
|
||||
// 有界队列 + 拒绝即降级:大批量(几千品牌)检查时队列洪峰不无界堆内存。
|
||||
// 队列满时绝不能回退到 CallerRuns——那会让 Tomcat 回传/心跳线程替上游跑完整条重试阶梯,
|
||||
// 正是 2026-10-20 分片回传被拖到 20 分钟的直接成因。拒绝策略只置位提交标记,由提交方
|
||||
// 换成「查询失败」结果转补测;不抛异常,也不在调用线程执行。
|
||||
private final ExecutorService checkExecutor = new java.util.concurrent.ThreadPoolExecutor(
|
||||
BRAND_CHECK_CONCURRENCY, BRAND_CHECK_CONCURRENCY,
|
||||
60L, java.util.concurrent.TimeUnit.SECONDS,
|
||||
new java.util.concurrent.LinkedBlockingQueue<>(1000),
|
||||
new java.util.concurrent.LinkedBlockingQueue<>(BRAND_CHECK_QUEUE_CAPACITY),
|
||||
namedThreadFactory("brand-check"),
|
||||
new java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy());
|
||||
(task, executor) -> {
|
||||
AtomicBoolean rejected = rejectedSubmission.get();
|
||||
if (rejected != null) {
|
||||
rejected.set(true);
|
||||
}
|
||||
});
|
||||
|
||||
@PreDestroy
|
||||
void shutdownCheckExecutor() {
|
||||
@@ -132,8 +158,9 @@ public class BrandCheckClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用方自带整批耗时预算(毫秒,{@code <= 0} 表示不限制):预算只掐「重试」,
|
||||
* 每个品牌的首轮请求始终执行,所以上游只是慢而不是坏时不会因此丢结论。
|
||||
* 调用方自带整批耗时预算(毫秒,{@code <= 0} 表示不限制):预算内等结果,用尽后不再重试;
|
||||
* 尚未开始的品牌再给 1 秒排队宽限拿首轮机会,仍排不上队就按查询失败转补测。
|
||||
* 排队时间也计入上界——否则慢上游会把同步回传线程拖到分钟级。
|
||||
*
|
||||
* <p>分片回传用它把一次 POST 的等待上界收到几秒:{@code total-timeout-millis} 那 90 秒
|
||||
* 是给后台批量任务(外观合批、品牌补测调度)的预算,同步等响应的桌面客户端不该替它扛
|
||||
@@ -223,17 +250,42 @@ public class BrandCheckClient {
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
long firstAttemptDeadlineNanos = deadlineNanos == Long.MAX_VALUE
|
||||
? Long.MAX_VALUE
|
||||
: deadlineNanos + FIRST_ATTEMPT_QUEUE_GRACE_MILLIS * 1_000_000L;
|
||||
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(brands.size());
|
||||
List<AtomicBoolean> startedFlags = new ArrayList<>(brands.size());
|
||||
// 一次批量一份判定:本批里只要有一个品牌跑满阶梯仍被判查询失败,其余品牌就不再
|
||||
// 各自烧完整条重试阶梯(见 checkOneBrand 的短路分支)。
|
||||
AtomicBoolean upstreamRejecting = new AtomicBoolean(false);
|
||||
for (String brand : brands) {
|
||||
futures.add(CompletableFuture.supplyAsync(
|
||||
() -> checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId),
|
||||
checkExecutor));
|
||||
AtomicBoolean started = new AtomicBoolean(false);
|
||||
AtomicBoolean rejected = new AtomicBoolean(false);
|
||||
rejectedSubmission.set(rejected);
|
||||
CompletableFuture<BrandCheckOutcome> future;
|
||||
try {
|
||||
future = CompletableFuture.supplyAsync(
|
||||
() -> {
|
||||
started.set(true);
|
||||
return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId);
|
||||
},
|
||||
checkExecutor);
|
||||
} finally {
|
||||
rejectedSubmission.remove();
|
||||
}
|
||||
if (rejected.get()) {
|
||||
// 队列满:立刻降级,不许在调用线程(Tomcat 回传/心跳线程)里同步跑检查。
|
||||
future.cancel(false);
|
||||
log.warn("[brand-check] 检查队列已满,按查询失败转补测 brand={} capacity={}",
|
||||
brand, BRAND_CHECK_QUEUE_CAPACITY);
|
||||
future = CompletableFuture.completedFuture(queryFailedOutcome(brand, REASON_QUEUE_FULL));
|
||||
}
|
||||
futures.add(future);
|
||||
startedFlags.add(started);
|
||||
}
|
||||
for (int index = 0; index < brands.size(); index++) {
|
||||
outcomes.put(brands.get(index), awaitOutcome(futures.get(index)));
|
||||
outcomes.put(brands.get(index), awaitOutcome(brands.get(index), futures.get(index),
|
||||
startedFlags.get(index), deadlineNanos, firstAttemptDeadlineNanos));
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
@@ -347,16 +399,97 @@ public class BrandCheckClient {
|
||||
}
|
||||
}
|
||||
|
||||
private BrandCheckOutcome awaitOutcome(CompletableFuture<BrandCheckOutcome> future) {
|
||||
try {
|
||||
return future.join();
|
||||
} catch (CompletionException ex) {
|
||||
log.warn("[brand-check] unexpected batch failure err={}",
|
||||
ex.getCause() == null ? ex.getMessage() : String.valueOf(ex.getCause().getMessage()));
|
||||
return new BrandCheckOutcome(List.of(), List.of(), "");
|
||||
/**
|
||||
* 等一个品牌检查结果:预算内等完成;预算用尽后只给尚未开始的检查一个短暂的首轮宽限,
|
||||
* 仍排不上队就取消并按查询失败转补测。已在执行的请求最多再等一次读超时。
|
||||
*
|
||||
* <p>原实现用 {@code join()} 无限等队列:上游 16890 卡死时 3 个检查线程被慢请求占满,
|
||||
* 后续分片回传全排在 1000 深的队列里 join,实测一次 chunk 被拖到 1284 秒;客户端
|
||||
* 180 秒超时后 5 次重试耗尽,任务被判「中间分批回传失败」,同批心跳也一起被拖住。
|
||||
*/
|
||||
private BrandCheckOutcome awaitOutcome(String brand,
|
||||
CompletableFuture<BrandCheckOutcome> future,
|
||||
AtomicBoolean started,
|
||||
long deadlineNanos,
|
||||
long firstAttemptDeadlineNanos) {
|
||||
// 0=预算内等待 1=排队宽限 2=取消或放行 3=等读超时 4=不限预算(旧行为)
|
||||
int stage = deadlineNanos == Long.MAX_VALUE ? 4 : 0;
|
||||
while (true) {
|
||||
try {
|
||||
switch (stage) {
|
||||
case 4:
|
||||
return future.get();
|
||||
case 0: {
|
||||
long budgetWait = deadlineNanos - System.nanoTime();
|
||||
if (budgetWait > 0L) {
|
||||
return future.get(budgetWait, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
stage = 1;
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
if (future.isDone()) {
|
||||
return future.get();
|
||||
}
|
||||
long graceWait = firstAttemptDeadlineNanos - System.nanoTime();
|
||||
if (graceWait > 0L) {
|
||||
return future.get(graceWait, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
stage = 2;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
if (!started.get() && future.cancel(false)) {
|
||||
log.warn("[brand-check] 排队超过预算+{}ms 宽限,取消未开始的检查并转补测 brand={}",
|
||||
FIRST_ATTEMPT_QUEUE_GRACE_MILLIS, brand);
|
||||
return queryFailedOutcome(brand, REASON_QUEUE_TIMEOUT);
|
||||
}
|
||||
stage = 3;
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
long readWait = Math.max(1L, properties.getReadTimeoutMillis()) * 1_000_000L;
|
||||
return future.get(readWait, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
default:
|
||||
return queryFailedOutcome(brand, REASON_UNEXPECTED_FAILURE);
|
||||
}
|
||||
} catch (Exception failure) {
|
||||
if (failure instanceof TimeoutException) {
|
||||
if (stage == 0) {
|
||||
stage = 1;
|
||||
continue;
|
||||
}
|
||||
if (stage == 1) {
|
||||
stage = 2;
|
||||
continue;
|
||||
}
|
||||
if (stage == 3) {
|
||||
future.cancel(true);
|
||||
log.warn("[brand-check] 检查已开始但超过读超时仍未返回,取消并转补测 brand={}",
|
||||
brand);
|
||||
return queryFailedOutcome(brand, REASON_RESPONSE_TIMEOUT);
|
||||
}
|
||||
}
|
||||
if (failure instanceof InterruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
future.cancel(true);
|
||||
log.warn("[brand-check] 等待检查结果被中断,按查询失败转补测 brand={}", brand);
|
||||
return queryFailedOutcome(brand, REASON_INTERRUPTED);
|
||||
}
|
||||
// 检查线程异常退出不能当成「品牌没问题」:旧实现返回空结论,调用方会把该品牌放行。
|
||||
Throwable cause = failure instanceof ExecutionException && failure.getCause() != null
|
||||
? failure.getCause() : failure;
|
||||
log.warn("[brand-check] 批量检查异常,按查询失败转补测 brand={} err={}", brand, cause.getMessage());
|
||||
return queryFailedOutcome(brand, REASON_UNEXPECTED_FAILURE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private BrandCheckOutcome queryFailedOutcome(String brand, String reason) {
|
||||
return new BrandCheckOutcome(List.of(), List.of(brand), reason);
|
||||
}
|
||||
|
||||
/**
|
||||
* 单个品牌的结论。
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user