diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java index 4a50336cc..13d1c430d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java @@ -27,7 +27,8 @@ public class BrandCheckProperties { * 一次批量检查的总耗时上限(毫秒):整批用尽后不再重试,剩余品牌按「查询失败」收尾。 * 上限的意义不是省时间,而是给「分片回传」这类同步调用方一个时延上界——上游卡死时 * 单品牌 10 次重试曾把一次回传拖到 103.5 秒(taskId 28599),客户端重试预算耗尽后 - * 中止了整个采集。首轮请求始终执行,故上游只是略慢时不会误降级。 + * 中止了整个采集。首轮请求在「预算 + 1 秒排队宽限」内执行;检查池被慢请求占满时 + * 不再无限等队列,剩余品牌按查询失败转补测,避免把同步回传线程拖到分钟级。 * 设为 0 或负数表示不限制。 */ private int totalTimeoutMillis = 90000; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java index ab2c35ce2..9b07ddceb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java @@ -22,9 +22,11 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicBoolean; @@ -37,12 +39,29 @@ public class BrandCheckClient { /** 品牌检查并发度:避免一次批量检查把第三方服务打满。 */ private static final int BRAND_CHECK_CONCURRENCY = 3; + /** 检查队列容量:有界防 OOM;满员时提交方立即降级,不再回落到调用线程执行。 */ + private static final int BRAND_CHECK_QUEUE_CAPACITY = 1000; + + /** + * 预算用尽后,尚未开始的检查最多再等这么久去拿首轮机会。 + * 超过即取消并转补测:排队等待必须有上界,否则慢上游会把同步回传线程拖到分钟级。 + */ + private static final long FIRST_ATTEMPT_QUEUE_GRACE_MILLIS = 1_000L; + + private static final String REASON_QUEUE_TIMEOUT = "品牌检查排队超预算"; + private static final String REASON_QUEUE_FULL = "品牌检查队列已满"; + private static final String REASON_RESPONSE_TIMEOUT = "品牌检查响应超时"; + private static final String REASON_INTERRUPTED = "品牌检查被中断"; + private static final String REASON_UNEXPECTED_FAILURE = "品牌检查异常"; + /** 用户没配代理时交回的原因文案;采集侧与补测队列都按它把行判查询失败。 */ public static final String NO_PROXY_REASON = "用户未配置代理"; private final BrandCheckProperties properties; private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics; private final UserApiSecretService userApiSecretService; + /** 提交线程上的拒绝标记:队列满时由拒绝策略置位,提交方据此立即降级(不新增 catch 出口)。 */ + private final ThreadLocal rejectedSubmission = new ThreadLocal<>(); private volatile RestClient sharedRestClient; public BrandCheckClient(BrandCheckProperties properties, @@ -53,14 +72,21 @@ public class BrandCheckClient { this.userApiSecretService = userApiSecretService; } - // 有界队列 + CallerRuns:大批量(几千品牌)检查时队列洪峰不无界堆内存, - // 队列打满时提交线程自己执行一个检查任务,上游天然限速。 + // 有界队列 + 拒绝即降级:大批量(几千品牌)检查时队列洪峰不无界堆内存。 + // 队列满时绝不能回退到 CallerRuns——那会让 Tomcat 回传/心跳线程替上游跑完整条重试阶梯, + // 正是 2026-10-20 分片回传被拖到 20 分钟的直接成因。拒绝策略只置位提交标记,由提交方 + // 换成「查询失败」结果转补测;不抛异常,也不在调用线程执行。 private final ExecutorService checkExecutor = new java.util.concurrent.ThreadPoolExecutor( BRAND_CHECK_CONCURRENCY, BRAND_CHECK_CONCURRENCY, 60L, java.util.concurrent.TimeUnit.SECONDS, - new java.util.concurrent.LinkedBlockingQueue<>(1000), + new java.util.concurrent.LinkedBlockingQueue<>(BRAND_CHECK_QUEUE_CAPACITY), namedThreadFactory("brand-check"), - new java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy()); + (task, executor) -> { + AtomicBoolean rejected = rejectedSubmission.get(); + if (rejected != null) { + rejected.set(true); + } + }); @PreDestroy void shutdownCheckExecutor() { @@ -132,8 +158,9 @@ public class BrandCheckClient { } /** - * 调用方自带整批耗时预算(毫秒,{@code <= 0} 表示不限制):预算只掐「重试」, - * 每个品牌的首轮请求始终执行,所以上游只是慢而不是坏时不会因此丢结论。 + * 调用方自带整批耗时预算(毫秒,{@code <= 0} 表示不限制):预算内等结果,用尽后不再重试; + * 尚未开始的品牌再给 1 秒排队宽限拿首轮机会,仍排不上队就按查询失败转补测。 + * 排队时间也计入上界——否则慢上游会把同步回传线程拖到分钟级。 * *

分片回传用它把一次 POST 的等待上界收到几秒:{@code total-timeout-millis} 那 90 秒 * 是给后台批量任务(外观合批、品牌补测调度)的预算,同步等响应的桌面客户端不该替它扛 @@ -223,17 +250,42 @@ public class BrandCheckClient { } return outcomes; } + long firstAttemptDeadlineNanos = deadlineNanos == Long.MAX_VALUE + ? Long.MAX_VALUE + : deadlineNanos + FIRST_ATTEMPT_QUEUE_GRACE_MILLIS * 1_000_000L; List> futures = new ArrayList<>(brands.size()); + List startedFlags = new ArrayList<>(brands.size()); // 一次批量一份判定:本批里只要有一个品牌跑满阶梯仍被判查询失败,其余品牌就不再 // 各自烧完整条重试阶梯(见 checkOneBrand 的短路分支)。 AtomicBoolean upstreamRejecting = new AtomicBoolean(false); for (String brand : brands) { - futures.add(CompletableFuture.supplyAsync( - () -> checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId), - checkExecutor)); + AtomicBoolean started = new AtomicBoolean(false); + AtomicBoolean rejected = new AtomicBoolean(false); + rejectedSubmission.set(rejected); + CompletableFuture future; + try { + future = CompletableFuture.supplyAsync( + () -> { + started.set(true); + return checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId); + }, + checkExecutor); + } finally { + rejectedSubmission.remove(); + } + if (rejected.get()) { + // 队列满:立刻降级,不许在调用线程(Tomcat 回传/心跳线程)里同步跑检查。 + future.cancel(false); + log.warn("[brand-check] 检查队列已满,按查询失败转补测 brand={} capacity={}", + brand, BRAND_CHECK_QUEUE_CAPACITY); + future = CompletableFuture.completedFuture(queryFailedOutcome(brand, REASON_QUEUE_FULL)); + } + futures.add(future); + startedFlags.add(started); } for (int index = 0; index < brands.size(); index++) { - outcomes.put(brands.get(index), awaitOutcome(futures.get(index))); + outcomes.put(brands.get(index), awaitOutcome(brands.get(index), futures.get(index), + startedFlags.get(index), deadlineNanos, firstAttemptDeadlineNanos)); } return outcomes; } @@ -347,16 +399,97 @@ public class BrandCheckClient { } } - private BrandCheckOutcome awaitOutcome(CompletableFuture future) { - try { - return future.join(); - } catch (CompletionException ex) { - log.warn("[brand-check] unexpected batch failure err={}", - ex.getCause() == null ? ex.getMessage() : String.valueOf(ex.getCause().getMessage())); - return new BrandCheckOutcome(List.of(), List.of(), ""); + /** + * 等一个品牌检查结果:预算内等完成;预算用尽后只给尚未开始的检查一个短暂的首轮宽限, + * 仍排不上队就取消并按查询失败转补测。已在执行的请求最多再等一次读超时。 + * + *

原实现用 {@code join()} 无限等队列:上游 16890 卡死时 3 个检查线程被慢请求占满, + * 后续分片回传全排在 1000 深的队列里 join,实测一次 chunk 被拖到 1284 秒;客户端 + * 180 秒超时后 5 次重试耗尽,任务被判「中间分批回传失败」,同批心跳也一起被拖住。 + */ + private BrandCheckOutcome awaitOutcome(String brand, + CompletableFuture future, + AtomicBoolean started, + long deadlineNanos, + long firstAttemptDeadlineNanos) { + // 0=预算内等待 1=排队宽限 2=取消或放行 3=等读超时 4=不限预算(旧行为) + int stage = deadlineNanos == Long.MAX_VALUE ? 4 : 0; + while (true) { + try { + switch (stage) { + case 4: + return future.get(); + case 0: { + long budgetWait = deadlineNanos - System.nanoTime(); + if (budgetWait > 0L) { + return future.get(budgetWait, TimeUnit.NANOSECONDS); + } + stage = 1; + break; + } + case 1: { + if (future.isDone()) { + return future.get(); + } + long graceWait = firstAttemptDeadlineNanos - System.nanoTime(); + if (graceWait > 0L) { + return future.get(graceWait, TimeUnit.NANOSECONDS); + } + stage = 2; + break; + } + case 2: { + if (!started.get() && future.cancel(false)) { + log.warn("[brand-check] 排队超过预算+{}ms 宽限,取消未开始的检查并转补测 brand={}", + FIRST_ATTEMPT_QUEUE_GRACE_MILLIS, brand); + return queryFailedOutcome(brand, REASON_QUEUE_TIMEOUT); + } + stage = 3; + break; + } + case 3: { + long readWait = Math.max(1L, properties.getReadTimeoutMillis()) * 1_000_000L; + return future.get(readWait, TimeUnit.NANOSECONDS); + } + default: + return queryFailedOutcome(brand, REASON_UNEXPECTED_FAILURE); + } + } catch (Exception failure) { + if (failure instanceof TimeoutException) { + if (stage == 0) { + stage = 1; + continue; + } + if (stage == 1) { + stage = 2; + continue; + } + if (stage == 3) { + future.cancel(true); + log.warn("[brand-check] 检查已开始但超过读超时仍未返回,取消并转补测 brand={}", + brand); + return queryFailedOutcome(brand, REASON_RESPONSE_TIMEOUT); + } + } + if (failure instanceof InterruptedException) { + Thread.currentThread().interrupt(); + future.cancel(true); + log.warn("[brand-check] 等待检查结果被中断,按查询失败转补测 brand={}", brand); + return queryFailedOutcome(brand, REASON_INTERRUPTED); + } + // 检查线程异常退出不能当成「品牌没问题」:旧实现返回空结论,调用方会把该品牌放行。 + Throwable cause = failure instanceof ExecutionException && failure.getCause() != null + ? failure.getCause() : failure; + log.warn("[brand-check] 批量检查异常,按查询失败转补测 brand={} err={}", brand, cause.getMessage()); + return queryFailedOutcome(brand, REASON_UNEXPECTED_FAILURE); + } } } + private BrandCheckOutcome queryFailedOutcome(String brand, String reason) { + return new BrandCheckOutcome(List.of(), List.of(brand), reason); + } + /** * 单个品牌的结论。 * diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java index 28a596e73..3531a9af7 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java @@ -4,12 +4,16 @@ import com.nanri.aiimage.config.BrandCheckProperties; import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService; import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; import java.io.IOException; import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; @@ -260,6 +264,71 @@ class BrandCheckClientTest { } } + /** + * 回归:品牌检查共享池被慢请求占满时,排队中的检查不得把同步回传线程拖到预算之外。 + * + *

2026-10-20 现网:16890 超时后 3 个检查线程被慢请求占住,后续分片回传在 + * {@code future.join()} 上等队列,品牌检测耗时从 447 秒一路涨到 2131 秒;客户端 + * 180 秒超时、5 次重试耗尽后报「中间分批回传失败」,同批心跳请求也被回传线程拖住。 + * 契约:排队等待必须有上界;预算用尽且仍未开始的检查取消后按「查询失败」返回, + * 由补测队列接手,而不是让 Tomcat 线程分钟级挂着等。 + */ + @Test + void saturatedCheckPoolDoesNotHoldCallerBeyondBatchBudget() throws Exception { + BrandCheckProperties props = new BrandCheckProperties(); + // 绝不指向生产 16890:即使回归用例失败,也不允许打出真实请求。 + props.setBaseUrl("http://127.0.0.1:1"); + props.setPath("/brand_check"); + props.setRetryTimes(1); + props.setRetryIntervalMillis(1); + props.setReadTimeoutMillis(1_000); + props.setTotalTimeoutMillis(0); + BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets()); + + ExecutorService pool = (ExecutorService) ReflectionTestUtils.getField(client, "checkExecutor"); + CountDownLatch occupied = new CountDownLatch(3); + CountDownLatch release = new CountDownLatch(1); + Thread releaser = new Thread(() -> { + try { + Thread.sleep(3_000); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + release.countDown(); + }, "brand-check-regression-releaser"); + releaser.setDaemon(true); + try { + for (int i = 0; i < 3; i++) { + pool.submit(() -> { + occupied.countDown(); + try { + release.await(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + }); + } + assertThat(occupied.await(2, TimeUnit.SECONDS)).as("3 个品牌检查线程应被占满").isTrue(); + + releaser.start(); + long startedAt = System.nanoTime(); + BrandCheckClient.BrandCheckBatchResult result = + client.checkAll(List.of("queued-brand"), "Terms", 200L, 7L); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + + assertThat(elapsedMillis) + .as("排队等待必须计入整批预算,不能跟着慢请求一起被拖到分钟级") + .isLessThan(2_000L); + assertThat(result.queryFaildData()) + .as("没轮到的品牌按查询失败返回,交给补测队列而不是静默放行") + .contains("queued-brand"); + assertThat(result.reason()).as("失败原因要能看出是排队超预算").contains("排队"); + } finally { + release.countDown(); + pool.shutdownNow(); + } + } + private HttpServer startBrandCheckServer(com.sun.net.httpserver.HttpHandler handler) throws IOException { HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); server.createContext("/brand_check", handler);