fix(brand-check): 检查池排队纳入批预算,队列满不再拖死回传

共享 3 线程池的排队等待不受采集批 10s 预算约束;16890 超时叠高后
单次品牌检查排队 7–35 分钟,Tomcat 线程被占满,客户端 180s×5 重试
耗尽,采集任务以「中间分批回传失败」整单失败。

现在排队最多等「批预算 + 1s 首轮宽限」,未开跑的品牌取消并按查询
失败交补测;队列满立即降级,不再 CallerRuns 到请求线程。
This commit is contained in:
2026-10-21 10:19:30 +08:00
parent 4d70a0d3f8
commit 73838916ef
3 changed files with 222 additions and 19 deletions
@@ -4,12 +4,16 @@ import com.nanri.aiimage.config.BrandCheckProperties;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
import com.sun.net.httpserver.HttpServer;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
@@ -260,6 +264,71 @@ class BrandCheckClientTest {
}
}
/**
* 回归:品牌检查共享池被慢请求占满时,排队中的检查不得把同步回传线程拖到预算之外。
*
* <p>2026-10-20 现网:16890 超时后 3 个检查线程被慢请求占住,后续分片回传在
* {@code future.join()} 上等队列,品牌检测耗时从 447 秒一路涨到 2131 秒;客户端
* 180 秒超时、5 次重试耗尽后报「中间分批回传失败」,同批心跳请求也被回传线程拖住。
* 契约:排队等待必须有上界;预算用尽且仍未开始的检查取消后按「查询失败」返回,
* 由补测队列接手,而不是让 Tomcat 线程分钟级挂着等。
*/
@Test
void saturatedCheckPoolDoesNotHoldCallerBeyondBatchBudget() throws Exception {
BrandCheckProperties props = new BrandCheckProperties();
// 绝不指向生产 16890:即使回归用例失败,也不允许打出真实请求。
props.setBaseUrl("http://127.0.0.1:1");
props.setPath("/brand_check");
props.setRetryTimes(1);
props.setRetryIntervalMillis(1);
props.setReadTimeoutMillis(1_000);
props.setTotalTimeoutMillis(0);
BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets());
ExecutorService pool = (ExecutorService) ReflectionTestUtils.getField(client, "checkExecutor");
CountDownLatch occupied = new CountDownLatch(3);
CountDownLatch release = new CountDownLatch(1);
Thread releaser = new Thread(() -> {
try {
Thread.sleep(3_000);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
release.countDown();
}, "brand-check-regression-releaser");
releaser.setDaemon(true);
try {
for (int i = 0; i < 3; i++) {
pool.submit(() -> {
occupied.countDown();
try {
release.await();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
});
}
assertThat(occupied.await(2, TimeUnit.SECONDS)).as("3 个品牌检查线程应被占满").isTrue();
releaser.start();
long startedAt = System.nanoTime();
BrandCheckClient.BrandCheckBatchResult result =
client.checkAll(List.of("queued-brand"), "Terms", 200L, 7L);
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
assertThat(elapsedMillis)
.as("排队等待必须计入整批预算,不能跟着慢请求一起被拖到分钟级")
.isLessThan(2_000L);
assertThat(result.queryFaildData())
.as("没轮到的品牌按查询失败返回,交给补测队列而不是静默放行")
.contains("queued-brand");
assertThat(result.reason()).as("失败原因要能看出是排队超预算").contains("排队");
} finally {
release.countDown();
pool.shutdownNow();
}
}
private HttpServer startBrandCheckServer(com.sun.net.httpserver.HttpHandler handler) throws IOException {
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/brand_check", handler);