fix(brand-check): 检查池排队纳入批预算,队列满不再拖死回传
共享 3 线程池的排队等待不受采集批 10s 预算约束;16890 超时叠高后 单次品牌检查排队 7–35 分钟,Tomcat 线程被占满,客户端 180s×5 重试 耗尽,采集任务以「中间分批回传失败」整单失败。 现在排队最多等「批预算 + 1s 首轮宽限」,未开跑的品牌取消并按查询 失败交补测;队列满立即降级,不再 CallerRuns 到请求线程。
This commit is contained in:
+69
@@ -4,12 +4,16 @@ import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -260,6 +264,71 @@ class BrandCheckClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 回归:品牌检查共享池被慢请求占满时,排队中的检查不得把同步回传线程拖到预算之外。
|
||||
*
|
||||
* <p>2026-10-20 现网:16890 超时后 3 个检查线程被慢请求占住,后续分片回传在
|
||||
* {@code future.join()} 上等队列,品牌检测耗时从 447 秒一路涨到 2131 秒;客户端
|
||||
* 180 秒超时、5 次重试耗尽后报「中间分批回传失败」,同批心跳请求也被回传线程拖住。
|
||||
* 契约:排队等待必须有上界;预算用尽且仍未开始的检查取消后按「查询失败」返回,
|
||||
* 由补测队列接手,而不是让 Tomcat 线程分钟级挂着等。
|
||||
*/
|
||||
@Test
|
||||
void saturatedCheckPoolDoesNotHoldCallerBeyondBatchBudget() throws Exception {
|
||||
BrandCheckProperties props = new BrandCheckProperties();
|
||||
// 绝不指向生产 16890:即使回归用例失败,也不允许打出真实请求。
|
||||
props.setBaseUrl("http://127.0.0.1:1");
|
||||
props.setPath("/brand_check");
|
||||
props.setRetryTimes(1);
|
||||
props.setRetryIntervalMillis(1);
|
||||
props.setReadTimeoutMillis(1_000);
|
||||
props.setTotalTimeoutMillis(0);
|
||||
BrandCheckClient client = new BrandCheckClient(props, null, stubbedUserSecrets());
|
||||
|
||||
ExecutorService pool = (ExecutorService) ReflectionTestUtils.getField(client, "checkExecutor");
|
||||
CountDownLatch occupied = new CountDownLatch(3);
|
||||
CountDownLatch release = new CountDownLatch(1);
|
||||
Thread releaser = new Thread(() -> {
|
||||
try {
|
||||
Thread.sleep(3_000);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
release.countDown();
|
||||
}, "brand-check-regression-releaser");
|
||||
releaser.setDaemon(true);
|
||||
try {
|
||||
for (int i = 0; i < 3; i++) {
|
||||
pool.submit(() -> {
|
||||
occupied.countDown();
|
||||
try {
|
||||
release.await();
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
}
|
||||
assertThat(occupied.await(2, TimeUnit.SECONDS)).as("3 个品牌检查线程应被占满").isTrue();
|
||||
|
||||
releaser.start();
|
||||
long startedAt = System.nanoTime();
|
||||
BrandCheckClient.BrandCheckBatchResult result =
|
||||
client.checkAll(List.of("queued-brand"), "Terms", 200L, 7L);
|
||||
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
|
||||
|
||||
assertThat(elapsedMillis)
|
||||
.as("排队等待必须计入整批预算,不能跟着慢请求一起被拖到分钟级")
|
||||
.isLessThan(2_000L);
|
||||
assertThat(result.queryFaildData())
|
||||
.as("没轮到的品牌按查询失败返回,交给补测队列而不是静默放行")
|
||||
.contains("queued-brand");
|
||||
assertThat(result.reason()).as("失败原因要能看出是排队超预算").contains("排队");
|
||||
} finally {
|
||||
release.countDown();
|
||||
pool.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private HttpServer startBrandCheckServer(com.sun.net.httpserver.HttpHandler handler) throws IOException {
|
||||
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/brand_check", handler);
|
||||
|
||||
Reference in New Issue
Block a user