9c07b28a94
现网 31888/31856/31725 等任务永久停在 RUNNING,读数链是: java.net.http 的请求超时只封顶到响应头,响应体 read() 走 ArrayBlockingQueue.take(), 上游只发半截就发呆时这一读永久 park(12 条 similar-asin-llm-row 虚拟线程全停在那) ⇒ 无超时的 join() 把 task-queue 线程一起挂住 ⇒ 它手里的 task:similar-asin:<id> 被看门狗无条件续期(实测 11.3 小时,放过留痕 0 次)⇒ 该任务每次结果回传判 40902 (两小时 2370 次),且 12 个 taskQueueExecutor 许可被占满、44 条排队。 两道界各配非零对照: - HttpClientPool.readBodyBounded 把响应体读取挪进一次性虚拟线程、由调用方拿预算去等, 到点中断+关流退成可重试 IOException;三处同型 readResponseBodyBounded 改为委托它。 - LockHandle 直接绑住 HeldLock,任意线程 close 都真放手(旧实现按取锁线程的 ThreadLocal 找锁,跨线程 close 静默 no-op 就是无主锁本体);加持锁封顶 max(30 分钟, 6×TTL), 越界停止续期并 error 留痕,让键在 TTL 内自然过期。 裸跳过例外台账随之清空:它保的正是那条被现网证伪的静默 no-op。 门禁:干净 worktree(HEAD 97312821+本笔)后端全量 4686 例 / 0 红 / 17 skip / rc=0。
179 lines
7.3 KiB
Java
179 lines
7.3 KiB
Java
package com.nanri.aiimage.config;
|
||
|
||
import org.junit.jupiter.api.AfterAll;
|
||
import org.junit.jupiter.api.BeforeAll;
|
||
import org.junit.jupiter.api.Test;
|
||
|
||
import java.io.IOException;
|
||
import java.io.InputStream;
|
||
import java.io.OutputStream;
|
||
import java.net.InetSocketAddress;
|
||
import java.net.ServerSocket;
|
||
import java.net.Socket;
|
||
import java.net.URI;
|
||
import java.net.http.HttpRequest;
|
||
import java.net.http.HttpResponse;
|
||
import java.nio.charset.StandardCharsets;
|
||
import java.time.Duration;
|
||
import java.util.concurrent.atomic.AtomicInteger;
|
||
|
||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||
|
||
/**
|
||
* 响应体读取必须有时间界(2026-10-22 现网事故的直接复现件)。
|
||
*
|
||
* <p>旧形状:三处 {@code readResponseBodyBounded} 就地 while-read,只量体积不量时间。
|
||
* JDK 的请求超时封顶到「响应头」,头之后的流式 read() 落在
|
||
* {@code ArrayBlockingQueue.take()} 上——上游只发半截就发呆时这一读是永久 park,
|
||
* 现网 12 条 LLM 行线程全卡在这里,连带焊住任务锁与 task-queue 并发许可。
|
||
*
|
||
* <p>{@code HttpClientTimeoutEffectiveTest} 早就写下「流式 body 读取不受请求超时约束」这条
|
||
* 平台边界,但当时只登记了边界、没补界。这里补的就是那一格:断流必须失败,不是挂死。
|
||
*/
|
||
class HttpClientPoolBodyReadDeadlineTest {
|
||
|
||
private static final int BODY_CAP = 1 << 20;
|
||
|
||
private static ServerSocket stallServer;
|
||
private static ServerSocket okServer;
|
||
private static final AtomicInteger served = new AtomicInteger();
|
||
|
||
@BeforeAll
|
||
static void startServers() throws IOException {
|
||
stallServer = new ServerSocket(0);
|
||
okServer = new ServerSocket(0);
|
||
// 只发 1 个字节就发呆:Content-Length 声明 100000,客户端 read() 必然等不到结尾
|
||
acceptLoop(stallServer, true);
|
||
acceptLoop(okServer, false);
|
||
}
|
||
|
||
@AfterAll
|
||
static void stopServers() {
|
||
closeQuietly(stallServer);
|
||
closeQuietly(okServer);
|
||
}
|
||
|
||
private static void acceptLoop(ServerSocket server, boolean stall) {
|
||
Thread thread = new Thread(() -> {
|
||
while (!server.isClosed()) {
|
||
try {
|
||
Socket socket = server.accept();
|
||
served.incrementAndGet();
|
||
Thread connection = new Thread(() -> serve(socket, stall), "fake-upstream-connection");
|
||
connection.setDaemon(true);
|
||
connection.start();
|
||
} catch (Exception ex) {
|
||
return;
|
||
}
|
||
}
|
||
}, "body-read-fake-upstream");
|
||
thread.setDaemon(true);
|
||
thread.start();
|
||
}
|
||
|
||
private static void serve(Socket socket, boolean stall) {
|
||
try (Socket connection = socket) {
|
||
drainRequestHeaders(connection.getInputStream());
|
||
OutputStream out = connection.getOutputStream();
|
||
byte[] payload = stall ? "{".getBytes(StandardCharsets.UTF_8)
|
||
: "{\"ok\":true}\n".getBytes(StandardCharsets.UTF_8);
|
||
// 断流那一台:Content-Length 报到 100000,实际只发 1 个字节就留着连接不关,
|
||
// 模拟上游半开发呆;正常那一台的长度必须逐字节算,报多了客户端就在收尾那一读上等到连接被关。
|
||
int declared = stall ? 100_000 : payload.length;
|
||
byte[] head = ("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
|
||
+ "Content-Length: " + declared + "\r\n\r\n").getBytes(StandardCharsets.UTF_8);
|
||
out.write(head);
|
||
out.write(payload);
|
||
out.flush();
|
||
if (stall) {
|
||
Thread.sleep(20_000L);
|
||
}
|
||
} catch (Exception ex) {
|
||
// 客户端掐断是预期内的,不影响判据
|
||
}
|
||
}
|
||
|
||
/** 读到头部结束的空行(\r\n\r\n)为止:等满整份请求体是假上游最常见的自杀方式。 */
|
||
private static void drainRequestHeaders(InputStream in) throws IOException {
|
||
int newlines = 0;
|
||
int b;
|
||
while ((b = in.read()) != -1) {
|
||
if (b == '\n') {
|
||
if (++newlines >= 2) {
|
||
return;
|
||
}
|
||
} else if (b != '\r') {
|
||
newlines = 0;
|
||
}
|
||
}
|
||
}
|
||
|
||
private static InputStream openBody(ServerSocket server) throws Exception {
|
||
HttpRequest request = HttpRequest.newBuilder(
|
||
URI.create("http://127.0.0.1:" + server.getLocalPort() + "/v1/chat/completions"))
|
||
.timeout(Duration.ofSeconds(30))
|
||
.POST(HttpRequest.BodyPublishers.ofString("{}"))
|
||
.build();
|
||
HttpResponse<InputStream> response = HttpClientPool.sharedHttpClient()
|
||
.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||
assertEquals(200, response.statusCode());
|
||
return response.body();
|
||
}
|
||
|
||
@Test
|
||
void stalledResponseBodyFailsWithinBudgetInsteadOfHangingForever() throws Exception {
|
||
InputStream body = openBody(stallServer);
|
||
long startedAt = System.nanoTime();
|
||
IOException expected = assertThrows(IOException.class,
|
||
() -> HttpClientPool.readBodyBounded(body, Duration.ofMillis(400), BODY_CAP));
|
||
long elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000L;
|
||
assertTrue(expected.getMessage().contains("预算"), "应报掐断原因,实际 " + expected.getMessage());
|
||
assertTrue(elapsedMillis >= 300L, "不应早于配置的预算就失败:" + elapsedMillis + "ms");
|
||
assertTrue(elapsedMillis < 5_000L,
|
||
"断流必须在预算附近失败;挂到这儿就是永久 park(实测 " + elapsedMillis + "ms)");
|
||
}
|
||
|
||
@Test
|
||
void completeResponseBodyStillReadsThrough() throws Exception {
|
||
InputStream body = openBody(okServer);
|
||
String text = HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), BODY_CAP);
|
||
assertTrue(text.contains("\"ok\":true"), "正常响应不该被掐,实际读到 " + text);
|
||
}
|
||
|
||
@Test
|
||
void oversizedResponseBodyStillFailsOnByteCap() throws Exception {
|
||
InputStream body = openBody(okServer);
|
||
assertThrows(IOException.class,
|
||
() -> HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), 4));
|
||
}
|
||
|
||
@Test
|
||
void nullBodyIsStillAnEmptyReadNotATimeout() throws IOException {
|
||
assertEquals("", HttpClientPool.readBodyBounded(null, Duration.ofMillis(400), BODY_CAP));
|
||
}
|
||
|
||
@Test
|
||
void budgetFloorKeepsNonPositiveConfigFromBecomingInstantAbort() throws Exception {
|
||
InputStream body = openBody(okServer);
|
||
String text = HttpClientPool.readBodyBounded(body, Duration.ofMillis(-1), BODY_CAP);
|
||
assertTrue(text.contains("\"ok\":true"), "预算非法时钳到下限,不该把正常响应判成超时");
|
||
}
|
||
|
||
@Test
|
||
void fakeUpstreamsActuallyServedRequests() throws Exception {
|
||
assertTrue(served.get() >= 1, "假上游没被连上过,上面几条断言就都是空的");
|
||
}
|
||
|
||
private static void closeQuietly(ServerSocket server) {
|
||
try {
|
||
if (server != null) {
|
||
server.close();
|
||
}
|
||
} catch (IOException ignored) {
|
||
// 收尾关不掉不影响判据
|
||
}
|
||
}
|
||
}
|