Files
crawler-plugin/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientPoolBodyReadDeadlineTest.java
T
huangzd1997 9c07b28a94 fix(task-lock): LLM 响应体补时间界+任务锁跨线程可放与持封顶,掐掉 11 小时无主锁
现网 31888/31856/31725 等任务永久停在 RUNNING,读数链是:
java.net.http 的请求超时只封顶到响应头,响应体 read() 走 ArrayBlockingQueue.take(),
上游只发半截就发呆时这一读永久 park(12 条 similar-asin-llm-row 虚拟线程全停在那)
⇒ 无超时的 join() 把 task-queue 线程一起挂住 ⇒ 它手里的 task:similar-asin:<id>
被看门狗无条件续期(实测 11.3 小时,放过留痕 0 次)⇒ 该任务每次结果回传判 40902
(两小时 2370 次),且 12 个 taskQueueExecutor 许可被占满、44 条排队。

两道界各配非零对照:
- HttpClientPool.readBodyBounded 把响应体读取挪进一次性虚拟线程、由调用方拿预算去等,
  到点中断+关流退成可重试 IOException;三处同型 readResponseBodyBounded 改为委托它。
- LockHandle 直接绑住 HeldLock,任意线程 close 都真放手(旧实现按取锁线程的 ThreadLocal
  找锁,跨线程 close 静默 no-op 就是无主锁本体);加持锁封顶 max(30 分钟, 6×TTL),
  越界停止续期并 error 留痕,让键在 TTL 内自然过期。

裸跳过例外台账随之清空:它保的正是那条被现网证伪的静默 no-op。
门禁:干净 worktree(HEAD 97312821+本笔)后端全量 4686 例 / 0 红 / 17 skip / rc=0。
2026-10-22 15:49:24 +08:00

179 lines
7.3 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package com.nanri.aiimage.config;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URI;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 响应体读取必须有时间界(2026-10-22 现网事故的直接复现件)。
*
* <p>旧形状:三处 {@code readResponseBodyBounded} 就地 while-read,只量体积不量时间。
* JDK 的请求超时封顶到「响应头」,头之后的流式 read() 落在
* {@code ArrayBlockingQueue.take()} 上——上游只发半截就发呆时这一读是永久 park,
* 现网 12 条 LLM 行线程全卡在这里,连带焊住任务锁与 task-queue 并发许可。
*
* <p>{@code HttpClientTimeoutEffectiveTest} 早就写下「流式 body 读取不受请求超时约束」这条
* 平台边界,但当时只登记了边界、没补界。这里补的就是那一格:断流必须失败,不是挂死。
*/
class HttpClientPoolBodyReadDeadlineTest {
private static final int BODY_CAP = 1 << 20;
private static ServerSocket stallServer;
private static ServerSocket okServer;
private static final AtomicInteger served = new AtomicInteger();
@BeforeAll
static void startServers() throws IOException {
stallServer = new ServerSocket(0);
okServer = new ServerSocket(0);
// 只发 1 个字节就发呆:Content-Length 声明 100000,客户端 read() 必然等不到结尾
acceptLoop(stallServer, true);
acceptLoop(okServer, false);
}
@AfterAll
static void stopServers() {
closeQuietly(stallServer);
closeQuietly(okServer);
}
private static void acceptLoop(ServerSocket server, boolean stall) {
Thread thread = new Thread(() -> {
while (!server.isClosed()) {
try {
Socket socket = server.accept();
served.incrementAndGet();
Thread connection = new Thread(() -> serve(socket, stall), "fake-upstream-connection");
connection.setDaemon(true);
connection.start();
} catch (Exception ex) {
return;
}
}
}, "body-read-fake-upstream");
thread.setDaemon(true);
thread.start();
}
private static void serve(Socket socket, boolean stall) {
try (Socket connection = socket) {
drainRequestHeaders(connection.getInputStream());
OutputStream out = connection.getOutputStream();
byte[] payload = stall ? "{".getBytes(StandardCharsets.UTF_8)
: "{\"ok\":true}\n".getBytes(StandardCharsets.UTF_8);
// 断流那一台:Content-Length 报到 100000,实际只发 1 个字节就留着连接不关,
// 模拟上游半开发呆;正常那一台的长度必须逐字节算,报多了客户端就在收尾那一读上等到连接被关。
int declared = stall ? 100_000 : payload.length;
byte[] head = ("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
+ "Content-Length: " + declared + "\r\n\r\n").getBytes(StandardCharsets.UTF_8);
out.write(head);
out.write(payload);
out.flush();
if (stall) {
Thread.sleep(20_000L);
}
} catch (Exception ex) {
// 客户端掐断是预期内的,不影响判据
}
}
/** 读到头部结束的空行(\r\n\r\n)为止:等满整份请求体是假上游最常见的自杀方式。 */
private static void drainRequestHeaders(InputStream in) throws IOException {
int newlines = 0;
int b;
while ((b = in.read()) != -1) {
if (b == '\n') {
if (++newlines >= 2) {
return;
}
} else if (b != '\r') {
newlines = 0;
}
}
}
private static InputStream openBody(ServerSocket server) throws Exception {
HttpRequest request = HttpRequest.newBuilder(
URI.create("http://127.0.0.1:" + server.getLocalPort() + "/v1/chat/completions"))
.timeout(Duration.ofSeconds(30))
.POST(HttpRequest.BodyPublishers.ofString("{}"))
.build();
HttpResponse<InputStream> response = HttpClientPool.sharedHttpClient()
.send(request, HttpResponse.BodyHandlers.ofInputStream());
assertEquals(200, response.statusCode());
return response.body();
}
@Test
void stalledResponseBodyFailsWithinBudgetInsteadOfHangingForever() throws Exception {
InputStream body = openBody(stallServer);
long startedAt = System.nanoTime();
IOException expected = assertThrows(IOException.class,
() -> HttpClientPool.readBodyBounded(body, Duration.ofMillis(400), BODY_CAP));
long elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000L;
assertTrue(expected.getMessage().contains("预算"), "应报掐断原因,实际 " + expected.getMessage());
assertTrue(elapsedMillis >= 300L, "不应早于配置的预算就失败:" + elapsedMillis + "ms");
assertTrue(elapsedMillis < 5_000L,
"断流必须在预算附近失败;挂到这儿就是永久 park(实测 " + elapsedMillis + "ms)");
}
@Test
void completeResponseBodyStillReadsThrough() throws Exception {
InputStream body = openBody(okServer);
String text = HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), BODY_CAP);
assertTrue(text.contains("\"ok\":true"), "正常响应不该被掐,实际读到 " + text);
}
@Test
void oversizedResponseBodyStillFailsOnByteCap() throws Exception {
InputStream body = openBody(okServer);
assertThrows(IOException.class,
() -> HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), 4));
}
@Test
void nullBodyIsStillAnEmptyReadNotATimeout() throws IOException {
assertEquals("", HttpClientPool.readBodyBounded(null, Duration.ofMillis(400), BODY_CAP));
}
@Test
void budgetFloorKeepsNonPositiveConfigFromBecomingInstantAbort() throws Exception {
InputStream body = openBody(okServer);
String text = HttpClientPool.readBodyBounded(body, Duration.ofMillis(-1), BODY_CAP);
assertTrue(text.contains("\"ok\":true"), "预算非法时钳到下限,不该把正常响应判成超时");
}
@Test
void fakeUpstreamsActuallyServedRequests() throws Exception {
assertTrue(served.get() >= 1, "假上游没被连上过,上面几条断言就都是空的");
}
private static void closeQuietly(ServerSocket server) {
try {
if (server != null) {
server.close();
}
} catch (IOException ignored) {
// 收尾关不掉不影响判据
}
}
}