fix(task-lock): LLM 响应体补时间界+任务锁跨线程可放与持封顶,掐掉 11 小时无主锁
现网 31888/31856/31725 等任务永久停在 RUNNING,读数链是: java.net.http 的请求超时只封顶到响应头,响应体 read() 走 ArrayBlockingQueue.take(), 上游只发半截就发呆时这一读永久 park(12 条 similar-asin-llm-row 虚拟线程全停在那) ⇒ 无超时的 join() 把 task-queue 线程一起挂住 ⇒ 它手里的 task:similar-asin:<id> 被看门狗无条件续期(实测 11.3 小时,放过留痕 0 次)⇒ 该任务每次结果回传判 40902 (两小时 2370 次),且 12 个 taskQueueExecutor 许可被占满、44 条排队。 两道界各配非零对照: - HttpClientPool.readBodyBounded 把响应体读取挪进一次性虚拟线程、由调用方拿预算去等, 到点中断+关流退成可重试 IOException;三处同型 readResponseBodyBounded 改为委托它。 - LockHandle 直接绑住 HeldLock,任意线程 close 都真放手(旧实现按取锁线程的 ThreadLocal 找锁,跨线程 close 静默 no-op 就是无主锁本体);加持锁封顶 max(30 分钟, 6×TTL), 越界停止续期并 error 留痕,让键在 TTL 内自然过期。 裸跳过例外台账随之清空:它保的正是那条被现网证伪的静默 no-op。 门禁:干净 worktree(HEAD 97312821+本笔)后端全量 4686 例 / 0 红 / 17 skip / rc=0。
This commit is contained in:
@@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.client.ClientHttpRequestFactory;
|
||||
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.Authenticator;
|
||||
@@ -15,7 +16,14 @@ import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
/**
|
||||
* Task 77:外部 HTTP 客户端统一连接复用池。
|
||||
@@ -29,6 +37,9 @@ public class HttpClientPool {
|
||||
|
||||
private static volatile HttpClient sharedHttpClient;
|
||||
private static volatile long configuredCallTimeoutMillis;
|
||||
/** 响应体读取的掐断执行器:一次性虚拟线程,只在读取期间存在,不占业务线程池。 */
|
||||
private static final ExecutorService BODY_READ_EXECUTOR =
|
||||
Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name("http-body-read-", 0).factory());
|
||||
|
||||
/** 由 Spring 配置属性在启动阶段调用,确保共享客户端使用实际的 connect/call 配置。 */
|
||||
public static void configure(long connectTimeoutMillis, long callTimeoutMillis) {
|
||||
@@ -92,6 +103,73 @@ public class HttpClientPool {
|
||||
return response.body();
|
||||
}
|
||||
|
||||
/**
|
||||
* 在预算内读完响应体(UTF-8),超过 maxBytes 直接判体积越界。
|
||||
*
|
||||
* <p>为什么不能就地 while-read:{@code java.net.http} 的请求超时只封顶到「响应头」,
|
||||
* 响应体是流式的,{@code HttpResponseInputStream.read()} 内部是
|
||||
* {@code ArrayBlockingQueue.take()} —— 上游只发了半截就发呆时,这一读就是**永久 park**,
|
||||
* 配多长的 readTimeout 都管不到它。现网实证:12 条 {@code similar-asin-llm-row} 虚拟线程
|
||||
* 全卡在这一读上,连带把上层 {@code task-queue} 线程和它手里的 {@code task:*} 分布式锁
|
||||
* 一起焊死(看门狗还在无条件续期),该任务此后每次结果回传都判 40902。
|
||||
*
|
||||
* <p>所以把读取挪进一次性虚拟线程,由调用方拿预算去等:到点就中断读取并关流,
|
||||
* 把「上游发呆」从永久挂死降级成一次可重试的 IOException。
|
||||
*/
|
||||
public static String readBodyBounded(InputStream input, Duration budget, int maxBytes) throws IOException {
|
||||
if (input == null) {
|
||||
return "";
|
||||
}
|
||||
long safeBudgetMillis = Math.max(1_000L, budget == null ? 0L : budget.toMillis());
|
||||
Future<String> future = BODY_READ_EXECUTOR.submit(() -> drainUtf8(input, maxBytes));
|
||||
try {
|
||||
return future.get(safeBudgetMillis, TimeUnit.MILLISECONDS);
|
||||
} catch (TimeoutException ex) {
|
||||
future.cancel(true);
|
||||
closeQuietly(input);
|
||||
throw new IOException("响应体读取超过 " + safeBudgetMillis + "ms 预算,已掐断(上游未发完)");
|
||||
} catch (InterruptedException ex) {
|
||||
future.cancel(true);
|
||||
closeQuietly(input);
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("响应体读取被中断", ex);
|
||||
} catch (ExecutionException ex) {
|
||||
closeQuietly(input);
|
||||
Throwable cause = ex.getCause();
|
||||
if (cause instanceof IOException ioException) {
|
||||
throw ioException;
|
||||
}
|
||||
throw new IOException("响应体读取失败: " + (cause == null ? ex.getMessage() : cause.getMessage()), cause);
|
||||
}
|
||||
}
|
||||
|
||||
private static String drainUtf8(InputStream input, int maxBytes) throws IOException {
|
||||
try (InputStream body = input; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int read;
|
||||
int total = 0;
|
||||
while ((read = body.read(buffer)) != -1) {
|
||||
if (read == 0) {
|
||||
continue;
|
||||
}
|
||||
if ((long) total + read > maxBytes) {
|
||||
throw new IOException("响应体超过 " + maxBytes + " 字节上限");
|
||||
}
|
||||
output.write(buffer, 0, read);
|
||||
total += read;
|
||||
}
|
||||
return output.toString(StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
private static void closeQuietly(InputStream input) {
|
||||
try {
|
||||
input.close();
|
||||
} catch (Exception ignored) {
|
||||
// 掐断路径上关不掉不额外报:连接由 JDK 的取消动作负责回收
|
||||
}
|
||||
}
|
||||
|
||||
/** 按 readTimeout(毫秒)创建共享连接池工厂;非法值钳制到最小正数。 */
|
||||
public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis) {
|
||||
return requestFactory(readTimeoutMillis, null);
|
||||
|
||||
Reference in New Issue
Block a user