fix(task-lock): LLM 响应体补时间界+任务锁跨线程可放与持封顶,掐掉 11 小时无主锁
现网 31888/31856/31725 等任务永久停在 RUNNING,读数链是: java.net.http 的请求超时只封顶到响应头,响应体 read() 走 ArrayBlockingQueue.take(), 上游只发半截就发呆时这一读永久 park(12 条 similar-asin-llm-row 虚拟线程全停在那) ⇒ 无超时的 join() 把 task-queue 线程一起挂住 ⇒ 它手里的 task:similar-asin:<id> 被看门狗无条件续期(实测 11.3 小时,放过留痕 0 次)⇒ 该任务每次结果回传判 40902 (两小时 2370 次),且 12 个 taskQueueExecutor 许可被占满、44 条排队。 两道界各配非零对照: - HttpClientPool.readBodyBounded 把响应体读取挪进一次性虚拟线程、由调用方拿预算去等, 到点中断+关流退成可重试 IOException;三处同型 readResponseBodyBounded 改为委托它。 - LockHandle 直接绑住 HeldLock,任意线程 close 都真放手(旧实现按取锁线程的 ThreadLocal 找锁,跨线程 close 静默 no-op 就是无主锁本体);加持锁封顶 max(30 分钟, 6×TTL), 越界停止续期并 error 留痕,让键在 TTL 内自然过期。 裸跳过例外台账随之清空:它保的正是那条被现网证伪的静默 no-op。 门禁:干净 worktree(HEAD 97312821+本笔)后端全量 4686 例 / 0 红 / 17 skip / rc=0。
This commit is contained in:
+18
-25
@@ -23,8 +23,9 @@ import org.junit.jupiter.api.Test;
|
||||
* <p>要拦的行为很具体:{@code if (<某个 lock 变量> == null) { continue; }} /
|
||||
* {@code { return; }} —— 巡检任务每轮跑、被锁挡掉的任务直接消失,一行日志都不留。
|
||||
* 实测基线(2026-10-10 自己数出来的,不是评审给的 14):全树 {@code src/main/java} 共
|
||||
* <b>29</b> 处这种形状,28 处已接 {@link LockSkipWitness},1 处判定不该接(见
|
||||
* {@link #LEDGERED_EXCEPTIONS})。
|
||||
* <b>29</b> 处这种形状,28 处已接 {@link LockSkipWitness},剩下 1 处当年记进了例外台账。
|
||||
* <b>2026-10-22 那条例外被现网证伪并已删除</b>(它保的正是"跨线程 close 静默不放手"这条
|
||||
* 造成 11.3 小时僵尸锁的路径),台账现为空,见 {@link #LEDGERED_EXCEPTIONS}。
|
||||
*
|
||||
* <p>判据打在<b>分支体</b>上而不是打在"某个类里出现过锚点"上:分支体剥掉注释后必须只剩
|
||||
* {@code continue;} / {@code return;}(顶多一行 {@code LockSkipWitness.*} 调用)。
|
||||
@@ -46,12 +47,15 @@ class LockSkipSilentExitGuardrailTest {
|
||||
/**
|
||||
* 例外台账:<b>文件名 -> 判定不该接的原因</b>。加一行要有现网证据,删一行就是补了出口。
|
||||
*
|
||||
* <p>{@code TaskDistributedLockService.close()} 里的 {@code heldLock == null} 不是"跳过任务":
|
||||
* 它是重复 close 的幂等护栏(第二次 close 时本地登记已经没了,无事可做),不挡任何工作、
|
||||
* 也不丢数据。给它加锚点等于每次正常重复 close 都刷一条 WARN,把真正在响的那 28 处淹掉。
|
||||
* <p>2026-10-22 清空。原先唯一那行是 {@code TaskDistributedLockService.close()} 里的
|
||||
* {@code heldLock == null},当时的判读是"重复 close 的幂等护栏,不挡工作也不丢数据"。
|
||||
* 现网把它证伪了:那句判空按<b>取锁线程</b>的 ThreadLocal 找锁,于是
|
||||
* <b>跨线程 close 会走进这条"无事可做"分支静默不放手</b>——锁留在原地、续期任务留在原地,
|
||||
* 三把 {@code task:similar-asin:*} 被看门狗喂到 11.3 小时不放,期间该任务每次结果回传
|
||||
* 都判 40902(两小时 2370 次),任务永久停在 RUNNING。修复后句柄直接绑住持锁对象,
|
||||
* 这条分支已不存在,台账随之归零:今后任何文件再写一处裸跳过,上面那条判据直接判红。
|
||||
*/
|
||||
private static final Set<String> LEDGERED_EXCEPTIONS =
|
||||
Set.of("TaskDistributedLockService.java");
|
||||
private static final Set<String> LEDGERED_EXCEPTIONS = Set.of();
|
||||
|
||||
/** 已接线点位数的硬分母:只许升(新增出口),不许降(出口被摘)。 */
|
||||
private static final int MIN_WITNESSED_SITES = 28;
|
||||
@@ -84,24 +88,13 @@ class LockSkipSilentExitGuardrailTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void lockedExceptionsStayExactlyOneAndCannotGrowQuietly() {
|
||||
List<String> bareExceptions = new ArrayList<>();
|
||||
for (Path file : mainSources()) {
|
||||
if (!LEDGERED_EXCEPTIONS.contains(file.getFileName().toString())) {
|
||||
continue;
|
||||
}
|
||||
String source = SourceScan.withoutComments(read(file));
|
||||
for (String body : lockNullBranches(source)) {
|
||||
if (isSkipOnlyBranch(body) && !body.contains("LockSkipWitness.")) {
|
||||
bareExceptions.add(file.getFileName() + ":" + body.strip());
|
||||
}
|
||||
}
|
||||
}
|
||||
// 台账里那个例外只剩 1 处(close() 的幂等护栏)。变成 2 处说明同一个文件里又新写了一处裸跳过,
|
||||
// 那不是"台账里已论证过的形状",必须停下来看。
|
||||
assertThat(bareExceptions)
|
||||
.as("例外台账文件里的裸跳过分支数(基线:TaskDistributedLockService.close() 一处)")
|
||||
.hasSize(1);
|
||||
void exceptionLedgerCannotBeReopenedWithoutAStory() {
|
||||
// 台账一旦有名字,上面那条"每处裸跳过都要出口"就对该文件失效——等于自己给自己发免检。
|
||||
// 2026-10-22 那把 11.3 小时的僵尸锁就是这张免检牌保下来的形状,所以这里要求:
|
||||
// 想加回去,必须带着现网证据来改这段判据,而不是悄悄加一行文件名。
|
||||
assertThat(LEDGERED_EXCEPTIONS)
|
||||
.as("例外台账必须为空(唯一那条例外在无主锁事故里被证伪并已随修复删除)")
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.ServerSocket;
|
||||
import java.net.Socket;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 响应体读取必须有时间界(2026-10-22 现网事故的直接复现件)。
|
||||
*
|
||||
* <p>旧形状:三处 {@code readResponseBodyBounded} 就地 while-read,只量体积不量时间。
|
||||
* JDK 的请求超时封顶到「响应头」,头之后的流式 read() 落在
|
||||
* {@code ArrayBlockingQueue.take()} 上——上游只发半截就发呆时这一读是永久 park,
|
||||
* 现网 12 条 LLM 行线程全卡在这里,连带焊住任务锁与 task-queue 并发许可。
|
||||
*
|
||||
* <p>{@code HttpClientTimeoutEffectiveTest} 早就写下「流式 body 读取不受请求超时约束」这条
|
||||
* 平台边界,但当时只登记了边界、没补界。这里补的就是那一格:断流必须失败,不是挂死。
|
||||
*/
|
||||
class HttpClientPoolBodyReadDeadlineTest {
|
||||
|
||||
private static final int BODY_CAP = 1 << 20;
|
||||
|
||||
private static ServerSocket stallServer;
|
||||
private static ServerSocket okServer;
|
||||
private static final AtomicInteger served = new AtomicInteger();
|
||||
|
||||
@BeforeAll
|
||||
static void startServers() throws IOException {
|
||||
stallServer = new ServerSocket(0);
|
||||
okServer = new ServerSocket(0);
|
||||
// 只发 1 个字节就发呆:Content-Length 声明 100000,客户端 read() 必然等不到结尾
|
||||
acceptLoop(stallServer, true);
|
||||
acceptLoop(okServer, false);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
static void stopServers() {
|
||||
closeQuietly(stallServer);
|
||||
closeQuietly(okServer);
|
||||
}
|
||||
|
||||
private static void acceptLoop(ServerSocket server, boolean stall) {
|
||||
Thread thread = new Thread(() -> {
|
||||
while (!server.isClosed()) {
|
||||
try {
|
||||
Socket socket = server.accept();
|
||||
served.incrementAndGet();
|
||||
Thread connection = new Thread(() -> serve(socket, stall), "fake-upstream-connection");
|
||||
connection.setDaemon(true);
|
||||
connection.start();
|
||||
} catch (Exception ex) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}, "body-read-fake-upstream");
|
||||
thread.setDaemon(true);
|
||||
thread.start();
|
||||
}
|
||||
|
||||
private static void serve(Socket socket, boolean stall) {
|
||||
try (Socket connection = socket) {
|
||||
drainRequestHeaders(connection.getInputStream());
|
||||
OutputStream out = connection.getOutputStream();
|
||||
byte[] payload = stall ? "{".getBytes(StandardCharsets.UTF_8)
|
||||
: "{\"ok\":true}\n".getBytes(StandardCharsets.UTF_8);
|
||||
// 断流那一台:Content-Length 报到 100000,实际只发 1 个字节就留着连接不关,
|
||||
// 模拟上游半开发呆;正常那一台的长度必须逐字节算,报多了客户端就在收尾那一读上等到连接被关。
|
||||
int declared = stall ? 100_000 : payload.length;
|
||||
byte[] head = ("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
|
||||
+ "Content-Length: " + declared + "\r\n\r\n").getBytes(StandardCharsets.UTF_8);
|
||||
out.write(head);
|
||||
out.write(payload);
|
||||
out.flush();
|
||||
if (stall) {
|
||||
Thread.sleep(20_000L);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// 客户端掐断是预期内的,不影响判据
|
||||
}
|
||||
}
|
||||
|
||||
/** 读到头部结束的空行(\r\n\r\n)为止:等满整份请求体是假上游最常见的自杀方式。 */
|
||||
private static void drainRequestHeaders(InputStream in) throws IOException {
|
||||
int newlines = 0;
|
||||
int b;
|
||||
while ((b = in.read()) != -1) {
|
||||
if (b == '\n') {
|
||||
if (++newlines >= 2) {
|
||||
return;
|
||||
}
|
||||
} else if (b != '\r') {
|
||||
newlines = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static InputStream openBody(ServerSocket server) throws Exception {
|
||||
HttpRequest request = HttpRequest.newBuilder(
|
||||
URI.create("http://127.0.0.1:" + server.getLocalPort() + "/v1/chat/completions"))
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.POST(HttpRequest.BodyPublishers.ofString("{}"))
|
||||
.build();
|
||||
HttpResponse<InputStream> response = HttpClientPool.sharedHttpClient()
|
||||
.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
assertEquals(200, response.statusCode());
|
||||
return response.body();
|
||||
}
|
||||
|
||||
@Test
|
||||
void stalledResponseBodyFailsWithinBudgetInsteadOfHangingForever() throws Exception {
|
||||
InputStream body = openBody(stallServer);
|
||||
long startedAt = System.nanoTime();
|
||||
IOException expected = assertThrows(IOException.class,
|
||||
() -> HttpClientPool.readBodyBounded(body, Duration.ofMillis(400), BODY_CAP));
|
||||
long elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000L;
|
||||
assertTrue(expected.getMessage().contains("预算"), "应报掐断原因,实际 " + expected.getMessage());
|
||||
assertTrue(elapsedMillis >= 300L, "不应早于配置的预算就失败:" + elapsedMillis + "ms");
|
||||
assertTrue(elapsedMillis < 5_000L,
|
||||
"断流必须在预算附近失败;挂到这儿就是永久 park(实测 " + elapsedMillis + "ms)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void completeResponseBodyStillReadsThrough() throws Exception {
|
||||
InputStream body = openBody(okServer);
|
||||
String text = HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), BODY_CAP);
|
||||
assertTrue(text.contains("\"ok\":true"), "正常响应不该被掐,实际读到 " + text);
|
||||
}
|
||||
|
||||
@Test
|
||||
void oversizedResponseBodyStillFailsOnByteCap() throws Exception {
|
||||
InputStream body = openBody(okServer);
|
||||
assertThrows(IOException.class,
|
||||
() -> HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), 4));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullBodyIsStillAnEmptyReadNotATimeout() throws IOException {
|
||||
assertEquals("", HttpClientPool.readBodyBounded(null, Duration.ofMillis(400), BODY_CAP));
|
||||
}
|
||||
|
||||
@Test
|
||||
void budgetFloorKeepsNonPositiveConfigFromBecomingInstantAbort() throws Exception {
|
||||
InputStream body = openBody(okServer);
|
||||
String text = HttpClientPool.readBodyBounded(body, Duration.ofMillis(-1), BODY_CAP);
|
||||
assertTrue(text.contains("\"ok\":true"), "预算非法时钳到下限,不该把正常响应判成超时");
|
||||
}
|
||||
|
||||
@Test
|
||||
void fakeUpstreamsActuallyServedRequests() throws Exception {
|
||||
assertTrue(served.get() >= 1, "假上游没被连上过,上面几条断言就都是空的");
|
||||
}
|
||||
|
||||
private static void closeQuietly(ServerSocket server) {
|
||||
try {
|
||||
if (server != null) {
|
||||
server.close();
|
||||
}
|
||||
} catch (IOException ignored) {
|
||||
// 收尾关不掉不影响判据
|
||||
}
|
||||
}
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 无主锁的防回归判据(2026-10-22 现网事故)。
|
||||
*
|
||||
* <p>现场形状:一条 {@code task-queue} 虚拟线程卡在 LLM 响应体读取上(无超时),
|
||||
* 它手里那把 {@code task:similar-asin:<id>} 因此永远不放,而看门狗还在无条件续期
|
||||
* ——实测三把锁压到 11.3 小时,期间该任务每一次结果回传都判 40902(两小时 2370 次),
|
||||
* 任务永久停在 RUNNING。两条线各自挡一种成因:
|
||||
* ① 释放不许依赖「在哪个线程上 close」;② 就算持有方真的卡死,锁也不能是永久的。
|
||||
*/
|
||||
class TaskDistributedLockServiceLeakTest {
|
||||
|
||||
private static final Duration TTL = Duration.ofMinutes(5);
|
||||
private static final long TASK_ID = 31888L;
|
||||
|
||||
/** @return 服务;renewCalls 计 renew 次数,用于判「续期有没有停」。 */
|
||||
private TaskDistributedLockService serviceWithRenewCounting(AtomicInteger renewCalls,
|
||||
DistributedJobLockService.LockHandle[] sink) {
|
||||
DistributedJobLockService delegate = mock(DistributedJobLockService.class);
|
||||
DistributedJobLockService.LockHandle inner = mock(DistributedJobLockService.LockHandle.class);
|
||||
when(inner.renew(any(Duration.class))).thenAnswer(invocation -> {
|
||||
renewCalls.incrementAndGet();
|
||||
return true;
|
||||
});
|
||||
when(delegate.tryLock(anyString(), any(Duration.class))).thenReturn(inner);
|
||||
sink[0] = inner;
|
||||
return new TaskDistributedLockService(delegate);
|
||||
}
|
||||
|
||||
@Test
|
||||
void closeFromAnotherThreadStillReleasesTheLock() throws Exception {
|
||||
AtomicInteger renewCalls = new AtomicInteger();
|
||||
DistributedJobLockService.LockHandle[] inner = new DistributedJobLockService.LockHandle[1];
|
||||
TaskDistributedLockService service = serviceWithRenewCounting(renewCalls, inner);
|
||||
|
||||
TaskDistributedLockService.LockHandle handle = service.acquire("similar-asin", TASK_ID);
|
||||
assertTrue(service.activeHeldMillis().containsKey("task:similar-asin:" + TASK_ID), "取到锁必须读得到在持");
|
||||
|
||||
// 换一条线程去放:旧实现按取锁线程的 ThreadLocal 找锁,在别人那张表里查不到自己,
|
||||
// 于是 close 静默 no-op —— 锁留在原地、续期任务留在原地,Redis 键被永久喂活。
|
||||
Thread closer = new Thread(handle::close, "other-thread-closer");
|
||||
closer.start();
|
||||
closer.join(5_000L);
|
||||
|
||||
assertTrue(service.activeHeldMillis().isEmpty(),
|
||||
"跨线程 close 必须真的放手:否则这条锁就是一条没人认领的僵尸锁");
|
||||
verify(inner[0], atLeastOnce()).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void nestedHandlesReleasedOnDifferentThreadsStillLetGo() throws Exception {
|
||||
AtomicInteger renewCalls = new AtomicInteger();
|
||||
DistributedJobLockService.LockHandle[] inner = new DistributedJobLockService.LockHandle[1];
|
||||
TaskDistributedLockService service = serviceWithRenewCounting(renewCalls, inner);
|
||||
|
||||
TaskDistributedLockService.LockHandle outer = service.acquire("similar-asin", TASK_ID);
|
||||
TaskDistributedLockService.LockHandle nested = service.acquire("similar-asin", TASK_ID);
|
||||
assertEquals(1, service.activeHeldMillis().size(), "重入只登记一次");
|
||||
nested.close();
|
||||
assertEquals(1, service.activeHeldMillis().size(), "只放了一层");
|
||||
|
||||
Thread closer = new Thread(outer::close, "other-thread-closer");
|
||||
closer.start();
|
||||
closer.join(5_000L);
|
||||
assertTrue(service.activeHeldMillis().isEmpty(), "最后一层在别的线程上放,也必须算放过");
|
||||
}
|
||||
|
||||
@Test
|
||||
void renewalStopsOnceHoldPassesTheCeiling() throws Exception {
|
||||
AtomicInteger renewCalls = new AtomicInteger();
|
||||
DistributedJobLockService.LockHandle[] inner = new DistributedJobLockService.LockHandle[1];
|
||||
TaskDistributedLockService service = serviceWithRenewCounting(renewCalls, inner);
|
||||
// ttl 300ms → 续期节拍钳到下限 1s,封顶 = max(200ms, 6×300ms) = 1.8s:
|
||||
// 第一次续期(≈1s)还在界内,第二次(≈2s)越界即停。
|
||||
service.setMaxHoldFloorMillisForTest(200L);
|
||||
TaskDistributedLockService.LockHandle handle =
|
||||
service.acquire("similar-asin", TASK_ID, Duration.ofMillis(300), 0L);
|
||||
|
||||
long deadline = System.currentTimeMillis() + 6_000L;
|
||||
while (renewCalls.get() == 0 && System.currentTimeMillis() < deadline) {
|
||||
Thread.sleep(50L);
|
||||
}
|
||||
assertTrue(renewCalls.get() >= 1, "没续过期就说明这条用例什么都没量到");
|
||||
int renewalsAtStopPoint = renewCalls.get();
|
||||
Thread.sleep(3_000L);
|
||||
assertEquals(renewalsAtStopPoint, renewCalls.get(),
|
||||
"越界后还在续期 = 锁永远不会自己解开,这正是 11.3 小时僵尸锁的成因");
|
||||
|
||||
// 到线只「放弃续期」,不替卡死的持有方去 del 键:键在 TTL 内自然过期,
|
||||
// 持有方将来真跑完时那次带 token 的 RELEASE 若已不属于它也只会返回 0,不会误删接管者的锁。
|
||||
verify(inner[0], never()).close();
|
||||
assertTrue(service.activeHeldMillis().containsKey("task:similar-asin:" + TASK_ID),
|
||||
"放弃续期的锁必须还留在在持读数里(标成续期已失败),看门狗不能对这种锁失明");
|
||||
handle.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user