diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java b/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java index 7bcddb415..f23fd70ca 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java @@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.http.client.JdkClientHttpRequestFactory; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.Authenticator; @@ -15,7 +16,14 @@ import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** * Task 77:外部 HTTP 客户端统一连接复用池。 @@ -29,6 +37,9 @@ public class HttpClientPool { private static volatile HttpClient sharedHttpClient; private static volatile long configuredCallTimeoutMillis; + /** 响应体读取的掐断执行器:一次性虚拟线程,只在读取期间存在,不占业务线程池。 */ + private static final ExecutorService BODY_READ_EXECUTOR = + Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name("http-body-read-", 0).factory()); /** 由 Spring 配置属性在启动阶段调用,确保共享客户端使用实际的 connect/call 配置。 */ public static void configure(long connectTimeoutMillis, long callTimeoutMillis) { @@ -92,6 +103,73 @@ public class HttpClientPool { return response.body(); } + /** + * 在预算内读完响应体(UTF-8),超过 maxBytes 直接判体积越界。 + * + *

为什么不能就地 while-read:{@code java.net.http} 的请求超时只封顶到「响应头」, + * 响应体是流式的,{@code HttpResponseInputStream.read()} 内部是 + * {@code ArrayBlockingQueue.take()} —— 上游只发了半截就发呆时,这一读就是**永久 park**, + * 配多长的 readTimeout 都管不到它。现网实证:12 条 {@code similar-asin-llm-row} 虚拟线程 + * 全卡在这一读上,连带把上层 {@code task-queue} 线程和它手里的 {@code task:*} 分布式锁 + * 一起焊死(看门狗还在无条件续期),该任务此后每次结果回传都判 40902。 + * + *

所以把读取挪进一次性虚拟线程,由调用方拿预算去等:到点就中断读取并关流, + * 把「上游发呆」从永久挂死降级成一次可重试的 IOException。 + */ + public static String readBodyBounded(InputStream input, Duration budget, int maxBytes) throws IOException { + if (input == null) { + return ""; + } + long safeBudgetMillis = Math.max(1_000L, budget == null ? 0L : budget.toMillis()); + Future future = BODY_READ_EXECUTOR.submit(() -> drainUtf8(input, maxBytes)); + try { + return future.get(safeBudgetMillis, TimeUnit.MILLISECONDS); + } catch (TimeoutException ex) { + future.cancel(true); + closeQuietly(input); + throw new IOException("响应体读取超过 " + safeBudgetMillis + "ms 预算,已掐断(上游未发完)"); + } catch (InterruptedException ex) { + future.cancel(true); + closeQuietly(input); + Thread.currentThread().interrupt(); + throw new IOException("响应体读取被中断", ex); + } catch (ExecutionException ex) { + closeQuietly(input); + Throwable cause = ex.getCause(); + if (cause instanceof IOException ioException) { + throw ioException; + } + throw new IOException("响应体读取失败: " + (cause == null ? ex.getMessage() : cause.getMessage()), cause); + } + } + + private static String drainUtf8(InputStream input, int maxBytes) throws IOException { + try (InputStream body = input; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) { + byte[] buffer = new byte[8192]; + int read; + int total = 0; + while ((read = body.read(buffer)) != -1) { + if (read == 0) { + continue; + } + if ((long) total + read > maxBytes) { + throw new IOException("响应体超过 " + maxBytes + " 字节上限"); + } + output.write(buffer, 0, read); + total += read; + } + return output.toString(StandardCharsets.UTF_8); + } + } + + private static void closeQuietly(InputStream input) { + try { + input.close(); + } catch (Exception ignored) { + // 掐断路径上关不掉不额外报:连接由 JDK 的取消动作负责回收 + } + } + /** 按 readTimeout(毫秒)创建共享连接池工厂;非法值钳制到最小正数。 */ public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis) { return requestFactory(readTimeoutMillis, null); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java index 4d6188809..16d580eb5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentLlmClient.java @@ -19,10 +19,10 @@ import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.web.client.RestClient; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; @@ -673,7 +673,7 @@ public class AppearancePatentLlmClient { }); request.body(body); String responseText = request.exchange((clientRequest, clientResponse) -> { - String responseBody = readResponseBodyBounded(clientResponse.getBody()); + String responseBody = readResponseBodyBounded(clientResponse.getBody(), attempt); if (!clientResponse.getStatusCode().is2xxSuccessful()) { throw new IllegalStateException("LLM http " + clientResponse.getStatusCode().value() + ": " + abbreviate(responseBody, 500)); @@ -842,26 +842,12 @@ public class AppearancePatentLlmClient { } } - private String readResponseBodyBounded(InputStream inputStream) throws IOException { - if (inputStream == null) { - return ""; - } - try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) { - byte[] buffer = new byte[8192]; - int read; - int total = 0; - while ((read = input.read(buffer)) != -1) { - if (read == 0) { - continue; - } - if ((long) total + read > MAX_LLM_RESPONSE_BYTES) { - throw new IOException("LLM response exceeds " + MAX_LLM_RESPONSE_BYTES + " bytes"); - } - output.write(buffer, 0, read); - total += read; - } - return output.toString(StandardCharsets.UTF_8); - } + /** 响应体读取预算与本次实际用的那套超时对齐;JDK 的超时只到响应头,这一腿原先无界。 */ + private String readResponseBodyBounded(InputStream inputStream, int attempt) throws IOException { + long budget = attempt == 1 + ? Math.min(firstAttemptTimeoutMillis(), Math.max(1, properties.getLlmReadTimeoutMillis())) + : Math.max(1, properties.getLlmReadTimeoutMillis()); + return HttpClientPool.readBodyBounded(inputStream, Duration.ofMillis(budget), MAX_LLM_RESPONSE_BYTES); } private JsonNode parseJsonOrThrow(String value) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java index cfbf50a54..2514300c5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinLlmClient.java @@ -15,10 +15,10 @@ import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.web.client.RestClient; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -126,7 +126,7 @@ public class SimilarAsinLlmClient { }); request.body(body); String responseText = request.exchange((clientRequest, clientResponse) -> { - String responseBody = readResponseBodyBounded(clientResponse.getBody()); + String responseBody = readResponseBodyBounded(clientResponse.getBody(), attempt); if (!clientResponse.getStatusCode().is2xxSuccessful()) { throw new IllegalStateException("LLM http " + clientResponse.getStatusCode().value() + ": " + abbreviate(responseBody, 500)); @@ -273,26 +273,15 @@ public class SimilarAsinLlmClient { } } - private String readResponseBodyBounded(InputStream inputStream) throws IOException { - if (inputStream == null) { - return ""; - } - try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) { - byte[] buffer = new byte[8192]; - int read; - int total = 0; - while ((read = input.read(buffer)) != -1) { - if (read == 0) { - continue; - } - if ((long) total + read > MAX_LLM_RESPONSE_BYTES) { - throw new IOException("LLM response exceeds " + MAX_LLM_RESPONSE_BYTES + " bytes"); - } - output.write(buffer, 0, read); - total += read; - } - return output.toString(StandardCharsets.UTF_8); - } + /** + * 响应体的读取预算:与本次实际用的那个 RestClient 的超时对齐(首试用短超时那套), + * 但**必须自己另算**——JDK 的超时只到响应头,响应体这一腿原先没有任何界。 + */ + private String readResponseBodyBounded(InputStream inputStream, int attempt) throws IOException { + long budget = attempt == 1 + ? Math.min(firstAttemptTimeoutMillis(), Math.max(1, properties.getLlmReadTimeoutMillis())) + : Math.max(1, properties.getLlmReadTimeoutMillis()); + return HttpClientPool.readBodyBounded(inputStream, Duration.ofMillis(budget), MAX_LLM_RESPONSE_BYTES); } private JsonNode parseJsonOrThrow(String value) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java index 992a62d0a..f3018be3f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java @@ -70,6 +70,15 @@ final class ActiveTaskLockRegistry { active.remove(lockName); } + /** + * 只有登记的就是这一任时才摘除。封顶放弃过的那把锁,持有线程可能过了很久才走到 close, + * 这期间新持有者已经登记同名锁——按名字无条件删就会把新那一任的读数抹掉(看门狗就此失明)。 + */ + void unregisterIfCurrent(String lockName, long acquiredAtMillis) { + active.computeIfPresent(lockName, + (key, lock) -> lock.acquiredAtMillis == acquiredAtMillis ? null : lock); + } + void markRenewalLost(String lockName) { ActiveLock lock = active.get(lockName); if (lock != null) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java index bfe498b88..cdece47e1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java @@ -11,10 +11,13 @@ import java.time.Duration; import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; @Service @RequiredArgsConstructor @@ -30,9 +33,24 @@ public class TaskDistributedLockService { /** 持锁时长留痕:任务锁被 LLM 批处理占用是 40902 的来源,此前只有 WAIT_LLM 一支量得出时长。 */ private static final long HELD_REPORT_MILLIS = 5000L; private static final long HELD_WARN_MILLIS = 60000L; + /** + * 持锁封顶=max(30 分钟, 6×TTL)。越过这条线还续着期,说明持有方已经卡死而不是在干活: + * 现网实测三把 {@code task:similar-asin:*} 被看门狗喂到 11.3 小时不放,期间该任务的 + * 每一次结果回传都判 40902(两小时窗口 2370 次),任务永远停在 RUNNING。 + * 到线后停止续期、让 Redis 键在 TTL 内自然过期,把"永久焊死"降级成"最多卡 TTL 量级"。 + */ + private static final long MAX_HOLD_FLOOR_MILLIS = 30 * 60_000L; + private static final int MAX_HOLD_TTL_MULTIPLE = 6; private final DistributedJobLockService distributedJobLockService; - private final ThreadLocal> localLocks = ThreadLocal.withInitial(LinkedHashMap::new); + /** 持锁封顶的下限线,见 {@link #MAX_HOLD_FLOOR_MILLIS};测试会临时压低它来走封顶分支。 */ + private volatile long maxHoldFloorMillis = MAX_HOLD_FLOOR_MILLIS; + /** + * 取锁线程 → 该线程在持的锁。它只负责**重入判定**(同一线程再次取同一把锁不能把自己挡住), + * 释放一律走 {@link LockHandle} 上直接绑定的 {@link HeldLock}:早先 release 也按这张表找, + * 跨线程 close 会在别人的表里查不到自己的锁而静默 no-op,锁和续期任务就双双留在原地没人管。 + */ + private final ThreadLocal> localLocks = ThreadLocal.withInitial(ConcurrentHashMap::new); /** * 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。 * 线程数从 4 提到 16(2026-09 全维度审查):并发持锁任务数上升或 Redis 抖动时, @@ -70,19 +88,19 @@ public class TaskDistributedLockService { Map heldLocks = localLocks.get(); HeldLock heldLock = heldLocks.get(lockName); if (heldLock != null) { - heldLock.depth++; - return new LockHandle(lockName); + heldLock.depth.incrementAndGet(); + return new LockHandle(heldLock); } long deadline = System.nanoTime() + Math.max(0L, waitMillis) * 1_000_000L; do { DistributedJobLockService.LockHandle delegate = distributedJobLockService.tryLock(lockName, ttl); if (delegate != null) { - HeldLock newHeldLock = new HeldLock(delegate, ttl); + HeldLock newHeldLock = new HeldLock(lockName, delegate, ttl, heldLocks); scheduleRenewal(lockName, newHeldLock, ttl); heldLocks.put(lockName, newHeldLock); activeLocks.register(lockName, newHeldLock.acquiredAtMillis, ttl.toMillis()); - return new LockHandle(lockName); + return new LockHandle(newHeldLock); } if (waitMillis <= 0L || System.nanoTime() >= deadline) { return null; @@ -118,18 +136,30 @@ public class TaskDistributedLockService { long ttlMillis = Math.max(1000L, ttl.toMillis()); long renewalDelayMillis = Math.max(1000L, ttlMillis / 3L); heldLock.renewalFuture = renewalExecutor.scheduleWithFixedDelay(() -> { - if (heldLock.closed) { + if (heldLock.closed.get()) { + return; + } + long heldMillis = System.currentTimeMillis() - heldLock.acquiredAtMillis; + // 封顶按调用方**要的那条 TTL**算,不按下面钳过下限的那条: + // 下限是给续期节拍兜底的,拿它去放大封顶线会让短 TTL 的锁反而能压更久。 + if (heldMillis > maxHoldMillis(ttl.toMillis())) { + // 封顶:不再续期,让键在 TTL 内自己过期。故意不主动 del——持有方可能只是慢, + // 由它跑完时那次带 token 的 RELEASE 去收尾更安全(token 不匹配即 0,不会误删接管者的锁)。 + heldLock.closed.set(true); + activeLocks.markRenewalLost(lockName); + cancelRenewal(heldLock); + log.error("[task-lock] 持锁超过上限,停止续期 lockName={} heldMs={} ceilingMs={} ttlMs={} —— " + + "持有方已卡死,本节点不应再对该资源做破坏性写操作;" + + "本锁最迟 TTL 后可被其它实例接管", + lockName, heldMillis, maxHoldMillis(ttl.toMillis()), ttlMillis); return; } if (renewWithRetry(lockName, heldLock)) { return; } - heldLock.closed = true; + heldLock.closed.set(true); activeLocks.markRenewalLost(lockName); - ScheduledFuture future = heldLock.renewalFuture; - if (future != null) { - future.cancel(false); - } + cancelRenewal(heldLock); // 此前单次失败即静默停止续期:本节点会继续以「仍持锁」的假设做破坏性操作, // 而锁最迟在 TTL 后已可被另一实例获取 —— 必须留下 error 级线索便于事后追溯 log.error("[task-lock] 续期连续 {} 次失败,锁已失效 lockName={} —— " @@ -138,6 +168,23 @@ public class TaskDistributedLockService { }, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS); } + private static void cancelRenewal(HeldLock heldLock) { + ScheduledFuture future = heldLock.renewalFuture; + if (future != null) { + future.cancel(false); + } + } + + /** 封顶随 TTL 缩放:调用方给了更长的 TTL,就该配更长的容忍线。 */ + private long maxHoldMillis(long ttlMillis) { + return Math.max(maxHoldFloorMillis, MAX_HOLD_TTL_MULTIPLE * ttlMillis); + } + + /** 测试接缝:把 30 分钟的下限压到毫秒级,才能在用例里真走到封顶那一条路。 */ + void setMaxHoldFloorMillisForTest(long floorMillis) { + this.maxHoldFloorMillis = Math.max(1L, floorMillis); + } + /** * 续期重试:单次失败可能是 Redis 抖动 / Sentinel 主从切换造成的瞬时问题, * 立即放弃会让本节点在锁已过期的情况下继续运行;重试仍失败才判锁丢失。 @@ -191,35 +238,39 @@ public class TaskDistributedLockService { private static final class HeldLock { + private final String lockName; private final DistributedJobLockService.LockHandle delegate; private final Duration ttl; private final long acquiredAtMillis; - private ScheduledFuture renewalFuture; - private int depth = 1; - private volatile boolean closed; + /** 取锁线程那张表:释放要从这里摘掉自己,摘的动作与在哪个线程上做无关。 */ + private final Map ownerLocks; + private volatile ScheduledFuture renewalFuture; + private final AtomicInteger depth = new AtomicInteger(1); + private final AtomicBoolean closed = new AtomicBoolean(); - private HeldLock(DistributedJobLockService.LockHandle delegate, Duration ttl) { + private HeldLock(String lockName, DistributedJobLockService.LockHandle delegate, Duration ttl, + Map ownerLocks) { + this.lockName = lockName; this.delegate = delegate; this.ttl = ttl; this.acquiredAtMillis = System.currentTimeMillis(); + this.ownerLocks = ownerLocks; } private void close() { - closed = true; - if (renewalFuture != null) { - renewalFuture.cancel(false); - } + closed.set(true); + cancelRenewal(this); delegate.close(); } } public final class LockHandle implements AutoCloseable { - private final String lockName; + private final HeldLock heldLock; private boolean closed; - private LockHandle(String lockName) { - this.lockName = lockName; + private LockHandle(HeldLock heldLock) { + this.heldLock = heldLock; } @Override @@ -228,26 +279,18 @@ public class TaskDistributedLockService { return; } closed = true; - Map heldLocks = localLocks.get(); - HeldLock heldLock = heldLocks.get(lockName); - if (heldLock == null) { - return; + if (heldLock.depth.decrementAndGet() > 0) { + return; // 只放了一层,锁还在手里 } - heldLock.depth--; - if (heldLock.depth <= 0) { - heldLocks.remove(lockName); - activeLocks.unregister(lockName); - if (heldLocks.isEmpty()) { - localLocks.remove(); - } - long heldMillis = System.currentTimeMillis() - heldLock.acquiredAtMillis; - heldLock.close(); - if (heldMillis >= HELD_WARN_MILLIS) { - log.warn("[task-lock-held] lockName={} heldMs={} threshold={}ms", - lockName, heldMillis, HELD_WARN_MILLIS); - } else if (heldMillis >= HELD_REPORT_MILLIS) { - log.info("[task-lock-held] lockName={} heldMs={}", lockName, heldMillis); - } + heldLock.ownerLocks.remove(heldLock.lockName, heldLock); + activeLocks.unregisterIfCurrent(heldLock.lockName, heldLock.acquiredAtMillis); + long heldMillis = System.currentTimeMillis() - heldLock.acquiredAtMillis; + heldLock.close(); + if (heldMillis >= HELD_WARN_MILLIS) { + log.warn("[task-lock-held] lockName={} heldMs={} threshold={}ms", + heldLock.lockName, heldMillis, HELD_WARN_MILLIS); + } else if (heldMillis >= HELD_REPORT_MILLIS) { + log.info("[task-lock-held] lockName={} heldMs={}", heldLock.lockName, heldMillis); } } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserApiSecretCheckService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserApiSecretCheckService.java index f3addfc53..b217390d9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserApiSecretCheckService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserApiSecretCheckService.java @@ -16,11 +16,11 @@ import org.springframework.http.MediaType; import org.springframework.stereotype.Service; import org.springframework.web.client.RestClient; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -472,26 +472,9 @@ public class UserApiSecretCheckService { } } + /** 探测响应体的读取界:JDK 的 readTimeout 只封顶到响应头,断流时这一读会永久 park。 */ private String readResponseBodyBounded(InputStream inputStream) throws IOException { - if (inputStream == null) { - return ""; - } - try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) { - byte[] buffer = new byte[8192]; - int read; - int total = 0; - while ((read = input.read(buffer)) != -1) { - if (read == 0) { - continue; - } - if ((long) total + read > MAX_RESPONSE_BYTES) { - throw new IOException("检测响应超过 " + MAX_RESPONSE_BYTES + " 字节"); - } - output.write(buffer, 0, read); - total += read; - } - return output.toString(StandardCharsets.UTF_8); - } + return HttpClientPool.readBodyBounded(inputStream, Duration.ofMillis(READ_TIMEOUT_MILLIS), MAX_RESPONSE_BYTES); } private String joinUrl(String baseUrl, String path) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/util/LockSkipSilentExitGuardrailTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/util/LockSkipSilentExitGuardrailTest.java index 0b1e14f90..32301f3c7 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/common/util/LockSkipSilentExitGuardrailTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/common/util/LockSkipSilentExitGuardrailTest.java @@ -23,8 +23,9 @@ import org.junit.jupiter.api.Test; *

要拦的行为很具体:{@code if (<某个 lock 变量> == null) { continue; }} / * {@code { return; }} —— 巡检任务每轮跑、被锁挡掉的任务直接消失,一行日志都不留。 * 实测基线(2026-10-10 自己数出来的,不是评审给的 14):全树 {@code src/main/java} 共 - * 29 处这种形状,28 处已接 {@link LockSkipWitness},1 处判定不该接(见 - * {@link #LEDGERED_EXCEPTIONS})。 + * 29 处这种形状,28 处已接 {@link LockSkipWitness},剩下 1 处当年记进了例外台账。 + * 2026-10-22 那条例外被现网证伪并已删除(它保的正是"跨线程 close 静默不放手"这条 + * 造成 11.3 小时僵尸锁的路径),台账现为空,见 {@link #LEDGERED_EXCEPTIONS}。 * *

判据打在分支体上而不是打在"某个类里出现过锚点"上:分支体剥掉注释后必须只剩 * {@code continue;} / {@code return;}(顶多一行 {@code LockSkipWitness.*} 调用)。 @@ -46,12 +47,15 @@ class LockSkipSilentExitGuardrailTest { /** * 例外台账:文件名 -> 判定不该接的原因。加一行要有现网证据,删一行就是补了出口。 * - *

{@code TaskDistributedLockService.close()} 里的 {@code heldLock == null} 不是"跳过任务": - * 它是重复 close 的幂等护栏(第二次 close 时本地登记已经没了,无事可做),不挡任何工作、 - * 也不丢数据。给它加锚点等于每次正常重复 close 都刷一条 WARN,把真正在响的那 28 处淹掉。 + *

2026-10-22 清空。原先唯一那行是 {@code TaskDistributedLockService.close()} 里的 + * {@code heldLock == null},当时的判读是"重复 close 的幂等护栏,不挡工作也不丢数据"。 + * 现网把它证伪了:那句判空按取锁线程的 ThreadLocal 找锁,于是 + * 跨线程 close 会走进这条"无事可做"分支静默不放手——锁留在原地、续期任务留在原地, + * 三把 {@code task:similar-asin:*} 被看门狗喂到 11.3 小时不放,期间该任务每次结果回传 + * 都判 40902(两小时 2370 次),任务永久停在 RUNNING。修复后句柄直接绑住持锁对象, + * 这条分支已不存在,台账随之归零:今后任何文件再写一处裸跳过,上面那条判据直接判红。 */ - private static final Set LEDGERED_EXCEPTIONS = - Set.of("TaskDistributedLockService.java"); + private static final Set LEDGERED_EXCEPTIONS = Set.of(); /** 已接线点位数的硬分母:只许升(新增出口),不许降(出口被摘)。 */ private static final int MIN_WITNESSED_SITES = 28; @@ -84,24 +88,13 @@ class LockSkipSilentExitGuardrailTest { } @Test - void lockedExceptionsStayExactlyOneAndCannotGrowQuietly() { - List bareExceptions = new ArrayList<>(); - for (Path file : mainSources()) { - if (!LEDGERED_EXCEPTIONS.contains(file.getFileName().toString())) { - continue; - } - String source = SourceScan.withoutComments(read(file)); - for (String body : lockNullBranches(source)) { - if (isSkipOnlyBranch(body) && !body.contains("LockSkipWitness.")) { - bareExceptions.add(file.getFileName() + ":" + body.strip()); - } - } - } - // 台账里那个例外只剩 1 处(close() 的幂等护栏)。变成 2 处说明同一个文件里又新写了一处裸跳过, - // 那不是"台账里已论证过的形状",必须停下来看。 - assertThat(bareExceptions) - .as("例外台账文件里的裸跳过分支数(基线:TaskDistributedLockService.close() 一处)") - .hasSize(1); + void exceptionLedgerCannotBeReopenedWithoutAStory() { + // 台账一旦有名字,上面那条"每处裸跳过都要出口"就对该文件失效——等于自己给自己发免检。 + // 2026-10-22 那把 11.3 小时的僵尸锁就是这张免检牌保下来的形状,所以这里要求: + // 想加回去,必须带着现网证据来改这段判据,而不是悄悄加一行文件名。 + assertThat(LEDGERED_EXCEPTIONS) + .as("例外台账必须为空(唯一那条例外在无主锁事故里被证伪并已随修复删除)") + .isEmpty(); } /** diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientPoolBodyReadDeadlineTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientPoolBodyReadDeadlineTest.java new file mode 100644 index 000000000..6bbed8216 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/config/HttpClientPoolBodyReadDeadlineTest.java @@ -0,0 +1,178 @@ +package com.nanri.aiimage.config; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URI; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * 响应体读取必须有时间界(2026-10-22 现网事故的直接复现件)。 + * + *

旧形状:三处 {@code readResponseBodyBounded} 就地 while-read,只量体积不量时间。 + * JDK 的请求超时封顶到「响应头」,头之后的流式 read() 落在 + * {@code ArrayBlockingQueue.take()} 上——上游只发半截就发呆时这一读是永久 park, + * 现网 12 条 LLM 行线程全卡在这里,连带焊住任务锁与 task-queue 并发许可。 + * + *

{@code HttpClientTimeoutEffectiveTest} 早就写下「流式 body 读取不受请求超时约束」这条 + * 平台边界,但当时只登记了边界、没补界。这里补的就是那一格:断流必须失败,不是挂死。 + */ +class HttpClientPoolBodyReadDeadlineTest { + + private static final int BODY_CAP = 1 << 20; + + private static ServerSocket stallServer; + private static ServerSocket okServer; + private static final AtomicInteger served = new AtomicInteger(); + + @BeforeAll + static void startServers() throws IOException { + stallServer = new ServerSocket(0); + okServer = new ServerSocket(0); + // 只发 1 个字节就发呆:Content-Length 声明 100000,客户端 read() 必然等不到结尾 + acceptLoop(stallServer, true); + acceptLoop(okServer, false); + } + + @AfterAll + static void stopServers() { + closeQuietly(stallServer); + closeQuietly(okServer); + } + + private static void acceptLoop(ServerSocket server, boolean stall) { + Thread thread = new Thread(() -> { + while (!server.isClosed()) { + try { + Socket socket = server.accept(); + served.incrementAndGet(); + Thread connection = new Thread(() -> serve(socket, stall), "fake-upstream-connection"); + connection.setDaemon(true); + connection.start(); + } catch (Exception ex) { + return; + } + } + }, "body-read-fake-upstream"); + thread.setDaemon(true); + thread.start(); + } + + private static void serve(Socket socket, boolean stall) { + try (Socket connection = socket) { + drainRequestHeaders(connection.getInputStream()); + OutputStream out = connection.getOutputStream(); + byte[] payload = stall ? "{".getBytes(StandardCharsets.UTF_8) + : "{\"ok\":true}\n".getBytes(StandardCharsets.UTF_8); + // 断流那一台:Content-Length 报到 100000,实际只发 1 个字节就留着连接不关, + // 模拟上游半开发呆;正常那一台的长度必须逐字节算,报多了客户端就在收尾那一读上等到连接被关。 + int declared = stall ? 100_000 : payload.length; + byte[] head = ("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + + "Content-Length: " + declared + "\r\n\r\n").getBytes(StandardCharsets.UTF_8); + out.write(head); + out.write(payload); + out.flush(); + if (stall) { + Thread.sleep(20_000L); + } + } catch (Exception ex) { + // 客户端掐断是预期内的,不影响判据 + } + } + + /** 读到头部结束的空行(\r\n\r\n)为止:等满整份请求体是假上游最常见的自杀方式。 */ + private static void drainRequestHeaders(InputStream in) throws IOException { + int newlines = 0; + int b; + while ((b = in.read()) != -1) { + if (b == '\n') { + if (++newlines >= 2) { + return; + } + } else if (b != '\r') { + newlines = 0; + } + } + } + + private static InputStream openBody(ServerSocket server) throws Exception { + HttpRequest request = HttpRequest.newBuilder( + URI.create("http://127.0.0.1:" + server.getLocalPort() + "/v1/chat/completions")) + .timeout(Duration.ofSeconds(30)) + .POST(HttpRequest.BodyPublishers.ofString("{}")) + .build(); + HttpResponse response = HttpClientPool.sharedHttpClient() + .send(request, HttpResponse.BodyHandlers.ofInputStream()); + assertEquals(200, response.statusCode()); + return response.body(); + } + + @Test + void stalledResponseBodyFailsWithinBudgetInsteadOfHangingForever() throws Exception { + InputStream body = openBody(stallServer); + long startedAt = System.nanoTime(); + IOException expected = assertThrows(IOException.class, + () -> HttpClientPool.readBodyBounded(body, Duration.ofMillis(400), BODY_CAP)); + long elapsedMillis = (System.nanoTime() - startedAt) / 1_000_000L; + assertTrue(expected.getMessage().contains("预算"), "应报掐断原因,实际 " + expected.getMessage()); + assertTrue(elapsedMillis >= 300L, "不应早于配置的预算就失败:" + elapsedMillis + "ms"); + assertTrue(elapsedMillis < 5_000L, + "断流必须在预算附近失败;挂到这儿就是永久 park(实测 " + elapsedMillis + "ms)"); + } + + @Test + void completeResponseBodyStillReadsThrough() throws Exception { + InputStream body = openBody(okServer); + String text = HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), BODY_CAP); + assertTrue(text.contains("\"ok\":true"), "正常响应不该被掐,实际读到 " + text); + } + + @Test + void oversizedResponseBodyStillFailsOnByteCap() throws Exception { + InputStream body = openBody(okServer); + assertThrows(IOException.class, + () -> HttpClientPool.readBodyBounded(body, Duration.ofSeconds(5), 4)); + } + + @Test + void nullBodyIsStillAnEmptyReadNotATimeout() throws IOException { + assertEquals("", HttpClientPool.readBodyBounded(null, Duration.ofMillis(400), BODY_CAP)); + } + + @Test + void budgetFloorKeepsNonPositiveConfigFromBecomingInstantAbort() throws Exception { + InputStream body = openBody(okServer); + String text = HttpClientPool.readBodyBounded(body, Duration.ofMillis(-1), BODY_CAP); + assertTrue(text.contains("\"ok\":true"), "预算非法时钳到下限,不该把正常响应判成超时"); + } + + @Test + void fakeUpstreamsActuallyServedRequests() throws Exception { + assertTrue(served.get() >= 1, "假上游没被连上过,上面几条断言就都是空的"); + } + + private static void closeQuietly(ServerSocket server) { + try { + if (server != null) { + server.close(); + } + } catch (IOException ignored) { + // 收尾关不掉不影响判据 + } + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceLeakTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceLeakTest.java new file mode 100644 index 000000000..e584f33ea --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceLeakTest.java @@ -0,0 +1,113 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.common.service.DistributedJobLockService; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 无主锁的防回归判据(2026-10-22 现网事故)。 + * + *

现场形状:一条 {@code task-queue} 虚拟线程卡在 LLM 响应体读取上(无超时), + * 它手里那把 {@code task:similar-asin:} 因此永远不放,而看门狗还在无条件续期 + * ——实测三把锁压到 11.3 小时,期间该任务每一次结果回传都判 40902(两小时 2370 次), + * 任务永久停在 RUNNING。两条线各自挡一种成因: + * ① 释放不许依赖「在哪个线程上 close」;② 就算持有方真的卡死,锁也不能是永久的。 + */ +class TaskDistributedLockServiceLeakTest { + + private static final Duration TTL = Duration.ofMinutes(5); + private static final long TASK_ID = 31888L; + + /** @return 服务;renewCalls 计 renew 次数,用于判「续期有没有停」。 */ + private TaskDistributedLockService serviceWithRenewCounting(AtomicInteger renewCalls, + DistributedJobLockService.LockHandle[] sink) { + DistributedJobLockService delegate = mock(DistributedJobLockService.class); + DistributedJobLockService.LockHandle inner = mock(DistributedJobLockService.LockHandle.class); + when(inner.renew(any(Duration.class))).thenAnswer(invocation -> { + renewCalls.incrementAndGet(); + return true; + }); + when(delegate.tryLock(anyString(), any(Duration.class))).thenReturn(inner); + sink[0] = inner; + return new TaskDistributedLockService(delegate); + } + + @Test + void closeFromAnotherThreadStillReleasesTheLock() throws Exception { + AtomicInteger renewCalls = new AtomicInteger(); + DistributedJobLockService.LockHandle[] inner = new DistributedJobLockService.LockHandle[1]; + TaskDistributedLockService service = serviceWithRenewCounting(renewCalls, inner); + + TaskDistributedLockService.LockHandle handle = service.acquire("similar-asin", TASK_ID); + assertTrue(service.activeHeldMillis().containsKey("task:similar-asin:" + TASK_ID), "取到锁必须读得到在持"); + + // 换一条线程去放:旧实现按取锁线程的 ThreadLocal 找锁,在别人那张表里查不到自己, + // 于是 close 静默 no-op —— 锁留在原地、续期任务留在原地,Redis 键被永久喂活。 + Thread closer = new Thread(handle::close, "other-thread-closer"); + closer.start(); + closer.join(5_000L); + + assertTrue(service.activeHeldMillis().isEmpty(), + "跨线程 close 必须真的放手:否则这条锁就是一条没人认领的僵尸锁"); + verify(inner[0], atLeastOnce()).close(); + } + + @Test + void nestedHandlesReleasedOnDifferentThreadsStillLetGo() throws Exception { + AtomicInteger renewCalls = new AtomicInteger(); + DistributedJobLockService.LockHandle[] inner = new DistributedJobLockService.LockHandle[1]; + TaskDistributedLockService service = serviceWithRenewCounting(renewCalls, inner); + + TaskDistributedLockService.LockHandle outer = service.acquire("similar-asin", TASK_ID); + TaskDistributedLockService.LockHandle nested = service.acquire("similar-asin", TASK_ID); + assertEquals(1, service.activeHeldMillis().size(), "重入只登记一次"); + nested.close(); + assertEquals(1, service.activeHeldMillis().size(), "只放了一层"); + + Thread closer = new Thread(outer::close, "other-thread-closer"); + closer.start(); + closer.join(5_000L); + assertTrue(service.activeHeldMillis().isEmpty(), "最后一层在别的线程上放,也必须算放过"); + } + + @Test + void renewalStopsOnceHoldPassesTheCeiling() throws Exception { + AtomicInteger renewCalls = new AtomicInteger(); + DistributedJobLockService.LockHandle[] inner = new DistributedJobLockService.LockHandle[1]; + TaskDistributedLockService service = serviceWithRenewCounting(renewCalls, inner); + // ttl 300ms → 续期节拍钳到下限 1s,封顶 = max(200ms, 6×300ms) = 1.8s: + // 第一次续期(≈1s)还在界内,第二次(≈2s)越界即停。 + service.setMaxHoldFloorMillisForTest(200L); + TaskDistributedLockService.LockHandle handle = + service.acquire("similar-asin", TASK_ID, Duration.ofMillis(300), 0L); + + long deadline = System.currentTimeMillis() + 6_000L; + while (renewCalls.get() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(50L); + } + assertTrue(renewCalls.get() >= 1, "没续过期就说明这条用例什么都没量到"); + int renewalsAtStopPoint = renewCalls.get(); + Thread.sleep(3_000L); + assertEquals(renewalsAtStopPoint, renewCalls.get(), + "越界后还在续期 = 锁永远不会自己解开,这正是 11.3 小时僵尸锁的成因"); + + // 到线只「放弃续期」,不替卡死的持有方去 del 键:键在 TTL 内自然过期, + // 持有方将来真跑完时那次带 token 的 RELEASE 若已不属于它也只会返回 0,不会误删接管者的锁。 + verify(inner[0], never()).close(); + assertTrue(service.activeHeldMillis().containsKey("task:similar-asin:" + TASK_ID), + "放弃续期的锁必须还留在在持读数里(标成续期已失败),看门狗不能对这种锁失明"); + handle.close(); + } +}