fix(task-lock): LLM 响应体补时间界+任务锁跨线程可放与持封顶,掐掉 11 小时无主锁
现网 31888/31856/31725 等任务永久停在 RUNNING,读数链是: java.net.http 的请求超时只封顶到响应头,响应体 read() 走 ArrayBlockingQueue.take(), 上游只发半截就发呆时这一读永久 park(12 条 similar-asin-llm-row 虚拟线程全停在那) ⇒ 无超时的 join() 把 task-queue 线程一起挂住 ⇒ 它手里的 task:similar-asin:<id> 被看门狗无条件续期(实测 11.3 小时,放过留痕 0 次)⇒ 该任务每次结果回传判 40902 (两小时 2370 次),且 12 个 taskQueueExecutor 许可被占满、44 条排队。 两道界各配非零对照: - HttpClientPool.readBodyBounded 把响应体读取挪进一次性虚拟线程、由调用方拿预算去等, 到点中断+关流退成可重试 IOException;三处同型 readResponseBodyBounded 改为委托它。 - LockHandle 直接绑住 HeldLock,任意线程 close 都真放手(旧实现按取锁线程的 ThreadLocal 找锁,跨线程 close 静默 no-op 就是无主锁本体);加持锁封顶 max(30 分钟, 6×TTL), 越界停止续期并 error 留痕,让键在 TTL 内自然过期。 裸跳过例外台账随之清空:它保的正是那条被现网证伪的静默 no-op。 门禁:干净 worktree(HEAD 97312821+本笔)后端全量 4686 例 / 0 红 / 17 skip / rc=0。
This commit is contained in:
+8
-22
@@ -19,10 +19,10 @@ import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -673,7 +673,7 @@ public class AppearancePatentLlmClient {
|
||||
});
|
||||
request.body(body);
|
||||
String responseText = request.exchange((clientRequest, clientResponse) -> {
|
||||
String responseBody = readResponseBodyBounded(clientResponse.getBody());
|
||||
String responseBody = readResponseBodyBounded(clientResponse.getBody(), attempt);
|
||||
if (!clientResponse.getStatusCode().is2xxSuccessful()) {
|
||||
throw new IllegalStateException("LLM http " + clientResponse.getStatusCode().value()
|
||||
+ ": " + abbreviate(responseBody, 500));
|
||||
@@ -842,26 +842,12 @@ public class AppearancePatentLlmClient {
|
||||
}
|
||||
}
|
||||
|
||||
private String readResponseBodyBounded(InputStream inputStream) throws IOException {
|
||||
if (inputStream == null) {
|
||||
return "";
|
||||
}
|
||||
try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int read;
|
||||
int total = 0;
|
||||
while ((read = input.read(buffer)) != -1) {
|
||||
if (read == 0) {
|
||||
continue;
|
||||
}
|
||||
if ((long) total + read > MAX_LLM_RESPONSE_BYTES) {
|
||||
throw new IOException("LLM response exceeds " + MAX_LLM_RESPONSE_BYTES + " bytes");
|
||||
}
|
||||
output.write(buffer, 0, read);
|
||||
total += read;
|
||||
}
|
||||
return output.toString(StandardCharsets.UTF_8);
|
||||
}
|
||||
/** 响应体读取预算与本次实际用的那套超时对齐;JDK 的超时只到响应头,这一腿原先无界。 */
|
||||
private String readResponseBodyBounded(InputStream inputStream, int attempt) throws IOException {
|
||||
long budget = attempt == 1
|
||||
? Math.min(firstAttemptTimeoutMillis(), Math.max(1, properties.getLlmReadTimeoutMillis()))
|
||||
: Math.max(1, properties.getLlmReadTimeoutMillis());
|
||||
return HttpClientPool.readBodyBounded(inputStream, Duration.ofMillis(budget), MAX_LLM_RESPONSE_BYTES);
|
||||
}
|
||||
|
||||
private JsonNode parseJsonOrThrow(String value) {
|
||||
|
||||
+11
-22
@@ -15,10 +15,10 @@ import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
@@ -126,7 +126,7 @@ public class SimilarAsinLlmClient {
|
||||
});
|
||||
request.body(body);
|
||||
String responseText = request.exchange((clientRequest, clientResponse) -> {
|
||||
String responseBody = readResponseBodyBounded(clientResponse.getBody());
|
||||
String responseBody = readResponseBodyBounded(clientResponse.getBody(), attempt);
|
||||
if (!clientResponse.getStatusCode().is2xxSuccessful()) {
|
||||
throw new IllegalStateException("LLM http " + clientResponse.getStatusCode().value()
|
||||
+ ": " + abbreviate(responseBody, 500));
|
||||
@@ -273,26 +273,15 @@ public class SimilarAsinLlmClient {
|
||||
}
|
||||
}
|
||||
|
||||
private String readResponseBodyBounded(InputStream inputStream) throws IOException {
|
||||
if (inputStream == null) {
|
||||
return "";
|
||||
}
|
||||
try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int read;
|
||||
int total = 0;
|
||||
while ((read = input.read(buffer)) != -1) {
|
||||
if (read == 0) {
|
||||
continue;
|
||||
}
|
||||
if ((long) total + read > MAX_LLM_RESPONSE_BYTES) {
|
||||
throw new IOException("LLM response exceeds " + MAX_LLM_RESPONSE_BYTES + " bytes");
|
||||
}
|
||||
output.write(buffer, 0, read);
|
||||
total += read;
|
||||
}
|
||||
return output.toString(StandardCharsets.UTF_8);
|
||||
}
|
||||
/**
|
||||
* 响应体的读取预算:与本次实际用的那个 RestClient 的超时对齐(首试用短超时那套),
|
||||
* 但**必须自己另算**——JDK 的超时只到响应头,响应体这一腿原先没有任何界。
|
||||
*/
|
||||
private String readResponseBodyBounded(InputStream inputStream, int attempt) throws IOException {
|
||||
long budget = attempt == 1
|
||||
? Math.min(firstAttemptTimeoutMillis(), Math.max(1, properties.getLlmReadTimeoutMillis()))
|
||||
: Math.max(1, properties.getLlmReadTimeoutMillis());
|
||||
return HttpClientPool.readBodyBounded(inputStream, Duration.ofMillis(budget), MAX_LLM_RESPONSE_BYTES);
|
||||
}
|
||||
|
||||
private JsonNode parseJsonOrThrow(String value) {
|
||||
|
||||
+9
@@ -70,6 +70,15 @@ final class ActiveTaskLockRegistry {
|
||||
active.remove(lockName);
|
||||
}
|
||||
|
||||
/**
|
||||
* 只有登记的就是这一任时才摘除。封顶放弃过的那把锁,持有线程可能过了很久才走到 close,
|
||||
* 这期间新持有者已经登记同名锁——按名字无条件删就会把新那一任的读数抹掉(看门狗就此失明)。
|
||||
*/
|
||||
void unregisterIfCurrent(String lockName, long acquiredAtMillis) {
|
||||
active.computeIfPresent(lockName,
|
||||
(key, lock) -> lock.acquiredAtMillis == acquiredAtMillis ? null : lock);
|
||||
}
|
||||
|
||||
void markRenewalLost(String lockName) {
|
||||
ActiveLock lock = active.get(lockName);
|
||||
if (lock != null) {
|
||||
|
||||
+84
-41
@@ -11,10 +11,13 @@ import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -30,9 +33,24 @@ public class TaskDistributedLockService {
|
||||
/** 持锁时长留痕:任务锁被 LLM 批处理占用是 40902 的来源,此前只有 WAIT_LLM 一支量得出时长。 */
|
||||
private static final long HELD_REPORT_MILLIS = 5000L;
|
||||
private static final long HELD_WARN_MILLIS = 60000L;
|
||||
/**
|
||||
* 持锁封顶=max(30 分钟, 6×TTL)。越过这条线还续着期,说明持有方已经卡死而不是在干活:
|
||||
* 现网实测三把 {@code task:similar-asin:*} 被看门狗喂到 11.3 小时不放,期间该任务的
|
||||
* 每一次结果回传都判 40902(两小时窗口 2370 次),任务永远停在 RUNNING。
|
||||
* 到线后停止续期、让 Redis 键在 TTL 内自然过期,把"永久焊死"降级成"最多卡 TTL 量级"。
|
||||
*/
|
||||
private static final long MAX_HOLD_FLOOR_MILLIS = 30 * 60_000L;
|
||||
private static final int MAX_HOLD_TTL_MULTIPLE = 6;
|
||||
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
|
||||
/** 持锁封顶的下限线,见 {@link #MAX_HOLD_FLOOR_MILLIS};测试会临时压低它来走封顶分支。 */
|
||||
private volatile long maxHoldFloorMillis = MAX_HOLD_FLOOR_MILLIS;
|
||||
/**
|
||||
* 取锁线程 → 该线程在持的锁。它只负责**重入判定**(同一线程再次取同一把锁不能把自己挡住),
|
||||
* 释放一律走 {@link LockHandle} 上直接绑定的 {@link HeldLock}:早先 release 也按这张表找,
|
||||
* 跨线程 close 会在别人的表里查不到自己的锁而静默 no-op,锁和续期任务就双双留在原地没人管。
|
||||
*/
|
||||
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(ConcurrentHashMap::new);
|
||||
/**
|
||||
* 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。
|
||||
* 线程数从 4 提到 16(2026-09 全维度审查):并发持锁任务数上升或 Redis 抖动时,
|
||||
@@ -70,19 +88,19 @@ public class TaskDistributedLockService {
|
||||
Map<String, HeldLock> heldLocks = localLocks.get();
|
||||
HeldLock heldLock = heldLocks.get(lockName);
|
||||
if (heldLock != null) {
|
||||
heldLock.depth++;
|
||||
return new LockHandle(lockName);
|
||||
heldLock.depth.incrementAndGet();
|
||||
return new LockHandle(heldLock);
|
||||
}
|
||||
|
||||
long deadline = System.nanoTime() + Math.max(0L, waitMillis) * 1_000_000L;
|
||||
do {
|
||||
DistributedJobLockService.LockHandle delegate = distributedJobLockService.tryLock(lockName, ttl);
|
||||
if (delegate != null) {
|
||||
HeldLock newHeldLock = new HeldLock(delegate, ttl);
|
||||
HeldLock newHeldLock = new HeldLock(lockName, delegate, ttl, heldLocks);
|
||||
scheduleRenewal(lockName, newHeldLock, ttl);
|
||||
heldLocks.put(lockName, newHeldLock);
|
||||
activeLocks.register(lockName, newHeldLock.acquiredAtMillis, ttl.toMillis());
|
||||
return new LockHandle(lockName);
|
||||
return new LockHandle(newHeldLock);
|
||||
}
|
||||
if (waitMillis <= 0L || System.nanoTime() >= deadline) {
|
||||
return null;
|
||||
@@ -118,18 +136,30 @@ public class TaskDistributedLockService {
|
||||
long ttlMillis = Math.max(1000L, ttl.toMillis());
|
||||
long renewalDelayMillis = Math.max(1000L, ttlMillis / 3L);
|
||||
heldLock.renewalFuture = renewalExecutor.scheduleWithFixedDelay(() -> {
|
||||
if (heldLock.closed) {
|
||||
if (heldLock.closed.get()) {
|
||||
return;
|
||||
}
|
||||
long heldMillis = System.currentTimeMillis() - heldLock.acquiredAtMillis;
|
||||
// 封顶按调用方**要的那条 TTL**算,不按下面钳过下限的那条:
|
||||
// 下限是给续期节拍兜底的,拿它去放大封顶线会让短 TTL 的锁反而能压更久。
|
||||
if (heldMillis > maxHoldMillis(ttl.toMillis())) {
|
||||
// 封顶:不再续期,让键在 TTL 内自己过期。故意不主动 del——持有方可能只是慢,
|
||||
// 由它跑完时那次带 token 的 RELEASE 去收尾更安全(token 不匹配即 0,不会误删接管者的锁)。
|
||||
heldLock.closed.set(true);
|
||||
activeLocks.markRenewalLost(lockName);
|
||||
cancelRenewal(heldLock);
|
||||
log.error("[task-lock] 持锁超过上限,停止续期 lockName={} heldMs={} ceilingMs={} ttlMs={} —— "
|
||||
+ "持有方已卡死,本节点不应再对该资源做破坏性写操作;"
|
||||
+ "本锁最迟 TTL 后可被其它实例接管",
|
||||
lockName, heldMillis, maxHoldMillis(ttl.toMillis()), ttlMillis);
|
||||
return;
|
||||
}
|
||||
if (renewWithRetry(lockName, heldLock)) {
|
||||
return;
|
||||
}
|
||||
heldLock.closed = true;
|
||||
heldLock.closed.set(true);
|
||||
activeLocks.markRenewalLost(lockName);
|
||||
ScheduledFuture<?> future = heldLock.renewalFuture;
|
||||
if (future != null) {
|
||||
future.cancel(false);
|
||||
}
|
||||
cancelRenewal(heldLock);
|
||||
// 此前单次失败即静默停止续期:本节点会继续以「仍持锁」的假设做破坏性操作,
|
||||
// 而锁最迟在 TTL 后已可被另一实例获取 —— 必须留下 error 级线索便于事后追溯
|
||||
log.error("[task-lock] 续期连续 {} 次失败,锁已失效 lockName={} —— "
|
||||
@@ -138,6 +168,23 @@ public class TaskDistributedLockService {
|
||||
}, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
private static void cancelRenewal(HeldLock heldLock) {
|
||||
ScheduledFuture<?> future = heldLock.renewalFuture;
|
||||
if (future != null) {
|
||||
future.cancel(false);
|
||||
}
|
||||
}
|
||||
|
||||
/** 封顶随 TTL 缩放:调用方给了更长的 TTL,就该配更长的容忍线。 */
|
||||
private long maxHoldMillis(long ttlMillis) {
|
||||
return Math.max(maxHoldFloorMillis, MAX_HOLD_TTL_MULTIPLE * ttlMillis);
|
||||
}
|
||||
|
||||
/** 测试接缝:把 30 分钟的下限压到毫秒级,才能在用例里真走到封顶那一条路。 */
|
||||
void setMaxHoldFloorMillisForTest(long floorMillis) {
|
||||
this.maxHoldFloorMillis = Math.max(1L, floorMillis);
|
||||
}
|
||||
|
||||
/**
|
||||
* 续期重试:单次失败可能是 Redis 抖动 / Sentinel 主从切换造成的瞬时问题,
|
||||
* 立即放弃会让本节点在锁已过期的情况下继续运行;重试仍失败才判锁丢失。
|
||||
@@ -191,35 +238,39 @@ public class TaskDistributedLockService {
|
||||
|
||||
private static final class HeldLock {
|
||||
|
||||
private final String lockName;
|
||||
private final DistributedJobLockService.LockHandle delegate;
|
||||
private final Duration ttl;
|
||||
private final long acquiredAtMillis;
|
||||
private ScheduledFuture<?> renewalFuture;
|
||||
private int depth = 1;
|
||||
private volatile boolean closed;
|
||||
/** 取锁线程那张表:释放要从这里摘掉自己,摘的动作与在哪个线程上做无关。 */
|
||||
private final Map<String, HeldLock> ownerLocks;
|
||||
private volatile ScheduledFuture<?> renewalFuture;
|
||||
private final AtomicInteger depth = new AtomicInteger(1);
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
private HeldLock(DistributedJobLockService.LockHandle delegate, Duration ttl) {
|
||||
private HeldLock(String lockName, DistributedJobLockService.LockHandle delegate, Duration ttl,
|
||||
Map<String, HeldLock> ownerLocks) {
|
||||
this.lockName = lockName;
|
||||
this.delegate = delegate;
|
||||
this.ttl = ttl;
|
||||
this.acquiredAtMillis = System.currentTimeMillis();
|
||||
this.ownerLocks = ownerLocks;
|
||||
}
|
||||
|
||||
private void close() {
|
||||
closed = true;
|
||||
if (renewalFuture != null) {
|
||||
renewalFuture.cancel(false);
|
||||
}
|
||||
closed.set(true);
|
||||
cancelRenewal(this);
|
||||
delegate.close();
|
||||
}
|
||||
}
|
||||
|
||||
public final class LockHandle implements AutoCloseable {
|
||||
|
||||
private final String lockName;
|
||||
private final HeldLock heldLock;
|
||||
private boolean closed;
|
||||
|
||||
private LockHandle(String lockName) {
|
||||
this.lockName = lockName;
|
||||
private LockHandle(HeldLock heldLock) {
|
||||
this.heldLock = heldLock;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -228,26 +279,18 @@ public class TaskDistributedLockService {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
Map<String, HeldLock> heldLocks = localLocks.get();
|
||||
HeldLock heldLock = heldLocks.get(lockName);
|
||||
if (heldLock == null) {
|
||||
return;
|
||||
if (heldLock.depth.decrementAndGet() > 0) {
|
||||
return; // 只放了一层,锁还在手里
|
||||
}
|
||||
heldLock.depth--;
|
||||
if (heldLock.depth <= 0) {
|
||||
heldLocks.remove(lockName);
|
||||
activeLocks.unregister(lockName);
|
||||
if (heldLocks.isEmpty()) {
|
||||
localLocks.remove();
|
||||
}
|
||||
long heldMillis = System.currentTimeMillis() - heldLock.acquiredAtMillis;
|
||||
heldLock.close();
|
||||
if (heldMillis >= HELD_WARN_MILLIS) {
|
||||
log.warn("[task-lock-held] lockName={} heldMs={} threshold={}ms",
|
||||
lockName, heldMillis, HELD_WARN_MILLIS);
|
||||
} else if (heldMillis >= HELD_REPORT_MILLIS) {
|
||||
log.info("[task-lock-held] lockName={} heldMs={}", lockName, heldMillis);
|
||||
}
|
||||
heldLock.ownerLocks.remove(heldLock.lockName, heldLock);
|
||||
activeLocks.unregisterIfCurrent(heldLock.lockName, heldLock.acquiredAtMillis);
|
||||
long heldMillis = System.currentTimeMillis() - heldLock.acquiredAtMillis;
|
||||
heldLock.close();
|
||||
if (heldMillis >= HELD_WARN_MILLIS) {
|
||||
log.warn("[task-lock-held] lockName={} heldMs={} threshold={}ms",
|
||||
heldLock.lockName, heldMillis, HELD_WARN_MILLIS);
|
||||
} else if (heldMillis >= HELD_REPORT_MILLIS) {
|
||||
log.info("[task-lock-held] lockName={} heldMs={}", heldLock.lockName, heldMillis);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+3
-20
@@ -16,11 +16,11 @@ import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
@@ -472,26 +472,9 @@ public class UserApiSecretCheckService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 探测响应体的读取界:JDK 的 readTimeout 只封顶到响应头,断流时这一读会永久 park。 */
|
||||
private String readResponseBodyBounded(InputStream inputStream) throws IOException {
|
||||
if (inputStream == null) {
|
||||
return "";
|
||||
}
|
||||
try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int read;
|
||||
int total = 0;
|
||||
while ((read = input.read(buffer)) != -1) {
|
||||
if (read == 0) {
|
||||
continue;
|
||||
}
|
||||
if ((long) total + read > MAX_RESPONSE_BYTES) {
|
||||
throw new IOException("检测响应超过 " + MAX_RESPONSE_BYTES + " 字节");
|
||||
}
|
||||
output.write(buffer, 0, read);
|
||||
total += read;
|
||||
}
|
||||
return output.toString(StandardCharsets.UTF_8);
|
||||
}
|
||||
return HttpClientPool.readBodyBounded(inputStream, Duration.ofMillis(READ_TIMEOUT_MILLIS), MAX_RESPONSE_BYTES);
|
||||
}
|
||||
|
||||
private String joinUrl(String baseUrl, String path) {
|
||||
|
||||
Reference in New Issue
Block a user