在持锁补读数:heldMs 此前只在放手那一刻打,锁还压着多久读不到
第 3/4 步(拆锁、缩合批)要立论,必须知道"此刻这把锁已经压了多久",而 [task-lock-held] 只在 LockHandle.close() 打——正在持有的那一段是盲区(货源 40902×24h RUNNING 就是这么看不见的)。 - ActiveTaskLockRegistry:登记表跟着取锁/放锁动;单独标 renewalLost——续期连续失败的锁最迟在 TTL 后就该被另一实例拿走,本节点却还在按"仍持锁"跑,挂着多久不等于还被保护。 - 看门狗 task-lock-watch 每 60 秒扫一轮,留痕走阶梯(首个 60 秒、之后每 5 分钟), 一次 30 分钟持锁总共 6 行而不是 30 行——与 10-06 通知放大同一口径。 - 15 例(9 节奏 + 6 接线),变异自证 M1 丢 unregister/M2 阶梯归零/M3 丢 register 三轮全 KILLED; 后端全量 4320/0 红/跳过 7。
This commit is contained in:
+113
@@ -0,0 +1,113 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
/**
|
||||
* 「正在持有」的任务锁登记表。
|
||||
*
|
||||
* <p>动机:{@code [task-lock-held]} 那条留痕只在 {@code LockHandle.close()} 打,
|
||||
* 于是"锁此刻还压着多久"这一维在生产上读不到——40902 连着出现时,分不清"马上要放了"
|
||||
* 和"已经挂了 24 小时"(货源那单就是后者)。这里补的正是那半句读数。</p>
|
||||
*
|
||||
* <p>另外单独标出"续期已连续失败":那种锁最迟在 TTL 后就该被另一实例拿走,
|
||||
* 本节点却还在按"仍持锁"的假设跑——挂着多久不等于还被谁保护,这个区别只能靠这条读数看出来。</p>
|
||||
*
|
||||
* <p>留痕按阶梯走(首个 60 秒、之后每 5 分钟一次),一次长持锁不会把日志灌满,
|
||||
* 与 10-06 通知放大那次处置同一口径。</p>
|
||||
*/
|
||||
final class ActiveTaskLockRegistry {
|
||||
|
||||
/** 60 秒以内的持锁不留痕:任务锁的正常生命周期本来就以秒到分钟计。 */
|
||||
static final long FIRST_REPORT_MILLIS = 60_000L;
|
||||
/** 首次之后每 5 分钟复报一次(一次 30 分钟的持锁总共 6 行,不是 30 行)。 */
|
||||
static final long REPEAT_REPORT_MILLIS = 300_000L;
|
||||
|
||||
static final class ActiveLock {
|
||||
|
||||
private final long acquiredAtMillis;
|
||||
private final long ttlMillis;
|
||||
private volatile boolean renewalLost;
|
||||
private volatile long lastReportAtMillis;
|
||||
|
||||
private ActiveLock(long acquiredAtMillis, long ttlMillis) {
|
||||
this.acquiredAtMillis = acquiredAtMillis;
|
||||
this.ttlMillis = ttlMillis;
|
||||
}
|
||||
|
||||
long heldMillis(long nowMillis) {
|
||||
return Math.max(0L, nowMillis - acquiredAtMillis);
|
||||
}
|
||||
|
||||
long ttlMillis() {
|
||||
return ttlMillis;
|
||||
}
|
||||
|
||||
/** 持有时长已超过 TTL:只有在续期也失败时才有意义(续期成功会让锁一直有效)。 */
|
||||
boolean ttlExpired(long nowMillis) {
|
||||
return heldMillis(nowMillis) > ttlMillis;
|
||||
}
|
||||
|
||||
boolean isRenewalLost() {
|
||||
return renewalLost;
|
||||
}
|
||||
}
|
||||
|
||||
private final ConcurrentMap<String, ActiveLock> active = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 登记一次持锁。同名重复登记以最后一次为准——上一任若是没走到 {@code unregister}
|
||||
* 就崩了,留下的旧时间戳会把"新持有者刚拿到锁"读成"已经压了很久"。
|
||||
*/
|
||||
void register(String lockName, long acquiredAtMillis, long ttlMillis) {
|
||||
active.put(lockName, new ActiveLock(acquiredAtMillis, ttlMillis));
|
||||
}
|
||||
|
||||
void unregister(String lockName) {
|
||||
active.remove(lockName);
|
||||
}
|
||||
|
||||
void markRenewalLost(String lockName) {
|
||||
ActiveLock lock = active.get(lockName);
|
||||
if (lock != null) {
|
||||
lock.renewalLost = true;
|
||||
}
|
||||
}
|
||||
|
||||
int size() {
|
||||
return active.size();
|
||||
}
|
||||
|
||||
/** 当前在持的锁快照(底层是 ConcurrentHashMap,不保证顺序);要不要留痕交给 shouldReport 逐条判。 */
|
||||
Map<String, ActiveLock> snapshot() {
|
||||
Map<String, ActiveLock> copy = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, ActiveLock> entry : active.entrySet()) {
|
||||
copy.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return Collections.unmodifiableMap(copy);
|
||||
}
|
||||
|
||||
/**
|
||||
* 这一轮要不要留痕。返回 true 的同时把复报基准推到 {@code nowMillis},
|
||||
* 所以调用方只在真要写日志时才调它(判一次不写一次会让下一条等到 5 分钟后)。
|
||||
*/
|
||||
boolean shouldReport(String lockName, long nowMillis) {
|
||||
ActiveLock lock = active.get(lockName);
|
||||
if (lock == null) {
|
||||
return false;
|
||||
}
|
||||
long held = lock.heldMillis(nowMillis);
|
||||
if (held < FIRST_REPORT_MILLIS) {
|
||||
return false;
|
||||
}
|
||||
long last = lock.lastReportAtMillis;
|
||||
if (last != 0L && nowMillis - last < REPEAT_REPORT_MILLIS) {
|
||||
return false;
|
||||
}
|
||||
lock.lastReportAtMillis = nowMillis;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+45
-1
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -43,6 +44,16 @@ public class TaskDistributedLockService {
|
||||
return thread;
|
||||
});
|
||||
|
||||
/** [task-lock-held] 那条只在放手时打;这一张补上「现在还压着多久」的读数。 */
|
||||
private final ActiveTaskLockRegistry activeLocks = new ActiveTaskLockRegistry();
|
||||
/** 看门狗 60 秒一轮,要不要写由 registry 的阶梯判(一次 30 分钟持锁总共 6 行,不是 30 行)。 */
|
||||
private static final long ACTIVE_REPORT_INTERVAL_MILLIS = 60_000L;
|
||||
private final ScheduledExecutorService activeReportExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-lock-watch");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
public LockHandle acquire(String moduleType, Long taskId) {
|
||||
return acquire(moduleType, taskId, DEFAULT_WAIT_MILLIS);
|
||||
}
|
||||
@@ -70,6 +81,7 @@ public class TaskDistributedLockService {
|
||||
HeldLock newHeldLock = new HeldLock(delegate, ttl);
|
||||
scheduleRenewal(lockName, newHeldLock, ttl);
|
||||
heldLocks.put(lockName, newHeldLock);
|
||||
activeLocks.register(lockName, newHeldLock.acquiredAtMillis, ttl.toMillis());
|
||||
return new LockHandle(lockName);
|
||||
}
|
||||
if (waitMillis <= 0L || System.nanoTime() >= deadline) {
|
||||
@@ -92,6 +104,16 @@ public class TaskDistributedLockService {
|
||||
return "task:" + normalizedModule + ":" + taskId;
|
||||
}
|
||||
|
||||
/** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */
|
||||
Map<String, Long> activeHeldMillis() {
|
||||
long now = System.currentTimeMillis();
|
||||
Map<String, Long> readings = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, ActiveTaskLockRegistry.ActiveLock> entry : activeLocks.snapshot().entrySet()) {
|
||||
readings.put(entry.getKey(), entry.getValue().heldMillis(now));
|
||||
}
|
||||
return readings;
|
||||
}
|
||||
|
||||
private void scheduleRenewal(String lockName, HeldLock heldLock, Duration ttl) {
|
||||
long ttlMillis = Math.max(1000L, ttl.toMillis());
|
||||
long renewalDelayMillis = Math.max(1000L, ttlMillis / 3L);
|
||||
@@ -103,6 +125,7 @@ public class TaskDistributedLockService {
|
||||
return;
|
||||
}
|
||||
heldLock.closed = true;
|
||||
activeLocks.markRenewalLost(lockName);
|
||||
ScheduledFuture<?> future = heldLock.renewalFuture;
|
||||
if (future != null) {
|
||||
future.cancel(false);
|
||||
@@ -141,9 +164,29 @@ public class TaskDistributedLockService {
|
||||
return false;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
void startActiveLockWatch() {
|
||||
activeReportExecutor.scheduleWithFixedDelay(this::reportActiveLocks,
|
||||
ACTIVE_REPORT_INTERVAL_MILLIS, ACTIVE_REPORT_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
/** 在持留痕:读得到「这把锁还没放、已经压了多久」;续期失败的那类单独标 renewalLost=true。 */
|
||||
void reportActiveLocks() {
|
||||
long now = System.currentTimeMillis();
|
||||
for (Map.Entry<String, ActiveTaskLockRegistry.ActiveLock> entry : activeLocks.snapshot().entrySet()) {
|
||||
if (!activeLocks.shouldReport(entry.getKey(), now)) {
|
||||
continue;
|
||||
}
|
||||
ActiveTaskLockRegistry.ActiveLock lock = entry.getValue();
|
||||
log.warn("[task-lock-active] lockName={} heldMs={} ttlMs={} renewalLost={}",
|
||||
entry.getKey(), lock.heldMillis(now), lock.ttlMillis(), lock.isRenewalLost());
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdownRenewalExecutor() {
|
||||
public void shutdownExecutors() {
|
||||
renewalExecutor.shutdownNow();
|
||||
activeReportExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
private static final class HeldLock {
|
||||
@@ -193,6 +236,7 @@ public class TaskDistributedLockService {
|
||||
heldLock.depth--;
|
||||
if (heldLock.depth <= 0) {
|
||||
heldLocks.remove(lockName);
|
||||
activeLocks.unregister(lockName);
|
||||
if (heldLocks.isEmpty()) {
|
||||
localLocks.remove();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user