From 975e053d2c98f05524246b3fadeb06004529f669 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com>
Date: Wed, 14 Oct 2026 01:06:38 +0800
Subject: [PATCH] =?UTF-8?q?=E5=9C=A8=E6=8C=81=E9=94=81=E8=A1=A5=E8=AF=BB?=
=?UTF-8?q?=E6=95=B0=EF=BC=9AheldMs=20=E6=AD=A4=E5=89=8D=E5=8F=AA=E5=9C=A8?=
=?UTF-8?q?=E6=94=BE=E6=89=8B=E9=82=A3=E4=B8=80=E5=88=BB=E6=89=93=EF=BC=8C?=
=?UTF-8?q?=E9=94=81=E8=BF=98=E5=8E=8B=E7=9D=80=E5=A4=9A=E4=B9=85=E8=AF=BB?=
=?UTF-8?q?=E4=B8=8D=E5=88=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
第 3/4 步(拆锁、缩合批)要立论,必须知道"此刻这把锁已经压了多久",而 [task-lock-held] 只在
LockHandle.close() 打——正在持有的那一段是盲区(货源 40902×24h RUNNING 就是这么看不见的)。
- ActiveTaskLockRegistry:登记表跟着取锁/放锁动;单独标 renewalLost——续期连续失败的锁最迟在
TTL 后就该被另一实例拿走,本节点却还在按"仍持锁"跑,挂着多久不等于还被保护。
- 看门狗 task-lock-watch 每 60 秒扫一轮,留痕走阶梯(首个 60 秒、之后每 5 分钟),
一次 30 分钟持锁总共 6 行而不是 30 行——与 10-06 通知放大同一口径。
- 15 例(9 节奏 + 6 接线),变异自证 M1 丢 unregister/M2 阶梯归零/M3 丢 register 三轮全 KILLED;
后端全量 4320/0 红/跳过 7。
---
.../task/service/ActiveTaskLockRegistry.java | 113 +++++++++++++++++
.../service/TaskDistributedLockService.java | 46 ++++++-
.../service/ActiveTaskLockRegistryTest.java | 114 ++++++++++++++++++
...stributedLockServiceActiveReadingTest.java | 102 ++++++++++++++++
4 files changed, 374 insertions(+), 1 deletion(-)
create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java
create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistryTest.java
create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceActiveReadingTest.java
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java
new file mode 100644
index 000000000..992a62d0a
--- /dev/null
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java
@@ -0,0 +1,113 @@
+package com.nanri.aiimage.modules.task.service;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * 「正在持有」的任务锁登记表。
+ *
+ *
动机:{@code [task-lock-held]} 那条留痕只在 {@code LockHandle.close()} 打,
+ * 于是"锁此刻还压着多久"这一维在生产上读不到——40902 连着出现时,分不清"马上要放了"
+ * 和"已经挂了 24 小时"(货源那单就是后者)。这里补的正是那半句读数。
+ *
+ * 另外单独标出"续期已连续失败":那种锁最迟在 TTL 后就该被另一实例拿走,
+ * 本节点却还在按"仍持锁"的假设跑——挂着多久不等于还被谁保护,这个区别只能靠这条读数看出来。
+ *
+ * 留痕按阶梯走(首个 60 秒、之后每 5 分钟一次),一次长持锁不会把日志灌满,
+ * 与 10-06 通知放大那次处置同一口径。
+ */
+final class ActiveTaskLockRegistry {
+
+ /** 60 秒以内的持锁不留痕:任务锁的正常生命周期本来就以秒到分钟计。 */
+ static final long FIRST_REPORT_MILLIS = 60_000L;
+ /** 首次之后每 5 分钟复报一次(一次 30 分钟的持锁总共 6 行,不是 30 行)。 */
+ static final long REPEAT_REPORT_MILLIS = 300_000L;
+
+ static final class ActiveLock {
+
+ private final long acquiredAtMillis;
+ private final long ttlMillis;
+ private volatile boolean renewalLost;
+ private volatile long lastReportAtMillis;
+
+ private ActiveLock(long acquiredAtMillis, long ttlMillis) {
+ this.acquiredAtMillis = acquiredAtMillis;
+ this.ttlMillis = ttlMillis;
+ }
+
+ long heldMillis(long nowMillis) {
+ return Math.max(0L, nowMillis - acquiredAtMillis);
+ }
+
+ long ttlMillis() {
+ return ttlMillis;
+ }
+
+ /** 持有时长已超过 TTL:只有在续期也失败时才有意义(续期成功会让锁一直有效)。 */
+ boolean ttlExpired(long nowMillis) {
+ return heldMillis(nowMillis) > ttlMillis;
+ }
+
+ boolean isRenewalLost() {
+ return renewalLost;
+ }
+ }
+
+ private final ConcurrentMap active = new ConcurrentHashMap<>();
+
+ /**
+ * 登记一次持锁。同名重复登记以最后一次为准——上一任若是没走到 {@code unregister}
+ * 就崩了,留下的旧时间戳会把"新持有者刚拿到锁"读成"已经压了很久"。
+ */
+ void register(String lockName, long acquiredAtMillis, long ttlMillis) {
+ active.put(lockName, new ActiveLock(acquiredAtMillis, ttlMillis));
+ }
+
+ void unregister(String lockName) {
+ active.remove(lockName);
+ }
+
+ void markRenewalLost(String lockName) {
+ ActiveLock lock = active.get(lockName);
+ if (lock != null) {
+ lock.renewalLost = true;
+ }
+ }
+
+ int size() {
+ return active.size();
+ }
+
+ /** 当前在持的锁快照(底层是 ConcurrentHashMap,不保证顺序);要不要留痕交给 shouldReport 逐条判。 */
+ Map snapshot() {
+ Map copy = new LinkedHashMap<>();
+ for (Map.Entry entry : active.entrySet()) {
+ copy.put(entry.getKey(), entry.getValue());
+ }
+ return Collections.unmodifiableMap(copy);
+ }
+
+ /**
+ * 这一轮要不要留痕。返回 true 的同时把复报基准推到 {@code nowMillis},
+ * 所以调用方只在真要写日志时才调它(判一次不写一次会让下一条等到 5 分钟后)。
+ */
+ boolean shouldReport(String lockName, long nowMillis) {
+ ActiveLock lock = active.get(lockName);
+ if (lock == null) {
+ return false;
+ }
+ long held = lock.heldMillis(nowMillis);
+ if (held < FIRST_REPORT_MILLIS) {
+ return false;
+ }
+ long last = lock.lastReportAtMillis;
+ if (last != 0L && nowMillis - last < REPEAT_REPORT_MILLIS) {
+ return false;
+ }
+ lock.lastReportAtMillis = nowMillis;
+ return true;
+ }
+}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java
index 51dfe734e..bfe498b88 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
+import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -43,6 +44,16 @@ public class TaskDistributedLockService {
return thread;
});
+ /** [task-lock-held] 那条只在放手时打;这一张补上「现在还压着多久」的读数。 */
+ private final ActiveTaskLockRegistry activeLocks = new ActiveTaskLockRegistry();
+ /** 看门狗 60 秒一轮,要不要写由 registry 的阶梯判(一次 30 分钟持锁总共 6 行,不是 30 行)。 */
+ private static final long ACTIVE_REPORT_INTERVAL_MILLIS = 60_000L;
+ private final ScheduledExecutorService activeReportExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
+ Thread thread = new Thread(runnable, "task-lock-watch");
+ thread.setDaemon(true);
+ return thread;
+ });
+
public LockHandle acquire(String moduleType, Long taskId) {
return acquire(moduleType, taskId, DEFAULT_WAIT_MILLIS);
}
@@ -70,6 +81,7 @@ public class TaskDistributedLockService {
HeldLock newHeldLock = new HeldLock(delegate, ttl);
scheduleRenewal(lockName, newHeldLock, ttl);
heldLocks.put(lockName, newHeldLock);
+ activeLocks.register(lockName, newHeldLock.acquiredAtMillis, ttl.toMillis());
return new LockHandle(lockName);
}
if (waitMillis <= 0L || System.nanoTime() >= deadline) {
@@ -92,6 +104,16 @@ public class TaskDistributedLockService {
return "task:" + normalizedModule + ":" + taskId;
}
+ /** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */
+ Map activeHeldMillis() {
+ long now = System.currentTimeMillis();
+ Map readings = new LinkedHashMap<>();
+ for (Map.Entry entry : activeLocks.snapshot().entrySet()) {
+ readings.put(entry.getKey(), entry.getValue().heldMillis(now));
+ }
+ return readings;
+ }
+
private void scheduleRenewal(String lockName, HeldLock heldLock, Duration ttl) {
long ttlMillis = Math.max(1000L, ttl.toMillis());
long renewalDelayMillis = Math.max(1000L, ttlMillis / 3L);
@@ -103,6 +125,7 @@ public class TaskDistributedLockService {
return;
}
heldLock.closed = true;
+ activeLocks.markRenewalLost(lockName);
ScheduledFuture> future = heldLock.renewalFuture;
if (future != null) {
future.cancel(false);
@@ -141,9 +164,29 @@ public class TaskDistributedLockService {
return false;
}
+ @PostConstruct
+ void startActiveLockWatch() {
+ activeReportExecutor.scheduleWithFixedDelay(this::reportActiveLocks,
+ ACTIVE_REPORT_INTERVAL_MILLIS, ACTIVE_REPORT_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
+ }
+
+ /** 在持留痕:读得到「这把锁还没放、已经压了多久」;续期失败的那类单独标 renewalLost=true。 */
+ void reportActiveLocks() {
+ long now = System.currentTimeMillis();
+ for (Map.Entry entry : activeLocks.snapshot().entrySet()) {
+ if (!activeLocks.shouldReport(entry.getKey(), now)) {
+ continue;
+ }
+ ActiveTaskLockRegistry.ActiveLock lock = entry.getValue();
+ log.warn("[task-lock-active] lockName={} heldMs={} ttlMs={} renewalLost={}",
+ entry.getKey(), lock.heldMillis(now), lock.ttlMillis(), lock.isRenewalLost());
+ }
+ }
+
@PreDestroy
- public void shutdownRenewalExecutor() {
+ public void shutdownExecutors() {
renewalExecutor.shutdownNow();
+ activeReportExecutor.shutdownNow();
}
private static final class HeldLock {
@@ -193,6 +236,7 @@ public class TaskDistributedLockService {
heldLock.depth--;
if (heldLock.depth <= 0) {
heldLocks.remove(lockName);
+ activeLocks.unregister(lockName);
if (heldLocks.isEmpty()) {
localLocks.remove();
}
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistryTest.java
new file mode 100644
index 000000000..c052c8cad
--- /dev/null
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistryTest.java
@@ -0,0 +1,114 @@
+package com.nanri.aiimage.modules.task.service;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * 「正在持有」读数的判据(10-14 补的埋点)。
+ *
+ * 要钉住的是原来读不到的那一维:锁**还没放**的时候它已经压了多久,以及那把锁的续期是否已经失败
+ * (后者意味着锁最迟在 TTL 后就该被另一实例拿走,本节点却还在按"仍持锁"跑)。
+ * 顺带钉住留痕节奏:阶梯不成立就会把一次长持锁灌成每分钟一行,与 10-06 通知放大同族。
+ */
+class ActiveTaskLockRegistryTest {
+
+ private static final long T0 = 1_700_000_000_000L;
+ private static final long TTL = Duration.ofMinutes(5).toMillis();
+
+ private static ActiveTaskLockRegistry registered() {
+ ActiveTaskLockRegistry registry = new ActiveTaskLockRegistry();
+ registry.register("similar-asin:30853", T0, TTL);
+ return registry;
+ }
+
+ @Test
+ void ladderIsPinnedAtOneMinuteThenFiveMinutes() {
+ assertEquals(60_000L, ActiveTaskLockRegistry.FIRST_REPORT_MILLIS);
+ assertEquals(300_000L, ActiveTaskLockRegistry.REPEAT_REPORT_MILLIS);
+ }
+
+ @Test
+ void holdUnderOneMinuteStaysSilent() {
+ ActiveTaskLockRegistry registry = registered();
+ assertFalse(registry.shouldReport("similar-asin:30853", T0 + 59_999L),
+ "60 秒以内的持锁是常态,读数不该在这一档说话");
+ assertFalse(registry.shouldReport("别的锁", T0 + 999_999L), "没登记过的名字恒不报");
+ }
+
+ @Test
+ void firstReportHappensOnceAtTheMinuteMarkThenWaits() {
+ ActiveTaskLockRegistry registry = registered();
+ assertTrue(registry.shouldReport("similar-asin:30853", T0 + 60_000L));
+ assertFalse(registry.shouldReport("similar-asin:30853", T0 + 60_001L), "同一轮不许复报");
+ assertFalse(registry.shouldReport("similar-asin:30853", T0 + 359_999L));
+ assertTrue(registry.shouldReport("similar-asin:30853", T0 + 360_000L), "满 5 分钟再报一次");
+ }
+
+ @Test
+ void releasedLockLeavesTheReading() {
+ ActiveTaskLockRegistry registry = registered();
+ registry.unregister("similar-asin:30853");
+ assertEquals(0, registry.size());
+ assertFalse(registry.shouldReport("similar-asin:30853", T0 + 900_000L),
+ "已经放手的锁不能还在'在持'里计数");
+ assertTrue(registry.snapshot().isEmpty());
+ }
+
+ @Test
+ void renewalLostIsReadableWhileTheHoldKeepsGrowing() {
+ ActiveTaskLockRegistry registry = registered();
+ registry.markRenewalLost("similar-asin:30853");
+ Map snap = registry.snapshot();
+ ActiveTaskLockRegistry.ActiveLock lock = snap.get("similar-asin:30853");
+ assertTrue(lock.isRenewalLost(), "续期失败必须能从读数里读出来");
+ assertEquals(T0 + 900_000L - T0, lock.heldMillis(T0 + 900_000L),
+ "锁丢失不等于时间停住——它还在被本节点按'仍持有'用着");
+ assertTrue(lock.ttlExpired(T0 + 900_000L));
+ }
+
+ @Test
+ void ttlNeedsStrictlyMoreThanTheTtlToCountAsExpired() {
+ ActiveTaskLockRegistry registry = registered();
+ assertFalse(registry.snapshot().get("similar-asin:30853").ttlExpired(T0 + TTL),
+ "刚好等于 TTL 还在续期窗口内");
+ assertTrue(registry.snapshot().get("similar-asin:30853").ttlExpired(T0 + TTL + 1L));
+ }
+
+ @Test
+ void reRegistrationResetsTheClock() {
+ ActiveTaskLockRegistry registry = registered();
+ // 上一任没走到 unregister 就崩了:新持有者刚拿到锁,读数不能显示"已经压了很久"
+ registry.register("similar-asin:30853", T0 + 86_400_000L, TTL);
+ ActiveTaskLockRegistry.ActiveLock lock = registry.snapshot().get("similar-asin:30853");
+ assertEquals(0L, lock.heldMillis(T0 + 86_400_000L));
+ assertFalse(registry.shouldReport("similar-asin:30853", T0 + 86_400_000L + 30_000L));
+ }
+
+ @Test
+ void thirtyMinuteHoldProducesSixLinesNotThirty() {
+ ActiveTaskLockRegistry registry = registered();
+ int reports = 0;
+ for (long elapsed = 0L; elapsed <= 1_800_000L; elapsed += 60_000L) {
+ if (registry.shouldReport("similar-asin:30853", T0 + elapsed)) {
+ reports++;
+ }
+ }
+ assertEquals(6, reports, "60s/360s/660s/960s/1260s/1560s 共 6 行");
+ }
+
+ @Test
+ void snapshotCannotBeMutatedByCallers() {
+ ActiveTaskLockRegistry registry = registered();
+ Map snap = registry.snapshot();
+ assertThrows(UnsupportedOperationException.class,
+ () -> snap.remove("similar-asin:30853"), "读数不能反过来改登记表");
+ assertEquals(1, registry.size());
+ }
+}
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceActiveReadingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceActiveReadingTest.java
new file mode 100644
index 000000000..16743bb7e
--- /dev/null
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceActiveReadingTest.java
@@ -0,0 +1,102 @@
+package com.nanri.aiimage.modules.task.service;
+
+import com.nanri.aiimage.common.service.DistributedJobLockService;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * 「在持读数」的接线判据(10-14)。
+ *
+ * 阶梯节奏本身在 {@link ActiveTaskLockRegistryTest} 里钉;这里只证一件事:
+ * 登记表真的跟着**取锁/放锁**在动。少一条,看门狗读到的就是上一任留下的旧时间戳,
+ * 那正是"锁挂了 24 小时"最容易骗人的形状。
+ */
+class TaskDistributedLockServiceActiveReadingTest {
+
+ private static final Duration TTL = Duration.ofMinutes(5);
+ private static final String LOCK_NAME = "task:similar-asin:30853";
+
+ private TaskDistributedLockService serviceWithLockAvailable() {
+ DistributedJobLockService delegate = mock(DistributedJobLockService.class);
+ when(delegate.tryLock(anyString(), any(Duration.class)))
+ .thenReturn(mock(DistributedJobLockService.LockHandle.class));
+ return new TaskDistributedLockService(delegate);
+ }
+
+ private TaskDistributedLockService serviceWithLockBusy() {
+ DistributedJobLockService delegate = mock(DistributedJobLockService.class);
+ when(delegate.tryLock(anyString(), any(Duration.class))).thenReturn(null);
+ return new TaskDistributedLockService(delegate);
+ }
+
+ @Test
+ void heldLockIsReadableBeforeItIsReleased() {
+ TaskDistributedLockService service = serviceWithLockAvailable();
+ TaskDistributedLockService.LockHandle handle = service.acquire("SIMILAR_ASIN", 30853L);
+ assertNotNull(handle, "锁可用时应当取到");
+ Map readings = service.activeHeldMillis();
+ assertEquals(Set.of(LOCK_NAME), readings.keySet(),
+ "模块名要归一成 task:similar-asin:30853,读数才对得上 grep 的锚");
+ assertTrue(readings.get(LOCK_NAME) >= 0L, "还没放的锁必须读得到时长");
+ }
+
+ @Test
+ void releaseTakesTheLockOutOfTheReading() {
+ TaskDistributedLockService service = serviceWithLockAvailable();
+ service.acquire("similar-asin", 30853L).close();
+ assertTrue(service.activeHeldMillis().isEmpty(), "放手的锁不能继续算在持");
+ }
+
+ @Test
+ void nestedAcquireRegistersOnceAndNeedsEveryClose() {
+ TaskDistributedLockService service = serviceWithLockAvailable();
+ TaskDistributedLockService.LockHandle outer = service.acquire("similar-asin", 30853L);
+ TaskDistributedLockService.LockHandle inner = service.acquire("similar-asin", 30853L);
+ assertEquals(1, service.activeHeldMillis().size(), "同一把锁重入只登记一次");
+ inner.close();
+ assertEquals(1, service.activeHeldMillis().size(), "只放了一层,锁还在本线程手里");
+ outer.close();
+ assertTrue(service.activeHeldMillis().isEmpty());
+ }
+
+ @Test
+ void contestedAcquireRegistersNothing() {
+ TaskDistributedLockService service = serviceWithLockBusy();
+ assertNull(service.acquire("similar-asin", 30853L, TTL, 0L), "等不到锁就是没取到");
+ assertTrue(service.activeHeldMillis().isEmpty(), "没取到的锁不许出现在在持读数里");
+ }
+
+ @Test
+ void badTaskIdentityRegistersNothing() {
+ TaskDistributedLockService service = serviceWithLockAvailable();
+ assertNull(service.acquire("similar-asin", null));
+ assertNull(service.acquire(" ", 30853L));
+ assertTrue(service.activeHeldMillis().isEmpty());
+ }
+
+ @Test
+ void reacquiringStartsTheClockOver() {
+ TaskDistributedLockService service = serviceWithLockAvailable();
+ service.acquire("similar-asin", 30853L).close();
+ long heldAfterReacquire;
+ TaskDistributedLockService.LockHandle again = service.acquire("similar-asin", 30853L);
+ heldAfterReacquire = service.activeHeldMillis().get(LOCK_NAME);
+ again.close();
+ assertTrue(heldAfterReacquire < 5_000L,
+ "上一任的时间戳不能留给新持有者(否则一条刚取到锁的读数会显示成已压了很久)");
+ assertFalse(service.activeHeldMillis().containsKey(LOCK_NAME));
+ }
+}