diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java new file mode 100644 index 000000000..992a62d0a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistry.java @@ -0,0 +1,113 @@ +package com.nanri.aiimage.modules.task.service; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * 「正在持有」的任务锁登记表。 + * + *

动机:{@code [task-lock-held]} 那条留痕只在 {@code LockHandle.close()} 打, + * 于是"锁此刻还压着多久"这一维在生产上读不到——40902 连着出现时,分不清"马上要放了" + * 和"已经挂了 24 小时"(货源那单就是后者)。这里补的正是那半句读数。

+ * + *

另外单独标出"续期已连续失败":那种锁最迟在 TTL 后就该被另一实例拿走, + * 本节点却还在按"仍持锁"的假设跑——挂着多久不等于还被谁保护,这个区别只能靠这条读数看出来。

+ * + *

留痕按阶梯走(首个 60 秒、之后每 5 分钟一次),一次长持锁不会把日志灌满, + * 与 10-06 通知放大那次处置同一口径。

+ */ +final class ActiveTaskLockRegistry { + + /** 60 秒以内的持锁不留痕:任务锁的正常生命周期本来就以秒到分钟计。 */ + static final long FIRST_REPORT_MILLIS = 60_000L; + /** 首次之后每 5 分钟复报一次(一次 30 分钟的持锁总共 6 行,不是 30 行)。 */ + static final long REPEAT_REPORT_MILLIS = 300_000L; + + static final class ActiveLock { + + private final long acquiredAtMillis; + private final long ttlMillis; + private volatile boolean renewalLost; + private volatile long lastReportAtMillis; + + private ActiveLock(long acquiredAtMillis, long ttlMillis) { + this.acquiredAtMillis = acquiredAtMillis; + this.ttlMillis = ttlMillis; + } + + long heldMillis(long nowMillis) { + return Math.max(0L, nowMillis - acquiredAtMillis); + } + + long ttlMillis() { + return ttlMillis; + } + + /** 持有时长已超过 TTL:只有在续期也失败时才有意义(续期成功会让锁一直有效)。 */ + boolean ttlExpired(long nowMillis) { + return heldMillis(nowMillis) > ttlMillis; + } + + boolean isRenewalLost() { + return renewalLost; + } + } + + private final ConcurrentMap active = new ConcurrentHashMap<>(); + + /** + * 登记一次持锁。同名重复登记以最后一次为准——上一任若是没走到 {@code unregister} + * 就崩了,留下的旧时间戳会把"新持有者刚拿到锁"读成"已经压了很久"。 + */ + void register(String lockName, long acquiredAtMillis, long ttlMillis) { + active.put(lockName, new ActiveLock(acquiredAtMillis, ttlMillis)); + } + + void unregister(String lockName) { + active.remove(lockName); + } + + void markRenewalLost(String lockName) { + ActiveLock lock = active.get(lockName); + if (lock != null) { + lock.renewalLost = true; + } + } + + int size() { + return active.size(); + } + + /** 当前在持的锁快照(底层是 ConcurrentHashMap,不保证顺序);要不要留痕交给 shouldReport 逐条判。 */ + Map snapshot() { + Map copy = new LinkedHashMap<>(); + for (Map.Entry entry : active.entrySet()) { + copy.put(entry.getKey(), entry.getValue()); + } + return Collections.unmodifiableMap(copy); + } + + /** + * 这一轮要不要留痕。返回 true 的同时把复报基准推到 {@code nowMillis}, + * 所以调用方只在真要写日志时才调它(判一次不写一次会让下一条等到 5 分钟后)。 + */ + boolean shouldReport(String lockName, long nowMillis) { + ActiveLock lock = active.get(lockName); + if (lock == null) { + return false; + } + long held = lock.heldMillis(nowMillis); + if (held < FIRST_REPORT_MILLIS) { + return false; + } + long last = lock.lastReportAtMillis; + if (last != 0L && nowMillis - last < REPEAT_REPORT_MILLIS) { + return false; + } + lock.lastReportAtMillis = nowMillis; + return true; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java index 51dfe734e..bfe498b88 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockService.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.task.service; import com.nanri.aiimage.common.service.DistributedJobLockService; +import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -43,6 +44,16 @@ public class TaskDistributedLockService { return thread; }); + /** [task-lock-held] 那条只在放手时打;这一张补上「现在还压着多久」的读数。 */ + private final ActiveTaskLockRegistry activeLocks = new ActiveTaskLockRegistry(); + /** 看门狗 60 秒一轮,要不要写由 registry 的阶梯判(一次 30 分钟持锁总共 6 行,不是 30 行)。 */ + private static final long ACTIVE_REPORT_INTERVAL_MILLIS = 60_000L; + private final ScheduledExecutorService activeReportExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread(runnable, "task-lock-watch"); + thread.setDaemon(true); + return thread; + }); + public LockHandle acquire(String moduleType, Long taskId) { return acquire(moduleType, taskId, DEFAULT_WAIT_MILLIS); } @@ -70,6 +81,7 @@ public class TaskDistributedLockService { HeldLock newHeldLock = new HeldLock(delegate, ttl); scheduleRenewal(lockName, newHeldLock, ttl); heldLocks.put(lockName, newHeldLock); + activeLocks.register(lockName, newHeldLock.acquiredAtMillis, ttl.toMillis()); return new LockHandle(lockName); } if (waitMillis <= 0L || System.nanoTime() >= deadline) { @@ -92,6 +104,16 @@ public class TaskDistributedLockService { return "task:" + normalizedModule + ":" + taskId; } + /** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */ + Map activeHeldMillis() { + long now = System.currentTimeMillis(); + Map readings = new LinkedHashMap<>(); + for (Map.Entry entry : activeLocks.snapshot().entrySet()) { + readings.put(entry.getKey(), entry.getValue().heldMillis(now)); + } + return readings; + } + private void scheduleRenewal(String lockName, HeldLock heldLock, Duration ttl) { long ttlMillis = Math.max(1000L, ttl.toMillis()); long renewalDelayMillis = Math.max(1000L, ttlMillis / 3L); @@ -103,6 +125,7 @@ public class TaskDistributedLockService { return; } heldLock.closed = true; + activeLocks.markRenewalLost(lockName); ScheduledFuture future = heldLock.renewalFuture; if (future != null) { future.cancel(false); @@ -141,9 +164,29 @@ public class TaskDistributedLockService { return false; } + @PostConstruct + void startActiveLockWatch() { + activeReportExecutor.scheduleWithFixedDelay(this::reportActiveLocks, + ACTIVE_REPORT_INTERVAL_MILLIS, ACTIVE_REPORT_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); + } + + /** 在持留痕:读得到「这把锁还没放、已经压了多久」;续期失败的那类单独标 renewalLost=true。 */ + void reportActiveLocks() { + long now = System.currentTimeMillis(); + for (Map.Entry entry : activeLocks.snapshot().entrySet()) { + if (!activeLocks.shouldReport(entry.getKey(), now)) { + continue; + } + ActiveTaskLockRegistry.ActiveLock lock = entry.getValue(); + log.warn("[task-lock-active] lockName={} heldMs={} ttlMs={} renewalLost={}", + entry.getKey(), lock.heldMillis(now), lock.ttlMillis(), lock.isRenewalLost()); + } + } + @PreDestroy - public void shutdownRenewalExecutor() { + public void shutdownExecutors() { renewalExecutor.shutdownNow(); + activeReportExecutor.shutdownNow(); } private static final class HeldLock { @@ -193,6 +236,7 @@ public class TaskDistributedLockService { heldLock.depth--; if (heldLock.depth <= 0) { heldLocks.remove(lockName); + activeLocks.unregister(lockName); if (heldLocks.isEmpty()) { localLocks.remove(); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistryTest.java new file mode 100644 index 000000000..c052c8cad --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ActiveTaskLockRegistryTest.java @@ -0,0 +1,114 @@ +package com.nanri.aiimage.modules.task.service; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * 「正在持有」读数的判据(10-14 补的埋点)。 + * + * 要钉住的是原来读不到的那一维:锁**还没放**的时候它已经压了多久,以及那把锁的续期是否已经失败 + * (后者意味着锁最迟在 TTL 后就该被另一实例拿走,本节点却还在按"仍持锁"跑)。 + * 顺带钉住留痕节奏:阶梯不成立就会把一次长持锁灌成每分钟一行,与 10-06 通知放大同族。 + */ +class ActiveTaskLockRegistryTest { + + private static final long T0 = 1_700_000_000_000L; + private static final long TTL = Duration.ofMinutes(5).toMillis(); + + private static ActiveTaskLockRegistry registered() { + ActiveTaskLockRegistry registry = new ActiveTaskLockRegistry(); + registry.register("similar-asin:30853", T0, TTL); + return registry; + } + + @Test + void ladderIsPinnedAtOneMinuteThenFiveMinutes() { + assertEquals(60_000L, ActiveTaskLockRegistry.FIRST_REPORT_MILLIS); + assertEquals(300_000L, ActiveTaskLockRegistry.REPEAT_REPORT_MILLIS); + } + + @Test + void holdUnderOneMinuteStaysSilent() { + ActiveTaskLockRegistry registry = registered(); + assertFalse(registry.shouldReport("similar-asin:30853", T0 + 59_999L), + "60 秒以内的持锁是常态,读数不该在这一档说话"); + assertFalse(registry.shouldReport("别的锁", T0 + 999_999L), "没登记过的名字恒不报"); + } + + @Test + void firstReportHappensOnceAtTheMinuteMarkThenWaits() { + ActiveTaskLockRegistry registry = registered(); + assertTrue(registry.shouldReport("similar-asin:30853", T0 + 60_000L)); + assertFalse(registry.shouldReport("similar-asin:30853", T0 + 60_001L), "同一轮不许复报"); + assertFalse(registry.shouldReport("similar-asin:30853", T0 + 359_999L)); + assertTrue(registry.shouldReport("similar-asin:30853", T0 + 360_000L), "满 5 分钟再报一次"); + } + + @Test + void releasedLockLeavesTheReading() { + ActiveTaskLockRegistry registry = registered(); + registry.unregister("similar-asin:30853"); + assertEquals(0, registry.size()); + assertFalse(registry.shouldReport("similar-asin:30853", T0 + 900_000L), + "已经放手的锁不能还在'在持'里计数"); + assertTrue(registry.snapshot().isEmpty()); + } + + @Test + void renewalLostIsReadableWhileTheHoldKeepsGrowing() { + ActiveTaskLockRegistry registry = registered(); + registry.markRenewalLost("similar-asin:30853"); + Map snap = registry.snapshot(); + ActiveTaskLockRegistry.ActiveLock lock = snap.get("similar-asin:30853"); + assertTrue(lock.isRenewalLost(), "续期失败必须能从读数里读出来"); + assertEquals(T0 + 900_000L - T0, lock.heldMillis(T0 + 900_000L), + "锁丢失不等于时间停住——它还在被本节点按'仍持有'用着"); + assertTrue(lock.ttlExpired(T0 + 900_000L)); + } + + @Test + void ttlNeedsStrictlyMoreThanTheTtlToCountAsExpired() { + ActiveTaskLockRegistry registry = registered(); + assertFalse(registry.snapshot().get("similar-asin:30853").ttlExpired(T0 + TTL), + "刚好等于 TTL 还在续期窗口内"); + assertTrue(registry.snapshot().get("similar-asin:30853").ttlExpired(T0 + TTL + 1L)); + } + + @Test + void reRegistrationResetsTheClock() { + ActiveTaskLockRegistry registry = registered(); + // 上一任没走到 unregister 就崩了:新持有者刚拿到锁,读数不能显示"已经压了很久" + registry.register("similar-asin:30853", T0 + 86_400_000L, TTL); + ActiveTaskLockRegistry.ActiveLock lock = registry.snapshot().get("similar-asin:30853"); + assertEquals(0L, lock.heldMillis(T0 + 86_400_000L)); + assertFalse(registry.shouldReport("similar-asin:30853", T0 + 86_400_000L + 30_000L)); + } + + @Test + void thirtyMinuteHoldProducesSixLinesNotThirty() { + ActiveTaskLockRegistry registry = registered(); + int reports = 0; + for (long elapsed = 0L; elapsed <= 1_800_000L; elapsed += 60_000L) { + if (registry.shouldReport("similar-asin:30853", T0 + elapsed)) { + reports++; + } + } + assertEquals(6, reports, "60s/360s/660s/960s/1260s/1560s 共 6 行"); + } + + @Test + void snapshotCannotBeMutatedByCallers() { + ActiveTaskLockRegistry registry = registered(); + Map snap = registry.snapshot(); + assertThrows(UnsupportedOperationException.class, + () -> snap.remove("similar-asin:30853"), "读数不能反过来改登记表"); + assertEquals(1, registry.size()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceActiveReadingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceActiveReadingTest.java new file mode 100644 index 000000000..16743bb7e --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskDistributedLockServiceActiveReadingTest.java @@ -0,0 +1,102 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.common.service.DistributedJobLockService; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * 「在持读数」的接线判据(10-14)。 + * + * 阶梯节奏本身在 {@link ActiveTaskLockRegistryTest} 里钉;这里只证一件事: + * 登记表真的跟着**取锁/放锁**在动。少一条,看门狗读到的就是上一任留下的旧时间戳, + * 那正是"锁挂了 24 小时"最容易骗人的形状。 + */ +class TaskDistributedLockServiceActiveReadingTest { + + private static final Duration TTL = Duration.ofMinutes(5); + private static final String LOCK_NAME = "task:similar-asin:30853"; + + private TaskDistributedLockService serviceWithLockAvailable() { + DistributedJobLockService delegate = mock(DistributedJobLockService.class); + when(delegate.tryLock(anyString(), any(Duration.class))) + .thenReturn(mock(DistributedJobLockService.LockHandle.class)); + return new TaskDistributedLockService(delegate); + } + + private TaskDistributedLockService serviceWithLockBusy() { + DistributedJobLockService delegate = mock(DistributedJobLockService.class); + when(delegate.tryLock(anyString(), any(Duration.class))).thenReturn(null); + return new TaskDistributedLockService(delegate); + } + + @Test + void heldLockIsReadableBeforeItIsReleased() { + TaskDistributedLockService service = serviceWithLockAvailable(); + TaskDistributedLockService.LockHandle handle = service.acquire("SIMILAR_ASIN", 30853L); + assertNotNull(handle, "锁可用时应当取到"); + Map readings = service.activeHeldMillis(); + assertEquals(Set.of(LOCK_NAME), readings.keySet(), + "模块名要归一成 task:similar-asin:30853,读数才对得上 grep 的锚"); + assertTrue(readings.get(LOCK_NAME) >= 0L, "还没放的锁必须读得到时长"); + } + + @Test + void releaseTakesTheLockOutOfTheReading() { + TaskDistributedLockService service = serviceWithLockAvailable(); + service.acquire("similar-asin", 30853L).close(); + assertTrue(service.activeHeldMillis().isEmpty(), "放手的锁不能继续算在持"); + } + + @Test + void nestedAcquireRegistersOnceAndNeedsEveryClose() { + TaskDistributedLockService service = serviceWithLockAvailable(); + TaskDistributedLockService.LockHandle outer = service.acquire("similar-asin", 30853L); + TaskDistributedLockService.LockHandle inner = service.acquire("similar-asin", 30853L); + assertEquals(1, service.activeHeldMillis().size(), "同一把锁重入只登记一次"); + inner.close(); + assertEquals(1, service.activeHeldMillis().size(), "只放了一层,锁还在本线程手里"); + outer.close(); + assertTrue(service.activeHeldMillis().isEmpty()); + } + + @Test + void contestedAcquireRegistersNothing() { + TaskDistributedLockService service = serviceWithLockBusy(); + assertNull(service.acquire("similar-asin", 30853L, TTL, 0L), "等不到锁就是没取到"); + assertTrue(service.activeHeldMillis().isEmpty(), "没取到的锁不许出现在在持读数里"); + } + + @Test + void badTaskIdentityRegistersNothing() { + TaskDistributedLockService service = serviceWithLockAvailable(); + assertNull(service.acquire("similar-asin", null)); + assertNull(service.acquire(" ", 30853L)); + assertTrue(service.activeHeldMillis().isEmpty()); + } + + @Test + void reacquiringStartsTheClockOver() { + TaskDistributedLockService service = serviceWithLockAvailable(); + service.acquire("similar-asin", 30853L).close(); + long heldAfterReacquire; + TaskDistributedLockService.LockHandle again = service.acquire("similar-asin", 30853L); + heldAfterReacquire = service.activeHeldMillis().get(LOCK_NAME); + again.close(); + assertTrue(heldAfterReacquire < 5_000L, + "上一任的时间戳不能留给新持有者(否则一条刚取到锁的读数会显示成已压了很久)"); + assertFalse(service.activeHeldMillis().containsKey(LOCK_NAME)); + } +}