fix(task-lock): 持锁封顶改按模块分档,两根平线都实测砍到过合法大单
30 分钟那版在主机A 留下 12 条 heldMs≈1,800,011 的精确撞线,被掐的 31725/31888 照常推进、最终在 43.7/49.3/49.8 分钟正常放过;改成一根 2 小时平线又会去砍 appearance-patent——两台 7 天日志的「放过」分布:similar-asin 7845 次最长 105.9 分, appearance-patent 660 次最长 453.8 分(对应任务墙钟 9.4 小时、SUCCESS), collect-data 1079 次最长 9.2 分。 现按模块分档(默认 2h/similar-asin 4h/appearance-patent 12h),6×TTL 项不变; 分档表整表只有一个出处,用例既钉静态线也钉「续期按这把锁的模块查线」的形状。
This commit is contained in:
+76
-2
@@ -4,6 +4,7 @@ import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -82,14 +83,87 @@ class TaskDistributedLockServiceLeakTest {
|
||||
assertTrue(service.activeHeldMillis().isEmpty(), "最后一层在别的线程上放,也必须算放过");
|
||||
}
|
||||
|
||||
@Test
|
||||
void eachModuleCeilingSitsAboveItsOwnLegitLongestHold() {
|
||||
// 校准样本=两台 7 天容器日志里 [task-lock-held](=真跑完并释放)的分布,10-22 取。
|
||||
// 只认「放过」不认「在持」:在持那一路按定义包含还没放的锁,拿它定线等于把封顶定成无限。
|
||||
// similar-asin 主机A 7845 次放过,最长 105.9 分(新代码上线后 2 小时窗口内 46 次,最长 49.8 分)
|
||||
// appearance-patent 主机B 660 次放过,最长 453.8 分;对应任务墙钟 9.4 小时、状态 SUCCESS
|
||||
// collect-data 两台 1079 次放过,最长 9.2 分
|
||||
// 这条判据防的是以后有人拿「锁压 30 分钟太久了」当理由,把某一档调回该模块合法区间以内。
|
||||
Map<String, Long> floors = TaskDistributedLockService.shippedMaxHoldFloorsMillis();
|
||||
long similarAsinLegitMax = Duration.ofMinutes(105).toMillis() + 54_000L;
|
||||
long patentLegitMax = Duration.ofMinutes(453).toMillis() + 48_000L;
|
||||
long defaultFloor = floors.get(TaskDistributedLockService.DEFAULT_FLOOR_KEY);
|
||||
assertTrue(defaultFloor >= 2 * 60 * 60_000L,
|
||||
"默认档不得低于 2 小时:similar-asin 合法放过实测到 105.9 分,"
|
||||
+ "把所有一档压回 30 分钟那种线,现网已实测会把慢但活着的大单当卡死治");
|
||||
assertTrue(floors.get("similar-asin") >= similarAsinLegitMax * 3 / 2,
|
||||
"similar-asin 必须落在自身合法最长(105.9 分)之上且留有余量");
|
||||
assertTrue(floors.get("appearance-patent") >= patentLegitMax * 3 / 2,
|
||||
"appearance-patent 的大单本来就要跑 7.6 小时,用一根平线去套会把它当僵尸掐");
|
||||
// 反向也钉:分档不是「干脆不设」。封顶存在的唯一意义是终结 11.3 小时那种永久僵尸锁。
|
||||
floors.forEach((module, millis) -> assertTrue(millis <= 12 * 60 * 60_000L,
|
||||
module + " 的封顶线被抬成「几乎没有」了:" + millis));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ceilingIsLookedUpPerModuleNotGlobally() throws Exception {
|
||||
// 只钉静态表还不够:得证明续期真的按**这把锁所属模块**去查线。
|
||||
// 两档一起放到毫秒级,越界的那把必须停、还在界内的那把必须继续——
|
||||
// 若封顶退回读一根全局线,两个读数会一起停,这条用例就抓不住。
|
||||
AtomicInteger similarAsinRenews = new AtomicInteger();
|
||||
AtomicInteger patentRenews = new AtomicInteger();
|
||||
TaskDistributedLockService service = twoModuleService(similarAsinRenews, patentRenews);
|
||||
service.setMaxHoldFloorsForTest(Map.of(
|
||||
TaskDistributedLockService.DEFAULT_FLOOR_KEY, 1L, // 没单独建档的模块走这条
|
||||
"similar-asin", 1L, // ttl 300ms → 封顶 = 6×300 = 1.8s
|
||||
"appearance-patent", 10 * 60 * 60_000L)); // 10 小时:本用例时长内绝不会越界
|
||||
TaskDistributedLockService.LockHandle similarAsin =
|
||||
service.acquire("similar-asin", TASK_ID, Duration.ofMillis(300), 0L);
|
||||
TaskDistributedLockService.LockHandle patent =
|
||||
service.acquire("APPEARANCE_PATENT", 31891L, Duration.ofMillis(300), 0L);
|
||||
|
||||
Thread.sleep(5_000L);
|
||||
assertTrue(similarAsinRenews.get() >= 1, "similar-asin 没续过期=什么都没量到");
|
||||
assertTrue(patentRenews.get() >= 1, "appearance-patent 没续过期=什么都没量到");
|
||||
int similarAsinAtStop = similarAsinRenews.get();
|
||||
int patentAtStop = patentRenews.get();
|
||||
Thread.sleep(3_000L);
|
||||
assertEquals(similarAsinAtStop, similarAsinRenews.get(),
|
||||
"similar-asin 过 1.8 秒还在续期=分档线没生效");
|
||||
assertTrue(patentRenews.get() > patentAtStop,
|
||||
"appearance-patent 停下来了=封顶退成了全局一根线,把合法长档也一起掐了");
|
||||
|
||||
similarAsin.close();
|
||||
patent.close();
|
||||
}
|
||||
|
||||
/** 两个模块各自计数 renew,用于判「哪一把停了、哪一把还在续」。 */
|
||||
private TaskDistributedLockService twoModuleService(AtomicInteger similarAsinRenews,
|
||||
AtomicInteger patentRenews) {
|
||||
DistributedJobLockService delegate = mock(DistributedJobLockService.class);
|
||||
when(delegate.tryLock(anyString(), any(Duration.class))).thenAnswer(invocation -> {
|
||||
boolean similarAsin = ((String) invocation.getArgument(0)).startsWith("task:similar-asin:");
|
||||
AtomicInteger counter = similarAsin ? similarAsinRenews : patentRenews;
|
||||
DistributedJobLockService.LockHandle inner = mock(DistributedJobLockService.LockHandle.class);
|
||||
when(inner.renew(any(Duration.class))).thenAnswer(renew -> {
|
||||
counter.incrementAndGet();
|
||||
return true;
|
||||
});
|
||||
return inner;
|
||||
});
|
||||
return new TaskDistributedLockService(delegate);
|
||||
}
|
||||
|
||||
@Test
|
||||
void renewalStopsOnceHoldPassesTheCeiling() throws Exception {
|
||||
AtomicInteger renewCalls = new AtomicInteger();
|
||||
DistributedJobLockService.LockHandle[] inner = new DistributedJobLockService.LockHandle[1];
|
||||
TaskDistributedLockService service = serviceWithRenewCounting(renewCalls, inner);
|
||||
// ttl 300ms → 续期节拍钳到下限 1s,封顶 = max(200ms, 6×300ms) = 1.8s:
|
||||
// ttl 300ms → 续期节拍钳到下限 1s,封顶 = max(1ms, 6×300ms) = 1.8s:
|
||||
// 第一次续期(≈1s)还在界内,第二次(≈2s)越界即停。
|
||||
service.setMaxHoldFloorMillisForTest(200L);
|
||||
service.setMaxHoldFloorsForTest(Map.of("similar-asin", 1L));
|
||||
TaskDistributedLockService.LockHandle handle =
|
||||
service.acquire("similar-asin", TASK_ID, Duration.ofMillis(300), 0L);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user