fix(长任务判死): 封口前增加「客户端仍在推进」豁免,避免还在采集就被结案丢数据

服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢
就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行,
丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进:
ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端
无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand /
collect-data 五条判死线。
This commit is contained in:
2026-09-23 10:52:45 +08:00
parent 8d73377b25
commit afa45d51c3
16 changed files with 348 additions and 6 deletions
@@ -306,7 +306,9 @@ class RollbackSemanticsContractTest {
storageProperties, fileTaskMapper, fileResultMapper,
taskScopeStateMapper, taskChunkMapper, objectMapper, mock(AppearancePatentLlmClient.class),
mock(AppearancePatentTaskCacheService.class), mock(com.nanri.aiimage.config.AppearancePatentProperties.class),
taskFileJobService, taskProgressSnapshotService, transientPayloadStorageService, transactionManager,
taskFileJobService, taskProgressSnapshotService,
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
transientPayloadStorageService, transactionManager,
distributedJobLockService, taskDistributedLockService, instanceMetadata,
mock(TaskProgressLightAssembler.class),
mock(UserApiSecretService.class));
@@ -13,7 +13,9 @@ import java.time.Duration;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
@@ -111,4 +113,81 @@ class TaskHeartbeatPositionServiceTest {
assertNull(service().describe(123L));
}
// ===== 停滞判定(封口前的豁免依据)=====
@Test
void stallUnknownWhenNoPosition() {
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123")).thenReturn(Map.of());
assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L));
}
@Test
void stallUnknownOnRedisError() {
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenThrow(new RuntimeException("redis down"));
assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L));
}
@Test
void stallAdvancingWhenPositionChangedAndSnapshotRefreshed() {
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
when(hashOperations.entries("task:progress:stall:123"))
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(400/1300)", "at", String.valueOf(Long.MAX_VALUE)));
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L));
verify(hashOperations).putAll(eq("task:progress:stall:123"), any());
}
@Test
void stallAdvancingWhenPositionUnchangedWithinWindow() {
// 位置没变但还没到停滞窗口:仍按「在推进」处理,不得封口
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
when(hashOperations.entries("task:progress:stall:123"))
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(5).toMillis())));
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L));
}
@Test
void stallStalledWhenPositionUnchangedBeyondWindow() {
// 位置长时间不动才是真卡死,允许调用方按原判据封口
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
when(hashOperations.entries("task:progress:stall:123"))
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis())));
assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L));
}
@Test
void clientAdvancingOnlyForAdvancingState() {
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
// 停滞(位置长时间未变)不构成豁免
when(hashOperations.entries("task:heartbeat:position:123"))
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
when(hashOperations.entries("task:progress:stall:123"))
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis())));
assertFalse(service().isClientAdvancing(123L, 180L));
// 无位置信息(老客户端)同样不构成豁免,调用方保留原判据
assertFalse(service().isClientAdvancing(null, 180L));
// 位置在变才是豁免
when(hashOperations.entries("task:progress:stall:123"))
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(1/1300)", "at", String.valueOf(Long.MAX_VALUE)));
assertTrue(service().isClientAdvancing(123L, 180L));
}
}