fix(长任务判死): 封口前增加「客户端仍在推进」豁免,避免还在采集就被结案丢数据
服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢 就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行, 丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进: ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端 无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand / collect-data 五条判死线。
This commit is contained in:
+2
@@ -104,6 +104,7 @@ class AppearancePatentTaskServiceDelegationTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||
properties, taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, transactionManager, distributedJobLockService,
|
||||
taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
@@ -117,6 +118,7 @@ class AppearancePatentTaskServiceDelegationTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||
properties, taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, null, distributedJobLockService,
|
||||
taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
|
||||
+1
@@ -213,6 +213,7 @@ class AppearancePatentTaskServiceHistoryBatchTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||
properties, taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, transactionManager, distributedJobLockService,
|
||||
taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
|
||||
+3
-1
@@ -79,7 +79,9 @@ class BrandTaskAbortServiceTest {
|
||||
|
||||
private BrandTaskService service() {
|
||||
return new BrandTaskService(brandCrawlTaskMapper, ossStorageService, storageProperties,
|
||||
brandProgressProperties, brandTaskProgressCacheService, brandTaskStorageService,
|
||||
brandProgressProperties, brandTaskProgressCacheService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
brandTaskStorageService,
|
||||
distributedJobLockService, taskDistributedLockService, new ObjectMapper(),
|
||||
taskFileJobService, taskProgressSnapshotService);
|
||||
}
|
||||
|
||||
+1
@@ -136,6 +136,7 @@ class CollectDataActivateGuardTest {
|
||||
mock(InvalidAsinDataMapper.class),
|
||||
mock(TaskChunkMapper.class), mock(TaskScopeStateMapper.class), mock(TaskResultItemMapper.class),
|
||||
mock(TaskDistributedLockService.class), mock(TaskFileJobService.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
mock(TransientPayloadStorageService.class),
|
||||
mock(CollectDataExcelAssemblyService.class), mock(OssStorageService.class),
|
||||
new ObjectMapper(), new TransactionTemplate(transactionManager),
|
||||
|
||||
+3
-1
@@ -104,7 +104,9 @@ class CollectDataDeleteConsistencyTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper.class),
|
||||
mock(com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper.class),
|
||||
taskChunkMapper, taskScopeStateMapper, taskResultItemMapper,
|
||||
taskDistributedLockService, taskFileJobService, transientPayloadStorageService,
|
||||
taskDistributedLockService, taskFileJobService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService,
|
||||
mock(com.nanri.aiimage.modules.collectdata.service.CollectDataExcelAssemblyService.class),
|
||||
mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class),
|
||||
objectMapper, txTemplate,
|
||||
|
||||
+3
@@ -8,6 +8,7 @@ import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -45,6 +46,8 @@ class CollectDataNoUploadStaleTest {
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
/** 默认(Mockito 布尔默认值 false)= 不构成「客户端仍在推进」豁免,与原判据行为一致。 */
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TaskDistributedLockService.LockHandle lockHandle;
|
||||
|
||||
@Spy
|
||||
|
||||
+3
-1
@@ -103,7 +103,9 @@ class CollectDataStorageCallCountTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper.class),
|
||||
mock(com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper.class),
|
||||
taskChunkMapper, taskScopeStateMapper, taskResultItemMapper,
|
||||
taskDistributedLockService, taskFileJobService, transientPayloadStorageService,
|
||||
taskDistributedLockService, taskFileJobService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService,
|
||||
mock(com.nanri.aiimage.modules.collectdata.service.CollectDataExcelAssemblyService.class),
|
||||
mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class),
|
||||
objectMapper, txTemplate,
|
||||
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 货源二次判死线的「客户端仍在推进」豁免。
|
||||
*
|
||||
* <p>事故背景(2026-09-22 任务 28919):客户端每 1.5~3 小时才回传一次分片(1300 行 / 12 小时),
|
||||
* 服务端按「180 分钟无结果分片上报」把仍在采集的任务封口结案,后面到的 22 个分片全被拒收,
|
||||
* 905 行数据丢失。修复后:位置文本仍在变(客户端在推进)就不封口;位置长时间不变或无位置信息
|
||||
* 才走原判据。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinNoUploadStaleTest {
|
||||
|
||||
private static final Long TASK_ID = 28919L;
|
||||
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@InjectMocks
|
||||
private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
when(properties.getNoResultUploadTimeoutMinutes()).thenReturn(180);
|
||||
when(properties.getStaleTimeoutMinutes()).thenReturn(30);
|
||||
}
|
||||
|
||||
/** 心跳新鲜(30 分钟内有心跳)+ 分片已 200 分钟没上传 = 命中二次判死线的候选条件。 */
|
||||
private void givenStaleChunkUpload() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setStatus("RUNNING");
|
||||
task.setCreatedAt(LocalDateTime.now().minusDays(1));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(1));
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskCacheService.getTaskHeartbeatMillis(TASK_ID)).thenReturn(System.currentTimeMillis());
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any()))
|
||||
.thenReturn(List.of(new TaskScopeLastChunkDto(TASK_ID, LocalDateTime.now().minusMinutes(200))));
|
||||
}
|
||||
|
||||
private void invokeNoUploadCheck() {
|
||||
ReflectionTestUtils.invokeMethod(service, "finalizeNoUploadStaleTasks");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 客户端仍在推进时不封口() {
|
||||
givenStaleChunkUpload();
|
||||
when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L))
|
||||
.thenReturn(TaskHeartbeatPositionService.StallState.ADVANCING);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
// 豁免生效:连任务锁都不去抢,更不会封口结案(也就不会拒收后续分片)
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 客户端停滞时保留原封口行为() {
|
||||
givenStaleChunkUpload();
|
||||
when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L))
|
||||
.thenReturn(TaskHeartbeatPositionService.StallState.STALLED);
|
||||
when(taskDistributedLockService.acquire(anyString(), anyLong(), any(Duration.class), anyLong()))
|
||||
.thenReturn(null);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
// 位置长时间不变(真卡死)仍按原判据走:进入锁内封口流程
|
||||
verify(taskDistributedLockService).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -306,7 +306,9 @@ class RollbackSemanticsContractTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, objectMapper, mock(AppearancePatentLlmClient.class),
|
||||
mock(AppearancePatentTaskCacheService.class), mock(com.nanri.aiimage.config.AppearancePatentProperties.class),
|
||||
taskFileJobService, taskProgressSnapshotService, transientPayloadStorageService, transactionManager,
|
||||
taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, transactionManager,
|
||||
distributedJobLockService, taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(UserApiSecretService.class));
|
||||
|
||||
+79
@@ -13,7 +13,9 @@ import java.time.Duration;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
@@ -111,4 +113,81 @@ class TaskHeartbeatPositionServiceTest {
|
||||
|
||||
assertNull(service().describe(123L));
|
||||
}
|
||||
|
||||
// ===== 停滞判定(封口前的豁免依据)=====
|
||||
|
||||
@Test
|
||||
void stallUnknownWhenNoPosition() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123")).thenReturn(Map.of());
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallUnknownOnRedisError() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenThrow(new RuntimeException("redis down"));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallAdvancingWhenPositionChangedAndSnapshotRefreshed() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(400/1300)", "at", String.valueOf(Long.MAX_VALUE)));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L));
|
||||
verify(hashOperations).putAll(eq("task:progress:stall:123"), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallAdvancingWhenPositionUnchangedWithinWindow() {
|
||||
// 位置没变但还没到停滞窗口:仍按「在推进」处理,不得封口
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
|
||||
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(5).toMillis())));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallStalledWhenPositionUnchangedBeyondWindow() {
|
||||
// 位置长时间不动才是真卡死,允许调用方按原判据封口
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
|
||||
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis())));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientAdvancingOnlyForAdvancingState() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
// 停滞(位置长时间未变)不构成豁免
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
|
||||
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis())));
|
||||
assertFalse(service().isClientAdvancing(123L, 180L));
|
||||
|
||||
// 无位置信息(老客户端)同样不构成豁免,调用方保留原判据
|
||||
assertFalse(service().isClientAdvancing(null, 180L));
|
||||
|
||||
// 位置在变才是豁免
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(1/1300)", "at", String.valueOf(Long.MAX_VALUE)));
|
||||
assertTrue(service().isClientAdvancing(123L, 180L));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user