feat(任务恢复): 客户端中断的任务自动续跑(保留失败记录 + 重新排队)
2026-09-18 任务 28587(跟价,uid 977 店铺「张美莺」)在客户端 09:51/09:57 被更新脚本 重启后中断,任务一直挂 RUNNING 到 stale 兜底;跟价循环还有第二重问题——子任务失败 即整个循环判 FAILED,一次客户端更新就让无限循环彻底停下。 契约:中断任务**保留失败记录**(用户看得到「因客户端重启中断」),同时由服务端自动 重新排队续跑,长任务不再因为一次更新整个白跑。 - V129:biz_file_task 加 resume_of_task_id / resume_attempt, biz_price_track_loop_run 加 resume_attempt(代数封顶用); - 新增 TaskResumeService:扫描最近 30 分钟内因客户端中断而失败、未续跑过、代数未超限的 任务,复制请求参数重新排队成 PENDING,交给已有兜底拉取通道(客户端每分钟 pull-pending) 领走执行——不新造第二套派发机制。只对注册了 ClientTaskPullSpi 的模块生效 (相似ASIN/采集/外观专利),上架/改价等写操作模块刻意排除,避免盲目重跑; - 幂等:按 resume_of_task_id 反查,同一原任务不会重复排队; - 挂在 stale-check 巡检线(每 2 分钟、有分布式锁),并用隔离壳包住异常: 续跑失败绝不拖垮判死主流程(判死优先级更高); - PriceTrackLoopRunService:子任务因「客户端异常中断」失败时不终止循环, 清 active_task_id 后保持 RUNNING,客户端下次 dispatchNext 拿到同一店铺/同一轮, 即原地续跑(页内已处理 ASIN 由服务端 skip_asins 去重,不会重复改价); 连续重派超过 3 次才真正判失败,避免会话持续不可用时无限重开浏览器 (28587 就是这样白烧了 5 小时)。子任务成功一轮后计数归零; - application.yml:aiimage.task-resume.enabled 默认跟随 client-task-pull 开关 (兜底拉取关着时续跑任务无人领取,只会积压被告死)。 测试:TaskResumeServiceTest 6 例(排队/幂等/开关/无实现/插入失败/无归属用户)、 PriceTrackLoopRunServiceTest 新增 4 例(中断续跑/达上限/非中断仍终止/成功归零), 连带更新 DeleteBrandStaleTaskServiceTest 与 TaskModuleCoverageTest 的构造参数。
This commit is contained in:
+6
-1
@@ -14,6 +14,7 @@ import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -59,6 +60,7 @@ class DeleteBrandStaleTaskServiceTest {
|
||||
@Mock private ProductRiskTaskService productRiskTaskService;
|
||||
@Mock private ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TaskResumeService taskResumeService;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
@@ -374,6 +376,9 @@ class DeleteBrandStaleTaskServiceTest {
|
||||
private void noUploadEnabled() {
|
||||
when(deleteBrandProgressProperties.isNoResultUploadCheckEnabled()).thenReturn(true);
|
||||
when(deleteBrandProgressProperties.getNoResultUploadTimeoutMinutes()).thenReturn(180L);
|
||||
// 续跑巡检在判死主流程里被调用:默认返回空统计(无中断任务可续跑);
|
||||
// 只有跑到 failStaleRunningTasks 的用例会用到,故 lenient
|
||||
lenient().when(taskResumeService.resumeInterruptedTasks()).thenReturn(new TaskResumeService.ResumeStats());
|
||||
}
|
||||
|
||||
private void noUploadTaskLockAvailable() {
|
||||
@@ -398,7 +403,7 @@ class DeleteBrandStaleTaskServiceTest {
|
||||
null, null, null, null, null, null, null, null, null, null,
|
||||
null, null, null,
|
||||
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService, null,
|
||||
taskScopeStateMapper, taskHeartbeatPositionService);
|
||||
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
|
||||
+1
-1
@@ -82,7 +82,7 @@ class TaskModuleCoverageTest {
|
||||
mock(SimilarAsinTaskService.class),
|
||||
null, null, null, null,
|
||||
mock(ShopDataCrawlTaskService.class),
|
||||
null, null);
|
||||
null, null, null);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+157
@@ -14,6 +14,7 @@ import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
@@ -120,6 +121,162 @@ class PriceTrackLoopRunServiceTest {
|
||||
verify(loopRunMapper, never()).updateById(loop);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 子任务因客户端中断失败时循环保持运行并重派当前轮() throws Exception {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||
|
||||
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||
loop.setId(2699L);
|
||||
loop.setUserId(977L);
|
||||
loop.setStatus("RUNNING");
|
||||
loop.setExecutionMode("INFINITE");
|
||||
loop.setCurrentRound(1);
|
||||
loop.setCurrentShopIndex(0);
|
||||
loop.setActiveTaskId(28587L);
|
||||
loop.setResumeAttempt(0);
|
||||
loop.setStopRequested(false);
|
||||
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(28587L);
|
||||
task.setUserId(977L);
|
||||
task.setModuleType("PRICE_TRACK");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报(上次任务类型: price-track-run)");
|
||||
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||
|
||||
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||
|
||||
service.syncLoopRunAfterChildTerminal(28587L);
|
||||
|
||||
assertEquals("RUNNING", loop.getStatus(), "中断不该终止循环");
|
||||
assertNull(loop.getActiveTaskId(), "必须清空 active_task_id 才能重派当前轮");
|
||||
assertNull(loop.getFinishedAt());
|
||||
assertNull(loop.getErrorMessage());
|
||||
assertEquals(1, loop.getCurrentRound(), "轮次不变:原地续跑");
|
||||
assertEquals(0, loop.getCurrentShopIndex(), "店铺不变:原地续跑");
|
||||
assertEquals(1, loop.getResumeAttempt(), "中断计数递增");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 中断重派达上限后循环判失败() throws Exception {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||
|
||||
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||
loop.setId(2699L);
|
||||
loop.setUserId(977L);
|
||||
loop.setStatus("RUNNING");
|
||||
loop.setExecutionMode("INFINITE");
|
||||
loop.setCurrentRound(1);
|
||||
loop.setCurrentShopIndex(0);
|
||||
loop.setActiveTaskId(28587L);
|
||||
loop.setResumeAttempt(3);
|
||||
loop.setStopRequested(false);
|
||||
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(28587L);
|
||||
task.setUserId(977L);
|
||||
task.setModuleType("PRICE_TRACK");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报");
|
||||
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||
|
||||
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||
|
||||
service.syncLoopRunAfterChildTerminal(28587L);
|
||||
|
||||
assertEquals("FAILED", loop.getStatus(), "达上限后不再重派,交人工介入");
|
||||
assertNotNull(loop.getErrorMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 非中断原因的子任务失败仍终止循环() throws Exception {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||
|
||||
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||
loop.setId(2699L);
|
||||
loop.setUserId(977L);
|
||||
loop.setStatus("RUNNING");
|
||||
loop.setExecutionMode("INFINITE");
|
||||
loop.setCurrentRound(1);
|
||||
loop.setCurrentShopIndex(0);
|
||||
loop.setActiveTaskId(28587L);
|
||||
loop.setResumeAttempt(0);
|
||||
loop.setStopRequested(false);
|
||||
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(28587L);
|
||||
task.setUserId(977L);
|
||||
task.setModuleType("PRICE_TRACK");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("张美莺: 国家 德国 未跑完:重试后仍未获取到页码信息");
|
||||
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||
|
||||
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||
|
||||
service.syncLoopRunAfterChildTerminal(28587L);
|
||||
|
||||
assertEquals("FAILED", loop.getStatus(), "业务失败照旧终止循环,只有中断才续跑");
|
||||
assertEquals(0, loop.getResumeAttempt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 子任务成功后中断计数归零() throws Exception {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||
|
||||
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||
loop.setId(2699L);
|
||||
loop.setUserId(977L);
|
||||
loop.setStatus("RUNNING");
|
||||
loop.setExecutionMode("FINITE");
|
||||
loop.setTargetRounds(1);
|
||||
loop.setCurrentRound(1);
|
||||
loop.setCurrentShopIndex(0);
|
||||
loop.setActiveTaskId(28587L);
|
||||
loop.setResumeAttempt(2);
|
||||
loop.setStopRequested(false);
|
||||
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(28587L);
|
||||
task.setUserId(977L);
|
||||
task.setModuleType("PRICE_TRACK");
|
||||
task.setStatus("SUCCESS");
|
||||
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||
|
||||
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||
|
||||
service.syncLoopRunAfterChildTerminal(28587L);
|
||||
|
||||
assertEquals(0, loop.getResumeAttempt(), "成功一轮后中断计数归零,避免历史中断占用额度");
|
||||
}
|
||||
|
||||
private PriceTrackMatchShopsVo.PriceTrackShopQueueItem shop(String name) {
|
||||
PriceTrackMatchShopsVo.PriceTrackShopQueueItem item = new PriceTrackMatchShopsVo.PriceTrackShopQueueItem();
|
||||
item.setShopName(name);
|
||||
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 客户端中断任务的自动续跑:保留失败记录 + 重新排队 PENDING 续跑任务。
|
||||
*
|
||||
* <p>背景:2026-09-18 任务 28587 因客户端被更新脚本重启而中断后一直挂 RUNNING;
|
||||
* 中断上报把任务标 FAILED 之后,还需要有人把长任务放回队列,否则一遇客户端更新就整个白跑。
|
||||
*/
|
||||
class TaskResumeServiceTest {
|
||||
|
||||
private static ClientTaskPullSpi spi(String moduleType) {
|
||||
return new ClientTaskPullSpi() {
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return moduleType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> buildQueuePayload(FileTaskEntity task) {
|
||||
return Map.of("type", "x-run", "data", Map.of("taskId", task.getId()));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static TaskResumeService service(FileTaskMapper mapper, List<ClientTaskPullSpi> handlers,
|
||||
boolean enabled, int maxAttempt) {
|
||||
TaskResumeService service = new TaskResumeService(mapper, handlers);
|
||||
ReflectionTestUtils.setField(service, "enabled", enabled);
|
||||
ReflectionTestUtils.setField(service, "maxAttempt", maxAttempt);
|
||||
ReflectionTestUtils.setField(service, "windowMinutes", 30L);
|
||||
ReflectionTestUtils.setField(service, "limit", 20);
|
||||
return service;
|
||||
}
|
||||
|
||||
private static FileTaskEntity interruptedTask(Long id, String moduleType) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setModuleType(moduleType);
|
||||
task.setUserId(977L);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报(上次任务类型: similar-asin-run)");
|
||||
task.setRequestJson("{\"items\":[]}");
|
||||
task.setResumeAttempt(0);
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
return task;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 中断任务会被重新排队为待执行的续跑任务() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.scannedTaskCount);
|
||||
assertEquals(1, stats.resumedTaskCount);
|
||||
ArgumentCaptor<FileTaskEntity> captor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(mapper).insert(captor.capture());
|
||||
FileTaskEntity resumed = captor.getValue();
|
||||
assertEquals("PENDING", resumed.getStatus(), "续跑任务必须是待领取状态,交给客户端兜底拉取");
|
||||
assertEquals(28587L, resumed.getResumeOfTaskId(), "必须回指原任务,用于追溯与幂等");
|
||||
assertEquals(1, resumed.getResumeAttempt());
|
||||
assertEquals(977L, resumed.getUserId());
|
||||
assertEquals("SIMILAR_ASIN", resumed.getModuleType());
|
||||
assertEquals("{\"items\":[]}", resumed.getRequestJson(), "原任务参数必须整体带走");
|
||||
assertTrue(resumed.getTaskNo().startsWith("SIMILAR_ASIN-"), "任务号沿用模块前缀");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 已有续跑任务时跳过避免重复排队() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||
when(mapper.selectCount(any())).thenReturn(1L, 1L);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
verify(mapper, never()).insert(any(FileTaskEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 开关关闭时不扫描() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), false, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(0, stats.scannedTaskCount);
|
||||
verify(mapper, never()).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有注册模块兜底实现时不扫描() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
TaskResumeService service = service(mapper, List.of(), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(0, stats.scannedTaskCount);
|
||||
verify(mapper, never()).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 插入失败只计入跳过不抛出() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
doThrow(new RuntimeException("库挂了")).when(mapper).insert(any(FileTaskEntity.class));
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有归属用户的中断任务跳过() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
FileTaskEntity orphan = interruptedTask(28587L, "SIMILAR_ASIN");
|
||||
orphan.setUserId(null);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(orphan));
|
||||
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
verify(mapper, never()).insert(any(FileTaskEntity.class));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user