diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java index 3277aca7c..2c58dc773 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java @@ -1576,7 +1576,9 @@ public class BrandTaskService { *
既有心跳线候选条件是 updated_at/last_heartbeat_at 陈旧,而前端心跳会持续刷新它们—— * 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。 * 本线候选取「心跳新鲜 + 创建超过 N 分钟」,判据用结果上报时写入 progress hash 的 - * last_result_at(仅 saveProgressFromResult 写);从未上报(无该字段)跳过,避免误杀首批较慢的正常任务。 + * last_result_at(仅 saveProgressFromResult 写);从未上报(无该字段)不再静默跳过, + * 而是用「处理位置是否仍在推进」区分「首批较慢」与「主线程停在原地」—— + * 2026-10-06 任务 2626 正是靠无条件跳过挂到 9 小时。无位置信息时仍保守跳过。 */ private void failNoUploadStaleRunningTasks() { long minutes = brandProgressProperties.getNoResultUploadTimeoutMinutes(); @@ -1603,7 +1605,30 @@ public class BrandTaskService { } catch (Exception ignored) { } if (lastResultAt <= 0L) { - // 从未上报结果:保守跳过(首批可能较慢) + // 从未上报结果分片:原先这里无条件 continue,是 2026-10-06 任务 2626 + // 挂 9 小时不被判死的出口之一——客户端在 30 轮失败重跑里心跳照打, + // updated_at 永远新鲜,三条判死线全部躲过。现改按「处理位置是否仍在推进」判: + // 在推进 = 只是攒着没回传(首批较慢的正常任务),跳过; + // 停在原地 = 采集主线程没进展,判死; + // 完全没有位置信息(老客户端不传 progressText)= 证据不足,保守跳过但留日志。 + String position = taskHeartbeatPositionService.describe(task.getId()); + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[brand-stale-check] no-result skip because client still advancing " + + "taskId={} position={} createdBefore={} minutes={}", + task.getId(), position, task.getCreatedAt(), minutes); + continue; + } + if (position == null || position.isBlank()) { + log.warn("[brand-stale-check] no-result skip without position evidence " + + "taskId={} createdBefore={} minutes={}", + task.getId(), task.getCreatedAt(), minutes); + continue; + } + log.warn("[brand-stale-check] no-result failing taskId={} position={} timeoutMinutes={}", + task.getId(), position, minutes); + failStaleRunningTask(task.getId(), + "任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「" + + position + "」未推进,疑似客户端任务卡死,任务已自动失败"); continue; } LocalDateTime lastResult = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastResultAt), ZoneId.systemDefault()); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskStaleCheckTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskStaleCheckTest.java new file mode 100644 index 000000000..d0b5e586f --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskStaleCheckTest.java @@ -0,0 +1,206 @@ +package com.nanri.aiimage.modules.brand.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.config.BrandProgressProperties; +import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; +import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 品牌「无结果回传」判死线的从未上报分支(2026-10-06 任务 2626 的静默出口)。 + * + *
2626:1915 个品牌,客户端因响应解密失败在 30 轮重跑里空转,一次结果都没回传, + * 但 15 秒心跳照打 ⇒ updated_at 永远新鲜、last_result_at 恒为 0。原实现在 + * {@code lastResultAt <= 0} 处无条件 continue,任务因此挂 9 小时不判死、前端只看得到 0/1915。 + * 现按「处理位置是否仍在推进」区分:在推进 = 首批较慢,跳过;停在原地 = 判死并带位置; + * 无位置信息(老客户端)= 证据不足,保守跳过。 + */ +@ExtendWith(MockitoExtension.class) +class BrandTaskStaleCheckTest { + + private static final long NO_RESULT_TIMEOUT_MINUTES = 180L; + + @Mock private BrandCrawlTaskMapper brandCrawlTaskMapper; + @Mock private BrandProgressProperties brandProgressProperties; + @Mock private BrandTaskProgressCacheService brandTaskProgressCacheService; + @Mock private TaskHeartbeatPositionService taskHeartbeatPositionService; + @Mock private TaskDistributedLockService taskDistributedLockService; + + private BrandTaskService service; + + @BeforeAll + static void initializeTableInfo() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + BrandCrawlTaskEntity.class); + } + + @BeforeEach + void setUp() { + // lenient:timeoutDisabledByConfigSkipsScan 会改配置,且它压根不读这两个值 + lenient().when(brandProgressProperties.getNoResultUploadTimeoutMinutes()) + .thenReturn(NO_RESULT_TIMEOUT_MINUTES); + lenient().when(brandProgressProperties.getHeartbeatTimeoutMinutes()).thenReturn(30L); + lenient().when(taskDistributedLockService.acquire(any(), any(), anyLong())) + .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); + service = newService(); + } + + private BrandTaskService newService() { + return new BrandTaskService(brandCrawlTaskMapper, null, null, brandProgressProperties, + brandTaskProgressCacheService, taskHeartbeatPositionService, null, null, + taskDistributedLockService, null, null, null, null); + } + + private void runNoUploadLine(BrandCrawlTaskEntity task, Map