diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 69e9268ca..319439bc9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -8,12 +8,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.task.TaskResultAdmission; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentExcelParser; -import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.common.util.GroupResultPropagator; -import com.nanri.aiimage.common.util.LockSkipWitness; import com.nanri.aiimage.common.util.FailedStatusRowFilter; -import com.nanri.aiimage.common.util.StaleTaskDeathReason; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.StorageProperties; @@ -36,6 +33,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskD import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskItemVo; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentRowOutcome; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentSheetBuilder; +import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentStaleTaskScanner; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentTaskOwnership; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; @@ -43,7 +41,6 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; -import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; @@ -72,14 +69,11 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.time.Duration; -import java.time.Instant; import java.time.LocalDateTime; -import java.time.ZoneId; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -655,214 +649,36 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline tasks = listStaleFinalizeCandidates(); - for (FileTaskEntity task : tasks) { - // P1-8(双实例 stale 判死盲区修复,与 similar-asin 同源):本方法由 - // DeleteBrandStaleTaskService 的 "delete-brand:stale-check" 分布式 job 锁保护, - // 单实例执行;历史上 owner 非本实例的任务直接跳过,若任务 owner 是另一实例则无人判死, - // 只能等 job 级 stuck 扫描误判。现改为全局判死:跨实例并发互斥由 task 分布式锁 - // (acquire 不等待、TTL 5 分钟)保证,owner 实例仍在正常推进的任务心跳不会 stale。 - if (!isHeartbeatStale(task, thresholdMillis)) { - continue; - } - AppearancePatentTaskOwnership ownership = taskOwnership(); - String owner = ownership.ownerFromTask(task); - if (!ownership.isOwnerCurrent(owner)) { - log.warn("[appearance-patent] 心跳超时任务归其他实例所有,由本实例(job 锁持有者)接管判死/恢复 taskId={} owner={} current={}", - task.getId(), owner, currentInstanceId()); - } - TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L); - if (lockHandle == null) { - // 锁被 owner 实例/文件任务 worker 占用说明其仍活跃,静默跳过等下一轮 - LockSkipWitness.recordSkip("appearance-patent:running-stale-scan", MODULE_TYPE, task.getId()); - continue; - } - try (lockHandle) { - finalizeStaleTask(task.getId(), describeStaleDeath(task.getId())); - } - } - finalizeNoUploadStaleTasks(); + staleTaskScanner().sweep(this::finalizeStaleTask); } public void debugFinalizeStaleTask(Long taskId) { - if (taskId == null || taskId <= 0) { - return; - } - TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L); - if (lockHandle == null) { - LockSkipWitness.recordAbandoned("appearance-patent:debug-finalize-stale", MODULE_TYPE, taskId); - return; - } - try (lockHandle) { - finalizeStaleTask(taskId, describeStaleDeath(taskId)); - } + staleTaskScanner().finalizeOne(taskId, this::finalizeStaleTask); } /** - * 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。 - * 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result", - * 把「没开跑」说成「跑挂了」。 + * 只跑二次判死那条线。这个入口的存在理由是测试: + * {@code AppearancePatentNoUploadStaleTest} 一直按名字点名这枚私有方法,单独验「心跳新鲜、 + * 分片久不来」那条出口——A 聚类抽到扫描件后若把它摘掉,三支用例直接 + * {@code Method not found}(本笔第一版就栽在这儿:抽件前只扫了调用点,没扫反射点名)。 + * 保留同名入口=保留测试接缝,也让「两条线各跑一次」这件事在服务侧仍然读得出来。 */ - private String describeStaleDeath(Long taskId) { - long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId); - return new StaleTaskDeathReason( - persistedResultChunkCount(taskId), - lastResultChunkAt(taskId), - heartbeatMillis > 0L - ? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault()) - : null, - taskHeartbeatPositionService.describe(taskId), - taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE), - properties.getStaleTimeoutMinutes()).describe(); - } - - private long persistedResultChunkCount(Long taskId) { - if (taskId == null) { - return 0L; - } - Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); - return count == null ? 0L : count; - } - - /** 最后一次结果分片回传时间;从未回传时为 null。 */ - private LocalDateTime lastResultChunkAt(Long taskId) { - List rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId)); - return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt(); - } - - private List listStaleFinalizeCandidates() { - // P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。 - // APPEARANCE_PATENT 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。 - return fileTaskMapper.selectList(new LambdaQueryWrapper() - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .orderByAsc(FileTaskEntity::getUpdatedAt) - .last("limit 200")); - } - - /** - * P1-7:判断 APPEARANCE_PATENT 任务是否已 stale。 - * 主信号:Redis heartbeat(仅 Python 心跳/上传 result 时刷新,不会被 Java 内部循环续命)。 - * Redis 命中:以 heartbeatMillis 与 thresholdMillis 比较。 - * Redis 缺失/降级(返回 0):回落到 file_task.updated_at,避免 Redis 故障时全量误杀。 - */ - private boolean isHeartbeatStale(FileTaskEntity task, long thresholdMillis) { - if (task == null || task.getId() == null) { - return false; - } - long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); - if (heartbeatMillis > 0L) { - return heartbeatMillis <= thresholdMillis; - } - LocalDateTime updatedAt = task.getUpdatedAt(); - if (updatedAt == null) { - return false; - } - long updatedMillis = updatedAt.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - return updatedMillis <= thresholdMillis; - } - - /** - * 二次判死:Python 心跳正常(Redis heartbeat 新鲜)但连续 N 分钟无结果分片上报。 - * - *

既有判定以 Redis heartbeat 为 stale 主信号(P1-7),但该心跳随 Python 的 HTTP 心跳 - * 每分钟刷新——主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。 - * 本线改看 biz_task_scope_state.last_chunk_at(仅在 persistSubmittedChunk 上传分片时刷新), - * 命中后走既有 finalizeStaleTask(封口上传 + LLM 收尾,不粗暴杀任务)。 - * 从未上报(无 scope 行或全 NULL)的任务不再无条件跳过,改按「处理位置是否仍在推进」区分: - * 位置在变=首批较慢,跳过;位置长时间不变=判死并写清停在哪;无位置信息(老客户端)=证据不足, - * 保守跳过但留 WARN —— 2026-10-06 品牌任务 2626 正是靠无条件跳过挂到 9 小时。 - */ - private void finalizeNoUploadStaleTasks() { - long minutes = properties.getNoResultUploadTimeoutMinutes(); - if (minutes <= 0) { - return; - } - List tasks = listStaleFinalizeCandidates(); - if (tasks.isEmpty()) { - return; - } - LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes); - long heartbeatThresholdMillis = LocalDateTime.now() - .minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())) - .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - Map lastResultAtByTaskId = new HashMap<>(); - for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds( - tasks.stream().map(FileTaskEntity::getId).toList())) { - if (dto.taskId() != null && dto.lastChunkAt() != null) { - lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt()); - } - } - for (FileTaskEntity task : tasks) { - if (isHeartbeatStale(task, heartbeatThresholdMillis)) { - // 心跳已 stale:归既有心跳线处理 - continue; - } - LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId()); - if (lastResultAt == null) { - // 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue, - // 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块同样心跳照打)。 - // 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪; - // 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。 - String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId()); - if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { - log.info("[appearance-patent] 从未上报但客户端仍在推进,本轮不封口 taskId={} position={} minutes={}", - task.getId(), neverReportedPosition, minutes); - continue; - } - if (neverReportedPosition == null || neverReportedPosition.isBlank()) { - log.warn("[appearance-patent] 从未上报且无位置信息,证据不足暂不封口 taskId={} createdAt={} minutes={}", - task.getId(), task.getCreatedAt(), minutes); - continue; - } - TaskDistributedLockService.LockHandle neverReportedLock = acquireTaskLock(task.getId(), 0L); - if (neverReportedLock == null) { - LockSkipWitness.recordSkip("appearance-patent:never-reported-stale-scan", - MODULE_TYPE, task.getId()); - continue; - } - try (neverReportedLock) { - log.warn("[appearance-patent] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}", - neverReportedPosition, task.getId(), minutes); - finalizeStaleTask(task.getId(), "超过 " + minutes + " 分钟无任何结果分片上报,处理位置长时间停在「" - + neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败"); - } - continue; - } - if (lastResultAt.isAfter(cutoff)) { - continue; - } - // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。 - // 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。 - // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。 - if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { - log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}", - task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId())); - continue; - } - TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L); - if (lockHandle == null) { - LockSkipWitness.recordSkip("appearance-patent:no-result-shard-stale-scan", MODULE_TYPE, task.getId()); - continue; - } - try (lockHandle) { - log.warn("[appearance-patent] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}", - minutes, task.getId(), lastResultAt); - String error = "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes"; - finalizeStaleTask(task.getId(), error); - } - } + void finalizeNoUploadStaleTasks() { + staleTaskScanner().sweepNoUpload(this::finalizeStaleTask); } /** diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentStaleTaskScanner.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentStaleTaskScanner.java new file mode 100644 index 000000000..af79c5459 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentStaleTaskScanner.java @@ -0,0 +1,312 @@ +package com.nanri.aiimage.modules.appearancepatent.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.util.LockSkipWitness; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.common.util.StaleTaskDeathReason; +import com.nanri.aiimage.config.AppearancePatentProperties; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import lombok.extern.slf4j.Slf4j; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 外观专利的陈旧任务扫描与二次判死(A 聚类),从 {@code AppearancePatentTaskService} + * 原样搬移(3A 拆薄第三笔):判据、日志文案、{@code LockSkipWitness} 的 tag、 + * 取锁参数与两条线的先后顺序一个字没改。 + * + *

搬的是扫描与判定,不是收尾。真正给任务判死那一步({@code finalizeStaleTask} + * → 恢复探针 → 封口 → LLM 批次 → {@code finalizeTask})留在服务侧,以 + * {@link StaleDeathFinalizer} 回调交进来。理由不是审美:那段要动事务边界 + * ({@code inNewTransaction})、LLM 批管线与结果行读取,那些件还在服务里; + * 把它们一起拖出来就得先给它们再造一套构造器,而 3A 的账已经证明 + * 「结构性成本 > 可沉件收益」是要避免的事。回调是这一笔里唯一的新形状。 + * + *

两条线的分工不能并。心跳线({@code sweep})看 Redis heartbeat 是否过期; + * 无上报线({@code sweepNoUpload})专治「心跳照打、主线程卡死」那类缺口(生产 28131 同型), + * 看的是 {@code biz_task_scope_state.last_chunk_at}。同一条任务在两把尺上的期望相反—— + * 心跳新鲜但分片久不来,前者放过、后者封口。合并成一个判定就会有一边失明。 + */ +@Slf4j +public class AppearancePatentStaleTaskScanner { + + /** 状态词一律引用公共词表:这里曾写死字面量,被 TaskStatusVocabularyRatchetTest 当场判红 + * (src/main 的 STATUS_XXX 标量声明是只许降的棘轮,降法就是接进 TaskStatuses)。*/ + private static final String STATUS_RUNNING = TaskStatuses.FileTask.RUNNING; + private static final int CANDIDATE_SCAN_LIMIT = 200; + + /** 扫到该判死时怎么收尾,由服务侧决定(事务边界与 LLM 收尾都留在它那儿)。 */ + @FunctionalInterface + public interface StaleDeathFinalizer { + void finalizeStale(Long taskId, String error); + } + + private final String moduleType; + private final Duration taskLockTtl; + private final FileTaskMapper fileTaskMapper; + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final AppearancePatentTaskCacheService taskCacheService; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; + private final TaskFileJobService taskFileJobService; + private final TaskDistributedLockService taskDistributedLockService; + private final AppearancePatentProperties properties; + private final AppearancePatentTaskOwnership ownership; + + public AppearancePatentStaleTaskScanner(String moduleType, + Duration taskLockTtl, + FileTaskMapper fileTaskMapper, + TaskChunkMapper taskChunkMapper, + TaskScopeStateMapper taskScopeStateMapper, + AppearancePatentTaskCacheService taskCacheService, + TaskHeartbeatPositionService taskHeartbeatPositionService, + TaskFileJobService taskFileJobService, + TaskDistributedLockService taskDistributedLockService, + AppearancePatentProperties properties, + AppearancePatentTaskOwnership ownership) { + this.moduleType = moduleType; + this.taskLockTtl = taskLockTtl; + this.fileTaskMapper = fileTaskMapper; + this.taskChunkMapper = taskChunkMapper; + this.taskScopeStateMapper = taskScopeStateMapper; + this.taskCacheService = taskCacheService; + this.taskHeartbeatPositionService = taskHeartbeatPositionService; + this.taskFileJobService = taskFileJobService; + this.taskDistributedLockService = taskDistributedLockService; + this.properties = properties; + this.ownership = ownership; + } + + public void sweep(StaleDeathFinalizer finalizer) { + // P1-7:stale 判定改用 Redis heartbeat 作为主信号。 + // 历史实现先按 file_task.updated_at 过滤再二次校验 Redis 心跳, + // 但 file_task.updated_at 会被 Java 端在 "等 Python 上传" 分支自我续命 + // (resetStuckJobs 把 file_job 改回 PENDING → 重跑 processResultFileJob → touchJavaSideTaskActivity)。 + // Redis heartbeat 仅在 Python 心跳/上传 result 时写入,不会被 Java 内部循环刷新。 + long thresholdMillis = LocalDateTime.now() + .minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())) + .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + List tasks = listStaleFinalizeCandidates(); + for (FileTaskEntity task : tasks) { + // P1-8(双实例 stale 判死盲区修复,与 similar-asin 同源):本方法由 + // DeleteBrandStaleTaskService 的 "delete-brand:stale-check" 分布式 job 锁保护, + // 单实例执行;历史上 owner 非本实例的任务直接跳过,若任务 owner 是另一实例则无人判死, + // 只能等 job 级 stuck 扫描误判。现改为全局判死:跨实例并发互斥由 task 分布式锁 + // (acquire 不等待、TTL 5 分钟)保证,owner 实例仍在正常推进的任务心跳不会 stale。 + if (!isHeartbeatStale(task, thresholdMillis)) { + continue; + } + String owner = ownership.ownerFromTask(task); + if (!ownership.isOwnerCurrent(owner)) { + log.warn("[appearance-patent] 心跳超时任务归其他实例所有,由本实例(job 锁持有者)接管判死/恢复 taskId={} owner={} current={}", + task.getId(), owner, ownership.currentInstanceId()); + } + TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L); + if (lockHandle == null) { + // 锁被 owner 实例/文件任务 worker 占用说明其仍活跃,静默跳过等下一轮 + LockSkipWitness.recordSkip("appearance-patent:running-stale-scan", moduleType, task.getId()); + continue; + } + try (lockHandle) { + finalizer.finalizeStale(task.getId(), describeStaleDeath(task.getId())); + } + } + sweepNoUpload(finalizer); + } + + public void finalizeOne(Long taskId, StaleDeathFinalizer finalizer) { + if (taskId == null || taskId <= 0) { + return; + } + TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L); + if (lockHandle == null) { + LockSkipWitness.recordAbandoned("appearance-patent:debug-finalize-stale", moduleType, taskId); + return; + } + try (lockHandle) { + finalizer.finalizeStale(taskId, describeStaleDeath(taskId)); + } + } + + /** + * 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。 + * 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result", + * 把「没开跑」说成「跑挂了」。 + */ + public String describeStaleDeath(Long taskId) { + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId); + return new StaleTaskDeathReason( + persistedResultChunkCount(taskId), + lastResultChunkAt(taskId), + heartbeatMillis > 0L + ? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault()) + : null, + taskHeartbeatPositionService.describe(taskId), + taskFileJobService.hasExhaustedAssembleJob(taskId, moduleType), + properties.getStaleTimeoutMinutes()).describe(); + } + + private long persistedResultChunkCount(Long taskId) { + if (taskId == null) { + return 0L; + } + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, moduleType)); + return count == null ? 0L : count; + } + + /** 最后一次结果分片回传时间;从未回传时为 null。 */ + private LocalDateTime lastResultChunkAt(Long taskId) { + List rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId)); + return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt(); + } + + private List listStaleFinalizeCandidates() { + // P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。 + // APPEARANCE_PATENT 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。 + return fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, moduleType) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .orderByAsc(FileTaskEntity::getUpdatedAt) + .last("limit " + CANDIDATE_SCAN_LIMIT)); + } + + /** + * P1-7:判断 APPEARANCE_PATENT 任务是否已 stale。 + * 主信号:Redis heartbeat(仅 Python 心跳/上传 result 时刷新,不会被 Java 内部循环续命)。 + * Redis 命中:以 heartbeatMillis 与 thresholdMillis 比较。 + * Redis 缺失/降级(返回 0):回落到 file_task.updated_at,避免 Redis 故障时全量误杀。 + */ + private boolean isHeartbeatStale(FileTaskEntity task, long thresholdMillis) { + if (task == null || task.getId() == null) { + return false; + } + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); + if (heartbeatMillis > 0L) { + return heartbeatMillis <= thresholdMillis; + } + LocalDateTime updatedAt = task.getUpdatedAt(); + if (updatedAt == null) { + return false; + } + long updatedMillis = updatedAt.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + return updatedMillis <= thresholdMillis; + } + + /** + * 二次判死:Python 心跳正常(Redis heartbeat 新鲜)但连续 N 分钟无结果分片上报。 + * + *

既有判定以 Redis heartbeat 为 stale 主信号(P1-7),但该心跳随 Python 的 HTTP 心跳 + * 每分钟刷新——主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。 + * 本线改看 biz_task_scope_state.last_chunk_at(仅在 persistSubmittedChunk 上传分片时刷新), + * 命中后走既有 finalizeStaleTask(封口上传 + LLM 收尾,不粗暴杀任务)。 + * 从未上报(无 scope 行或全 NULL)的任务不再无条件跳过,改按「处理位置是否仍在推进」区分: + * 位置在变=首批较慢,跳过;位置长时间不变=判死并写清停在哪;无位置信息(老客户端)=证据不足, + * 保守跳过但留 WARN —— 2026-10-06 品牌任务 2626 正是靠无条件跳过挂到 9 小时。 + */ + public void sweepNoUpload(StaleDeathFinalizer finalizer) { + long minutes = properties.getNoResultUploadTimeoutMinutes(); + if (minutes <= 0) { + return; + } + List tasks = listStaleFinalizeCandidates(); + if (tasks.isEmpty()) { + return; + } + LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes); + long heartbeatThresholdMillis = LocalDateTime.now() + .minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())) + .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + Map lastResultAtByTaskId = new HashMap<>(); + for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds( + tasks.stream().map(FileTaskEntity::getId).toList())) { + if (dto.taskId() != null && dto.lastChunkAt() != null) { + lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt()); + } + } + for (FileTaskEntity task : tasks) { + if (isHeartbeatStale(task, heartbeatThresholdMillis)) { + // 心跳已 stale:归既有心跳线处理 + continue; + } + LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId()); + if (lastResultAt == null) { + // 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue, + // 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块同样心跳照打)。 + // 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪; + // 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。 + String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId()); + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[appearance-patent] 从未上报但客户端仍在推进,本轮不封口 taskId={} position={} minutes={}", + task.getId(), neverReportedPosition, minutes); + continue; + } + if (neverReportedPosition == null || neverReportedPosition.isBlank()) { + log.warn("[appearance-patent] 从未上报且无位置信息,证据不足暂不封口 taskId={} createdAt={} minutes={}", + task.getId(), task.getCreatedAt(), minutes); + continue; + } + TaskDistributedLockService.LockHandle neverReportedLock = acquireTaskLock(task.getId(), 0L); + if (neverReportedLock == null) { + LockSkipWitness.recordSkip("appearance-patent:never-reported-stale-scan", + moduleType, task.getId()); + continue; + } + try (neverReportedLock) { + log.warn("[appearance-patent] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}", + neverReportedPosition, task.getId(), minutes); + finalizer.finalizeStale(task.getId(), "超过 " + minutes + " 分钟无任何结果分片上报,处理位置长时间停在「" + + neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败"); + } + continue; + } + if (lastResultAt.isAfter(cutoff)) { + continue; + } + // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。 + // 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。 + // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。 + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}", + task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId())); + continue; + } + TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L); + if (lockHandle == null) { + LockSkipWitness.recordSkip("appearance-patent:no-result-shard-stale-scan", moduleType, task.getId()); + continue; + } + try (lockHandle) { + log.warn("[appearance-patent] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}", + minutes, task.getId(), lastResultAt); + String error = "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes"; + finalizer.finalizeStale(task.getId(), error); + } + } + } + + /** + * 与 {@code AppearancePatentTaskService#acquireTaskLock} 同一枚调用(不等待、拿不到就跳过等下一轮)。 + * 没有把这枚也抽成回调:它只是一行转发,多一个回调位反而让「谁在什么锁上跳过」更难追。 + * TTL 由服务侧交进来,不在这里重复写死 5 分钟。 + */ + private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) { + return taskDistributedLockService.acquire(moduleType, taskId, taskLockTtl, waitMillis); + } +}