From 0168a30ae461941c038bbbf78df6fea343d22424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 19 Oct 2026 14:55:32 +0800 Subject: [PATCH] =?UTF-8?q?refactor(phase-3):=203C=20=E6=8B=86=E8=96=84?= =?UTF-8?q?=E7=AC=AC=E4=B8=80=E7=AC=94=E2=80=94=E2=80=94A=20=E9=99=88?= =?UTF-8?q?=E6=97=A7=E6=89=AB=E6=8F=8F=E8=81=9A=E7=B1=BB=E6=8A=BD=E6=88=90?= =?UTF-8?q?=20support=20=E4=BB=B6=EF=BC=8C=E6=9C=8D=E5=8A=A1=E6=A3=98?= =?UTF-8?q?=E8=BD=AE=202209=E2=86=922080?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 把 CollectDataService 的心跳线与二次判死线的「扫描+判定」搬到 modules/collectdata/service/support/CollectDataStaleTaskScanner(287 行),收尾 (ensureTaskResult / loadStats / hasReceivedChunks / enqueueFinalWorkbook / markTaskFailed)留在服务侧经两枚 @FunctionalInterface 回调交回——与 AppearancePatentStaleTaskScanner 同一形状。判据、步序、日志文案逐字保持; 状态字面量改走 TaskStatuses.FileTask.RUNNING;两处 LockSkipWitness 点位仍写死 字面量在调用处(护栏尺不许常量)。服务侧留 finalizeNoUploadStaleTasks 薄委托, 因 CollectDataNoUploadStaleTest 按名字反射点名它。 目的是给 3C 迁移的 29 枚显式构造器(≈+47 行)腾出「同笔抵平」额度: 本笔净减 129 行,基线一格未改。 门禁:定向 349 例/0 红/2 skipped;后端全量 4528 例/0 红/14 skipped (较上一读数 +10 = 3E/3C 两支新 golden 各 5 枚 @Test);ops ruff All checks passed、readability 棘轮 9 passed。 --- .../service/CollectDataService.java | 211 +++---------- .../support/CollectDataStaleTaskScanner.java | 286 ++++++++++++++++++ 2 files changed, 327 insertions(+), 170 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataStaleTaskScanner.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index 4fb46f812..9f1df0a09 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -12,7 +12,6 @@ import com.nanri.aiimage.common.exception.BusinessCodes; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.security.AdminAuthSupport; import com.nanri.aiimage.common.util.ExcelStreamReader; -import com.nanri.aiimage.common.util.LockSkipWitness; import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService; import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper; import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; @@ -37,6 +36,7 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo; +import com.nanri.aiimage.modules.collectdata.service.support.CollectDataStaleTaskScanner; import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery; import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter; import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec; @@ -55,7 +55,6 @@ import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; -import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; @@ -565,179 +564,55 @@ public class CollectDataService { @Scheduled(cron = "${aiimage.collect-data.stale-check-cron:0 */2 * * * *}") public void finalizeStaleTasks() { - // 双节点互斥:Redis 不可用时 fail-open(仅记日志),兜底靠下方 per-task DB 锁,不会双写终态 - DistributedJobLockService.LockHandle jobLock = - distributedJobLockService.tryLock("collect-data:stale-check", java.time.Duration.ofMinutes(2)); - if (jobLock == null) { - log.info("[collect-data] stale-check skipped, another instance holds the distributed lock"); - return; - } - try (jobLock) { - long timeoutMinutes = Math.max(1L, staleTimeoutMinutes); - LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes); - List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .lt(FileTaskEntity::getUpdatedAt, threshold) - .orderByAsc(FileTaskEntity::getUpdatedAt) - .last("limit 200")); - for (FileTaskEntity task : tasks) { - finalizeStaleTask(task.getId(), threshold, timeoutMinutes); - } - finalizeNoUploadStaleTasks(threshold); - } + staleTaskScanner().sweep(this::finalizeHeartbeatStale, this::finalizeNoUploadStale); } - private void finalizeStaleTask(Long taskId, LocalDateTime threshold, long timeoutMinutes) { - try (TaskDistributedLockService.LockHandle lock = - taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) { - if (lock == null) { - LockSkipWitness.recordAbandoned("collect-data:stale-finalize", MODULE_TYPE, taskId); - return; - } - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task == null - || !MODULE_TYPE.equals(task.getModuleType()) - || !STATUS_RUNNING.equals(task.getStatus()) - || task.getUpdatedAt() == null - || !task.getUpdatedAt().isBefore(threshold)) { - return; - } - // 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点 - if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat( - taskId, 0L, task.getUpdatedAt(), LocalDateTime.now())) { - log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}", - taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(), - task.getUpdatedAt()); - return; - } - if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) { - return; - } - - FileResultEntity result = ensureTaskResult(task); - CollectDataStats stats = loadStats(task); - if (hasReceivedChunks(taskId)) { - enqueueFinalWorkbook(task, result, stats); - log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}", - taskId, timeoutMinutes, stats.finalRowCount); - return; - } - - markTaskFailed(task, result, STALE_TASK_ERROR, stats); - log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}", - taskId, timeoutMinutes); - } catch (Exception ex) { - log.warn("[collect-data] stale task finalization failed taskId={} msg={}", - taskId, ex.getMessage(), ex); + /** 心跳线判死的收尾:已收到分片 ⇒ 挂部分工作簿;一片都没收到 ⇒ 直接标失败。 */ + private void finalizeHeartbeatStale(FileTaskEntity task, long timeoutMinutes) { + FileResultEntity result = ensureTaskResult(task); + CollectDataStats stats = loadStats(task); + if (hasReceivedChunks(task.getId())) { + enqueueFinalWorkbook(task, result, stats); + log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}", + task.getId(), timeoutMinutes, stats.finalRowCount); + return; } + markTaskFailed(task, result, STALE_TASK_ERROR, stats); + log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}", + task.getId(), timeoutMinutes); + } + + /** 二次判死线的收尾(扫描与判定在 {@code CollectDataStaleTaskScanner}):文案与心跳线分开写。 */ + private void finalizeNoUploadStale(FileTaskEntity task, LocalDateTime lastResultAt, String position, long minutes) { + FileResultEntity result = ensureTaskResult(task); + CollectDataStats stats = loadStats(task); + if (hasReceivedChunks(task.getId())) { + enqueueFinalWorkbook(task, result, stats); + log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}", + task.getId(), lastResultAt, minutes, stats.finalRowCount); + return; + } + markTaskFailed(task, result, noUploadFailReason(minutes, lastResultAt, position), stats); + log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}", + task.getId(), lastResultAt, minutes); } /** - * 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。 - * - *

既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at—— - * 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。 - * 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用 - * biz_task_scope_state.last_chunk_at(只随结果分片上报刷新); - * 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。 + * 这个入口的存在理由是测试:{@code CollectDataNoUploadStaleTest} 按名字反射点名它, + * 扫描与判定本体在 {@code CollectDataStaleTaskScanner#sweepNoUpload}。 */ - private void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) { - long minutes = noResultUploadTimeoutMinutes; - if (minutes <= 0) { - return; - } - LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes); - List candidates = fileTaskMapper.selectList(new LambdaQueryWrapper() - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold) - .lt(FileTaskEntity::getCreatedAt, cutoff) - .orderByAsc(FileTaskEntity::getCreatedAt) - .last("limit 200")); - if (candidates.isEmpty()) { - return; - } - Map lastResultAtByTaskId = new HashMap<>(); - for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds( - candidates.stream().map(FileTaskEntity::getId).toList())) { - if (dto.taskId() != null && dto.lastChunkAt() != null) { - lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt()); - } - } - for (FileTaskEntity task : candidates) { - LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId()); - if (lastResultAt == null) { - // 从未上报任何结果分片:原先与「上报仍新鲜」合并成静默 continue,是 2626 那型出口。 - // 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪; - // 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。 - String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId()); - if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { - log.info("[collect-data] never-reported skip because client still advancing taskId={} position={} minutes={}", - task.getId(), neverReportedPosition, minutes); - continue; - } - if (neverReportedPosition == null || neverReportedPosition.isBlank()) { - log.warn("[collect-data] never-reported skip without position evidence taskId={} createdAt={} minutes={}", - task.getId(), task.getCreatedAt(), minutes); - continue; - } - log.warn("[collect-data] never-reported but position stuck, failing taskId={} position={} minutes={}", - task.getId(), neverReportedPosition, minutes); - finalizeNoUploadStaleTask(task.getId(), null, neverReportedPosition, minutes); - continue; - } - if (lastResultAt.isAfter(cutoff)) { - continue; - } - // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。 - // 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。 - // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。 - if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { - log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}", - task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId())); - continue; - } - finalizeNoUploadStaleTask(task.getId(), lastResultAt, null, minutes); - } + void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) { + staleTaskScanner().sweepNoUpload(heartbeatThreshold, this::finalizeNoUploadStale); } - /** 单个任务的二次判死收尾:锁内复查心跳后,有分片→组装部分工作簿;无分片→标失败。 */ - private void finalizeNoUploadStaleTask(Long taskId, LocalDateTime lastResultAt, String position, long minutes) { - try (TaskDistributedLockService.LockHandle lock = - taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) { - if (lock == null) { - LockSkipWitness.recordAbandoned("collect-data:no-upload-stale-finalize", MODULE_TYPE, taskId); - return; - } - FileTaskEntity task = fileTaskMapper.selectById(taskId); - LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes)); - if (task == null - || !MODULE_TYPE.equals(task.getModuleType()) - || !STATUS_RUNNING.equals(task.getStatus()) - || task.getUpdatedAt() == null - || task.getUpdatedAt().isBefore(heartbeatFreshAfter)) { - // 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理) - return; - } - if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) { - return; - } - FileResultEntity result = ensureTaskResult(task); - CollectDataStats stats = loadStats(task); - if (hasReceivedChunks(taskId)) { - enqueueFinalWorkbook(task, result, stats); - log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}", - taskId, lastResultAt, minutes, stats.finalRowCount); - return; - } - markTaskFailed(task, result, noUploadFailReason(minutes, lastResultAt, position), stats); - log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}", - taskId, lastResultAt, minutes); - } catch (Exception ex) { - log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}", - taskId, ex.getMessage(), ex); - } + /** + * 每次入口新建一枚(不是 bean):两枚阈值是 {@code @Value} 注进本支的非 final 字段, + * 测试用 {@code ReflectionTestUtils.setField} 改它们——烤进更早构造的缓存就会读旧数。 + */ + private CollectDataStaleTaskScanner staleTaskScanner() { + return new CollectDataStaleTaskScanner(MODULE_TYPE, staleTimeoutMinutes, noResultUploadTimeoutMinutes, + distributedJobLockService, fileTaskMapper, taskDistributedLockService, taskFileJobService, + taskScopeStateMapper, taskHeartbeatPositionService, staleTaskHeartbeatPolicy); } /** 二次判死原因文案;从未上报与已有上报分开写(后者不能只说「最后结果上报 null」)。 */ @@ -1524,10 +1399,6 @@ public class CollectDataService { return normalize(value).toUpperCase(Locale.ROOT); } - private String normalizeBrand(String value) { - return normalize(value).toLowerCase(Locale.ROOT); - } - /** * 将 Python 回传的配送方式标准化为 FBA / FBM / AMZ; * 空值或未识别值统一返回空字符串,由前端 / Excel 端展示为“无配送方式”。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataStaleTaskScanner.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataStaleTaskScanner.java new file mode 100644 index 000000000..728c19040 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataStaleTaskScanner.java @@ -0,0 +1,286 @@ +package com.nanri.aiimage.modules.collectdata.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.common.util.LockSkipWitness; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import lombok.extern.slf4j.Slf4j; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 采集数据陈旧任务扫描器:心跳线({@code updated_at} 超期)+二次判死线(心跳新鲜但连续无结果分片)。 + * + *

搬的是「扫描+判定」,不是「收尾」。本类决定一个任务该不该死——取锁、锁内复查、 + * 首次心跳排队宽限、未完成的组装 Job 闸门、以及二次线那套「从未上报」的三分证据判定; + * 而「死了要做什么」留在 {@code CollectDataService}(要动结果行、chunk 计数、部分工作簿与 + * resultJson 统计,那些件都还在服务里)。判完通过 {@link HeartbeatStaleFinalizer} 与 + * {@link NoUploadStaleFinalizer} 两枚回调交回去,与 {@code AppearancePatentStaleTaskScanner} + * 同一形状——把收尾一起拖出来就得给结果行读取、 + * 工作簿组装与统计再套一套构造器,那是刚结清过的那种亏。 + * + *

两条线刻意不合并。心跳线判「{@code updated_at} 陈旧」,二次线判「心跳新鲜但 + * {@code last_chunk_at} 久未变」——同一条任务在两条线上的期望相反,合并判定必有一边失明 + * (2626 那型:从未上报的行与上报仍新鲜的行被并成一句静默 continue,假活任务杀不掉)。 + * + *

入参为什么逐枚交进来而不是注一个 properties。{@code staleTimeoutMinutes} 与 + * {@code noResultUploadTimeoutMinutes} 在服务侧是 {@code @Value} 注入的非 final 字段, + * 测试用 {@code ReflectionTestUtils.setField} 改它们;本类由服务每次入口调用时新建, + * 读的就是当时那一枚值——把阈值烤进构造更早的缓存里,那三枚既有用例当场就会读旧数。 + * + *

{@code moduleType()} 由调用方交出(本支是 {@code COLLECT_DATA}),状态字面量一律走 + * {@link TaskStatuses}:{@code src/main} 里手写的 {@code STATUS_XXX} 标量是只许降的棘轮, + * 降法就是接进这里。 + */ +@Slf4j +public class CollectDataStaleTaskScanner { + + /** 双节点互斥的作业锁名,逐字沿用服务侧现网那句(改名等于换一把锁)。 */ + private static final String STALE_CHECK_JOB_LOCK = "collect-data:stale-check"; + + /** 每轮作业锁的持有时长;Redis 不可用时上层 fail-open,兜底靠 per-task DB 锁。 */ + private static final Duration STALE_CHECK_JOB_LOCK_TTL = Duration.ofMinutes(2); + + /** 单轮候选上限,逐字沿用现网那句 {@code limit 200}。 */ + private static final String CANDIDATE_LIMIT = "limit 200"; + + /** 本模块没有自己的 Redis 心跳读数,闸门按「没有首心跳记录」这一形传 0。 */ + private static final long NO_REDIS_HEARTBEAT_READING = 0L; + + /** + * 判死之后的收尾归服务。两条线各一枚接缝而不是一个双方法接口——本类的 {@link #sweep} + * 要在心跳线之后接着跑二次线,两枚接缝分开交,服务侧那两枚私有方法才能原样留私有 + * (合成一枚双方法接口就得让服务 implements 它,那两枚方法当场被迫变 public,白扩对外面)。 + */ + @FunctionalInterface + public interface HeartbeatStaleFinalizer { + /** 心跳线判死:阈值与超时分钟数已算好,收尾按「有无已收分片」分两条路走。 */ + void finalizeHeartbeatStale(FileTaskEntity task, long timeoutMinutes); + } + + @FunctionalInterface + public interface NoUploadStaleFinalizer { + /** + * 二次判死线收尾。{@code lastResultAt} 为 null 表示从未上报过结果分片, + * 此时 {@code position} 是唯一的卡死证据(心跳位置文本),两者恰好一个非 null 是现网形状。 + */ + void finalizeNoUploadStale(FileTaskEntity task, LocalDateTime lastResultAt, String position, long minutes); + } + + private final String moduleType; + private final long staleTimeoutMinutes; + private final long noResultUploadTimeoutMinutes; + private final DistributedJobLockService distributedJobLockService; + private final FileTaskMapper fileTaskMapper; + private final TaskDistributedLockService taskDistributedLockService; + private final TaskFileJobService taskFileJobService; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; + private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy; + + public CollectDataStaleTaskScanner(String moduleType, + long staleTimeoutMinutes, + long noResultUploadTimeoutMinutes, + DistributedJobLockService distributedJobLockService, + FileTaskMapper fileTaskMapper, + TaskDistributedLockService taskDistributedLockService, + TaskFileJobService taskFileJobService, + TaskScopeStateMapper taskScopeStateMapper, + TaskHeartbeatPositionService taskHeartbeatPositionService, + StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy) { + this.moduleType = moduleType; + this.staleTimeoutMinutes = staleTimeoutMinutes; + this.noResultUploadTimeoutMinutes = noResultUploadTimeoutMinutes; + this.distributedJobLockService = distributedJobLockService; + this.fileTaskMapper = fileTaskMapper; + this.taskDistributedLockService = taskDistributedLockService; + this.taskFileJobService = taskFileJobService; + this.taskScopeStateMapper = taskScopeStateMapper; + this.taskHeartbeatPositionService = taskHeartbeatPositionService; + this.staleTaskHeartbeatPolicy = staleTaskHeartbeatPolicy; + } + + /** + * 一轮完整扫描:先跑心跳线,再在同一把作业锁内跑二次判死线(现网就是这个次序, + * 且二次线拿的是心跳线那枚 {@code threshold})。 + */ + public void sweep(HeartbeatStaleFinalizer heartbeat, NoUploadStaleFinalizer noUpload) { + // 双节点互斥:Redis 不可用时 fail-open(仅记日志),兜底靠下方 per-task DB 锁,不会双写终态 + DistributedJobLockService.LockHandle jobLock = + distributedJobLockService.tryLock(STALE_CHECK_JOB_LOCK, STALE_CHECK_JOB_LOCK_TTL); + if (jobLock == null) { + log.info("[collect-data] stale-check skipped, another instance holds the distributed lock"); + return; + } + try (jobLock) { + long timeoutMinutes = Math.max(1L, staleTimeoutMinutes); + LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes); + for (FileTaskEntity task : listHeartbeatStaleCandidates(threshold)) { + finalizeHeartbeatStale(task.getId(), threshold, timeoutMinutes, heartbeat); + } + sweepNoUpload(threshold, noUpload); + } + } + + private List listHeartbeatStaleCandidates(LocalDateTime threshold) { + return fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, moduleType) + .eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING) + .lt(FileTaskEntity::getUpdatedAt, threshold) + .orderByAsc(FileTaskEntity::getUpdatedAt) + .last(CANDIDATE_LIMIT)); + } + + /** + * 心跳线单任务:锁内复查 ⇒ 排队宽限 ⇒ 未完成组装 Job 闸门 ⇒ 交回收尾。 + * 每一格都是「放过、不判死」的形状,判据与顺序逐字保持现网写法。 + */ + private void finalizeHeartbeatStale(Long taskId, + LocalDateTime threshold, + long timeoutMinutes, + HeartbeatStaleFinalizer finalizer) { + try (TaskDistributedLockService.LockHandle lock = + taskDistributedLockService.acquire(moduleType, taskId, 0L)) { + if (lock == null) { + LockSkipWitness.recordAbandoned("collect-data:stale-finalize", moduleType, taskId); + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null + || !moduleType.equals(task.getModuleType()) + || !TaskStatuses.FileTask.RUNNING.equals(task.getStatus()) + || task.getUpdatedAt() == null + || !task.getUpdatedAt().isBefore(threshold)) { + return; + } + // 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点 + if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat( + taskId, NO_REDIS_HEARTBEAT_READING, task.getUpdatedAt(), LocalDateTime.now())) { + log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}", + taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(), + task.getUpdatedAt()); + return; + } + if (taskFileJobService.countUnfinishedAssembleJobs(taskId, moduleType) > 0L) { + return; + } + finalizer.finalizeHeartbeatStale(task, timeoutMinutes); + } catch (Exception ex) { + log.warn("[collect-data] stale task finalization failed taskId={} msg={}", + taskId, ex.getMessage(), ex); + } + } + + /** + * 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。 + * + *

既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at—— + * 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。 + * 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用 + * biz_task_scope_state.last_chunk_at(只随结果分片上报刷新); + * 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。 + */ + public void sweepNoUpload(LocalDateTime heartbeatThreshold, NoUploadStaleFinalizer finalizer) { + long minutes = noResultUploadTimeoutMinutes; + if (minutes <= 0) { + return; + } + LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes); + List candidates = fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, moduleType) + .eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING) + .ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold) + .lt(FileTaskEntity::getCreatedAt, cutoff) + .orderByAsc(FileTaskEntity::getCreatedAt) + .last(CANDIDATE_LIMIT)); + if (candidates.isEmpty()) { + return; + } + Map lastResultAtByTaskId = new HashMap<>(); + for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds( + candidates.stream().map(FileTaskEntity::getId).toList())) { + if (dto.taskId() != null && dto.lastChunkAt() != null) { + lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt()); + } + } + for (FileTaskEntity task : candidates) { + LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId()); + if (lastResultAt == null) { + // 从未上报任何结果分片:原先与「上报仍新鲜」合并成静默 continue,是 2626 那型出口。 + // 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪; + // 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。 + String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId()); + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[collect-data] never-reported skip because client still advancing taskId={} position={} minutes={}", + task.getId(), neverReportedPosition, minutes); + continue; + } + if (neverReportedPosition == null || neverReportedPosition.isBlank()) { + log.warn("[collect-data] never-reported skip without position evidence taskId={} createdAt={} minutes={}", + task.getId(), task.getCreatedAt(), minutes); + continue; + } + log.warn("[collect-data] never-reported but position stuck, failing taskId={} position={} minutes={}", + task.getId(), neverReportedPosition, minutes); + finalizeNoUploadStale(task.getId(), null, neverReportedPosition, minutes, finalizer); + continue; + } + if (lastResultAt.isAfter(cutoff)) { + continue; + } + // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。 + // 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。 + // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。 + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}", + task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId())); + continue; + } + finalizeNoUploadStale(task.getId(), lastResultAt, null, minutes, finalizer); + } + } + + /** 二次线单任务:锁内复查心跳是否仍新鲜、是否还有未完成的组装 Job,然后才交回收尾。 */ + private void finalizeNoUploadStale(Long taskId, + LocalDateTime lastResultAt, + String position, + long minutes, + NoUploadStaleFinalizer finalizer) { + try (TaskDistributedLockService.LockHandle lock = + taskDistributedLockService.acquire(moduleType, taskId, 0L)) { + if (lock == null) { + LockSkipWitness.recordAbandoned("collect-data:no-upload-stale-finalize", moduleType, taskId); + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes)); + if (task == null + || !moduleType.equals(task.getModuleType()) + || !TaskStatuses.FileTask.RUNNING.equals(task.getStatus()) + || task.getUpdatedAt() == null + || task.getUpdatedAt().isBefore(heartbeatFreshAfter)) { + // 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理) + return; + } + if (taskFileJobService.countUnfinishedAssembleJobs(taskId, moduleType) > 0L) { + return; + } + finalizer.finalizeNoUploadStale(task, lastResultAt, position, minutes); + } catch (Exception ex) { + log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}", + taskId, ex.getMessage(), ex); + } + } +}