refactor(phase-3): 3C 拆薄第一笔——A 陈旧扫描聚类抽成 support 件,服务棘轮 2209→2080

把 CollectDataService 的心跳线与二次判死线的「扫描+判定」搬到
modules/collectdata/service/support/CollectDataStaleTaskScanner(287 行),收尾
(ensureTaskResult / loadStats / hasReceivedChunks / enqueueFinalWorkbook /
markTaskFailed)留在服务侧经两枚 @FunctionalInterface 回调交回——与
AppearancePatentStaleTaskScanner 同一形状。判据、步序、日志文案逐字保持;
状态字面量改走 TaskStatuses.FileTask.RUNNING;两处 LockSkipWitness 点位仍写死
字面量在调用处(护栏尺不许常量)。服务侧留 finalizeNoUploadStaleTasks 薄委托,
因 CollectDataNoUploadStaleTest 按名字反射点名它。

目的是给 3C 迁移的 29 枚显式构造器(≈+47 行)腾出「同笔抵平」额度:
本笔净减 129 行,基线一格未改。

门禁:定向 349 例/0 红/2 skipped;后端全量 4528 例/0 红/14 skipped
(较上一读数 +10 = 3E/3C 两支新 golden 各 5 枚 @Test);ops ruff All checks
passed、readability 棘轮 9 passed。
This commit is contained in:
2026-10-19 14:55:32 +08:00
parent 465ee06b7c
commit 0168a30ae4
2 changed files with 327 additions and 170 deletions
@@ -12,7 +12,6 @@ import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.common.util.LockSkipWitness;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
@@ -37,6 +36,7 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
import com.nanri.aiimage.modules.collectdata.service.support.CollectDataStaleTaskScanner;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
@@ -55,7 +55,6 @@ import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
@@ -565,179 +564,55 @@ public class CollectDataService {
@Scheduled(cron = "${aiimage.collect-data.stale-check-cron:0 */2 * * * *}")
public void finalizeStaleTasks() {
// 双节点互斥:Redis 不可用时 fail-open(仅记日志),兜底靠下方 per-task DB 锁,不会双写终态
DistributedJobLockService.LockHandle jobLock =
distributedJobLockService.tryLock("collect-data:stale-check", java.time.Duration.ofMinutes(2));
if (jobLock == null) {
log.info("[collect-data] stale-check skipped, another instance holds the distributed lock");
return;
}
try (jobLock) {
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.lt(FileTaskEntity::getUpdatedAt, threshold)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 200"));
for (FileTaskEntity task : tasks) {
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
}
finalizeNoUploadStaleTasks(threshold);
}
staleTaskScanner().sweep(this::finalizeHeartbeatStale, this::finalizeNoUploadStale);
}
private void finalizeStaleTask(Long taskId, LocalDateTime threshold, long timeoutMinutes) {
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
if (lock == null) {
LockSkipWitness.recordAbandoned("collect-data:stale-finalize", MODULE_TYPE, taskId);
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null
|| !MODULE_TYPE.equals(task.getModuleType())
|| !STATUS_RUNNING.equals(task.getStatus())
|| task.getUpdatedAt() == null
|| !task.getUpdatedAt().isBefore(threshold)) {
return;
}
// 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
taskId, 0L, task.getUpdatedAt(), LocalDateTime.now())) {
log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(),
task.getUpdatedAt());
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
return;
}
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
if (hasReceivedChunks(taskId)) {
enqueueFinalWorkbook(task, result, stats);
log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
taskId, timeoutMinutes, stats.finalRowCount);
return;
}
markTaskFailed(task, result, STALE_TASK_ERROR, stats);
log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
taskId, timeoutMinutes);
} catch (Exception ex) {
log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
taskId, ex.getMessage(), ex);
/** 心跳线判死的收尾:已收到分片 ⇒ 挂部分工作簿;一片都没收到 ⇒ 直接标失败。 */
private void finalizeHeartbeatStale(FileTaskEntity task, long timeoutMinutes) {
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
if (hasReceivedChunks(task.getId())) {
enqueueFinalWorkbook(task, result, stats);
log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
task.getId(), timeoutMinutes, stats.finalRowCount);
return;
}
markTaskFailed(task, result, STALE_TASK_ERROR, stats);
log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
task.getId(), timeoutMinutes);
}
/** 二次判死线的收尾(扫描与判定在 {@code CollectDataStaleTaskScanner}):文案与心跳线分开写。 */
private void finalizeNoUploadStale(FileTaskEntity task, LocalDateTime lastResultAt, String position, long minutes) {
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
if (hasReceivedChunks(task.getId())) {
enqueueFinalWorkbook(task, result, stats);
log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}",
task.getId(), lastResultAt, minutes, stats.finalRowCount);
return;
}
markTaskFailed(task, result, noUploadFailReason(minutes, lastResultAt, position), stats);
log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}",
task.getId(), lastResultAt, minutes);
}
/**
* 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
*
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。
* 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用
* biz_task_scope_state.last_chunk_at(只随结果分片上报刷新);
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
* 这个入口的存在理由是测试:{@code CollectDataNoUploadStaleTest} 按名字反射点名它,
* 扫描与判定本体在 {@code CollectDataStaleTaskScanner#sweepNoUpload}。
*/
private void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) {
long minutes = noResultUploadTimeoutMinutes;
if (minutes <= 0) {
return;
}
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
.lt(FileTaskEntity::getCreatedAt, cutoff)
.orderByAsc(FileTaskEntity::getCreatedAt)
.last("limit 200"));
if (candidates.isEmpty()) {
return;
}
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
candidates.stream().map(FileTaskEntity::getId).toList())) {
if (dto.taskId() != null && dto.lastChunkAt() != null) {
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
}
}
for (FileTaskEntity task : candidates) {
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
if (lastResultAt == null) {
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成静默 continue,是 2626 那型出口。
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
log.info("[collect-data] never-reported skip because client still advancing taskId={} position={} minutes={}",
task.getId(), neverReportedPosition, minutes);
continue;
}
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
log.warn("[collect-data] never-reported skip without position evidence taskId={} createdAt={} minutes={}",
task.getId(), task.getCreatedAt(), minutes);
continue;
}
log.warn("[collect-data] never-reported but position stuck, failing taskId={} position={} minutes={}",
task.getId(), neverReportedPosition, minutes);
finalizeNoUploadStaleTask(task.getId(), null, neverReportedPosition, minutes);
continue;
}
if (lastResultAt.isAfter(cutoff)) {
continue;
}
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
continue;
}
finalizeNoUploadStaleTask(task.getId(), lastResultAt, null, minutes);
}
void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) {
staleTaskScanner().sweepNoUpload(heartbeatThreshold, this::finalizeNoUploadStale);
}
/** 单个任务的二次判死收尾:锁内复查心跳后,有分片→组装部分工作簿;无分片→标失败。 */
private void finalizeNoUploadStaleTask(Long taskId, LocalDateTime lastResultAt, String position, long minutes) {
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
if (lock == null) {
LockSkipWitness.recordAbandoned("collect-data:no-upload-stale-finalize", MODULE_TYPE, taskId);
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes));
if (task == null
|| !MODULE_TYPE.equals(task.getModuleType())
|| !STATUS_RUNNING.equals(task.getStatus())
|| task.getUpdatedAt() == null
|| task.getUpdatedAt().isBefore(heartbeatFreshAfter)) {
// 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理)
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
return;
}
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
if (hasReceivedChunks(taskId)) {
enqueueFinalWorkbook(task, result, stats);
log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}",
taskId, lastResultAt, minutes, stats.finalRowCount);
return;
}
markTaskFailed(task, result, noUploadFailReason(minutes, lastResultAt, position), stats);
log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}",
taskId, lastResultAt, minutes);
} catch (Exception ex) {
log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}",
taskId, ex.getMessage(), ex);
}
/**
* 每次入口新建一枚(不是 bean):两枚阈值是 {@code @Value} 注进本支的非 final 字段,
* 测试用 {@code ReflectionTestUtils.setField} 改它们——烤进更早构造的缓存就会读旧数。
*/
private CollectDataStaleTaskScanner staleTaskScanner() {
return new CollectDataStaleTaskScanner(MODULE_TYPE, staleTimeoutMinutes, noResultUploadTimeoutMinutes,
distributedJobLockService, fileTaskMapper, taskDistributedLockService, taskFileJobService,
taskScopeStateMapper, taskHeartbeatPositionService, staleTaskHeartbeatPolicy);
}
/** 二次判死原因文案;从未上报与已有上报分开写(后者不能只说「最后结果上报 null」)。 */
@@ -1524,10 +1399,6 @@ public class CollectDataService {
return normalize(value).toUpperCase(Locale.ROOT);
}
private String normalizeBrand(String value) {
return normalize(value).toLowerCase(Locale.ROOT);
}
/**
* 将 Python 回传的配送方式标准化为 FBA / FBM / AMZ;
* 空值或未识别值统一返回空字符串,由前端 / Excel 端展示为“无配送方式”。
@@ -0,0 +1,286 @@
package com.nanri.aiimage.modules.collectdata.service.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.task.TaskStatuses;
import com.nanri.aiimage.common.util.LockSkipWitness;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import lombok.extern.slf4j.Slf4j;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 采集数据陈旧任务扫描器:心跳线({@code updated_at} 超期)+二次判死线(心跳新鲜但连续无结果分片)。
*
* <p><b>搬的是「扫描+判定」,不是「收尾」。</b>本类决定一个任务该不该死——取锁、锁内复查、
* 首次心跳排队宽限、未完成的组装 Job 闸门、以及二次线那套「从未上报」的三分证据判定;
* 而「死了要做什么」留在 {@code CollectDataService}(要动结果行、chunk 计数、部分工作簿与
* resultJson 统计,那些件都还在服务里)。判完通过 {@link HeartbeatStaleFinalizer} 与
* {@link NoUploadStaleFinalizer} 两枚回调交回去,与 {@code AppearancePatentStaleTaskScanner}
* 同一形状——把收尾一起拖出来就得给结果行读取、
* 工作簿组装与统计再套一套构造器,那是刚结清过的那种亏。
*
* <p><b>两条线刻意不合并。</b>心跳线判「{@code updated_at} 陈旧」,二次线判「心跳新鲜但
* {@code last_chunk_at} 久未变」——同一条任务在两条线上的期望相反,合并判定必有一边失明
* (2626 那型:从未上报的行与上报仍新鲜的行被并成一句静默 continue,假活任务杀不掉)。
*
* <p><b>入参为什么逐枚交进来而不是注一个 properties。</b>{@code staleTimeoutMinutes} 与
* {@code noResultUploadTimeoutMinutes} 在服务侧是 {@code @Value} 注入的非 final 字段,
* 测试用 {@code ReflectionTestUtils.setField} 改它们;本类由服务每次入口调用时新建,
* 读的就是当时那一枚值——把阈值烤进构造更早的缓存里,那三枚既有用例当场就会读旧数。
*
* <p>{@code moduleType()} 由调用方交出(本支是 {@code COLLECT_DATA}),状态字面量一律走
* {@link TaskStatuses}:{@code src/main} 里手写的 {@code STATUS_XXX} 标量是只许降的棘轮,
* 降法就是接进这里。
*/
@Slf4j
public class CollectDataStaleTaskScanner {
/** 双节点互斥的作业锁名,逐字沿用服务侧现网那句(改名等于换一把锁)。 */
private static final String STALE_CHECK_JOB_LOCK = "collect-data:stale-check";
/** 每轮作业锁的持有时长;Redis 不可用时上层 fail-open,兜底靠 per-task DB 锁。 */
private static final Duration STALE_CHECK_JOB_LOCK_TTL = Duration.ofMinutes(2);
/** 单轮候选上限,逐字沿用现网那句 {@code limit 200}。 */
private static final String CANDIDATE_LIMIT = "limit 200";
/** 本模块没有自己的 Redis 心跳读数,闸门按「没有首心跳记录」这一形传 0。 */
private static final long NO_REDIS_HEARTBEAT_READING = 0L;
/**
* 判死之后的收尾归服务。两条线各一枚接缝而不是一个双方法接口——本类的 {@link #sweep}
* 要在心跳线之后接着跑二次线,两枚接缝分开交,服务侧那两枚私有方法才能原样留私有
* (合成一枚双方法接口就得让服务 implements 它,那两枚方法当场被迫变 public,白扩对外面)。
*/
@FunctionalInterface
public interface HeartbeatStaleFinalizer {
/** 心跳线判死:阈值与超时分钟数已算好,收尾按「有无已收分片」分两条路走。 */
void finalizeHeartbeatStale(FileTaskEntity task, long timeoutMinutes);
}
@FunctionalInterface
public interface NoUploadStaleFinalizer {
/**
* 二次判死线收尾。{@code lastResultAt} 为 null 表示从未上报过结果分片,
* 此时 {@code position} 是唯一的卡死证据(心跳位置文本),两者恰好一个非 null 是现网形状。
*/
void finalizeNoUploadStale(FileTaskEntity task, LocalDateTime lastResultAt, String position, long minutes);
}
private final String moduleType;
private final long staleTimeoutMinutes;
private final long noResultUploadTimeoutMinutes;
private final DistributedJobLockService distributedJobLockService;
private final FileTaskMapper fileTaskMapper;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService;
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
public CollectDataStaleTaskScanner(String moduleType,
long staleTimeoutMinutes,
long noResultUploadTimeoutMinutes,
DistributedJobLockService distributedJobLockService,
FileTaskMapper fileTaskMapper,
TaskDistributedLockService taskDistributedLockService,
TaskFileJobService taskFileJobService,
TaskScopeStateMapper taskScopeStateMapper,
TaskHeartbeatPositionService taskHeartbeatPositionService,
StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy) {
this.moduleType = moduleType;
this.staleTimeoutMinutes = staleTimeoutMinutes;
this.noResultUploadTimeoutMinutes = noResultUploadTimeoutMinutes;
this.distributedJobLockService = distributedJobLockService;
this.fileTaskMapper = fileTaskMapper;
this.taskDistributedLockService = taskDistributedLockService;
this.taskFileJobService = taskFileJobService;
this.taskScopeStateMapper = taskScopeStateMapper;
this.taskHeartbeatPositionService = taskHeartbeatPositionService;
this.staleTaskHeartbeatPolicy = staleTaskHeartbeatPolicy;
}
/**
* 一轮完整扫描:先跑心跳线,再在同一把作业锁内跑二次判死线(现网就是这个次序,
* 且二次线拿的是心跳线那枚 {@code threshold})。
*/
public void sweep(HeartbeatStaleFinalizer heartbeat, NoUploadStaleFinalizer noUpload) {
// 双节点互斥:Redis 不可用时 fail-open(仅记日志),兜底靠下方 per-task DB 锁,不会双写终态
DistributedJobLockService.LockHandle jobLock =
distributedJobLockService.tryLock(STALE_CHECK_JOB_LOCK, STALE_CHECK_JOB_LOCK_TTL);
if (jobLock == null) {
log.info("[collect-data] stale-check skipped, another instance holds the distributed lock");
return;
}
try (jobLock) {
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
for (FileTaskEntity task : listHeartbeatStaleCandidates(threshold)) {
finalizeHeartbeatStale(task.getId(), threshold, timeoutMinutes, heartbeat);
}
sweepNoUpload(threshold, noUpload);
}
}
private List<FileTaskEntity> listHeartbeatStaleCandidates(LocalDateTime threshold) {
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, moduleType)
.eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)
.lt(FileTaskEntity::getUpdatedAt, threshold)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last(CANDIDATE_LIMIT));
}
/**
* 心跳线单任务:锁内复查 ⇒ 排队宽限 ⇒ 未完成组装 Job 闸门 ⇒ 交回收尾。
* 每一格都是「放过、不判死」的形状,判据与顺序逐字保持现网写法。
*/
private void finalizeHeartbeatStale(Long taskId,
LocalDateTime threshold,
long timeoutMinutes,
HeartbeatStaleFinalizer finalizer) {
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(moduleType, taskId, 0L)) {
if (lock == null) {
LockSkipWitness.recordAbandoned("collect-data:stale-finalize", moduleType, taskId);
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null
|| !moduleType.equals(task.getModuleType())
|| !TaskStatuses.FileTask.RUNNING.equals(task.getStatus())
|| task.getUpdatedAt() == null
|| !task.getUpdatedAt().isBefore(threshold)) {
return;
}
// 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
taskId, NO_REDIS_HEARTBEAT_READING, task.getUpdatedAt(), LocalDateTime.now())) {
log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(),
task.getUpdatedAt());
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, moduleType) > 0L) {
return;
}
finalizer.finalizeHeartbeatStale(task, timeoutMinutes);
} catch (Exception ex) {
log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
taskId, ex.getMessage(), ex);
}
}
/**
* 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
*
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。
* 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用
* biz_task_scope_state.last_chunk_at(只随结果分片上报刷新);
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
*/
public void sweepNoUpload(LocalDateTime heartbeatThreshold, NoUploadStaleFinalizer finalizer) {
long minutes = noResultUploadTimeoutMinutes;
if (minutes <= 0) {
return;
}
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, moduleType)
.eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
.lt(FileTaskEntity::getCreatedAt, cutoff)
.orderByAsc(FileTaskEntity::getCreatedAt)
.last(CANDIDATE_LIMIT));
if (candidates.isEmpty()) {
return;
}
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
candidates.stream().map(FileTaskEntity::getId).toList())) {
if (dto.taskId() != null && dto.lastChunkAt() != null) {
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
}
}
for (FileTaskEntity task : candidates) {
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
if (lastResultAt == null) {
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成静默 continue,是 2626 那型出口。
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
log.info("[collect-data] never-reported skip because client still advancing taskId={} position={} minutes={}",
task.getId(), neverReportedPosition, minutes);
continue;
}
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
log.warn("[collect-data] never-reported skip without position evidence taskId={} createdAt={} minutes={}",
task.getId(), task.getCreatedAt(), minutes);
continue;
}
log.warn("[collect-data] never-reported but position stuck, failing taskId={} position={} minutes={}",
task.getId(), neverReportedPosition, minutes);
finalizeNoUploadStale(task.getId(), null, neverReportedPosition, minutes, finalizer);
continue;
}
if (lastResultAt.isAfter(cutoff)) {
continue;
}
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
continue;
}
finalizeNoUploadStale(task.getId(), lastResultAt, null, minutes, finalizer);
}
}
/** 二次线单任务:锁内复查心跳是否仍新鲜、是否还有未完成的组装 Job,然后才交回收尾。 */
private void finalizeNoUploadStale(Long taskId,
LocalDateTime lastResultAt,
String position,
long minutes,
NoUploadStaleFinalizer finalizer) {
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(moduleType, taskId, 0L)) {
if (lock == null) {
LockSkipWitness.recordAbandoned("collect-data:no-upload-stale-finalize", moduleType, taskId);
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes));
if (task == null
|| !moduleType.equals(task.getModuleType())
|| !TaskStatuses.FileTask.RUNNING.equals(task.getStatus())
|| task.getUpdatedAt() == null
|| task.getUpdatedAt().isBefore(heartbeatFreshAfter)) {
// 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理)
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, moduleType) > 0L) {
return;
}
finalizer.finalizeNoUploadStale(task, lastResultAt, position, minutes);
} catch (Exception ex) {
log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}",
taskId, ex.getMessage(), ex);
}
}
}