refactor(phase-3): 3C 拆薄第一笔——A 陈旧扫描聚类抽成 support 件,服务棘轮 2209→2080
把 CollectDataService 的心跳线与二次判死线的「扫描+判定」搬到 modules/collectdata/service/support/CollectDataStaleTaskScanner(287 行),收尾 (ensureTaskResult / loadStats / hasReceivedChunks / enqueueFinalWorkbook / markTaskFailed)留在服务侧经两枚 @FunctionalInterface 回调交回——与 AppearancePatentStaleTaskScanner 同一形状。判据、步序、日志文案逐字保持; 状态字面量改走 TaskStatuses.FileTask.RUNNING;两处 LockSkipWitness 点位仍写死 字面量在调用处(护栏尺不许常量)。服务侧留 finalizeNoUploadStaleTasks 薄委托, 因 CollectDataNoUploadStaleTest 按名字反射点名它。 目的是给 3C 迁移的 29 枚显式构造器(≈+47 行)腾出「同笔抵平」额度: 本笔净减 129 行,基线一格未改。 门禁:定向 349 例/0 红/2 skipped;后端全量 4528 例/0 红/14 skipped (较上一读数 +10 = 3E/3C 两支新 golden 各 5 枚 @Test);ops ruff All checks passed、readability 棘轮 9 passed。
This commit is contained in:
+41
-170
@@ -12,7 +12,6 @@ import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.common.util.LockSkipWitness;
|
||||
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
@@ -37,6 +36,7 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
|
||||
import com.nanri.aiimage.modules.collectdata.service.support.CollectDataStaleTaskScanner;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
|
||||
@@ -55,7 +55,6 @@ import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
@@ -565,179 +564,55 @@ public class CollectDataService {
|
||||
|
||||
@Scheduled(cron = "${aiimage.collect-data.stale-check-cron:0 */2 * * * *}")
|
||||
public void finalizeStaleTasks() {
|
||||
// 双节点互斥:Redis 不可用时 fail-open(仅记日志),兜底靠下方 per-task DB 锁,不会双写终态
|
||||
DistributedJobLockService.LockHandle jobLock =
|
||||
distributedJobLockService.tryLock("collect-data:stale-check", java.time.Duration.ofMinutes(2));
|
||||
if (jobLock == null) {
|
||||
log.info("[collect-data] stale-check skipped, another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (jobLock) {
|
||||
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
|
||||
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 200"));
|
||||
for (FileTaskEntity task : tasks) {
|
||||
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
|
||||
}
|
||||
finalizeNoUploadStaleTasks(threshold);
|
||||
}
|
||||
staleTaskScanner().sweep(this::finalizeHeartbeatStale, this::finalizeNoUploadStale);
|
||||
}
|
||||
|
||||
private void finalizeStaleTask(Long taskId, LocalDateTime threshold, long timeoutMinutes) {
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
|
||||
if (lock == null) {
|
||||
LockSkipWitness.recordAbandoned("collect-data:stale-finalize", MODULE_TYPE, taskId);
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null
|
||||
|| !MODULE_TYPE.equals(task.getModuleType())
|
||||
|| !STATUS_RUNNING.equals(task.getStatus())
|
||||
|| task.getUpdatedAt() == null
|
||||
|| !task.getUpdatedAt().isBefore(threshold)) {
|
||||
return;
|
||||
}
|
||||
// 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点
|
||||
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
taskId, 0L, task.getUpdatedAt(), LocalDateTime.now())) {
|
||||
log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
|
||||
taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(),
|
||||
task.getUpdatedAt());
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
if (hasReceivedChunks(taskId)) {
|
||||
enqueueFinalWorkbook(task, result, stats);
|
||||
log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
|
||||
taskId, timeoutMinutes, stats.finalRowCount);
|
||||
return;
|
||||
}
|
||||
|
||||
markTaskFailed(task, result, STALE_TASK_ERROR, stats);
|
||||
log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
|
||||
taskId, timeoutMinutes);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
|
||||
taskId, ex.getMessage(), ex);
|
||||
/** 心跳线判死的收尾:已收到分片 ⇒ 挂部分工作簿;一片都没收到 ⇒ 直接标失败。 */
|
||||
private void finalizeHeartbeatStale(FileTaskEntity task, long timeoutMinutes) {
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
if (hasReceivedChunks(task.getId())) {
|
||||
enqueueFinalWorkbook(task, result, stats);
|
||||
log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
|
||||
task.getId(), timeoutMinutes, stats.finalRowCount);
|
||||
return;
|
||||
}
|
||||
markTaskFailed(task, result, STALE_TASK_ERROR, stats);
|
||||
log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
|
||||
task.getId(), timeoutMinutes);
|
||||
}
|
||||
|
||||
/** 二次判死线的收尾(扫描与判定在 {@code CollectDataStaleTaskScanner}):文案与心跳线分开写。 */
|
||||
private void finalizeNoUploadStale(FileTaskEntity task, LocalDateTime lastResultAt, String position, long minutes) {
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
if (hasReceivedChunks(task.getId())) {
|
||||
enqueueFinalWorkbook(task, result, stats);
|
||||
log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}",
|
||||
task.getId(), lastResultAt, minutes, stats.finalRowCount);
|
||||
return;
|
||||
}
|
||||
markTaskFailed(task, result, noUploadFailReason(minutes, lastResultAt, position), stats);
|
||||
log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}",
|
||||
task.getId(), lastResultAt, minutes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
|
||||
*
|
||||
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。
|
||||
* 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用
|
||||
* biz_task_scope_state.last_chunk_at(只随结果分片上报刷新);
|
||||
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||
* 这个入口的存在理由是测试:{@code CollectDataNoUploadStaleTest} 按名字反射点名它,
|
||||
* 扫描与判定本体在 {@code CollectDataStaleTaskScanner#sweepNoUpload}。
|
||||
*/
|
||||
private void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) {
|
||||
long minutes = noResultUploadTimeoutMinutes;
|
||||
if (minutes <= 0) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
|
||||
.lt(FileTaskEntity::getCreatedAt, cutoff)
|
||||
.orderByAsc(FileTaskEntity::getCreatedAt)
|
||||
.last("limit 200"));
|
||||
if (candidates.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
|
||||
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||
candidates.stream().map(FileTaskEntity::getId).toList())) {
|
||||
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||
}
|
||||
}
|
||||
for (FileTaskEntity task : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成静默 continue,是 2626 那型出口。
|
||||
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[collect-data] never-reported skip because client still advancing taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[collect-data] never-reported skip without position evidence taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
log.warn("[collect-data] never-reported but position stuck, failing taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
finalizeNoUploadStaleTask(task.getId(), null, neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
finalizeNoUploadStaleTask(task.getId(), lastResultAt, null, minutes);
|
||||
}
|
||||
void finalizeNoUploadStaleTasks(LocalDateTime heartbeatThreshold) {
|
||||
staleTaskScanner().sweepNoUpload(heartbeatThreshold, this::finalizeNoUploadStale);
|
||||
}
|
||||
|
||||
/** 单个任务的二次判死收尾:锁内复查心跳后,有分片→组装部分工作簿;无分片→标失败。 */
|
||||
private void finalizeNoUploadStaleTask(Long taskId, LocalDateTime lastResultAt, String position, long minutes) {
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
|
||||
if (lock == null) {
|
||||
LockSkipWitness.recordAbandoned("collect-data:no-upload-stale-finalize", MODULE_TYPE, taskId);
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes));
|
||||
if (task == null
|
||||
|| !MODULE_TYPE.equals(task.getModuleType())
|
||||
|| !STATUS_RUNNING.equals(task.getStatus())
|
||||
|| task.getUpdatedAt() == null
|
||||
|| task.getUpdatedAt().isBefore(heartbeatFreshAfter)) {
|
||||
// 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理)
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
if (hasReceivedChunks(taskId)) {
|
||||
enqueueFinalWorkbook(task, result, stats);
|
||||
log.warn("[collect-data] no-upload stale task enqueued partial workbook taskId={} lastResultAt={} timeoutMinutes={} finalRows={}",
|
||||
taskId, lastResultAt, minutes, stats.finalRowCount);
|
||||
return;
|
||||
}
|
||||
markTaskFailed(task, result, noUploadFailReason(minutes, lastResultAt, position), stats);
|
||||
log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}",
|
||||
taskId, lastResultAt, minutes);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}",
|
||||
taskId, ex.getMessage(), ex);
|
||||
}
|
||||
/**
|
||||
* 每次入口新建一枚(不是 bean):两枚阈值是 {@code @Value} 注进本支的非 final 字段,
|
||||
* 测试用 {@code ReflectionTestUtils.setField} 改它们——烤进更早构造的缓存就会读旧数。
|
||||
*/
|
||||
private CollectDataStaleTaskScanner staleTaskScanner() {
|
||||
return new CollectDataStaleTaskScanner(MODULE_TYPE, staleTimeoutMinutes, noResultUploadTimeoutMinutes,
|
||||
distributedJobLockService, fileTaskMapper, taskDistributedLockService, taskFileJobService,
|
||||
taskScopeStateMapper, taskHeartbeatPositionService, staleTaskHeartbeatPolicy);
|
||||
}
|
||||
|
||||
/** 二次判死原因文案;从未上报与已有上报分开写(后者不能只说「最后结果上报 null」)。 */
|
||||
@@ -1524,10 +1399,6 @@ public class CollectDataService {
|
||||
return normalize(value).toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String normalizeBrand(String value) {
|
||||
return normalize(value).toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 Python 回传的配送方式标准化为 FBA / FBM / AMZ;
|
||||
* 空值或未识别值统一返回空字符串,由前端 / Excel 端展示为“无配送方式”。
|
||||
|
||||
+286
@@ -0,0 +1,286 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.common.util.LockSkipWitness;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 采集数据陈旧任务扫描器:心跳线({@code updated_at} 超期)+二次判死线(心跳新鲜但连续无结果分片)。
|
||||
*
|
||||
* <p><b>搬的是「扫描+判定」,不是「收尾」。</b>本类决定一个任务该不该死——取锁、锁内复查、
|
||||
* 首次心跳排队宽限、未完成的组装 Job 闸门、以及二次线那套「从未上报」的三分证据判定;
|
||||
* 而「死了要做什么」留在 {@code CollectDataService}(要动结果行、chunk 计数、部分工作簿与
|
||||
* resultJson 统计,那些件都还在服务里)。判完通过 {@link HeartbeatStaleFinalizer} 与
|
||||
* {@link NoUploadStaleFinalizer} 两枚回调交回去,与 {@code AppearancePatentStaleTaskScanner}
|
||||
* 同一形状——把收尾一起拖出来就得给结果行读取、
|
||||
* 工作簿组装与统计再套一套构造器,那是刚结清过的那种亏。
|
||||
*
|
||||
* <p><b>两条线刻意不合并。</b>心跳线判「{@code updated_at} 陈旧」,二次线判「心跳新鲜但
|
||||
* {@code last_chunk_at} 久未变」——同一条任务在两条线上的期望相反,合并判定必有一边失明
|
||||
* (2626 那型:从未上报的行与上报仍新鲜的行被并成一句静默 continue,假活任务杀不掉)。
|
||||
*
|
||||
* <p><b>入参为什么逐枚交进来而不是注一个 properties。</b>{@code staleTimeoutMinutes} 与
|
||||
* {@code noResultUploadTimeoutMinutes} 在服务侧是 {@code @Value} 注入的非 final 字段,
|
||||
* 测试用 {@code ReflectionTestUtils.setField} 改它们;本类由服务每次入口调用时新建,
|
||||
* 读的就是当时那一枚值——把阈值烤进构造更早的缓存里,那三枚既有用例当场就会读旧数。
|
||||
*
|
||||
* <p>{@code moduleType()} 由调用方交出(本支是 {@code COLLECT_DATA}),状态字面量一律走
|
||||
* {@link TaskStatuses}:{@code src/main} 里手写的 {@code STATUS_XXX} 标量是只许降的棘轮,
|
||||
* 降法就是接进这里。
|
||||
*/
|
||||
@Slf4j
|
||||
public class CollectDataStaleTaskScanner {
|
||||
|
||||
/** 双节点互斥的作业锁名,逐字沿用服务侧现网那句(改名等于换一把锁)。 */
|
||||
private static final String STALE_CHECK_JOB_LOCK = "collect-data:stale-check";
|
||||
|
||||
/** 每轮作业锁的持有时长;Redis 不可用时上层 fail-open,兜底靠 per-task DB 锁。 */
|
||||
private static final Duration STALE_CHECK_JOB_LOCK_TTL = Duration.ofMinutes(2);
|
||||
|
||||
/** 单轮候选上限,逐字沿用现网那句 {@code limit 200}。 */
|
||||
private static final String CANDIDATE_LIMIT = "limit 200";
|
||||
|
||||
/** 本模块没有自己的 Redis 心跳读数,闸门按「没有首心跳记录」这一形传 0。 */
|
||||
private static final long NO_REDIS_HEARTBEAT_READING = 0L;
|
||||
|
||||
/**
|
||||
* 判死之后的收尾归服务。两条线各一枚接缝而不是一个双方法接口——本类的 {@link #sweep}
|
||||
* 要在心跳线之后接着跑二次线,两枚接缝分开交,服务侧那两枚私有方法才能原样留私有
|
||||
* (合成一枚双方法接口就得让服务 implements 它,那两枚方法当场被迫变 public,白扩对外面)。
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface HeartbeatStaleFinalizer {
|
||||
/** 心跳线判死:阈值与超时分钟数已算好,收尾按「有无已收分片」分两条路走。 */
|
||||
void finalizeHeartbeatStale(FileTaskEntity task, long timeoutMinutes);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface NoUploadStaleFinalizer {
|
||||
/**
|
||||
* 二次判死线收尾。{@code lastResultAt} 为 null 表示从未上报过结果分片,
|
||||
* 此时 {@code position} 是唯一的卡死证据(心跳位置文本),两者恰好一个非 null 是现网形状。
|
||||
*/
|
||||
void finalizeNoUploadStale(FileTaskEntity task, LocalDateTime lastResultAt, String position, long minutes);
|
||||
}
|
||||
|
||||
private final String moduleType;
|
||||
private final long staleTimeoutMinutes;
|
||||
private final long noResultUploadTimeoutMinutes;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
|
||||
public CollectDataStaleTaskScanner(String moduleType,
|
||||
long staleTimeoutMinutes,
|
||||
long noResultUploadTimeoutMinutes,
|
||||
DistributedJobLockService distributedJobLockService,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
TaskDistributedLockService taskDistributedLockService,
|
||||
TaskFileJobService taskFileJobService,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
TaskHeartbeatPositionService taskHeartbeatPositionService,
|
||||
StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy) {
|
||||
this.moduleType = moduleType;
|
||||
this.staleTimeoutMinutes = staleTimeoutMinutes;
|
||||
this.noResultUploadTimeoutMinutes = noResultUploadTimeoutMinutes;
|
||||
this.distributedJobLockService = distributedJobLockService;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskDistributedLockService = taskDistributedLockService;
|
||||
this.taskFileJobService = taskFileJobService;
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.taskHeartbeatPositionService = taskHeartbeatPositionService;
|
||||
this.staleTaskHeartbeatPolicy = staleTaskHeartbeatPolicy;
|
||||
}
|
||||
|
||||
/**
|
||||
* 一轮完整扫描:先跑心跳线,再在同一把作业锁内跑二次判死线(现网就是这个次序,
|
||||
* 且二次线拿的是心跳线那枚 {@code threshold})。
|
||||
*/
|
||||
public void sweep(HeartbeatStaleFinalizer heartbeat, NoUploadStaleFinalizer noUpload) {
|
||||
// 双节点互斥:Redis 不可用时 fail-open(仅记日志),兜底靠下方 per-task DB 锁,不会双写终态
|
||||
DistributedJobLockService.LockHandle jobLock =
|
||||
distributedJobLockService.tryLock(STALE_CHECK_JOB_LOCK, STALE_CHECK_JOB_LOCK_TTL);
|
||||
if (jobLock == null) {
|
||||
log.info("[collect-data] stale-check skipped, another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (jobLock) {
|
||||
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
|
||||
for (FileTaskEntity task : listHeartbeatStaleCandidates(threshold)) {
|
||||
finalizeHeartbeatStale(task.getId(), threshold, timeoutMinutes, heartbeat);
|
||||
}
|
||||
sweepNoUpload(threshold, noUpload);
|
||||
}
|
||||
}
|
||||
|
||||
private List<FileTaskEntity> listHeartbeatStaleCandidates(LocalDateTime threshold) {
|
||||
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, moduleType)
|
||||
.eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last(CANDIDATE_LIMIT));
|
||||
}
|
||||
|
||||
/**
|
||||
* 心跳线单任务:锁内复查 ⇒ 排队宽限 ⇒ 未完成组装 Job 闸门 ⇒ 交回收尾。
|
||||
* 每一格都是「放过、不判死」的形状,判据与顺序逐字保持现网写法。
|
||||
*/
|
||||
private void finalizeHeartbeatStale(Long taskId,
|
||||
LocalDateTime threshold,
|
||||
long timeoutMinutes,
|
||||
HeartbeatStaleFinalizer finalizer) {
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(moduleType, taskId, 0L)) {
|
||||
if (lock == null) {
|
||||
LockSkipWitness.recordAbandoned("collect-data:stale-finalize", moduleType, taskId);
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null
|
||||
|| !moduleType.equals(task.getModuleType())
|
||||
|| !TaskStatuses.FileTask.RUNNING.equals(task.getStatus())
|
||||
|| task.getUpdatedAt() == null
|
||||
|| !task.getUpdatedAt().isBefore(threshold)) {
|
||||
return;
|
||||
}
|
||||
// 排队宽限内暂不判死:这个模块没有自己的 Redis 心跳读数(传 0),闸门改读共享心跳打点
|
||||
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
taskId, NO_REDIS_HEARTBEAT_READING, task.getUpdatedAt(), LocalDateTime.now())) {
|
||||
log.info("[collect-data] 未收到首次心跳、排队宽限内暂不判死 taskId={} timeoutMinutes={} graceMinutes={} updatedAt={}",
|
||||
taskId, timeoutMinutes, staleTaskHeartbeatPolicy.firstHeartbeatGraceMinutes(),
|
||||
task.getUpdatedAt());
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, moduleType) > 0L) {
|
||||
return;
|
||||
}
|
||||
finalizer.finalizeHeartbeatStale(task, timeoutMinutes);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
|
||||
taskId, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 二次判死:心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。
|
||||
*
|
||||
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中。
|
||||
* 本线候选取「心跳新鲜(既有线放过)+ 创建超过 N 分钟」,判据用
|
||||
* biz_task_scope_state.last_chunk_at(只随结果分片上报刷新);
|
||||
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||
*/
|
||||
public void sweepNoUpload(LocalDateTime heartbeatThreshold, NoUploadStaleFinalizer finalizer) {
|
||||
long minutes = noResultUploadTimeoutMinutes;
|
||||
if (minutes <= 0) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, moduleType)
|
||||
.eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)
|
||||
.ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold)
|
||||
.lt(FileTaskEntity::getCreatedAt, cutoff)
|
||||
.orderByAsc(FileTaskEntity::getCreatedAt)
|
||||
.last(CANDIDATE_LIMIT));
|
||||
if (candidates.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
|
||||
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||
candidates.stream().map(FileTaskEntity::getId).toList())) {
|
||||
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||
}
|
||||
}
|
||||
for (FileTaskEntity task : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成静默 continue,是 2626 那型出口。
|
||||
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[collect-data] never-reported skip because client still advancing taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[collect-data] never-reported skip without position evidence taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
log.warn("[collect-data] never-reported but position stuck, failing taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
finalizeNoUploadStale(task.getId(), null, neverReportedPosition, minutes, finalizer);
|
||||
continue;
|
||||
}
|
||||
if (lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
finalizeNoUploadStale(task.getId(), lastResultAt, null, minutes, finalizer);
|
||||
}
|
||||
}
|
||||
|
||||
/** 二次线单任务:锁内复查心跳是否仍新鲜、是否还有未完成的组装 Job,然后才交回收尾。 */
|
||||
private void finalizeNoUploadStale(Long taskId,
|
||||
LocalDateTime lastResultAt,
|
||||
String position,
|
||||
long minutes,
|
||||
NoUploadStaleFinalizer finalizer) {
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(moduleType, taskId, 0L)) {
|
||||
if (lock == null) {
|
||||
LockSkipWitness.recordAbandoned("collect-data:no-upload-stale-finalize", moduleType, taskId);
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
LocalDateTime heartbeatFreshAfter = LocalDateTime.now().minusMinutes(Math.max(1L, staleTimeoutMinutes));
|
||||
if (task == null
|
||||
|| !moduleType.equals(task.getModuleType())
|
||||
|| !TaskStatuses.FileTask.RUNNING.equals(task.getStatus())
|
||||
|| task.getUpdatedAt() == null
|
||||
|| task.getUpdatedAt().isBefore(heartbeatFreshAfter)) {
|
||||
// 已终态、或心跳在排队期间回落到陈旧(交回既有心跳线处理)
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, moduleType) > 0L) {
|
||||
return;
|
||||
}
|
||||
finalizer.finalizeNoUploadStale(task, lastResultAt, position, minutes);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] no-upload stale task finalization failed taskId={} msg={}",
|
||||
taskId, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user