feat(跟价): 循环轮次卡死兜底回收,挂 stale-check 巡检线
跟价轮次的推进(下一店/下一轮/收尾)全部由前端页面驱动,页面一关就再没有 任何东西推进它,轮次永久停在 RUNNING。生产库存量 146 条 RUNNING 中 125 条 是这种僵尸,最早的可追溯到 4 月。 判据刻意不落在 loop_run.updated_at:子任务长跑期间轮次表根本不被刷新 (心跳只写 biz_file_task 与 Redis),正在正常跟价的轮次同样表现为 updated_at 陈旧,按它判死会误杀长任务。改为落在 active_task_id 指向的子任务上: active 为空超阈值 / 子任务行不存在 → 判失败;子任务已终态但轮次没推进 → 复用 syncLoopRunAfterChildTerminal 补推进;子任务仍在跑则交给跟价子任务判死线。 用户已请求停止的一律落 STOPPED,与 dispatchNext 既有语义一致。 刻意不做自动续跑:跟价是写操作(会真实改价),让用户早已关掉页面的轮次 自己跑起来风险不可接受。
This commit is contained in:
+9
-1
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackLoopRunStaleService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
@@ -75,6 +76,8 @@ public class DeleteBrandStaleTaskService {
|
||||
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||
private final PriceTrackTaskService priceTrackTaskService;
|
||||
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||
/** 跟价循环批次(biz_price_track_loop_run)的卡死轮次兜底回收。 */
|
||||
private final PriceTrackLoopRunStaleService priceTrackLoopRunStaleService;
|
||||
private final ShopMatchTaskService shopMatchTaskService;
|
||||
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
|
||||
private final PatrolDeleteTaskService patrolDeleteTaskService;
|
||||
@@ -112,6 +115,10 @@ public class DeleteBrandStaleTaskService {
|
||||
failStaleDeleteBrandTasks();
|
||||
ProductRiskStaleCheckStats stats = failStaleProductRiskResolveTasks();
|
||||
PriceTrackStaleCheckStats priceTrackStats = failStalePriceTrackTasks();
|
||||
// 跟价轮次(父实体)的卡死回收:上面那条只覆盖 biz_file_task 子任务,
|
||||
// 轮次本身停 RUNNING 无人管(前端页面一关就再没有任何驱动)
|
||||
PriceTrackLoopRunStaleService.StaleLoopRunStats priceTrackLoopStats =
|
||||
priceTrackLoopRunStaleService.failStaleLoopRuns();
|
||||
ShopMatchStaleCheckStats shopMatchStats = failStaleShopMatchTasks();
|
||||
ShopMatchStaleCheckStats patrolDeleteStats = failStalePatrolDeleteTasks();
|
||||
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
||||
@@ -126,9 +133,10 @@ public class DeleteBrandStaleTaskService {
|
||||
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
||||
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
||||
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
||||
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) no-upload(c={} f={} x={}) resume(s={} r={} k={} u={}) elapsedMs={} thread={}",
|
||||
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) price-track-loop(s={} f={} p={} r={} x={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) no-upload(c={} f={} x={}) resume(s={} r={} k={} u={}) elapsedMs={} thread={}",
|
||||
stats.scannedTaskCount, stats.finalizedTaskCount, stats.failedTaskCount, stats.skippedTaskCount,
|
||||
priceTrackStats.scannedTaskCount, priceTrackStats.finalizedTaskCount, priceTrackStats.failedTaskCount, priceTrackStats.skippedTaskCount,
|
||||
priceTrackLoopStats.scannedLoopRunCount, priceTrackLoopStats.failedLoopRunCount, priceTrackLoopStats.stoppedLoopRunCount, priceTrackLoopStats.finalizedLoopRunCount, priceTrackLoopStats.skippedLoopRunCount,
|
||||
shopMatchStats.scannedTaskCount, shopMatchStats.finalizedTaskCount, shopMatchStats.failedTaskCount, shopMatchStats.skippedTaskCount,
|
||||
patrolDeleteStats.scannedTaskCount, patrolDeleteStats.finalizedTaskCount, patrolDeleteStats.failedTaskCount, patrolDeleteStats.skippedTaskCount,
|
||||
queryAsinStats.scannedTaskCount, queryAsinStats.finalizedTaskCount, queryAsinStats.failedTaskCount, queryAsinStats.skippedTaskCount,
|
||||
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.modules.pricetrack.mapper.PriceTrackLoopRunMapper;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackLoopRunEntity;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 跟价循环批次(biz_price_track_loop_run)「卡死轮次」的兜底回收。
|
||||
*
|
||||
* <p><b>为什么需要</b>:轮次的推进(下一店 / 下一轮 / 收尾)全部由前端页面驱动——
|
||||
* 前端在 while 循环里调 dispatch-next 拿子任务、轮询子任务结束后调 child-finished。
|
||||
* 页面一关这个驱动就没了,轮次会永久停在 RUNNING:既没有子任务在跑,也没有任何人
|
||||
* 来推进它。生产库 2026-09-22 存量 146 条 RUNNING,其中 125 条是这种僵尸,
|
||||
* 最早的可追溯到 4 月。
|
||||
*
|
||||
* <p><b>关键约束:不能拿 loop_run.updated_at 当判据</b>。子任务长跑期间没有任何东西
|
||||
* 会刷新轮次表(心跳只写 biz_file_task 与 Redis 的 price-track:task:heartbeat),
|
||||
* 所以一个正在正常跟价、子任务已跑了 36 小时的轮次,它的 updated_at 同样停在
|
||||
* 36 小时前——按 updated_at 判死会误杀长任务。判据必须落在 active_task_id 指向的
|
||||
* 那条子任务上。
|
||||
*
|
||||
* <p><b>处置分四种</b>:
|
||||
* <ul>
|
||||
* <li>用户已请求停止(stop_requested=1)→ STOPPED,尊重停止意图(与 dispatchNext /
|
||||
* handleChildFinished 的既有语义一致)</li>
|
||||
* <li>active_task_id 为空且超过阈值 → FAILED,前端已停止派发下一店</li>
|
||||
* <li>active_task_id 指向的子任务行已不存在 → FAILED,事件链断了(用户回来也会在
|
||||
* child-finished 报「子任务不存在」)</li>
|
||||
* <li>子任务已终态但轮次没推进 → 补推进,复用 {@link PriceTrackLoopRunService#syncLoopRunAfterChildTerminal}
|
||||
* 把前端漏掉的那次 child-finished 补上</li>
|
||||
* </ul>
|
||||
* 子任务仍在 RUNNING 的轮次一律跳过:子任务自身由跟价子任务判死线
|
||||
* (DeleteBrandStaleTaskService.failStalePriceTrackTasks)负责,它转终态后本服务
|
||||
* 下一轮自然接住。
|
||||
*
|
||||
* <p><b>刻意不做自动续跑</b>:跟价是写操作(会真实改价),让一个用户早就关掉页面的
|
||||
* 轮次自己重新跑起来,风险不可接受。这也与 TaskResumeService 刻意排除写操作模块的
|
||||
* 原则一致。
|
||||
*
|
||||
* <p>由 DeleteBrandStaleTaskService 的 stale-check 巡检线每 2 分钟调用一次
|
||||
* (沿用 P1-8 约定:委派式判死统一挂该线,不自建 @Scheduled)。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PriceTrackLoopRunStaleService {
|
||||
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String STATUS_FAILED = "FAILED";
|
||||
private static final String STATUS_STOPPED = "STOPPED";
|
||||
|
||||
/** 子任务终态集合,与 PriceTrackLoopRunService.isChildTaskTerminal 口径一致。 */
|
||||
private static final Set<String> CHILD_TERMINAL_STATUSES = Set.of("SUCCESS", "FAILED", "STOPPED");
|
||||
|
||||
/** 单轮最多扫描的轮次数,其余留给下一轮(每 2 分钟一次),避免一次扫太多拖慢巡检。 */
|
||||
static final int MAX_LOOP_RUNS_PER_SCAN = 200;
|
||||
|
||||
private final PriceTrackLoopRunMapper loopRunMapper;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final PriceTrackLoopRunService loopRunService;
|
||||
|
||||
/**
|
||||
* 「轮次在等前端派发下一店」超过多少分钟算废弃(默认 120)。
|
||||
*
|
||||
* <p>这个阈值等于「放弃等待用户回来」的时间:前端把轮次 id 存在 localStorage,
|
||||
* 用户重新打开页面会自动 resume,所以只要没判死就还能接着跑;判死后用户再打开
|
||||
* 会看到失败,需要重新配置店铺重跑。取 2 小时是因为正常的两店间隔只有几秒
|
||||
* (前端 while 循环紧接着调 dispatch-next),2 小时足以排除任何正常瞬态。
|
||||
*/
|
||||
@Value("${aiimage.price-track.loop-run-stale-minutes:120}")
|
||||
private long staleMinutes = 120;
|
||||
|
||||
/**
|
||||
* 回收卡死的跟价轮次。
|
||||
*
|
||||
* @return 单轮统计,合并进 stale-check 的 summary 日志
|
||||
*/
|
||||
public StaleLoopRunStats failStaleLoopRuns() {
|
||||
StaleLoopRunStats stats = new StaleLoopRunStats();
|
||||
long minutes = Math.max(1L, staleMinutes);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||
|
||||
// 候选取「RUNNING 且(active 为空但已陈旧,或 active 非空)」:
|
||||
// active 为空且 updated_at 很新 = 前端正在两店之间正常派发(正常只持续几秒),绝不碰。
|
||||
List<PriceTrackLoopRunEntity> candidates = loopRunMapper.selectList(
|
||||
new LambdaQueryWrapper<PriceTrackLoopRunEntity>()
|
||||
.eq(PriceTrackLoopRunEntity::getStatus, STATUS_RUNNING)
|
||||
.and(w -> w.isNull(PriceTrackLoopRunEntity::getActiveTaskId)
|
||||
.lt(PriceTrackLoopRunEntity::getUpdatedAt, threshold)
|
||||
.or().isNotNull(PriceTrackLoopRunEntity::getActiveTaskId))
|
||||
.last("limit " + MAX_LOOP_RUNS_PER_SCAN));
|
||||
stats.scannedLoopRunCount = candidates.size();
|
||||
if (candidates.isEmpty()) {
|
||||
return stats;
|
||||
}
|
||||
|
||||
// 批量取子任务,避免逐条 selectById(候选最多 200 条)
|
||||
Set<Long> childIds = candidates.stream()
|
||||
.map(PriceTrackLoopRunEntity::getActiveTaskId)
|
||||
.filter(id -> id != null && id > 0)
|
||||
.collect(Collectors.toSet());
|
||||
Map<Long, FileTaskEntity> childById = childIds.isEmpty()
|
||||
? Map.of()
|
||||
: fileTaskMapper.selectBatchIds(childIds).stream()
|
||||
.collect(Collectors.toMap(FileTaskEntity::getId, task -> task,
|
||||
(a, b) -> a, LinkedHashMap::new));
|
||||
|
||||
log.info("[stale-check] price-track-loop candidates={} threshold={} staleMinutes={}",
|
||||
candidates.size(), threshold, minutes);
|
||||
|
||||
for (PriceTrackLoopRunEntity entity : candidates) {
|
||||
try {
|
||||
reclaimLoopRun(entity, childById, now, stats);
|
||||
} catch (Exception ex) {
|
||||
// 单条失败不影响其余:本轮跳过,2 分钟后下一轮自然重试
|
||||
stats.skippedLoopRunCount++;
|
||||
log.warn("[stale-check] price-track-loop 回收单条失败 loopRunId={} msg={}",
|
||||
entity.getId(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
/** 单条轮次的回收分流。 */
|
||||
private void reclaimLoopRun(PriceTrackLoopRunEntity entity, Map<Long, FileTaskEntity> childById,
|
||||
LocalDateTime now, StaleLoopRunStats stats) {
|
||||
boolean stopRequested = Boolean.TRUE.equals(entity.getStopRequested());
|
||||
Long activeTaskId = entity.getActiveTaskId();
|
||||
|
||||
// ① active 为空:前端在两店之间停止了(正常只持续几秒,超过阈值说明页面已关且没再打开)
|
||||
if (activeTaskId == null || activeTaskId <= 0) {
|
||||
if (stopRequested) {
|
||||
markStopped(entity, now, stats);
|
||||
} else {
|
||||
markFailed(entity, "循环长时间无进展(前端已停止派发下一店),已自动结束", now, stats);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
FileTaskEntity child = childById.get(activeTaskId);
|
||||
// ② 子任务行已不存在:事件链已断,用户回来也会在 child-finished 报「子任务不存在」
|
||||
if (child == null) {
|
||||
if (stopRequested) {
|
||||
markStopped(entity, now, stats);
|
||||
} else {
|
||||
markFailed(entity, "循环子任务已不存在,循环已自动结束", now, stats);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ③ 子任务还在跑:判死交给跟价子任务判死线,本轮不动轮次(它终态后下一轮自然接住)
|
||||
if (!isChildTerminal(child)) {
|
||||
stats.skippedLoopRunCount++;
|
||||
return;
|
||||
}
|
||||
|
||||
// ④ 子任务已终态但轮次仍 RUNNING:前端漏掉了那次 child-finished,服务端补推进
|
||||
if (stopRequested) {
|
||||
// 停止意图优先于补推进(否则点完停止循环还会自己转起来)
|
||||
markStopped(entity, now, stats);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
loopRunService.syncLoopRunAfterChildTerminal(activeTaskId);
|
||||
stats.finalizedLoopRunCount++;
|
||||
log.info("[stale-check] price-track-loop 补推进漏掉的收尾 loopRunId={} childTaskId={} childStatus={}",
|
||||
entity.getId(), activeTaskId, child.getStatus());
|
||||
} catch (Exception ex) {
|
||||
// 补推进失败(子任务不属于该循环 / 上下文缺失等)→ 判死,不留孤儿
|
||||
markFailed(entity, "循环子任务收尾异常,循环已自动结束", now, stats);
|
||||
log.warn("[stale-check] price-track-loop 补推进失败改判死 loopRunId={} childTaskId={} msg={}",
|
||||
entity.getId(), activeTaskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void markStopped(PriceTrackLoopRunEntity entity, LocalDateTime now, StaleLoopRunStats stats) {
|
||||
if (casUpdateStatus(entity.getId(), STATUS_STOPPED, null, now)) {
|
||||
stats.stoppedLoopRunCount++;
|
||||
log.info("[stale-check] price-track-loop 回收为已停止 loopRunId={} userId={}",
|
||||
entity.getId(), entity.getUserId());
|
||||
} else {
|
||||
// 影响 0 行 = 这几秒内已被并发改写(前端刚好推进了轮次),不覆盖
|
||||
stats.skippedLoopRunCount++;
|
||||
}
|
||||
}
|
||||
|
||||
private void markFailed(PriceTrackLoopRunEntity entity, String reason, LocalDateTime now, StaleLoopRunStats stats) {
|
||||
if (casUpdateStatus(entity.getId(), STATUS_FAILED, reason, now)) {
|
||||
stats.failedLoopRunCount++;
|
||||
log.warn("[stale-check] price-track-loop 判失败 loopRunId={} userId={} round={}/{} shopIndex={} reason={}",
|
||||
entity.getId(), entity.getUserId(), entity.getCurrentRound(), entity.getTargetRounds(),
|
||||
entity.getCurrentShopIndex(), reason);
|
||||
} else {
|
||||
stats.skippedLoopRunCount++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 字段级 CAS 更新:只从 RUNNING 改。
|
||||
*
|
||||
* <p>不用整行 updateById——那会拿读取时的快照覆盖并发写入的字段(前端在这几秒内
|
||||
* 刚好推进轮次、或用户刚好点了停止),既有 bindChildTask / requestStop 也出于同样
|
||||
* 原因走字段级更新。影响 0 行即视为已被并发改写,调用方按跳过处理。
|
||||
*/
|
||||
private boolean casUpdateStatus(Long loopRunId, String status, String errorMessage, LocalDateTime now) {
|
||||
return loopRunMapper.update(null, new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
|
||||
.eq(PriceTrackLoopRunEntity::getId, loopRunId)
|
||||
.eq(PriceTrackLoopRunEntity::getStatus, STATUS_RUNNING)
|
||||
.set(PriceTrackLoopRunEntity::getStatus, status)
|
||||
.set(PriceTrackLoopRunEntity::getActiveTaskId, null)
|
||||
.set(PriceTrackLoopRunEntity::getErrorMessage, errorMessage)
|
||||
.set(PriceTrackLoopRunEntity::getFinishedAt, now)
|
||||
.set(PriceTrackLoopRunEntity::getUpdatedAt, now)) > 0;
|
||||
}
|
||||
|
||||
private static boolean isChildTerminal(FileTaskEntity child) {
|
||||
String status = child.getStatus();
|
||||
return status != null && CHILD_TERMINAL_STATUSES.contains(status.trim().toUpperCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
/** 单轮统计(合并进 stale-check 的 summary 日志,避免定期刷屏)。 */
|
||||
public static final class StaleLoopRunStats {
|
||||
public int scannedLoopRunCount;
|
||||
/** 补推进(子任务已终态、轮次漏了收尾)的条数。 */
|
||||
public int finalizedLoopRunCount;
|
||||
public int failedLoopRunCount;
|
||||
public int stoppedLoopRunCount;
|
||||
public int skippedLoopRunCount;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user