refactor(phase-3): 3A 拆薄第三笔——A 聚类(陈旧扫描与二次判死)抽成 support 件
AppearancePatentTaskService 棘轮读数 3349→3165(本会话累计 3458→3165,净减 293 行); 新落 service/support/AppearancePatentStaleTaskScanner 313 行(<800 报警线,不进超长清单)。 搬走的是「扫描+判定」:候选列举、心跳是否过期、判死文案按证据分述、二次判死那条线 (心跳新鲜但分片久不来,看 biz_task_scope_state.last_chunk_at)、取锁与 LockSkipWitness 留痕。 收尾那一步 finalizeStaleTask 留在服务侧、以 StaleDeathFinalizer 回调交进来——它要动事务边界、 封口与 LLM 批管线,那些件还在服务里,一起拖出去就得再造一套构造器,正是上一笔刚结清的那种亏。 两条线的先后与分工原样保留(sweep 末尾照旧接 sweepNoUpload;同一条任务在两把尺上的期望相反, 合并判定就会有一边失明)。日志文案、tag 字符串、取锁参数(不等待、TTL 由服务侧交进来)逐字没动。 两处当场判红、当场修,都是本笔的账: - AppearancePatentNoUploadStaleTest 三支 Method not found:那支测试用 ReflectionTestUtils 按名字点名 finalizeNoUploadStaleTasks 这枚私有方法。抽件前只扫了调用点没扫反射点名—— AGENTS.md §1 铁律 2 写的就是这个面。处置是保留同名入口(包内可见)转调到扫描件, 测试接缝与原语义都不动,理由写在方法注释里。 - TaskStatusVocabularyRatchetTest:扫描件里那行 STATUS_RUNNING = "RUNNING" 让 src/main 手写 状态标量从 70 涨到 71。这格是只许降的棘轮,降法本来就写在该尺的报错文案里——改成引用 TaskStatuses.FileTask.RUNNING,一格判据没放宽。 门禁:定向批 435 例/0 红/0 错/5 skipped(golden 生成器),BUILD SUCCESS,取码走无管道写法; 含 *Stale*Test 七支与 ForbiddenLogAuditTest(搬移带 WARN,日志面也过)。八条 golden 逐字节相等。
This commit is contained in:
+23
-207
@@ -8,12 +8,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.task.TaskResultAdmission;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentExcelParser;
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.util.GroupResultPropagator;
|
||||
import com.nanri.aiimage.common.util.LockSkipWitness;
|
||||
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
|
||||
import com.nanri.aiimage.common.util.StaleTaskDeathReason;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
@@ -36,6 +33,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskD
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskItemVo;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentRowOutcome;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentSheetBuilder;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentStaleTaskScanner;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentTaskOwnership;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -43,7 +41,6 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
@@ -72,14 +69,11 @@ import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
@@ -655,214 +649,36 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
: row.getResultFilename();
|
||||
}
|
||||
|
||||
/**
|
||||
* A 聚类(陈旧任务的扫描与判定)住在 {@link AppearancePatentStaleTaskScanner}
|
||||
* (3A 拆薄第三笔,判据与文案逐字搬移)。收尾那一步留在本类、以回调交出:
|
||||
* 恢复探针、封口事务与 LLM 批管线都还在服务侧,一起拖出去就得再造一套构造器,
|
||||
* 那是上一笔刚结清过的那种亏(结构性成本 > 可沉件收益)。
|
||||
*/
|
||||
private AppearancePatentStaleTaskScanner staleTaskScanner() {
|
||||
return new AppearancePatentStaleTaskScanner(MODULE_TYPE, TASK_LOCK_TTL,
|
||||
fileTaskMapper, taskChunkMapper, taskScopeStateMapper, taskCacheService,
|
||||
taskHeartbeatPositionService, taskFileJobService, taskDistributedLockService,
|
||||
properties, taskOwnership());
|
||||
}
|
||||
|
||||
public void finalizeStaleTasks() {
|
||||
// P1-7:stale 判定改用 Redis heartbeat 作为主信号。
|
||||
// 历史实现先按 file_task.updated_at 过滤再二次校验 Redis 心跳,
|
||||
// 但 file_task.updated_at 会被 Java 端在 "等 Python 上传" 分支自我续命
|
||||
// (resetStuckJobs 把 file_job 改回 PENDING → 重跑 processResultFileJob → touchJavaSideTaskActivity)。
|
||||
// Redis heartbeat 仅在 Python 心跳/上传 result 时写入,不会被 Java 内部循环刷新。
|
||||
long thresholdMillis = LocalDateTime.now()
|
||||
.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()))
|
||||
.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates();
|
||||
for (FileTaskEntity task : tasks) {
|
||||
// P1-8(双实例 stale 判死盲区修复,与 similar-asin 同源):本方法由
|
||||
// DeleteBrandStaleTaskService 的 "delete-brand:stale-check" 分布式 job 锁保护,
|
||||
// 单实例执行;历史上 owner 非本实例的任务直接跳过,若任务 owner 是另一实例则无人判死,
|
||||
// 只能等 job 级 stuck 扫描误判。现改为全局判死:跨实例并发互斥由 task 分布式锁
|
||||
// (acquire 不等待、TTL 5 分钟)保证,owner 实例仍在正常推进的任务心跳不会 stale。
|
||||
if (!isHeartbeatStale(task, thresholdMillis)) {
|
||||
continue;
|
||||
}
|
||||
AppearancePatentTaskOwnership ownership = taskOwnership();
|
||||
String owner = ownership.ownerFromTask(task);
|
||||
if (!ownership.isOwnerCurrent(owner)) {
|
||||
log.warn("[appearance-patent] 心跳超时任务归其他实例所有,由本实例(job 锁持有者)接管判死/恢复 taskId={} owner={} current={}",
|
||||
task.getId(), owner, currentInstanceId());
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
|
||||
if (lockHandle == null) {
|
||||
// 锁被 owner 实例/文件任务 worker 占用说明其仍活跃,静默跳过等下一轮
|
||||
LockSkipWitness.recordSkip("appearance-patent:running-stale-scan", MODULE_TYPE, task.getId());
|
||||
continue;
|
||||
}
|
||||
try (lockHandle) {
|
||||
finalizeStaleTask(task.getId(), describeStaleDeath(task.getId()));
|
||||
}
|
||||
}
|
||||
finalizeNoUploadStaleTasks();
|
||||
staleTaskScanner().sweep(this::finalizeStaleTask);
|
||||
}
|
||||
|
||||
public void debugFinalizeStaleTask(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L);
|
||||
if (lockHandle == null) {
|
||||
LockSkipWitness.recordAbandoned("appearance-patent:debug-finalize-stale", MODULE_TYPE, taskId);
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
finalizeStaleTask(taskId, describeStaleDeath(taskId));
|
||||
}
|
||||
staleTaskScanner().finalizeOne(taskId, this::finalizeStaleTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。
|
||||
* 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result",
|
||||
* 把「没开跑」说成「跑挂了」。
|
||||
* 只跑二次判死那条线。<b>这个入口的存在理由是测试</b>:
|
||||
* {@code AppearancePatentNoUploadStaleTest} 一直按名字点名这枚私有方法,单独验「心跳新鲜、
|
||||
* 分片久不来」那条出口——A 聚类抽到扫描件后若把它摘掉,三支用例直接
|
||||
* {@code Method not found}(本笔第一版就栽在这儿:抽件前只扫了调用点,没扫反射点名)。
|
||||
* 保留同名入口=保留测试接缝,也让「两条线各跑一次」这件事在服务侧仍然读得出来。
|
||||
*/
|
||||
private String describeStaleDeath(Long taskId) {
|
||||
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId);
|
||||
return new StaleTaskDeathReason(
|
||||
persistedResultChunkCount(taskId),
|
||||
lastResultChunkAt(taskId),
|
||||
heartbeatMillis > 0L
|
||||
? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault())
|
||||
: null,
|
||||
taskHeartbeatPositionService.describe(taskId),
|
||||
taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE),
|
||||
properties.getStaleTimeoutMinutes()).describe();
|
||||
}
|
||||
|
||||
private long persistedResultChunkCount(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
/** 最后一次结果分片回传时间;从未回传时为 null。 */
|
||||
private LocalDateTime lastResultChunkAt(Long taskId) {
|
||||
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
|
||||
return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt();
|
||||
}
|
||||
|
||||
private List<FileTaskEntity> listStaleFinalizeCandidates() {
|
||||
// P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。
|
||||
// APPEARANCE_PATENT 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。
|
||||
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 200"));
|
||||
}
|
||||
|
||||
/**
|
||||
* P1-7:判断 APPEARANCE_PATENT 任务是否已 stale。
|
||||
* 主信号:Redis heartbeat(仅 Python 心跳/上传 result 时刷新,不会被 Java 内部循环续命)。
|
||||
* Redis 命中:以 heartbeatMillis 与 thresholdMillis 比较。
|
||||
* Redis 缺失/降级(返回 0):回落到 file_task.updated_at,避免 Redis 故障时全量误杀。
|
||||
*/
|
||||
private boolean isHeartbeatStale(FileTaskEntity task, long thresholdMillis) {
|
||||
if (task == null || task.getId() == null) {
|
||||
return false;
|
||||
}
|
||||
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId());
|
||||
if (heartbeatMillis > 0L) {
|
||||
return heartbeatMillis <= thresholdMillis;
|
||||
}
|
||||
LocalDateTime updatedAt = task.getUpdatedAt();
|
||||
if (updatedAt == null) {
|
||||
return false;
|
||||
}
|
||||
long updatedMillis = updatedAt.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
return updatedMillis <= thresholdMillis;
|
||||
}
|
||||
|
||||
/**
|
||||
* 二次判死:Python 心跳正常(Redis heartbeat 新鲜)但连续 N 分钟无结果分片上报。
|
||||
*
|
||||
* <p>既有判定以 Redis heartbeat 为 stale 主信号(P1-7),但该心跳随 Python 的 HTTP 心跳
|
||||
* 每分钟刷新——主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。
|
||||
* 本线改看 biz_task_scope_state.last_chunk_at(仅在 persistSubmittedChunk 上传分片时刷新),
|
||||
* 命中后走既有 finalizeStaleTask(封口上传 + LLM 收尾,不粗暴杀任务)。
|
||||
* 从未上报(无 scope 行或全 NULL)的任务不再无条件跳过,改按「处理位置是否仍在推进」区分:
|
||||
* 位置在变=首批较慢,跳过;位置长时间不变=判死并写清停在哪;无位置信息(老客户端)=证据不足,
|
||||
* 保守跳过但留 WARN —— 2026-10-06 品牌任务 2626 正是靠无条件跳过挂到 9 小时。
|
||||
*/
|
||||
private void finalizeNoUploadStaleTasks() {
|
||||
long minutes = properties.getNoResultUploadTimeoutMinutes();
|
||||
if (minutes <= 0) {
|
||||
return;
|
||||
}
|
||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates();
|
||||
if (tasks.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||
long heartbeatThresholdMillis = LocalDateTime.now()
|
||||
.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()))
|
||||
.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
|
||||
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||
tasks.stream().map(FileTaskEntity::getId).toList())) {
|
||||
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||
}
|
||||
}
|
||||
for (FileTaskEntity task : tasks) {
|
||||
if (isHeartbeatStale(task, heartbeatThresholdMillis)) {
|
||||
// 心跳已 stale:归既有心跳线处理
|
||||
continue;
|
||||
}
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue,
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块同样心跳照打)。
|
||||
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[appearance-patent] 从未上报但客户端仍在推进,本轮不封口 taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[appearance-patent] 从未上报且无位置信息,证据不足暂不封口 taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle neverReportedLock = acquireTaskLock(task.getId(), 0L);
|
||||
if (neverReportedLock == null) {
|
||||
LockSkipWitness.recordSkip("appearance-patent:never-reported-stale-scan",
|
||||
MODULE_TYPE, task.getId());
|
||||
continue;
|
||||
}
|
||||
try (neverReportedLock) {
|
||||
log.warn("[appearance-patent] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
||||
neverReportedPosition, task.getId(), minutes);
|
||||
finalizeStaleTask(task.getId(), "超过 " + minutes + " 分钟无任何结果分片上报,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
|
||||
if (lockHandle == null) {
|
||||
LockSkipWitness.recordSkip("appearance-patent:no-result-shard-stale-scan", MODULE_TYPE, task.getId());
|
||||
continue;
|
||||
}
|
||||
try (lockHandle) {
|
||||
log.warn("[appearance-patent] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}",
|
||||
minutes, task.getId(), lastResultAt);
|
||||
String error = "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes";
|
||||
finalizeStaleTask(task.getId(), error);
|
||||
}
|
||||
}
|
||||
void finalizeNoUploadStaleTasks() {
|
||||
staleTaskScanner().sweepNoUpload(this::finalizeStaleTask);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.util.LockSkipWitness;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.common.util.StaleTaskDeathReason;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 外观专利的陈旧任务扫描与二次判死(A 聚类),从 {@code AppearancePatentTaskService}
|
||||
* <b>原样搬移</b>(3A 拆薄第三笔):判据、日志文案、{@code LockSkipWitness} 的 tag、
|
||||
* 取锁参数与<b>两条线的先后顺序</b>一个字没改。
|
||||
*
|
||||
* <p><b>搬的是扫描与判定,不是收尾。</b>真正给任务判死那一步({@code finalizeStaleTask}
|
||||
* → 恢复探针 → 封口 → LLM 批次 → {@code finalizeTask})留在服务侧,以
|
||||
* {@link StaleDeathFinalizer} 回调交进来。理由不是审美:那段要动事务边界
|
||||
* ({@code inNewTransaction})、LLM 批管线与结果行读取,那些件还在服务里;
|
||||
* 把它们一起拖出来就得先给它们再造一套构造器,而 3A 的账已经证明
|
||||
* 「结构性成本 > 可沉件收益」是要避免的事。回调是这一笔里唯一的新形状。
|
||||
*
|
||||
* <p><b>两条线的分工不能并。</b>心跳线({@code sweep})看 Redis heartbeat 是否过期;
|
||||
* 无上报线({@code sweepNoUpload})专治「心跳照打、主线程卡死」那类缺口(生产 28131 同型),
|
||||
* 看的是 {@code biz_task_scope_state.last_chunk_at}。同一条任务在两把尺上的期望相反——
|
||||
* 心跳新鲜但分片久不来,前者放过、后者封口。合并成一个判定就会有一边失明。
|
||||
*/
|
||||
@Slf4j
|
||||
public class AppearancePatentStaleTaskScanner {
|
||||
|
||||
/** 状态词一律引用公共词表:这里曾写死字面量,被 TaskStatusVocabularyRatchetTest 当场判红
|
||||
* (src/main 的 STATUS_XXX 标量声明是只许降的棘轮,降法就是接进 TaskStatuses)。*/
|
||||
private static final String STATUS_RUNNING = TaskStatuses.FileTask.RUNNING;
|
||||
private static final int CANDIDATE_SCAN_LIMIT = 200;
|
||||
|
||||
/** 扫到该判死时怎么收尾,由服务侧决定(事务边界与 LLM 收尾都留在它那儿)。 */
|
||||
@FunctionalInterface
|
||||
public interface StaleDeathFinalizer {
|
||||
void finalizeStale(Long taskId, String error);
|
||||
}
|
||||
|
||||
private final String moduleType;
|
||||
private final Duration taskLockTtl;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final AppearancePatentTaskCacheService taskCacheService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final AppearancePatentProperties properties;
|
||||
private final AppearancePatentTaskOwnership ownership;
|
||||
|
||||
public AppearancePatentStaleTaskScanner(String moduleType,
|
||||
Duration taskLockTtl,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
AppearancePatentTaskCacheService taskCacheService,
|
||||
TaskHeartbeatPositionService taskHeartbeatPositionService,
|
||||
TaskFileJobService taskFileJobService,
|
||||
TaskDistributedLockService taskDistributedLockService,
|
||||
AppearancePatentProperties properties,
|
||||
AppearancePatentTaskOwnership ownership) {
|
||||
this.moduleType = moduleType;
|
||||
this.taskLockTtl = taskLockTtl;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.taskCacheService = taskCacheService;
|
||||
this.taskHeartbeatPositionService = taskHeartbeatPositionService;
|
||||
this.taskFileJobService = taskFileJobService;
|
||||
this.taskDistributedLockService = taskDistributedLockService;
|
||||
this.properties = properties;
|
||||
this.ownership = ownership;
|
||||
}
|
||||
|
||||
public void sweep(StaleDeathFinalizer finalizer) {
|
||||
// P1-7:stale 判定改用 Redis heartbeat 作为主信号。
|
||||
// 历史实现先按 file_task.updated_at 过滤再二次校验 Redis 心跳,
|
||||
// 但 file_task.updated_at 会被 Java 端在 "等 Python 上传" 分支自我续命
|
||||
// (resetStuckJobs 把 file_job 改回 PENDING → 重跑 processResultFileJob → touchJavaSideTaskActivity)。
|
||||
// Redis heartbeat 仅在 Python 心跳/上传 result 时写入,不会被 Java 内部循环刷新。
|
||||
long thresholdMillis = LocalDateTime.now()
|
||||
.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()))
|
||||
.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates();
|
||||
for (FileTaskEntity task : tasks) {
|
||||
// P1-8(双实例 stale 判死盲区修复,与 similar-asin 同源):本方法由
|
||||
// DeleteBrandStaleTaskService 的 "delete-brand:stale-check" 分布式 job 锁保护,
|
||||
// 单实例执行;历史上 owner 非本实例的任务直接跳过,若任务 owner 是另一实例则无人判死,
|
||||
// 只能等 job 级 stuck 扫描误判。现改为全局判死:跨实例并发互斥由 task 分布式锁
|
||||
// (acquire 不等待、TTL 5 分钟)保证,owner 实例仍在正常推进的任务心跳不会 stale。
|
||||
if (!isHeartbeatStale(task, thresholdMillis)) {
|
||||
continue;
|
||||
}
|
||||
String owner = ownership.ownerFromTask(task);
|
||||
if (!ownership.isOwnerCurrent(owner)) {
|
||||
log.warn("[appearance-patent] 心跳超时任务归其他实例所有,由本实例(job 锁持有者)接管判死/恢复 taskId={} owner={} current={}",
|
||||
task.getId(), owner, ownership.currentInstanceId());
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
|
||||
if (lockHandle == null) {
|
||||
// 锁被 owner 实例/文件任务 worker 占用说明其仍活跃,静默跳过等下一轮
|
||||
LockSkipWitness.recordSkip("appearance-patent:running-stale-scan", moduleType, task.getId());
|
||||
continue;
|
||||
}
|
||||
try (lockHandle) {
|
||||
finalizer.finalizeStale(task.getId(), describeStaleDeath(task.getId()));
|
||||
}
|
||||
}
|
||||
sweepNoUpload(finalizer);
|
||||
}
|
||||
|
||||
public void finalizeOne(Long taskId, StaleDeathFinalizer finalizer) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, 0L);
|
||||
if (lockHandle == null) {
|
||||
LockSkipWitness.recordAbandoned("appearance-patent:debug-finalize-stale", moduleType, taskId);
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
finalizer.finalizeStale(taskId, describeStaleDeath(taskId));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。
|
||||
* 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result",
|
||||
* 把「没开跑」说成「跑挂了」。
|
||||
*/
|
||||
public String describeStaleDeath(Long taskId) {
|
||||
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(taskId);
|
||||
return new StaleTaskDeathReason(
|
||||
persistedResultChunkCount(taskId),
|
||||
lastResultChunkAt(taskId),
|
||||
heartbeatMillis > 0L
|
||||
? LocalDateTime.ofInstant(Instant.ofEpochMilli(heartbeatMillis), ZoneId.systemDefault())
|
||||
: null,
|
||||
taskHeartbeatPositionService.describe(taskId),
|
||||
taskFileJobService.hasExhaustedAssembleJob(taskId, moduleType),
|
||||
properties.getStaleTimeoutMinutes()).describe();
|
||||
}
|
||||
|
||||
private long persistedResultChunkCount(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
/** 最后一次结果分片回传时间;从未回传时为 null。 */
|
||||
private LocalDateTime lastResultChunkAt(Long taskId) {
|
||||
List<TaskScopeLastChunkDto> rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId));
|
||||
return rows == null || rows.isEmpty() ? null : rows.get(0).lastChunkAt();
|
||||
}
|
||||
|
||||
private List<FileTaskEntity> listStaleFinalizeCandidates() {
|
||||
// P1-7:不再用 file_task.updated_at 预过滤,由调用方按 Redis heartbeat 判 stale。
|
||||
// APPEARANCE_PATENT 在线 RUNNING 任务量级有限(限 200),全扫成本可忽略。
|
||||
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, moduleType)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit " + CANDIDATE_SCAN_LIMIT));
|
||||
}
|
||||
|
||||
/**
|
||||
* P1-7:判断 APPEARANCE_PATENT 任务是否已 stale。
|
||||
* 主信号:Redis heartbeat(仅 Python 心跳/上传 result 时刷新,不会被 Java 内部循环续命)。
|
||||
* Redis 命中:以 heartbeatMillis 与 thresholdMillis 比较。
|
||||
* Redis 缺失/降级(返回 0):回落到 file_task.updated_at,避免 Redis 故障时全量误杀。
|
||||
*/
|
||||
private boolean isHeartbeatStale(FileTaskEntity task, long thresholdMillis) {
|
||||
if (task == null || task.getId() == null) {
|
||||
return false;
|
||||
}
|
||||
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId());
|
||||
if (heartbeatMillis > 0L) {
|
||||
return heartbeatMillis <= thresholdMillis;
|
||||
}
|
||||
LocalDateTime updatedAt = task.getUpdatedAt();
|
||||
if (updatedAt == null) {
|
||||
return false;
|
||||
}
|
||||
long updatedMillis = updatedAt.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
return updatedMillis <= thresholdMillis;
|
||||
}
|
||||
|
||||
/**
|
||||
* 二次判死:Python 心跳正常(Redis heartbeat 新鲜)但连续 N 分钟无结果分片上报。
|
||||
*
|
||||
* <p>既有判定以 Redis heartbeat 为 stale 主信号(P1-7),但该心跳随 Python 的 HTTP 心跳
|
||||
* 每分钟刷新——主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。
|
||||
* 本线改看 biz_task_scope_state.last_chunk_at(仅在 persistSubmittedChunk 上传分片时刷新),
|
||||
* 命中后走既有 finalizeStaleTask(封口上传 + LLM 收尾,不粗暴杀任务)。
|
||||
* 从未上报(无 scope 行或全 NULL)的任务不再无条件跳过,改按「处理位置是否仍在推进」区分:
|
||||
* 位置在变=首批较慢,跳过;位置长时间不变=判死并写清停在哪;无位置信息(老客户端)=证据不足,
|
||||
* 保守跳过但留 WARN —— 2026-10-06 品牌任务 2626 正是靠无条件跳过挂到 9 小时。
|
||||
*/
|
||||
public void sweepNoUpload(StaleDeathFinalizer finalizer) {
|
||||
long minutes = properties.getNoResultUploadTimeoutMinutes();
|
||||
if (minutes <= 0) {
|
||||
return;
|
||||
}
|
||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates();
|
||||
if (tasks.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes);
|
||||
long heartbeatThresholdMillis = LocalDateTime.now()
|
||||
.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()))
|
||||
.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
Map<Long, LocalDateTime> lastResultAtByTaskId = new HashMap<>();
|
||||
for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds(
|
||||
tasks.stream().map(FileTaskEntity::getId).toList())) {
|
||||
if (dto.taskId() != null && dto.lastChunkAt() != null) {
|
||||
lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt());
|
||||
}
|
||||
}
|
||||
for (FileTaskEntity task : tasks) {
|
||||
if (isHeartbeatStale(task, heartbeatThresholdMillis)) {
|
||||
// 心跳已 stale:归既有心跳线处理
|
||||
continue;
|
||||
}
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue,
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块同样心跳照打)。
|
||||
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[appearance-patent] 从未上报但客户端仍在推进,本轮不封口 taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[appearance-patent] 从未上报且无位置信息,证据不足暂不封口 taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle neverReportedLock = acquireTaskLock(task.getId(), 0L);
|
||||
if (neverReportedLock == null) {
|
||||
LockSkipWitness.recordSkip("appearance-patent:never-reported-stale-scan",
|
||||
moduleType, task.getId());
|
||||
continue;
|
||||
}
|
||||
try (neverReportedLock) {
|
||||
log.warn("[appearance-patent] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
||||
neverReportedPosition, task.getId(), minutes);
|
||||
finalizer.finalizeStale(task.getId(), "超过 " + minutes + " 分钟无任何结果分片上报,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
|
||||
if (lockHandle == null) {
|
||||
LockSkipWitness.recordSkip("appearance-patent:no-result-shard-stale-scan", moduleType, task.getId());
|
||||
continue;
|
||||
}
|
||||
try (lockHandle) {
|
||||
log.warn("[appearance-patent] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}",
|
||||
minutes, task.getId(), lastResultAt);
|
||||
String error = "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes";
|
||||
finalizer.finalizeStale(task.getId(), error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 与 {@code AppearancePatentTaskService#acquireTaskLock} 同一枚调用(不等待、拿不到就跳过等下一轮)。
|
||||
* 没有把这枚也抽成回调:它只是一行转发,多一个回调位反而让「谁在什么锁上跳过」更难追。
|
||||
* TTL 由服务侧交进来,不在这里重复写死 5 分钟。
|
||||
*/
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) {
|
||||
return taskDistributedLockService.acquire(moduleType, taskId, taskLockTtl, waitMillis);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user