From de38c3ec584c57afbf21d6a64e42a7701e055fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 19 Oct 2026 18:56:22 +0800 Subject: [PATCH] =?UTF-8?q?refactor(phase-3):=203D=20=E5=89=8D=E7=BD=AE?= =?UTF-8?q?=E6=8B=86=E8=96=84=E2=80=94=E2=80=94publish=20=E9=99=88?= =?UTF-8?q?=E6=97=A7=E6=89=AB=E6=8F=8F=E8=81=9A=E7=B1=BB=E6=8A=BD=E5=87=BA?= =?UTF-8?q?=20PublishStaleTaskScanner=EF=BC=8C=E6=9C=8D=E5=8A=A1=202250?= =?UTF-8?q?=E2=86=922109?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 用户拍板的路线:接底座净亏 +8~15 行、必须先靠一笔 ③ 前置抽件把账做平。本笔就是那件抽件,为下一笔(publish extends 底座,实测成本 ≈+35 行)腾出 141 行额度。 搬的是「扫描+判定」,不是「收尾」: - 进 `publish/service/support/PublishStaleTaskScanner.java`(293 行):作业锁 `publish:stale-check` 与其 fail-open 日志、两条候选 SQL(心跳线 `updated_at` 陈旧/二次线 `last_chunk_at` 停摆)、每任务分布式锁、未完成组装 Job 闸门、「从未上报」三分证据判定、两条线各自的复查谓词与对外文案。 - 留在服务:`failStaleTaskLockedInternal` 收尾本体**零改动**(部分回传行抢救、结果行终态、组装 Job 入队都在 machinery 侧),`failStaleTasks` 与 `failNoUploadStaleTasks` 各留一枚 delegator。 - 两枚 delegator 不是风格选择:`failStaleTasks` 被 `PublishTaskServiceTest` 三处公有调用;`failNoUploadStaleTasks` 被 `PublishNoUploadStaleTest:123` 用 `ReflectionTestUtils.invokeMethod` **按名字**反射。删名字等于把回归网拆掉。 - 扫描器**每次入口新建、不注入**:两枚阈值是服务侧 `@Value` 非 final 字段,测试靠 `setField` 改它们;烤进构造更早的缓存里,那三支既有用例当场读旧数。 等值证明不靠眼看:把搬走的两段与 `git show HEAD` 的原文做符号归一后逐行对照,A 线与 B 线的差异**只剩** `MODULE_TYPE`→`moduleType`、`STATUS_RUNNING`→`TaskStatuses.FileTask.RUNNING`、`currentInstanceId()`→`Supplier.get()`、收尾改走 `StaleFinalizer` 回调、`"limit 100"`/`0L`/`Math.max(5,…)` 提常量——没有第六种。当前实例 id 交 Supplier 而非定值是为了不改「查询那一刻才取值」的现网语义。 顺手的一处真减:两条候选查询的 owner 三条件块逐字相同,收成一枚 `ownerScoped` ⇒ 全树 wrapper 内裸 SQL **15→12**。 ⚠ 一处如实登记的既有缺口:二次线「有过上报但停摆」那一支没有测试覆盖(三支用例全在「从未上报」三分上)。它不是本笔造成的,但意味着那一支只有逐行对照当证据、没有测试当证据。 门禁与两处读数账: - 后端全量 **`Tests run: 4540, Failures: 0, Errors: 0, Skipped: 15` + BUILD SUCCESS,真实 rc=0**(取自重定向)。⚠ 4537→4540 这 +3 是另一路 `ProductRiskTaskServiceTest`/`ProductRiskExcelAssemblyServiceTest` 的在飞改动带来的,**本笔零新增 `@Test`**——第十一处读数坑的同族,引用例数变化先查归属。 - **抓到一枚本笔自己的红**:`SqlLayeringConventionTest.rawSqlInsideWrappersDoesNotGrow` 按文件名精确记 wrapper 内函数级 SQL,在册 `PublishTaskService=6`,抽件后实测 `PublishStaleTaskScanner=3`。按测文规定「迁走 ⇒ 同步减基线」改登记键,方向是减、不是放宽(不是阈值动作,无需请示)。教训入册:定向清单是按「上一笔红过什么」攒的那七类,**不含这一类**,新建 `src/main` 文件会唤醒按文件记账的尺——定向全绿给不出门禁全绿,这条只有全量能兜住。 - ops `ruff check .` All checks passed;readability 棘轮 2 failed/7 passed,三处增长全不归本笔(`ProductRiskTaskService` 1485→1495、`BrandProductRiskTab.vue` 1722→1724 属另一路 crawler-plugin 在飞件;`app_client/amazon/approve_v2.py` 1279→1284 与 `approve.py` 三枚长函数属 app_client 面在飞件)⇒ 不动基线、不 `--write`、不代收。本笔 `PublishTaskService` 2250→2109 是减,天然合法、不回写基线。格式步未纳管 ⇒ 如实跳过。真机启动验收本格仍欠。 --- .../publish/service/PublishTaskService.java | 165 +--------- .../support/PublishStaleTaskScanner.java | 293 ++++++++++++++++++ .../aiimage/SqlLayeringConventionTest.java | 3 +- 3 files changed, 307 insertions(+), 154 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/support/PublishStaleTaskScanner.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java index 60c262fca..3fa493484 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; import com.nanri.aiimage.common.security.AdminAuthSupport; -import com.nanri.aiimage.common.util.LockSkipWitness; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; @@ -22,6 +21,7 @@ import com.nanri.aiimage.modules.publish.model.dto.PublishParseRequest; import com.nanri.aiimage.modules.publish.model.dto.PublishResultFileDto; import com.nanri.aiimage.modules.publish.model.dto.PublishRowDto; import com.nanri.aiimage.modules.publish.model.dto.PublishSourceFileDto; +import com.nanri.aiimage.modules.publish.service.support.PublishStaleTaskScanner; import com.nanri.aiimage.modules.publish.model.dto.PublishSubmitResultRequest; import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity; import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity; @@ -38,7 +38,6 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; -import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; @@ -492,46 +491,7 @@ public class PublishTaskService { @Scheduled(fixedDelayString = "${aiimage.publish.stale-scan-delay-ms:60000}") public void failStaleTasks() { - // 双节点互斥:避免 ownerInstanceId 为空的 stale 行被两节点同时扫出双写错误信息; - // Redis 不可用时 fail-open,兜底靠下方 per-task 锁 - DistributedJobLockService.LockHandle jobLock = - distributedJobLockService.tryLock("publish:stale-check", java.time.Duration.ofMinutes(2)); - if (jobLock == null) { - log.info("[publish] stale-check skipped, another instance holds the distributed lock"); - return; - } - try (jobLock) { - LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, staleTimeoutMinutes)); - List staleTasks = fileTaskMapper.selectList(new LambdaQueryWrapper() - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .and(owner -> owner - .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL") - .or() - .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''") - .or() - .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId())) - .lt(FileTaskEntity::getUpdatedAt, threshold) - .orderByAsc(FileTaskEntity::getUpdatedAt) - .last("limit 100")); - for (FileTaskEntity candidate : staleTasks) { - if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) { - continue; - } - try (TaskDistributedLockService.LockHandle lock = - taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) { - if (lock == null) { - LockSkipWitness.recordSkip("publish:stale-heartbeat-scan", MODULE_TYPE, candidate.getId()); - continue; - } - transactionTemplate.executeWithoutResult(status -> failStaleTaskLocked(candidate.getId(), threshold)); - } catch (Exception ex) { - log.warn("[publish] stale task cleanup failed taskId={} msg={}", - candidate.getId(), ex.getMessage()); - } - } - failNoUploadStaleTasks(); - } + newStaleTaskScanner().sweep(this::failStaleTaskLockedInternal); } public void processResultFileJob(TaskFileJobEntity job) { @@ -868,122 +828,21 @@ public class PublishTaskService { return prepared; } - private void failStaleTaskLocked(Long taskId, LocalDateTime threshold) { - failStaleTaskLockedInternal(taskId, - task -> task.getUpdatedAt() != null && task.getUpdatedAt().isBefore(threshold), - "任务心跳超时"); - } - /** - * 二次判死:Python 心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。 - * - *

既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at—— - * 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。 - * owner 过滤与 A 线保持一致(避免双节点同时处理非本实例任务); - * 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报不再无条件跳过—— - * 与上报停摆同样按处理位置三分:仍在推进=首批较慢跳过、位置久未变=判死、无位置信息=证据不足保守跳过但留 WARN - * (原先的无条件跳过是 2026-10-06 品牌任务 2626 挂 9 小时的同类出口)。 + * 二次判死线的入口。名字必须留在本类上——{@code PublishNoUploadStaleTest} 拿 + * {@code ReflectionTestUtils.invokeMethod(service, "failNoUploadStaleTasks")} 按名字反射它。 + * 判定与扫描都在 {@link PublishStaleTaskScanner},这里只负责每次入口新建扫描器(阈值当场读) + * 并把收尾交回本类的 {@link #failStaleTaskLockedInternal}。 */ private void failNoUploadStaleTasks() { - long minutes = noResultUploadTimeoutMinutes; - if (minutes <= 0) { - return; - } - LocalDateTime heartbeatThreshold = LocalDateTime.now().minusMinutes(Math.max(5, staleTimeoutMinutes)); - LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes); - List candidates = fileTaskMapper.selectList(new LambdaQueryWrapper() - .eq(FileTaskEntity::getModuleType, MODULE_TYPE) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .and(owner -> owner - .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL") - .or() - .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''") - .or() - .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId())) - .ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold) - .lt(FileTaskEntity::getCreatedAt, cutoff) - .orderByAsc(FileTaskEntity::getCreatedAt) - .last("limit 100")); - if (candidates.isEmpty()) { - return; - } - Map lastResultAtByTaskId = new LinkedHashMap<>(); - for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds( - candidates.stream().map(FileTaskEntity::getId).toList())) { - if (dto.taskId() != null && dto.lastChunkAt() != null) { - lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt()); - } - } - for (FileTaskEntity candidate : candidates) { - LocalDateTime lastResultAt = lastResultAtByTaskId.get(candidate.getId()); - if (lastResultAt != null && lastResultAt.isAfter(cutoff)) { - // 上报仍新鲜:正常推进,不必处理 - continue; - } - if (lastResultAt == null) { - // 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue(零日志), - // 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块心跳同样照打)。 - // 现按处理位置三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪; - // 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。 - String neverReportedPosition = taskHeartbeatPositionService.describe(candidate.getId()); - if (taskHeartbeatPositionService.isClientAdvancing(candidate.getId(), minutes)) { - log.info("[publish] 从未上报但客户端仍在推进,本轮不判死 taskId={} position={} minutes={}", - candidate.getId(), neverReportedPosition, minutes); - continue; - } - if (neverReportedPosition == null || neverReportedPosition.isBlank()) { - log.warn("[publish] 从未上报且无位置信息,证据不足暂不判死 taskId={} createdAt={} minutes={}", - candidate.getId(), candidate.getCreatedAt(), minutes); - continue; - } - failNoUploadStaleTask(candidate, - "任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「" - + neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败", - task -> { - // 排队期间复查:心跳回落陈旧 → 交回 A 线;出现任一结果上报 → 已恢复回传,交回原线评估 - if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) { - return false; - } - return queryLastChunkAt(task.getId()) == null; - }); - continue; - } - failNoUploadStaleTask(candidate, - "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")", - task -> { - // 排队期间复查:心跳回落陈旧 → 交回 A 线;出现新结果上报 → 任务已恢复 - if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) { - return false; - } - LocalDateTime freshLastChunk = queryLastChunkAt(task.getId()); - return freshLastChunk != null && !freshLastChunk.isAfter(cutoff); - }); - } + newStaleTaskScanner().sweepNoUpload(this::failStaleTaskLockedInternal); } - /** 二次判死共同尾部:待组装检查 → 任务锁 → 收尾;error 与复查谓词由两条路各自提供。 */ - private void failNoUploadStaleTask(FileTaskEntity candidate, String error, Predicate staleCheck) { - if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) { - return; - } - try (TaskDistributedLockService.LockHandle lock = - taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) { - if (lock == null) { - LockSkipWitness.recordAbandoned("publish:no-upload-stale-finalize", MODULE_TYPE, candidate.getId()); - return; - } - transactionTemplate.executeWithoutResult(status -> failStaleTaskLockedInternal( - candidate.getId(), staleCheck, error)); - } catch (Exception ex) { - log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}", - candidate.getId(), ex.getMessage()); - } - } - - /** 单任务最后一次结果分片上报时间;无记录返回 null。 */ - private LocalDateTime queryLastChunkAt(Long taskId) { - List rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId)); - return rows.isEmpty() ? null : rows.get(0).lastChunkAt(); + /** 扫描器不注入、每次入口新建:两枚阈值是 {@code @Value} 非 final 字段,测试按字段反射改它们。 */ + private PublishStaleTaskScanner newStaleTaskScanner() { + return new PublishStaleTaskScanner(MODULE_TYPE, staleTimeoutMinutes, noResultUploadTimeoutMinutes, + this::currentInstanceId, distributedJobLockService, fileTaskMapper, taskScopeStateMapper, + taskDistributedLockService, taskFileJobService, taskHeartbeatPositionService, transactionTemplate); } /** 收尾条件由调用方判定(A 线=心跳陈旧;B 线=心跳新鲜但结果上报陈旧),条件通过后走同一收尾路径。 */ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/support/PublishStaleTaskScanner.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/support/PublishStaleTaskScanner.java new file mode 100644 index 000000000..f8bc35361 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/support/PublishStaleTaskScanner.java @@ -0,0 +1,293 @@ +package com.nanri.aiimage.modules.publish.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.common.util.LockSkipWitness; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** + * 上架陈旧任务扫描器:心跳线({@code updated_at} 超期)+二次判死线(心跳新鲜但连续无结果分片上报)。 + * + *

搬的是「扫描+判定」,不是「收尾」。本类决定一个任务该不该死——作业锁、候选 SQL、 + * 每任务分布式锁、未完成的组装 Job 闸门、以及二次线那套「从未上报」的三分证据判定与复查谓词; + * 而「死了要做什么」留在 {@code PublishTaskService}(要动结果行、部分回传行的抢救、 + * 任务与结果行的终态与组装 Job 入队,那些件都还在服务里)。判完通过 {@link StaleFinalizer} + * 交回去,与 {@code CollectDataStaleTaskScanner} 同一形状。 + * + *

本支只用一枚回调,collectdata 用两枚——差别在收尾:上架两条线判死后走的是 + * 同一段收尾({@code failStaleTaskLockedInternal}),只有「复查谓词」和「错误文案」不同, + * 所以那一格由扫描侧当参数交出即可;采集那支两条线的收尾本身就不同,才需要两枚接缝。 + * 这不是可以互相照抄的对称形状,别为了统一把上架的收尾拆成两段。 + * + *

两条线刻意不合并。心跳线判「{@code updated_at} 陈旧」,二次线判「心跳新鲜但 + * {@code last_chunk_at} 久未变」——同一条任务在两条线上的期望相反,合并判定必有一边失明 + * (2626 那型:从未上报的行与上报仍新鲜的行被并成一句静默 continue,假活任务杀不掉)。 + * + *

阈值逐枚由调用方在每次入口交进来,而不是注一份 properties。 + * {@code staleTimeoutMinutes} 与 {@code noResultUploadTimeoutMinutes} 在服务侧是 {@code @Value} + * 注入的非 final 字段,测试用 {@code ReflectionTestUtils.setField} 改它们;本类由服务每次入口调用时新建, + * 读的就是当时那一枚值——把阈值烤进构造更早的缓存里,测试当场会读旧数。 + * + *

本支独有的一格:候选 SQL 上的 owner 过滤。上架是多实例转发支,两节点各扫自己 + * {@code ownerInstanceId} 的行({@code result_json} 里那枚),所以两条线的候选查询共用同一个 + * {@code IS NULL / = '' / = 当前实例} 三条件块;本件把那一块收成一枚私有方法(两处逐字相同)。 + * 当前实例 id 用 {@link Supplier} 逐次取而不是构造时定值——它读的是 {@code InstanceMetadata}, + * 现网在查询发生的那一刻才取值,烤进来就把「注册后立刻生效」改成「重启才生效」。 + * + *

{@code moduleType()} 由调用方交出(本支是 {@code PUBLISH}),状态字面量一律走 + * {@link TaskStatuses}:{@code src/main} 里手写的 {@code STATUS_XXX} 标量是只许降的棘轮,降法就是接进这里。 + */ +@Slf4j +public class PublishStaleTaskScanner { + + /** 双节点互斥的作业锁名,逐字沿用服务侧现网那句(改名等于换一把锁)。 */ + private static final String STALE_CHECK_JOB_LOCK = "publish:stale-check"; + + /** 每轮作业锁的持有时长;Redis 不可用时 fail-open(只记一行日志),兜底靠下方 per-task 锁。 */ + private static final Duration STALE_CHECK_JOB_LOCK_TTL = Duration.ofMinutes(2); + + /** 单轮候选上限,逐字沿用现网那两句 {@code limit 100}。 */ + private static final String CANDIDATE_LIMIT = "limit 100"; + + /** 阈值下限:配置填了小于 5 的值也按 5 分钟判,逐字沿用现网那句 {@code Math.max(5, …)}。 */ + private static final long MIN_TIMEOUT_MINUTES = 5L; + + /** 每任务锁的等待时长,0=抢不到立刻放弃(不排队、不降级)。 */ + private static final long TASK_LOCK_NO_WAIT = 0L; + + /** 心跳线的对外错误文案,golden 与既有用例都按这句逐字对撞。 */ + private static final String HEARTBEAT_STALE_ERROR = "任务心跳超时"; + + /** + * 判死之后的收尾交回服务:{@code staleCheck} 是扫描侧算好的复查谓词, + * {@code error} 是那一线对外的文案。两者都由调用线各自给出,收尾路径同一条。 + */ + @FunctionalInterface + public interface StaleFinalizer { + void finalizeStale(Long taskId, Predicate staleCheck, String error); + } + + private final String moduleType; + private final long staleTimeoutMinutes; + private final long noResultUploadTimeoutMinutes; + private final Supplier currentInstanceId; + private final DistributedJobLockService distributedJobLockService; + private final FileTaskMapper fileTaskMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TaskDistributedLockService taskDistributedLockService; + private final TaskFileJobService taskFileJobService; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; + private final TransactionTemplate transactionTemplate; + + public PublishStaleTaskScanner(String moduleType, + long staleTimeoutMinutes, + long noResultUploadTimeoutMinutes, + Supplier currentInstanceId, + DistributedJobLockService distributedJobLockService, + FileTaskMapper fileTaskMapper, + TaskScopeStateMapper taskScopeStateMapper, + TaskDistributedLockService taskDistributedLockService, + TaskFileJobService taskFileJobService, + TaskHeartbeatPositionService taskHeartbeatPositionService, + TransactionTemplate transactionTemplate) { + this.moduleType = moduleType; + this.staleTimeoutMinutes = staleTimeoutMinutes; + this.noResultUploadTimeoutMinutes = noResultUploadTimeoutMinutes; + this.currentInstanceId = currentInstanceId; + this.distributedJobLockService = distributedJobLockService; + this.fileTaskMapper = fileTaskMapper; + this.taskScopeStateMapper = taskScopeStateMapper; + this.taskDistributedLockService = taskDistributedLockService; + this.taskFileJobService = taskFileJobService; + this.taskHeartbeatPositionService = taskHeartbeatPositionService; + this.transactionTemplate = transactionTemplate; + } + + /** + * 一轮完整扫描:先跑心跳线,再在同一把作业锁内跑二次判死线(现网就是这个次序)。 + * 二次线自己再取一次 {@code now()} 算心跳阈值——那不是疏漏,现网两条线各算各的, + * 本件照搬,不改判据也不改取值时刻。 + */ + public void sweep(StaleFinalizer finalizer) { + DistributedJobLockService.LockHandle jobLock = + distributedJobLockService.tryLock(STALE_CHECK_JOB_LOCK, STALE_CHECK_JOB_LOCK_TTL); + if (jobLock == null) { + log.info("[publish] stale-check skipped, another instance holds the distributed lock"); + return; + } + try (jobLock) { + LocalDateTime threshold = LocalDateTime.now().minusMinutes(effectiveStaleTimeoutMinutes()); + List staleTasks = fileTaskMapper.selectList(ownerScoped( + new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, moduleType) + .eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)) + .lt(FileTaskEntity::getUpdatedAt, threshold) + .orderByAsc(FileTaskEntity::getUpdatedAt) + .last(CANDIDATE_LIMIT)); + for (FileTaskEntity candidate : staleTasks) { + if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), moduleType) > 0L) { + continue; + } + try (TaskDistributedLockService.LockHandle lock = + taskDistributedLockService.acquire(moduleType, candidate.getId(), TASK_LOCK_NO_WAIT)) { + if (lock == null) { + LockSkipWitness.recordSkip("publish:stale-heartbeat-scan", moduleType, candidate.getId()); + continue; + } + transactionTemplate.executeWithoutResult(status -> finalizer.finalizeStale( + candidate.getId(), + task -> task.getUpdatedAt() != null && task.getUpdatedAt().isBefore(threshold), + HEARTBEAT_STALE_ERROR)); + } catch (Exception ex) { + log.warn("[publish] stale task cleanup failed taskId={} msg={}", + candidate.getId(), ex.getMessage()); + } + } + sweepNoUpload(finalizer); + } + } + + /** + * 二次判死:Python 心跳正常但连续 N 分钟无结果分片上报(治「心跳续命」的假活任务)。 + * + *

既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at—— + * 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。 + * owner 过滤与心跳线保持一致(避免双节点同时处理非本实例任务); + * 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报不再无条件跳过—— + * 与上报停摆同样按处理位置三分:仍在推进=首批较慢跳过、位置久未变=判死、无位置信息=证据不足保守跳过但留 WARN + * (原先的无条件跳过是 2026-10-06 品牌任务 2626 挂 9 小时的同类出口)。 + */ + public void sweepNoUpload(StaleFinalizer finalizer) { + long minutes = noResultUploadTimeoutMinutes; + if (minutes <= 0) { + return; + } + LocalDateTime heartbeatThreshold = LocalDateTime.now().minusMinutes(effectiveStaleTimeoutMinutes()); + LocalDateTime cutoff = LocalDateTime.now().minusMinutes(minutes); + List candidates = fileTaskMapper.selectList(ownerScoped( + new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, moduleType) + .eq(FileTaskEntity::getStatus, TaskStatuses.FileTask.RUNNING)) + .ge(FileTaskEntity::getUpdatedAt, heartbeatThreshold) + .lt(FileTaskEntity::getCreatedAt, cutoff) + .orderByAsc(FileTaskEntity::getCreatedAt) + .last(CANDIDATE_LIMIT)); + if (candidates.isEmpty()) { + return; + } + Map lastResultAtByTaskId = new LinkedHashMap<>(); + for (TaskScopeLastChunkDto dto : taskScopeStateMapper.selectLastChunkAtByTaskIds( + candidates.stream().map(FileTaskEntity::getId).toList())) { + if (dto.taskId() != null && dto.lastChunkAt() != null) { + lastResultAtByTaskId.put(dto.taskId(), dto.lastChunkAt()); + } + } + for (FileTaskEntity candidate : candidates) { + LocalDateTime lastResultAt = lastResultAtByTaskId.get(candidate.getId()); + if (lastResultAt != null && lastResultAt.isAfter(cutoff)) { + // 上报仍新鲜:正常推进,不必处理 + continue; + } + if (lastResultAt == null) { + // 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue(零日志), + // 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块心跳同样照打)。 + // 现按处理位置三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪; + // 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。 + String neverReportedPosition = taskHeartbeatPositionService.describe(candidate.getId()); + if (taskHeartbeatPositionService.isClientAdvancing(candidate.getId(), minutes)) { + log.info("[publish] 从未上报但客户端仍在推进,本轮不判死 taskId={} position={} minutes={}", + candidate.getId(), neverReportedPosition, minutes); + continue; + } + if (neverReportedPosition == null || neverReportedPosition.isBlank()) { + log.warn("[publish] 从未上报且无位置信息,证据不足暂不判死 taskId={} createdAt={} minutes={}", + candidate.getId(), candidate.getCreatedAt(), minutes); + continue; + } + finalizeNoUploadStale(candidate, + "任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「" + + neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败", + task -> { + // 排队期间复查:心跳回落陈旧 → 交回心跳线;出现任一结果上报 → 已恢复回传,交回原线评估 + if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) { + return false; + } + return queryLastChunkAt(task.getId()) == null; + }, finalizer); + continue; + } + finalizeNoUploadStale(candidate, + "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")", + task -> { + // 排队期间复查:心跳回落陈旧 → 交回心跳线;出现新结果上报 → 任务已恢复 + if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) { + return false; + } + LocalDateTime freshLastChunk = queryLastChunkAt(task.getId()); + return freshLastChunk != null && !freshLastChunk.isAfter(cutoff); + }, finalizer); + } + } + + /** 二次判死共同尾部:待组装检查 → 任务锁 → 事务内收尾;error 与复查谓词由两条路各自提供。 */ + private void finalizeNoUploadStale(FileTaskEntity candidate, + String error, + Predicate staleCheck, + StaleFinalizer finalizer) { + if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), moduleType) > 0L) { + return; + } + try (TaskDistributedLockService.LockHandle lock = + taskDistributedLockService.acquire(moduleType, candidate.getId(), TASK_LOCK_NO_WAIT)) { + if (lock == null) { + LockSkipWitness.recordAbandoned("publish:no-upload-stale-finalize", moduleType, candidate.getId()); + return; + } + transactionTemplate.executeWithoutResult(status -> finalizer.finalizeStale( + candidate.getId(), staleCheck, error)); + } catch (Exception ex) { + log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}", + candidate.getId(), ex.getMessage()); + } + } + + /** 单任务最后一次结果分片上报时间;无记录返回 null。 */ + private LocalDateTime queryLastChunkAt(Long taskId) { + List rows = taskScopeStateMapper.selectLastChunkAtByTaskIds(List.of(taskId)); + return rows.isEmpty() ? null : rows.get(0).lastChunkAt(); + } + + /** + * 两条候选查询共用的 owner 作用域块:本实例的行,或 {@code ownerInstanceId} 还没写上的行。 + * 现网两处逐字相同,收成一处;SQL 文本与参数序一字未动。 + */ + private LambdaQueryWrapper ownerScoped(LambdaQueryWrapper query) { + return query.and(owner -> owner + .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL") + .or() + .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''") + .or() + .apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId.get())); + } + + private long effectiveStaleTimeoutMinutes() { + return Math.max(MIN_TIMEOUT_MINUTES, staleTimeoutMinutes); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java index 9331b07bd..b919e4e5c 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java @@ -100,7 +100,8 @@ class SqlLayeringConventionTest { private static Map rawSqlInWrappersBaseline() { Map baseline = new LinkedHashMap<>(); - baseline.put("PublishTaskService", 6); // ownerInstanceId 的 JSON_EXTRACT 谓词 + baseline.put("PublishStaleTaskScanner", 3); // ownerInstanceId 的 JSON_EXTRACT 谓词:3D 前置抽件把 PublishTaskService 那 6 处 + // (心跳线+二次判死线各 3 处逐字相同)收成共用一枚 ownerScoped ⇒ 整树 6→3。 baseline.put("SimilarAsinTaskOwnershipSupport", 3); // 同上,state_json 维度 baseline.put("ShopDataCrawlDailyFileService", 4); // last("FOR UPDATE") 行锁 baseline.put("TaskScopePayloadStorageService", 1); // last("limit 1 FOR UPDATE") 混写