fix(b28): 终态任务的结果作业不再被领取、锁停止续期

现网 FAILED 任务 32217/32220/32226/32229 的合批 job 仍在逐行申领,
在持锁续到 86 分钟。两条 claim 入口先读父任务状态:终态则拉满重试并打
终态回调,不再翻转 RUNNING;执行中由心跳复查,终态后停止看门狗续期
(不释放锁,仍由带 token 的 close 收尾)。父任务状态查询进 mapper XML,
SQL 分层账本 128→129、等价账本 88→89 同步登记。
This commit is contained in:
2026-10-25 00:22:09 +08:00
parent f93e7d95fc
commit 8881f63839
10 changed files with 292 additions and 4 deletions
@@ -3,7 +3,15 @@ package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@Mapper
public interface TaskFileJobMapper extends BaseMapper<TaskFileJobEntity> {
/**
* 只读父任务状态:结果作业的 taskId 落在 12 个客户端模块共用的 {@code biz_file_task}。
* 品牌任务的 job 查不到行(返回 null),调用方按"非终态"处理,行为不变。
* 语句在 {@code mapper/TaskFileJobMapper.xml}(本仓约定:自定义 SQL 一律进 XML)。
*/
String selectFileTaskStatus(@Param("taskId") Long taskId);
}
@@ -324,6 +324,21 @@ public class TaskDistributedLockService {
this.heldLock = heldLock;
}
/**
* 停止看门狗续期,但**不释放**锁:父任务已终态时,本节点不该再给这把锁续命;
* 仍由 {@link #close()} 那次带 token 的 RELEASE 收尾,避免把接管者的锁误删。
*/
public void stopRenewal(String reason) {
if (heldLock.closed.compareAndSet(false, true)) {
cancelRenewal(heldLock);
activeLocks.markRenewalLost(heldLock.lockName);
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={}",
heldLock.lockName,
System.currentTimeMillis() - heldLock.acquiredAtMillis,
reason);
}
}
@Override
public void close() {
if (closed) {
@@ -221,6 +221,26 @@ public class TaskFileJobService {
return claimCandidates(candidates);
}
/**
* 父任务是否已终态。终态任务的作业不得再被领取、锁不得再续期(B28):
* 现网 FAILED 任务 32217/32220/32226/32229 的合批 job 仍在逐行申领,锁续到 86 分钟。
*/
public boolean isOwningTaskTerminal(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
String status = taskFileJobMapper.selectFileTaskStatus(taskId);
return status != null && TaskStatuses.FileTask.TERMINAL.contains(status);
}
/** 终态父任务的 job 直接拉满重试并打终态回调:不执行、不再被下一轮扫到。 */
private void skipJobOfTerminalTask(TaskFileJobEntity job, String stage) {
String message = "父任务已终态,结果作业不再执行";
markFailedPermanent(job, message);
log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage={}",
job.getId(), job.getTaskId(), job.getModuleType(), stage);
}
private List<TaskFileJobEntity> claimCandidates(List<TaskFileJobEntity> candidates) {
if (candidates == null || candidates.isEmpty()) {
return List.of();
@@ -230,6 +250,10 @@ public class TaskFileJobService {
if (candidate == null || candidate.getId() == null) {
continue;
}
if (isOwningTaskTerminal(candidate.getTaskId())) {
skipJobOfTerminalTask(candidate, "CLAIM");
continue;
}
if (markRunning(candidate.getId())) {
claimedIds.add(candidate.getId());
}
@@ -547,6 +571,14 @@ public class TaskFileJobService {
/** Returns the persisted RUNNING row so queued execution can later fence itself with updatedAt. */
public TaskFileJobEntity claimRunning(Long jobId) {
if (jobId == null || jobId <= 0) {
return null;
}
TaskFileJobEntity existing = taskFileJobMapper.selectById(jobId);
if (existing != null && isOwningTaskTerminal(existing.getTaskId())) {
skipJobOfTerminalTask(existing, "CLAIM_RUNNING");
return null;
}
if (!markRunning(jobId)) {
return null;
}
@@ -65,6 +65,10 @@ public class TaskResultFileJobWorker {
return thread;
});
/** 本进程正在执行的 job → 它持有的任务锁;心跳发现父任务终态时用它停续期(B28)。 */
private final java.util.concurrent.ConcurrentHashMap<Long, TaskDistributedLockService.LockHandle> activeLockHandles =
new java.util.concurrent.ConcurrentHashMap<>();
@Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}")
public void runPendingJobs() {
if (!localWorkerEnabled) {
@@ -197,6 +201,14 @@ public class TaskResultFileJobWorker {
long interval = Math.max(1000L, heartbeatIntervalMillis);
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
try {
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
if (handle != null) {
handle.stopRenewal("parent-task-terminal");
log.info("[task-file-job] stopped lock renewal because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
job.getId(), job.getTaskId(), job.getModuleType());
}
}
taskFileJobService.touchRunning(job.getId());
} catch (Exception ex) {
log.warn("[task-file-job] heartbeat failed jobId={} taskId={} moduleType={} msg={} stage=HEARTBEAT",
@@ -206,6 +218,12 @@ public class TaskResultFileJobWorker {
}
private void processClaimedWithHeartbeat(TaskFileJobEntity job, TaskFileJobEntity claim) {
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_SKIP",
job.getId(), job.getTaskId(), job.getModuleType());
return;
}
if (!taskFileJobService.activateRunningClaim(claim)) {
log.info("[task-file-job] queued claim expired before execution jobId={} taskId={} moduleType={} claimUpdatedAt={}",
job.getId(), job.getTaskId(), job.getModuleType(), claim.getUpdatedAt());
@@ -247,6 +265,15 @@ public class TaskResultFileJobWorker {
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
return;
}
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
try (lockHandle) {
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
}
log.info("[task-file-job] skipped after acquiring lock because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_AFTER_LOCK",
job.getId(), job.getTaskId(), job.getModuleType());
return;
}
activeLockHandles.put(job.getId(), lockHandle);
try (lockHandle) {
latest = taskFileJobService.findById(job.getId());
if (latest != null && "SUCCESS".equals(latest.getStatus())) {
@@ -283,6 +310,8 @@ public class TaskResultFileJobWorker {
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt, resultFileUrl);
}
} finally {
activeLockHandles.remove(job.getId(), lockHandle);
}
// 成功后回调解耦(原 WITHDRAW 特判):时机与历史一致——锁已释放、markSuccess 之后;
// 各模块需要"任务收尾"时实现 ResultFileJobHandler.onSuccess,task 侧不再 import 业务模块
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper">
<!-- B28:终态父任务的 job 不得再被领取;只读一列,按主键点查。 -->
<select id="selectFileTaskStatus" resultType="java.lang.String">
SELECT status FROM biz_file_task WHERE id = #{taskId}
</select>
</mapper>
@@ -61,7 +61,9 @@ class MigratedSqlEquivalenceTest {
// (与 SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 99 → 115 → 124)。
// 86 → 88:损耗监控在 UserSecretUsageMapper 新写 2 条聚合 select,逐条点入
// (SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 124 → 126)。
private static final int XML_SELECT_TOTAL = 88;
// 88 → 89:B28 新增 TaskFileJobMapper#selectFileTaskStatus(按主键读父任务状态),逐条点入
// (SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 128 → 129)。
private static final int XML_SELECT_TOTAL = 89;
/**
* 不是「从注解搬过来」的语句:注解时代没有原文可对撞,等价性这支只能跳过它们。
@@ -97,7 +99,9 @@ class MigratedSqlEquivalenceTest {
"BrandGatingMapper#countGroupBySource",
// 损耗监控(2026-10-24)新写的 2 条聚合 select:当日全量扫描 + 单用户读数。
"UserSecretUsageMapper#sumByUserAndModule",
"UserSecretUsageMapper#sumForUser");
"UserSecretUsageMapper#sumForUser",
// B28(2026-10-25)新写的 1 条主键点查:父任务是否已终态。
"TaskFileJobMapper#selectFileTaskStatus");
/** 账本 5 列:接口 / 方法 / @Select 种类 / 是否含常量插值 / 注解原文(Java 反转义后那条串)。 */
private static final int LEDGER_COLUMNS = 5;
@@ -78,7 +78,9 @@ class SqlLayeringConventionTest {
// 124 → 126:损耗监控在 {@code UserSecretUsageMapper} 新写 2 条聚合 select
// (当日全量扫描 + 单用户读数),同样逐条登记进 NOT_MIGRATED(86 → 88)。
// 126 → 128:同轮 {@code UserTaskGuardMapper} 的 pause/resume 两条写侧语句从注解迁进 XML。
private static final int XML_STATEMENT_TOTAL = 128;
// 128 → 129:B28 新增 {@code TaskFileJobMapper#selectFileTaskStatus}(按主键读父任务状态),
// 新写语句按约定直接进 XML,同轮登记进 MigratedSqlEquivalenceTest.NOT_MIGRATED(88 → 89)。
private static final int XML_STATEMENT_TOTAL = 129;
/**
* wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数:
@@ -100,8 +100,10 @@ class TaskResultFileJobWorkerLogAuditTest {
}
// 阶段字面量来自允许集合(防新增阶段拼写漂移)
// SUCCESS_LOST:结果文件已生成但作业行不在本次执行手里(2026-10-09 任务 30456 的静默丢失)
// 2026-10-24 B28:父任务终态后不再领取/执行/续期,三条出口各有一个阶段字面量。
List<String> allowed = List.of("SKIP_OWNER", "REQUEUE", "WAIT_LLM", "DEFER", "SUCCESS",
"SUCCESS_LOST", "ORPHAN", "FAILED", "FINALIZE", "HEARTBEAT", "FALLBACK");
"SUCCESS_LOST", "ORPHAN", "FAILED", "FINALIZE", "HEARTBEAT", "FALLBACK",
"TERMINAL_SKIP", "TERMINAL_AFTER_LOCK", "TERMINAL_STOP_RENEW");
for (String l : perJob) {
if (l.contains("stage=")) {
int idx = l.indexOf("stage=");
@@ -0,0 +1,48 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* B28:父任务终态后,心跳调用的 {@code stopRenewal} 必须真的让看门狗停下,
* 但**不释放**锁(释放仍由带 token 的 close 收尾,避免误删接管者的锁)。
*/
class TaskDistributedLockServiceStopRenewalTest {
@Test
void stopRenewalStopsTheWatchdogWithoutReleasingTheLock() throws Exception {
AtomicInteger renewCalls = new AtomicInteger();
DistributedJobLockService delegate = mock(DistributedJobLockService.class);
DistributedJobLockService.LockHandle inner = mock(DistributedJobLockService.LockHandle.class);
when(inner.renew(any(Duration.class))).thenAnswer(invocation -> {
renewCalls.incrementAndGet();
return true;
});
when(delegate.tryLock(anyString(), any(Duration.class))).thenReturn(inner);
TaskDistributedLockService service = new TaskDistributedLockService(delegate);
TaskDistributedLockService.LockHandle handle =
service.acquire("appearance-patent", 32220L, Duration.ofSeconds(1), 0);
assertTrue(service.activeHeldMillis().containsKey("task:appearance-patent:32220"));
handle.stopRenewal("parent-task-terminal");
Thread.sleep(1_400L);
assertEquals(0, renewCalls.get(), "stopRenewal 之后看门狗不得再续期");
assertTrue(service.activeHeldMillis().containsKey("task:appearance-patent:32220"),
"停续期不等于释放:锁仍在持,由 close 收尾");
handle.close();
assertTrue(service.activeHeldMillis().isEmpty(), "close 仍要正常释放");
}
}
@@ -0,0 +1,138 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.ApplicationEventPublisher;
import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* B28:父任务进入终态后,其结果作业不得再被领取。
*
* <p>现网读数(2026-10-24):任务 32217/32220/32226/32229 已是 FAILED,
* 同刻仍在打「候选收集逐行申领」、锁续到 86 分钟。这里把两条 claim 入口
* (claimCandidates / claimRunning)都钉住:终态任务的行不再翻转成 RUNNING,
* 而是拉满重试并打终态回调,下一轮扫描不会再捞起来。
*/
@ExtendWith(MockitoExtension.class)
class TaskFileJobTerminalTaskGuardTest {
@Mock private TaskFileJobMapper taskFileJobMapper;
@Mock private ApplicationEventPublisher applicationEventPublisher;
private TaskFileJobService service;
@BeforeAll
static void initializeTableInfo() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
TaskFileJobEntity.class);
}
@BeforeEach
void setUp() {
service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
}
private static TaskFileJobEntity job(Long id, Long taskId) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setTaskId(taskId);
job.setResultId(2000L + id);
job.setModuleType("APPEARANCE_PATENT");
job.setJobType("ASSEMBLE_RESULT");
job.setStatus("PENDING");
job.setRetryCount(0);
job.setUpdatedAt(LocalDateTime.now());
return job;
}
private static boolean hasParam(LambdaUpdateWrapper<TaskFileJobEntity> wrapper, Object value) {
return wrapper.getParamNameValuePairs().containsValue(value);
}
@Test
void claimRunningRejectsJobWhoseTaskIsFailed() {
TaskFileJobEntity failedTaskJob = job(11L, 500L);
when(taskFileJobMapper.selectById(11L)).thenReturn(failedTaskJob);
when(taskFileJobMapper.selectFileTaskStatus(500L)).thenReturn("FAILED");
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
assertNull(service.claimRunning(11L), "终态任务的作业不得被 claim 成 RUNNING");
@SuppressWarnings("unchecked")
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> captor =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(taskFileJobMapper, times(1)).update(isNull(), captor.capture());
LambdaUpdateWrapper<TaskFileJobEntity> wrapper = captor.getValue();
assertTrue(hasParam(wrapper, "FAILED"), "必须把作业拉成 FAILED,下一轮扫描才捞不到");
assertTrue(hasParam(wrapper, 5), "retry_count 必须一次拉满到上限");
assertFalse(hasParam(wrapper, "RUNNING"), "不得再翻转成 RUNNING");
}
@Test
void claimCandidatesSkipsTerminalTaskAndKeepsActiveOne() {
TaskFileJobEntity terminal = job(21L, 500L);
TaskFileJobEntity active = job(22L, 501L);
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
.thenReturn(List.of(terminal, active));
when(taskFileJobMapper.selectFileTaskStatus(500L)).thenReturn("FAILED");
when(taskFileJobMapper.selectFileTaskStatus(501L)).thenReturn("RUNNING");
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1);
TaskFileJobEntity claimedActive = job(22L, 501L);
claimedActive.setStatus("RUNNING");
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(claimedActive));
List<TaskFileJobEntity> claimed = service.claimRunnableJobs(10);
assertEquals(1, claimed.size(), "终态任务的 job 不入结果集");
assertEquals(22L, claimed.get(0).getId());
@SuppressWarnings("unchecked")
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> captor =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(taskFileJobMapper, times(2)).update(isNull(), captor.capture());
assertTrue(hasParam(captor.getAllValues().get(0), "FAILED"), "第一笔是终态收尾");
assertTrue(hasParam(captor.getAllValues().get(1), "RUNNING"), "第二笔才是正常 claim");
}
@Test
void unknownTaskRowIsNotTreatedAsTerminal() {
// 品牌任务的 taskId 不在 biz_file_task(查询返回 null):按非终态走,行为不变。
when(taskFileJobMapper.selectFileTaskStatus(900L)).thenReturn(null);
assertFalse(service.isOwningTaskTerminal(900L));
verify(taskFileJobMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
}
@Test
void nullAndNonPositiveTaskIdAreNotTerminal() {
assertFalse(service.isOwningTaskTerminal(null));
assertFalse(service.isOwningTaskTerminal(0L));
assertFalse(service.isOwningTaskTerminal(-1L));
}
}