From 37f51df37df99ce9a733a9668725bbcee638db91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 2 Nov 2026 22:53:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(owner):=20=E8=B7=A8=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E5=BD=92=E5=B1=9E=E8=83=8C=E7=A6=BB=E6=94=B9=E4=B8=BA=E8=BD=AC?= =?UTF-8?q?=E4=BA=A4=E4=BD=9C=E4=B8=9A=EF=BC=8C=E4=B8=8D=E5=86=8D=E6=97=A0?= =?UTF-8?q?=E9=99=90=20defer=20=E7=A9=BA=E8=BD=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 领取按 biz_task_file_job.owner、执行时校验按 biz_file_task.owner_instance_id, 任务转交后两列一旦背离就是互斥死锁:能领到这行的实例必被校验拒,校验放过的 实例必领不到这行;而拒绝分支 deferRunning 既不计 retry_count 也不计 stuck_requeue_count 还每轮刷新 updated_at,卡死重排与 PENDING 遗骸两条兜底 因此永不触发。现网外观专利 33179/33149 停在 PENDING 每 15 秒被拒一次,单机 12 小时 6381 条拒绝,进度快照冻结 4 小时 19 分,既不失败也不完成。 - TaskFileJobService: 新增 handOffToOwner(owner 列与 scope_key 尾段一起改挂, CAS 按观测到的旧 owner 回撞)与 withOwnerScopeKey(ownerFromScopeKey 的逆操作); enqueueAssembleResult 的 PENDING 分支改为按入队 scopeKey 对账归属——Python 超时恢复每 2 分钟走这一格一次,原先只补发事件不对账等于每 2 分钟确认死锁还在。 - TaskResultFileJobWorker: 归属拒绝改为先改挂再退回。转交后两列重合,拒绝条件 按构造不再成立,因此不需要额外计数兜底。 - 相似ASIN/发布 ownerFromTask 改为 owner_instance_id 列优先:原先只读 resultJson 快照(转交后不再同步),转交目标会算错、死锁只是换个方向。店铺采集本就列优先,未动。 判据:TaskFileJobOwnerHandOffTest(11) + TaskResultFileJobWorkerOwnerHandOffTest(3) + TaskOwnerReadSourceGuardrailTest(5,含四台校验源同源的源码绊线,已反向验证 "改回只读快照即红")。SourceScanTest 台账按设计登记新消费方:裸名 22→23、 调用方 16→17。后端整量 5206 支 0 失败 19 skipped。 --- .../publish/service/PublishTaskService.java | 18 +- .../SimilarAsinTaskOwnershipSupport.java | 22 +- .../task/service/TaskFileJobService.java | 93 +++++++- .../task/service/TaskResultFileJobWorker.java | 15 +- .../service/TaskFileJobOwnerHandOffTest.java | 225 ++++++++++++++++++ .../TaskOwnerReadSourceGuardrailTest.java | 88 +++++++ ...skResultFileJobWorkerOwnerHandOffTest.java | 132 ++++++++++ .../aiimage/testutil/SourceScanTest.java | 20 +- 8 files changed, 597 insertions(+), 16 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobOwnerHandOffTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskOwnerReadSourceGuardrailTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerHandOffTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java index fc1bdb5d6..ebb77fb65 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java @@ -1322,8 +1322,24 @@ public class PublishTaskService extends AbstractFileTaskPipeline task == null ? null : task.getId(), operation, owner, currentInstanceId()); } + /** + * 归属实例 id:优先读显式列 {@code owner_instance_id},列缺失时才回落 {@code resultJson} 快照。 + * + *

原先只读快照,而快照是建单那一刻写死的、任务转交后不再同步(转交写的是列)。 + * 本方法给出的值同时是 worker 侧转交作业的目标实例({@code TaskFileJobService#handOffToOwner} + * 按 {@code TaskOwnerMismatchException#getOwnerInstanceId()} 改挂),读陈旧快照会把作业转交给 + * 已经不管这行的实例,死锁只是换了个方向。与 {@code AppearancePatentTaskOwnership#ownerFromTask} + * 同一口径。 + */ private String ownerFromTask(FileTaskEntity task) { - if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) { + if (task == null) { + return null; + } + String column = task.getOwnerInstanceId(); + if (column != null && !column.isBlank()) { + return column.trim(); + } + if (task.getResultJson() == null || task.getResultJson().isBlank()) { return null; } try { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskOwnershipSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskOwnershipSupport.java index e240fb3b4..82a5ce271 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskOwnershipSupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinTaskOwnershipSupport.java @@ -115,8 +115,25 @@ public class SimilarAsinTaskOwnershipSupport { return context == null ? null : context.ownerInstanceId(); } + /** + * 归属实例 id:优先读显式列 {@code owner_instance_id},列缺失时才回落 {@code resultJson} 快照。 + * + *

原先只读快照,而快照是 parse 那一刻写死的、任务转交后不再同步 + * ({@code TaskClientPullService.claim} 转交的是列)。两列一旦背离,本方法会把已不归属本机的 + * 任务判给本机、或把已归属本机的任务判给别台;worker 侧按本方法给出的值转交作业时会转错实例 + * (外观专利 2026-11-02 现网 33179/33149 即此形状,见 + * {@code TaskFileJobService#handOffToOwner})。与 {@code AppearancePatentTaskOwnership#ownerFromTask} + * 同一口径,四台不得再各读各的。 + */ public String ownerFromTask(FileTaskEntity task) { - if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) { + if (task == null) { + return null; + } + String column = task.getOwnerInstanceId(); + if (column != null && !column.isBlank()) { + return column.trim(); + } + if (task.getResultJson() == null || task.getResultJson().isBlank()) { return null; } try { @@ -125,6 +142,7 @@ public class SimilarAsinTaskOwnershipSupport { } catch (Exception ex) { log.warn("[similar-asin] read task owner failed taskId={} err={}", task.getId(), ex.getMessage()); return null; + } } /** @@ -133,8 +151,6 @@ public class SimilarAsinTaskOwnershipSupport { * isJobOwnedByCurrentInstance / ownerFromTask 兼容这条 ownerInstanceId)。 * 同时把 fallback 信号写到 task_scope_state.state_json,便于排查。 */ - } - public void bindTaskToCurrentOwnerForLocalFallback(FileTaskEntity task, String scopeHash, Integer chunkIndex) { if (task == null || task.getId() == null) { throw new IllegalStateException("Cannot bind local fallback without a task ID"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index 1813a2431..1b9f752d4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -71,11 +71,36 @@ public class TaskFileJobService { return existing; } if ("PENDING".equals(existing.getStatus())) { + // 归属对账:调用方传进来的 scopeKey 是按任务**当前** owner 现算的 + // (各模块的 buildTaskOwnerScopeKey),而 existing 那两列是入队那一刻的快照。 + // 任务转交过之后不在这里对齐,就会留下「领取按 job.owner、执行校验按 + // biz_file_task.owner_instance_id」的互斥死锁(2026-11-02 现网 33179/33149: + // 单机 12 小时 6381 条拒绝、作业永不终态),而 Python 超时恢复每 2 分钟 + // 就会走到这一格一次——只补发事件不修归属,等于每 2 分钟确认一遍死锁还在。 + String expectedOwner = ownerFromScopeKey(scopeKey); + TaskFileJobEntity pending = existing; + if (expectedOwner != null && !expectedOwner.equals( + existing.getOwner() == null ? null : existing.getOwner().trim())) { + taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, existing.getId()) + .eq(TaskFileJobEntity::getStatus, "PENDING") + .set(TaskFileJobEntity::getScopeKey, scopeKey) + .set(TaskFileJobEntity::getOwner, expectedOwner) + .set(TaskFileJobEntity::getErrorMessage, null) + .set(TaskFileJobEntity::getUpdatedAt, now)); + pending = taskFileJobMapper.selectById(existing.getId()); + log.warn("[task-file-job] PENDING 作业归属与任务当前 owner 背离,已按入队 scopeKey 对账" + + " jobId={} taskId={} moduleType={} from={} to={}", + existing.getId(), taskId, moduleType, existing.getOwner(), expectedOwner); + if (pending == null) { + return null; + } + } // PENDING 僵尸兜底:job 入队时若 MQ 发送失败且本地兜底未生效(或消息被 // 非 owner 实例静默吞掉),job 会永远停留在 PENDING、无人处理。 // 补发一次 dispatch 事件,让调度链重试(幂等:consumer 侧 claim 有状态翻转保护)。 - publishDispatchEvent(existing); - return existing; + publishDispatchEvent(pending); + return pending; } // 终态失败:retryCount 已达上限的 FAILED job 不再重置为 PENDING, // 否则 finalize 路径会把它反复拉回排队,导致 retryCount 无限累加、任务永远卡在 RUNNING。 @@ -1019,6 +1044,70 @@ public class TaskFileJobService { return owner.isBlank() ? null : owner; } + /** + * {@link #ownerFromScopeKey} 的逆操作:把 scope_key 最后一个 {@code :owner:} 之后的尾段换成指定实例。 + * 无标记时原样返回——那种 scope_key 本来就不携带归属信息,不该被凭空造出一个 owner 段。 + */ + static String withOwnerInScopeKey(String scopeKey, String owner) { + if (scopeKey == null || scopeKey.isBlank() || owner == null || owner.isBlank()) { + return scopeKey; + } + int index = scopeKey.lastIndexOf(OWNER_MARKER); + if (index < 0) { + return scopeKey; + } + return (scopeKey.substring(0, index + OWNER_MARKER.length()) + owner.trim()).trim(); + } + + /** + * 把作业改挂到任务的当前归属实例:{@code owner} 列与 {@code scope_key} 的尾段一起换。 + * + *

为什么必须两个来源一起改。领取按 {@code biz_task_file_job.owner} + * ({@link #claimRunnableJobsForOwner}),执行时的归属校验却按 {@code biz_file_task.owner_instance_id}。 + * 任务转交后两列一旦背离,就是互斥死锁:能领到这行的实例必被校验拒,校验放过的实例必领不到这行。 + * 2026-11-02 现网任务 33179/33149 即此形状——作业停在 PENDING、每 15 秒被拒一次, + * 单机 12 小时 6381 条拒绝,而拒绝走的 {@link #deferRunning} 既不计 retry_count + * 也不计 stuck_requeue_count 还每轮刷新 updated_at,两条兜底(卡死重排 20 次判死、 + * PENDING 遗骸回收)因此永远不触发:这行既不失败也不完成。 + * + *

为什么不需要额外计数兜底。转交后 {@code job.owner} 与校验读的那个值重合, + * 拒绝条件按构造不再成立;要再触发一次,必须任务先合法地再转交一回。 + * + *

CAS 按调用方观测到的旧 owner 回撞:期间任务又转交、或作业被别的实例改挂,本次就不生效, + * 由下一轮拒绝重新转交,不覆盖别人的改动。 + * + * @return true=本次确实改挂了;false=无需改挂、行已不在预期状态或目标实例为空 + */ + public boolean handOffToOwner(Long jobId, String observedOwner, String targetOwner) { + if (jobId == null || jobId <= 0 || targetOwner == null || targetOwner.isBlank()) { + return false; + } + TaskFileJobEntity job = taskFileJobMapper.selectById(jobId); + if (job == null) { + return false; + } + String normalizedTarget = targetOwner.trim(); + String currentOwner = job.getOwner() == null ? "" : job.getOwner().trim(); + if (normalizedTarget.equals(currentOwner)) { + return false; + } + String normalizedObserved = observedOwner == null ? "" : observedOwner.trim(); + LambdaUpdateWrapper wrapper = new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, jobId) + .eq(TaskFileJobEntity::getStatus, "RUNNING") + .set(TaskFileJobEntity::getOwner, normalizedTarget) + .set(TaskFileJobEntity::getScopeKey, withOwnerInScopeKey(job.getScopeKey(), normalizedTarget)) + .set(TaskFileJobEntity::getErrorMessage, null) + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()); + if (normalizedObserved.isEmpty()) { + wrapper.and(w -> w.isNull(TaskFileJobEntity::getOwner) + .or().eq(TaskFileJobEntity::getOwner, "")); + } else { + wrapper.eq(TaskFileJobEntity::getOwner, normalizedObserved); + } + return taskFileJobMapper.update(null, wrapper) > 0; + } + private TaskFileJobEntity findJob(Long taskId, String moduleType, Long resultId, String jobType) { if (taskId == null || moduleType == null || moduleType.isBlank() || resultId == null || jobType == null || jobType.isBlank()) { return null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index a3b0325a9..17c4d8b46 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -375,9 +375,18 @@ public class TaskResultFileJobWorker { } } catch (TaskOwnerMismatchException ex) { // 归属校验失败不是"生成失败":不能计入 retryCount,否则任务会在归属机 - // 一切正常的情况下被本机重试耗尽判死。退回 PENDING 交还归属实例。 - log.warn("[task-file-job] process rejected because task belongs to another instance jobId={} taskId={} moduleType={} owner={} current={}", - job.getId(), job.getTaskId(), job.getModuleType(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId()); + // 一切正常的情况下被本机重试耗尽判死。 + // 但也不能只退回 PENDING —— 那是无限空转:领取按 job.owner、这一道校验按任务的 + // owner_instance_id,两列一旦背离,能领到这行的实例必被校验拒、校验放过的实例 + // 必领不到这行(2026-11-02 现网 33179/33149:作业停 PENDING 每 15 秒被拒一次, + // 单机 12 小时 6381 条拒绝,而 deferRunning 不计任何计数还刷 updated_at, + // 卡死重排与 PENDING 遗骸两条兜底因此永不触发——既不失败也不完成)。 + // 异常携带的 ownerInstanceId 就是校验刚读到的任务当前归属,按它改挂即可闭合。 + boolean handedOff = taskFileJobService.handOffToOwner( + job.getId(), ownerFromScopeKey(job.getScopeKey()), ex.getOwnerInstanceId()); + log.warn("[task-file-job] process rejected because task belongs to another instance jobId={} taskId={} moduleType={} owner={} current={} handedOff={}", + job.getId(), job.getTaskId(), job.getModuleType(), ex.getOwnerInstanceId(), + ex.getCurrentInstanceId(), handedOff); taskFileJobService.deferRunning(job.getId(), ex.getMessage()); } catch (Exception ex) { TaskFileJobEntity latest = taskFileJobService.findById(job.getId()); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobOwnerHandOffTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobOwnerHandOffTest.java new file mode 100644 index 000000000..7c39a0d3f --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobOwnerHandOffTest.java @@ -0,0 +1,225 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; + +import java.time.LocalDateTime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 跨实例 owner 死锁的防回归判据(2026-11-02 现网任务 33179/33149)。 + * + *

形状:领取按 {@code biz_task_file_job.owner},执行时的归属校验按 + * {@code biz_file_task.owner_instance_id}。任务转交后两列背离——能领到这行的实例必被校验拒, + * 校验放过的实例必领不到这行;拒绝走的 {@code deferRunning} 既不计 retry_count 也不计 + * stuck_requeue_count 还每轮刷新 updated_at,两条兜底因此永不触发,作业既不失败也不完成 + * (单机 12 小时 6381 条拒绝)。 + * + *

把 {@link TaskFileJobService#handOffToOwner} 改回「只退回 PENDING 不改挂」、 + * 或把 {@link TaskFileJobService#enqueueAssembleResult} 的 PENDING 分支改回「只补发事件不对账」, + * 本类即红。 + */ +@ExtendWith(MockitoExtension.class) +class TaskFileJobOwnerHandOffTest { + + private static final String STALE_OWNER = "server-110"; + private static final String TASK_OWNER = "server-121"; + + @Mock private TaskFileJobMapper taskFileJobMapper; + @Mock private ApplicationEventPublisher applicationEventPublisher; + + private TaskFileJobService service; + + @BeforeAll + static void initializeTableInfo() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + TaskFileJobEntity.class); + } + + @BeforeEach + void setUp() { + service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher); + } + + private static TaskFileJobEntity job(Long id, Long taskId, String status, String owner) { + TaskFileJobEntity entity = new TaskFileJobEntity(); + entity.setId(id); + entity.setTaskId(taskId); + entity.setModuleType("APPEARANCE_PATENT"); + entity.setResultId(36102L); + entity.setJobType("ASSEMBLE_RESULT"); + entity.setStatus(status); + entity.setOwner(owner); + entity.setScopeKey(owner == null ? "task:" + taskId : "task:" + taskId + ":owner:" + owner); + entity.setRetryCount(0); + entity.setUpdatedAt(LocalDateTime.now()); + return entity; + } + + // ==================== withOwnerInScopeKey:ownerFromScopeKey 的逆操作 ==================== + + @Test + void test_scope_key_只换最后一个_owner_尾段() { + assertEquals("task:33179:owner:" + TASK_OWNER, + TaskFileJobService.withOwnerInScopeKey("task:33179:owner:" + STALE_OWNER, TASK_OWNER)); + // Java 侧 ownerFromScopeKey 用 lastIndexOf,所以逆操作也必须只改最后一段 + assertEquals("task:1:owner:x:owner:" + TASK_OWNER, + TaskFileJobService.withOwnerInScopeKey("task:1:owner:x:owner:" + STALE_OWNER, TASK_OWNER)); + } + + @Test + void test_scope_key_无标记或目标为空时不得凭空造_owner_段() { + assertEquals("task:1:plain", TaskFileJobService.withOwnerInScopeKey("task:1:plain", TASK_OWNER)); + assertEquals("task:1:owner:" + STALE_OWNER, + TaskFileJobService.withOwnerInScopeKey("task:1:owner:" + STALE_OWNER, " ")); + assertNull(TaskFileJobService.withOwnerInScopeKey(null, TASK_OWNER)); + } + + @Test + void test_逆操作与提取函数必须互逆() { + String rebuilt = TaskFileJobService.withOwnerInScopeKey( + "task:33179:owner:" + STALE_OWNER, TASK_OWNER); + assertEquals(TASK_OWNER, TaskFileJobService.ownerFromScopeKey(rebuilt), + "改挂之后按同一枚谓词读回来必须是目标实例,否则转交等于没转"); + } + + // ==================== handOffToOwner ==================== + + @Test + void test_改挂必须同时落_owner_列与_scope_key_且按旧_owner_做_cas() { + when(taskFileJobMapper.selectById(23815L)) + .thenReturn(job(23815L, 33179L, "RUNNING", STALE_OWNER)); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + + assertTrue(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER)); + + ArgumentCaptor> captor = + ArgumentCaptor.forClass(LambdaUpdateWrapper.class); + verify(taskFileJobMapper).update(isNull(), captor.capture()); + LambdaUpdateWrapper wrapper = captor.getValue(); + String setClause = wrapper.getSqlSet(); + assertTrue(setClause.contains("owner"), "必须改 owner 列: " + setClause); + assertTrue(setClause.contains("scope_key"), "必须同步改 scope_key,否则 lastIndexOf 读回的还是旧实例: " + setClause); + assertTrue(setClause.contains("error_message"), "必须清掉那条误导性的归属拒绝文案: " + setClause); + assertTrue(wrapper.getCustomSqlSegment().contains("status = "), "只能改挂自己手里的 RUNNING 行: " + wrapper.getCustomSqlSegment()); + assertTrue(wrapper.getCustomSqlSegment().contains("owner = "), "必须按观测到的旧 owner 回撞: " + wrapper.getCustomSqlSegment()); + assertTrue(wrapper.getParamNameValuePairs().containsValue("task:33179:owner:" + TASK_OWNER), + "scope_key 参数必须是换过尾段的值: " + wrapper.getParamNameValuePairs()); + } + + @Test + void test_已经挂在目标实例上时不得再发更新() { + // 空转的代价就是每轮刷 updated_at:两列重合后必须一口都不动 + when(taskFileJobMapper.selectById(23815L)) + .thenReturn(job(23815L, 33179L, "RUNNING", TASK_OWNER)); + + assertFalse(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER)); + verify(taskFileJobMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class)); + } + + @Test + void test_目标实例为空或行已消失时拒绝改挂() { + assertFalse(service.handOffToOwner(23815L, STALE_OWNER, null)); + assertFalse(service.handOffToOwner(23815L, STALE_OWNER, " ")); + assertFalse(service.handOffToOwner(null, STALE_OWNER, TASK_OWNER)); + verify(taskFileJobMapper, never()).selectById(any()); + + when(taskFileJobMapper.selectById(23815L)).thenReturn(null); + assertFalse(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER)); + } + + @Test + void test_cas_未命中时返回失败而不抛() { + // 期间任务又转交、或行被别的实例改挂:update 打空,由下一轮拒绝重新转交 + when(taskFileJobMapper.selectById(23815L)) + .thenReturn(job(23815L, 33179L, "RUNNING", STALE_OWNER)); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0); + + assertFalse(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER)); + } + + @Test + void test_旧_owner_为空时_cas_必须走_is_null_而不是空串等值() { + when(taskFileJobMapper.selectById(1L)).thenReturn(job(1L, 900L, "RUNNING", null)); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + + assertTrue(service.handOffToOwner(1L, null, TASK_OWNER)); + + ArgumentCaptor> captor = + ArgumentCaptor.forClass(LambdaUpdateWrapper.class); + verify(taskFileJobMapper).update(isNull(), captor.capture()); + assertTrue(captor.getValue().getCustomSqlSegment().contains("owner IS NULL"), + "存量行 owner 为 NULL,等值空串永远打不中: " + captor.getValue().getCustomSqlSegment()); + } + + // ==================== enqueueAssembleResult 的 PENDING 对账 ==================== + + @Test + void test_pending_作业重新入队时必须按入队_scope_key_对账归属() { + // Python 超时恢复每 2 分钟就会走到这一格一次。只补发事件不对账,等于每 2 分钟确认一遍死锁还在。 + TaskFileJobEntity pending = job(23815L, 33179L, "PENDING", STALE_OWNER); + when(taskFileJobMapper.selectOne(any())).thenReturn(pending); + when(taskFileJobMapper.selectById(23815L)) + .thenReturn(job(23815L, 33179L, "PENDING", TASK_OWNER)); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + + TaskFileJobEntity result = service.enqueueAssembleResult( + 33179L, "APPEARANCE_PATENT", 36102L, "task:33179:owner:" + TASK_OWNER); + + assertEquals(TASK_OWNER, result.getOwner(), "返回的必须是对账后的行"); + ArgumentCaptor> captor = + ArgumentCaptor.forClass(LambdaUpdateWrapper.class); + verify(taskFileJobMapper, times(1)).update(isNull(), captor.capture()); + assertTrue(captor.getValue().getSqlSet().contains("owner"), "对账必须写 owner 列"); + assertTrue(captor.getValue().getCustomSqlSegment().contains("status = "), + "只能对账仍在 PENDING 的行,别抢正在跑的那台: " + captor.getValue().getCustomSqlSegment()); + verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class)); + } + + @Test + void test_pending_作业归属已重合时不得发更新() { + TaskFileJobEntity pending = job(23815L, 33179L, "PENDING", TASK_OWNER); + when(taskFileJobMapper.selectOne(any())).thenReturn(pending); + + TaskFileJobEntity result = service.enqueueAssembleResult( + 33179L, "APPEARANCE_PATENT", 36102L, "task:33179:owner:" + TASK_OWNER); + + assertEquals(23815L, result.getId()); + verify(taskFileJobMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class)); + verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class)); + } + + @Test + void test_pending_作业入队_scope_key_不带_owner_段时不得凭空对账() { + TaskFileJobEntity pending = job(23815L, 33179L, "PENDING", STALE_OWNER); + when(taskFileJobMapper.selectOne(any())).thenReturn(pending); + + service.enqueueAssembleResult(33179L, "APPEARANCE_PATENT", 36102L, "task:33179"); + + verify(taskFileJobMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class)); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskOwnerReadSourceGuardrailTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskOwnerReadSourceGuardrailTest.java new file mode 100644 index 000000000..c0b7d9841 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskOwnerReadSourceGuardrailTest.java @@ -0,0 +1,88 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.testutil.SourceScan; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 「任务归属实例」校验源必须四台同源的绊线(2026-11-02)。 + * + *

起因:现网任务 33179/33149 的跨实例死锁。领取按 {@code biz_task_file_job.owner}、 + * 执行时校验按 {@code biz_file_task.owner_instance_id},两列一旦背离就是互斥死锁 + * (能领到这行的实例必被校验拒、校验放过的实例必领不到这行)。而四台的校验源当时是三样: + * 外观专利读列(正确)、相似ASIN 与发布只读 {@code resultJson} 快照、 + * 店铺采集回落 {@code requestJson}。快照是建单那一刻写死的、任务转交后不再同步, + * 于是 worker 侧按异常携带的 owner 转交作业时会转交给已经不管这行的实例——死锁只是换了个方向。 + * + *

判据形状:每台的归属读取方法都必须先读 {@code owner_instance_id} 列, + * 只在列缺失时才回落 JSON 快照。把任何一台改回「只读快照」,本类即红。 + * 之所以打在源码上而不是四份单测夹具:这条谓词的正确性在于「读哪个字段」, + * 用 mock 造四份任务实体各测一遍,改回读快照时那四份夹具照样全绿。 + */ +class TaskOwnerReadSourceGuardrailTest { + + /** 从方法签名起做括号配对,取出方法体(注释已由 withoutComments 剥掉)。 */ + private static String methodBody(String source, String signature) { + int start = source.indexOf(signature); + assertThat(start).as("必须存在方法 " + signature).isGreaterThan(-1); + int open = source.indexOf('{', start); + assertThat(open).as(signature + " 的方法体起点").isGreaterThan(-1); + int depth = 0; + for (int i = open; i < source.length(); i++) { + char c = source.charAt(i); + if (c == '{') { + depth++; + } else if (c == '}') { + depth--; + if (depth == 0) { + return source.substring(open + 1, i); + } + } + } + throw new AssertionError(signature + " 括号未闭合"); + } + + @ParameterizedTest(name = "{0} 的 {1} 必须先读 owner_instance_id 列") + @CsvSource({ + "AppearancePatentTaskOwnership.java, ownerFromTask(FileTaskEntity task)", + "SimilarAsinTaskOwnershipSupport.java, ownerFromTask(FileTaskEntity task)", + "PublishTaskService.java, ownerFromTask(FileTaskEntity task)", + "ShopDataCrawlTaskService.java, ownerInstanceIdOf(FileTaskEntity task)", + }) + void ownershipReadMustStartFromTheExplicitColumn(String fileName, String signature) { + String source = SourceScan.withoutComments(SourceScan.sourceText(fileName)); + String body = methodBody(source, signature); + + assertThat(body) + .as(fileName + " 的 " + signature + " 必须读显式列,而不是只读 JSON 快照") + .contains("getOwnerInstanceId()"); + int columnRead = body.indexOf("getOwnerInstanceId()"); + int jsonRead = body.indexOf("readTree("); + if (jsonRead >= 0) { + assertThat(columnRead) + .as(fileName + " 必须先读列再读 JSON 快照(列是转交后唯一被更新的来源)") + .isLessThan(jsonRead); + } + // jsonRead < 0 也合法:那一台已经把快照回落后移到别的方法里,列就是唯一来源。 + // 真正不允许的是反过来——只读快照,那正是相似ASIN 与发布当时的形状。 + } + + @Test + void 转交作业的目标实例必须来自异常携带的任务当前_owner() { + String source = SourceScan.withoutComments(SourceScan.sourceText("TaskResultFileJobWorker.java")); + String body = methodBody(source, "catch (TaskOwnerMismatchException ex)"); + + assertThat(body) + .as("只 deferRunning 就是 2026-11-02 那 6381 条空转拒绝的形状") + .contains("handOffToOwner"); + assertThat(body) + .as("改挂目标必须取任务当前 owner,取作业自己的旧 owner 等于原地不动") + .contains("ex.getOwnerInstanceId()"); + assertThat(body.indexOf("handOffToOwner")) + .as("必须先改挂再退回:deferRunning 把行翻成 PENDING,而改挂的 CAS 只认 RUNNING") + .isLessThan(body.indexOf("deferRunning")); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerHandOffTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerHandOffTest.java new file mode 100644 index 000000000..b2fe0be30 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerHandOffTest.java @@ -0,0 +1,132 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentResultFileJobHandler; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.LocalDateTime; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * worker 侧归属拒绝的处理顺序判据(2026-11-02 现网任务 33179/33149)。 + * + *

被 {@code TaskOwnerMismatchException} 拒掉时,必须先把作业改挂到任务当前 owner、 + * 再退回 PENDING。顺序反了就不生效:{@code handOffToOwner} 的 CAS 只认 RUNNING 行, + * 而 {@code deferRunning} 恰好把行翻成 PENDING——先 defer 的话改挂永远打不中, + * 于是回到「每 15 秒领一次、每次都被拒、永不终态」的死锁原状。 + * + *

改挂目标必须取异常携带的 {@code ownerInstanceId}(校验刚读到的任务当前归属), + * 不能取作业自己的旧 owner——那是原地不动,等于没修。 + */ +@ExtendWith(MockitoExtension.class) +class TaskResultFileJobWorkerOwnerHandOffTest { + + @Mock private TaskFileJobService taskFileJobService; + @Mock private TaskDistributedLockService taskDistributedLockService; + @Mock private InstanceMetadata instanceMetadata; + @Mock private AppearancePatentTaskService appearancePatentTaskService; + + private TaskResultFileJobWorker worker; + private TaskFileJobEntity job; + + @BeforeEach + void setUp() { + ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry( + List.of(new AppearancePatentResultFileJobHandler(appearancePatentTaskService))); + worker = new TaskResultFileJobWorker( + taskFileJobService, + taskDistributedLockService, + mock(FileResultMapper.class), + mock(TaskFileJobLocalDispatcher.class), + instanceMetadata, + registry); + job = new TaskFileJobEntity(); + job.setId(23815L); + job.setTaskId(33179L); + job.setResultId(36102L); + job.setModuleType("APPEARANCE_PATENT"); + job.setJobType("ASSEMBLE_RESULT"); + job.setStatus("RUNNING"); + job.setOwner("server-110"); + job.setScopeKey("task:33179:owner:server-110"); + job.setRetryCount(1); + job.setUpdatedAt(LocalDateTime.now()); + } + + private void stubRejectedByTaskOwner() throws Exception { + when(taskFileJobService.findById(23815L)).thenReturn(null); + when(taskDistributedLockService.acquire(eq("APPEARANCE_PATENT"), eq(33179L), anyLong())) + .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); + when(appearancePatentTaskService.processResultFileJob(job)).thenThrow( + new TaskOwnerMismatchException(33179L, "assemble result file", "server-121", "server-110")); + } + + private void processInternal() throws Exception { + Method method = TaskResultFileJobWorker.class + .getDeclaredMethod("processInternal", TaskFileJobEntity.class); + method.setAccessible(true); + try { + method.invoke(worker, job); + } catch (InvocationTargetException ex) { + throw (Exception) ex.getCause(); + } + } + + @Test + void test_归属被拒时必须先改挂再退回() throws Exception { + stubRejectedByTaskOwner(); + + assertDoesNotThrow(this::processInternal); + + InOrder order = inOrder(taskFileJobService); + order.verify(taskFileJobService).handOffToOwner(23815L, "server-110", "server-121"); + order.verify(taskFileJobService).deferRunning(eq(23815L), anyString()); + } + + @Test + void test_改挂目标必须取任务当前_owner_而不是作业旧_owner() throws Exception { + stubRejectedByTaskOwner(); + + processInternal(); + + ArgumentCaptor target = ArgumentCaptor.forClass(String.class); + verify(taskFileJobService).handOffToOwner(eq(23815L), anyString(), target.capture()); + assertEquals("server-121", target.getValue(), + "取成 server-110 就是原地转交,死锁只是换了个方向"); + } + + @Test + void test_归属被拒不得计入_retry_预算() throws Exception { + // 计入 retry_count 会让任务在归属机一切正常时被本机重试耗尽判死(原注释的语义,改动后仍须成立) + stubRejectedByTaskOwner(); + + processInternal(); + + verify(taskFileJobService, never()).markFailed(any(TaskFileJobEntity.class), anyString()); + verify(taskFileJobService, never()).requeue(anyLong(), anyString()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java index de6896230..b2d0fe9e6 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java @@ -39,7 +39,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; * 那条数是 {@code grep -rl 'sourceText('} 数出来的,而第 9 支是 {@code SourceScan.java:22} 里 * {@code public static String sourceText(String fileName)} 这行被调方自己的声明——它不是调用点。 * 所以真实形状是 调用点/调用方/引用方三类数各自成立(2026-10-28 v1.1 Phase 0 起调用方 16 支、 - * 裸名 22 个,增量逐笔登记在 {@code REGISTERED_BARE_NAMES} 的注释里)。下面第 3 格把这些数同时钉住, + * 裸名 23 个,增量逐笔登记在 {@code REGISTERED_BARE_NAMES} 的注释里)。下面第 3 格把这些数同时钉住, * 免得下一个人拿 {@code grep} 再数一遍又把声明算进去。

*/ class SourceScanTest { @@ -77,7 +77,12 @@ class SourceScanTest { "TaskCenterQueryService.java", "TaskHeartbeatService.java", "AppearancePatentDeliverySupport.java", - "RequestTraceFilter.java"); + "RequestTraceFilter.java", + // —— 2026-11-02 跨实例归属死锁根治追加(1 名)—— + // TaskOwnerReadSourceGuardrailTest 要钉「被拒时必须先改挂再退回」这一枚顺序, + // 顺序错在 deferRunning 把行翻成 PENDING 之后 handOffToOwner 的 CAS 就打不中, + // 用 mock 造不出这个先后(它要跑的正是 catch 块里两行的次序),只能打在源码连线上。 + "TaskResultFileJobWorker.java"); @TempDir Path tmp; @@ -93,7 +98,7 @@ class SourceScanTest { } @Test - void census_唯一裸名恰为22且与已登记集合相等() { + void census_唯一裸名恰为23且与已登记集合相等() { Set found = SourceScan.censusSourceTextCallSites().stream() .map(SourceScan.CallSite::bareName) .collect(Collectors.toSet()); @@ -101,11 +106,11 @@ class SourceScanTest { assertThat(found) .as("唯一裸名集合必须与类头台账逐字相等——多一个说明有人新引了单文件口径,少一个说明尺子漏扫") .isEqualTo(REGISTERED_BARE_NAMES) - .hasSize(22); + .hasSize(23); } @Test - void census_调用方文件恰为16而声明件在外() { + void census_调用方文件恰为17而声明件在外() { List sites = SourceScan.censusSourceTextCallSites(); Set callerFiles = sites.stream() .map(SourceScan.CallSite::relativeFile) @@ -114,8 +119,9 @@ class SourceScanTest { String declarationFile = "src/test/java/com/nanri/aiimage/testutil/SourceScan.java"; assertThat(callerFiles) - .as("真实字面量调用点落在 16 支调用方文件里(8 支既有 + 2026-10-28 v1.1 Phase 0 预置契约 8 支)") - .hasSize(16); + .as("真实字面量调用点落在 17 支调用方文件里(8 支既有 + 2026-10-28 v1.1 Phase 0 预置契约 8 支" + + " + 2026-11-02 归属死锁绊线 1 支)") + .hasSize(17); assertThat(callerFiles) .as("plan 原文记的「9 支」= grep -rl 'sourceText(' 的条数,第 9 支是被调方自己的声明行" + "(SourceScan.java:22 的 `public static String sourceText(String fileName)`),"