修复双实例 owner-scoped 结果文件 job 被非归属实例误杀

线上现象：货源查询任务 26402/26400（归属 server-121）在归属机正常推进，却被 server-110 判为「结果生成失败」，错误为「该任务已绑定到另一台服务实例处理」。两个缺陷叠加： 1. 两台实例共用一个 RocketMQ consumer group，dispatch 消息被轮询投递； worker 对"PENDING + owner 不匹配"的 job 选择本地抢跑，模块层 ensureTaskOwnedByCurrentInstance 直接抛 TaskOwnerMismatchException，异常落进通用 catch 被当成生成失败，5 次抢跑即耗尽 retryCount 并终态失败。 2. runPendingJobs 双重 claim：claimRunnableJobsForOwner 已把 job 翻成 RUNNING，随后又走 claimRunning（条件 PENDING/FAILED）必然失败 → job 被静默丢弃，归属机的定时兜底从未真正生效。修复： - worker：非归属实例一律跳过 owner-scoped job；抽出 processClaimed 供已持有 claim 的调用方使用；processInternal 单独捕获 TaskOwnerMismatchException，只 deferRunning 退回 PENDING，不计入重试。 - TaskFileJobLocalDispatcher 新增 dispatchClaimed。 - dispatch 事件带上 owner，coordinator 按归属路由：本机 owner 直接本地派发不发 MQ，他机 owner 不发不跑交由归属机轮询接手，owner 为空维持原有 MQ 行为。 - 新增 owner 路由回归测试两组，回滚配合"本地抢跑"的旧断言。同时提交此前工作区内已随 JAR 上线的 backend-java 改动：SimilarAsin 解析载荷 groups 内嵌 items（Python 旧链路兼容）与 chunk 类型不匹配跳过、 PermissionMenuSchemaInitializer 与 V103 两级菜单分组迁移、相关测试。
2026-09-01 21:07:25 +08:00
parent 55517f996b
commit 5f4fcad2ef
16 changed files with 679 additions and 44 deletions
@@ -0,0 +1,85 @@
+package com.nanri.aiimage.modules.task.service;
+
+import com.nanri.aiimage.config.InstanceMetadata;
+import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
+import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.task.TaskExecutor;
+
+import java.time.LocalDateTime;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * owner-scoped job 的派发路由（2026-09-01 事故）：两台实例共用一个 RocketMQ consumer group，
+ * 消息被轮询投递到非归属实例后只能丢弃。归属实例改为直接本地派发，非归属实例不再发消息。
+ */
+class TaskFileJobDispatchCoordinatorOwnerRoutingTest {
+
+    private final TaskFileJobPublisher publisher = mock(TaskFileJobPublisher.class);
+    private final TaskFileJobLocalDispatcher localDispatcher = mock(TaskFileJobLocalDispatcher.class);
+    private final InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
+    private final TaskExecutor inlineExecutor = Runnable::run;
+
+    private TaskFileJobDispatchCoordinator buildCoordinator() {
+        return new TaskFileJobDispatchCoordinator(publisher, localDispatcher, instanceMetadata, inlineExecutor);
+    }
+
+    private static TaskFileJobDispatchEvent event(String owner) {
+        return new TaskFileJobDispatchEvent(18524L, 26402L, "SIMILAR_ASIN", 29405L,
+                owner == null ? "task:26402" : "task:26402:owner:" + owner,
+                TaskFileJobService.JOB_TYPE_ASSEMBLE_RESULT, owner, LocalDateTime.now());
+    }
+
+    @Test
+    void ownerJobDispatchedLocallyWithoutMq() {
+        when(instanceMetadata.getInstanceId()).thenReturn("server-121");
+        when(localDispatcher.dispatch(18524L, 26402L, "SIMILAR_ASIN", true)).thenReturn(true);
+
+        buildCoordinator().onTaskFileJobDispatch(event("server-121"));
+
+        verify(localDispatcher).dispatch(18524L, 26402L, "SIMILAR_ASIN", true);
+        verifyNoInteractions(publisher);
+    }
+
+    @Test
+    void ownerJobFallsBackToMqWhenLocalDispatchDeclined() {
+        when(instanceMetadata.getInstanceId()).thenReturn("server-121");
+        when(localDispatcher.dispatch(18524L, 26402L, "SIMILAR_ASIN", true)).thenReturn(false, true);
+        when(publisher.publish(any(TaskFileJobMessage.class))).thenReturn(false);
+
+        buildCoordinator().onTaskFileJobDispatch(event("server-121"));
+
+        verify(publisher).publish(any(TaskFileJobMessage.class));
+    }
+
+    @Test
+    void otherInstanceJobIsNeitherPublishedNorRun() {
+        when(instanceMetadata.getInstanceId()).thenReturn("server-110");
+
+        buildCoordinator().onTaskFileJobDispatch(event("server-121"));
+
+        verifyNoInteractions(publisher);
+        verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), any(), anyBoolean());
+        verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), any());
+    }
+
+    @Test
+    void jobWithoutOwnerStillGoesThroughMq() {
+        when(instanceMetadata.getInstanceId()).thenReturn("server-110");
+        when(publisher.publish(any(TaskFileJobMessage.class))).thenReturn(true);
+
+        buildCoordinator().onTaskFileJobDispatch(event(null));
+
+        verify(publisher).publish(any(TaskFileJobMessage.class));
+        verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), eq("SIMILAR_ASIN"), anyBoolean());
+    }
+}
@@ -0,0 +1,163 @@
+package com.nanri.aiimage.modules.task.service;
+
+import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
+import com.nanri.aiimage.config.InstanceMetadata;
+import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
+import com.nanri.aiimage.modules.brand.service.BrandTaskService;
+import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
+import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
+import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
+import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
+import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
+import com.nanri.aiimage.modules.publish.service.PublishTaskService;
+import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
+import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
+import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
+import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
+import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
+import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
+import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Field;
+import java.time.LocalDateTime;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * 双实例归属路由回归测试（2026-09-01 货源查询任务被非归属实例误杀）：
+ * 1) 定时 worker 已 claim 的 job 不能再走 claimRunning，否则条件 UPDATE 失败被静默丢弃；
+ * 2) 处理中撞上归属校验时只退回 PENDING，不得计入 retryCount（markFailed）。
+ */
+class TaskResultFileJobWorkerOwnerRoutingTest {
+
+    private final ShopMatchTaskService shopMatch = mock(ShopMatchTaskService.class);
+    private final PriceTrackTaskService priceTrack = mock(PriceTrackTaskService.class);
+    private final ProductRiskTaskService productRisk = mock(ProductRiskTaskService.class);
+    private final PublishTaskService publish = mock(PublishTaskService.class);
+    private final QueryAsinTaskService queryAsin = mock(QueryAsinTaskService.class);
+    private final ShopDataCrawlTaskService shopDataCrawl = mock(ShopDataCrawlTaskService.class);
+    private final WithdrawTaskService withdraw = mock(WithdrawTaskService.class);
+    private final PatrolDeleteTaskService patrolDelete = mock(PatrolDeleteTaskService.class);
+    private final AppearancePatentTaskService appearance = mock(AppearancePatentTaskService.class);
+    private final SimilarAsinTaskService similar = mock(SimilarAsinTaskService.class);
+    private final DeleteBrandRunService deleteBrand = mock(DeleteBrandRunService.class);
+    private final BrandTaskService brand = mock(BrandTaskService.class);
+    private final CollectDataService collectData = mock(CollectDataService.class);
+    private final TaskResultPayloadService payload = mock(TaskResultPayloadService.class);
+    private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
+    private final TaskDistributedLockService taskDistributedLockService = mock(TaskDistributedLockService.class);
+    private final TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
+    private final TaskFileJobLocalDispatcher localDispatcher = mock(TaskFileJobLocalDispatcher.class);
+    private final InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
+
+    private TaskResultFileJobWorker buildWorker() throws Exception {
+        List<ResultFileJobHandler> handlers = List.of(
+                new ShopMatchResultFileJobHandler(shopMatch, payload),
+                new PriceTrackResultFileJobHandler(priceTrack, payload),
+                new ProductRiskResultFileJobHandler(productRisk, payload),
+                new PublishResultFileJobHandler(publish),
+                new QueryAsinResultFileJobHandler(queryAsin, payload),
+                new ShopDataCrawlResultFileJobHandler(shopDataCrawl, payload),
+                new WithdrawResultFileJobHandler(withdraw, payload),
+                new PatrolDeleteResultFileJobHandler(patrolDelete, payload),
+                new AppearancePatentResultFileJobHandler(appearance),
+                new SimilarAsinResultFileJobHandler(similar),
+                new DeleteBrandResultFileJobHandler(deleteBrand),
+                new BrandResultFileJobHandler(brand, payload),
+                new CollectDataResultFileJobHandler(collectData));
+        TaskResultFileJobWorker worker = new TaskResultFileJobWorker(
+                taskFileJobService,
+                taskDistributedLockService,
+                mock(FileResultMapper.class),
+                localDispatcher,
+                instanceMetadata,
+                withdraw, brand,
+                new ResultFileJobHandlerRegistry(handlers));
+        set(worker, "taskQueueExecutor", (org.springframework.core.task.TaskExecutor) Runnable::run);
+        set(worker, "localWorkerEnabled", true);
+        set(worker, "batchSize", 20);
+        return worker;
+    }
+
+    private static void set(TaskResultFileJobWorker worker, String fieldName, Object value) throws Exception {
+        Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
+        field.setAccessible(true);
+        field.set(worker, value);
+    }
+
+    private static TaskFileJobEntity claimedJob(String moduleType, long jobId, long taskId, String scopeKey) {
+        TaskFileJobEntity entity = new TaskFileJobEntity();
+        entity.setId(jobId);
+        entity.setTaskId(taskId);
+        entity.setResultId(taskId + 1000L);
+        entity.setModuleType(moduleType);
+        entity.setScopeKey(scopeKey);
+        entity.setStatus("RUNNING");
+        entity.setUpdatedAt(LocalDateTime.now());
+        return entity;
+    }
+
+    @Test
+    void scheduledWorkerProcessesAlreadyClaimedJobWithoutReclaiming() throws Exception {
+        TaskResultFileJobWorker worker = buildWorker();
+        when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
+        TaskFileJobEntity claim = claimedJob(PublishTaskService.MODULE_TYPE, 18524L, 26402L, "task:26402:owner:instance-a");
+        when(taskFileJobService.claimRunnableJobsForOwner(20, "instance-a")).thenReturn(List.of(claim));
+        when(taskFileJobService.activateRunningClaim(claim)).thenReturn(true);
+        when(localDispatcher.dispatchClaimed(claim.getId(), claim.getTaskId(), claim.getModuleType())).thenReturn(false);
+        when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, claim.getTaskId(),
+                TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
+
+        worker.runPendingJobs();
+
+        verify(taskFileJobService, never()).claimRunning(anyLong());
+        verify(publish).processResultFileJob(claim);
+        verify(taskFileJobService).markSuccess(eq(claim), any());
+    }
+
+    @Test
+    void scheduledWorkerPrefersClaimedDispatch() throws Exception {
+        TaskResultFileJobWorker worker = buildWorker();
+        when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
+        TaskFileJobEntity claim = claimedJob(PublishTaskService.MODULE_TYPE, 18525L, 26403L, "task:26403:owner:instance-a");
+        when(taskFileJobService.claimRunnableJobsForOwner(20, "instance-a")).thenReturn(List.of(claim));
+        when(localDispatcher.dispatchClaimed(claim.getId(), claim.getTaskId(), claim.getModuleType())).thenReturn(true);
+
+        worker.runPendingJobs();
+
+        verify(localDispatcher).dispatchClaimed(claim.getId(), claim.getTaskId(), claim.getModuleType());
+        verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), any());
+        verify(taskFileJobService, never()).claimRunning(anyLong());
+        verify(publish, never()).processResultFileJob(any());
+    }
+
+    @Test
+    void ownerMismatchDuringProcessingDefersInsteadOfFailing() throws Exception {
+        TaskResultFileJobWorker worker = buildWorker();
+        when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
+        // scopeKey 归属本机（通过 process 的前置校验），但模块层读到的 task owner 已经变成他机
+        TaskFileJobEntity job = claimedJob(PublishTaskService.MODULE_TYPE, 18526L, 26404L, "task:26404:owner:instance-a");
+        when(taskFileJobService.claimRunning(job.getId())).thenReturn(job);
+        when(taskFileJobService.activateRunningClaim(job)).thenReturn(true);
+        when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, job.getTaskId(),
+                TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
+        doThrow(new TaskOwnerMismatchException(job.getTaskId(), "assemble result file", "instance-b", "instance-a"))
+                .when(publish).processResultFileJob(job);
+
+        worker.process(job);
+
+        verify(taskFileJobService).deferRunning(eq(job.getId()), any());
+        verify(taskFileJobService, never()).markFailed(any(), any());
+        verify(taskFileJobService, never()).markFailedPermanent(any(), any());
+        verify(taskFileJobService, never()).isRetryExhausted(anyLong());
+    }
+}
@@ -128,6 +128,20 @@ class TaskResultFileJobWorkerOwnerScopedTest {
        TaskResultFileJobWorker worker = buildWorker();
        when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
        TaskFileJobEntity job = job("PUBLISH", 2L, 12L, "task:12:owner:instance-b");
+        job.setStatus("RUNNING");
+        worker.process(job);
+        verifyNoInteractions(taskFileJobService, taskDistributedLockService, publish);
+    }
+
+    @Test
+    void ownerScopedOtherOwnerPendingSkip() {
+        // PENDING + owner 不匹配也必须跳过：模块层 ensureTaskOwnedByCurrentInstance 会抛
+        // TaskOwnerMismatchException，本机抢跑只会耗尽 job 重试次数并误杀归属机上的任务。
+        // job 由归属实例的 runPendingJobs（每 15 秒扫自己 owner 的 PENDING/FAILED）接手。
+        TaskResultFileJobWorker worker = buildWorker();
+        when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
+        TaskFileJobEntity job = job("PUBLISH", 2L, 12L, "task:12:owner:instance-b");
+        job.setStatus("PENDING");
        worker.process(job);
        verifyNoInteractions(taskFileJobService, taskDistributedLockService, publish);
    }