修复双实例 owner-scoped 结果文件 job 被非归属实例误杀
线上现象:货源查询任务 26402/26400(归属 server-121)在归属机正常推进, 却被 server-110 判为「结果生成失败」,错误为「该任务已绑定到另一台服务实例处理」。 两个缺陷叠加: 1. 两台实例共用一个 RocketMQ consumer group,dispatch 消息被轮询投递; worker 对"PENDING + owner 不匹配"的 job 选择本地抢跑,模块层 ensureTaskOwnedByCurrentInstance 直接抛 TaskOwnerMismatchException, 异常落进通用 catch 被当成生成失败,5 次抢跑即耗尽 retryCount 并终态失败。 2. runPendingJobs 双重 claim:claimRunnableJobsForOwner 已把 job 翻成 RUNNING, 随后又走 claimRunning(条件 PENDING/FAILED)必然失败 → job 被静默丢弃, 归属机的定时兜底从未真正生效。 修复: - worker:非归属实例一律跳过 owner-scoped job;抽出 processClaimed 供已持有 claim 的调用方使用;processInternal 单独捕获 TaskOwnerMismatchException, 只 deferRunning 退回 PENDING,不计入重试。 - TaskFileJobLocalDispatcher 新增 dispatchClaimed。 - dispatch 事件带上 owner,coordinator 按归属路由:本机 owner 直接本地派发不发 MQ, 他机 owner 不发不跑交由归属机轮询接手,owner 为空维持原有 MQ 行为。 - 新增 owner 路由回归测试两组,回滚配合"本地抢跑"的旧断言。 同时提交此前工作区内已随 JAR 上线的 backend-java 改动:SimilarAsin 解析载荷 groups 内嵌 items(Python 旧链路兼容)与 chunk 类型不匹配跳过、 PermissionMenuSchemaInitializer 与 V103 两级菜单分组迁移、相关测试。
This commit is contained in:
+1
@@ -9,6 +9,7 @@ public record TaskFileJobDispatchEvent(
|
||||
Long resultId,
|
||||
String scopeKey,
|
||||
String jobType,
|
||||
String owner,
|
||||
LocalDateTime createdAt
|
||||
) {
|
||||
}
|
||||
|
||||
+29
@@ -1,5 +1,6 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -17,6 +18,7 @@ public class TaskFileJobDispatchCoordinator {
|
||||
|
||||
private final TaskFileJobPublisher taskFileJobPublisher;
|
||||
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
@Qualifier("taskFileJobDispatchExecutor")
|
||||
private final TaskExecutor taskFileJobDispatchExecutor;
|
||||
|
||||
@@ -43,6 +45,28 @@ public class TaskFileJobDispatchCoordinator {
|
||||
}
|
||||
|
||||
private void dispatchAfterCommit(TaskFileJobDispatchEvent event, TaskFileJobMessage message) {
|
||||
// owner 非空 ⇔ owner-scoped job(只有 SIMILAR_ASIN/APPEARANCE_PATENT/PUBLISH/
|
||||
// SHOP_DATA_CRAWL 会写 "task:{id}:owner:{instance}" 形式的 scopeKey)。
|
||||
// 这类 job 必须在归属实例上执行,走共享 consumer group 的 MQ 会被轮询投递到
|
||||
// 另一台,那边只能跳过,白白多绕一圈。
|
||||
String owner = event.owner() == null ? "" : event.owner().trim();
|
||||
if (!owner.isBlank()) {
|
||||
if (owner.equals(currentInstanceId())) {
|
||||
boolean dispatched = taskFileJobLocalDispatcher.dispatch(
|
||||
event.jobId(), event.taskId(), event.moduleType(), true);
|
||||
if (dispatched) {
|
||||
return;
|
||||
}
|
||||
log.warn("[task-file-job] owner-scoped local dispatch declined, fallback to mq jobId={} taskId={} moduleType={} owner={}",
|
||||
event.jobId(), event.taskId(), event.moduleType(), owner);
|
||||
} else {
|
||||
// 归属实例每 15 秒扫一次自己 owner 的 PENDING/FAILED job,这里不发 MQ、
|
||||
// 不本地跑,避免消息落到非归属实例后被丢弃。
|
||||
log.info("[task-file-job] skip dispatch because job belongs to another instance jobId={} taskId={} moduleType={} owner={} current={}",
|
||||
event.jobId(), event.taskId(), event.moduleType(), owner, currentInstanceId());
|
||||
return;
|
||||
}
|
||||
}
|
||||
boolean published = taskFileJobPublisher.publish(message);
|
||||
if (published) {
|
||||
return;
|
||||
@@ -56,4 +80,9 @@ public class TaskFileJobDispatchCoordinator {
|
||||
event.jobId(), event.taskId(), event.moduleType());
|
||||
}
|
||||
}
|
||||
|
||||
private String currentInstanceId() {
|
||||
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
|
||||
return instanceId == null || instanceId.isBlank() ? "unknown-instance" : instanceId;
|
||||
}
|
||||
}
|
||||
|
||||
+20
-4
@@ -42,7 +42,19 @@ public class TaskFileJobLocalDispatcher {
|
||||
return dispatch(jobId, taskId, moduleType, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用方已把 job claim 成 RUNNING 时使用:复用同样的背压/去重/异步执行,
|
||||
* 但落到 worker.processClaimed,避免二次 claimRunning 失败导致 job 被静默丢弃。
|
||||
*/
|
||||
public boolean dispatchClaimed(Long jobId, Long taskId, String moduleType) {
|
||||
return submit(jobId, taskId, moduleType, false, true);
|
||||
}
|
||||
|
||||
public boolean dispatch(Long jobId, Long taskId, String moduleType, boolean force) {
|
||||
return submit(jobId, taskId, moduleType, force, false);
|
||||
}
|
||||
|
||||
private boolean submit(Long jobId, Long taskId, String moduleType, boolean force, boolean alreadyClaimed) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
}
|
||||
@@ -63,7 +75,7 @@ public class TaskFileJobLocalDispatcher {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
taskFileJobDispatchExecutor.execute(() -> processLocally(jobId, taskId, moduleType));
|
||||
taskFileJobDispatchExecutor.execute(() -> processLocally(jobId, taskId, moduleType, alreadyClaimed));
|
||||
return true;
|
||||
} catch (RuntimeException ex) {
|
||||
inflightJobIds.remove(jobId);
|
||||
@@ -71,7 +83,7 @@ public class TaskFileJobLocalDispatcher {
|
||||
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||
if (force) {
|
||||
try {
|
||||
processLocally(jobId, taskId, moduleType);
|
||||
processLocally(jobId, taskId, moduleType, alreadyClaimed);
|
||||
return true;
|
||||
} finally {
|
||||
inflightJobIds.remove(jobId);
|
||||
@@ -81,7 +93,7 @@ public class TaskFileJobLocalDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
private void processLocally(Long jobId, Long taskId, String moduleType) {
|
||||
private void processLocally(Long jobId, Long taskId, String moduleType, boolean alreadyClaimed) {
|
||||
try {
|
||||
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
|
||||
if (job == null) {
|
||||
@@ -95,7 +107,11 @@ public class TaskFileJobLocalDispatcher {
|
||||
jobId, taskId, moduleType);
|
||||
return;
|
||||
}
|
||||
taskResultFileJobWorker.process(job);
|
||||
if (alreadyClaimed) {
|
||||
taskResultFileJobWorker.processClaimed(job);
|
||||
} else {
|
||||
taskResultFileJobWorker.process(job);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
|
||||
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||
|
||||
+34
-1
@@ -57,7 +57,14 @@ public class TaskFileJobService {
|
||||
if (existing == null) {
|
||||
return null;
|
||||
}
|
||||
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus()) || "PENDING".equals(existing.getStatus())) {
|
||||
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
|
||||
return existing;
|
||||
}
|
||||
if ("PENDING".equals(existing.getStatus())) {
|
||||
// PENDING 僵尸兜底:job 入队时若 MQ 发送失败且本地兜底未生效(或消息被
|
||||
// 非 owner 实例静默吞掉),job 会永远停留在 PENDING、无人处理。
|
||||
// 补发一次 dispatch 事件,让调度链重试(幂等:consumer 侧 claim 有状态翻转保护)。
|
||||
publishDispatchEvent(existing);
|
||||
return existing;
|
||||
}
|
||||
// 终态失败:retryCount 已达上限的 FAILED job 不再重置为 PENDING,
|
||||
@@ -237,6 +244,13 @@ public class TaskFileJobService {
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
|
||||
.or()
|
||||
.isNull(TaskFileJobEntity::getUpdatedAt)))
|
||||
.or(stuckPending -> stuckPending
|
||||
.eq(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.ne(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.and(age -> age
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
|
||||
.or()
|
||||
.isNull(TaskFileJobEntity::getUpdatedAt)))
|
||||
.or(exhausted -> exhausted
|
||||
.eq(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
@@ -251,6 +265,24 @@ public class TaskFileJobService {
|
||||
exhaustedJobs.add(job);
|
||||
continue;
|
||||
}
|
||||
if ("PENDING".equals(job.getStatus())) {
|
||||
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),
|
||||
// 并累加 retryCount 防无限重发;注意不翻转状态,
|
||||
// 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1);
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
@@ -642,6 +674,7 @@ public class TaskFileJobService {
|
||||
entity.getResultId(),
|
||||
entity.getScopeKey(),
|
||||
entity.getJobType(),
|
||||
entity.getOwner(),
|
||||
LocalDateTime.now()
|
||||
));
|
||||
}
|
||||
|
||||
+37
-6
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
|
||||
@@ -76,13 +77,16 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
log.info("[task-file-job] scheduled worker claimed jobs count={}", jobs.size());
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
boolean dispatched = taskFileJobLocalDispatcher.dispatch(
|
||||
// claimRunnableJobsForOwner 已把 job 原子翻转成 RUNNING,这里必须走"已 claim"
|
||||
// 通道:若再走 process() 的 claimRunning,条件 UPDATE 会因状态已是 RUNNING 而
|
||||
// 失败,job 被静默丢弃并挂到 stuck 扫描才回收,定时兜底等于失效。
|
||||
boolean dispatched = taskFileJobLocalDispatcher.dispatchClaimed(
|
||||
job.getId(),
|
||||
job.getTaskId(),
|
||||
job.getModuleType()
|
||||
);
|
||||
if (!dispatched) {
|
||||
process(job);
|
||||
processClaimed(job);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -104,18 +108,39 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
|
||||
public void process(TaskFileJobEntity job) {
|
||||
if (isOwnerScopedJob(job) && !isOwnedByCurrentInstance(job)) {
|
||||
log.debug("[task-file-job] skip owner-scoped job because owner is another instance jobId={} taskId={} moduleType={} owner={} current={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
}
|
||||
if (job == null || job.getId() == null) {
|
||||
if (isOwnerScopedJob(job) && !isOwnedByCurrentInstance(job)) {
|
||||
// 非归属实例一律不碰 owner-scoped job:模块层 ensureTaskOwnedByCurrentInstance
|
||||
// 会直接抛 TaskOwnerMismatchException,抢跑只会白白消耗 job 的重试次数,
|
||||
// 重试耗尽后把归属机上跑得好好的任务判成失败。
|
||||
// job 不会因此卡死:归属实例的 runPendingJobs 每 15 秒扫一次自己 owner 的
|
||||
// PENDING/FAILED job,enqueueAssembleResult / stuck 扫描也会补发 dispatch。
|
||||
log.info("[task-file-job] skip owner-scoped job because owner is another instance jobId={} taskId={} moduleType={} status={} owner={} current={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getStatus(),
|
||||
ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
|
||||
return;
|
||||
}
|
||||
TaskFileJobEntity claim = taskFileJobService.claimRunning(job.getId());
|
||||
if (claim == null) {
|
||||
return;
|
||||
}
|
||||
offloadOrRun(job, claim);
|
||||
}
|
||||
|
||||
/**
|
||||
* 供"调用方已持有 RUNNING claim"的路径使用(定时 worker 的 claimRunnableJobsForOwner):
|
||||
* 跳过 claimRunning,直接按 handler 能力异步/内联执行。
|
||||
*/
|
||||
public void processClaimed(TaskFileJobEntity claim) {
|
||||
if (claim == null || claim.getId() == null) {
|
||||
return;
|
||||
}
|
||||
offloadOrRun(claim, claim);
|
||||
}
|
||||
|
||||
private void offloadOrRun(TaskFileJobEntity job, TaskFileJobEntity claim) {
|
||||
ResultFileJobHandler handler = handlerRegistry.asMap().get(job.getModuleType());
|
||||
if (handler != null && handler.supportsAsyncOffload()) {
|
||||
try {
|
||||
@@ -206,6 +231,12 @@ public class TaskResultFileJobWorker {
|
||||
if (finalizeWithdraw) {
|
||||
withdrawTaskService.tryFinalizeTask(job.getTaskId(), false);
|
||||
}
|
||||
} catch (TaskOwnerMismatchException ex) {
|
||||
// 归属校验失败不是"生成失败":不能计入 retryCount,否则任务会在归属机
|
||||
// 一切正常的情况下被本机重试耗尽判死。退回 PENDING 交还归属实例。
|
||||
log.warn("[task-file-job] process rejected because task belongs to another instance jobId={} taskId={} moduleType={} owner={} current={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId());
|
||||
taskFileJobService.deferRunning(job.getId(), ex.getMessage());
|
||||
} catch (Exception ex) {
|
||||
TaskFileJobEntity latest = taskFileJobService.findById(job.getId());
|
||||
if (latest != null && "SUCCESS".equals(latest.getStatus())) {
|
||||
|
||||
Reference in New Issue
Block a user