task-72: 文件作业数据库原子 claim，避免重复派发同一 job

2026-08-30 20:01:18 +08:00
parent c07e26a400
commit baa43d80ba
3 changed files with 282 additions and 2 deletions
@@ -86,6 +86,72 @@ public class TaskFileJobService {
                .last("limit " + Math.max(1, Math.min(limit, 100))));
    }

+    /**
+     * 原子 claim：先查候选（PENDING/FAILED 且未达重试上限），再逐个以条件
+     * UPDATE（状态+retryCount 匹配 → RUNNING）翻转；只有翻转成功的行才返回。
+     * 多个 worker/实例并发取数时，同一 job 只会被一个消费者 claim 到，
+     * 从源头避免重复派发同一 job。
+     */
+    public List<TaskFileJobEntity> claimRunnableJobs(int limit) {
+        List<TaskFileJobEntity> candidates = listRunnableJobs(limit);
+        return claimCandidates(candidates);
+    }
+
+    /** 带 owner 过滤的原子 claim：owner 任务优先，其余补足；同 {@link #claimRunnableJobs} 的原子性。 */
+    public List<TaskFileJobEntity> claimRunnableJobsForOwner(int limit, String owner) {
+        String normalizedOwner = owner == null ? "" : owner.trim();
+        if (normalizedOwner.isBlank()) {
+            return claimRunnableJobs(limit);
+        }
+        String ownerMarker = ":owner:" + normalizedOwner;
+        int safeLimit = Math.max(1, Math.min(limit, 100));
+
+        List<TaskFileJobEntity> ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
+                .in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
+                .lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
+                .in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
+                .like(TaskFileJobEntity::getScopeKey, ownerMarker)
+                .orderByAsc(TaskFileJobEntity::getUpdatedAt)
+                .last("limit " + safeLimit));
+
+        List<TaskFileJobEntity> candidates = new ArrayList<>(ownerJobs);
+        if (ownerJobs.size() < safeLimit) {
+            LambdaQueryWrapper<TaskFileJobEntity> genericWrapper = new LambdaQueryWrapper<TaskFileJobEntity>()
+                    .in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
+                    .lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
+                    .and(wrapper -> wrapper
+                            .notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
+                            .or()
+                            .isNull(TaskFileJobEntity::getScopeKey)
+                            .or()
+                            .notLike(TaskFileJobEntity::getScopeKey, ":owner:"))
+                    .orderByAsc(TaskFileJobEntity::getUpdatedAt)
+                    .last("limit " + (safeLimit - ownerJobs.size()));
+            candidates.addAll(taskFileJobMapper.selectList(genericWrapper));
+        }
+        return claimCandidates(candidates);
+    }
+
+    private List<TaskFileJobEntity> claimCandidates(List<TaskFileJobEntity> candidates) {
+        if (candidates == null || candidates.isEmpty()) {
+            return List.of();
+        }
+        List<TaskFileJobEntity> claimed = new ArrayList<>();
+        for (TaskFileJobEntity candidate : candidates) {
+            if (candidate == null || candidate.getId() == null) {
+                continue;
+            }
+            if (!markRunning(candidate.getId())) {
+                continue;
+            }
+            TaskFileJobEntity claim = taskFileJobMapper.selectById(candidate.getId());
+            if (claim != null && "RUNNING".equals(claim.getStatus())) {
+                claimed.add(claim);
+            }
+        }
+        return claimed;
+    }
+
    public List<TaskFileJobEntity> listRunnableJobsForOwner(int limit, String owner) {
        String normalizedOwner = owner == null ? "" : owner.trim();
        if (normalizedOwner.isBlank()) {
@@ -85,11 +85,13 @@ public class TaskResultFileJobWorker {
        if (!localWorkerEnabled) {
            return;
        }
-        List<TaskFileJobEntity> jobs = taskFileJobService.listRunnableJobsForOwner(batchSize, currentInstanceId());
+        // 原子 claim：每个候选以条件 UPDATE 翻转为 RUNNING，只处理 claim 成功的行，
+        // 并发 worker/实例不会重复派发同一 job。
+        List<TaskFileJobEntity> jobs = taskFileJobService.claimRunnableJobsForOwner(batchSize, currentInstanceId());
        if (jobs == null || jobs.isEmpty()) {
            return;
        }
-        log.info("[task-file-job] scheduled worker picked jobs count={}", jobs.size());
+        log.info("[task-file-job] scheduled worker claimed jobs count={}", jobs.size());
        for (TaskFileJobEntity job : jobs) {
            boolean dispatched = taskFileJobLocalDispatcher.dispatch(
                    job.getId(),