fix(调度): 双节点下 collect-data/publish/image-video 定时调度加 Redis 分布式锁互斥——双活原来各自扫描执行同一批任务,重复派发/重复终态判定

This commit is contained in:
2026-09-11 16:09:46 +08:00
parent b9ecddce70
commit c2fcf4fb1c
3 changed files with 155 additions and 104 deletions
@@ -50,6 +50,7 @@ import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.scheduling.annotation.Scheduled;
@@ -108,6 +109,7 @@ public class PublishTaskService {
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
private final DistributedJobLockService distributedJobLockService;
private final InstanceMetadata instanceMetadata;
private final TaskProgressLightAssembler taskProgressLightAssembler;
@@ -420,32 +422,42 @@ public class PublishTaskService {
@Scheduled(fixedDelayString = "${aiimage.publish.stale-scan-delay-ms:60000}")
public void failStaleTasks() {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, staleTimeoutMinutes));
List<FileTaskEntity> staleTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.and(owner -> owner
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId()))
.lt(FileTaskEntity::getUpdatedAt, threshold)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 100"));
for (FileTaskEntity candidate : staleTasks) {
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
continue;
}
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
if (lock == null) {
// 双节点互斥:避免 ownerInstanceId 为空的 stale 行被两节点同时扫出双写错误信息;
// Redis 不可用时 fail-open,兜底靠下方 per-task 锁
DistributedJobLockService.LockHandle jobLock =
distributedJobLockService.tryLock("publish:stale-check", java.time.Duration.ofMinutes(2));
if (jobLock == null) {
log.info("[publish] stale-check skipped, another instance holds the distributed lock");
return;
}
try (jobLock) {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, staleTimeoutMinutes));
List<FileTaskEntity> staleTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.and(owner -> owner
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId()))
.lt(FileTaskEntity::getUpdatedAt, threshold)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 100"));
for (FileTaskEntity candidate : staleTasks) {
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
continue;
}
transactionTemplate.executeWithoutResult(status -> failStaleTaskLocked(candidate.getId(), threshold));
} catch (Exception ex) {
log.warn("[publish] stale task cleanup failed taskId={} msg={}",
candidate.getId(), ex.getMessage());
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
if (lock == null) {
continue;
}
transactionTemplate.executeWithoutResult(status -> failStaleTaskLocked(candidate.getId(), threshold));
} catch (Exception ex) {
log.warn("[publish] stale task cleanup failed taskId={} msg={}",
candidate.getId(), ex.getMessage());
}
}
}
}