fix(调度): 双节点下 collect-data/publish/image-video 定时调度加 Redis 分布式锁互斥——双活原来各自扫描执行同一批任务,重复派发/重复终态判定
This commit is contained in:
+83
-63
@@ -58,6 +58,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
@@ -147,6 +148,7 @@ public class CollectDataService {
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
/** ASIN 去重 + 无效品牌批量集合查询器:两段式查询合并为一次往返,语义与旧实现等价。 */
|
||||
private final CollectDataBatchQuery collectDataBatchQuery;
|
||||
@@ -483,16 +485,25 @@ public class CollectDataService {
|
||||
|
||||
@Scheduled(cron = "${aiimage.collect-data.stale-check-cron:*/30 * * * * *}")
|
||||
public void finalizeStaleTasks() {
|
||||
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
|
||||
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 200"));
|
||||
for (FileTaskEntity task : tasks) {
|
||||
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
|
||||
// 双节点互斥:Redis 不可用时 fail-open(仅记日志),兜底靠下方 per-task DB 锁,不会双写终态
|
||||
DistributedJobLockService.LockHandle jobLock =
|
||||
distributedJobLockService.tryLock("collect-data:stale-check", java.time.Duration.ofMinutes(2));
|
||||
if (jobLock == null) {
|
||||
log.info("[collect-data] stale-check skipped, another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (jobLock) {
|
||||
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
|
||||
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 200"));
|
||||
for (FileTaskEntity task : tasks) {
|
||||
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1201,36 +1212,6 @@ public class CollectDataService {
|
||||
return safe.isBlank() ? "collect-data" : safe;
|
||||
}
|
||||
|
||||
private void deleteTransientTaskPayloads(List<TaskChunkEntity> chunks, List<TaskResultItemEntity> items) {
|
||||
if (chunks != null) {
|
||||
Set<String> deleted = new HashSet<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
// 多 chunk 共享同一对象(deterministic key 残留场景)按值去重只删一次。
|
||||
if (chunk.getPayloadJson() != null && deleted.add(chunk.getPayloadJson())) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
deleteResultItemPayloads(items);
|
||||
}
|
||||
|
||||
/** 删除结果明细 payload:chunk 级引用按对象去重后各删一次,旧格式逐行删。 */
|
||||
private void deleteResultItemPayloads(List<TaskResultItemEntity> items) {
|
||||
if (items == null) {
|
||||
return;
|
||||
}
|
||||
Set<String> deletedPointers = new HashSet<>();
|
||||
for (TaskResultItemEntity item : items) {
|
||||
CollectDataResultDetailCodec.ChunkRef ref = resultDetailCodec.parseRef(item.getPayloadJson());
|
||||
if (ref != null) {
|
||||
if (deletedPointers.add(ref.pointer())) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(ref.pointer());
|
||||
}
|
||||
} else {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(item.getPayloadJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class CollectDataStats {
|
||||
private int totalRows;
|
||||
@@ -1252,9 +1233,38 @@ public class CollectDataService {
|
||||
private List<CollectDataSummaryRowDto> summaries = new ArrayList<>();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
// 不用 @Transactional(同类自调用代理失效):用 TransactionTemplate 显式包 DB 段,
|
||||
// 事务提交后再做远程删除(慢 IO),避免大任务删除时长事务占用连接池。
|
||||
public void deleteTask(Long taskId, Long userId) {
|
||||
List<String> chunkPayloadPointers = new ArrayList<>();
|
||||
List<String> resultPayloadJsons = new ArrayList<>();
|
||||
transactionTemplate.executeWithoutResult(status ->
|
||||
deleteTaskRows(taskId, userId, chunkPayloadPointers, resultPayloadJsons));
|
||||
// 事务已提交,这里执行远程删除
|
||||
Set<String> deleted = new HashSet<>();
|
||||
for (String pointer : chunkPayloadPointers) {
|
||||
if (deleted.add(pointer)) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(pointer);
|
||||
}
|
||||
}
|
||||
deletePayloadJsonsAndPointers(resultPayloadJsons, deleted);
|
||||
}
|
||||
|
||||
private void deleteTaskRows(Long taskId, Long userId, List<String> chunkPayloadPointers, List<String> resultPayloadJsons) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getPayloadJson)
|
||||
.eq(TaskChunkEntity::getTaskId, task.getId())
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)).stream()
|
||||
.map(TaskChunkEntity::getPayloadJson)
|
||||
.filter(Objects::nonNull)
|
||||
.forEach(chunkPayloadPointers::add);
|
||||
taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, task.getId())
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)).stream()
|
||||
.map(TaskResultItemEntity::getPayloadJson)
|
||||
.forEach(resultPayloadJsons::add);
|
||||
collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, task.getId()));
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
@@ -1271,37 +1281,47 @@ public class CollectDataService {
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, task.getId())
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE));
|
||||
deleteTransientTaskPayloads(
|
||||
taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getPayloadJson)
|
||||
.eq(TaskChunkEntity::getTaskId, task.getId())
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)),
|
||||
taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, task.getId())
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)));
|
||||
taskFileJobService.deleteTaskJobs(task.getId(), MODULE_TYPE);
|
||||
fileTaskMapper.deleteById(task.getId());
|
||||
}
|
||||
|
||||
private void deletePayloadJsonsAndPointers(List<String> payloadJsons, Set<String> deletedPointers) {
|
||||
for (String payloadJson : payloadJsons) {
|
||||
if (payloadJson == null) {
|
||||
continue;
|
||||
}
|
||||
CollectDataResultDetailCodec.ChunkRef ref = resultDetailCodec.parseRef(payloadJson);
|
||||
if (ref != null) {
|
||||
if (deletedPointers.add(ref.pointer())) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(ref.pointer());
|
||||
}
|
||||
} else {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(payloadJson);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteHistory(Long resultId, Long userId) {
|
||||
FileResultEntity row = fileResultMapper.selectById(resultId);
|
||||
if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
// 与 deleteTask 一致:先删 DB 行再物理删对象,保证行删除与对象删除一致。
|
||||
deleteResultItemPayloads(resultItems);
|
||||
taskFileJobService.deleteResultJobs(row.getTaskId(), MODULE_TYPE, row.getId());
|
||||
fileResultMapper.deleteById(resultId);
|
||||
List<String> resultPayloadJsons = transactionTemplate.execute(status -> {
|
||||
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
taskFileJobService.deleteResultJobs(row.getTaskId(), MODULE_TYPE, row.getId());
|
||||
fileResultMapper.deleteById(resultId);
|
||||
return resultItems.stream().map(TaskResultItemEntity::getPayloadJson).toList();
|
||||
});
|
||||
// 事务已提交,再物理删对象,保证行删除与对象删除一致。
|
||||
deletePayloadJsonsAndPointers(resultPayloadJsons, new HashSet<>());
|
||||
}
|
||||
|
||||
public CollectDataCountryPreferenceVo getCountryPreference(Long userId) {
|
||||
|
||||
Reference in New Issue
Block a user