task-70: 历史清理改为 keyset 分页、小批量和短事务
This commit is contained in:
@@ -12,6 +12,11 @@ public class ModuleCleanupProperties {
|
||||
private boolean enabled = true;
|
||||
private String cron = "0 0 0 * * *";
|
||||
private long retentionDays = 7;
|
||||
/**
|
||||
* keyset 分页每批处理的任务数：每批一个短事务，批内按 id 升序，
|
||||
* 批间无行级排他锁占用；0/负值回退默认 500。
|
||||
*/
|
||||
private int batchSize = 500;
|
||||
// SHOP_DATA_CRAWL keeps one per-shop daily workbook in its task service
|
||||
// and must not be removed by the age-based sweep.
|
||||
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
|
||||
|
||||
+152
-92
@@ -22,12 +22,12 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -38,12 +38,12 @@ import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ModuleHistoryCleanupService {
|
||||
|
||||
private static final Set<String> TERMINAL_STATUSES = Set.of("SUCCESS", "FAILED", "CANCELLED", "CANCELED");
|
||||
private static final Duration CLEANUP_LOCK_TTL = Duration.ofHours(2);
|
||||
private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA";
|
||||
private static final int DEFAULT_BATCH_SIZE = 500;
|
||||
|
||||
private final ModuleCleanupProperties moduleCleanupProperties;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
@@ -57,15 +57,47 @@ public class ModuleHistoryCleanupService {
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
|
||||
/**
|
||||
* 单次清理最多收集的 payload 指针数：超过即截断（保底可重试），
|
||||
* 单批最多收集的 payload 指针数：超过即截断（保底可重试），
|
||||
* 防止单任务行数异常巨大时无界收集造成内存增长。
|
||||
*/
|
||||
@Value("${aiimage.module-cleanup.max-collect-payloads:10000}")
|
||||
private int maxCollectPayloadsPerRun = 10000;
|
||||
|
||||
@Transactional
|
||||
public ModuleHistoryCleanupService(ModuleCleanupProperties moduleCleanupProperties,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
FileResultMapper fileResultMapper,
|
||||
TaskFileJobMapper taskFileJobMapper,
|
||||
TaskResultItemMapper taskResultItemMapper,
|
||||
TaskProgressSnapshotMapper taskProgressSnapshotMapper,
|
||||
TaskResultPayloadMapper taskResultPayloadMapper,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
CollectDataItemMapper collectDataItemMapper,
|
||||
DistributedJobLockService distributedJobLockService,
|
||||
TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator,
|
||||
PlatformTransactionManager platformTransactionManager) {
|
||||
this.moduleCleanupProperties = moduleCleanupProperties;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.fileResultMapper = fileResultMapper;
|
||||
this.taskFileJobMapper = taskFileJobMapper;
|
||||
this.taskResultItemMapper = taskResultItemMapper;
|
||||
this.taskProgressSnapshotMapper = taskProgressSnapshotMapper;
|
||||
this.taskResultPayloadMapper = taskResultPayloadMapper;
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.collectDataItemMapper = collectDataItemMapper;
|
||||
this.distributedJobLockService = distributedJobLockService;
|
||||
this.transientPayloadDeleteOrchestrator = transientPayloadDeleteOrchestrator;
|
||||
this.transactionTemplate = new TransactionTemplate(platformTransactionManager);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 id 升序 keyset 分页拉取过期终态任务，每页一个小批量、一个短事务；
|
||||
* 页间不持有事务与行锁，失败回滚当页并中止，未处理页可下次重试。
|
||||
*/
|
||||
@Scheduled(cron = "${aiimage.module-cleanup.cron:0 0 0 * * *}")
|
||||
public void cleanupConfiguredModules() {
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
@@ -83,101 +115,129 @@ public class ModuleHistoryCleanupService {
|
||||
return;
|
||||
}
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusDays(Math.max(0, moduleCleanupProperties.getRetentionDays()));
|
||||
int batchSize = moduleCleanupProperties.getBatchSize();
|
||||
if (batchSize <= 0) {
|
||||
batchSize = DEFAULT_BATCH_SIZE;
|
||||
}
|
||||
|
||||
List<FileTaskEntity> moduleTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getModuleType, moduleTypes)
|
||||
.select(FileTaskEntity::getId, FileTaskEntity::getModuleType, FileTaskEntity::getStatus,
|
||||
FileTaskEntity::getUpdatedAt, FileTaskEntity::getFinishedAt));
|
||||
|
||||
List<Long> cleanupTaskIds = new ArrayList<>();
|
||||
List<Long> cleanupCollectDataTaskIds = new ArrayList<>();
|
||||
long cursor = 0L;
|
||||
int totalBatches = 0;
|
||||
int totalDeletedTasks = 0;
|
||||
int totalCollectedPointers = 0;
|
||||
List<Long> skippedActiveTaskIds = new ArrayList<>();
|
||||
List<Long> skippedRetainedTaskIds = new ArrayList<>();
|
||||
for (FileTaskEntity task : moduleTasks) {
|
||||
if (task == null || task.getId() == null) {
|
||||
continue;
|
||||
while (true) {
|
||||
List<FileTaskEntity> page = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getModuleType, moduleTypes)
|
||||
.gt(FileTaskEntity::getId, cursor)
|
||||
.orderByAsc(FileTaskEntity::getId)
|
||||
.last("LIMIT " + batchSize));
|
||||
if (page.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
if (!isTerminalStatus(task.getStatus())) {
|
||||
skippedActiveTaskIds.add(task.getId());
|
||||
continue;
|
||||
}
|
||||
if (isExpired(task, cutoff)) {
|
||||
cleanupTaskIds.add(task.getId());
|
||||
if (COLLECT_DATA_MODULE_TYPE.equals(task.getModuleType())) {
|
||||
cleanupCollectDataTaskIds.add(task.getId());
|
||||
long pageMaxId = cursor;
|
||||
List<Long> batchTaskIds = new ArrayList<>();
|
||||
List<Long> batchCollectDataTaskIds = new ArrayList<>();
|
||||
for (FileTaskEntity task : page) {
|
||||
if (task == null || task.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
pageMaxId = Math.max(pageMaxId, task.getId());
|
||||
if (!isTerminalStatus(task.getStatus())) {
|
||||
skippedActiveTaskIds.add(task.getId());
|
||||
continue;
|
||||
}
|
||||
if (isExpired(task, cutoff)) {
|
||||
batchTaskIds.add(task.getId());
|
||||
if (COLLECT_DATA_MODULE_TYPE.equals(task.getModuleType())) {
|
||||
batchCollectDataTaskIds.add(task.getId());
|
||||
}
|
||||
} else {
|
||||
skippedRetainedTaskIds.add(task.getId());
|
||||
}
|
||||
} else {
|
||||
skippedRetainedTaskIds.add(task.getId());
|
||||
}
|
||||
if (!batchTaskIds.isEmpty()) {
|
||||
final List<Long> taskIds = batchTaskIds;
|
||||
final List<Long> collectDataTaskIds = batchCollectDataTaskIds;
|
||||
final List<String> types = moduleTypes;
|
||||
final List<String> collected = new ArrayList<>();
|
||||
transactionTemplate.executeWithoutResult(status -> {
|
||||
collected.addAll(collectPayloadPointers(types, taskIds, maxCollectPayloadsPerRun));
|
||||
int deletedRows = deleteRows(types, taskIds, collectDataTaskIds);
|
||||
submitAndFlush(collected);
|
||||
log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}",
|
||||
taskIds, deletedRows, collected.size());
|
||||
});
|
||||
totalBatches++;
|
||||
totalDeletedTasks += taskIds.size();
|
||||
totalCollectedPointers += collected.size();
|
||||
}
|
||||
if (pageMaxId <= cursor) {
|
||||
log.warn("[module-cleanup] keyset cursor did not advance, abort loop cursor={}", cursor);
|
||||
break;
|
||||
}
|
||||
cursor = pageMaxId;
|
||||
}
|
||||
|
||||
if (cleanupTaskIds.isEmpty()) {
|
||||
log.info("[module-cleanup] skipped: moduleTypes={}, retentionDays={}, cutoff={}, activeTaskIds={}, retainedTaskIds={}, reason=no-expired-terminal-tasks",
|
||||
moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff, skippedActiveTaskIds, skippedRetainedTaskIds);
|
||||
return;
|
||||
}
|
||||
|
||||
// 行删除前先收集将随行删除的 transient payload 指针（上限截断），
|
||||
// 全部删除成功后再提交清理队列并 flush，避免删除失败留下已提交的孤儿清理。
|
||||
List<String> collectedPayloadPointers = collectPayloadPointers(cleanupTaskIds, maxCollectPayloadsPerRun);
|
||||
|
||||
int deletedFileJobs = taskFileJobMapper.delete(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getModuleType, moduleTypes)
|
||||
.in(TaskFileJobEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
int deletedResultItems = taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.in(TaskResultItemEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultItemEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
int deletedProgressSnapshots = taskProgressSnapshotMapper.delete(new LambdaQueryWrapper<TaskProgressSnapshotEntity>()
|
||||
.in(TaskProgressSnapshotEntity::getModuleType, moduleTypes)
|
||||
.in(TaskProgressSnapshotEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
int deletedResultPayloads = taskResultPayloadMapper.delete(new LambdaQueryWrapper<TaskResultPayloadEntity>()
|
||||
.in(TaskResultPayloadEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
int deletedScopeStates = taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.in(TaskScopeStateEntity::getModuleType, moduleTypes)
|
||||
.in(TaskScopeStateEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
int deletedChunks = taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getModuleType, moduleTypes)
|
||||
.in(TaskChunkEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
int deletedCollectDataItems = 0;
|
||||
if (!cleanupCollectDataTaskIds.isEmpty()) {
|
||||
deletedCollectDataItems = collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.in(CollectDataItemEntity::getTaskId, cleanupCollectDataTaskIds));
|
||||
}
|
||||
|
||||
int deletedResults = fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getModuleType, moduleTypes)
|
||||
.in(FileResultEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
int resetTasks = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, cleanupTaskIds)
|
||||
.set(FileTaskEntity::getResultJson, null)
|
||||
.set(FileTaskEntity::getRequestJson, null)
|
||||
.set(FileTaskEntity::getErrorMessage, null));
|
||||
|
||||
int deletedTasks = fileTaskMapper.delete(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, cleanupTaskIds));
|
||||
|
||||
// 行已全部删除，此时引用检查将判定这些指针不再被引用 → 提交清理队列并 flush。
|
||||
if (!collectedPayloadPointers.isEmpty()) {
|
||||
transientPayloadDeleteOrchestrator.submitDeletes(collectedPayloadPointers);
|
||||
transientPayloadDeleteOrchestrator.flushPendingDeletes();
|
||||
}
|
||||
|
||||
log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, deletedFileJobs={}, deletedResultItems={}, deletedProgressSnapshots={}, deletedResultPayloads={}, deletedScopeStates={}, deletedChunks={}, deletedCollectDataItems={}, deletedResults={}, resetTasks={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}",
|
||||
log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, batches={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}",
|
||||
moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff,
|
||||
deletedFileJobs, deletedResultItems, deletedProgressSnapshots, deletedResultPayloads,
|
||||
deletedScopeStates, deletedChunks, deletedCollectDataItems, deletedResults, resetTasks, deletedTasks,
|
||||
collectedPayloadPointers.size(), skippedActiveTaskIds, skippedRetainedTaskIds);
|
||||
totalBatches, totalDeletedTasks, totalCollectedPointers,
|
||||
skippedActiveTaskIds, skippedRetainedTaskIds);
|
||||
}
|
||||
}
|
||||
|
||||
/** 一个事务内完成：收集指针 → 删除本批行 → 行删完后提交清理队列并 flush。 */
|
||||
private int deleteRows(List<String> moduleTypes, List<Long> cleanupTaskIds, List<Long> collectDataTaskIds) {
|
||||
taskFileJobMapper.delete(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getModuleType, moduleTypes)
|
||||
.in(TaskFileJobEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.in(TaskResultItemEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultItemEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
taskProgressSnapshotMapper.delete(new LambdaQueryWrapper<TaskProgressSnapshotEntity>()
|
||||
.in(TaskProgressSnapshotEntity::getModuleType, moduleTypes)
|
||||
.in(TaskProgressSnapshotEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
taskResultPayloadMapper.delete(new LambdaQueryWrapper<TaskResultPayloadEntity>()
|
||||
.in(TaskResultPayloadEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.in(TaskScopeStateEntity::getModuleType, moduleTypes)
|
||||
.in(TaskScopeStateEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getModuleType, moduleTypes)
|
||||
.in(TaskChunkEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
if (!collectDataTaskIds.isEmpty()) {
|
||||
collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.in(CollectDataItemEntity::getTaskId, collectDataTaskIds));
|
||||
}
|
||||
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getModuleType, moduleTypes)
|
||||
.in(FileResultEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, cleanupTaskIds)
|
||||
.set(FileTaskEntity::getResultJson, null)
|
||||
.set(FileTaskEntity::getRequestJson, null)
|
||||
.set(FileTaskEntity::getErrorMessage, null));
|
||||
|
||||
return fileTaskMapper.delete(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, cleanupTaskIds));
|
||||
}
|
||||
|
||||
private void submitAndFlush(List<String> collectedPointers) {
|
||||
if (collectedPointers.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
transientPayloadDeleteOrchestrator.submitDeletes(collectedPointers);
|
||||
transientPayloadDeleteOrchestrator.flushPendingDeletes();
|
||||
}
|
||||
|
||||
private boolean isExpired(FileTaskEntity task, LocalDateTime cutoff) {
|
||||
LocalDateTime completedAt = task.getFinishedAt() != null ? task.getFinishedAt() : task.getUpdatedAt();
|
||||
return completedAt != null && !completedAt.isAfter(cutoff);
|
||||
@@ -195,16 +255,16 @@ public class ModuleHistoryCleanupService {
|
||||
* scope_state.parsedPayloadJson / stateJson），去重并保持稳定顺序；
|
||||
* 达到 {@code max} 上限即截断，防止异常巨大的任务行数引发无界收集。
|
||||
*/
|
||||
private List<String> collectPayloadPointers(List<Long> cleanupTaskIds, int max) {
|
||||
private List<String> collectPayloadPointers(List<String> moduleTypes, List<Long> cleanupTaskIds, int max) {
|
||||
java.util.Set<String> pointers = new java.util.LinkedHashSet<>();
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getModuleType, moduleCleanupProperties.getModuleTypes())
|
||||
.in(TaskChunkEntity::getModuleType, moduleTypes)
|
||||
.in(TaskChunkEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
collectPointer(pointers, chunk.getPayloadJson(), max);
|
||||
}
|
||||
List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.in(TaskScopeStateEntity::getModuleType, moduleCleanupProperties.getModuleTypes())
|
||||
.in(TaskScopeStateEntity::getModuleType, moduleTypes)
|
||||
.in(TaskScopeStateEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskScopeStateEntity scopeState : scopeStates) {
|
||||
collectPointer(pointers, scopeState.getParsedPayloadJson(), max);
|
||||
|
||||
Reference in New Issue
Block a user