task-69: 历史清理任务收集 transient payload 指针并提交删除队列

2026-08-30 19:29:32 +08:00
parent 54260551ee
commit 5749a421cb
3 changed files with 469 additions and 2 deletions
@@ -24,6 +24,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
 import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -55,6 +56,14 @@ public class ModuleHistoryCleanupService {
    private final TaskChunkMapper taskChunkMapper;
    private final CollectDataItemMapper collectDataItemMapper;
    private final DistributedJobLockService distributedJobLockService;
+    private final TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator;
+
+    /**
+     * 单次清理最多收集的 payload 指针数：超过即截断（保底可重试），
+     * 防止单任务行数异常巨大时无界收集造成内存增长。
+     */
+    @Value("${aiimage.module-cleanup.max-collect-payloads:10000}")
+    private int maxCollectPayloadsPerRun = 10000;

    @Transactional
    @Scheduled(cron = "${aiimage.module-cleanup.cron:0 0 0 * * *}")
@@ -108,6 +117,10 @@ public class ModuleHistoryCleanupService {
                return;
            }

+            // 行删除前先收集将随行删除的 transient payload 指针（上限截断），
+            // 全部删除成功后再提交清理队列并 flush，避免删除失败留下已提交的孤儿清理。
+            List<String> collectedPayloadPointers = collectPayloadPointers(cleanupTaskIds, maxCollectPayloadsPerRun);
+
            int deletedFileJobs = taskFileJobMapper.delete(new LambdaQueryWrapper<TaskFileJobEntity>()
                    .in(TaskFileJobEntity::getModuleType, moduleTypes)
                    .in(TaskFileJobEntity::getTaskId, cleanupTaskIds));
@@ -151,11 +164,17 @@ public class ModuleHistoryCleanupService {
            int deletedTasks = fileTaskMapper.delete(new LambdaQueryWrapper<FileTaskEntity>()
                    .in(FileTaskEntity::getId, cleanupTaskIds));

-            log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, deletedFileJobs={}, deletedResultItems={}, deletedProgressSnapshots={}, deletedResultPayloads={}, deletedScopeStates={}, deletedChunks={}, deletedCollectDataItems={}, deletedResults={}, resetTasks={}, deletedTasks={}, skippedActiveTaskIds={}, retainedTaskIds={}",
+            // 行已全部删除，此时引用检查将判定这些指针不再被引用 → 提交清理队列并 flush。
+            if (!collectedPayloadPointers.isEmpty()) {
+                transientPayloadDeleteOrchestrator.submitDeletes(collectedPayloadPointers);
+                transientPayloadDeleteOrchestrator.flushPendingDeletes();
+            }
+
+            log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, deletedFileJobs={}, deletedResultItems={}, deletedProgressSnapshots={}, deletedResultPayloads={}, deletedScopeStates={}, deletedChunks={}, deletedCollectDataItems={}, deletedResults={}, resetTasks={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}",
                    moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff,
                    deletedFileJobs, deletedResultItems, deletedProgressSnapshots, deletedResultPayloads,
                    deletedScopeStates, deletedChunks, deletedCollectDataItems, deletedResults, resetTasks, deletedTasks,
-                    skippedActiveTaskIds, skippedRetainedTaskIds);
+                    collectedPayloadPointers.size(), skippedActiveTaskIds, skippedRetainedTaskIds);
        }
    }

@@ -170,4 +189,40 @@ public class ModuleHistoryCleanupService {
        }
        return TERMINAL_STATUSES.contains(status.trim().toUpperCase(Locale.ROOT));
    }
+
+    /**
+     * 删除前批量收集将随行删除的 payload 指针（chunk.payloadJson、
+     * scope_state.parsedPayloadJson / stateJson），去重并保持稳定顺序；
+     * 达到 {@code max} 上限即截断，防止异常巨大的任务行数引发无界收集。
+     */
+    private List<String> collectPayloadPointers(List<Long> cleanupTaskIds, int max) {
+        java.util.Set<String> pointers = new java.util.LinkedHashSet<>();
+        List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
+                .in(TaskChunkEntity::getModuleType, moduleCleanupProperties.getModuleTypes())
+                .in(TaskChunkEntity::getTaskId, cleanupTaskIds));
+        for (TaskChunkEntity chunk : chunks) {
+            collectPointer(pointers, chunk.getPayloadJson(), max);
+        }
+        List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
+                .in(TaskScopeStateEntity::getModuleType, moduleCleanupProperties.getModuleTypes())
+                .in(TaskScopeStateEntity::getTaskId, cleanupTaskIds));
+        for (TaskScopeStateEntity scopeState : scopeStates) {
+            collectPointer(pointers, scopeState.getParsedPayloadJson(), max);
+            collectPointer(pointers, scopeState.getStateJson(), max);
+        }
+        if (pointers.size() >= max) {
+            log.warn("[module-cleanup] payload pointer collection truncated at max={}", max);
+        }
+        return new ArrayList<>(pointers);
+    }
+
+    private void collectPointer(java.util.Set<String> pointers, String value, int max) {
+        if (pointers.size() >= max || value == null || value.isBlank()) {
+            return;
+        }
+        String pointer = transientPayloadDeleteOrchestrator.extractPointer(value);
+        if (pointer != null) {
+            pointers.add(pointer);
+        }
+    }
 }
@@ -128,6 +128,14 @@ public class TransientPayloadDeleteOrchestrator {
        }
    }

+    /**
+     * 从任意存储值（指针或 JSON 编码指针）中提取 rustfs 指针，非指针返回 null。
+     * 供调用方在提交前过滤待删值（如清理任务收集 payload 字段）。
+     */
+    public String extractPointer(String value) {
+        return transientPayloadStorageService.extractPointer(value);
+    }
+
    private void deleteObjects(List<String> pointers) {
        for (String pointer : pointers) {
            if (pointer == null || !pointer.startsWith(RUSTFS_POINTER_PREFIX)) {