feat(保留期清理): 补齐 7 处只增不删的数据;修结果对象孤儿
审核发现一批表/元数据只增不删,且对象存储存在活跃泄漏: - module-cleanup 漏收 task_result_item / task_result_payload 的 payload 指针, 每晚删完行就在 json-server 桶留下孤儿对象;指针收集触顶从"截断后照样删整组" 改为任务组二分拆分,拆到单任务仍触顶则整组不删并记 error(截断会让指针随行消失) - 结果文件对象此前从不回收(只删 DB 行、对象留在桶里),现改为事务提交后 按 file_result / task_file_job 的 result_file_url 逐个回收 - 新增 7 个保留期清理:device_log_file 元数据、price_track_loop_run、撞款扫描行、 密钥用量日统计、紫鸟记忆过期行、PUBLISH 任务、BRAND 任务。后两者刻意不复用 module-cleanup(BRAND 不写 biz_file_task;PUBLISH 单任务可达数万行), 改为调各自既有的业务删除入口,保证结果对象与子表一起回收 - 未读通知补保留期(180 天,已读仍 90 天);ZiniaoMemoryStoreService.deleteExpired 此前全库无调用方(实现了没接线),现已挂定时任务 全部沿用既有范式:@Scheduled + 分布式锁单实例 + 分批 + 单轮批次数上限 + 失败只记日志。
This commit is contained in:
@@ -3,7 +3,35 @@ package com.nanri.aiimage.modules.task.mapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface FileTaskMapper extends BaseMapper<FileTaskEntity> {
|
||||
|
||||
/**
|
||||
* 查询某模块下超过保留期的终态任务 id(保留期清理用,只取 id 不拉整行)。
|
||||
*
|
||||
* <p>终态集合与 ModuleHistoryCleanupService 的 TERMINAL_STATUSES 一致:PENDING/RUNNING 绝不返回——
|
||||
* 正在跑的任务被删会让分片回传、结果组装找不到任务行。
|
||||
*
|
||||
* <p>时间线只用 updated_at:该列由 DB 的 ON UPDATE CURRENT_TIMESTAMP 维护,任何一次行更新
|
||||
* (含终态写入)都会刷新,不会早于 finished_at(finished_at 在任务重置时会被置空,不可单独依赖);
|
||||
* 且能命中 V120 的 idx_biz_file_task_status_updated(status, updated_at) 索引,避免每轮全表扫描。
|
||||
* 注意不要在这里包 COALESCE(finished_at, updated_at):表达式会让该索引失效。
|
||||
*/
|
||||
@Select("""
|
||||
SELECT id FROM biz_file_task
|
||||
WHERE module_type = #{moduleType}
|
||||
AND status IN ('SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED')
|
||||
AND updated_at < #{cutoff}
|
||||
ORDER BY id ASC
|
||||
LIMIT #{batchSize}
|
||||
""")
|
||||
List<Long> selectExpiredTerminalTaskIds(@Param("moduleType") String moduleType,
|
||||
@Param("cutoff") LocalDateTime cutoff,
|
||||
@Param("batchSize") int batchSize);
|
||||
}
|
||||
|
||||
+133
-21
@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
@@ -44,6 +45,8 @@ public class ModuleHistoryCleanupService {
|
||||
private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA";
|
||||
private static final int DEFAULT_BATCH_SIZE = 500;
|
||||
private static final int LOG_SAMPLE_IDS = 5;
|
||||
/** 指针收集触顶时任务组的最大二分拆分深度(500 任务拆到单个任务约需 9 层)。 */
|
||||
private static final int MAX_POINTER_SPLIT_DEPTH = 20;
|
||||
|
||||
private final ModuleCleanupProperties moduleCleanupProperties;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
@@ -57,14 +60,15 @@ public class ModuleHistoryCleanupService {
|
||||
private final CollectDataItemCleanupSpi collectDataItemCleanupSpi;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
|
||||
/**
|
||||
* 单批最多收集的 payload 指针数:超过即截断(保底可重试),
|
||||
* 单组最多收集的 payload 指针数:超过即把任务组二分拆分(见 {@link #cleanupTaskGroup}),
|
||||
* 防止单任务行数异常巨大时无界收集造成内存增长。
|
||||
*/
|
||||
@Value("${aiimage.module-cleanup.max-collect-payloads:10000}")
|
||||
private int maxCollectPayloadsPerRun = 10000;
|
||||
@Value("${aiimage.module-cleanup.max-collect-payloads:50000}")
|
||||
private int maxCollectPayloadsPerRun = 50000;
|
||||
|
||||
public ModuleHistoryCleanupService(ModuleCleanupProperties moduleCleanupProperties,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
@@ -78,6 +82,7 @@ public class ModuleHistoryCleanupService {
|
||||
CollectDataItemCleanupSpi collectDataItemCleanupSpi,
|
||||
DistributedJobLockService distributedJobLockService,
|
||||
TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator,
|
||||
OssStorageService ossStorageService,
|
||||
PlatformTransactionManager platformTransactionManager) {
|
||||
this.moduleCleanupProperties = moduleCleanupProperties;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
@@ -91,6 +96,7 @@ public class ModuleHistoryCleanupService {
|
||||
this.collectDataItemCleanupSpi = collectDataItemCleanupSpi;
|
||||
this.distributedJobLockService = distributedJobLockService;
|
||||
this.transientPayloadDeleteOrchestrator = transientPayloadDeleteOrchestrator;
|
||||
this.ossStorageService = ossStorageService;
|
||||
this.transactionTemplate = new TransactionTemplate(platformTransactionManager);
|
||||
}
|
||||
|
||||
@@ -161,20 +167,11 @@ public class ModuleHistoryCleanupService {
|
||||
}
|
||||
}
|
||||
if (!batchTaskIds.isEmpty()) {
|
||||
final List<Long> taskIds = batchTaskIds;
|
||||
final List<Long> collectDataTaskIds = batchCollectDataTaskIds;
|
||||
final List<String> types = moduleTypes;
|
||||
final List<String> collected = new ArrayList<>();
|
||||
transactionTemplate.executeWithoutResult(status -> {
|
||||
collected.addAll(collectPayloadPointers(types, taskIds, maxCollectPayloadsPerRun));
|
||||
int deletedRows = deleteRows(types, taskIds, collectDataTaskIds);
|
||||
submitAndFlush(collected);
|
||||
log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}",
|
||||
formatIdSample(taskIds, LOG_SAMPLE_IDS), deletedRows, collected.size());
|
||||
});
|
||||
totalBatches++;
|
||||
totalDeletedTasks += taskIds.size();
|
||||
totalCollectedPointers += collected.size();
|
||||
BatchTally tally = new BatchTally();
|
||||
cleanupTaskGroup(moduleTypes, batchTaskIds, batchCollectDataTaskIds, 0, tally);
|
||||
totalBatches += tally.batches;
|
||||
totalDeletedTasks += tally.deletedTasks;
|
||||
totalCollectedPointers += tally.collectedPointers;
|
||||
}
|
||||
if (pageMaxId <= cursor) {
|
||||
log.warn("[module-cleanup] keyset cursor did not advance, abort loop cursor={}", cursor);
|
||||
@@ -190,7 +187,107 @@ public class ModuleHistoryCleanupService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 一个事务内完成:收集指针 → 删除本批行 → 行删完后提交清理队列并 flush。 */
|
||||
/**
|
||||
* 清理一组过期任务:收集 payload 指针 → 删除行 → 提交对象回收。
|
||||
*
|
||||
* <p>指针数触顶时把任务组二分拆分重试,保证「凡被删除的行,其 transient 指针一定已进回收队列」——
|
||||
* 截断会让指针随行一起消失,对象再无引用可查、永久留在桶里。
|
||||
* 拆到单个任务仍触顶则整组不删并记 error(宁可留脏行待下轮重试,也不制造孤儿对象);
|
||||
* 运维可据此调大 {@code aiimage.module-cleanup.max-collect-payloads} 后自动收敛。
|
||||
*/
|
||||
private void cleanupTaskGroup(List<String> moduleTypes, List<Long> taskIds, List<Long> collectDataTaskIds,
|
||||
int depth, BatchTally tally) {
|
||||
if (taskIds.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<String> collected = collectPayloadPointers(moduleTypes, taskIds, maxCollectPayloadsPerRun);
|
||||
boolean capped = collected.size() >= maxCollectPayloadsPerRun;
|
||||
if (capped && taskIds.size() > 1 && depth < MAX_POINTER_SPLIT_DEPTH) {
|
||||
int half = taskIds.size() / 2;
|
||||
log.warn("[module-cleanup] payload 指针数触顶 {},任务组二分拆分重试: depth={}, size={}",
|
||||
maxCollectPayloadsPerRun, depth, taskIds.size());
|
||||
cleanupTaskGroup(moduleTypes, new ArrayList<>(taskIds.subList(0, half)),
|
||||
collectDataTaskIds, depth + 1, tally);
|
||||
cleanupTaskGroup(moduleTypes, new ArrayList<>(taskIds.subList(half, taskIds.size())),
|
||||
collectDataTaskIds, depth + 1, tally);
|
||||
return;
|
||||
}
|
||||
if (capped) {
|
||||
log.error("[module-cleanup] payload 指针数触顶且无法再拆分,本轮保留任务行待下轮重试: taskIds={}",
|
||||
formatIdSample(taskIds, LOG_SAMPLE_IDS));
|
||||
return;
|
||||
}
|
||||
final List<Long> ids = List.copyOf(taskIds);
|
||||
final Set<Long> collectDataIdSet = Set.copyOf(collectDataTaskIds);
|
||||
final List<Long> groupCollectDataIds = ids.stream().filter(collectDataIdSet::contains).toList();
|
||||
List<String> resultObjectKeys = collectResultObjectKeys(moduleTypes, ids);
|
||||
transactionTemplate.executeWithoutResult(status -> {
|
||||
int deletedRows = deleteRows(moduleTypes, ids, groupCollectDataIds);
|
||||
log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}, resultObjects={}",
|
||||
formatIdSample(ids, LOG_SAMPLE_IDS), deletedRows, collected.size(), resultObjectKeys.size());
|
||||
});
|
||||
// 远端删除一律放在事务提交之后:事务里做,一旦回滚就会出现「行还在、对象已被删」的悬空引用
|
||||
submitAndFlush(collected);
|
||||
deleteResultObjects(resultObjectKeys);
|
||||
tally.batches++;
|
||||
tally.deletedTasks += ids.size();
|
||||
tally.collectedPointers += collected.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 收集随行删除的结果文件对象 key(file_result.result_file_url、task_file_job.result_file_url)。
|
||||
*
|
||||
* <p>这些对象此前从不回收——行删掉后再没有任何地方记录过它们,桶里只能靠生命周期规则兜底;
|
||||
* 一旦某天桶规则被调整(历史上就误配过全桶 30 天过期),就会变成永久垃圾。
|
||||
*/
|
||||
private List<String> collectResultObjectKeys(List<String> moduleTypes, List<Long> cleanupTaskIds) {
|
||||
java.util.Set<String> keys = new java.util.LinkedHashSet<>();
|
||||
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getModuleType, moduleTypes)
|
||||
.in(FileResultEntity::getTaskId, cleanupTaskIds));
|
||||
for (FileResultEntity result : results) {
|
||||
addObjectKey(keys, result.getResultFileUrl());
|
||||
}
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getModuleType, moduleTypes)
|
||||
.in(TaskFileJobEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
addObjectKey(keys, job.getResultFileUrl());
|
||||
}
|
||||
return new ArrayList<>(keys);
|
||||
}
|
||||
|
||||
private void addObjectKey(java.util.Set<String> keys, String value) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
keys.add(value.trim());
|
||||
}
|
||||
}
|
||||
|
||||
/** 逐个回收结果对象;单个失败只记日志,不影响其余对象与后续批次。 */
|
||||
private void deleteResultObjects(List<String> objectKeys) {
|
||||
if (objectKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int failed = 0;
|
||||
for (String objectKey : objectKeys) {
|
||||
try {
|
||||
ossStorageService.deleteObject(objectKey);
|
||||
} catch (Exception ex) {
|
||||
failed++;
|
||||
log.warn("[module-cleanup] 结果对象删除失败 key={} msg={}", objectKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
log.info("[module-cleanup] 结果对象回收完成 count={} failed={}", objectKeys.size(), failed);
|
||||
}
|
||||
|
||||
/** 分组拆分后回传计数:批次数、删除任务数、收集指针数。 */
|
||||
private static final class BatchTally {
|
||||
private int batches;
|
||||
private int deletedTasks;
|
||||
private int collectedPointers;
|
||||
}
|
||||
|
||||
/** 一个事务内完成:删除本批行 → 行删完后提交清理队列并 flush(指针已由调用方收集)。 */
|
||||
private int deleteRows(List<String> moduleTypes, List<Long> cleanupTaskIds, List<Long> collectDataTaskIds) {
|
||||
taskFileJobMapper.delete(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getModuleType, moduleTypes)
|
||||
@@ -257,8 +354,11 @@ public class ModuleHistoryCleanupService {
|
||||
|
||||
/**
|
||||
* 删除前批量收集将随行删除的 payload 指针(chunk.payloadJson、
|
||||
* scope_state.parsedPayloadJson / stateJson),去重并保持稳定顺序;
|
||||
* 达到 {@code max} 上限即截断,防止异常巨大的任务行数引发无界收集。
|
||||
* scope_state.parsedPayloadJson / stateJson、result_item.payloadJson、
|
||||
* result_payload.payloadJson),去重并保持稳定顺序。
|
||||
*
|
||||
* <p>必须覆盖**所有**存放 transient 指针的列:行一旦删除,漏收的指针就再也无法
|
||||
* 定位对象,桶里会永久残留孤儿对象。新增写入 payload 指针的列时必须同步加进来。
|
||||
*/
|
||||
private List<String> collectPayloadPointers(List<String> moduleTypes, List<Long> cleanupTaskIds, int max) {
|
||||
java.util.Set<String> pointers = new java.util.LinkedHashSet<>();
|
||||
@@ -275,8 +375,20 @@ public class ModuleHistoryCleanupService {
|
||||
collectPointer(pointers, scopeState.getParsedPayloadJson(), max);
|
||||
collectPointer(pointers, scopeState.getStateJson(), max);
|
||||
}
|
||||
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.in(TaskResultItemEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultItemEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskResultItemEntity resultItem : resultItems) {
|
||||
collectPointer(pointers, resultItem.getPayloadJson(), max);
|
||||
}
|
||||
List<TaskResultPayloadEntity> resultPayloads = taskResultPayloadMapper.selectList(new LambdaQueryWrapper<TaskResultPayloadEntity>()
|
||||
.in(TaskResultPayloadEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskResultPayloadEntity resultPayload : resultPayloads) {
|
||||
collectPointer(pointers, resultPayload.getPayloadJson(), max);
|
||||
}
|
||||
if (pointers.size() >= max) {
|
||||
log.warn("[module-cleanup] payload pointer collection truncated at max={}", max);
|
||||
log.warn("[module-cleanup] payload pointer collection reached max={}, 交由调用方拆分重试", max);
|
||||
}
|
||||
return new ArrayList<>(pointers);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user