feat(保留期清理): 补齐 7 处只增不删的数据;修结果对象孤儿
审核发现一批表/元数据只增不删,且对象存储存在活跃泄漏: - module-cleanup 漏收 task_result_item / task_result_payload 的 payload 指针, 每晚删完行就在 json-server 桶留下孤儿对象;指针收集触顶从"截断后照样删整组" 改为任务组二分拆分,拆到单任务仍触顶则整组不删并记 error(截断会让指针随行消失) - 结果文件对象此前从不回收(只删 DB 行、对象留在桶里),现改为事务提交后 按 file_result / task_file_job 的 result_file_url 逐个回收 - 新增 7 个保留期清理:device_log_file 元数据、price_track_loop_run、撞款扫描行、 密钥用量日统计、紫鸟记忆过期行、PUBLISH 任务、BRAND 任务。后两者刻意不复用 module-cleanup(BRAND 不写 biz_file_task;PUBLISH 单任务可达数万行), 改为调各自既有的业务删除入口,保证结果对象与子表一起回收 - 未读通知补保留期(180 天,已读仍 90 天);ZiniaoMemoryStoreService.deleteExpired 此前全库无调用方(实现了没接线),现已挂定时任务 全部沿用既有范式:@Scheduled + 分布式锁单实例 + 分批 + 单轮批次数上限 + 失败只记日志。
This commit is contained in:
+97
@@ -0,0 +1,97 @@
|
||||
package com.nanri.aiimage.modules.publish.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 上架任务的保留期清理(2026-09 审核:PUBLISH 的任务历史此前完全没有清理,永久累积)。
|
||||
*
|
||||
* <p>涉及 biz_file_task / biz_file_result / biz_publish_item(每个上传 Excel 的每一行落一行,
|
||||
* 单任务可达数万行)/ biz_publish_file,此前只有用户手动删任务才回收。这里刻意不复用
|
||||
* ModuleHistoryCleanupService:上架单任务行数太大,且删除必须经过业务删除入口回收结果对象。
|
||||
*
|
||||
* <p>删除动作复用 {@link PublishTaskService#deleteTaskForRetention(Long)}(与用户删任务
|
||||
* 同一套删除实现:明细/结果/分片载荷 + 事务提交后回收结果对象),本类只做「查一批过期 id →
|
||||
* 逐个调用 → 计数」。该入口保留状态校验,只删终态任务,PENDING/RUNNING 不会被碰。
|
||||
*
|
||||
* <p>双节点用 job 锁保证单实例执行;每批小批量(默认 50)逐个删,避免单次事务过长。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PublishTaskRetentionService {
|
||||
|
||||
/** 单轮最多处理的批次数(每批 batchSize 个任务),剩余留给下一轮。 */
|
||||
static final int MAX_BATCHES_PER_RUN = 20;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final PublishTaskService publishTaskService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Value("${aiimage.publish.task-retention-days:90}")
|
||||
private int retentionDays = 90;
|
||||
|
||||
@Value("${aiimage.publish.task-retention-batch-size:50}")
|
||||
private int retentionBatchSize = 50;
|
||||
|
||||
@Scheduled(cron = "${aiimage.publish.task-retention-cron:0 30 4 * * *}")
|
||||
public void purgeExpiredTasks() {
|
||||
int days = Math.max(1, retentionDays);
|
||||
int batchSize = Math.max(1, retentionBatchSize);
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusDays(days);
|
||||
|
||||
// 单轮 20 批 × 50 个任务的域内删除(含结果对象回收)可能跑较久,锁 TTL 给足 30 分钟
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("publish:task-retention", Duration.ofMinutes(30));
|
||||
if (lockHandle == null) {
|
||||
log.info("[publish-retention] 任务保留清理跳过:另一实例持锁");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
int totalDeleted = 0;
|
||||
int totalFailed = 0;
|
||||
int batches = 0;
|
||||
while (batches < MAX_BATCHES_PER_RUN) {
|
||||
// 只取 id:上架任务行可携带很大的 request_json/result_json,不拉整行
|
||||
List<Long> taskIds = fileTaskMapper.selectExpiredTerminalTaskIds(
|
||||
PublishTaskService.MODULE_TYPE, cutoff, batchSize);
|
||||
if (taskIds.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
batches++;
|
||||
int deletedInBatch = 0;
|
||||
for (Long taskId : taskIds) {
|
||||
try {
|
||||
publishTaskService.deleteTaskForRetention(taskId);
|
||||
deletedInBatch++;
|
||||
} catch (Exception ex) {
|
||||
// 单个任务删除失败只记日志继续:一个坏任务不能卡住整轮
|
||||
log.warn("[publish-retention] 任务删除失败 taskId={} msg={}", taskId, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
totalDeleted += deletedInBatch;
|
||||
totalFailed += taskIds.size() - deletedInBatch;
|
||||
if (deletedInBatch == 0) {
|
||||
log.warn("[publish-retention] 整批任务均删除失败,提前结束本轮以免反复重试同一批");
|
||||
break;
|
||||
}
|
||||
if (taskIds.size() < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
log.info("[publish-retention] 任务保留清理完成 cutoff={} retentionDays={} deleted={} failed={} batches={}",
|
||||
cutoff, days, totalDeleted, totalFailed, batches);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish-retention] 任务保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -96,6 +96,12 @@ public class PublishTaskService {
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||
private static final String STATUS_FAILED = "FAILED";
|
||||
/**
|
||||
* 保留期清理允许删除的终态集合(与 ModuleHistoryCleanupService 的 TERMINAL_STATUSES 一致)。
|
||||
* PENDING/RUNNING 绝不在此列;PUBLISH 实际只会写 SUCCESS/FAILED,取消态是无害的超集。
|
||||
*/
|
||||
private static final Set<String> RETENTION_TERMINAL_STATUSES =
|
||||
Set.of(STATUS_SUCCESS, STATUS_FAILED, "CANCELLED", "CANCELED");
|
||||
private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
private final LocalFileStorageService localFileStorageService;
|
||||
@@ -566,6 +572,37 @@ public class PublishTaskService {
|
||||
@Transactional
|
||||
public void deleteTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTaskForDeletion(taskId, userId);
|
||||
deleteTaskWithRelatedRows(task);
|
||||
}
|
||||
|
||||
/**
|
||||
* 保留期清理专用删除入口(内部调用,定时任务没有用户身份)。
|
||||
*
|
||||
* <p>与 {@link #deleteTask(Long, Long)} 复用同一套删除实现(明细行/结果记录/分片载荷 +
|
||||
* 事务提交后回收结果对象),仅跳过用户归属校验;状态校验保留 —— 只允许删终态任务,
|
||||
* PENDING/RUNNING 一律不删(正在跑的任务被删会让分片回传、结果组装找不到任务行)。
|
||||
* 行已被删除或非上架任务时幂等跳过,不抛异常。
|
||||
*/
|
||||
@Transactional
|
||||
public void deleteTaskForRetention(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("taskId 不合法");
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
log.info("[publish] 保留期删除跳过:任务已不存在或非上架任务 taskId={}", taskId);
|
||||
return;
|
||||
}
|
||||
if (!isRetentionTerminal(task.getStatus())) {
|
||||
log.warn("[publish] 保留期删除跳过非终态任务 taskId={} status={}", taskId, task.getStatus());
|
||||
return;
|
||||
}
|
||||
deleteTaskWithRelatedRows(task);
|
||||
}
|
||||
|
||||
/** 删除任务行与全部关联数据,并在事务提交后回收远端对象(deleteTask 与保留期清理共用)。 */
|
||||
private void deleteTaskWithRelatedRows(FileTaskEntity task) {
|
||||
Long taskId = task.getId();
|
||||
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, taskId)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
@@ -1953,6 +1990,11 @@ public class PublishTaskService {
|
||||
return STATUS_SUCCESS.equals(status) || STATUS_FAILED.equals(status);
|
||||
}
|
||||
|
||||
/** 保留期清理的终态判定:大小写不敏感,PENDING/RUNNING 一律返回 false。 */
|
||||
private static boolean isRetentionTerminal(String status) {
|
||||
return status != null && RETENTION_TERMINAL_STATUSES.contains(status.trim().toUpperCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
private List<Long> normalizeTaskIds(List<Long> taskIds) {
|
||||
if (taskIds == null) {
|
||||
return List.of();
|
||||
|
||||
Reference in New Issue
Block a user