feat(保留期清理): 补齐 7 处只增不删的数据;修结果对象孤儿

审核发现一批表/元数据只增不删,且对象存储存在活跃泄漏:

- module-cleanup 漏收 task_result_item / task_result_payload 的 payload 指针,
  每晚删完行就在 json-server 桶留下孤儿对象;指针收集触顶从"截断后照样删整组"
  改为任务组二分拆分,拆到单任务仍触顶则整组不删并记 error(截断会让指针随行消失)
- 结果文件对象此前从不回收(只删 DB 行、对象留在桶里),现改为事务提交后
  按 file_result / task_file_job 的 result_file_url 逐个回收
- 新增 7 个保留期清理:device_log_file 元数据、price_track_loop_run、撞款扫描行、
  密钥用量日统计、紫鸟记忆过期行、PUBLISH 任务、BRAND 任务。后两者刻意不复用
  module-cleanup(BRAND 不写 biz_file_task;PUBLISH 单任务可达数万行),
  改为调各自既有的业务删除入口,保证结果对象与子表一起回收
- 未读通知补保留期(180 天,已读仍 90 天);ZiniaoMemoryStoreService.deleteExpired
  此前全库无调用方(实现了没接线),现已挂定时任务

全部沿用既有范式:@Scheduled + 分布式锁单实例 + 分批 + 单轮批次数上限 + 失败只记日志。
This commit is contained in:
2026-09-16 01:59:28 +08:00
parent b7684dbbde
commit eca0ea0c18
29 changed files with 1680 additions and 32 deletions
@@ -2,8 +2,26 @@ package com.nanri.aiimage.modules.pricetrack.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackLoopRunEntity;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.time.LocalDateTime;
@Mapper
public interface PriceTrackLoopRunMapper extends BaseMapper<PriceTrackLoopRunEntity> {
}
/**
* 分批删除已结束且超过保留期的循环批次行。
*
* <p>只删终态行:RUNNING 的批次删了会让后续 dispatch / childFinished 找不到记录。
* 时间线用 COALESCE 兜底——终态行本应写 finished_at,历史行可能只有 updated_at。
*/
@Delete("""
DELETE FROM biz_price_track_loop_run
WHERE status IN ('SUCCESS', 'FAILED', 'STOPPED')
AND COALESCE(finished_at, updated_at, created_at) < #{cutoff}
LIMIT #{batchSize}
""")
int deleteFinishedBefore(@Param("cutoff") LocalDateTime cutoff, @Param("batchSize") int batchSize);
}
@@ -0,0 +1,70 @@
package com.nanri.aiimage.modules.pricetrack.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.pricetrack.mapper.PriceTrackLoopRunMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
/**
* 跟价循环批次表(biz_price_track_loop_run)的保留期清理。
*
* <p>每次「循环跟价」插一行,且只在请求停止时更新——全模块此前没有任何删除路径(纯遗漏)。
* 读取侧只有两个入口:按 id 取单条(用户当下正在看的那一轮)与「当前是否有活跃循环」,
* 没有任何历史列表查询,所以终态批次过了保留期即可安全删除。
*
* <p>只删终态(SUCCESS/FAILED/STOPPED)行,RUNNING 一律不动——正在跑的循环被删会让
* 后续 dispatch / childFinished 找不到记录而中断。
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class PriceTrackLoopRunRetentionService {
/** 单轮最多删除的批次数,剩余留给下一轮。 */
static final int MAX_BATCHES_PER_RUN = 20;
private final PriceTrackLoopRunMapper loopRunMapper;
private final DistributedJobLockService distributedJobLockService;
@Value("${aiimage.price-track.loop-run-retention-days:30}")
private int retentionDays = 30;
@Value("${aiimage.price-track.loop-run-retention-batch-size:500}")
private int retentionBatchSize = 500;
@Scheduled(cron = "${aiimage.price-track.loop-run-retention-cron:0 20 4 * * *}")
public void purgeExpiredLoopRuns() {
int days = Math.max(1, retentionDays);
int batchSize = Math.max(50, retentionBatchSize);
LocalDateTime cutoff = LocalDateTime.now().minusDays(days);
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("price-track:loop-run-retention", Duration.ofMinutes(15));
if (lockHandle == null) {
log.info("[price-track-loop] 批次保留清理跳过:另一实例持锁");
return;
}
try (lockHandle) {
int totalDeleted = 0;
int batches = 0;
while (batches < MAX_BATCHES_PER_RUN) {
int deleted = loopRunMapper.deleteFinishedBefore(cutoff, batchSize);
batches++;
totalDeleted += deleted;
if (deleted < batchSize) {
break;
}
}
log.info("[price-track-loop] 批次保留清理完成 cutoff={} retentionDays={} deleted={} batches={}",
cutoff, days, totalDeleted, batches);
} catch (Exception ex) {
log.warn("[price-track-loop] 批次保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
}
}
}