feat(保留期清理): 补齐 7 处只增不删的数据;修结果对象孤儿
审核发现一批表/元数据只增不删,且对象存储存在活跃泄漏: - module-cleanup 漏收 task_result_item / task_result_payload 的 payload 指针, 每晚删完行就在 json-server 桶留下孤儿对象;指针收集触顶从"截断后照样删整组" 改为任务组二分拆分,拆到单任务仍触顶则整组不删并记 error(截断会让指针随行消失) - 结果文件对象此前从不回收(只删 DB 行、对象留在桶里),现改为事务提交后 按 file_result / task_file_job 的 result_file_url 逐个回收 - 新增 7 个保留期清理:device_log_file 元数据、price_track_loop_run、撞款扫描行、 密钥用量日统计、紫鸟记忆过期行、PUBLISH 任务、BRAND 任务。后两者刻意不复用 module-cleanup(BRAND 不写 biz_file_task;PUBLISH 单任务可达数万行), 改为调各自既有的业务删除入口,保证结果对象与子表一起回收 - 未读通知补保留期(180 天,已读仍 90 天);ZiniaoMemoryStoreService.deleteExpired 此前全库无调用方(实现了没接线),现已挂定时任务 全部沿用既有范式:@Scheduled + 分布式锁单实例 + 分批 + 单轮批次数上限 + 失败只记日志。
This commit is contained in:
+17
@@ -2,8 +2,25 @@ package com.nanri.aiimage.modules.devicelog.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogFileEntity;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.time.LocalDate;
|
||||
|
||||
@Mapper
|
||||
public interface DeviceLogFileMapper extends BaseMapper<DeviceLogFileEntity> {
|
||||
|
||||
/**
|
||||
* 按日志日期分批删除保留期外的元数据行。
|
||||
*
|
||||
* <p>只删 {@code log_date} 早于 cutoff 的行:这条线正好是查询侧可见窗口的边界,
|
||||
* 即"早已看不见、只剩占位"的行,删掉不影响任何读取路径。
|
||||
*/
|
||||
@Delete("""
|
||||
DELETE FROM device_log_file
|
||||
WHERE log_date < #{cutoff}
|
||||
LIMIT #{batchSize}
|
||||
""")
|
||||
int deleteOlderThanBatch(@Param("cutoff") LocalDate cutoff, @Param("batchSize") int batchSize);
|
||||
}
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.nanri.aiimage.modules.devicelog.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.DeviceLogOssProperties;
|
||||
import com.nanri.aiimage.modules.devicelog.mapper.DeviceLogFileMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDate;
|
||||
|
||||
/**
|
||||
* 设备日志元数据的保留期清理。
|
||||
*
|
||||
* <p>对象侧一直有生命周期(主机B 独立 MinIO 桶,由运维 mc 定时任务按 7 天回收),
|
||||
* 但 {@code device_log_file} 的元数据行此前**只增不删**:客户端每 60 秒上报一轮,
|
||||
* 每台设备每天登记若干「来源+设备+文件名+日期」行,长期运行会无限累积。
|
||||
*
|
||||
* <p>查询侧本就只展示保留期内的行({@link DeviceLogService#page} 用同一个
|
||||
* {@code retentionDays} 过滤),这里按完全相同的边界删掉早已不可见的行。
|
||||
* 保留天数直接复用 {@code aiimage.device-log-oss.retention-days},
|
||||
* 与对象侧共用同一个配置项,避免两边各写一份天数后悄悄漂移。
|
||||
*
|
||||
* <p>对象本身仍由桶生命周期负责回收,本任务不碰对象:两条时间线按「对象修改时间」
|
||||
* 与「日志日期」衡量,可能有几天错位,但对象最终仍会被桶规则删除,不会永久残留。
|
||||
*
|
||||
* <p>双节点用 job 锁保证单实例执行;分批删除并限制单轮批次数,避免一次跑太久占住锁。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class DeviceLogRetentionService {
|
||||
|
||||
/** 单轮最多删除的批次数(每批 batchSize 行),剩余留给下一轮。 */
|
||||
static final int MAX_BATCHES_PER_RUN = 20;
|
||||
|
||||
private final DeviceLogFileMapper deviceLogFileMapper;
|
||||
private final DeviceLogOssProperties properties;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Value("${aiimage.device-log.retention-batch-size:2000}")
|
||||
private int retentionBatchSize = 2000;
|
||||
|
||||
@Scheduled(cron = "${aiimage.device-log.retention-cron:0 40 4 * * *}")
|
||||
public void purgeExpiredMetadata() {
|
||||
int days = properties.retentionDaysOrDefault();
|
||||
int batchSize = Math.max(100, retentionBatchSize);
|
||||
// 与查询侧可见窗口同一条边界:page() 只展示 log_date >= 今天-(days-1) 的行
|
||||
LocalDate cutoff = LocalDate.now().minusDays(days - 1L);
|
||||
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("device-log:metadata-retention", Duration.ofMinutes(15));
|
||||
if (lockHandle == null) {
|
||||
log.info("[device-log] 元数据保留清理跳过:另一实例持锁");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
int totalDeleted = 0;
|
||||
int batches = 0;
|
||||
while (batches < MAX_BATCHES_PER_RUN) {
|
||||
int deleted = deviceLogFileMapper.deleteOlderThanBatch(cutoff, batchSize);
|
||||
batches++;
|
||||
totalDeleted += deleted;
|
||||
if (deleted < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
log.info("[device-log] 元数据保留清理完成 cutoff={} retentionDays={} deleted={} batches={}",
|
||||
cutoff, days, totalDeleted, batches);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[device-log] 元数据保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user