feat(保留期清理): 补齐 7 处只增不删的数据;修结果对象孤儿

审核发现一批表/元数据只增不删,且对象存储存在活跃泄漏:

- module-cleanup 漏收 task_result_item / task_result_payload 的 payload 指针,
  每晚删完行就在 json-server 桶留下孤儿对象;指针收集触顶从"截断后照样删整组"
  改为任务组二分拆分,拆到单任务仍触顶则整组不删并记 error(截断会让指针随行消失)
- 结果文件对象此前从不回收(只删 DB 行、对象留在桶里),现改为事务提交后
  按 file_result / task_file_job 的 result_file_url 逐个回收
- 新增 7 个保留期清理:device_log_file 元数据、price_track_loop_run、撞款扫描行、
  密钥用量日统计、紫鸟记忆过期行、PUBLISH 任务、BRAND 任务。后两者刻意不复用
  module-cleanup(BRAND 不写 biz_file_task;PUBLISH 单任务可达数万行),
  改为调各自既有的业务删除入口,保证结果对象与子表一起回收
- 未读通知补保留期(180 天,已读仍 90 天);ZiniaoMemoryStoreService.deleteExpired
  此前全库无调用方(实现了没接线),现已挂定时任务

全部沿用既有范式:@Scheduled + 分布式锁单实例 + 分批 + 单轮批次数上限 + 失败只记日志。
This commit is contained in:
2026-09-16 01:59:28 +08:00
parent b7684dbbde
commit eca0ea0c18
29 changed files with 1680 additions and 32 deletions
@@ -167,7 +167,7 @@ class ModuleHistoryCleanupKeysetPaginationTest {
cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper,
taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper,
taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator,
transactionManager);
oss, transactionManager);
}
private static FileTaskEntity expiredTask(long id) {
@@ -158,7 +158,7 @@ class ModuleHistoryCleanupLoggingTest {
cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper,
taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper,
taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator,
transactionManager);
oss, transactionManager);
}
private static FileTaskEntity expiredTask(long id) {
@@ -56,6 +56,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -148,7 +149,7 @@ class ModuleHistoryCleanupPayloadCleanupTest {
cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper,
taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper,
taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator,
transactionManager);
oss, transactionManager);
}
private static FileTaskEntity expiredTask(long id) {
@@ -225,6 +226,38 @@ class ModuleHistoryCleanupPayloadCleanupTest {
return state;
}
private static TaskResultItemEntity resultItem(long taskId, String payloadJson) {
TaskResultItemEntity item = new TaskResultItemEntity();
item.setTaskId(taskId);
item.setModuleType("DEDUPE");
item.setPayloadJson(payloadJson);
return item;
}
private static TaskResultPayloadEntity resultPayload(long taskId, String payloadJson) {
TaskResultPayloadEntity payload = new TaskResultPayloadEntity();
payload.setTaskId(taskId);
payload.setModuleType("DEDUPE");
payload.setPayloadJson(payloadJson);
return payload;
}
private static FileResultEntity resultRow(long taskId, String resultFileUrl) {
FileResultEntity row = new FileResultEntity();
row.setTaskId(taskId);
row.setModuleType("DEDUPE");
row.setResultFileUrl(resultFileUrl);
return row;
}
private static TaskFileJobEntity resultJob(long taskId, String resultFileUrl) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setTaskId(taskId);
job.setModuleType("DEDUPE");
job.setResultFileUrl(resultFileUrl);
return job;
}
private void stubDeletes() {
when(taskFileJobMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
when(taskResultItemMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
@@ -355,7 +388,7 @@ class ModuleHistoryCleanupPayloadCleanupTest {
@Test
void test_task_069_payload_cleanup_boundary_limit_and_overflow() throws Exception {
// 上限/超限:收集达到上限即停止,不发生无界收集;
// flush 时引用检查失败则本批保留(保守),可下次重试。
// 单任务即触顶、无法再拆分时整组不删——宁可留脏行待下轮重试,也不让指针随行消失造成孤儿对象。
stubExpiredTasks(51);
stubDeletes();
ReflectionTestUtils.setField(cleanupService, "maxCollectPayloadsPerRun", 3);
@@ -363,14 +396,36 @@ class ModuleHistoryCleanupPayloadCleanupTest {
for (int i = 1; i <= 5; i++) {
chunks.add(chunk(51, jsonPointer(50 + i)));
}
doAnswer(sequenceThenFail(chunks, 1)).when(taskChunkMapper).selectList(any(LambdaQueryWrapper.class));
doAnswer(sequence(chunks, 1, List.of())).when(taskChunkMapper).selectList(any(LambdaQueryWrapper.class));
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
cleanupService.cleanupConfiguredModules();
assertEquals(3, orchestrator.pendingCount(), "仅保留上限内的指针");
assertEquals(0, orchestrator.pendingCount(), "触顶且无法拆分时不提交任何指针");
verify(rustfs, never()).deleteObject(anyString());
verify(taskChunkMapper).delete(any(LambdaQueryWrapper.class));
verify(taskChunkMapper, never()).delete(any(LambdaQueryWrapper.class));
}
@Test
void test_payload_cleanup_overflow_splits_group_instead_of_truncating() throws Exception {
// 触顶拆分:整组 4 个指针、上限 3 时不得"截断后照样把整组删掉"——
// 那会让未收集到的指针随行一起消失,对象再无引用可查、永久孤儿化;
// 正确行为是二分后分两批处理。两次 delete 即证明整组未被当作一批删除。
stubExpiredTasks(81, 82);
stubDeletes();
ReflectionTestUtils.setField(cleanupService, "maxCollectPayloadsPerRun", 3);
List<TaskChunkEntity> all = List.of(
chunk(81, pointer(81)), chunk(81, pointer(82)),
chunk(82, pointer(83)), chunk(82, pointer(84)));
doAnswer(sequence(all, 1, List.of())).when(taskChunkMapper).selectList(any(LambdaQueryWrapper.class));
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
CountDownLatch done = latchOnDelete(1);
cleanupService.cleanupConfiguredModules();
verify(taskChunkMapper, times(2)).delete(any(LambdaQueryWrapper.class));
assertEquals(1, done.getCount(), "拆分后的两批均未携带被截断的指针,无对象进入删除队列");
}
@Test
@@ -392,6 +447,69 @@ class ModuleHistoryCleanupPayloadCleanupTest {
verify(rustfs, never()).deleteObject(anyString());
}
@Test
void test_payload_cleanup_collects_result_item_and_result_payload_pointers() throws Exception {
// 回归:result_item / result_payload 的 payloadJson 同样存放 transient 指针
//(storeResultItemPayload / storeResultPayload),行删除前必须一并收集,
// 否则这两类对象随行消失、永久孤儿化在桶里。
stubExpiredTasks(91);
stubDeletes();
when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
doAnswer(sequence(List.of(resultItem(91, pointer(91))), 1, List.of()))
.when(taskResultItemMapper).selectList(any(LambdaQueryWrapper.class));
doAnswer(sequence(List.of(resultPayload(91, jsonPointer(92))), 1, List.of()))
.when(taskResultPayloadMapper).selectList(any(LambdaQueryWrapper.class));
CountDownLatch done = latchOnDelete(2);
cleanupService.cleanupConfiguredModules();
assertTrue(done.await(2, TimeUnit.SECONDS), "result_item / result_payload 指针各自异步删除一次");
verify(rustfs, times(2)).deleteObject(anyString());
verify(rustfs).deleteObject("task-parsed/test/91/scope/latest.json");
verify(rustfs).deleteObject("task-parsed/test/92/scope/latest.json");
}
@Test
void test_payload_cleanup_recycles_result_file_objects() throws Exception {
// 结果对象回收:file_result / task_file_job 的 result_file_url 指向的对象随行删除一并回收。
// 此前只删行、对象永久留在桶里(只能靠桶生命周期兜底)。
// 两个来源指向同一个 key 时只删一次(去重)。
stubExpiredTasks(101);
stubDeletes();
when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
when(fileResultMapper.selectList(any(LambdaQueryWrapper.class)))
.thenReturn(List.of(resultRow(101, "result/dedupe/101.xlsx")));
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
.thenReturn(List.of(resultJob(101, "result/dedupe/101.xlsx")));
cleanupService.cleanupConfiguredModules();
verify(oss).deleteObject("result/dedupe/101.xlsx");
verify(oss, times(1)).deleteObject(anyString());
verify(fileTaskMapper).delete(any(LambdaQueryWrapper.class));
}
@Test
void test_payload_cleanup_result_object_failure_does_not_abort_batch() throws Exception {
// 单个结果对象删不掉只记日志:后续对象照常回收,行照常删除,不影响下一批。
stubExpiredTasks(102);
stubDeletes();
when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
when(fileResultMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(
resultRow(102, "result/dedupe/102a.xlsx"),
resultRow(102, "result/dedupe/102b.xlsx")));
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
doThrow(new IllegalStateException("oss down")).when(oss).deleteObject("result/dedupe/102a.xlsx");
cleanupService.cleanupConfiguredModules();
verify(oss).deleteObject("result/dedupe/102b.xlsx");
verify(fileTaskMapper).delete(any(LambdaQueryWrapper.class));
}
@Test
void test_task_069_payload_cleanup_dependency_failure_releases_resources() throws Exception {
// 依赖失败:行删除抛异常时中止,指针不提交、不 flush、不删除对象;