feat(保留期清理): 补齐 7 处只增不删的数据;修结果对象孤儿
审核发现一批表/元数据只增不删,且对象存储存在活跃泄漏: - module-cleanup 漏收 task_result_item / task_result_payload 的 payload 指针, 每晚删完行就在 json-server 桶留下孤儿对象;指针收集触顶从"截断后照样删整组" 改为任务组二分拆分,拆到单任务仍触顶则整组不删并记 error(截断会让指针随行消失) - 结果文件对象此前从不回收(只删 DB 行、对象留在桶里),现改为事务提交后 按 file_result / task_file_job 的 result_file_url 逐个回收 - 新增 7 个保留期清理:device_log_file 元数据、price_track_loop_run、撞款扫描行、 密钥用量日统计、紫鸟记忆过期行、PUBLISH 任务、BRAND 任务。后两者刻意不复用 module-cleanup(BRAND 不写 biz_file_task;PUBLISH 单任务可达数万行), 改为调各自既有的业务删除入口,保证结果对象与子表一起回收 - 未读通知补保留期(180 天,已读仍 90 天);ZiniaoMemoryStoreService.deleteExpired 此前全库无调用方(实现了没接线),现已挂定时任务 全部沿用既有范式:@Scheduled + 分布式锁单实例 + 分批 + 单轮批次数上限 + 失败只记日志。
This commit is contained in:
+117
@@ -0,0 +1,117 @@
|
||||
package com.nanri.aiimage.modules.brand.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.stream.LongStream;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 品牌检测任务保留期清理(brand_crawl_tasks 自建表、不写 biz_file_task,此前没有任何删除路径)。
|
||||
*/
|
||||
class BrandTaskRetentionServiceTest {
|
||||
|
||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper = mock(BrandCrawlTaskMapper.class);
|
||||
private final BrandTaskService brandTaskService = mock(BrandTaskService.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private BrandTaskRetentionService service() {
|
||||
return new BrandTaskRetentionService(brandCrawlTaskMapper, brandTaskService, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
private List<Long> ids(long from, int count) {
|
||||
return LongStream.range(from, from + count).boxed().toList();
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(brandCrawlTaskMapper.selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(ids(1, 50), ids(51, 50), ids(101, 7));
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(brandCrawlTaskMapper, times(3)).selectExpiredTerminalTaskIds(any(LocalDateTime.class), eq(50));
|
||||
// 逐条走既有删除入口(它已处理任务行 + 存储数据 + 进度缓存)
|
||||
verify(brandTaskService, times(107)).deleteTask(anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(brandCrawlTaskMapper.selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(ids(1, 50));
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(brandCrawlTaskMapper, times(BrandTaskRetentionService.MAX_BATCHES_PER_RUN))
|
||||
.selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(brandCrawlTaskMapper, never()).selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt());
|
||||
verify(brandTaskService, never()).deleteTask(anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void singleTaskFailureDoesNotAbortTheBatch() {
|
||||
// 任务锁被占、任务已在执行中等原因都会让单条删除抛异常,不能因此卡住整轮
|
||||
lockAvailable();
|
||||
when(brandCrawlTaskMapper.selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(List.of(1L, 2L, 3L));
|
||||
doThrow(new IllegalStateException("锁被占用")).when(brandTaskService).deleteTask(2L);
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(brandTaskService).deleteTask(1L);
|
||||
verify(brandTaskService).deleteTask(2L);
|
||||
verify(brandTaskService).deleteTask(3L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void wholeBatchFailureStopsEarlyToAvoidRepeatingSameBatch() {
|
||||
lockAvailable();
|
||||
when(brandCrawlTaskMapper.selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(ids(1, 50));
|
||||
doThrow(new IllegalStateException("db down")).when(brandTaskService).deleteTask(anyLong());
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(brandCrawlTaskMapper, times(1)).selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(brandCrawlTaskMapper.selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(brandCrawlTaskMapper).selectExpiredTerminalTaskIds(any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
}
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
package com.nanri.aiimage.modules.devicelog.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.DeviceLogOssProperties;
|
||||
import com.nanri.aiimage.modules.devicelog.mapper.DeviceLogFileMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.time.LocalDate;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 设备日志元数据保留期清理(device_log_file 此前只增不删,对象侧有生命周期而元数据无)。
|
||||
*/
|
||||
class DeviceLogRetentionServiceTest {
|
||||
|
||||
private final DeviceLogFileMapper fileMapper = mock(DeviceLogFileMapper.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private DeviceLogRetentionService service(Integer retentionDays) {
|
||||
DeviceLogOssProperties properties = new DeviceLogOssProperties();
|
||||
properties.setRetentionDays(retentionDays);
|
||||
return new DeviceLogRetentionService(fileMapper, properties, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(fileMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())).thenReturn(2000, 2000, 120);
|
||||
|
||||
service(7).purgeExpiredMetadata();
|
||||
|
||||
verify(fileMapper, times(3)).deleteOlderThanBatch(any(LocalDate.class), eq(2000));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(fileMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())).thenReturn(2000);
|
||||
|
||||
service(7).purgeExpiredMetadata();
|
||||
|
||||
verify(fileMapper, times(DeviceLogRetentionService.MAX_BATCHES_PER_RUN))
|
||||
.deleteOlderThanBatch(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service(7).purgeExpiredMetadata();
|
||||
|
||||
verify(fileMapper, never()).deleteOlderThanBatch(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(fileMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service(7).purgeExpiredMetadata();
|
||||
|
||||
verify(fileMapper).deleteOlderThanBatch(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
/**
|
||||
* 契约:cutoff 必须与查询侧可见窗口完全同一条线——page() 展示 log_date >= 今天-(days-1),
|
||||
* 所以这里只删 log_date < 今天-(days-1),即"早已看不见"的行。
|
||||
* 天数取自 device-log-oss.retention-days,与对象侧生命周期共用同一配置项。
|
||||
*/
|
||||
@Test
|
||||
void cutoffMatchesVisibleWindowBoundary() {
|
||||
lockAvailable();
|
||||
when(fileMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())).thenReturn(0);
|
||||
|
||||
service(7).purgeExpiredMetadata();
|
||||
|
||||
ArgumentCaptor<LocalDate> cutoff = ArgumentCaptor.forClass(LocalDate.class);
|
||||
verify(fileMapper).deleteOlderThanBatch(cutoff.capture(), anyInt());
|
||||
assertEquals(LocalDate.now().minusDays(6), cutoff.getValue(), "保留 7 天 → 删 7 天前的日期线");
|
||||
}
|
||||
|
||||
@Test
|
||||
void retentionDaysFallsBackToDefaultWhenUnset() {
|
||||
lockAvailable();
|
||||
when(fileMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())).thenReturn(0);
|
||||
|
||||
service(null).purgeExpiredMetadata();
|
||||
|
||||
ArgumentCaptor<LocalDate> cutoff = ArgumentCaptor.forClass(LocalDate.class);
|
||||
verify(fileMapper).deleteOlderThanBatch(cutoff.capture(), anyInt());
|
||||
assertEquals(LocalDate.now().minusDays(6), cutoff.getValue(), "未配置时回退默认 7 天");
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.pricetrack.mapper.PriceTrackLoopRunMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 跟价循环批次表保留期清理(该表此前有 insert 无 delete,属纯遗漏)。
|
||||
*/
|
||||
class PriceTrackLoopRunRetentionServiceTest {
|
||||
|
||||
private final PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private PriceTrackLoopRunRetentionService service() {
|
||||
return new PriceTrackLoopRunRetentionService(loopRunMapper, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(loopRunMapper.deleteFinishedBefore(any(LocalDateTime.class), anyInt())).thenReturn(500, 500, 30);
|
||||
|
||||
service().purgeExpiredLoopRuns();
|
||||
|
||||
verify(loopRunMapper, times(3)).deleteFinishedBefore(any(LocalDateTime.class), eq(500));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(loopRunMapper.deleteFinishedBefore(any(LocalDateTime.class), anyInt())).thenReturn(500);
|
||||
|
||||
service().purgeExpiredLoopRuns();
|
||||
|
||||
verify(loopRunMapper, times(PriceTrackLoopRunRetentionService.MAX_BATCHES_PER_RUN))
|
||||
.deleteFinishedBefore(any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service().purgeExpiredLoopRuns();
|
||||
|
||||
verify(loopRunMapper, never()).deleteFinishedBefore(any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(loopRunMapper.deleteFinishedBefore(any(LocalDateTime.class), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service().purgeExpiredLoopRuns();
|
||||
|
||||
verify(loopRunMapper).deleteFinishedBefore(any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
package com.nanri.aiimage.modules.publish.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 上架任务保留期清理(PUBLISH 此前完全没有清理,biz_publish_item 单任务可达数万行)。
|
||||
*/
|
||||
class PublishTaskRetentionServiceTest {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final PublishTaskService publishTaskService = mock(PublishTaskService.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private PublishTaskRetentionService service() {
|
||||
return new PublishTaskRetentionService(fileTaskMapper, publishTaskService, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
private List<Long> ids(long from, int count) {
|
||||
return java.util.stream.LongStream.range(from, from + count).boxed().toList();
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(fileTaskMapper.selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(ids(1, 50), ids(51, 50), ids(101, 7));
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(fileTaskMapper, times(3)).selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), eq(50));
|
||||
verify(publishTaskService, times(107)).deleteTaskForRetention(anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(fileTaskMapper.selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(ids(1, 50));
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(fileTaskMapper, times(PublishTaskRetentionService.MAX_BATCHES_PER_RUN))
|
||||
.selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(fileTaskMapper, never()).selectExpiredTerminalTaskIds(anyString(), any(LocalDateTime.class), anyInt());
|
||||
verify(publishTaskService, never()).deleteTaskForRetention(anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void singleTaskFailureDoesNotAbortTheBatch() {
|
||||
// 一个坏任务不能卡住整轮:失败的只计数,同批其余任务照常删除
|
||||
lockAvailable();
|
||||
when(fileTaskMapper.selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(List.of(1L, 2L, 3L));
|
||||
doThrow(new IllegalStateException("delete failed")).when(publishTaskService).deleteTaskForRetention(2L);
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(publishTaskService).deleteTaskForRetention(1L);
|
||||
verify(publishTaskService).deleteTaskForRetention(2L);
|
||||
verify(publishTaskService).deleteTaskForRetention(3L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void wholeBatchFailureStopsEarlyToAvoidRepeatingSameBatch() {
|
||||
// 整批全失败说明是环境性故障(不是单个坏任务),继续循环只会反复重试同一批
|
||||
lockAvailable();
|
||||
when(fileTaskMapper.selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt()))
|
||||
.thenReturn(ids(1, 50));
|
||||
doThrow(new IllegalStateException("db down")).when(publishTaskService).deleteTaskForRetention(anyLong());
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(fileTaskMapper, times(1)).selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(fileTaskMapper.selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service().purgeExpiredTasks();
|
||||
|
||||
verify(fileTaskMapper).selectExpiredTerminalTaskIds(eq("PUBLISH"), any(LocalDateTime.class), anyInt());
|
||||
}
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
package com.nanri.aiimage.modules.shopduplicatecheck.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
/**
|
||||
* 撞款扫描结果表保留期清理(读取侧只认最新一行,历史行此前无删除路径)。
|
||||
*/
|
||||
class ShopDataDuplicateScanRetentionServiceTest {
|
||||
|
||||
private final ShopDataDuplicateScanMapper scanMapper = mock(ShopDataDuplicateScanMapper.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private ShopDataDuplicateScanRetentionService service() {
|
||||
return new ShopDataDuplicateScanRetentionService(scanMapper, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(scanMapper.selectNthNewestId(anyInt())).thenReturn(900L);
|
||||
when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt()))
|
||||
.thenReturn(200, 200, 40);
|
||||
|
||||
service().purgeExpiredScans();
|
||||
|
||||
verify(scanMapper, times(3)).deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), eq(200));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(scanMapper.selectNthNewestId(anyInt())).thenReturn(900L);
|
||||
when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt())).thenReturn(200);
|
||||
|
||||
service().purgeExpiredScans();
|
||||
|
||||
verify(scanMapper, times(ShopDataDuplicateScanRetentionService.MAX_BATCHES_PER_RUN))
|
||||
.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service().purgeExpiredScans();
|
||||
|
||||
verify(scanMapper, never()).deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(scanMapper.selectNthNewestId(anyInt())).thenReturn(900L);
|
||||
when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service().purgeExpiredScans();
|
||||
|
||||
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt());
|
||||
}
|
||||
|
||||
/** 行数不足保护量时不设保护线(Long.MAX_VALUE),此时只按时间线删,不会误伤最新几行。 */
|
||||
@Test
|
||||
void keepsAllRowsWhenFewerThanProtectedCount() {
|
||||
lockAvailable();
|
||||
when(scanMapper.selectNthNewestId(anyInt())).thenReturn(null);
|
||||
when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt())).thenReturn(0);
|
||||
|
||||
service().purgeExpiredScans();
|
||||
|
||||
ArgumentCaptor<Long> protectLine = ArgumentCaptor.forClass(Long.class);
|
||||
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
|
||||
assertTrue(protectLine.getValue() == Long.MAX_VALUE, "保护线为 Long.MAX_VALUE,等于不额外设限");
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -167,7 +167,7 @@ class ModuleHistoryCleanupKeysetPaginationTest {
|
||||
cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper,
|
||||
taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator,
|
||||
transactionManager);
|
||||
oss, transactionManager);
|
||||
}
|
||||
|
||||
private static FileTaskEntity expiredTask(long id) {
|
||||
|
||||
+1
-1
@@ -158,7 +158,7 @@ class ModuleHistoryCleanupLoggingTest {
|
||||
cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper,
|
||||
taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator,
|
||||
transactionManager);
|
||||
oss, transactionManager);
|
||||
}
|
||||
|
||||
private static FileTaskEntity expiredTask(long id) {
|
||||
|
||||
+123
-5
@@ -56,6 +56,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -148,7 +149,7 @@ class ModuleHistoryCleanupPayloadCleanupTest {
|
||||
cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper,
|
||||
taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator,
|
||||
transactionManager);
|
||||
oss, transactionManager);
|
||||
}
|
||||
|
||||
private static FileTaskEntity expiredTask(long id) {
|
||||
@@ -225,6 +226,38 @@ class ModuleHistoryCleanupPayloadCleanupTest {
|
||||
return state;
|
||||
}
|
||||
|
||||
private static TaskResultItemEntity resultItem(long taskId, String payloadJson) {
|
||||
TaskResultItemEntity item = new TaskResultItemEntity();
|
||||
item.setTaskId(taskId);
|
||||
item.setModuleType("DEDUPE");
|
||||
item.setPayloadJson(payloadJson);
|
||||
return item;
|
||||
}
|
||||
|
||||
private static TaskResultPayloadEntity resultPayload(long taskId, String payloadJson) {
|
||||
TaskResultPayloadEntity payload = new TaskResultPayloadEntity();
|
||||
payload.setTaskId(taskId);
|
||||
payload.setModuleType("DEDUPE");
|
||||
payload.setPayloadJson(payloadJson);
|
||||
return payload;
|
||||
}
|
||||
|
||||
private static FileResultEntity resultRow(long taskId, String resultFileUrl) {
|
||||
FileResultEntity row = new FileResultEntity();
|
||||
row.setTaskId(taskId);
|
||||
row.setModuleType("DEDUPE");
|
||||
row.setResultFileUrl(resultFileUrl);
|
||||
return row;
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity resultJob(long taskId, String resultFileUrl) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setTaskId(taskId);
|
||||
job.setModuleType("DEDUPE");
|
||||
job.setResultFileUrl(resultFileUrl);
|
||||
return job;
|
||||
}
|
||||
|
||||
private void stubDeletes() {
|
||||
when(taskFileJobMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
|
||||
when(taskResultItemMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
|
||||
@@ -355,7 +388,7 @@ class ModuleHistoryCleanupPayloadCleanupTest {
|
||||
@Test
|
||||
void test_task_069_payload_cleanup_boundary_limit_and_overflow() throws Exception {
|
||||
// 上限/超限:收集达到上限即停止,不发生无界收集;
|
||||
// flush 时引用检查失败则本批保留(保守),可下次重试。
|
||||
// 单任务即触顶、无法再拆分时整组不删——宁可留脏行待下轮重试,也不让指针随行消失造成孤儿对象。
|
||||
stubExpiredTasks(51);
|
||||
stubDeletes();
|
||||
ReflectionTestUtils.setField(cleanupService, "maxCollectPayloadsPerRun", 3);
|
||||
@@ -363,14 +396,36 @@ class ModuleHistoryCleanupPayloadCleanupTest {
|
||||
for (int i = 1; i <= 5; i++) {
|
||||
chunks.add(chunk(51, jsonPointer(50 + i)));
|
||||
}
|
||||
doAnswer(sequenceThenFail(chunks, 1)).when(taskChunkMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
doAnswer(sequence(chunks, 1, List.of())).when(taskChunkMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
cleanupService.cleanupConfiguredModules();
|
||||
|
||||
assertEquals(3, orchestrator.pendingCount(), "仅保留上限内的指针");
|
||||
assertEquals(0, orchestrator.pendingCount(), "触顶且无法拆分时不提交任何指针");
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
verify(taskChunkMapper).delete(any(LambdaQueryWrapper.class));
|
||||
verify(taskChunkMapper, never()).delete(any(LambdaQueryWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_payload_cleanup_overflow_splits_group_instead_of_truncating() throws Exception {
|
||||
// 触顶拆分:整组 4 个指针、上限 3 时不得"截断后照样把整组删掉"——
|
||||
// 那会让未收集到的指针随行一起消失,对象再无引用可查、永久孤儿化;
|
||||
// 正确行为是二分后分两批处理。两次 delete 即证明整组未被当作一批删除。
|
||||
stubExpiredTasks(81, 82);
|
||||
stubDeletes();
|
||||
ReflectionTestUtils.setField(cleanupService, "maxCollectPayloadsPerRun", 3);
|
||||
List<TaskChunkEntity> all = List.of(
|
||||
chunk(81, pointer(81)), chunk(81, pointer(82)),
|
||||
chunk(82, pointer(83)), chunk(82, pointer(84)));
|
||||
doAnswer(sequence(all, 1, List.of())).when(taskChunkMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
CountDownLatch done = latchOnDelete(1);
|
||||
|
||||
cleanupService.cleanupConfiguredModules();
|
||||
|
||||
verify(taskChunkMapper, times(2)).delete(any(LambdaQueryWrapper.class));
|
||||
assertEquals(1, done.getCount(), "拆分后的两批均未携带被截断的指针,无对象进入删除队列");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -392,6 +447,69 @@ class ModuleHistoryCleanupPayloadCleanupTest {
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_payload_cleanup_collects_result_item_and_result_payload_pointers() throws Exception {
|
||||
// 回归:result_item / result_payload 的 payloadJson 同样存放 transient 指针
|
||||
//(storeResultItemPayload / storeResultPayload),行删除前必须一并收集,
|
||||
// 否则这两类对象随行消失、永久孤儿化在桶里。
|
||||
stubExpiredTasks(91);
|
||||
stubDeletes();
|
||||
when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
doAnswer(sequence(List.of(resultItem(91, pointer(91))), 1, List.of()))
|
||||
.when(taskResultItemMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
doAnswer(sequence(List.of(resultPayload(91, jsonPointer(92))), 1, List.of()))
|
||||
.when(taskResultPayloadMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
|
||||
CountDownLatch done = latchOnDelete(2);
|
||||
cleanupService.cleanupConfiguredModules();
|
||||
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS), "result_item / result_payload 指针各自异步删除一次");
|
||||
verify(rustfs, times(2)).deleteObject(anyString());
|
||||
verify(rustfs).deleteObject("task-parsed/test/91/scope/latest.json");
|
||||
verify(rustfs).deleteObject("task-parsed/test/92/scope/latest.json");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_payload_cleanup_recycles_result_file_objects() throws Exception {
|
||||
// 结果对象回收:file_result / task_file_job 的 result_file_url 指向的对象随行删除一并回收。
|
||||
// 此前只删行、对象永久留在桶里(只能靠桶生命周期兜底)。
|
||||
// 两个来源指向同一个 key 时只删一次(去重)。
|
||||
stubExpiredTasks(101);
|
||||
stubDeletes();
|
||||
when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
when(fileResultMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenReturn(List.of(resultRow(101, "result/dedupe/101.xlsx")));
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenReturn(List.of(resultJob(101, "result/dedupe/101.xlsx")));
|
||||
|
||||
cleanupService.cleanupConfiguredModules();
|
||||
|
||||
verify(oss).deleteObject("result/dedupe/101.xlsx");
|
||||
verify(oss, times(1)).deleteObject(anyString());
|
||||
verify(fileTaskMapper).delete(any(LambdaQueryWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_payload_cleanup_result_object_failure_does_not_abort_batch() throws Exception {
|
||||
// 单个结果对象删不掉只记日志:后续对象照常回收,行照常删除,不影响下一批。
|
||||
stubExpiredTasks(102);
|
||||
stubDeletes();
|
||||
when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
when(fileResultMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(
|
||||
resultRow(102, "result/dedupe/102a.xlsx"),
|
||||
resultRow(102, "result/dedupe/102b.xlsx")));
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
doThrow(new IllegalStateException("oss down")).when(oss).deleteObject("result/dedupe/102a.xlsx");
|
||||
|
||||
cleanupService.cleanupConfiguredModules();
|
||||
|
||||
verify(oss).deleteObject("result/dedupe/102b.xlsx");
|
||||
verify(fileTaskMapper).delete(any(LambdaQueryWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_069_payload_cleanup_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:行删除抛异常时中止,指针不提交、不 flush、不删除对象;
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDate;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 密钥用量日统计表保留期清理(该表按「用户 × 模块 × 日」只增不删)。
|
||||
*/
|
||||
class UserSecretUsageRetentionServiceTest {
|
||||
|
||||
private final UserSecretUsageMapper usageMapper = mock(UserSecretUsageMapper.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private UserSecretUsageRetentionService service() {
|
||||
return new UserSecretUsageRetentionService(usageMapper, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(usageMapper.deleteBefore(any(LocalDate.class), anyInt())).thenReturn(2000, 2000, 300);
|
||||
|
||||
service().purgeExpiredUsage();
|
||||
|
||||
verify(usageMapper, times(3)).deleteBefore(any(LocalDate.class), eq(2000));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(usageMapper.deleteBefore(any(LocalDate.class), anyInt())).thenReturn(2000);
|
||||
|
||||
service().purgeExpiredUsage();
|
||||
|
||||
verify(usageMapper, times(UserSecretUsageRetentionService.MAX_BATCHES_PER_RUN))
|
||||
.deleteBefore(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service().purgeExpiredUsage();
|
||||
|
||||
verify(usageMapper, never()).deleteBefore(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(usageMapper.deleteBefore(any(LocalDate.class), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service().purgeExpiredUsage();
|
||||
|
||||
verify(usageMapper).deleteBefore(any(LocalDate.class), anyInt());
|
||||
}
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
package com.nanri.aiimage.modules.ziniao.memory.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 紫鸟记忆存储过期行清理(deleteExpired 此前无任何调用方,属"实现了没接线")。
|
||||
*/
|
||||
class ZiniaoMemoryExpiredCleanupServiceTest {
|
||||
|
||||
private final ZiniaoMemoryStoreService memoryStoreService = mock(ZiniaoMemoryStoreService.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private ZiniaoMemoryExpiredCleanupService service() {
|
||||
return new ZiniaoMemoryExpiredCleanupService(memoryStoreService, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(memoryStoreService.deleteExpired(anyInt())).thenReturn(500, 500, 20);
|
||||
|
||||
service().purgeExpired();
|
||||
|
||||
verify(memoryStoreService, times(3)).deleteExpired(eq(500));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(memoryStoreService.deleteExpired(anyInt())).thenReturn(500);
|
||||
|
||||
service().purgeExpired();
|
||||
|
||||
verify(memoryStoreService, times(ZiniaoMemoryExpiredCleanupService.MAX_BATCHES_PER_RUN))
|
||||
.deleteExpired(anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service().purgeExpired();
|
||||
|
||||
verify(memoryStoreService, never()).deleteExpired(anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(memoryStoreService.deleteExpired(anyInt())).thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service().purgeExpired();
|
||||
|
||||
verify(memoryStoreService).deleteExpired(anyInt());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user