From 6f32ed86e45599484280f29d32f1a2a6be3896b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sun, 30 Aug 2026 19:53:49 +0800 Subject: [PATCH] =?UTF-8?q?task-71:=20=E6=B8=85=E7=90=86=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=95=B0=E9=87=8F=E4=B8=8E=20sample=20ID?= =?UTF-8?q?=EF=BC=8C=E7=A6=81=E6=AD=A2=E8=BE=93=E5=87=BA=E8=B6=85=E9=95=BF?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=20ID=20=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/ModuleHistoryCleanupService.java | 29 +- .../ModuleHistoryCleanupLoggingTest.java | 297 ++++++++++++++++++ 2 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java index 1645d156..a1ed1d01 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java @@ -44,6 +44,7 @@ public class ModuleHistoryCleanupService { private static final Duration CLEANUP_LOCK_TTL = Duration.ofHours(2); private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA"; private static final int DEFAULT_BATCH_SIZE = 500; + private static final int LOG_SAMPLE_IDS = 5; private final ModuleCleanupProperties moduleCleanupProperties; private final FileTaskMapper fileTaskMapper; @@ -166,7 +167,7 @@ public class ModuleHistoryCleanupService { int deletedRows = deleteRows(types, taskIds, collectDataTaskIds); submitAndFlush(collected); log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}", - taskIds, deletedRows, collected.size()); + formatIdSample(taskIds, LOG_SAMPLE_IDS), deletedRows, collected.size()); }); totalBatches++; totalDeletedTasks += taskIds.size(); @@ -181,7 +182,8 @@ public class ModuleHistoryCleanupService { log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, batches={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}", moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff, totalBatches, totalDeletedTasks, totalCollectedPointers, - skippedActiveTaskIds, skippedRetainedTaskIds); + formatIdSample(skippedActiveTaskIds, LOG_SAMPLE_IDS), + formatIdSample(skippedRetainedTaskIds, LOG_SAMPLE_IDS)); } } @@ -285,4 +287,27 @@ public class ModuleHistoryCleanupService { pointers.add(pointer); } } + + /** + * 日志用 ID 摘要：只输出数量与最多 {@code sampleLimit} 个样本， + * 禁止把超长任务 ID 列表写进日志。非法上限回退到 1，null/空输出 count=0。 + */ + static String formatIdSample(List ids, int sampleLimit) { + int count = ids == null ? 0 : ids.size(); + if (count == 0) { + return "count=0"; + } + int limit = Math.max(1, sampleLimit); + StringBuilder sb = new StringBuilder("count=").append(count).append(", sample=["); + for (int i = 0; i < Math.min(count, limit); i++) { + if (i > 0) { + sb.append(','); + } + sb.append(ids.get(i)); + } + if (count > limit) { + sb.append(",..."); + } + return sb.append(']').toString(); + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java new file mode 100644 index 00000000..0dce452b --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java @@ -0,0 +1,297 @@ +package com.nanri.aiimage.modules.task.service; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.ModuleCleanupProperties; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.config.TransientStorageProperties; +import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; +import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper; +import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper; +import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentMatchers; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.LoggerFactory; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionStatus; + +import java.nio.file.Path; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Task 71：清理日志改为数量与 sample ID，禁止输出超长任务 ID 列表。 + * ModuleHistoryCleanupService 的日志只打印 count 与最多 5 个 sample id + * （超限追加省略号），不打印完整任务 ID 列表；formatIdSample 对空值、 + * 单元素、超限与非法参数均有确定行为。 + */ +class ModuleHistoryCleanupLoggingTest { + + @TempDir + Path tempDir; + + private static final LocalDateTime OLD_FINISHED_AT = LocalDateTime.of(2026, 1, 1, 0, 0); + private static final int LOG_SAMPLE_IDS = 5; + + private ModuleCleanupProperties cleanupProperties; + private FileTaskMapper fileTaskMapper; + private FileResultMapper fileResultMapper; + private TaskFileJobMapper taskFileJobMapper; + private TaskResultItemMapper taskResultItemMapper; + private TaskProgressSnapshotMapper taskProgressSnapshotMapper; + private TaskResultPayloadMapper taskResultPayloadMapper; + private TaskScopeStateMapper taskScopeStateMapper; + private TaskChunkMapper taskChunkMapper; + private CollectDataItemMapper collectDataItemMapper; + private DistributedJobLockService lockService; + private TransientPayloadStorageService storage; + private RustfsObjectStorageService rustfs; + private OssStorageService oss; + private TransientPayloadDeleteOrchestrator orchestrator; + private ExecutorService executor; + private PlatformTransactionManager transactionManager; + private ModuleHistoryCleanupService cleanupService; + + @BeforeAll + static void initializeMybatisMetadata() { + MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), ""); + TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class); + TableInfoHelper.initTableInfo(assistant, FileResultEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskFileJobEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskResultItemEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskProgressSnapshotEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskResultPayloadEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class); + TableInfoHelper.initTableInfo(assistant, CollectDataItemEntity.class); + } + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + cleanupProperties = new ModuleCleanupProperties(); + fileTaskMapper = mock(FileTaskMapper.class); + fileResultMapper = mock(FileResultMapper.class); + taskFileJobMapper = mock(TaskFileJobMapper.class); + taskResultItemMapper = mock(TaskResultItemMapper.class); + taskProgressSnapshotMapper = mock(TaskProgressSnapshotMapper.class); + taskResultPayloadMapper = mock(TaskResultPayloadMapper.class); + taskScopeStateMapper = mock(TaskScopeStateMapper.class); + taskChunkMapper = mock(TaskChunkMapper.class); + collectDataItemMapper = mock(CollectDataItemMapper.class); + StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class); + ValueOperations valueOperations = mock(ValueOperations.class); + when(redisTemplate.opsForValue()).thenReturn(valueOperations); + when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(Boolean.TRUE); + lockService = new DistributedJobLockService(redisTemplate); + + rustfs = mock(RustfsObjectStorageService.class); + oss = mock(OssStorageService.class); + TransientStorageProperties transientProperties = new TransientStorageProperties(); + transientProperties.setEnabled(true); + StorageProperties storageProperties = new StorageProperties(); + storageProperties.setLocalTempDir(tempDir.toString()); + storage = new TransientPayloadStorageService( + transientProperties, storageProperties, rustfs, oss, + new ObjectMapper(), new InstanceMetadata("test-instance"), + taskChunkMapper, taskScopeStateMapper); + executor = Executors.newFixedThreadPool(2); + orchestrator = new TransientPayloadDeleteOrchestrator( + storage, rustfs, taskChunkMapper, taskScopeStateMapper, new ObjectMapper(), executor); + ReflectionTestUtils.setField(orchestrator, "maxPendingDeletes", 1000L); + transactionManager = mock(PlatformTransactionManager.class); + doAnswer(invocation -> mock(TransactionStatus.class)) + .when(transactionManager).getTransaction(ArgumentMatchers.any(TransactionDefinition.class)); + doAnswer(invocation -> null).when(transactionManager).commit(any(TransactionStatus.class)); + doAnswer(invocation -> null).when(transactionManager).rollback(any(TransactionStatus.class)); + cleanupService = new ModuleHistoryCleanupService( + cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper, + taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper, + taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator, + transactionManager); + } + + private static FileTaskEntity expiredTask(long id) { + FileTaskEntity task = new FileTaskEntity(); + task.setId(id); + task.setModuleType("DEDUPE"); + task.setStatus("SUCCESS"); + task.setFinishedAt(OLD_FINISHED_AT); + return task; + } + + private static Answer> sequence(List rows, int rowCalls, List fallback) { + return new Answer>() { + private int calls; + + @Override + public List answer(InvocationOnMock invocation) { + return calls++ < rowCalls ? rows : fallback; + } + }; + } + + private ListAppender attachLogAppender() { + Logger logger = (Logger) LoggerFactory.getLogger(ModuleHistoryCleanupService.class); + ListAppender appender = new ListAppender<>(); + appender.setContext(logger.getLoggerContext()); + appender.start(); + logger.addAppender(appender); + return appender; + } + + private static void detachLogAppender(ListAppender appender) { + Logger logger = (Logger) LoggerFactory.getLogger(ModuleHistoryCleanupService.class); + logger.detachAppender(appender); + } + + @Test + void test_task_071_cleanup_logging_normal_default_path() { + // 默认路径：正常列表完整输出，数量与 sample 齐全。 + List ids = List.of(11L, 22L, 33L); + assertEquals("count=3, sample=[11,22,33]", + ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS)); + } + + @Test + void test_task_071_cleanup_logging_normal_multiple_items() { + // 批量场景：超过 sample 上限时输出截断列表与省略号，count 保持完整。 + List ids = List.of(1L, 2L, 3L, 4L, 5L, 6L, 7L); + assertEquals("count=7, sample=[1,2,3,4,5,...]", + ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS)); + } + + @Test + void test_task_071_cleanup_logging_normal_repeated_operation_is_idempotent() { + // 幂等：同一输入重复格式化结果一致，无随机性。 + List ids = List.of(9L, 8L, 7L, 6L, 5L, 4L); + String first = ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS); + String second = ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS); + assertEquals(first, second); + assertEquals("count=6, sample=[9,8,7,6,5,...]", first); + } + + @Test + void test_task_071_cleanup_logging_boundary_empty_input() { + // 空输入：null 与空列表输出 count=0，不输出列表。 + assertEquals("count=0", ModuleHistoryCleanupService.formatIdSample(null, LOG_SAMPLE_IDS)); + assertEquals("count=0", ModuleHistoryCleanupService.formatIdSample(List.of(), LOG_SAMPLE_IDS)); + } + + @Test + void test_task_071_cleanup_logging_boundary_single_item() { + // 单元素：单个 id 完整输出，不依赖批量路径。 + assertEquals("count=1, sample=[42]", + ModuleHistoryCleanupService.formatIdSample(List.of(42L), LOG_SAMPLE_IDS)); + } + + @Test + void test_task_071_cleanup_logging_boundary_limit_and_overflow() { + // 上限/超限：sample 上限为 1 时只输出首个 id 加省略号，不输出完整列表。 + List ids = List.of(1L, 2L, 3L, 4L, 5L); + assertEquals("count=5, sample=[1,...]", + ModuleHistoryCleanupService.formatIdSample(ids, 1)); + assertEquals("count=5, sample=[1,...]", + ModuleHistoryCleanupService.formatIdSample(ids, 0), "非法上限回退到 1"); + assertEquals("count=5, sample=[1,...]", + ModuleHistoryCleanupService.formatIdSample(ids, -3), "负上限回退到 1"); + } + + @Test + void test_task_071_cleanup_logging_invalid_input_rejected() { + // 非法参数：列表含 null 元素时不抛异常，输出稳定占位；count 保持实际元素数。 + assertEquals("count=2, sample=[null,22]", + ModuleHistoryCleanupService.formatIdSample(new ArrayList<>(java.util.Arrays.asList(null, 22L)), LOG_SAMPLE_IDS)); + } + + @Test + void test_task_071_cleanup_logging_dependency_failure_releases_resources() { + // 依赖失败：服务日志不使用超长列表；依赖异常传播且日志中无完整 ID 列表。 + ListAppender appender = attachLogAppender(); + try { + cleanupProperties.setBatchSize(100); + List ids = new ArrayList<>(); + for (long i = 1; i <= 60; i++) { + ids.add(i); + } + List tasks = new ArrayList<>(); + for (long id : ids) { + tasks.add(expiredTask(id)); + } + when(fileTaskMapper.selectList(any(LambdaQueryWrapper.class))) + .thenAnswer(sequence(tasks, 1, List.of())); + lenient().when(taskFileJobMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0); + lenient().when(taskResultItemMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0); + lenient().when(taskProgressSnapshotMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0); + lenient().when(taskResultPayloadMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0); + lenient().when(taskScopeStateMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0); + when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of()); + when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of()); + + when(taskChunkMapper.delete(any(LambdaQueryWrapper.class))) + .thenThrow(new RuntimeException("db down")); + + assertThrows(RuntimeException.class, () -> cleanupService.cleanupConfiguredModules()); + + for (ILoggingEvent event : appender.list) { + String message = event.getFormattedMessage(); + assertTrue(!message.contains("taskIds=["), "日志不输出超长任务 ID 列表: " + message); + assertTrue(!message.contains("skippedActiveTaskIds=["), "日志不输出超长活跃 ID 列表: " + message); + assertTrue(!message.contains("retainedTaskIds=["), "日志不输出超长保留 ID 列表: " + message); + } + verify(taskChunkMapper).selectList(any(LambdaQueryWrapper.class)); + } finally { + detachLogAppender(appender); + } + } +}