super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-71: 清理日志改为数量与 sample ID，禁止输出超长任务 ID 列表

Browse Source
This commit is contained in:
黄自达
2026-08-30 19:53:49 +08:00
parent f48407ec7e
commit 6f32ed86e4
2 changed files with 324 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java
+27 -2
View File
@@ -44,6 +44,7 @@ public class ModuleHistoryCleanupService {
private static final Duration CLEANUP_LOCK_TTL = Duration.ofHours(2); private static final Duration CLEANUP_LOCK_TTL = Duration.ofHours(2);
private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA"; private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA";
private static final int DEFAULT_BATCH_SIZE = 500; private static final int DEFAULT_BATCH_SIZE = 500;
private static final int LOG_SAMPLE_IDS = 5;
private final ModuleCleanupProperties moduleCleanupProperties; private final ModuleCleanupProperties moduleCleanupProperties;
private final FileTaskMapper fileTaskMapper; private final FileTaskMapper fileTaskMapper;
@@ -166,7 +167,7 @@ public class ModuleHistoryCleanupService {
int deletedRows = deleteRows(types, taskIds, collectDataTaskIds); int deletedRows = deleteRows(types, taskIds, collectDataTaskIds);
submitAndFlush(collected); submitAndFlush(collected);
log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}", log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}",
taskIds, deletedRows, collected.size()); formatIdSample(taskIds, LOG_SAMPLE_IDS), deletedRows, collected.size());
}); });
totalBatches++; totalBatches++;
totalDeletedTasks += taskIds.size(); totalDeletedTasks += taskIds.size();
@@ -181,7 +182,8 @@ public class ModuleHistoryCleanupService {
log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, batches={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}", log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, batches={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}",
moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff, moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff,
totalBatches, totalDeletedTasks, totalCollectedPointers, totalBatches, totalDeletedTasks, totalCollectedPointers,
skippedActiveTaskIds, skippedRetainedTaskIds); formatIdSample(skippedActiveTaskIds, LOG_SAMPLE_IDS),
formatIdSample(skippedRetainedTaskIds, LOG_SAMPLE_IDS));
} }
} }
@@ -285,4 +287,27 @@ public class ModuleHistoryCleanupService {
pointers.add(pointer); pointers.add(pointer);
} }
} }
/**
* 日志用 ID 摘要：只输出数量与最多 {@code sampleLimit} 个样本，
* 禁止把超长任务 ID 列表写进日志。非法上限回退到 1，null/空输出 count=0。
*/
static String formatIdSample(List<Long> ids, int sampleLimit) {
int count = ids == null ? 0 : ids.size();
if (count == 0) {
return "count=0";
}
int limit = Math.max(1, sampleLimit);
StringBuilder sb = new StringBuilder("count=").append(count).append(", sample=[");
for (int i = 0; i < Math.min(count, limit); i++) {
if (i > 0) {
sb.append(',');
}
sb.append(ids.get(i));
}
if (count > limit) {
sb.append(",...");
}
return sb.append(']').toString();
}
} }
backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java
+297
View File
@@ -0,0 +1,297 @@
package com.nanri.aiimage.modules.task.service;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.ModuleCleanupProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentMatchers;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.slf4j.LoggerFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import java.nio.file.Path;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 71：清理日志改为数量与 sample ID，禁止输出超长任务 ID 列表。
* ModuleHistoryCleanupService 的日志只打印 count 与最多 5 个 sample id
* （超限追加省略号），不打印完整任务 ID 列表；formatIdSample 对空值、
* 单元素、超限与非法参数均有确定行为。
*/
class ModuleHistoryCleanupLoggingTest {
@TempDir
Path tempDir;
private static final LocalDateTime OLD_FINISHED_AT = LocalDateTime.of(2026, 1, 1, 0, 0);
private static final int LOG_SAMPLE_IDS = 5;
private ModuleCleanupProperties cleanupProperties;
private FileTaskMapper fileTaskMapper;
private FileResultMapper fileResultMapper;
private TaskFileJobMapper taskFileJobMapper;
private TaskResultItemMapper taskResultItemMapper;
private TaskProgressSnapshotMapper taskProgressSnapshotMapper;
private TaskResultPayloadMapper taskResultPayloadMapper;
private TaskScopeStateMapper taskScopeStateMapper;
private TaskChunkMapper taskChunkMapper;
private CollectDataItemMapper collectDataItemMapper;
private DistributedJobLockService lockService;
private TransientPayloadStorageService storage;
private RustfsObjectStorageService rustfs;
private OssStorageService oss;
private TransientPayloadDeleteOrchestrator orchestrator;
private ExecutorService executor;
private PlatformTransactionManager transactionManager;
private ModuleHistoryCleanupService cleanupService;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskFileJobEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskResultItemEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskProgressSnapshotEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskResultPayloadEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, CollectDataItemEntity.class);
}
@BeforeEach
@SuppressWarnings("unchecked")
void setUp() {
cleanupProperties = new ModuleCleanupProperties();
fileTaskMapper = mock(FileTaskMapper.class);
fileResultMapper = mock(FileResultMapper.class);
taskFileJobMapper = mock(TaskFileJobMapper.class);
taskResultItemMapper = mock(TaskResultItemMapper.class);
taskProgressSnapshotMapper = mock(TaskProgressSnapshotMapper.class);
taskResultPayloadMapper = mock(TaskResultPayloadMapper.class);
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
taskChunkMapper = mock(TaskChunkMapper.class);
collectDataItemMapper = mock(CollectDataItemMapper.class);
StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
ValueOperations<String, String> valueOperations = mock(ValueOperations.class);
when(redisTemplate.opsForValue()).thenReturn(valueOperations);
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(Boolean.TRUE);
lockService = new DistributedJobLockService(redisTemplate);
rustfs = mock(RustfsObjectStorageService.class);
oss = mock(OssStorageService.class);
TransientStorageProperties transientProperties = new TransientStorageProperties();
transientProperties.setEnabled(true);
StorageProperties storageProperties = new StorageProperties();
storageProperties.setLocalTempDir(tempDir.toString());
storage = new TransientPayloadStorageService(
transientProperties, storageProperties, rustfs, oss,
new ObjectMapper(), new InstanceMetadata("test-instance"),
taskChunkMapper, taskScopeStateMapper);
executor = Executors.newFixedThreadPool(2);
orchestrator = new TransientPayloadDeleteOrchestrator(
storage, rustfs, taskChunkMapper, taskScopeStateMapper, new ObjectMapper(), executor);
ReflectionTestUtils.setField(orchestrator, "maxPendingDeletes", 1000L);
transactionManager = mock(PlatformTransactionManager.class);
doAnswer(invocation -> mock(TransactionStatus.class))
.when(transactionManager).getTransaction(ArgumentMatchers.any(TransactionDefinition.class));
doAnswer(invocation -> null).when(transactionManager).commit(any(TransactionStatus.class));
doAnswer(invocation -> null).when(transactionManager).rollback(any(TransactionStatus.class));
cleanupService = new ModuleHistoryCleanupService(
cleanupProperties, fileTaskMapper, fileResultMapper, taskFileJobMapper,
taskResultItemMapper, taskProgressSnapshotMapper, taskResultPayloadMapper,
taskScopeStateMapper, taskChunkMapper, collectDataItemMapper, lockService, orchestrator,
transactionManager);
}
private static FileTaskEntity expiredTask(long id) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType("DEDUPE");
task.setStatus("SUCCESS");
task.setFinishedAt(OLD_FINISHED_AT);
return task;
}
private static <T> Answer<List<T>> sequence(List<T> rows, int rowCalls, List<T> fallback) {
return new Answer<List<T>>() {
private int calls;
@Override
public List<T> answer(InvocationOnMock invocation) {
return calls++ < rowCalls ? rows : fallback;
}
};
}
private ListAppender<ILoggingEvent> attachLogAppender() {
Logger logger = (Logger) LoggerFactory.getLogger(ModuleHistoryCleanupService.class);
ListAppender<ILoggingEvent> appender = new ListAppender<>();
appender.setContext(logger.getLoggerContext());
appender.start();
logger.addAppender(appender);
return appender;
}
private static void detachLogAppender(ListAppender<ILoggingEvent> appender) {
Logger logger = (Logger) LoggerFactory.getLogger(ModuleHistoryCleanupService.class);
logger.detachAppender(appender);
}
@Test
void test_task_071_cleanup_logging_normal_default_path() {
// 默认路径：正常列表完整输出，数量与 sample 齐全。
List<Long> ids = List.of(11L, 22L, 33L);
assertEquals("count=3, sample=[11,22,33]",
ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS));
}
@Test
void test_task_071_cleanup_logging_normal_multiple_items() {
// 批量场景：超过 sample 上限时输出截断列表与省略号，count 保持完整。
List<Long> ids = List.of(1L, 2L, 3L, 4L, 5L, 6L, 7L);
assertEquals("count=7, sample=[1,2,3,4,5,...]",
ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS));
}
@Test
void test_task_071_cleanup_logging_normal_repeated_operation_is_idempotent() {
// 幂等：同一输入重复格式化结果一致，无随机性。
List<Long> ids = List.of(9L, 8L, 7L, 6L, 5L, 4L);
String first = ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS);
String second = ModuleHistoryCleanupService.formatIdSample(ids, LOG_SAMPLE_IDS);
assertEquals(first, second);
assertEquals("count=6, sample=[9,8,7,6,5,...]", first);
}
@Test
void test_task_071_cleanup_logging_boundary_empty_input() {
// 空输入：null 与空列表输出 count=0，不输出列表。
assertEquals("count=0", ModuleHistoryCleanupService.formatIdSample(null, LOG_SAMPLE_IDS));
assertEquals("count=0", ModuleHistoryCleanupService.formatIdSample(List.of(), LOG_SAMPLE_IDS));
}
@Test
void test_task_071_cleanup_logging_boundary_single_item() {
// 单元素：单个 id 完整输出，不依赖批量路径。
assertEquals("count=1, sample=[42]",
ModuleHistoryCleanupService.formatIdSample(List.of(42L), LOG_SAMPLE_IDS));
}
@Test
void test_task_071_cleanup_logging_boundary_limit_and_overflow() {
// 上限/超限：sample 上限为 1 时只输出首个 id 加省略号，不输出完整列表。
List<Long> ids = List.of(1L, 2L, 3L, 4L, 5L);
assertEquals("count=5, sample=[1,...]",
ModuleHistoryCleanupService.formatIdSample(ids, 1));
assertEquals("count=5, sample=[1,...]",
ModuleHistoryCleanupService.formatIdSample(ids, 0), "非法上限回退到 1");
assertEquals("count=5, sample=[1,...]",
ModuleHistoryCleanupService.formatIdSample(ids, -3), "负上限回退到 1");
}
@Test
void test_task_071_cleanup_logging_invalid_input_rejected() {
// 非法参数：列表含 null 元素时不抛异常，输出稳定占位；count 保持实际元素数。
assertEquals("count=2, sample=[null,22]",
ModuleHistoryCleanupService.formatIdSample(new ArrayList<>(java.util.Arrays.asList(null, 22L)), LOG_SAMPLE_IDS));
}
@Test
void test_task_071_cleanup_logging_dependency_failure_releases_resources() {
// 依赖失败：服务日志不使用超长列表；依赖异常传播且日志中无完整 ID 列表。
ListAppender<ILoggingEvent> appender = attachLogAppender();
try {
cleanupProperties.setBatchSize(100);
List<Long> ids = new ArrayList<>();
for (long i = 1; i <= 60; i++) {
ids.add(i);
}
List<FileTaskEntity> tasks = new ArrayList<>();
for (long id : ids) {
tasks.add(expiredTask(id));
}
when(fileTaskMapper.selectList(any(LambdaQueryWrapper.class)))
.thenAnswer(sequence(tasks, 1, List.of()));
lenient().when(taskFileJobMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
lenient().when(taskResultItemMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
lenient().when(taskProgressSnapshotMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
lenient().when(taskResultPayloadMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
lenient().when(taskScopeStateMapper.delete(any(LambdaQueryWrapper.class))).thenReturn(0);
when(taskChunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
when(taskScopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
when(taskChunkMapper.delete(any(LambdaQueryWrapper.class)))
.thenThrow(new RuntimeException("db down"));
assertThrows(RuntimeException.class, () -> cleanupService.cleanupConfiguredModules());
for (ILoggingEvent event : appender.list) {
String message = event.getFormattedMessage();
assertTrue(!message.contains("taskIds=["), "日志不输出超长任务 ID 列表: " + message);
assertTrue(!message.contains("skippedActiveTaskIds=["), "日志不输出超长活跃 ID 列表: " + message);
assertTrue(!message.contains("retainedTaskIds=["), "日志不输出超长保留 ID 列表: " + message);
}
verify(taskChunkMapper).selectList(any(LambdaQueryWrapper.class));
} finally {
detachLogAppender(appender);
}
}
}