task-68: payload 引用删除改为批量引用检查与异步物理删除
- 新增 TransientPayloadDeleteOrchestrator：提交去重入队（maxPendingDeletes 上限）， flush 时一次性 IN 批量反查 biz_task_chunk / biz_task_scope_state， 未引用对象交由后台线程池异步物理删除（剥 rustfs: 前缀） - 幂等：重复提交只入队一次；保守：查询失败整批保留可重试，不误删 - 空值/非指针忽略，本地/OSS 指针不在批量删除范围（各有归属与清理路径）
This commit is contained in:
+221
@@ -0,0 +1,221 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 68：payload 引用删除改为批量引用检查与异步物理删除。
|
||||
* TransientPayloadDeleteOrchestrator 把待删 pointer 去重入队，flush 时一次性
|
||||
* IN 批量反查 biz_task_chunk / biz_task_scope_state（而非逐条两次查询），
|
||||
* 未引用的对象交由后台线程池异步物理删除；重复提交幂等、失败保守保留。
|
||||
*/
|
||||
class TransientPayloadDeleteOrchestratorTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
private TaskChunkMapper chunkMapper;
|
||||
private TaskScopeStateMapper scopeStateMapper;
|
||||
private RustfsObjectStorageService rustfs;
|
||||
private OssStorageService oss;
|
||||
private TransientPayloadStorageService storage;
|
||||
private TransientPayloadDeleteOrchestrator orchestrator;
|
||||
private ExecutorService executor;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
rustfs = mock(RustfsObjectStorageService.class);
|
||||
oss = mock(OssStorageService.class);
|
||||
chunkMapper = mock(TaskChunkMapper.class);
|
||||
scopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||
TransientStorageProperties properties = new TransientStorageProperties();
|
||||
properties.setEnabled(true);
|
||||
StorageProperties storageProperties = new StorageProperties();
|
||||
storageProperties.setLocalTempDir(tempDir.toString());
|
||||
storage = new TransientPayloadStorageService(
|
||||
properties, storageProperties, rustfs, oss,
|
||||
new ObjectMapper(), new InstanceMetadata("test-instance"),
|
||||
chunkMapper, scopeStateMapper);
|
||||
executor = Executors.newFixedThreadPool(2);
|
||||
orchestrator = new TransientPayloadDeleteOrchestrator(
|
||||
storage, rustfs, chunkMapper, scopeStateMapper, new ObjectMapper(), executor);
|
||||
ReflectionTestUtils.setField(orchestrator, "maxPendingDeletes", 100L);
|
||||
}
|
||||
|
||||
private static String pointer(int taskId) {
|
||||
return "rustfs:task-parsed/test/" + taskId + "/scope/latest.json";
|
||||
}
|
||||
|
||||
private void stubNoReferences() {
|
||||
when(chunkMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
when(scopeStateMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
}
|
||||
|
||||
private CountDownLatch latchOnDelete(int count) {
|
||||
CountDownLatch latch = new CountDownLatch(count);
|
||||
doAnswer(invocation -> {
|
||||
latch.countDown();
|
||||
return null;
|
||||
}).when(rustfs).deleteObject(anyString());
|
||||
return latch;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_normal_default_path() throws Exception {
|
||||
// 默认路径：批量提交未引用 pointer，flush 后全部异步物理删除。
|
||||
stubNoReferences();
|
||||
int accepted = orchestrator.submitDeletes(List.of(pointer(1), pointer(2)));
|
||||
assertEquals(2, accepted);
|
||||
|
||||
CountDownLatch done = latchOnDelete(2);
|
||||
assertEquals(2, orchestrator.flushPendingDeletes(), "两个对象提交物理删除");
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS), "异步删除完成");
|
||||
verify(rustfs, times(2)).deleteObject(anyString());
|
||||
verify(chunkMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
verify(scopeStateMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_normal_multiple_items() throws Exception {
|
||||
// 批量场景：5 个对象批量检查，1 个仍被 scope_state 引用则跳过，其余 4 个删除。
|
||||
stubNoReferences();
|
||||
TaskScopeStateEntity referenced = new TaskScopeStateEntity();
|
||||
referenced.setParsedPayloadJson("\"rustfs:task-parsed/test/3/scope/latest.json\"");
|
||||
when(scopeStateMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenReturn(List.of(referenced));
|
||||
|
||||
List<String> values = new ArrayList<>();
|
||||
for (int i = 1; i <= 5; i++) {
|
||||
values.add(pointer(i));
|
||||
}
|
||||
assertEquals(5, orchestrator.submitDeletes(values));
|
||||
CountDownLatch done = latchOnDelete(4);
|
||||
assertEquals(4, orchestrator.flushPendingDeletes(), "仅删除未引用对象");
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS));
|
||||
verify(rustfs, times(4)).deleteObject(anyString());
|
||||
verify(rustfs, never()).deleteObject("task-parsed/test/3/scope/latest.json");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 幂等：重复提交同一对象只入队一次、只删除一次，不产生重复物理删除请求。
|
||||
stubNoReferences();
|
||||
assertEquals(1, orchestrator.submitDeletes(List.of(pointer(1))));
|
||||
assertEquals(0, orchestrator.submitDeletes(List.of(pointer(1))), "重复提交被去重");
|
||||
|
||||
CountDownLatch done = latchOnDelete(1);
|
||||
assertEquals(1, orchestrator.flushPendingDeletes());
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS));
|
||||
verify(rustfs, times(1)).deleteObject(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_boundary_empty_input() throws Exception {
|
||||
// 空输入：空集合/含空值集合不入队、不查询、不删除。
|
||||
assertEquals(0, orchestrator.submitDeletes(List.of()));
|
||||
assertEquals(0, orchestrator.submitDeletes(null));
|
||||
assertEquals(0, orchestrator.submitDeletes(java.util.Arrays.asList("", " ", null)));
|
||||
|
||||
assertEquals(0, orchestrator.flushPendingDeletes());
|
||||
verify(chunkMapper, never()).selectList(any());
|
||||
verify(scopeStateMapper, never()).selectList(any());
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_boundary_single_item() throws Exception {
|
||||
// 单元素：单个对象不依赖批量路径，检查一次删除一次。
|
||||
stubNoReferences();
|
||||
assertEquals(1, orchestrator.submitDeletes(List.of(pointer(7))));
|
||||
CountDownLatch done = latchOnDelete(1);
|
||||
assertEquals(1, orchestrator.flushPendingDeletes());
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS));
|
||||
verify(rustfs).deleteObject("task-parsed/test/7/scope/latest.json");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_boundary_limit_and_overflow() throws Exception {
|
||||
// 上限/超限：pending 队列达到 maxPendingDeletes 后拒绝新提交，不发生无界内存增长。
|
||||
stubNoReferences();
|
||||
ReflectionTestUtils.setField(orchestrator, "maxPendingDeletes", 3L);
|
||||
List<String> values = List.of(pointer(1), pointer(2), pointer(3), pointer(4), pointer(5));
|
||||
assertEquals(3, orchestrator.submitDeletes(values), "超过上限的提交被拒绝");
|
||||
|
||||
CountDownLatch done = latchOnDelete(3);
|
||||
assertEquals(3, orchestrator.flushPendingDeletes());
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS));
|
||||
verify(rustfs, times(3)).deleteObject(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_invalid_input_rejected() {
|
||||
// 非法参数：非指针值不入队，不发起引用检查。
|
||||
stubNoReferences();
|
||||
assertEquals(0, orchestrator.submitDeletes(List.of("not-a-pointer", "plain text")));
|
||||
assertEquals(0, orchestrator.flushPendingDeletes());
|
||||
verify(chunkMapper, never()).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_068_payload_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败：引用检查查询异常时保守不删、pending 保留可重试；恢复后删除成功。
|
||||
when(chunkMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenThrow(new RuntimeException("db down"));
|
||||
assertEquals(1, orchestrator.submitDeletes(List.of(pointer(1))));
|
||||
|
||||
assertEquals(0, orchestrator.flushPendingDeletes(), "查询失败保守不删");
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
|
||||
stubNoReferences();
|
||||
CountDownLatch done = latchOnDelete(1);
|
||||
assertEquals(1, orchestrator.flushPendingDeletes(), "恢复后重试成功");
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS));
|
||||
verify(rustfs).deleteObject("task-parsed/test/1/scope/latest.json");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user