完善任务存储、权限及货源查询流程
Build Backend JAR / build (push) Has been cancelled

This commit is contained in:
supernijia
2026-08-14 22:49:15 +08:00
parent 5b1ccad40e
commit 7a7f1dfa21
21 changed files with 2584 additions and 566 deletions
@@ -3,38 +3,148 @@ package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.config.TransientStorageProperties;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
class RustfsDeleteRetryServiceTest {
@Test
void retryPendingDeletesRefillsDeferredItemsBeforeQueueDrainsCompletely() {
void pendingUniqueKeysNeverExceedCapacityUnderConcurrentAdmission() throws Exception {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setDeleteRetryEnabled(true);
properties.setDeleteRetryQueueCapacity(2);
properties.setDeleteRetryBatchSize(1);
properties.setDeleteRetryQueueCapacity(3);
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
RustfsDeleteRetryService service = new RustfsDeleteRetryService(properties, rustfs);
ExecutorService executor = Executors.newFixedThreadPool(8);
CountDownLatch start = new CountDownLatch(1);
List<Future<?>> futures = new ArrayList<>();
try {
for (int i = 0; i < 20; i++) {
String objectKey = "object-" + i;
futures.add(executor.submit(() -> {
start.await();
service.enqueue(objectKey, new IllegalStateException("failed"));
return null;
}));
}
start.countDown();
for (Future<?> future : futures) {
future.get(5, TimeUnit.SECONDS);
}
} finally {
executor.shutdownNow();
}
assertEquals(3, service.pendingCount());
}
@Test
void failedKeyIsAttemptedOnlyOncePerScheduledInvocation() {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setDeleteRetryEnabled(true);
properties.setDeleteRetryQueueCapacity(10);
properties.setDeleteRetryBatchSize(10);
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
doAnswer(invocation -> {
String objectKey = invocation.getArgument(0);
if ("b".equals(objectKey)) {
throw new IllegalStateException("still failing");
}
return null;
}).when(rustfs).deleteObjectFromRetry(org.mockito.ArgumentMatchers.anyString());
throw new IllegalStateException("still failing");
}).when(rustfs).deleteObjectFromRetry(eq("a"), anyLong());
RustfsDeleteRetryService service = new RustfsDeleteRetryService(properties, rustfs);
service.enqueue("a", new IllegalStateException("failed-a"));
service.enqueue("b", new IllegalStateException("failed-b"));
service.enqueue("c", new IllegalStateException("failed-c"));
service.retryPendingDeletes();
service.retryPendingDeletes();
verify(rustfs).deleteObjectFromRetry(eq("a"), anyLong());
assertEquals(1, service.pendingCount());
service.retryPendingDeletes();
verify(rustfs).deleteObjectFromRetry("c");
verify(rustfs, times(2)).deleteObjectFromRetry(eq("a"), anyLong());
assertEquals(1, service.pendingCount());
}
@Test
void successfulRetryDoesNotRemoveConcurrentReenqueue() throws Exception {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setDeleteRetryEnabled(true);
properties.setDeleteRetryQueueCapacity(10);
properties.setDeleteRetryBatchSize(10);
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
CountDownLatch deleteStarted = new CountDownLatch(1);
CountDownLatch allowDeleteSuccess = new CountDownLatch(1);
doAnswer(invocation -> {
deleteStarted.countDown();
assertTrue(allowDeleteSuccess.await(5, TimeUnit.SECONDS));
return null;
}).when(rustfs).deleteObjectFromRetry(eq("a"), anyLong());
RustfsDeleteRetryService service = new RustfsDeleteRetryService(properties, rustfs);
service.enqueue("a", new IllegalStateException("first failure"));
ExecutorService executor = Executors.newSingleThreadExecutor();
try {
Future<?> retry = executor.submit(service::retryPendingDeletes);
assertTrue(deleteStarted.await(5, TimeUnit.SECONDS));
service.enqueue("a", new IllegalStateException("new failure while delete completes"));
allowDeleteSuccess.countDown();
retry.get(5, TimeUnit.SECONDS);
} finally {
allowDeleteSuccess.countDown();
executor.shutdownNow();
}
assertEquals(1, service.pendingCount());
service.retryPendingDeletes();
verify(rustfs, times(2)).deleteObjectFromRetry(eq("a"), anyLong());
assertEquals(0, service.pendingCount());
}
@Test
void batchDeadlineStopsBeforeNextItemAndPassesRemainingBudget() {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setDeleteRetryEnabled(true);
properties.setDeleteRetryQueueCapacity(10);
properties.setDeleteRetryBatchSize(10);
properties.setDeleteRetryBatchTimeoutSeconds(1);
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
AtomicLong receivedBudgetNanos = new AtomicLong();
doAnswer(invocation -> {
long budgetNanos = invocation.getArgument(1);
receivedBudgetNanos.set(budgetNanos);
long deadlineNanos = System.nanoTime() + budgetNanos;
while (System.nanoTime() < deadlineNanos) {
Thread.onSpinWait();
}
throw new IllegalStateException("timed out");
}).when(rustfs).deleteObjectFromRetry(eq("a"), anyLong());
RustfsDeleteRetryService service = new RustfsDeleteRetryService(properties, rustfs);
service.enqueue("a", new IllegalStateException("failed-a"));
service.enqueue("b", new IllegalStateException("failed-b"));
long startedAt = System.nanoTime();
service.retryPendingDeletes();
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
assertTrue(receivedBudgetNanos.get() > 0L);
assertTrue(receivedBudgetNanos.get() <= TimeUnit.SECONDS.toNanos(1));
assertTrue(elapsedMillis < 2000L, "batch should stop close to its configured deadline");
verify(rustfs, never()).deleteObjectFromRetry(eq("b"), anyLong());
assertEquals(2, service.pendingCount());
}
}
@@ -3,19 +3,34 @@ package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.config.TransientStorageProperties;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
import io.minio.StatObjectArgs;
import okhttp3.OkHttpClient;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.ObjectProvider;
import java.lang.reflect.Field;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.same;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -24,6 +39,9 @@ class RustfsObjectStorageServiceTest {
@Test
void httpClientUsesConfiguredConnectionPoolAndKeepsRetryEnabled() {
TransientStorageProperties properties = configuredProperties();
properties.setCallTimeoutSeconds(45);
properties.setDispatcherMaxRequests(12);
properties.setDispatcherMaxRequestsPerHost(9);
properties.setConnectionPoolMaxIdle(3);
properties.setConnectionPoolKeepAliveMillis(1500);
RustfsObjectStorageService service = new RustfsObjectStorageService(
@@ -33,6 +51,26 @@ class RustfsObjectStorageServiceTest {
assertTrue(client.retryOnConnectionFailure(), "RustFS OkHttp 应开启连接失败重试");
assertNotNull(client.connectionPool(), "RustFS OkHttp 应使用显式 connectionPool 配置");
assertEquals(TimeUnit.SECONDS.toMillis(45), client.callTimeoutMillis());
assertEquals(12, client.dispatcher().getMaxRequests());
assertEquals(9, client.dispatcher().getMaxRequestsPerHost());
}
@Test
void deadlineClientViewShortensCallTimeoutAndSharesResources() {
TransientStorageProperties properties = configuredProperties();
properties.setCallTimeoutSeconds(45);
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), null);
OkHttpClient shared = service.getHttpClient();
OkHttpClient bounded = service.getHttpClient(
System.nanoTime() + TimeUnit.SECONDS.toNanos(2));
assertTrue(bounded.callTimeoutMillis() > 0);
assertTrue(bounded.callTimeoutMillis() <= 2000);
assertTrue(shared.connectionPool() == bounded.connectionPool());
assertTrue(shared.dispatcher() == bounded.dispatcher());
}
@Test
@@ -106,6 +144,241 @@ class RustfsObjectStorageServiceTest {
assertTrue(rejected.getMessage().contains("cooldown active"));
}
@Test
void retryBackoffDoesNotHoldUploadPermit() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setMaxConcurrentUploads(1);
properties.setAcquirePermitTimeoutMillis(300);
properties.setUploadMaxRetries(2);
properties.setBaseRetryDelayMillis(1000);
properties.setMaxRetryDelayMillis(1000);
MinioClient client = mock(MinioClient.class);
CountDownLatch firstAttemptFailed = new CountDownLatch(1);
AtomicInteger calls = new AtomicInteger();
doAnswer(invocation -> {
if (calls.incrementAndGet() == 1) {
firstAttemptFailed.countDown();
throw new IllegalStateException("first attempt failed");
}
return null;
}).when(client).putObject(any(PutObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> client);
ExecutorService executor = Executors.newFixedThreadPool(2);
try {
Future<String> first = executor.submit(() -> service.uploadText("task/first.json", "{}", false));
assertTrue(firstAttemptFailed.await(2, TimeUnit.SECONDS));
Future<String> second = executor.submit(() -> service.uploadText("task/second.json", "{}", false));
assertEquals("task/second.json", second.get(800, TimeUnit.MILLISECONDS));
assertEquals("task/first.json", first.get(2, TimeUnit.SECONDS));
verify(client, times(3)).putObject(any(PutObjectArgs.class));
assertEquals(1, privateSemaphore(service, "uploadSemaphore").availablePermits());
} finally {
executor.shutdownNow();
}
}
@Test
void visibilityCheckDoesNotHoldUploadPermit() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setMaxConcurrentUploads(1);
properties.setAcquirePermitTimeoutMillis(300);
MinioClient client = mock(MinioClient.class);
CountDownLatch statStarted = new CountDownLatch(1);
CountDownLatch releaseStat = new CountDownLatch(1);
doAnswer(invocation -> {
statStarted.countDown();
assertTrue(releaseStat.await(2, TimeUnit.SECONDS));
return null;
}).when(client).statObject(any(StatObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> client);
ExecutorService executor = Executors.newFixedThreadPool(2);
try {
Future<String> first = executor.submit(() -> service.uploadText("task/first.json", "{}", true));
assertTrue(statStarted.await(2, TimeUnit.SECONDS));
Future<String> second = executor.submit(() -> service.uploadText("task/second.json", "{}", false));
assertEquals("task/second.json", second.get(800, TimeUnit.MILLISECONDS));
releaseStat.countDown();
assertEquals("task/first.json", first.get(2, TimeUnit.SECONDS));
assertEquals(1, privateSemaphore(service, "uploadSemaphore").availablePermits());
} finally {
releaseStat.countDown();
executor.shutdownNow();
}
}
@Test
void openedCircuitStopsRemainingOuterAttempts() {
TransientStorageProperties properties = configuredProperties();
properties.setUploadMaxRetries(3);
properties.setFailureWindowThreshold(1);
properties.setFailureCooldownMillis(10000);
AtomicInteger clientBuilds = new AtomicInteger();
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> {
clientBuilds.incrementAndGet();
throw new IllegalStateException("rustfs down");
});
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/a.json", "{}", false));
assertTrue(ex.getMessage().contains("cooldown active"));
assertEquals(1, clientBuilds.get());
}
@Test
void successfulDeleteDoesNotResetReadWriteFailureWindow() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setUploadMaxRetries(1);
properties.setFailureWindowThreshold(2);
properties.setFailureCooldownMillis(10000);
MinioClient client = mock(MinioClient.class);
doThrow(new IllegalStateException("upload failed"))
.when(client).putObject(any(PutObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> client);
assertThrows(IllegalStateException.class,
() -> service.uploadText("task/first.json", "{}", false));
service.deleteObject("task/old.json");
assertThrows(IllegalStateException.class,
() -> service.uploadText("task/second.json", "{}", false));
IllegalStateException rejected = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/third.json", "{}", false));
assertTrue(rejected.getMessage().contains("cooldown active"));
verify(client, times(2)).putObject(any(PutObjectArgs.class));
verify(client).removeObject(any(RemoveObjectArgs.class));
}
@Test
void failedDeleteDoesNotIncreaseReadWriteFailureWindow() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setUploadMaxRetries(1);
properties.setDeleteMaxRetries(1);
properties.setFailureWindowThreshold(2);
properties.setFailureCooldownMillis(10000);
MinioClient client = mock(MinioClient.class);
doThrow(new IllegalStateException("upload failed"))
.when(client).putObject(any(PutObjectArgs.class));
doThrow(new IllegalStateException("delete failed"))
.when(client).removeObject(any(RemoveObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> client);
assertThrows(IllegalStateException.class,
() -> service.uploadText("task/first.json", "{}", false));
assertThrows(IllegalStateException.class, () -> service.deleteObject("task/old.json"));
IllegalStateException opensCircuit = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/second.json", "{}", false));
assertTrue(opensCircuit.getMessage().contains("failed to upload"));
IllegalStateException rejected = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/third.json", "{}", false));
assertTrue(rejected.getMessage().contains("cooldown active"));
verify(client, times(2)).putObject(any(PutObjectArgs.class));
}
@Test
void repeatedStatFailuresOpenCircuitAndEnqueueOneCompensationPerObject() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setFailureWindowThreshold(5);
properties.setFailureCooldownMillis(10000);
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
MinioClient client = mock(MinioClient.class);
doThrow(new IllegalStateException("stat failed"))
.when(client).statObject(any(StatObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), provider(retryService), () -> client);
IllegalStateException firstFailure = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/a.json", "{}", true));
IllegalStateException secondFailure = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/b.json", "{}", true));
assertTrue(firstFailure.getMessage().contains("not visible"));
assertTrue(secondFailure.getMessage().contains("not visible"));
verify(client, times(5)).statObject(any(StatObjectArgs.class));
verify(retryService).enqueue(eq("task/a.json"), same(firstFailure));
verify(retryService).enqueue(eq("task/b.json"), same(secondFailure));
IllegalStateException rejected = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/c.json", "{}", false));
assertTrue(rejected.getMessage().contains("cooldown active"));
verify(client, times(2)).putObject(any(PutObjectArgs.class));
}
@Test
void retryDeleteBudgetCutsOffLongBackoff() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setDeleteMaxRetries(3);
properties.setBaseRetryDelayMillis(5000);
properties.setMaxRetryDelayMillis(5000);
MinioClient client = mock(MinioClient.class);
doThrow(new IllegalStateException("delete failed"))
.when(client).removeObject(any(RemoveObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> client);
long startedAt = System.nanoTime();
IllegalStateException failure = assertThrows(IllegalStateException.class,
() -> service.deleteObjectFromRetry("task/a.json", TimeUnit.MILLISECONDS.toNanos(200)));
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
assertTrue(failure.getMessage().contains("operation timeout"));
assertTrue(elapsedMillis < 1000, "operation budget should cut off the five-second backoff");
verify(client).removeObject(any(RemoveObjectArgs.class));
}
@Test
void configuredOperationBudgetCutsOffLongBackoff() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setOperationTimeoutSeconds(1);
properties.setUploadMaxRetries(3);
properties.setBaseRetryDelayMillis(5000);
properties.setMaxRetryDelayMillis(5000);
MinioClient client = mock(MinioClient.class);
doThrow(new IllegalStateException("upload failed"))
.when(client).putObject(any(PutObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> client);
long startedAt = System.nanoTime();
IllegalStateException failure = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/a.json", "{}", false));
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt);
assertTrue(failure.getMessage().contains("operation timeout"));
assertTrue(elapsedMillis < 2000, "configured budget should cut off the five-second backoff");
verify(client).putObject(any(PutObjectArgs.class));
}
@Test
void completedUploadIsEnqueuedWhenDeadlineExpiresAfterPut() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setOperationTimeoutSeconds(1);
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
MinioClient client = mock(MinioClient.class);
doAnswer(invocation -> {
Thread.sleep(1100);
return null;
}).when(client).putObject(any(PutObjectArgs.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), provider(retryService), () -> client);
IllegalStateException failure = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/a.json", "{}", false));
assertTrue(failure.getMessage().contains("operation timeout"));
verify(client).putObject(any(PutObjectArgs.class));
verify(retryService).enqueue(eq("task/a.json"), same(failure));
}
private static TransientStorageProperties configuredProperties() {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setEndpoint("http://127.0.0.1:9000");
@@ -1,8 +1,5 @@
package com.nanri.aiimage.modules.permission.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.mapper.PermissionMenuMapper;
@@ -17,11 +14,9 @@ import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUse
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import org.junit.jupiter.api.Test;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -43,7 +38,7 @@ class PermissionMenuServiceTest {
when(userMapper.selectById(7L)).thenReturn(new AdminUserEntity());
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(imageVideoPermission());
when(permissionMapper.selectCount(any())).thenReturn(1L);
when(permissionMapper.countByUserIdAndColumnId(any(), any())).thenReturn(1L);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
@@ -65,7 +60,7 @@ class PermissionMenuServiceTest {
when(userMapper.selectById(8L)).thenReturn(new AdminUserEntity());
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(imageVideoPermission());
when(permissionMapper.selectCount(any())).thenReturn(0L);
when(permissionMapper.countByUserIdAndColumnId(any(), any())).thenReturn(0L);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(75L));
@@ -83,7 +78,7 @@ class PermissionMenuServiceTest {
when(userMapper.selectById(7L)).thenReturn(new AdminUserEntity());
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission(), shopDataCrawlDataPermission());
when(permissionMapper.selectCount(any())).thenReturn(1L);
when(permissionMapper.countByUserIdAndColumnId(any(), any())).thenReturn(1L);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
@@ -104,7 +99,7 @@ class PermissionMenuServiceTest {
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(permissionMapper.selectByUserId(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11),
@@ -126,7 +121,7 @@ class PermissionMenuServiceTest {
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(permissionMapper.selectByUserId(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11)));
@@ -144,7 +139,7 @@ class PermissionMenuServiceTest {
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 2L)));
when(permissionMapper.selectByUserId(any())).thenReturn(List.of(grant(9L, 2L)));
PermissionMenuEntity root = menu(1L, null, "app", 1);
root.setColumnKey("brand_front_tools");
PermissionMenuEntity leaf = menu(2L, 1L, "app", 2);
@@ -184,7 +179,7 @@ class PermissionMenuServiceTest {
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(1L)).thenReturn(user(1L, "super_admin", 1));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 2L)));
when(permissionMapper.selectByUserId(any())).thenReturn(List.of(grant(1L, 2L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
@@ -209,7 +204,7 @@ class PermissionMenuServiceTest {
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(1L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId).containsExactly(1L, 2L);
verify(permissionMapper, never()).selectList(any());
verify(permissionMapper, never()).selectByUserId(any());
}
@Test
@@ -221,7 +216,7 @@ class PermissionMenuServiceTest {
AdminUserEntity currentUser = user(9L, "normal", 0);
when(userMapper.selectById(9L)).thenReturn(currentUser);
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(permissionMapper.selectByUserId(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11)));
@@ -245,7 +240,7 @@ class PermissionMenuServiceTest {
assertThatThrownBy(() -> service.getUserColumnPermissions(currentUser, 10L, "app"))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("管理员权限");
verify(permissionMapper, never()).selectList(any());
verify(permissionMapper, never()).selectByUserId(any());
}
@Test
@@ -327,7 +322,7 @@ class PermissionMenuServiceTest {
.isInstanceOf(BusinessException.class)
.hasMessageContaining("子菜单");
verify(menuMapper, never()).deleteById(any(Long.class));
verify(permissionMapper, never()).delete(any());
verify(permissionMapper, never()).deleteByColumnId(any());
}
@Test
@@ -346,7 +341,7 @@ class PermissionMenuServiceTest {
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectList(any())).thenReturn(List.of(menu(1L, null, "app", 1)));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L)));
when(permissionMapper.selectByUserId(any())).thenReturn(List.of(grant(10L, 1L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
@@ -354,7 +349,7 @@ class PermissionMenuServiceTest {
assertThatThrownBy(() -> service.updateUserColumnPermissions(operator, 20L, request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("自己已有");
verify(permissionMapper, never()).delete(any());
verify(permissionMapper, never()).deleteByUserId(any());
}
@Test
@@ -374,7 +369,7 @@ class PermissionMenuServiceTest {
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L)));
when(permissionMapper.selectByUserId(any())).thenReturn(List.of(grant(10L, 1L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
@@ -401,7 +396,7 @@ class PermissionMenuServiceTest {
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, null, "app", 2)));
when(permissionMapper.selectList(any()))
when(permissionMapper.selectByUserId(any()))
.thenReturn(List.of(grant(10L, 1L)), List.of(grant(20L, 2L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(1L));
@@ -454,15 +449,7 @@ class PermissionMenuServiceTest {
service.updateUserColumnPermissions(operator, 9L, request, PermissionMenuService.MENU_TYPE_APP);
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
UserColumnPermissionEntity.class);
ArgumentCaptor<LambdaUpdateWrapper<UserColumnPermissionEntity>> deleted =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(permissionMapper).delete(deleted.capture());
assertThat(deleted.getValue().getSqlSegment()).contains("column_id", "IN");
assertThat(deleted.getValue().getParamNameValuePairs().values())
.contains(9L, 11L, 12L);
verify(permissionMapper).deleteByUserIdAndColumnIds(9L, List.of(11L, 12L));
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(inserted.capture());
@@ -484,7 +471,7 @@ class PermissionMenuServiceTest {
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
verify(menuMapper, never()).selectOne(any());
verify(permissionMapper, never()).delete(any());
verify(permissionMapper, never()).deleteByColumnId(any());
}
@Test
@@ -513,7 +500,7 @@ class PermissionMenuServiceTest {
normal.setUsername("normal");
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission());
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 75L)));
when(permissionMapper.selectByColumnId(any())).thenReturn(List.of(grant(1L, 75L)));
when(userMapper.selectList(any())).thenReturn(List.of(operator, explicitAdmin, normal));
List<ImageVideoDataPermissionUserVo> result = service.listImageVideoDataPermissionUsers(operator);
@@ -539,7 +526,7 @@ class PermissionMenuServiceTest {
int grantedCount = service.updateImageVideoDataPermissionUsers(operator, List.of(2L));
assertThat(grantedCount).isEqualTo(1);
verify(permissionMapper).deleteByMap(Map.of("column_id", 75L));
verify(permissionMapper).deleteByColumnId(75L);
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
@@ -563,7 +550,7 @@ class PermissionMenuServiceTest {
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
verify(menuMapper, never()).selectOne(any());
verify(permissionMapper, never()).delete(any());
verify(permissionMapper, never()).deleteByColumnId(any());
}
@Test
@@ -579,7 +566,7 @@ class PermissionMenuServiceTest {
normal.setUsername("normal");
when(menuMapper.selectOne(any())).thenReturn(shopDataCrawlDataPermission());
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 76L)));
when(permissionMapper.selectByColumnId(any())).thenReturn(List.of(grant(1L, 76L)));
when(userMapper.selectList(any())).thenReturn(List.of(operator, admin, normal));
List<ImageVideoDataPermissionUserVo> users = service.listShopDataCrawlDataPermissionUsers(operator);
@@ -589,7 +576,7 @@ class PermissionMenuServiceTest {
assertThat(users.get(0).isGranted()).isTrue();
assertThat(users.get(1).isGranted()).isFalse();
assertThat(grantedCount).isEqualTo(1);
verify(permissionMapper).deleteByMap(Map.of("column_id", 76L));
verify(permissionMapper).deleteByColumnId(76L);
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(inserted.capture());
@@ -41,6 +41,7 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
@@ -53,9 +54,11 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -107,6 +110,8 @@ class PublishTaskServiceTest {
private final List<TaskChunkEntity> storedChunks = new ArrayList<>();
private final List<TaskScopeStateEntity> storedScopes = new ArrayList<>();
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
private final AtomicBoolean transactionActive = new AtomicBoolean();
private final List<Boolean> storageTransactionStates = new ArrayList<>();
private int nextPayloadId;
@BeforeEach
@@ -115,11 +120,21 @@ class PublishTaskServiceTest {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(transactionTemplate.execute(any())).thenAnswer(invocation -> {
TransactionCallback<?> callback = invocation.getArgument(0);
return callback.doInTransaction(null);
transactionActive.set(true);
try {
return callback.doInTransaction(null);
} finally {
transactionActive.set(false);
}
});
lenient().doAnswer(invocation -> {
Consumer<TransactionStatus> callback = invocation.getArgument(0);
callback.accept(null);
transactionActive.set(true);
try {
callback.accept(null);
} finally {
transactionActive.set(false);
}
return null;
}).when(transactionTemplate).executeWithoutResult(any());
}
@@ -399,6 +414,90 @@ class PublishTaskServiceTest {
verify(taskFileJobService).enqueueAssembleResult(
taskId, PublishTaskService.MODULE_TYPE, resultId,
"task:" + taskId + ":owner:instance-a");
assertFalse(storageTransactionStates.isEmpty());
assertTrue(storageTransactionStates.stream().noneMatch(Boolean::booleanValue));
}
@Test
void resultCallbackDeletesUploadedPayloadWhenTransactionFails() {
long taskId = 123L;
long fileId = 223L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "transaction-failure.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 323L));
org.mockito.Mockito.reset(transactionTemplate);
doAnswer(invocation -> {
Consumer<TransactionStatus> callback = invocation.getArgument(0);
transactionActive.set(true);
try {
callback.accept(null);
} finally {
transactionActive.set(false);
}
storedChunks.clear();
storedScopes.clear();
throw new IllegalStateException("commit failed");
}).when(transactionTemplate).executeWithoutResult(any());
assertThrows(IllegalStateException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))));
assertTrue(rustfsPayloads.isEmpty());
assertTrue(storedChunks.isEmpty());
assertEquals(List.of(false, false), storageTransactionStates);
org.mockito.InOrder order = org.mockito.Mockito.inOrder(
taskDistributedLockService, transientPayloadStorageService, transactionTemplate, lock);
order.verify(taskDistributedLockService).acquire(PublishTaskService.MODULE_TYPE, taskId);
order.verify(transientPayloadStorageService).storeChunkPayloadVersioned(
any(), any(), any(), any(), any());
order.verify(transactionTemplate).executeWithoutResult(any());
order.verify(transientPayloadStorageService).deletePayloadIfPresent(any());
order.verify(lock).close();
}
@Test
void resultCallbackDeletesLoserPayloadAfterDuplicateChunkCommit() {
long taskId = 124L;
long fileId = 224L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "duplicate-chunk.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 324L));
when(taskChunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> {
TaskChunkEntity attempted = invocation.getArgument(0);
TaskChunkEntity winner = new TaskChunkEntity();
winner.setId(999L);
winner.setTaskId(attempted.getTaskId());
winner.setModuleType(attempted.getModuleType());
winner.setScopeKey(attempted.getScopeKey());
winner.setScopeHash(attempted.getScopeHash());
winner.setChunkIndex(attempted.getChunkIndex());
winner.setChunkTotal(attempted.getChunkTotal());
winner.setPayloadJson("rustfs:test/publish/winner");
winner.setPayloadHash(attempted.getPayloadHash());
storedChunks.add(winner);
throw new DuplicateKeyException("duplicate chunk");
});
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
assertEquals(1, storedChunks.size());
assertEquals("rustfs:test/publish/winner", storedChunks.getFirst().getPayloadJson());
assertTrue(rustfsPayloads.isEmpty());
assertEquals(List.of(false, false), storageTransactionStates);
verify(transientPayloadStorageService).deletePayloadIfPresent(any());
verify(lock).close();
}
@Test
@@ -804,11 +903,14 @@ class PublishTaskServiceTest {
storedChunks.clear();
storedScopes.clear();
rustfsPayloads.clear();
storageTransactionStates.clear();
transactionActive.set(false);
nextPayloadId = 0;
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
any(), any(), any(), any(), any())).thenAnswer(invocation -> {
storageTransactionStates.add(transactionActive.get());
String pointer = "rustfs:test/publish/chunk-" + (++nextPayloadId);
rustfsPayloads.put(pointer, invocation.getArgument(4));
return pointer;
@@ -818,6 +920,7 @@ class PublishTaskServiceTest {
return value != null && value.startsWith("rustfs:") ? value : null;
});
lenient().when(transientPayloadStorageService.resolvePayload(any(), any())).thenAnswer(invocation -> {
storageTransactionStates.add(transactionActive.get());
String pointer = invocation.getArgument(0);
String payload = rustfsPayloads.get(pointer);
if (payload == null) {
@@ -826,6 +929,7 @@ class PublishTaskServiceTest {
return payload;
});
lenient().doAnswer(invocation -> {
storageTransactionStates.add(transactionActive.get());
rustfsPayloads.remove(invocation.getArgument(0));
return null;
}).when(transientPayloadStorageService).deletePayloadIfPresent(any());
@@ -848,6 +952,19 @@ class PublishTaskServiceTest {
});
lenient().when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
query.getSqlSegment();
List<String> payloadValues = query.getParamNameValuePairs().values().stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.filter(value -> value.contains("rustfs:")
|| value.contains("local:")
|| value.contains("oss:"))
.toList();
if (!payloadValues.isEmpty()) {
return storedChunks.stream()
.filter(chunk -> payloadValues.contains(chunk.getPayloadJson()))
.count();
}
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
@@ -0,0 +1,366 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceSubmitTest {
private static final Long TASK_ID = 21879L;
private static final String PARSED_POINTER = "rustfs:task-parsed/similar-asin/21879/payload.json";
private static final String CHUNK_POINTER = "rustfs:task-chunk/similar-asin/21879/chunk.json";
private static final String STORED_CHUNK_POINTER = "\"" + CHUNK_POINTER + "\"";
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private CozeCredentialPoolService cozeCredentialPoolService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@Mock private TransactionStatus transactionStatus;
@InjectMocks private SimilarAsinTaskService service;
private final AtomicBoolean transactionActive = new AtomicBoolean();
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUpTransactionAndLock() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE),
anyLong(),
any(Duration.class),
eq(10_000L)))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
transactionActive.set(true);
return transactionStatus;
});
lenient().doAnswer(invocation -> {
transactionActive.set(false);
return null;
}).when(transactionManager).commit(transactionStatus);
lenient().doAnswer(invocation -> {
transactionActive.set(false);
return null;
}).when(transactionManager).rollback(transactionStatus);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdownExecutors() {
service.shutdownAssembleExecutor();
}
@Test
void doneCallbackReadsPayloadOnlyBeforeShortTransaction() throws Exception {
FileTaskEntity task = runningTask("instance-a");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
configureNewChunkAndScope();
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
List<Boolean> storageCallTransactionStates = new ArrayList<>();
when(transientPayloadStorageService.resolvePayload(eq(PARSED_POINTER), anyString()))
.thenAnswer(invocation -> {
storageCallTransactionStates.add(transactionActive.get());
return parsedPayloadJson(List.of(new SimilarAsinParsedRowVo(), new SimilarAsinParsedRowVo()));
});
when(transientPayloadStorageService.storeChunkPayloadVersioned(
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
.thenAnswer(invocation -> {
storageCallTransactionStates.add(transactionActive.get());
return STORED_CHUNK_POINTER;
});
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(false);
doAnswer(invocation -> {
assertTrue(transactionActive.get());
FileResultEntity result = invocation.getArgument(0);
result.setId(501L);
return 1;
}).when(fileResultMapper).insert(any(FileResultEntity.class));
doAnswer(invocation -> {
storageCallTransactionStates.add(transactionActive.get());
return null;
}).when(taskCacheService).deleteTaskCache(TASK_ID);
service.submitResult(TASK_ID, request(true));
assertFalse(storageCallTransactionStates.isEmpty());
assertTrue(storageCallTransactionStates.stream().noneMatch(Boolean::booleanValue));
verify(transientPayloadStorageService).resolvePayload(eq(PARSED_POINTER), anyString());
ArgumentCaptor<FileResultEntity> resultCaptor = ArgumentCaptor.forClass(FileResultEntity.class);
verify(fileResultMapper).insert(resultCaptor.capture());
assertEquals(2, resultCaptor.getValue().getRowCount());
assertEquals("germany.xlsx", resultCaptor.getValue().getSourceFilename());
}
@Test
void unknownCommitOutcomeDoesNotDeletePossiblyCommittedChunk() throws Exception {
FileTaskEntity task = runningTask("instance-a");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
configureNewChunkAndScope();
when(taskChunkMapper.selectCount(any())).thenReturn(0L);
configureChunkStore(false);
doAnswer(invocation -> {
transactionActive.set(false);
throw new IllegalStateException("commit ACK lost");
}).when(transactionManager).commit(transactionStatus);
IllegalStateException thrown = assertThrows(IllegalStateException.class,
() -> service.submitResult(TASK_ID, request(false)));
assertEquals("commit ACK lost", thrown.getMessage());
verify(transientPayloadStorageService, never()).extractPointer(anyString());
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
verify(taskCacheService, never()).touchTaskHeartbeat(TASK_ID);
}
@Test
void duplicateAlreadyPersistedChunkDoesNotRunPayloadCleanup() throws Exception {
FileTaskEntity task = runningTask("instance-a");
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
when(taskChunkMapper.selectOne(any())).thenReturn(chunk("\"rustfs:winner\""));
configureScopeStorage(null);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.submitResult(TASK_ID, request(false));
verify(transientPayloadStorageService, never()).extractPointer(anyString());
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
}
@Test
void localFallbackOwnerFailureRollsBackChunkAndKeepsCandidate() throws Exception {
FileTaskEntity task = runningTask(null);
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
configureNewChunkAndScope();
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
configureChunkStore(true);
when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(0);
IllegalStateException thrown = assertThrows(IllegalStateException.class,
() -> service.submitResult(TASK_ID, request(false)));
assertTrue(thrown.getMessage().contains("Failed to bind local fallback task owner"));
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
verify(transactionManager).rollback(transactionStatus);
verify(transactionManager, never()).commit(transactionStatus);
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
}
@Test
void duplicateLocalCandidateIsNotBoundAndReferencedPayloadIsKept() throws Exception {
FileTaskEntity task = runningTask(null);
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
TaskChunkEntity winner = chunk(STORED_CHUNK_POINTER);
when(taskChunkMapper.selectOne(any())).thenReturn(null, winner);
configureScopeStorage(null);
configureChunkStore(true);
when(transientPayloadStorageService.extractPointer(STORED_CHUNK_POINTER)).thenReturn(CHUNK_POINTER);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.submitResult(TASK_ID, request(false));
verify(taskChunkMapper, never()).insert(any(TaskChunkEntity.class));
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(STORED_CHUNK_POINTER);
}
@Test
void repeatedNonFinalCallbackCannotClearCompletedScope() throws Exception {
FileTaskEntity task = runningTask("instance-a");
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
TaskChunkEntity existing = chunk("\"rustfs:winner\"");
when(taskChunkMapper.selectOne(any())).thenReturn(existing);
TaskScopeStateEntity scope = scope(1);
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.submitResult(TASK_ID, request(false));
assertEquals(1, scope.getCompleted());
verify(taskScopeStateMapper).updateById(scope);
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(
anyString(), anyLong(), anyString(), any(), anyString());
}
private String parsedPayloadJson(List<SimilarAsinParsedRowVo> rows) throws Exception {
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setAllItems(rows);
payload.setItems(rows);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey("uploads/germany.xlsx");
sourceFile.setOriginalFilename("germany.xlsx");
payload.setSourceFiles(List.of(sourceFile));
return objectMapper.writeValueAsString(payload);
}
private void configureNewChunkAndScope() {
AtomicReference<TaskChunkEntity> chunkRef = new AtomicReference<>();
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> chunkRef.get());
doAnswer(invocation -> {
TaskChunkEntity chunk = invocation.getArgument(0);
chunk.setId(301L);
chunkRef.set(chunk);
return 1;
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
configureScopeStorage(null);
}
private void configureScopeStorage(TaskScopeStateEntity initial) {
AtomicReference<TaskScopeStateEntity> scopeRef = new AtomicReference<>(initial);
when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> scopeRef.get());
doAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
scope.setId(401L);
scopeRef.set(scope);
return 1;
}).when(taskScopeStateMapper).insert(any(TaskScopeStateEntity.class));
}
private void configureChunkStore(boolean localFallback) {
when(transientPayloadStorageService.storeChunkPayloadVersioned(
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
.thenReturn(STORED_CHUNK_POINTER);
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(localFallback);
}
private FileTaskEntity runningTask(String owner) throws Exception {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
task.setStatus("RUNNING");
task.setUserId(7L);
String resultJson = "{\"parsedPayloadRef\":\"" + PARSED_POINTER + "\"";
if (owner != null) {
resultJson += ",\"ownerInstanceId\":\"" + owner + "\"";
}
task.setResultJson(resultJson + "}");
return task;
}
private SimilarAsinSubmitResultRequest request(boolean done) {
SimilarAsinSubmitResultRequest request = new SimilarAsinSubmitResultRequest();
request.setSubmissionId("similar-asin-21879");
request.setChunkIndex(0);
request.setChunkTotal(1);
request.setDone(done);
return request;
}
private TaskChunkEntity chunk(String payload) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(301L);
chunk.setTaskId(TASK_ID);
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
chunk.setScopeHash("existing-scope");
chunk.setChunkIndex(0);
chunk.setChunkTotal(1);
chunk.setPayloadJson(payload);
return chunk;
}
private TaskScopeStateEntity scope(int completed) {
TaskScopeStateEntity scope = new TaskScopeStateEntity();
scope.setId(401L);
scope.setTaskId(TASK_ID);
scope.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
scope.setScopeKey("similar-asin-21879");
scope.setScopeHash("existing-scope");
scope.setChunkTotal(1);
scope.setCompleted(completed);
scope.setStateJson("{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}");
return scope;
}
}
@@ -0,0 +1,208 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.SimpleTransactionStatus;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class TaskScopePayloadStorageServiceTest {
@BeforeAll
static void initializeMybatisMetadata() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
TaskScopeStateEntity.class);
}
@Test
void uploadsBeforeOpeningTransactionAndCleansReplacedPayloadAfterCommit() {
TaskScopeStateMapper mapper = mock(TaskScopeStateMapper.class);
TransientPayloadStorageService payloadStorage = mock(TransientPayloadStorageService.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
TaskScopePayloadStorageService service = new TaskScopePayloadStorageService(
mapper, new ObjectMapper(), payloadStorage, transactionManager);
List<String> events = new ArrayList<>();
AtomicBoolean transactionActive = new AtomicBoolean();
TaskScopeStateEntity existing = new TaskScopeStateEntity();
existing.setId(9L);
existing.setTaskId(7L);
existing.setModuleType("PRICE_TRACK");
existing.setScopeHash("scope-hash");
existing.setStateJson("rustfs:old.json");
when(payloadStorage.storeScopePayloadVersioned(
eq("PRICE_TRACK"), eq(7L), any(), any(), eq(true)))
.thenAnswer(invocation -> {
assertFalse(transactionActive.get());
events.add("upload");
return "rustfs:new.json";
});
when(payloadStorage.extractPointer("rustfs:new.json")).thenReturn("rustfs:new.json");
when(payloadStorage.extractPointer("rustfs:old.json")).thenReturn("rustfs:old.json");
when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
events.add("begin");
transactionActive.set(true);
return new SimpleTransactionStatus();
});
when(mapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<?> query = invocation.getArgument(0);
assertTrue(query.getSqlSegment().endsWith("limit 1 FOR UPDATE"));
return existing;
});
when(mapper.update(any(), any())).thenReturn(1);
doAnswer(invocation -> {
events.add("commit");
transactionActive.set(false);
return null;
}).when(transactionManager).commit(any(TransactionStatus.class));
doAnswer(invocation -> {
assertFalse(transactionActive.get());
events.add("cleanup");
return null;
}).when(payloadStorage).deleteReplacedPayloadIfNeeded("rustfs:old.json", "rustfs:new.json");
service.saveScopePayload(7L, "PRICE_TRACK", "shop-a", new Payload("ok"));
assertEquals(List.of("upload", "begin", "commit", "cleanup"), events);
verify(mapper).update(any(), any());
}
@Test
void deletesUploadedPayloadAfterRollbackWhenDatabaseConfirmsItIsUnreferenced() {
TaskScopeStateMapper mapper = mock(TaskScopeStateMapper.class);
TransientPayloadStorageService payloadStorage = mock(TransientPayloadStorageService.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
TaskScopePayloadStorageService service = new TaskScopePayloadStorageService(
mapper, new ObjectMapper(), payloadStorage, transactionManager);
List<String> events = new ArrayList<>();
AtomicBoolean transactionActive = new AtomicBoolean();
TaskScopeStateEntity existing = new TaskScopeStateEntity();
existing.setId(9L);
existing.setTaskId(7L);
existing.setModuleType("PRICE_TRACK");
existing.setScopeHash("scope-hash");
existing.setStateJson("rustfs:old.json");
when(payloadStorage.storeScopePayloadVersioned(
eq("PRICE_TRACK"), eq(7L), any(), any(), eq(true)))
.thenAnswer(invocation -> {
assertFalse(transactionActive.get());
events.add("upload");
return "rustfs:new.json";
});
when(payloadStorage.extractPointer("rustfs:new.json")).thenReturn("rustfs:new.json");
when(payloadStorage.extractPointer("rustfs:old.json")).thenReturn("rustfs:old.json");
when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
events.add("begin");
transactionActive.set(true);
return new SimpleTransactionStatus();
});
when(mapper.selectOne(any())).thenAnswer(invocation -> {
events.add(transactionActive.get() ? "transaction-read" : "cleanup-read");
return existing;
});
when(mapper.update(any(), any())).thenThrow(new IllegalStateException("database write failed"));
doAnswer(invocation -> {
events.add("rollback");
transactionActive.set(false);
return null;
}).when(transactionManager).rollback(any(TransactionStatus.class));
doAnswer(invocation -> {
assertFalse(transactionActive.get());
events.add("delete");
return null;
}).when(payloadStorage).deletePayloadIfPresent("rustfs:new.json");
assertThrows(IllegalStateException.class,
() -> service.saveScopePayload(7L, "PRICE_TRACK", "shop-a", new Payload("ok")));
assertEquals(List.of("upload", "begin", "transaction-read", "rollback", "cleanup-read", "delete"), events);
verify(payloadStorage).deletePayloadIfPresent("rustfs:new.json");
}
@Test
void keepsUploadedPayloadWhenCommitThrowsButDatabaseReferencesIt() {
TaskScopeStateMapper mapper = mock(TaskScopeStateMapper.class);
TransientPayloadStorageService payloadStorage = mock(TransientPayloadStorageService.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
TaskScopePayloadStorageService service = new TaskScopePayloadStorageService(
mapper, new ObjectMapper(), payloadStorage, transactionManager);
List<String> events = new ArrayList<>();
AtomicBoolean transactionActive = new AtomicBoolean();
TaskScopeStateEntity previous = new TaskScopeStateEntity();
previous.setId(9L);
previous.setTaskId(7L);
previous.setModuleType("PRICE_TRACK");
previous.setScopeHash("scope-hash");
previous.setStateJson("rustfs:old.json");
TaskScopeStateEntity committed = new TaskScopeStateEntity();
committed.setId(9L);
committed.setTaskId(7L);
committed.setModuleType("PRICE_TRACK");
committed.setScopeHash("scope-hash");
committed.setStateJson("rustfs:new.json");
when(payloadStorage.storeScopePayloadVersioned(
eq("PRICE_TRACK"), eq(7L), any(), any(), eq(true)))
.thenAnswer(invocation -> {
assertFalse(transactionActive.get());
events.add("upload");
return "rustfs:new.json";
});
when(payloadStorage.extractPointer("rustfs:new.json")).thenReturn("rustfs:new.json");
when(payloadStorage.extractPointer("rustfs:old.json")).thenReturn("rustfs:old.json");
when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
events.add("begin");
transactionActive.set(true);
return new SimpleTransactionStatus();
});
when(mapper.selectOne(any())).thenAnswer(invocation -> {
events.add(transactionActive.get() ? "transaction-read" : "cleanup-read");
return transactionActive.get() ? previous : committed;
});
when(mapper.update(any(), any())).thenReturn(1);
doAnswer(invocation -> {
events.add("commit");
transactionActive.set(false);
throw new IllegalStateException("commit outcome unknown");
}).when(transactionManager).commit(any(TransactionStatus.class));
IllegalStateException error = assertThrows(IllegalStateException.class,
() -> service.saveScopePayload(7L, "PRICE_TRACK", "shop-a", new Payload("ok")));
assertEquals("commit outcome unknown", error.getMessage());
assertEquals(List.of("upload", "begin", "transaction-read", "commit", "cleanup-read"), events);
verify(payloadStorage, never()).deletePayloadIfPresent("rustfs:new.json");
verify(payloadStorage, never()).deleteReplacedPayloadIfNeeded(any(), any());
}
private record Payload(String value) {
}
}
@@ -0,0 +1,63 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.nanri.aiimage.config.ZiniaoProperties;
import com.nanri.aiimage.modules.ziniao.client.ZiniaoClient;
import com.nanri.aiimage.modules.ziniao.memory.service.ZiniaoTransientCacheService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffItemVo;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class ZiniaoAuthServiceTest {
@Test
void invalidUserCacheSkipsUpstreamStoreLookup() {
ZiniaoClient client = mock(ZiniaoClient.class);
ZiniaoTransientCacheService cache = mock(ZiniaoTransientCacheService.class);
ZiniaoAuthService service = service(client, cache);
when(cache.get(eq("INVALID_USER_STORES"), anyString(), eq(Boolean.class)))
.thenReturn(Optional.of(Boolean.TRUE));
assertTrue(service.getOrLoadUserStoresForIndex("api-key", 12L, 34L).isEmpty());
verify(client, never()).listUserStores(anyString(), any(), any());
}
@Test
void markingInvalidUserWritesNegativeCacheAndRemovesStaleStaffEntry() {
ZiniaoTransientCacheService cache = mock(ZiniaoTransientCacheService.class);
ZiniaoAuthService service = service(mock(ZiniaoClient.class), cache);
ZiniaoStaffItemVo invalid = new ZiniaoStaffItemVo();
invalid.setUserId(34L);
ZiniaoStaffItemVo valid = new ZiniaoStaffItemVo();
valid.setUserId(35L);
when(cache.getList(eq("STAFF_LIST"), anyString(), eq(ZiniaoStaffItemVo.class)))
.thenReturn(Optional.of(List.of(invalid, valid)));
service.evictInvalidUserForIndex("api-key", 12L, 34L);
verify(cache).put(eq("INVALID_USER_STORES"), anyString(), eq(Boolean.TRUE), eq(Duration.ofMinutes(30)));
verify(cache).put(eq("STAFF_LIST"), anyString(), eq(List.of(valid)), eq(Duration.ofMinutes(30)));
verify(cache).delete(eq("USER_STORES"), anyString());
}
private ZiniaoAuthService service(ZiniaoClient client, ZiniaoTransientCacheService cache) {
return new ZiniaoAuthService(
new ZiniaoProperties(),
client,
mock(ZiniaoSessionCacheService.class),
cache,
mock(ZiniaoApiKeyProvider.class));
}
}