super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增内容，新增拉起软件层

Browse Source
This commit is contained in:
super
2026-06-08 09:19:15 +08:00
parent 8ab647c419
commit c85e5b278f
67 changed files with 1552 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClientTest.java
View File

@@ -25,6 +25,24 @@ class AppearancePatentCozeClientTest {
new BrandCheckClient(new BrandCheckProperties())
);
@Test
void markRowsFailedLeavesUserFacingResultBlankWhenAsyncPollTimeout() {
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setId("1");
List<AppearancePatentResultRowDto> failedRows =
client.markRowsFailed(List.of(row), "Coze \u5f02\u6b65\u5de5\u4f5c\u6d41\u8f6e\u8be2\u8d85\u65f6");
assertThat(failedRows).hasSize(1);
AppearancePatentResultRowDto failed = failedRows.get(0);
assertThat(failed.getError()).isEqualTo("Coze \u5f02\u6b65\u5de5\u4f5c\u6d41\u8f6e\u8be2\u8d85\u65f6");
assertThat(failed.getStatus()).isEqualTo("FAILED");
assertThat(failed.getTitleRisk()).isNull();
assertThat(failed.getAppearanceRisk()).isNull();
assertThat(failed.getPatentRisk()).isNull();
assertThat(failed.getConclusion()).isNull();
}
@Test
void titleRiskIsInfringementWhenBrandCheckHasFailedData() {
BrandCheckClient.BrandCheckBatchResult result =

40
backend-java/src/test/java/com/nanri/aiimage/modules/file/service/object/RustfsDeleteRetryServiceTest.java Normal file
View File

@@ -0,0 +1,40 @@
package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.config.TransientStorageProperties;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
class RustfsDeleteRetryServiceTest {
@Test
void retryPendingDeletesRefillsDeferredItemsBeforeQueueDrainsCompletely() {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setDeleteRetryEnabled(true);
properties.setDeleteRetryQueueCapacity(2);
properties.setDeleteRetryBatchSize(1);
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
doAnswer(invocation -> {
String objectKey = invocation.getArgument(0);
if ("b".equals(objectKey)) {
throw new IllegalStateException("still failing");
}
return null;
}).when(rustfs).deleteObjectFromRetry(org.mockito.ArgumentMatchers.anyString());
RustfsDeleteRetryService service = new RustfsDeleteRetryService(properties, rustfs);
service.enqueue("a", new IllegalStateException("failed-a"));
service.enqueue("b", new IllegalStateException("failed-b"));
service.enqueue("c", new IllegalStateException("failed-c"));
service.retryPendingDeletes();
service.retryPendingDeletes();
service.retryPendingDeletes();
verify(rustfs).deleteObjectFromRetry("c");
assertEquals(1, service.pendingCount());
}
}

138
backend-java/src/test/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageServiceTest.java Normal file
View File

@@ -0,0 +1,138 @@
package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.config.TransientStorageProperties;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import io.minio.MinioClient;
import okhttp3.OkHttpClient;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.ObjectProvider;
import java.lang.reflect.Field;
import java.util.concurrent.Semaphore;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class RustfsObjectStorageServiceTest {
@Test
void httpClientUsesConfiguredConnectionPoolAndKeepsRetryEnabled() {
TransientStorageProperties properties = configuredProperties();
properties.setConnectionPoolMaxIdle(3);
properties.setConnectionPoolKeepAliveMillis(1500);
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, provider(new SimpleMeterRegistry()), emptyProvider(), null);
OkHttpClient client = service.getHttpClient();
assertTrue(client.retryOnConnectionFailure(), "RustFS OkHttp 应开启连接失败重试");
assertNotNull(client.connectionPool(), "RustFS OkHttp 应使用显式 connectionPool 配置");
}
@Test
void deleteFailureRetriesAndEnqueuesCompensation() {
TransientStorageProperties properties = configuredProperties();
properties.setDeleteMaxRetries(2);
properties.setBaseRetryDelayMillis(0);
properties.setRetryJitterMillis(0);
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), provider(retryService), () -> {
throw new IllegalStateException("rustfs down");
});
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service.deleteObject("task/a.json"));
assertTrue(ex.getMessage().contains("delete"));
verify(retryService).enqueue(any(String.class), any(Throwable.class));
}
@Test
void deleteFromRetryDoesNotEnqueueAgain() {
TransientStorageProperties properties = configuredProperties();
properties.setDeleteMaxRetries(1);
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), provider(retryService), () -> {
throw new IllegalStateException("rustfs down");
});
assertThrows(IllegalStateException.class, () -> service.deleteObjectFromRetry("task/a.json"));
verify(retryService, never()).enqueue(any(String.class), any(Throwable.class));
}
@Test
void readPermitTimeoutRejectsImmediately() throws Exception {
TransientStorageProperties properties = configuredProperties();
properties.setAcquirePermitTimeoutMillis(0);
properties.setMaxConcurrentReads(1);
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> mock(MinioClient.class));
Semaphore readSemaphore = privateSemaphore(service, "readSemaphore");
assertTrue(readSemaphore.tryAcquire());
try {
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service.readObjectAsString("task/a.json"));
assertTrue(ex.getMessage().contains("concurrency limit"));
} finally {
readSemaphore.release();
}
}
@Test
void failureWindowOpensCooldownForUpload() {
TransientStorageProperties properties = configuredProperties();
properties.setUploadMaxRetries(1);
properties.setFailureWindowThreshold(1);
properties.setFailureCooldownMillis(10000);
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), emptyProvider(), () -> {
throw new IllegalStateException("rustfs down");
});
assertThrows(IllegalStateException.class, () -> service.uploadText("task/a.json", "{}", false));
IllegalStateException rejected = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/b.json", "{}", false));
assertTrue(rejected.getMessage().contains("cooldown active"));
}
private static TransientStorageProperties configuredProperties() {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setEndpoint("http://127.0.0.1:9000");
properties.setBucket("bucket");
properties.setAccessKeyId("ak");
properties.setAccessKeySecret("sk");
properties.setBaseRetryDelayMillis(0);
properties.setRetryJitterMillis(0);
return properties;
}
@SuppressWarnings("unchecked")
private static Semaphore privateSemaphore(RustfsObjectStorageService service, String fieldName) throws Exception {
Field field = RustfsObjectStorageService.class.getDeclaredField(fieldName);
field.setAccessible(true);
return (Semaphore) field.get(service);
}
private static <T> ObjectProvider<T> provider(T value) {
ObjectProvider<T> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(value);
return provider;
}
private static <T> ObjectProvider<T> emptyProvider() {
ObjectProvider<T> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(null);
return provider;
}
}

113
backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageServiceTest.java Normal file
View File

@@ -0,0 +1,113 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class TransientPayloadStorageServiceTest {
@TempDir
Path tempDir;
@Test
void oversizeRawPayloadFallsBackToLocalAndKeepsFallbackMarker() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(4, 1024, true));
String pointer = service.storeParsedPayloadFast("TEST", 1L, "scope", "abcdef", false);
assertTrue(pointer.startsWith("local:i/test-instance/task-parsed/test/1/scope/latest.json"));
assertTrue(service.wasLastStoreLocalFallback());
assertEquals("abcdef", service.resolvePayload(pointer, "read failed"));
verify(rustfs, never()).uploadText(anyString(), anyString(), anyBoolean());
verify(rustfs).recordLocalFallback();
}
@Test
void encodedPayloadLimitCanForceLocalFallback() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 1, true));
String pointer = service.storeParsedPayloadFast("TEST", 1L, "scope", "a", false);
assertTrue(pointer.startsWith("local:"));
assertTrue(service.wasLastStoreLocalFallback());
verify(rustfs, never()).uploadText(anyString(), anyString(), anyBoolean());
}
@Test
void oversizePayloadCanBeRejectedByConfiguration() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(4, 1024, false));
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service.storeParsedPayloadFast("TEST", 1L, "scope", "abcdef", false));
assertTrue(ex.getMessage().contains("exceeds configured size limit"));
verify(rustfs, never()).uploadText(anyString(), anyString(), anyBoolean());
}
@Test
void smallPayloadStillUsesRustfsAndClearsFallbackMarker() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
when(rustfs.uploadText(anyString(), anyString(), anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0));
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 4096, true));
String pointer = service.storeParsedPayloadFast("TEST", 1L, "scope", "abc", false);
assertTrue(pointer.startsWith("rustfs:task-parsed/test/1/scope/latest.json"));
assertFalse(service.wasLastStoreLocalFallback());
verify(rustfs).uploadText(anyString(), anyString(), anyBoolean());
}
private TransientPayloadStorageService newService(RustfsObjectStorageService rustfs,
TransientStorageProperties transientProperties) {
StorageProperties storageProperties = new StorageProperties();
storageProperties.setLocalTempDir(tempDir.toString());
return new TransientPayloadStorageService(
transientProperties,
storageProperties,
rustfs,
mock(OssStorageService.class),
new ObjectMapper(),
new InstanceMetadata("test-instance"),
mock(TaskChunkMapper.class),
mock(TaskScopeStateMapper.class));
}
private TransientStorageProperties propertiesWithLimit(long maxPayloadBytes,
long maxStoredPayloadBytes,
boolean fallbackToLocalOnOversize) {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setEnabled(true);
properties.setMaxPayloadBytes(maxPayloadBytes);
properties.setMaxStoredPayloadBytes(maxStoredPayloadBytes);
properties.setWarnPayloadBytes(1);
properties.setFallbackToLocalOnOversize(fallbackToLocalOnOversize);
return properties;
}
}