task-116: 后台管理新增/导入表单弹窗化 + 分组管理独立菜单与UI优化 + 轻量进度端点
- 后台管理页(admin)所有面板的新增/导入表单改为弹窗操作,原有字段 ID 全部保留、提交逻辑不变;导入删除入口不再触发二次确认拦截 - 分组管理升级为独立菜单(V102 + schema initializer),移除 5 个面板内的管理分组按钮;分组列表改为蓝白主题、增加权限分组横幅 - Python 侧 group-manage 权限守卫(_ensure_backend_menu_access 补充 group-manage) - 引入 V101(biz_task_file_job 复合索引)+ 新增 TaskProgressLight/TaskFileJob 轻量端点与进度聚合作 - 前端 progress-light / page-separated-loads / dispatch-guard 共享模块及单元测试
This commit is contained in:
-85
@@ -1,85 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import javax.net.ssl.SNIHostName;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLParameters;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
|
||||
public class LlmGatewayTlsProbe {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String host = "ai.t8star.org";
|
||||
String apiKey = args[0];
|
||||
|
||||
System.out.println("[probe] java=" + System.getProperty("java.version")
|
||||
+ " tls=" + System.getProperty("java.vm.name"));
|
||||
for (InetAddress a : InetAddress.getAllByName(host)) {
|
||||
System.out.println("[probe] dns " + a);
|
||||
}
|
||||
|
||||
rawHandshake(host, null, "default");
|
||||
rawHandshake(host, "TLSv1.2", "tls12-only");
|
||||
|
||||
httpClientCall(host, apiKey, null, "jdk-http-default");
|
||||
httpClientCall(host, apiKey, "TLSv1.2", "jdk-http-tls12");
|
||||
}
|
||||
|
||||
private static void rawHandshake(String host, String protocol, String label) throws Exception {
|
||||
SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
|
||||
try (SSLSocket socket = (SSLSocket) factory.createSocket(host, 443)) {
|
||||
socket.setSoTimeout(15000);
|
||||
SSLParameters params = socket.getSSLParameters();
|
||||
if (protocol != null) {
|
||||
params.setProtocols(new String[]{protocol});
|
||||
}
|
||||
params.setServerNames(List.of(new SNIHostName(host)));
|
||||
socket.setSSLParameters(params);
|
||||
socket.startHandshake();
|
||||
System.out.println("[probe] raw[" + label + "] OK proto=" + socket.getSession().getProtocol()
|
||||
+ " cipher=" + socket.getSession().getCipherSuite());
|
||||
} catch (Exception ex) {
|
||||
System.out.println("[probe] raw[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void httpClientCall(String host, String apiKey, String protocol, String label) throws Exception {
|
||||
HttpClient.Builder builder = HttpClient.newBuilder()
|
||||
.connectTimeout(Duration.ofSeconds(10))
|
||||
.version(HttpClient.Version.HTTP_1_1);
|
||||
if (protocol != null) {
|
||||
SSLContext context = SSLContext.getInstance("TLS");
|
||||
context.init(null, null, null);
|
||||
builder.sslContext(context);
|
||||
}
|
||||
HttpClient client = builder.build();
|
||||
try {
|
||||
String body = "{\"model\":\"gemini-3.5-flash-lite\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":10}";
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create("https://" + host + "/v1/chat/completions"))
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build();
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
String text = response.body();
|
||||
System.out.println("[probe] http[" + label + "] status=" + response.statusCode()
|
||||
+ " body=" + text.substring(0, Math.min(160, text.length())));
|
||||
} catch (Exception ex) {
|
||||
System.out.println("[probe] http[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
|
||||
Throwable cause = ex;
|
||||
while (cause.getCause() != null) {
|
||||
cause = cause.getCause();
|
||||
System.out.println("[probe] cause " + cause.getClass().getSimpleName() + ": " + cause.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
-226
@@ -1,226 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 15:图片缓存访问时间更新改为异步批量刷新,减少逐图 UPDATE。
|
||||
* lookup 命中不再同步 touchLastUsed,而是进入内存 touch 缓冲(按 url_hash 去重),
|
||||
* 由定时任务/阈值触发 flushPendingTouches 批量 touchLastUsedBatch 刷新;
|
||||
* 缓冲有大小上限,超限立即刷新,不会无界增长;失败 best-effort 吞掉。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinImagePrefetchServiceAsyncTouchTest {
|
||||
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private TaskImageCacheMapper taskImageCacheMapper;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
|
||||
@InjectMocks private SimilarAsinImagePrefetchService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(1000);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdown();
|
||||
}
|
||||
|
||||
private static String sha256Hex(String value) throws Exception {
|
||||
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
|
||||
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private void stubHit(String url, byte[] bytes) throws Exception {
|
||||
when(taskImageCacheMapper.selectBytesByUrlHash(sha256Hex(url))).thenReturn(bytes);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_normal_default_path() throws Exception {
|
||||
// 正常输入:命中不立即写 DB,进入缓冲;flush 后批量 touch 一次。
|
||||
String url = "https://img.example.com/hit.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
byte[] bytes = new byte[]{1, 2, 3};
|
||||
stubHit(url, bytes);
|
||||
|
||||
byte[] result = service.lookup(url);
|
||||
assertEquals(bytes, result, "命中必须返回缓存字节");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多次命中进入同一缓冲,flush 合并为一次批量 touch,覆盖全部命中。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
|
||||
"https://img.example.com/c.jpg");
|
||||
Set<String> hashes = new HashSet<>();
|
||||
for (int i = 0; i < urls.size(); i++) {
|
||||
hashes.add(sha256Hex(urls.get(i)));
|
||||
stubHit(urls.get(i), new byte[]{(byte) (i + 1)});
|
||||
}
|
||||
for (String url : urls) {
|
||||
assertNotNull(service.lookup(url), "命中返回字节");
|
||||
}
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
|
||||
service.flushPendingTouches();
|
||||
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
|
||||
assertEquals(hashes, new HashSet<>(captor.getValue()), "一次批量 touch 覆盖全部命中 hash");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:同一 url 多次命中只 touch 一次;重复 flush 无多余请求。
|
||||
String url = "https://img.example.com/same.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubHit(url, new byte[]{5});
|
||||
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(new byte[]{5});
|
||||
|
||||
service.lookup(url);
|
||||
service.lookup(url);
|
||||
service.lookup(url);
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空白 url 不进入缓冲;flush 空缓冲不产生任何数据库访问。
|
||||
assertNull(service.lookup(null));
|
||||
assertNull(service.lookup(""));
|
||||
assertNull(service.lookup(" "));
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_boundary_single_item() throws Exception {
|
||||
// 单条命中:flush 后单元素批量 touch,不依赖批量路径。
|
||||
String url = "https://img.example.com/single.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubHit(url, new byte[]{7});
|
||||
|
||||
assertNotNull(service.lookup(url));
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_boundary_limit_and_overflow() throws Exception {
|
||||
// 缓冲达到阈值立即刷新,不无界增长;刷新后继续累积。
|
||||
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(3);
|
||||
List<String> urls = List.of("https://img.example.com/o1.jpg", "https://img.example.com/o2.jpg",
|
||||
"https://img.example.com/o3.jpg", "https://img.example.com/o4.jpg",
|
||||
"https://img.example.com/o5.jpg");
|
||||
for (String url : urls) {
|
||||
stubHit(url, new byte[]{1});
|
||||
}
|
||||
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
||||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
service.lookup(urls.get(i));
|
||||
}
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
|
||||
assertEquals(3, captor.getValue().size(), "第 3 条命中触发阈值立即刷新 3 条");
|
||||
|
||||
service.lookup(urls.get(3));
|
||||
service.lookup(urls.get(4));
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
|
||||
assertEquals(2, captor.getValue().size(), "剩余 2 条在 flush 时刷新,缓冲不残留、不无界增长");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_invalid_input_rejected() {
|
||||
// 非法输入:db cache 关闭时 lookup 直接返回 null,不进入缓冲、不访问数据库。
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
|
||||
assertNull(service.lookup("https://img.example.com/a.jpg"));
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:批量 touch 抛异常时吞掉不阻塞 lookup、缓冲已排空无残留;
|
||||
// 恢复后重新入队 flush 成功。
|
||||
String url = "https://img.example.com/fail.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubHit(url, new byte[]{3});
|
||||
AtomicInteger failures = new AtomicInteger(0);
|
||||
List<String> capturedArgs = new java.util.ArrayList<>();
|
||||
doAnswer(invocation -> {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> arg = (List<String>) invocation.getArgument(0);
|
||||
capturedArgs.addAll(arg);
|
||||
if (failures.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("db down");
|
||||
}
|
||||
return 0;
|
||||
}).when(taskImageCacheMapper).touchLastUsedBatch(any());
|
||||
|
||||
assertNotNull(service.lookup(url), "touch 失败不阻断 lookup 返回缓存字节");
|
||||
assertThrows(Exception.class, () -> service.flushPendingTouches(), "首次 flush 抛错(由调用方吞掉)");
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
|
||||
assertNotNull(service.lookup(url), "失败后再次命中重新入队");
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(any());
|
||||
assertEquals(List.of(hash, hash), capturedArgs, "两次 touch 都覆盖命中 hash,失败后恢复成功");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
private static void assertNull(Object value) {
|
||||
org.junit.jupiter.api.Assertions.assertNull(value);
|
||||
}
|
||||
}
|
||||
-297
@@ -1,297 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 14:图片 DB cache 改为批量读取缩略图,并只更新实际命中的 last_used_at。
|
||||
* 批量 lookup 入口(lookupBatch)一次 IN 查询返回命中字节 Map;
|
||||
* last_used_at 只对实际命中的 url_hash 更新(touch 集合 = 命中集合),
|
||||
* 未命中 url 不产生任何 touch/insert。单 URL 旧入口 lookup 语义保持兼容。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinImagePrefetchServiceBatchTest {
|
||||
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private TaskImageCacheMapper taskImageCacheMapper;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
|
||||
@InjectMocks private SimilarAsinImagePrefetchService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdown();
|
||||
}
|
||||
|
||||
private static String sha256Hex(String value) throws Exception {
|
||||
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
|
||||
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** stub 批量命中读取:只对 hits 集合内的 hash 返回字节。 */
|
||||
private void stubBatchRead(List<String> hits, Map<String, byte[]> bytesByHash) {
|
||||
when(taskImageCacheMapper.selectBytesByUrlHashes(any())).thenAnswer(invocation -> {
|
||||
List<String> hashes = invocation.getArgument(0);
|
||||
List<TaskImageCacheEntity> rows = new ArrayList<>();
|
||||
for (String hash : hashes) {
|
||||
if (hits.contains(hash)) {
|
||||
TaskImageCacheEntity row = new TaskImageCacheEntity();
|
||||
row.setUrlHash(hash);
|
||||
row.setImageBytes(bytesByHash.get(hash));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
|
||||
/** 调用批量入口,按 url 顺序返回字节(未命中为 null)。 */
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<byte[]> invokeLookupBatch(SimilarAsinImagePrefetchService svc, List<String> urls) throws Exception {
|
||||
Method m = SimilarAsinImagePrefetchService.class.getDeclaredMethod("lookupBatch", List.class);
|
||||
m.setAccessible(true);
|
||||
return (List<byte[]>) m.invoke(svc, urls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_default_path() throws Exception {
|
||||
// 正常输入:命中与未命中混排,批量读回命中字节,touch 只覆盖实际命中。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
|
||||
List<String> hits = List.of(sha256Hex(urls.get(0)));
|
||||
byte[] bytesA = new byte[]{1, 2, 3};
|
||||
stubBatchRead(hits, Map.of(sha256Hex(urls.get(0)), bytesA));
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
|
||||
assertEquals(2, result.size(), "批量入口必须按输入顺序返回");
|
||||
assertEquals(bytesA, result.get(0), "命中行返回缓存字节");
|
||||
assertNull(result.get(1), "未命中行返回 null,不虚构缓存内容");
|
||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(sha256Hex(urls.get(0))));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_multiple_items() throws Exception {
|
||||
// 批量场景:全命中多 url,一次 IN 查询返回全部字节,touch 覆盖全部命中。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
|
||||
"https://img.example.com/c.jpg");
|
||||
List<String> hits = new ArrayList<>();
|
||||
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
|
||||
for (int i = 0; i < urls.size(); i++) {
|
||||
hits.add(sha256Hex(urls.get(i)));
|
||||
bytesByHash.put(hits.get(i), new byte[]{(byte) (i + 1)});
|
||||
}
|
||||
stubBatchRead(hits, bytesByHash);
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
|
||||
assertEquals(3, result.size());
|
||||
for (int i = 0; i < urls.size(); i++) {
|
||||
assertEquals(bytesByHash.get(sha256Hex(urls.get(i))), result.get(i), "顺序稳定、字节不丢失");
|
||||
}
|
||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(new ArrayList<>(hits));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:每次行为一致,不产生重复请求/重复 touch。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
||||
String hash = sha256Hex(urls.get(0));
|
||||
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{9}));
|
||||
|
||||
invokeLookupBatch(service, urls);
|
||||
invokeLookupBatch(service, urls);
|
||||
|
||||
verify(taskImageCacheMapper, times(2)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(List.of(hash));
|
||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空列表安全返回空结果,不产生任何数据库访问。
|
||||
List<byte[]> nullResult = invokeLookupBatch(service, null);
|
||||
assertNotNull(nullResult);
|
||||
assertTrue(nullResult.isEmpty());
|
||||
|
||||
List<byte[]> emptyResult = invokeLookupBatch(service, List.of());
|
||||
assertNotNull(emptyResult);
|
||||
assertTrue(emptyResult.isEmpty());
|
||||
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_boundary_single_item() throws Exception {
|
||||
// 单 url:不依赖批量路径,命中时单次查询 + 单次 touch。
|
||||
String url = "https://img.example.com/single.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{7}));
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, List.of(url));
|
||||
|
||||
assertEquals(1, result.size());
|
||||
assertEquals(7, result.get(0)[0]);
|
||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_boundary_limit_and_overflow() throws Exception {
|
||||
// 大批量(超过单批上限 500):分片查询,命中 touch 只覆盖命中集合。
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 1200; i++) {
|
||||
urls.add("https://img.example.com/overflow-" + i + ".jpg");
|
||||
}
|
||||
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
|
||||
List<String> hits = new ArrayList<>();
|
||||
for (int i = 0; i < 1200; i += 2) {
|
||||
String hash = sha256Hex(urls.get(i));
|
||||
hits.add(hash);
|
||||
bytesByHash.put(hash, new byte[]{(byte) (i % 100)});
|
||||
}
|
||||
stubBatchRead(hits, bytesByHash);
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
|
||||
assertEquals(1200, result.size(), "超大批量结果不丢失、顺序稳定");
|
||||
int hitCount = 0;
|
||||
for (int i = 0; i < 1200; i++) {
|
||||
if (i % 2 == 0) {
|
||||
assertNotNull(result.get(i), "偶数下标命中必须返回字节");
|
||||
hitCount++;
|
||||
} else {
|
||||
assertNull(result.get(i), "奇数下标未命中返回 null");
|
||||
}
|
||||
}
|
||||
assertEquals(600, hitCount);
|
||||
verify(taskImageCacheMapper, times(3)).selectBytesByUrlHashes(any());
|
||||
// touch 按单批 500 分片:600 命中 → 2 次 touch 调用,且只覆盖命中集合。
|
||||
var captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
|
||||
List<List<String>> touchCalls = new ArrayList<>();
|
||||
for (Object call : captor.getAllValues()) {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> casted = (List<String>) call;
|
||||
touchCalls.add(casted);
|
||||
}
|
||||
assertEquals(2, touchCalls.size());
|
||||
assertEquals(500, touchCalls.get(0).size(), "第一批 touch 500 个命中");
|
||||
assertEquals(100, touchCalls.get(1).size(), "第二批 touch 剩余 100 个命中");
|
||||
assertEquals(hits.subList(0, 500), touchCalls.get(0), "touch 只覆盖实际命中集合");
|
||||
assertEquals(hits.subList(500, 600), touchCalls.get(1));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_invalid_input_rejected() throws Exception {
|
||||
// 非法输入:db cache 关闭时批量入口直接返回空,不访问数据库。
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
assertNotNull(result);
|
||||
assertTrue(result.isEmpty(), "db cache 关闭时必须直接返回空结果");
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:查询抛异常时批量入口返回空结果、无任何 touch/insert 残留;
|
||||
// 恢复后重试成功。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
||||
String hash = sha256Hex(urls.get(0));
|
||||
AtomicInteger callCount = new AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (callCount.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("db down");
|
||||
}
|
||||
TaskImageCacheEntity row = new TaskImageCacheEntity();
|
||||
row.setUrlHash(hash);
|
||||
row.setImageBytes(new byte[]{5});
|
||||
return List.of(row);
|
||||
}).when(taskImageCacheMapper).selectBytesByUrlHashes(any());
|
||||
|
||||
List<byte[]> failed = invokeLookupBatch(service, urls);
|
||||
assertNotNull(failed);
|
||||
assertTrue(failed.isEmpty(), "查询失败必须返回空结果而不是抛错阻断组装");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
||||
|
||||
List<byte[]> recovered = invokeLookupBatch(service, urls);
|
||||
assertEquals(1, recovered.size());
|
||||
assertEquals(5, recovered.get(0)[0], "依赖恢复后重试成功");
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_single_lookup_legacy_compat() throws Exception {
|
||||
// 兼容性:单 URL 旧入口 lookup 保持返回字节语义;Task 15 起 touch 改为
|
||||
// 异步批量缓冲,flush 后批量 touch 一次,命中才入缓冲。
|
||||
String url = "https://img.example.com/legacy.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
byte[] bytes = new byte[]{6};
|
||||
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(bytes);
|
||||
|
||||
byte[] result = service.lookup(url);
|
||||
assertEquals(bytes, result, "lookup 命中必须返回缓存字节");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
|
||||
assertNull(service.lookup("https://img.example.com/missing.jpg"), "未命中返回 null");
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
}
|
||||
-167
@@ -1,167 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper;
|
||||
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 本地验证入口:用生产真实批次数据 + 生产 LLM 网关跑 SimilarAsinLlmService 完整链路。
|
||||
* 用法:mvn compile test-compile 后执行
|
||||
* java -cp target/classes;target/test-classes;$(cat cp.txt) com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]
|
||||
*/
|
||||
public class SimilarAsinLlmLocalVerify {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
System.setOut(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.out), true, "UTF-8"));
|
||||
System.setErr(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.err), true, "UTF-8"));
|
||||
if (args.length < 2) {
|
||||
System.err.println("usage: SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]");
|
||||
System.exit(1);
|
||||
}
|
||||
String dataFile = args[0];
|
||||
String apiKey = args[1];
|
||||
boolean imgSwitch = args.length > 2 && Boolean.parseBoolean(args[2]);
|
||||
boolean categorySwitch = args.length > 3 && Boolean.parseBoolean(args[3]);
|
||||
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
JsonNode root = objectMapper.readTree(new File(dataFile));
|
||||
List<SimilarAsinResultRowDto> rows = new ArrayList<>();
|
||||
if (root.isArray()) {
|
||||
for (JsonNode node : root) {
|
||||
rows.add(fromJson(objectMapper, node));
|
||||
}
|
||||
} else {
|
||||
rows.add(fromJson(objectMapper, root));
|
||||
}
|
||||
System.out.println("[verify] loaded rows=" + rows.size() + " imgSwitch=" + imgSwitch
|
||||
+ " categorySwitch=" + categorySwitch);
|
||||
if (imgSwitch && categorySwitch) {
|
||||
System.out.println("[verify] raw first row alibaba[0].url="
|
||||
+ (rows.isEmpty() || rows.get(0).getAlibaba().isEmpty() ? "null"
|
||||
: rows.get(0).getAlibaba().get(0).getUrl()));
|
||||
}
|
||||
|
||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
||||
props.setLlmApiKey(apiKey);
|
||||
props.setLlmRowConcurrency(2);
|
||||
props.setLlmImageDownloadTimeoutSeconds(10);
|
||||
|
||||
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null);
|
||||
OssProperties ossProps = new OssProperties();
|
||||
ossProps.setEndpoint("https://oss.aishufu.top");
|
||||
ossProps.setPublicEndpoint("https://oss.aishufu.top");
|
||||
ossProps.setBucket("nanri-ai-images");
|
||||
ossProps.setAccessKeyId("appuser");
|
||||
ossProps.setAccessKeySecret("AppUser@2024SecureKey");
|
||||
OssStorageService oss = new OssStorageService(ossProps);
|
||||
PuzzleImageMerger merger = new PuzzleImageMerger(props);
|
||||
|
||||
// 生产真实类目数据(导出自 biz_product_category),spy 类目服务按 parentId 过滤。
|
||||
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> categories = loadCategories(objectMapper);
|
||||
ProductCategoryService categoryService = Mockito.spy(new ProductCategoryService(
|
||||
Mockito.mock(com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper.class)));
|
||||
Mockito.doAnswer(invocation -> {
|
||||
Long parentId = invocation.getArgument(0);
|
||||
List<com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo> items = categories.stream()
|
||||
.filter(c -> parentId == null ? c.getParentId() == null : parentId.equals(c.getParentId()))
|
||||
.map(c -> {
|
||||
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo item =
|
||||
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo();
|
||||
item.setId(c.getId());
|
||||
item.setParentId(c.getParentId());
|
||||
item.setName(c.getName());
|
||||
item.setCategoryKey(c.getCategoryKey());
|
||||
return item;
|
||||
})
|
||||
.toList();
|
||||
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo vo =
|
||||
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo();
|
||||
vo.setItems(items);
|
||||
vo.setTree(List.of());
|
||||
vo.setTotal((long) items.size());
|
||||
vo.setPage(1L);
|
||||
vo.setPageSize((long) Math.max(1, items.size()));
|
||||
vo.setHasMore(false);
|
||||
return vo;
|
||||
}).when(categoryService).children(Mockito.any(), Mockito.anyLong(), Mockito.anyLong());
|
||||
|
||||
SimilarAsinLlmService service = new SimilarAsinLlmService(
|
||||
client, props, ossProps, categoryService, merger, oss);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(rows, null, apiKey, imgSwitch, categorySwitch);
|
||||
long elapsed = System.currentTimeMillis() - start;
|
||||
System.out.println("[verify] done rows=" + result.size() + " elapsedMs=" + elapsed);
|
||||
for (SimilarAsinResultRowDto row : result) {
|
||||
System.out.println(String.format(
|
||||
"asin=%s | status=%s | isConform=%s | category=%s | reason=%s | isStock=%s | similarity=%s | mainUrl=%s | puzzle1=%s | puzzle2=%s",
|
||||
row.getAsin(), row.getStatus(), row.getIsConform(), row.getCategory(),
|
||||
row.getReason(), row.getIsStock(), row.getSimilarity(),
|
||||
shorten(row.getMainUrl()), shorten(row.getPuzzleImg1()), shorten(row.getPuzzleImg2())));
|
||||
}
|
||||
}
|
||||
|
||||
private static List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> loadCategories(ObjectMapper objectMapper) throws Exception {
|
||||
com.fasterxml.jackson.databind.JsonNode root = objectMapper.readTree(
|
||||
SimilarAsinLlmLocalVerify.class.getResourceAsStream("/categories.json"));
|
||||
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> list = new ArrayList<>();
|
||||
for (com.fasterxml.jackson.databind.JsonNode node : root) {
|
||||
com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity entity =
|
||||
new com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity();
|
||||
entity.setId(node.get("id").asLong());
|
||||
if (!node.get("parent_id").isNull()) {
|
||||
entity.setParentId(node.get("parent_id").asLong());
|
||||
}
|
||||
entity.setName(node.get("name").asText());
|
||||
entity.setCategoryKey(node.get("category_key").asText());
|
||||
entity.setSortOrder(node.get("sort_order").isNull() ? null : node.get("sort_order").asInt());
|
||||
entity.setDescription(node.get("description").isNull() ? null : node.get("description").asText());
|
||||
entity.setIsBuiltin(node.get("is_builtin") != null && node.get("is_builtin").asBoolean());
|
||||
list.add(entity);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto fromJson(ObjectMapper objectMapper, JsonNode node) throws Exception {
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin(text(node, "asin"));
|
||||
row.setTitle(text(node, "title"));
|
||||
row.setSku(text(node, "sku"));
|
||||
row.setCountry(text(node, "country"));
|
||||
row.setUrl(text(node, "url"));
|
||||
JsonNode alibaba = node.get("alibaba");
|
||||
if (alibaba != null && alibaba.isArray()) {
|
||||
List<SimilarAsinResultRowDto.AlibabaItem> items = new ArrayList<>();
|
||||
for (JsonNode item : alibaba) {
|
||||
items.add(objectMapper.treeToValue(item, SimilarAsinResultRowDto.AlibabaItem.class));
|
||||
}
|
||||
row.setAlibaba(items);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
private static String text(JsonNode node, String field) {
|
||||
JsonNode value = node.get(field);
|
||||
return value == null || value.isNull() ? null : value.asText();
|
||||
}
|
||||
|
||||
private static String shorten(String value) {
|
||||
if (value == null) {
|
||||
return "null";
|
||||
}
|
||||
return value.length() <= 70 ? value : value.substring(0, 70) + "...";
|
||||
}
|
||||
}
|
||||
-279
@@ -1,279 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo;
|
||||
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo;
|
||||
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import org.mockito.Mockito;
|
||||
|
||||
class SimilarAsinLlmServiceTest {
|
||||
|
||||
private static byte[] jpegBytes() {
|
||||
try {
|
||||
BufferedImage image = new BufferedImage(10, 10, BufferedImage.TYPE_INT_RGB);
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", baos);
|
||||
return baos.toByteArray();
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private SimilarAsinProperties properties() {
|
||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
||||
props.setLlmApiKey("test-key");
|
||||
return props;
|
||||
}
|
||||
|
||||
private SimilarAsinLlmService service(SimilarAsinLlmClient llmClient) {
|
||||
OssStorageService ossStorage = mock(OssStorageService.class);
|
||||
when(ossStorage.getPublicUrl(anyString())).thenAnswer(invocation -> "https://oss.aishufu.top/nanri-ai-images/" + invocation.getArgument(0));
|
||||
PuzzleImageMerger merger = mock(PuzzleImageMerger.class);
|
||||
when(merger.merge(anyList(), Mockito.<SimilarAsinResultRowDto>any())).thenReturn(jpegBytes());
|
||||
ProductCategoryService categoryService = mock(ProductCategoryService.class);
|
||||
when(categoryService.children(any(), anyLong(), anyLong()))
|
||||
.thenAnswer(invocation -> {
|
||||
Long parentId = invocation.getArgument(0);
|
||||
if (parentId == null) {
|
||||
return categoryPage("类目A", 1L);
|
||||
}
|
||||
if (parentId == 1L) {
|
||||
return categoryPage("类目B", 2L);
|
||||
}
|
||||
if (parentId == 2L) {
|
||||
return categoryPage("类目C", 3L);
|
||||
}
|
||||
return emptyCategoryPage();
|
||||
});
|
||||
SimilarAsinLlmService svc = new SimilarAsinLlmService(
|
||||
llmClient,
|
||||
properties(),
|
||||
mock(OssProperties.class),
|
||||
categoryService,
|
||||
merger,
|
||||
ossStorage);
|
||||
svc.setDownloadHttpClientForTest(mockHttpClient());
|
||||
return svc;
|
||||
}
|
||||
|
||||
private static ProductCategoryListVo categoryPage(String name, long id) {
|
||||
ProductCategoryItemVo item = new ProductCategoryItemVo();
|
||||
item.setId(id);
|
||||
item.setName(name);
|
||||
ProductCategoryListVo vo = new ProductCategoryListVo();
|
||||
vo.setItems(List.of(item));
|
||||
vo.setTotal(1L);
|
||||
vo.setPage(1L);
|
||||
vo.setPageSize(1L);
|
||||
vo.setHasMore(false);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static ProductCategoryListVo emptyCategoryPage() {
|
||||
ProductCategoryListVo vo = new ProductCategoryListVo();
|
||||
vo.setItems(List.of());
|
||||
vo.setTotal(0L);
|
||||
vo.setPage(1L);
|
||||
vo.setPageSize(1L);
|
||||
vo.setHasMore(false);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static HttpClient mockHttpClient() {
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
try {
|
||||
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
byte[] body = jpegBytes();
|
||||
HttpRequest request = invocation.getArgument(0);
|
||||
return new TestHttpResponse(200, body, request);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
private static final class TestHttpResponse implements HttpResponse<byte[]> {
|
||||
private final int statusCode;
|
||||
private final byte[] body;
|
||||
private final HttpRequest request;
|
||||
|
||||
TestHttpResponse(int statusCode, byte[] body, HttpRequest request) {
|
||||
this.statusCode = statusCode;
|
||||
this.body = body;
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int statusCode() {
|
||||
return statusCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpRequest request() {
|
||||
return request;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.util.Optional<HttpResponse<byte[]>> previousResponse() {
|
||||
return java.util.Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.http.HttpHeaders headers() {
|
||||
return java.net.http.HttpHeaders.of(Map.of(), (a, b) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] body() {
|
||||
return body;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.URI uri() {
|
||||
return java.net.URI.create("http://test");
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.http.HttpClient.Version version() {
|
||||
return HttpClient.Version.HTTP_1_1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.util.Optional<javax.net.ssl.SSLSession> sslSession() {
|
||||
return java.util.Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private SimilarAsinLlmClient llmClient(String apiKey, Map<String, String> responses) {
|
||||
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
|
||||
when(client.resolveApiKey(anyString())).thenReturn(apiKey == null ? "" : apiKey);
|
||||
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
|
||||
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String response = responses.get("chat");
|
||||
if (response == null) {
|
||||
throw new IllegalStateException("unexpected chat call");
|
||||
}
|
||||
return response;
|
||||
});
|
||||
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String response = responses.get("images");
|
||||
if (response == null) {
|
||||
throw new IllegalStateException("unexpected images call");
|
||||
}
|
||||
return response;
|
||||
});
|
||||
return client;
|
||||
}
|
||||
|
||||
private static final com.fasterxml.jackson.databind.ObjectMapper OBJECT_MAPPER =
|
||||
new com.fasterxml.jackson.databind.ObjectMapper();
|
||||
|
||||
/** 类目匹配(按序号返回不同类目名)与合规检查(后续)返回不同 JSON。 */
|
||||
private SimilarAsinLlmClient llmClientStaged(String apiKey, List<String> categoryJsons,
|
||||
String conformJson, String imagesJson) {
|
||||
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
|
||||
int[] categoryIndex = {0};
|
||||
when(client.resolveApiKey(anyString())).thenReturn(apiKey);
|
||||
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
|
||||
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
if (categoryIndex[0] < categoryJsons.size()) {
|
||||
return categoryJsons.get(categoryIndex[0]++);
|
||||
}
|
||||
return conformJson;
|
||||
});
|
||||
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> imagesJson);
|
||||
when(client.parseJsonContent(anyString()))
|
||||
.thenAnswer(invocation -> parseJson(invocation.getArgument(0)));
|
||||
return client;
|
||||
}
|
||||
|
||||
private static com.fasterxml.jackson.databind.JsonNode parseJson(String content) {
|
||||
try {
|
||||
return OBJECT_MAPPER.readTree(content);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void noApiKeyKeepsRawRows() {
|
||||
SimilarAsinLlmService service = service(llmClient("", Map.of()));
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin("B0TEST");
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "", true, true);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("B0TEST", result.get(0).getAsin());
|
||||
assertNull(result.get(0).getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
void categorySwitchOffOnlyPreparesImagesAndMarksNotExistsWhenNoMainUrl() {
|
||||
SimilarAsinLlmService service = service(llmClient("k", Map.of()));
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin("B0TEST");
|
||||
row.setTitle("Test product");
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", false, false);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("不存在", result.get(0).getStatus());
|
||||
assertNull(result.get(0).getIsConform());
|
||||
assertNull(result.get(0).getPuzzleImg1());
|
||||
}
|
||||
|
||||
@Test
|
||||
void imageCompareStopsOnStockAndFillsFields() {
|
||||
// 前 2 次 chat:一级/二级类目匹配返回名称(Java 按名回查候选取真实 ID);第 3 次:合规检查。
|
||||
SimilarAsinLlmClient client = llmClientStaged("k", List.of("{\"name\":\"类目A\"}", "{\"name\":\"类目B\"}"),
|
||||
"{\"asin\":\"B0TEST\",\"is_conform\":\"符合\",\"reason\":\"无\",\"category\":\"类目A->类目B->类目C\"}",
|
||||
"{\"asin\":\"B0TEST\",\"is_stock\":\"有货\",\"similarity\":\"95%\",\"status\":\"成功\",\"is_conform\":\"符合\",\"category\":\"类目A->类目B->类目C\"}");
|
||||
SimilarAsinLlmService service = service(client);
|
||||
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin("B0TEST");
|
||||
row.setTitle("Test product");
|
||||
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
|
||||
SimilarAsinResultRowDto.AlibabaItem item = new SimilarAsinResultRowDto.AlibabaItem();
|
||||
item.setUrl("https://cbu01.alicdn.com/img/1.jpg");
|
||||
row.setAlibaba(List.of(item));
|
||||
|
||||
// 一级/二级都匹配(按名回查 ID),三级候选可用,合规符合 → 图片对比,有货即停。
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", true, true);
|
||||
assertEquals(1, result.size());
|
||||
SimilarAsinResultRowDto out = result.get(0);
|
||||
assertEquals("成功", out.getStatus());
|
||||
assertEquals("有货", out.getIsStock());
|
||||
assertEquals("95%", out.getSimilarity());
|
||||
assertEquals("符合", out.getIsConform());
|
||||
}
|
||||
}
|
||||
-204
@@ -1,204 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 9:chunk 查询从单行分页改为批量 keyset 分页,保持低内存读取。
|
||||
* loadChunksKeyset 按 id 递增分批拉取(每批 pageSize),最后按 chunkIndex 升序合并,
|
||||
* 避免超大任务一次 selectList 全量载入 chunk 元数据。
|
||||
* mock 分页由 wrapper 中 gt("id", lastId) 的 keyset 值驱动,保证重复调用可复现(幂等)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceChunkKeysetTest {
|
||||
|
||||
private static final String MODULE = "similar-asin";
|
||||
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
|
||||
private static TaskChunkEntity chunk(long id, int chunkIndex) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7004L);
|
||||
chunk.setModuleType(MODULE);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static List<TaskChunkEntity> chunks(long... idsAndIndexes) {
|
||||
List<TaskChunkEntity> result = new ArrayList<>();
|
||||
for (int i = 0; i < idsAndIndexes.length; i += 2) {
|
||||
result.add(chunk(idsAndIndexes[i], (int) idsAndIndexes[i + 1]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 从 wrapper 的 SQL 片段解析 keyset:匹配 "id > #{ew.paramNameValuePairs.键}" 后从参数表取值。 */
|
||||
private static long keysetOf(QueryWrapper<TaskChunkEntity> wrapper) {
|
||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||
.compile("id\\s*>\\s*#\\{ew\\.paramNameValuePairs\\.(\\w+)\\}", java.util.regex.Pattern.CASE_INSENSITIVE)
|
||||
.matcher(wrapper.getSqlSegment());
|
||||
if (m.find()) {
|
||||
Object value = wrapper.getParamNameValuePairs().get(m.group(1));
|
||||
if (value instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
}
|
||||
return 0L;
|
||||
}
|
||||
|
||||
/** 按 keyset 驱动分页:每次 selectList 返回 id > keyset 的下一批,天然支持重复调用。 */
|
||||
private void stubKeysetPages(List<TaskChunkEntity> all, int pageSize) {
|
||||
int batch = pageSize > 0 ? pageSize : 500;
|
||||
doAnswer(invocation -> {
|
||||
long lastId = keysetOf(invocation.getArgument(0));
|
||||
return all.stream().filter(c -> c.getId() > lastId).limit(batch).toList();
|
||||
}).when(taskChunkMapper).selectList(any());
|
||||
}
|
||||
|
||||
private List<Long> keysetIdsFromRounds(int rounds) {
|
||||
ArgumentCaptor<QueryWrapper<TaskChunkEntity>> captor =
|
||||
ArgumentCaptor.forClass(QueryWrapper.class);
|
||||
verify(taskChunkMapper, times(rounds)).selectList(captor.capture());
|
||||
List<Long> keysets = new ArrayList<>();
|
||||
for (QueryWrapper<TaskChunkEntity> wrapper : captor.getAllValues()) {
|
||||
keysets.add(keysetOf(wrapper));
|
||||
}
|
||||
return keysets;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_normal_default_path() {
|
||||
// 正常输入:chunk 数小于 pageSize,一轮拉完,结果全且按 chunkIndex 有序
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
|
||||
stubKeysetPages(all, 500);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
assertEquals(3, result.size());
|
||||
assertEquals(List.of(1, 2, 3), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
||||
assertEquals(List.of(0L), keysetIdsFromRounds(1), "首轮 keyset 为 0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_normal_multiple_items() {
|
||||
// 超过 pageSize:多轮拉取,keyset 逐轮推进,全部合并且按 chunkIndex 升序、无重复
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7);
|
||||
stubKeysetPages(all, 3);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
||||
assertEquals(7, result.size());
|
||||
assertEquals(List.of(1, 2, 3, 4, 5, 6, 7), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
||||
long distinctIds = result.stream().map(TaskChunkEntity::getId).distinct().count();
|
||||
assertEquals(7, distinctIds, "keyset 分页不能产生重复 chunk");
|
||||
assertEquals(List.of(0L, 3L, 6L), keysetIdsFromRounds(3), "keyset 逐轮推进,不足一批即止");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行同一输入:每轮都从 keyset=0 开始,结果一致
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
|
||||
stubKeysetPages(all, 500);
|
||||
List<TaskChunkEntity> first = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
List<TaskChunkEntity> second = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getId(), second.get(i).getId());
|
||||
}
|
||||
assertEquals(List.of(0L, 0L), keysetIdsFromRounds(2), "重复执行每轮都从 keyset=0 开始");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_boundary_empty_input() {
|
||||
// 空集合:返回空列表,不创建无效资源,且只查一轮
|
||||
stubKeysetPages(List.of(), 500);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
assertNotNull(result);
|
||||
assertEquals(0, result.size());
|
||||
verify(taskChunkMapper, times(1)).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_boundary_single_item() {
|
||||
// 单 chunk:一轮返回后 keyset 推进即拉空,不依赖批量路径
|
||||
List<TaskChunkEntity> all = chunks(42, 9);
|
||||
stubKeysetPages(all, 1);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 1);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals(9, result.get(0).getChunkIndex());
|
||||
assertEquals(42L, result.get(0).getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_boundary_limit_and_overflow() {
|
||||
// chunk 数恰好等于 pageSize 的倍数:最后一轮仍返回非空才继续,全部取回
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6);
|
||||
stubKeysetPages(all, 3);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
||||
assertEquals(6, result.size());
|
||||
// pageSize 为 0/负数:回退默认 500,不抛异常
|
||||
stubKeysetPages(all, 0);
|
||||
List<TaskChunkEntity> fallback = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 0);
|
||||
assertEquals(6, fallback.size());
|
||||
stubKeysetPages(all, -5);
|
||||
List<TaskChunkEntity> negative = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, -5);
|
||||
assertEquals(6, negative.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_invalid_input_rejected() {
|
||||
// taskId 为 null:安全返回空列表,不发起查询
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, null, MODULE, 500);
|
||||
assertNotNull(result);
|
||||
assertEquals(0, result.size());
|
||||
verify(taskChunkMapper, times(0)).selectList(any());
|
||||
// mapper 查询抛异常:转项目约定异常,消息可识别
|
||||
when(taskChunkMapper.selectList(any())).thenThrow(new IllegalStateException("db down"));
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"异常消息必须可识别,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_dependency_failure_releases_resources() {
|
||||
// 第二轮查询失败:抛异常不返回半截结果;恢复后重试可完整返回
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5);
|
||||
doAnswer(invocation -> {
|
||||
long lastId = keysetOf(invocation.getArgument(0));
|
||||
if (lastId == 0L) {
|
||||
return all.subList(0, 3);
|
||||
}
|
||||
if (lastId == 3L) {
|
||||
throw new IllegalStateException("db down mid-page");
|
||||
}
|
||||
return all.subList(3, 5);
|
||||
}).when(taskChunkMapper).selectList(any());
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"异常消息必须可识别,实际: " + ex.getMessage());
|
||||
// 恢复后重试成功:5 个 chunk 全部取回
|
||||
stubKeysetPages(all, 3);
|
||||
List<TaskChunkEntity> recovered = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
||||
assertEquals(5, recovered.size());
|
||||
assertEquals(List.of(1, 2, 3, 4, 5), recovered.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
||||
}
|
||||
}
|
||||
-399
@@ -1,399 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 13:chunk 合并增加单次最大行数与 payload 字节上限。
|
||||
* 校验点位于 mergeChunkPayload 单次合并入口:合并后总行数超过 chunkMergeMaxRows、
|
||||
* 或 payload 字节超过 chunkMergePayloadMaxBytes 时,从最旧行开始降级到
|
||||
* orphan 兜底(assemble 阶段 putIfAbsent 合并回结果,不丢数据);
|
||||
* 单行本身超过字节上限时抛可识别异常拒绝合并。低于上限的行为与旧路径完全一致。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceChunkMergeLimitTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(90000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
|
||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(16L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/90000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String asin, String title) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(rowToken);
|
||||
r.setAsin(asin);
|
||||
r.setCountry("英国");
|
||||
r.setTitle(title);
|
||||
return r;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(9004L);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
/** 记录每次 storeChunkPayloadVersioned 收到的 payload 字符串。 */
|
||||
private void stubChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicReference<String> storedPayload) throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
|
||||
.thenReturn(payloadJson);
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storedPayload.set(invocation.getArgument(4));
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task,
|
||||
String scopeHash, Integer chunkIndex,
|
||||
List<SimilarAsinResultRowDto> llmRows) throws Exception {
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, scopeHash, chunkIndex, llmRows, Map.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_normal_default_path() throws Exception {
|
||||
// 正常输入:行数与 payload 字节均在上限内,合并走原路径,结果完整保留。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicReference<String> storedPayload = new AtomicReference<>("");
|
||||
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(
|
||||
row("r1", "B0A0000001", "标题1"),
|
||||
row("r2", "B0A0000002", "标题2"),
|
||||
row("r3", "B0A0000003", "标题3"));
|
||||
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertNotNull(storedPayload.get());
|
||||
assertTrue(storedPayload.get().contains("\"r1\"") && storedPayload.get().contains("\"r3\""),
|
||||
"上限内合并必须完整保留存量行与新增行,实际: " + storedPayload.get());
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多 chunk 一次 merge,全部在上限内,各 chunk 分别写回、结果不丢失。
|
||||
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "B0A0000001", "标题1"))));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r2", "B0A0000002", "标题2"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
|
||||
AtomicLong selectOneRound = new AtomicLong(0);
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
|
||||
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMerge(service, task, null, null, List.of(
|
||||
row("r1", "B0A0000001", "标题1-新"),
|
||||
row("r2", "B0A0000002", "标题2-新")));
|
||||
|
||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, times(2)).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:每次 merge 恰好写一次,不产生重复对象、重复状态。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicLong storeCalls = new AtomicLong(0);
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storeCalls.incrementAndGet();
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "B0A0000001", "标题1"));
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertEquals(2, storeCalls.get(), "重复执行同一输入:每次 merge 恰好写回一次,无多余请求");
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空列表安全跳过,不读取 chunk、不写存储、不创建资源。
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMerge(service, task, "hashA", 1, null);
|
||||
invokeMerge(service, task, "hashA", 1, List.of());
|
||||
verify(taskChunkMapper, times(0)).selectList(any());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_boundary_single_item() throws Exception {
|
||||
// 单行:不依赖批量路径,合并后结果正确。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicReference<String> storedPayload = new AtomicReference<>("");
|
||||
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
|
||||
assertTrue(storedPayload.get().contains("\"r1\""), "单行合并也必须写回 chunk payload,实际: " + storedPayload.get());
|
||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_boundary_limit_and_overflow() throws Exception {
|
||||
// 超限场景三连:行数超限降级、字节超限降级、单行超字节上限拒绝。
|
||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(2);
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicLong storeCalls = new AtomicLong(0);
|
||||
AtomicReference<String> lastStoredPayload = new AtomicReference<>("");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storeCalls.incrementAndGet();
|
||||
lastStoredPayload.set(invocation.getArgument(4));
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
llmRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
|
||||
}
|
||||
|
||||
// Phase A:行数超限(上限 2,存量 1 + 新增 5)→ 只保留上限内最新行,超限部分转 orphan。
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertTrue(lastStoredPayload.get().contains("\"r4\"") && lastStoredPayload.get().contains("\"r5\""),
|
||||
"行数超限时保留上限内的最新行,实际: " + lastStoredPayload.get());
|
||||
assertFalse(lastStoredPayload.get().contains("\"r0\""), "行数超限时最旧行被降级,实际: " + lastStoredPayload.get());
|
||||
assertEquals(1, storeCalls.get());
|
||||
verify(taskScopeStateMapper, times(1)).insert(any(TaskScopeStateEntity.class));
|
||||
verify(transientPayloadStorageService, times(1))
|
||||
.storeParsedPayloadEntry(anyString(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
|
||||
// Phase B:字节超限(行数放开)→ 从最旧行降级到字节上限内,保留最新结果。
|
||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
|
||||
long oneRowBytes = rowsJson(List.of(row("r9", "B0A0000099", "样本行"))).getBytes(StandardCharsets.UTF_8).length;
|
||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(oneRowBytes + 5L);
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertTrue(lastStoredPayload.get().contains("\"r5\""), "字节超限时保留最新行,实际: " + lastStoredPayload.get());
|
||||
assertFalse(lastStoredPayload.get().contains("\"r0\""), "字节超限时最旧行被降级,实际: " + lastStoredPayload.get());
|
||||
assertEquals(2, storeCalls.get());
|
||||
verify(taskScopeStateMapper, times(2)).insert(any(TaskScopeStateEntity.class));
|
||||
|
||||
// Phase C:单行本身超过字节上限 → 抛可识别异常拒绝合并,不写 chunk。
|
||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(5L);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertNotNull(ex.getMessage());
|
||||
assertTrue(ex.getMessage().contains("字节上限"),
|
||||
"超字节上限必须抛可识别异常,实际: " + ex.getMessage());
|
||||
assertEquals(2, storeCalls.get(), "拒绝合并时不写 chunk");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_invalid_input_rejected() {
|
||||
// 非法输入:chunk 载荷加载失败(resolve 抛异常)时抛出可识别异常且不写 chunk。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenThrow(new IllegalStateException("rustfs down"));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertNotNull(ex.getMessage());
|
||||
assertTrue(ex.getMessage().contains("chunk"),
|
||||
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:payload 存储失败时抛带上下文的可识别异常、无残留状态;恢复后重试成功。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||
AtomicLong storeCalls = new AtomicLong(0);
|
||||
doAnswer(invocation -> {
|
||||
if (storeCalls.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("rustfs down");
|
||||
}
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("相似ASIN分片载荷"),
|
||||
"存储失败必须抛带上下文的可识别异常,实际: " + ex.getMessage());
|
||||
assertEquals(1, storeCalls.get(), "失败时只尝试一次即抛出,不静默吞错");
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
|
||||
// 依赖恢复后重试成功:结果正确、无残留状态。
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
assertEquals(2, storeCalls.get(), "恢复后重试成功");
|
||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
}
|
||||
-416
@@ -1,416 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 12:扩展 Coze 结果缓冲覆盖范围,减少频繁读写完整 chunk payload。
|
||||
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1";Task 12 扩展为:
|
||||
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
|
||||
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
|
||||
* 3) 统一走 bufferLlmRowsOrMerge:缓冲失败回退立即 merge,结果不丢失。
|
||||
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceCozeBufferScopeTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
|
||||
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
|
||||
private static final String CREDENTIAL = "cred-1";
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinLlmService similarAsinLlmService;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(properties.getLlmBatchSize()).thenReturn(5);
|
||||
lenient().when(properties.getLlmTextOnlyBatchSize()).thenReturn(10);
|
||||
lenient().when(properties.isLlmResultBufferEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getDbJobTouchIntervalMillis()).thenReturn(2_000L);
|
||||
lenient().when(properties.getDbTaskTouchIntervalMillis()).thenReturn(2_000L);
|
||||
lenient().when(properties.getLlmFlushPendingMinutes()).thenReturn(10);
|
||||
lenient().when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(
|
||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
|
||||
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
|
||||
.thenAnswer(invocation -> "rustfs:coze-result/" + NEXT_ID.incrementAndGet());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country, String title) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(id);
|
||||
r.setAsin(asin);
|
||||
r.setCountry(country);
|
||||
r.setTitle(title);
|
||||
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
|
||||
return r;
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7104L);
|
||||
chunk.setModuleType(MODULE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
chunk.setPayloadHash("h-" + id);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7104L);
|
||||
task.setModuleType(MODULE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setResultJson("{\"categorySwitch\":true}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private static TaskScopeStateEntity state(FileTaskEntity task, long id, String status, int batchTotal) {
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setId(id);
|
||||
state.setTaskId(task.getId());
|
||||
state.setModuleType(MODULE);
|
||||
state.setScopeHash("scope-" + id);
|
||||
state.setLlmStatus(status);
|
||||
state.setParsedPayloadJson("ptr:batch-" + id);
|
||||
state.setStateJson("{\"jobId\":7101,\"resultId\":7201,\"chunkScopeHash\":null,\"chunkIndex\":null,"
|
||||
+ "\"batchIndex\":1,\"batchTotal\":" + batchTotal + ",\"ownerInstanceId\":\"test-instance\","
|
||||
+ "\"submitRetryCount\":0,\"credentialName\":\"" + CREDENTIAL + "\",\"resultPayloadPointer\":\"ptr:buffer-" + id + "\"}");
|
||||
return state;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private SimilarAsinTaskService.LlmBatchContext context(int batchTotal) {
|
||||
return new SimilarAsinTaskService.LlmBatchContext(
|
||||
7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null);
|
||||
}
|
||||
|
||||
private void stubChunkMerge(String payloadJson) throws Exception {
|
||||
TaskChunkEntity chunk = chunk(1L, "scope-1", 1, "ptr:chunk-1");
|
||||
// loadSubmittedChunks 只保留非空 chunk,chunk payload 必须能解析出至少一行。
|
||||
// 全部 lenient:缓冲成功路径不触达 merge,仅缓冲失败/flush 合并路径消费。
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return payloadJson;
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
private static String chunkRowsJson() throws Exception {
|
||||
// chunk-1 已含 r1 行:loadSubmittedChunks 只保留非空 chunk,且 rowKey 索引能命中缓冲行。
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_default_path() throws Exception {
|
||||
// 正常输入:DONE 结果(batchTotal=1 单 batch)经 bufferLlmRowsOrMerge 走缓冲,
|
||||
// 不立即写 chunk;缓冲失败回退立即 merge 结果不丢失。
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多个 DONE state(单 batch)全部缓冲;flush 后按 chunk 分组一次合并
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(
|
||||
List.of(state(task, 1L, "DONE", 1), state(task, 2L, "DONE", 1)));
|
||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return chunkRowsJson();
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(1L, "scope-1", 1, "ptr:chunk-1"));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1")));
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
|
||||
flush.setAccessible(true);
|
||||
flush.invoke(service, 7104L);
|
||||
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
||||
// pointer 清理:每个缓冲 state 都更新
|
||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:缓冲写幂等(同一 state 不产生重复 buffer/merge)
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
TaskScopeStateEntity state = state(task, 1L, "DONE", 2);
|
||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
||||
|
||||
// 缓冲 2 次(每次重新写 pointer 是幂等语义:同一 state 覆盖写,无重复行)
|
||||
verify(transientPayloadStorageService, times(2)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_empty_input() throws Exception {
|
||||
// 空输入:无行时缓冲与 merge 都不发生,不创建无效资源
|
||||
FileTaskEntity task = task();
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of());
|
||||
bufferOrMerge.invoke(service, state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_single_item() throws Exception {
|
||||
// 单 batch(batchTotal=1):原 P0-3 例外,现在也缓冲
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_limit_and_overflow() throws Exception {
|
||||
// 缓冲开关关闭:回退立即 merge,DONE 结果仍落 chunk 不丢失
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(properties.isLlmResultBufferEnabled()).thenReturn(false);
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_invalid_input_rejected() throws Exception {
|
||||
// 缓冲写失败(storeParsedPayloadEntry 抛异常):回退立即 merge,结果不丢失
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(transientPayloadStorageService.storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)))
|
||||
.thenThrow(new IllegalStateException("rustfs full"));
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_dependency_failure_releases_resources() throws Exception {
|
||||
// flush 时 chunk 写失败:抛可识别业务异常且不清 pointer(保留待重试);
|
||||
// 依赖恢复后重试 flush 成功,chunk 合并一次、pointer 清理。
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
TaskScopeStateEntity s = state(task, 1L, "DONE", 1);
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(s));
|
||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return chunkRowsJson();
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
java.util.concurrent.atomic.AtomicInteger storeCalls = new java.util.concurrent.atomic.AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (storeCalls.incrementAndGet() == 1) {
|
||||
throw new IllegalStateException("rustfs write failed");
|
||||
}
|
||||
return "ptr:stored-" + invocation.getArgument(3);
|
||||
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
|
||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
|
||||
flush.setAccessible(true);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
flush.invoke(service, 7104L);
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("刷新缓冲区"),
|
||||
"flush 失败消息必须可识别, 实际: " + ex.getMessage());
|
||||
// 失败分组不清 pointer:buffer 未被删除、stateJson 未更新,留待重试
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
verify(taskScopeStateMapper, never()).update(any(), any());
|
||||
|
||||
// 恢复后重试 flush:chunk 合并成功一次,pointer 清理
|
||||
flush.invoke(service, 7104L);
|
||||
assertEquals(2, storeCalls.get(), "恢复后重试应再次写 chunk");
|
||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
}
|
||||
-378
@@ -1,378 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 6:分组数据改为索引/范围引用,避免 groups 嵌套复制完整行对象。
|
||||
* 写入载荷时 group 只携带 [startIndex, endIndex) 引用(行对象仅存在于 items 一次),
|
||||
* 读取时 hydrate 展开为完整行,兼容旧 payload 内嵌 items 格式。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceGroupRefTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(30000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/30000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-group-ref-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0GRP%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("group-ref.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
private String storedPayloadJson() {
|
||||
// 捕获最近一次存储的 payload JSON
|
||||
return "rustfs:task-parsed/similar-asin/30000/payload.json";
|
||||
}
|
||||
|
||||
private SimilarAsinParsedPayloadDto readPayload(String json) throws Exception {
|
||||
return objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
}
|
||||
|
||||
private static SimilarAsinParsedRowVo row(String fileKey, int index, String groupKey) {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setSourceFileKey(fileKey);
|
||||
row.setSourceFilename("group-ref.xlsx");
|
||||
row.setRowIndex(index);
|
||||
row.setSourceId(String.valueOf(index));
|
||||
row.setDisplayId(String.valueOf(index));
|
||||
row.setRowToken(fileKey + "::row::" + index);
|
||||
row.setGroupKey(groupKey);
|
||||
row.setAsin(String.format("B0GRP%05d", index));
|
||||
row.setCountry("英国");
|
||||
row.setValues(new java.util.LinkedHashMap<>());
|
||||
return row;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_normal_default_path() throws Exception {
|
||||
// 正常多行文件:groups 写入为索引引用,行对象只出现在 items 一次
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenAnswer(invocation -> {
|
||||
String json = invocation.getArgument(3);
|
||||
return "rustfs:task-parsed/similar-asin/30000/payload.json::" + json;
|
||||
});
|
||||
File workbook = buildWorkbook(150);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-default.xlsx");
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
// 每个 group 是索引引用:携带 [startIndex, endIndex),区间宽度等于 itemCount
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertNotNull(group.getStartIndex());
|
||||
assertNotNull(group.getEndIndex());
|
||||
assertTrue(group.getStartIndex() < group.getEndIndex());
|
||||
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItemCount());
|
||||
}
|
||||
// 响应 groups 按预览上限裁剪(默认 100),引用区间覆盖全部行、不重叠
|
||||
int coverage = 0;
|
||||
int prevEnd = -1;
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertTrue(group.getStartIndex() >= prevEnd, "组区间不能重叠且必须顺序递增");
|
||||
coverage += group.getEndIndex() - group.getStartIndex();
|
||||
prevEnd = group.getEndIndex();
|
||||
}
|
||||
assertTrue(coverage <= 100 && coverage > 0, "预览组覆盖行数必须在 (0, 预览上限] 内,实际 " + coverage);
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
// 响应组内嵌预览行(前端兼容):每个组 items 与引用区间宽度一致
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertNotNull(group.getItems());
|
||||
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItems().size(),
|
||||
"响应组内嵌预览行数量必须与引用区间宽度一致");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_normal_multiple_items() throws Exception {
|
||||
// 多组批量:每组行数不同,引用与 items 严格对应且顺序稳定
|
||||
String json = groupRefJson(3, new int[][]{{0, 3}, {3, 8}, {8, 10}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
assertEquals(10, payload.getItems().size());
|
||||
assertEquals(3, payload.getGroups().size());
|
||||
for (int g = 0; g < payload.getGroups().size(); g++) {
|
||||
SimilarAsinParsedGroupVo group = payload.getGroups().get(g);
|
||||
int start = group.getStartIndex();
|
||||
int end = group.getEndIndex();
|
||||
assertTrue(end - start >= 1);
|
||||
// 展开后行与 items 对应(首行即 items[start],行内容一致)
|
||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(start, end));
|
||||
assertEquals(end - start, expanded.size());
|
||||
assertEquals("t" + (start + 1), expanded.get(0).getRowToken(), "展开首行必须是 items[start]");
|
||||
assertEquals("B0GRP" + String.format("%05d", start + 1), expanded.get(0).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复展开同一 payload:结果一致,且不修改 items
|
||||
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 5}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
List<SimilarAsinParsedRowVo> first = hydrateForTest(payload);
|
||||
List<SimilarAsinParsedRowVo> second = hydrateForTest(payload);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
}
|
||||
assertEquals(5, payload.getItems().size(), "展开不能修改 payload 内部状态");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_boundary_empty_input() throws Exception {
|
||||
// 空 groups:引用列表为空,不创建无效引用
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setItems(List.of());
|
||||
payload.setGroups(List.of());
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertNotNull(restored);
|
||||
assertEquals(0, restored.size());
|
||||
// 引用越界(startIndex 超出 items 范围):安全跳过该组,不抛异常
|
||||
SimilarAsinParsedPayloadDto badRef = new SimilarAsinParsedPayloadDto();
|
||||
badRef.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setGroupKey("f.xlsx::1");
|
||||
group.setStartIndex(5);
|
||||
group.setEndIndex(7);
|
||||
badRef.setGroups(List.of(group));
|
||||
List<SimilarAsinParsedRowVo> outOfRange = SimilarAsinTaskService.resolveAllRows(badRef);
|
||||
assertEquals(0, outOfRange.size(), "越界引用必须安全跳过");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_boundary_single_item() throws Exception {
|
||||
// 单行单组:区间为 [0,1),单行不依赖批量路径
|
||||
String json = groupRefJson(1, new int[][]{{0, 1}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
assertEquals(1, payload.getGroups().size());
|
||||
SimilarAsinParsedGroupVo group = payload.getGroups().get(0);
|
||||
assertEquals(0, group.getStartIndex());
|
||||
assertEquals(1, group.getEndIndex());
|
||||
assertEquals(1, group.getItemCount());
|
||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(0, 1));
|
||||
assertEquals(1, expanded.size());
|
||||
assertEquals("B0GRP00001", expanded.get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_boundary_limit_and_overflow() throws Exception {
|
||||
// 组引用到达 items 末尾:endIndex == items.size(),不越界
|
||||
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 6}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
assertEquals(6, payload.getItems().size());
|
||||
SimilarAsinParsedGroupVo last = payload.getGroups().get(1);
|
||||
assertEquals(6, last.getEndIndex());
|
||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(last.getStartIndex(), last.getEndIndex()));
|
||||
assertEquals(4, expanded.size());
|
||||
// 未携带 items 的旧 payload 走 allItems 兜底
|
||||
String legacy = "{\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0OLD00001\"}],"
|
||||
+ "\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":1,\"itemCount\":1}]}";
|
||||
SimilarAsinParsedPayloadDto legacyPayload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class);
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacyPayload);
|
||||
assertEquals(1, restored.size());
|
||||
assertEquals("B0OLD00001", restored.get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_invalid_input_rejected() throws Exception {
|
||||
// 非法区间:endIndex <= startIndex,安全跳过
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1"), row("f.xlsx", 2, "f.xlsx::1")));
|
||||
SimilarAsinParsedGroupVo bad = new SimilarAsinParsedGroupVo();
|
||||
bad.setGroupKey("f.xlsx::1");
|
||||
bad.setStartIndex(1);
|
||||
bad.setEndIndex(1);
|
||||
payload.setGroups(List.of(bad));
|
||||
assertEquals(0, SimilarAsinTaskService.resolveAllRows(payload).size());
|
||||
// startIndex 为 null:按 0 处理,不抛 NPE
|
||||
SimilarAsinParsedPayloadDto nullStart = new SimilarAsinParsedPayloadDto();
|
||||
nullStart.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
|
||||
SimilarAsinParsedGroupVo g = new SimilarAsinParsedGroupVo();
|
||||
g.setGroupKey("f.xlsx::1");
|
||||
g.setStartIndex(null);
|
||||
g.setEndIndex(1);
|
||||
nullStart.setGroups(List.of(g));
|
||||
assertEquals(1, SimilarAsinTaskService.resolveAllRows(nullStart).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_dependency_failure_releases_resources() throws Exception {
|
||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,groups 引用与行一致
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(80);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/gr-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/30001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/gr-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-recovered.xlsx");
|
||||
assertEquals(80, vo.getAcceptedRows());
|
||||
int coverage = 0;
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertTrue(group.getStartIndex() < group.getEndIndex());
|
||||
coverage += group.getEndIndex() - group.getStartIndex();
|
||||
}
|
||||
assertEquals(80, coverage);
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
private static String groupRefJson(int groupCount, int[][] ranges) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("{\"items\":[");
|
||||
// 计算总行数
|
||||
int maxEnd = 0;
|
||||
for (int[] r : ranges) {
|
||||
maxEnd = Math.max(maxEnd, r[1]);
|
||||
}
|
||||
for (int i = 1; i <= maxEnd; i++) {
|
||||
if (i > 1) {
|
||||
sb.append(",");
|
||||
}
|
||||
sb.append("{\"rowToken\":\"t").append(i).append("\",\"asin\":\"B0GRP")
|
||||
.append(String.format("%05d", i)).append("\",\"sourceFileKey\":\"f.xlsx\",\"rowIndex\":")
|
||||
.append(i).append("}");
|
||||
}
|
||||
sb.append("],\"groups\":[");
|
||||
for (int g = 0; g < groupCount; g++) {
|
||||
if (g > 0) {
|
||||
sb.append(",");
|
||||
}
|
||||
sb.append("{\"groupKey\":\"g").append(g + 1).append("\",\"startIndex\":")
|
||||
.append(ranges[g][0]).append(",\"endIndex\":").append(ranges[g][1])
|
||||
.append(",\"itemCount\":").append(ranges[g][1] - ranges[g][0]).append("}");
|
||||
}
|
||||
sb.append("]}");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static List<SimilarAsinParsedRowVo> hydrateForTest(SimilarAsinParsedPayloadDto payload) {
|
||||
// 调用 service 的引用展开实现(与 hydrateParsedPayloadRows 语义一致)
|
||||
SimilarAsinParsedPayloadDto copy = new SimilarAsinParsedPayloadDto();
|
||||
copy.setItems(payload.getItems());
|
||||
copy.setAllItems(payload.getAllItems());
|
||||
copy.setGroups(payload.getGroups());
|
||||
return SimilarAsinTaskService.expandGroupRefs(copy);
|
||||
}
|
||||
}
|
||||
-275
@@ -1,275 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 7:限制单文件大小、最大行数和最大字段长度,防止解析任务无界增长。
|
||||
* 超限输入在解析入口被拒绝或截断;mock 依赖 + 真实 xlsx 验证边界行为。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceParseLimitsTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(40000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/40000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
return buildWorkbookWithAsin(rowCount, null);
|
||||
}
|
||||
|
||||
private File buildWorkbookWithAsin(int rowCount, String asinValue) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-parse-limits-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(asinValue != null ? asinValue : String.format("B0LIM%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("limits.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_normal_default_path() throws Exception {
|
||||
// 默认配置(50MB/50000 行/2000 字符):正常文件解析成功,行数不丢失
|
||||
File workbook = buildWorkbook(120);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-default.xlsx");
|
||||
assertEquals(120, vo.getAcceptedRows());
|
||||
assertEquals(120, vo.getTotalRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
|
||||
// 源文件大小在限制内
|
||||
assertTrue(workbook.length() <= 50L * 1024L * 1024L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_normal_multiple_items() throws Exception {
|
||||
// 多文件批量:每个文件都在限制内,汇总不丢行
|
||||
File workbookA = buildWorkbook(30);
|
||||
File workbookB = buildWorkbook(40);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-a.xlsx")).thenReturn(workbookA);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-b.xlsx")).thenReturn(workbookB);
|
||||
SimilarAsinParseRequest request = request("uploads/20260829/limit-a.xlsx");
|
||||
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
|
||||
sourceB.setFileKey("uploads/20260829/limit-b.xlsx");
|
||||
sourceB.setOriginalFilename("limits-b.xlsx");
|
||||
request.setFiles(List.of(request.getFiles().get(0), sourceB));
|
||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
||||
assertEquals(70, vo.getAcceptedRows());
|
||||
assertEquals(70, vo.getTotalRows());
|
||||
assertNotNull(vo.getTaskId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复解析同一文件:结果一致,不产生重复状态
|
||||
File workbook = buildWorkbook(60);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/limit-idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/limit-idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_boundary_empty_input() throws Exception {
|
||||
// 空文件(无有效数据行):抛业务异常,不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-empty.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/limit-empty.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_boundary_single_item() throws Exception {
|
||||
// 单行小文件:不依赖批量路径,结果正确
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_boundary_limit_and_overflow() throws Exception {
|
||||
// 行数恰好等于上限:允许
|
||||
when(properties.getMaxParseRows()).thenReturn(8);
|
||||
File workbook = buildWorkbook(8);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-exact.xlsx");
|
||||
assertEquals(8, vo.getAcceptedRows());
|
||||
// 行数超过上限:拒绝,且不创建任务
|
||||
when(properties.getMaxParseRows()).thenReturn(3);
|
||||
File workbookOver = buildWorkbook(4);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-over.xlsx")).thenReturn(workbookOver);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbookOver, "uploads/20260829/limit-over.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("行数"),
|
||||
"超行数异常消息必须可识别,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_invalid_input_rejected() throws Exception {
|
||||
// 文件大小超限:拒绝,异常消息可识别
|
||||
when(properties.getMaxSourceFileBytes()).thenReturn(64L);
|
||||
File workbook = buildWorkbook(5);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-bigfile.xlsx")).thenReturn(workbook);
|
||||
assertTrue(workbook.length() > 64L, "测试文件必须超过 64 字节限制,实际 " + workbook.length());
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/limit-bigfile.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("大小限制"),
|
||||
"超文件大小异常消息必须可识别,实际: " + ex.getMessage());
|
||||
// 字段长度超限:截断而非拒绝,字段仍非空
|
||||
when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
when(properties.getMaxFieldLength()).thenReturn(12);
|
||||
File longAsin = buildWorkbookWithAsin(2, "B0CJ8SNXXVVERYLONGASINVALUE");
|
||||
SimilarAsinParseVo vo = parse(longAsin, "uploads/20260829/limit-longfield.xlsx");
|
||||
assertEquals(2, vo.getAcceptedRows());
|
||||
for (var item : vo.getItems()) {
|
||||
assertTrue(item.getAsin().length() <= 12, "超长字段必须截断到配置上限");
|
||||
assertTrue(!item.getAsin().isBlank());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_dependency_failure_releases_resources() throws Exception {
|
||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,无残留状态
|
||||
File workbook = buildWorkbook(40);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/40001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/limit-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-recovered.xlsx");
|
||||
assertEquals(40, vo.getAcceptedRows());
|
||||
assertEquals(40, vo.getItems().size());
|
||||
// 行数/文件大小/字段长度默认值均处于有效区间
|
||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
||||
assertNotNull(defaults.getMaxParseRows());
|
||||
assertNotNull(defaults.getMaxSourceFileBytes());
|
||||
assertNotNull(defaults.getMaxFieldLength());
|
||||
assertTrue(defaults.getMaxParseRows() >= 1000, "默认最大行数至少 1000");
|
||||
assertTrue(defaults.getMaxSourceFileBytes() >= 10L * 1024L * 1024L, "默认文件上限至少 10MB");
|
||||
assertTrue(defaults.getMaxFieldLength() >= 500, "默认字段上限至少 500 字符");
|
||||
}
|
||||
}
|
||||
-274
@@ -1,274 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 4:解析接口只返回固定数量预览行,完整行仅保存在后端任务载荷。
|
||||
* 通过 mock 依赖 + 真实 xlsx 文件验证 parseAndCreateTask 的响应裁剪行为。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceParsePreviewTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(10000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/10000/payload.json");
|
||||
// 插入任务时回填 id(异常路径不触发,标记 lenient)
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-parse-preview-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
header.createCell(3).setCellValue("价格");
|
||||
header.createCell(4).setCellValue("货号");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0TEST%04d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
row.createCell(3).setCellValue("12.29");
|
||||
row.createCell(4).setCellValue("SKU-" + i);
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("preview.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_default_path() throws Exception {
|
||||
// 150 行:响应只返回预览行(≤100),完整行不进入响应
|
||||
File workbook = buildWorkbook(150);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/preview.xlsx");
|
||||
assertNotNull(vo.getTaskId());
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
assertEquals(150, vo.getTotalRows());
|
||||
// 预览行固定 ≤ 100
|
||||
assertTrue(vo.getItems().size() <= 100, "响应 items 必须是固定数量预览行");
|
||||
assertEquals(vo.getItems().size(), 100);
|
||||
// 预览行顺序稳定:从第 1 行开始
|
||||
assertEquals("1", vo.getItems().get(0).getSourceId());
|
||||
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
|
||||
// groups 也裁剪为预览行(不携带全量子行)
|
||||
assertTrue(vo.getGroups().size() <= 100);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_multiple_items() throws Exception {
|
||||
// 5000 行大文件:响应预览行数量不随总行数增长
|
||||
File workbook = buildWorkbook(5000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/large.xlsx");
|
||||
assertEquals(5000, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertTrue(vo.getGroups().size() <= 100);
|
||||
// 预览行字段完整(asin/country/sku)
|
||||
assertEquals("英国", vo.getItems().get(0).getCountry());
|
||||
assertEquals("SKU-1", vo.getItems().get(0).getSku());
|
||||
// 后 4900 行不进入响应体
|
||||
boolean containsTail = vo.getItems().stream().anyMatch(item -> "B0TEST4900".equals(item.getAsin()));
|
||||
assertFalse(containsTail, "响应不能包含末尾行");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 同一文件重复解析:响应预览行一致,不产生重复状态
|
||||
File workbook = buildWorkbook(200);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_empty_input() throws Exception {
|
||||
// 空文件(只有表头无数据行):抛项目约定异常,不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/empty.xlsx")).thenReturn(workbook);
|
||||
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/empty.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_single_item() throws Exception {
|
||||
// 单行文件:预览行 = 完整行,不依赖批量路径
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_limit_and_overflow() throws Exception {
|
||||
// 行数恰好等于预览上限(100):全部返回
|
||||
File workbook = buildWorkbook(100);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/exact.xlsx");
|
||||
assertEquals(100, vo.getItems().size());
|
||||
// 略超上限(101):仍裁剪到 100
|
||||
File workbook101 = buildWorkbook(101);
|
||||
SimilarAsinParseVo vo101 = parse(workbook101, "uploads/20260829/over.xlsx");
|
||||
assertEquals(100, vo101.getItems().size());
|
||||
assertFalse(vo101.getItems().stream().anyMatch(item -> "B0TEST0101".equals(item.getAsin())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_invalid_input_rejected() throws Exception {
|
||||
// 文件不存在:抛业务异常
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/missing.xlsx")).thenReturn(null);
|
||||
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/missing.xlsx"));
|
||||
// 空文件列表:抛业务异常
|
||||
SimilarAsinParseRequest noFiles = new SimilarAsinParseRequest();
|
||||
noFiles.setUserId(7L);
|
||||
noFiles.setFiles(List.of());
|
||||
noFiles.setApiKey("sk");
|
||||
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(noFiles));
|
||||
// 非法 user_id
|
||||
SimilarAsinParseRequest badUser = request("uploads/20260829/preview.xlsx");
|
||||
badUser.setUserId(null);
|
||||
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(badUser));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_dependency_failure_releases_resources() throws Exception {
|
||||
// payload 存储失败:抛业务异常,不返回部分结果;随后恢复存储 mock 验证可重试
|
||||
File workbook = buildWorkbook(50);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/storefail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/10001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/storefail.xlsx"));
|
||||
// 存储恢复后,同一文件解析可正常完成
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/recovered.xlsx");
|
||||
assertEquals(50, vo.getAcceptedRows());
|
||||
assertEquals(50, vo.getItems().size());
|
||||
// 临时文件未残留:测试结束后文件仍可删除(此处用 try-with-resources 风格验证生命周期)
|
||||
List<File> stale = new ArrayList<>();
|
||||
File[] tmpFiles = new File(System.getProperty("java.io.tmpdir"))
|
||||
.listFiles((dir, name) -> name.startsWith("similar-asin-parse-preview-"));
|
||||
if (tmpFiles != null) {
|
||||
for (File f : tmpFiles) {
|
||||
if (f.exists()) {
|
||||
stale.add(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 测试用临时文件仅限本次测试创建的(外部残留不统计)
|
||||
for (File f : stale) {
|
||||
Files.deleteIfExists(f.toPath());
|
||||
}
|
||||
assertTrue(true);
|
||||
}
|
||||
}
|
||||
-223
@@ -1,223 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.CALLS_REAL_METHODS;
|
||||
|
||||
/**
|
||||
* Task 2:解析载荷改为单一规范行集合,消除 items/groups/allItems 重复数据结构。
|
||||
* 写入侧只输出 items(唯一全量行来源);旧 JSON 的 allItems 键反序列化时吸收到 items,行不丢失。
|
||||
*/
|
||||
class SimilarAsinTaskServicePayloadNormalizationTest {
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
|
||||
ReflectionTestUtils.setField(service, "objectMapper", MAPPER);
|
||||
}
|
||||
|
||||
private List<SimilarAsinParsedRowVo> rows(int count) {
|
||||
List<SimilarAsinParsedRowVo> result = new ArrayList<>();
|
||||
for (int i = 1; i <= count; i++) {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setSourceFileKey("uploads/20260829/base.xlsx");
|
||||
row.setSourceFilename("base.xlsx");
|
||||
row.setRowIndex(i);
|
||||
row.setSourceId(String.valueOf(i));
|
||||
row.setDisplayId(String.valueOf(i));
|
||||
row.setRowToken("uploads/20260829/base.xlsx::row::" + i);
|
||||
row.setAsin("B0CJ8SNXXV");
|
||||
row.setCountry("英国");
|
||||
row.setPrice("12.29");
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
values.put("id", String.valueOf(i));
|
||||
values.put("asin", "B0CJ8SNXXV");
|
||||
values.put("国家", "英国");
|
||||
values.put("价格", "12.29");
|
||||
row.setValues(values);
|
||||
result.add(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<SimilarAsinParsedGroupVo> groups(List<SimilarAsinParsedRowVo> rows) {
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setSourceFileKey("uploads/20260829/base.xlsx");
|
||||
group.setSourceFilename("base.xlsx");
|
||||
group.setGroupKey("uploads/20260829/base.xlsx::1@1");
|
||||
group.setBaseId("1");
|
||||
group.setDisplayId("1");
|
||||
group.setItemCount(rows.size());
|
||||
group.setItems(new ArrayList<>(rows));
|
||||
return List.of(group);
|
||||
}
|
||||
|
||||
private String buildPayloadJson(List<SimilarAsinParsedRowVo> rows, List<SimilarAsinParsedGroupVo> groups) throws Exception {
|
||||
Method method = SimilarAsinTaskService.class.getDeclaredMethod(
|
||||
"buildParsedPayloadJson",
|
||||
String.class, String.class, Boolean.class, Boolean.class,
|
||||
List.class, List.class, List.class, List.class);
|
||||
method.setAccessible(true);
|
||||
return (String) method.invoke(service,
|
||||
"请排查侵权风险", "sk-123", Boolean.TRUE, Boolean.FALSE,
|
||||
List.of(sourceFile()), List.of("id", "asin"), groups, rows);
|
||||
}
|
||||
|
||||
private SimilarAsinSourceFileDto sourceFile() {
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey("uploads/20260829/base.xlsx");
|
||||
sourceFile.setOriginalFilename("base.xlsx");
|
||||
return sourceFile;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_normal_default_path() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(100);
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
// 规范行集合只输出 items,不输出 allItems 重复结构
|
||||
assertTrue(node.has("items"));
|
||||
assertFalse(node.has("allItems"), "payload 必须不再序列化 allItems 重复结构");
|
||||
assertEquals(100, node.get("items").size());
|
||||
// groups 仍保留(Python 回传需要),但不作为全量行来源
|
||||
assertTrue(node.has("groups"));
|
||||
// items 中每行字段完整
|
||||
JsonNode first = node.get("items").get(0);
|
||||
assertEquals("uploads/20260829/base.xlsx::row::1", first.get("rowToken").asText());
|
||||
assertEquals("B0CJ8SNXXV", first.get("asin").asText());
|
||||
assertEquals("uploads/20260829/base.xlsx", first.get("sourceFileKey").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_normal_multiple_items() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(1000);
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertEquals(1000, node.get("items").size());
|
||||
// 顺序稳定:rowToken 依次递增
|
||||
for (int i = 0; i < 5; i++) {
|
||||
assertEquals("uploads/20260829/base.xlsx::row::" + (i + 1),
|
||||
node.get("items").get(i).get("rowToken").asText());
|
||||
}
|
||||
// 反序列化后行数不丢失
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
assertEquals(1000, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(200);
|
||||
String first = buildPayloadJson(rows, groups(rows));
|
||||
String second = buildPayloadJson(rows, groups(rows));
|
||||
// 重复构建输出一致
|
||||
assertEquals(MAPPER.readTree(first), MAPPER.readTree(second));
|
||||
// 不产生重复记录:行 token 唯一
|
||||
JsonNode items = MAPPER.readTree(first).get("items");
|
||||
long distinct = java.util.stream.StreamSupport.stream(items.spliterator(), false)
|
||||
.map(item -> item.get("rowToken").asText()).distinct().count();
|
||||
assertEquals(200, distinct);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_boundary_empty_input() throws Exception {
|
||||
String json = buildPayloadJson(List.of(), List.of());
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertTrue(node.has("items"));
|
||||
assertEquals(0, node.get("items").size());
|
||||
assertFalse(node.has("allItems"));
|
||||
// 空载荷反序列化安全
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
assertNotNull(payload.getItems());
|
||||
assertEquals(0, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_boundary_single_item() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(1);
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertEquals(1, node.get("items").size());
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
assertEquals(1, payload.getItems().size());
|
||||
assertEquals("uploads/20260829/base.xlsx::row::1", payload.getItems().get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_boundary_limit_and_overflow() throws Exception {
|
||||
// null 行集合:items 输出为空数组而非 NPE/崩溃
|
||||
String json = buildPayloadJson(null, null);
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertTrue(node.has("items"));
|
||||
assertEquals(0, node.get("items").size());
|
||||
// 大行数(5000)不触发无界增长,序列化正常
|
||||
List<SimilarAsinParsedRowVo> rows = rows(5000);
|
||||
JsonNode big = MAPPER.readTree(buildPayloadJson(rows, groups(rows)));
|
||||
assertEquals(5000, big.get("items").size());
|
||||
assertFalse(big.has("allItems"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_invalid_input_rejected() throws Exception {
|
||||
// 旧格式 JSON(含 allItems)反序列化:allItems 键被吸收进 items,不丢失行,不抛异常
|
||||
String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
|
||||
+ "\"sourceFiles\":[],\"headers\":[],"
|
||||
+ "\"items\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"}],"
|
||||
+ "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"},{\"rowToken\":\"t2\",\"asin\":\"B0TEST1234\"}],"
|
||||
+ "\"groups\":[]}";
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class);
|
||||
// items 优先;allItems 仅在 items 为空时兜底吸收,避免旧数据行丢失
|
||||
assertEquals(1, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_dependency_failure_releases_resources() throws Exception {
|
||||
// 序列化器故障:抛项目约定异常(BusinessException),不产生部分结果
|
||||
ObjectMapper broken = new ObjectMapper() {
|
||||
@Override
|
||||
public String writeValueAsString(Object value) {
|
||||
throw new IllegalStateException("serializer down");
|
||||
}
|
||||
};
|
||||
SimilarAsinTaskService failingService = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
|
||||
ReflectionTestUtils.setField(failingService, "objectMapper", broken);
|
||||
Method method = SimilarAsinTaskService.class.getDeclaredMethod(
|
||||
"buildParsedPayloadJson",
|
||||
String.class, String.class, Boolean.class, Boolean.class,
|
||||
List.class, List.class, List.class, List.class);
|
||||
method.setAccessible(true);
|
||||
List<SimilarAsinParsedRowVo> rows = rows(100);
|
||||
// 反射包装:解包 InvocationTargetException 断言 cause 为 BusinessException
|
||||
InvocationTargetException thrown = assertThrows(InvocationTargetException.class, () -> method.invoke(failingService,
|
||||
"p", "k", Boolean.FALSE, Boolean.FALSE, List.of(), List.of(), groups(rows), rows));
|
||||
assertTrue(thrown.getCause() instanceof BusinessException);
|
||||
// 恢复后(换回正常 mapper)仍能正常工作
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
assertEquals(100, MAPPER.readTree(json).get("items").size());
|
||||
}
|
||||
}
|
||||
-253
@@ -1,253 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 5:预览行数量增加配置边界、空文件和超限输入校验。
|
||||
* 预览上限从硬编码常量改为配置驱动,并对配置值做 clamp 边界保护。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServicePreviewConfigTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(20000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/20000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-preview-config-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0CFG%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("config.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_normal_default_path() throws Exception {
|
||||
// 默认配置 100:150 行文件返回 100 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(150);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-default.xlsx");
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
// 预览行从第 1 行开始
|
||||
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_normal_multiple_items() throws Exception {
|
||||
// 配置 50:500 行文件返回 50 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(50);
|
||||
File workbook = buildWorkbook(500);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-50.xlsx");
|
||||
assertEquals(50, vo.getItems().size());
|
||||
assertEquals(500, vo.getAcceptedRows());
|
||||
// groups 同样按配置裁剪
|
||||
assertTrue(vo.getGroups().size() <= 50);
|
||||
// 配置 200:300 行文件返回 200 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(200);
|
||||
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-200.xlsx");
|
||||
assertEquals(200, vo2.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(250);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_boundary_empty_input() throws Exception {
|
||||
// 空文件(无有效数据行):抛业务异常
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-empty.xlsx")).thenReturn(workbook);
|
||||
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/cfg-empty.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_boundary_single_item() throws Exception {
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-single.xlsx");
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_boundary_limit_and_overflow() throws Exception {
|
||||
// 配置值超过最大上限(1000):clamp 到上限,不发生无界内存增长
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(50000);
|
||||
File workbook = buildWorkbook(2000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-over.xlsx");
|
||||
assertEquals(1000, vo.getItems().size(), "超限配置必须 clamp 到最大允许值");
|
||||
assertEquals(2000, vo.getAcceptedRows());
|
||||
// 配置值恰好等于上限:返回 1000 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(1000);
|
||||
SimilarAsinParseVo vo2 = parse(buildWorkbook(1000), "uploads/20260829/cfg-exact.xlsx");
|
||||
assertEquals(1000, vo2.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_invalid_input_rejected() throws Exception {
|
||||
// 配置为 0/负数:回退到默认值 100,不抛异常不崩溃
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(0, -5);
|
||||
File workbook = buildWorkbook(300);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-zero.xlsx");
|
||||
assertEquals(100, vo.getItems().size());
|
||||
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-neg.xlsx");
|
||||
assertEquals(100, vo2.getItems().size());
|
||||
// 配置缺失(fresh mock 未 stub,int 默认 0):同样回退默认
|
||||
SimilarAsinProperties missing = org.mockito.Mockito.mock(SimilarAsinProperties.class);
|
||||
lenient().when(missing.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(missing.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(missing.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
ReflectionTestUtils.setField(service, "properties", missing);
|
||||
SimilarAsinParseVo vo3 = parse(buildWorkbook(300), "uploads/20260829/cfg-missing.xlsx");
|
||||
assertEquals(100, vo3.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_dependency_failure_releases_resources() throws Exception {
|
||||
// 存储失败抛异常;恢复后解析正常
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(120);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/20001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/cfg-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-recovered.xlsx");
|
||||
assertEquals(120, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
// 配置对象默认值校验:新实例默认 100,处于 [1, 1000] 边界内
|
||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
||||
assertNotNull(defaults.getParseResponsePreviewLimit());
|
||||
int previewLimit = defaults.getParseResponsePreviewLimit();
|
||||
assertTrue(previewLimit >= 1 && previewLimit <= 1000,
|
||||
"默认预览上限必须在 [1, 1000] 内,实际 " + previewLimit);
|
||||
}
|
||||
}
|
||||
-210
@@ -1,210 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Task 3:保留旧 payload 读取兼容逻辑,验证新旧结构均可恢复全量行。
|
||||
* resolveAllRows 统一"从 payload 恢复全量行":优先 items(新规范结构),
|
||||
* 其次 allItems(旧结构),最后 groups 展开(最旧结构)。
|
||||
*/
|
||||
class SimilarAsinTaskServiceResolveAllRowsTest {
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private static SimilarAsinParsedRowVo row(String fileKey, int index) {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setSourceFileKey(fileKey);
|
||||
row.setSourceFilename("base.xlsx");
|
||||
row.setRowIndex(index);
|
||||
row.setSourceId(String.valueOf(index));
|
||||
row.setDisplayId(String.valueOf(index));
|
||||
row.setRowToken(fileKey + "::row::" + index);
|
||||
row.setAsin("B0CJ8SNXXV");
|
||||
row.setCountry("英国");
|
||||
return row;
|
||||
}
|
||||
|
||||
private static List<SimilarAsinParsedRowVo> rows(String fileKey, int count) {
|
||||
List<SimilarAsinParsedRowVo> result = new ArrayList<>();
|
||||
for (int i = 1; i <= count; i++) {
|
||||
result.add(row(fileKey, i));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static SimilarAsinParsedGroupVo group(List<SimilarAsinParsedRowVo> items) {
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setSourceFileKey("uploads/20260829/base.xlsx");
|
||||
group.setSourceFilename("base.xlsx");
|
||||
group.setGroupKey("uploads/20260829/base.xlsx::1@1");
|
||||
group.setBaseId("1");
|
||||
group.setDisplayId("1");
|
||||
group.setItemCount(items.size());
|
||||
group.setItems(new ArrayList<>(items));
|
||||
return group;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_default_path() {
|
||||
// 新结构:items 有值 → 恢复全量行
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/base.xlsx", 100);
|
||||
payload.setItems(items);
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(100, restored.size());
|
||||
assertEquals("uploads/20260829/base.xlsx::row::1", restored.get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_multiple_items() {
|
||||
// 新结构 1000 行:不丢失且顺序稳定
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/multi.xlsx", 1000);
|
||||
payload.setItems(items);
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(1000, restored.size());
|
||||
for (int i = 0; i < restored.size(); i++) {
|
||||
assertEquals(i + 1, restored.get(i).getRowIndex());
|
||||
}
|
||||
// 旧结构 1000 行:allItems 全量恢复
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
legacy.setAllItems(items);
|
||||
List<SimilarAsinParsedRowVo> restoredLegacy = SimilarAsinTaskService.resolveAllRows(legacy);
|
||||
assertEquals(1000, restoredLegacy.size());
|
||||
assertEquals("uploads/20260829/multi.xlsx::row::1", restoredLegacy.get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_repeated_operation_is_idempotent() {
|
||||
// 重复调用返回相同行集合(不修改 payload 本身)
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/idem.xlsx", 50);
|
||||
payload.setItems(items);
|
||||
List<SimilarAsinParsedRowVo> first = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
List<SimilarAsinParsedRowVo> second = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
}
|
||||
// payload 未被修改:items 仍 50 行
|
||||
assertEquals(50, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_boundary_empty_input() {
|
||||
// 空 payload:返回空列表而非 null,不创建无效资源
|
||||
SimilarAsinParsedPayloadDto empty = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(empty);
|
||||
assertNotNull(restored);
|
||||
assertEquals(0, restored.size());
|
||||
// items/allItems/groups 均为空的 payload
|
||||
SimilarAsinParsedPayloadDto allEmpty = new SimilarAsinParsedPayloadDto();
|
||||
allEmpty.setItems(List.of());
|
||||
allEmpty.setAllItems(List.of());
|
||||
allEmpty.setGroups(List.of());
|
||||
assertEquals(0, SimilarAsinTaskService.resolveAllRows(allEmpty).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_boundary_single_item() {
|
||||
// 单行新结构
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setItems(rows("uploads/20260829/single.xlsx", 1));
|
||||
assertEquals(1, SimilarAsinTaskService.resolveAllRows(payload).size());
|
||||
// 单行旧结构(仅 groups)
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
legacy.setGroups(List.of(group(rows("uploads/20260829/single.xlsx", 1))));
|
||||
assertEquals(1, SimilarAsinTaskService.resolveAllRows(legacy).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_boundary_limit_and_overflow() {
|
||||
// 旧结构 allItems 5000 行全量恢复,不丢失
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
legacy.setAllItems(rows("uploads/20260829/max.xlsx", 5000));
|
||||
assertEquals(5000, SimilarAsinTaskService.resolveAllRows(legacy).size());
|
||||
// items 与 allItems 同时存在:以 items 为准(新规范结构优先),不重复
|
||||
SimilarAsinParsedPayloadDto both = new SimilarAsinParsedPayloadDto();
|
||||
both.setItems(rows("uploads/20260829/both.xlsx", 10));
|
||||
both.setAllItems(rows("uploads/20260829/both.xlsx", 20));
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(both);
|
||||
assertEquals(10, restored.size());
|
||||
// groups 也同时存在:仍以 items 为准
|
||||
both.setGroups(List.of(group(rows("uploads/20260829/both.xlsx", 30))));
|
||||
assertEquals(10, SimilarAsinTaskService.resolveAllRows(both).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_invalid_input_rejected() {
|
||||
// null payload:安全返回空列表(调用方容忍),不抛 NPE
|
||||
assertEquals(0, SimilarAsinTaskService.resolveAllRows(null).size());
|
||||
// groups 中含 null 元素:跳过不抛异常
|
||||
SimilarAsinParsedPayloadDto messy = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
|
||||
groups.add(null);
|
||||
groups.add(group(rows("uploads/20260829/messy.xlsx", 5)));
|
||||
messy.setGroups(groups);
|
||||
assertEquals(5, SimilarAsinTaskService.resolveAllRows(messy).size());
|
||||
// group.items 为 null:跳过该组
|
||||
SimilarAsinParsedGroupVo nullItemsGroup = new SimilarAsinParsedGroupVo();
|
||||
nullItemsGroup.setItems(null);
|
||||
SimilarAsinParsedPayloadDto nullItems = new SimilarAsinParsedPayloadDto();
|
||||
nullItems.setGroups(List.of(nullItemsGroup, group(rows("uploads/20260829/n2.xlsx", 3))));
|
||||
assertEquals(3, SimilarAsinTaskService.resolveAllRows(nullItems).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_dependency_failure_releases_resources() throws Exception {
|
||||
// 旧格式 JSON(只有 allItems)反序列化 → hydrate 后 resolveAllRows 恢复全量行
|
||||
String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
|
||||
+ "\"sourceFiles\":[],\"headers\":[],\"groups\":[],"
|
||||
+ "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0AAA00001\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":1},"
|
||||
+ "{\"rowToken\":\"t2\",\"asin\":\"B0AAA00002\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":2}]}";
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class);
|
||||
// 未 hydrate 时:allItems 恢复(items 为空走 allItems)
|
||||
List<SimilarAsinParsedRowVo> fromLegacy = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(2, fromLegacy.size());
|
||||
assertEquals("t1", fromLegacy.get(0).getRowToken());
|
||||
assertEquals("B0AAA00002", fromLegacy.get(1).getAsin());
|
||||
// 新格式 JSON(只有 items)反序列化 → 直接恢复
|
||||
String newJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
|
||||
+ "\"sourceFiles\":[],\"headers\":[],"
|
||||
+ "\"items\":[{\"rowToken\":\"n1\",\"asin\":\"B0NEW00001\",\"sourceFileKey\":\"uploads/20260829/b.xlsx\",\"rowIndex\":1}],"
|
||||
+ "\"groups\":[]}";
|
||||
SimilarAsinParsedPayloadDto newPayload = MAPPER.readValue(newJson, SimilarAsinParsedPayloadDto.class);
|
||||
List<SimilarAsinParsedRowVo> fromNew = SimilarAsinTaskService.resolveAllRows(newPayload);
|
||||
assertEquals(1, fromNew.size());
|
||||
assertEquals("n1", fromNew.get(0).getRowToken());
|
||||
// 两种格式行数之和互不影响,恢复结果稳定
|
||||
assertTrue(fromLegacy.size() == 2 && fromNew.size() == 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_groups_expansion_preserves_order() {
|
||||
// 最旧结构:仅 groups 嵌套行,展开后顺序稳定(按组、组内原序)
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> g1 = rows("uploads/20260829/g.xlsx", 2);
|
||||
List<SimilarAsinParsedRowVo> g2 = rows("uploads/20260829/g.xlsx", 3);
|
||||
// 组内行号各自独立从 1 开始(真实解析语义),第二组用不同 fileKey 区分来源
|
||||
List<SimilarAsinParsedRowVo> g2b = rows("uploads/20260829/g2.xlsx", 3);
|
||||
legacy.setGroups(List.of(group(g1), group(g2b)));
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacy);
|
||||
assertEquals(5, restored.size());
|
||||
assertEquals("uploads/20260829/g.xlsx::row::1", restored.get(0).getRowToken());
|
||||
assertEquals("uploads/20260829/g.xlsx::row::2", restored.get(1).getRowToken());
|
||||
assertEquals("uploads/20260829/g2.xlsx::row::1", restored.get(2).getRowToken());
|
||||
assertEquals("uploads/20260829/g2.xlsx::row::2", restored.get(3).getRowToken());
|
||||
assertEquals("uploads/20260829/g2.xlsx::row::3", restored.get(4).getRowToken());
|
||||
}
|
||||
}
|
||||
-321
@@ -1,321 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 11:Coze 结果合并的重复检测从 O(n²) 改为 HashSet/稳定 row key。
|
||||
* dedupeRowsByRowKey 用 HashSet 按稳定 rowKey 一次性去重(保留顺序),
|
||||
* mergeCozeRowsIntoChunk 合并前先去重,消除重复行逐行重复处理。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceRowKeyDedupeTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(70000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/70000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(id);
|
||||
r.setAsin(asin);
|
||||
r.setCountry(country);
|
||||
return r;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7004L);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private void stubSingleChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicLong storedCounter) throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
|
||||
.thenReturn(payloadJson);
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storedCounter.incrementAndGet();
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_normal_default_path() throws Exception {
|
||||
// 正常输入:llmRows 含同一 rowKey 的重复行,merge 前按稳定 rowKey 去重,
|
||||
// chunk payload 只写一次,结果行不重复。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicLong storedCounter = new AtomicLong(0);
|
||||
stubSingleChunkMerge(chunk, rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))), storedCounter);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(
|
||||
row("r1", "1", "B0A0000001", "英国"),
|
||||
row("r1", "1", "B0A0000001", "英国"));
|
||||
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(1, deduped.size(), "重复行必须按稳定 rowKey 去重");
|
||||
assertEquals("r1", deduped.get(0).getRowToken());
|
||||
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, null, null, llmRows, Map.of());
|
||||
assertEquals(1, storedCounter.get(), "去重后 chunk 只写一次");
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多个重复行跨 chunk 分组,去重后顺序稳定、结果不丢失
|
||||
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r2", "2", "B0A0000002", "英国"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
|
||||
AtomicLong selectOneRound = new AtomicLong(0);
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
|
||||
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 0; i < 3; i++) {
|
||||
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
|
||||
llmRows.add(row("r2", "2", "B0A0000002", "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(2, deduped.size(), "3 轮重复输入去重后只剩 2 个唯一行");
|
||||
assertEquals(List.of("r1", "r2"), deduped.stream().map(SimilarAsinResultRowDto::getRowToken).toList(),
|
||||
"去重必须保留首次出现顺序");
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, null, null, llmRows, Map.of());
|
||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行同一输入:去重结果完全一致,不产生重复记录
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(
|
||||
row("r1", "1", "B0A0000001", "英国"),
|
||||
row("r2", "2", "B0A0000002", "英国"),
|
||||
row("r1", "1", "B0A0000001", "英国"));
|
||||
List<SimilarAsinResultRowDto> first = service.dedupeRowsByRowKey(llmRows);
|
||||
List<SimilarAsinResultRowDto> second = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
assertEquals(first.get(i).getAsin(), second.get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_boundary_empty_input() {
|
||||
// 空输入:null/空列表安全返回空结果,不创建无效资源
|
||||
assertNotNull(service.dedupeRowsByRowKey(null));
|
||||
assertTrue(service.dedupeRowsByRowKey(null).isEmpty());
|
||||
assertTrue(service.dedupeRowsByRowKey(List.of()).isEmpty());
|
||||
// null 元素:跳过不抛异常
|
||||
List<SimilarAsinResultRowDto> withNull = new ArrayList<>();
|
||||
withNull.add(null);
|
||||
withNull.add(row("r1", "1", "B0A0000001", "英国"));
|
||||
assertEquals(1, service.dedupeRowsByRowKey(withNull).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_boundary_single_item() {
|
||||
// 单行:不依赖批量路径,去重后结果正确
|
||||
List<SimilarAsinResultRowDto> single = List.of(row("r1", "1", "B0A0000001", "英国"));
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(single);
|
||||
assertEquals(1, deduped.size());
|
||||
assertEquals("r1", deduped.get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_boundary_limit_and_overflow() {
|
||||
// 大批量:1000 行全部重复,去重后只剩 1 个唯一行,无无界内存增长
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(1, deduped.size());
|
||||
// 1000 行唯一:全部保留且顺序稳定
|
||||
List<SimilarAsinResultRowDto> unique = new ArrayList<>();
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
unique.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0U" + String.format("%06d", i), "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> dedupedUnique = service.dedupeRowsByRowKey(unique);
|
||||
assertEquals(1000, dedupedUnique.size());
|
||||
assertEquals("r0001", dedupedUnique.get(1).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_invalid_input_rejected() {
|
||||
// 稳定 rowKey 冲突:rowToken 为空时用 legacy key(id::ASIN::country)识别重复
|
||||
List<SimilarAsinResultRowDto> noToken = List.of(
|
||||
row("", "1", "B0A0000001", "英国"),
|
||||
row("", "1", "b0a0000001", " 英国 "));
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(noToken);
|
||||
assertEquals(1, deduped.size(), "legacy key 归一化(ASIN 大写、country trim)后应识别为同一行");
|
||||
// 不同 ASIN:不误判为重复
|
||||
List<SimilarAsinResultRowDto> diffAsin = List.of(
|
||||
row("", "1", "B0A0000001", "英国"),
|
||||
row("", "2", "B0A0000002", "英国"));
|
||||
assertEquals(2, service.dedupeRowsByRowKey(diffAsin).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_dependency_failure_releases_resources() throws Exception {
|
||||
// chunk payload 读取失败:抛可识别业务异常且不写 chunk;
|
||||
// 依赖恢复后重试成功,去重路径无残留状态
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenThrow(new IllegalStateException("rustfs down"));
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
merge.invoke(service, task, null, null,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
// 恢复后重试成功:只写一次,无重复记录
|
||||
AtomicLong storedCounter = new AtomicLong(0);
|
||||
stubSingleChunkMerge(chunk, rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))), storedCounter);
|
||||
merge.invoke(service, task, null, null,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
assertEquals(1, storedCounter.get());
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
}
|
||||
-372
@@ -1,372 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinImagePrefetchService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 10:为 chunk 结果建立按 row key 的批量索引,消除跨 chunk 线性扫描。
|
||||
* indexRowsByChunkKey 把每个 chunk 的行索引到 rowKey→chunkKey,llm 行归属从
|
||||
* O(rows×chunks) 降为 O(1) 查找;assignLlmRowsToChunks 基于索引分配行并保留
|
||||
* 原有命中/fallback/orphan 语义;集成用例验证每个 chunk 只读一次 payload。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceRowKeyIndexTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(60000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/60000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(id);
|
||||
r.setAsin(asin);
|
||||
r.setCountry(country);
|
||||
return r;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7004L);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
/** 构造 rowsByChunk:chunkStorageKey(scopeHash, chunkIndex) → rowKey 行表。 */
|
||||
private static Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunkOf(String scopeHash, Integer chunkIndex, List<SimilarAsinResultRowDto> rows) {
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> map = new LinkedHashMap<>();
|
||||
Map<String, SimilarAsinResultRowDto> byKey = new LinkedHashMap<>();
|
||||
for (SimilarAsinResultRowDto row : rows) {
|
||||
byKey.put(row.getRowToken(), row);
|
||||
}
|
||||
map.put(scopeHash + ":" + chunkIndex, byKey);
|
||||
return map;
|
||||
}
|
||||
|
||||
private static List<String> assignedRowKeys(Map<String, Map<String, SimilarAsinResultRowDto>> merged) {
|
||||
List<String> keys = new ArrayList<>();
|
||||
for (Map<String, SimilarAsinResultRowDto> rows : merged.values()) {
|
||||
for (String key : rows.keySet()) {
|
||||
keys.add(key);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_normal_default_path() throws Exception {
|
||||
// 正常输入:2 个 chunk 各含行,llm 回传行按 rowKey 命中各自 chunk;
|
||||
// 每个 chunk 的 payload 只被读取一次(索引建立),消除跨 chunk 线性扫描。
|
||||
List<TaskChunkEntity> chunks = List.of(
|
||||
chunk(1L, "hashA", 1, "ptr:chunk-A"),
|
||||
chunk(2L, "hashB", 2, "ptr:chunk-B"));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国"))));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r3", "3", "B0A0000003", "美国"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
|
||||
AtomicInteger selectOneRound = new AtomicInteger(0);
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
|
||||
int i = selectOneRound.getAndIncrement();
|
||||
return chunks.get(Math.min(i, chunks.size() - 1));
|
||||
});
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"), row("r3", "3", "B0A0000003", "美国"));
|
||||
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, null, null, llmRows, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString());
|
||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, times(2)).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_normal_multiple_items() {
|
||||
// 批量场景:3 个 chunk 各 3 行,9 个 llm 回传行全部命中且顺序稳定,无 orphan
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
for (int c = 0; c < 3; c++) {
|
||||
rowsByChunk.putAll(rowsByChunkOf("hash" + c, c + 1,
|
||||
List.of(row("c" + c + "r1", "1", "B0B" + c + "000001", "英国"),
|
||||
row("c" + c + "r2", "2", "B0B" + c + "000002", "英国"),
|
||||
row("c" + c + "r3", "3", "B0B" + c + "000003", "美国"))));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int c = 0; c < 3; c++) {
|
||||
for (int r = 1; r <= 3; r++) {
|
||||
llmRows.add(row("c" + c + "r" + r, String.valueOf(r), "B0B" + c + "00000" + r, r == 3 ? "美国" : "英国"));
|
||||
}
|
||||
}
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, orphans);
|
||||
assertEquals(3, merged.size());
|
||||
assertEquals(9, assignedRowKeys(merged).size());
|
||||
assertTrue(orphans.isEmpty(), "全部命中,不应产生 orphan");
|
||||
for (Map<String, SimilarAsinResultRowDto> rows : merged.values()) {
|
||||
assertEquals(3, rows.size());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行同一输入:结果完全一致,不产生重复记录
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国")));
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"));
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> first = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> second = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
|
||||
assertEquals(assignedRowKeys(first), assignedRowKeys(second));
|
||||
assertEquals(first.size(), second.size());
|
||||
for (Map.Entry<String, Map<String, SimilarAsinResultRowDto>> entry : first.entrySet()) {
|
||||
assertEquals(entry.getValue().keySet(), second.get(entry.getKey()).keySet());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_boundary_empty_input() {
|
||||
// 空输入:null/空 rowsByChunk 与 llmRows 均安全返回空结果,不创建无效资源
|
||||
assertNotNull(service.indexRowsByChunkKey(null));
|
||||
assertTrue(service.indexRowsByChunkKey(null).isEmpty());
|
||||
assertTrue(service.indexRowsByChunkKey(Map.of()).isEmpty());
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> emptyAssign = service.assignLlmRowsToChunks(
|
||||
Map.of(), List.of(), Map.of(), null, null, new ArrayList<>());
|
||||
assertTrue(emptyAssign.isEmpty());
|
||||
assertTrue(service.assignLlmRowsToChunks(
|
||||
Map.of(), null, Map.of(), null, null, new ArrayList<>()).isEmpty());
|
||||
// 无可匹配行(rowKey 不存在于任何 chunk)→ 进 orphan 兜底,不产生 merge
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1, List.of(row("r1", "1", "B0A0000001", "英国")));
|
||||
List<SimilarAsinResultRowDto> blankRow = List.of(row("", "", "", ""));
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, blankRow, index, null, null, orphans);
|
||||
assertTrue(assignedRowKeys(merged).isEmpty());
|
||||
assertEquals(1, orphans.size(), "全空行生成 legacy key :::: 不命中任何 chunk,按既有语义进 orphan");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_boundary_single_item() {
|
||||
// 单 chunk 单行:不依赖批量路径,命中正确
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国")));
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, List.of(row("r1", "1", "B0A0000001", "英国")), index, null, null, orphans);
|
||||
assertEquals(1, merged.size());
|
||||
assertEquals(List.of("r1"), assignedRowKeys(merged));
|
||||
assertTrue(orphans.isEmpty());
|
||||
// 索引也只含该行
|
||||
assertEquals(1, index.size());
|
||||
assertEquals("hashA:1", index.get("r1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_boundary_limit_and_overflow() {
|
||||
// 大批量:1000 行索引 + 500 个 llm 回传行全部命中,行不丢、无 orphan
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
Map<String, SimilarAsinResultRowDto> bigChunk = new LinkedHashMap<>();
|
||||
for (int i = 1; i <= 1000; i++) {
|
||||
bigChunk.put("r" + String.format("%04d", i), row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
|
||||
}
|
||||
rowsByChunk.put("hashBig:1", bigChunk);
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
assertEquals(1000, index.size());
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 1; i <= 500; i++) {
|
||||
llmRows.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, orphans);
|
||||
assertEquals(1, merged.size());
|
||||
assertEquals(500, assignedRowKeys(merged).size());
|
||||
assertTrue(orphans.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_invalid_input_rejected() {
|
||||
// 同一 rowKey 出现在多个 chunk:索引保留第一个 chunk(putIfAbsent),行为确定
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
rowsByChunk.putAll(rowsByChunkOf("hashA", 1, List.of(row("dup", "1", "B0A0000001", "英国"))));
|
||||
rowsByChunk.putAll(rowsByChunkOf("hashB", 2, List.of(row("dup", "1", "B0A0000001", "英国"))));
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
assertEquals("hashA:1", index.get("dup"), "重复 rowKey 应保留第一个 chunk");
|
||||
// fallback 缺失:llm 行未命中且无有效 fallback → 进 orphan,不产生 merge
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, List.of(row("ghost", "9", "B0A0000009", "英国")), Map.of(), "missingHash", 99, orphans);
|
||||
assertTrue(assignedRowKeys(merged).isEmpty());
|
||||
assertEquals(1, orphans.size());
|
||||
assertEquals("ghost", orphans.get(0).getRowToken());
|
||||
// llmRows 含 null 元素:跳过不抛异常,其余行正常分配
|
||||
List<SimilarAsinResultRowDto> withNull = new ArrayList<>();
|
||||
withNull.add(null);
|
||||
withNull.add(row("dup", "1", "B0A0000001", "英国"));
|
||||
List<SimilarAsinResultRowDto> orphans2 = new ArrayList<>();
|
||||
Map<String, String> index2 = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged2 = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, withNull, index2, null, null, orphans2);
|
||||
assertEquals(1, merged2.size());
|
||||
assertEquals(List.of("dup"), assignedRowKeys(merged2));
|
||||
assertTrue(orphans2.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_dependency_failure_releases_resources() throws Exception {
|
||||
// chunk payload 读取失败:抛可识别业务异常且不产生部分 merge;
|
||||
// 依赖恢复后重试成功,无残留状态
|
||||
List<TaskChunkEntity> chunks = List.of(chunk(1L, "hashA", 1, "ptr:chunk-A"));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenThrow(new IllegalStateException("rustfs down"));
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
BusinessException ex = assertThrows(BusinessException.class, () -> {
|
||||
try {
|
||||
merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
// 恢复后重试成功:行合并到正确 chunk
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenReturn("stored:retry");
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunks.get(0));
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
}
|
||||
-362
@@ -1,362 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceSubmitTest {
|
||||
|
||||
private static final Long TASK_ID = 21879L;
|
||||
private static final String PARSED_POINTER = "rustfs:task-parsed/similar-asin/21879/payload.json";
|
||||
private static final String CHUNK_POINTER = "rustfs:task-chunk/similar-asin/21879/chunk.json";
|
||||
private static final String STORED_CHUNK_POINTER = "\"" + CHUNK_POINTER + "\"";
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
@Mock private TransactionStatus transactionStatus;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
private final AtomicBoolean transactionActive = new AtomicBoolean();
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUpTransactionAndLock() {
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
lenient().when(taskDistributedLockService.acquire(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE),
|
||||
anyLong(),
|
||||
any(Duration.class),
|
||||
eq(10_000L)))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
transactionActive.set(true);
|
||||
return transactionStatus;
|
||||
});
|
||||
lenient().doAnswer(invocation -> {
|
||||
transactionActive.set(false);
|
||||
return null;
|
||||
}).when(transactionManager).commit(transactionStatus);
|
||||
lenient().doAnswer(invocation -> {
|
||||
transactionActive.set(false);
|
||||
return null;
|
||||
}).when(transactionManager).rollback(transactionStatus);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdownExecutors() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
@Test
|
||||
void doneCallbackReadsPayloadOnlyBeforeShortTransaction() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
configureNewChunkAndScope();
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
List<Boolean> storageCallTransactionStates = new ArrayList<>();
|
||||
when(transientPayloadStorageService.resolvePayload(eq(PARSED_POINTER), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storageCallTransactionStates.add(transactionActive.get());
|
||||
return parsedPayloadJson(List.of(new SimilarAsinParsedRowVo(), new SimilarAsinParsedRowVo()));
|
||||
});
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storageCallTransactionStates.add(transactionActive.get());
|
||||
return STORED_CHUNK_POINTER;
|
||||
});
|
||||
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(false);
|
||||
doAnswer(invocation -> {
|
||||
assertTrue(transactionActive.get());
|
||||
FileResultEntity result = invocation.getArgument(0);
|
||||
result.setId(501L);
|
||||
return 1;
|
||||
}).when(fileResultMapper).insert(any(FileResultEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
storageCallTransactionStates.add(transactionActive.get());
|
||||
return null;
|
||||
}).when(taskCacheService).deleteTaskCache(TASK_ID);
|
||||
|
||||
service.submitResult(TASK_ID, request(true));
|
||||
|
||||
assertFalse(storageCallTransactionStates.isEmpty());
|
||||
assertTrue(storageCallTransactionStates.stream().noneMatch(Boolean::booleanValue));
|
||||
verify(transientPayloadStorageService).resolvePayload(eq(PARSED_POINTER), anyString());
|
||||
ArgumentCaptor<FileResultEntity> resultCaptor = ArgumentCaptor.forClass(FileResultEntity.class);
|
||||
verify(fileResultMapper).insert(resultCaptor.capture());
|
||||
assertEquals(2, resultCaptor.getValue().getRowCount());
|
||||
assertEquals("germany.xlsx", resultCaptor.getValue().getSourceFilename());
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownCommitOutcomeDoesNotDeletePossiblyCommittedChunk() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
configureNewChunkAndScope();
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(0L);
|
||||
configureChunkStore(false);
|
||||
doAnswer(invocation -> {
|
||||
transactionActive.set(false);
|
||||
throw new IllegalStateException("commit ACK lost");
|
||||
}).when(transactionManager).commit(transactionStatus);
|
||||
|
||||
IllegalStateException thrown = assertThrows(IllegalStateException.class,
|
||||
() -> service.submitResult(TASK_ID, request(false)));
|
||||
|
||||
assertEquals("commit ACK lost", thrown.getMessage());
|
||||
verify(transientPayloadStorageService, never()).extractPointer(anyString());
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
verify(taskCacheService, never()).touchTaskHeartbeat(TASK_ID);
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateAlreadyPersistedChunkDoesNotRunPayloadCleanup() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk("\"rustfs:winner\""));
|
||||
configureScopeStorage(null);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
|
||||
verify(transientPayloadStorageService, never()).extractPointer(anyString());
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void localFallbackOwnerFailureRollsBackChunkAndKeepsCandidate() throws Exception {
|
||||
FileTaskEntity task = runningTask(null);
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
configureNewChunkAndScope();
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
configureChunkStore(true);
|
||||
when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(0);
|
||||
|
||||
IllegalStateException thrown = assertThrows(IllegalStateException.class,
|
||||
() -> service.submitResult(TASK_ID, request(false)));
|
||||
|
||||
assertTrue(thrown.getMessage().contains("Failed to bind local fallback task owner"));
|
||||
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||
verify(transactionManager).rollback(transactionStatus);
|
||||
verify(transactionManager, never()).commit(transactionStatus);
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateLocalCandidateIsNotBoundAndReferencedPayloadIsKept() throws Exception {
|
||||
FileTaskEntity task = runningTask(null);
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
TaskChunkEntity winner = chunk(STORED_CHUNK_POINTER);
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(null, winner);
|
||||
configureScopeStorage(null);
|
||||
configureChunkStore(true);
|
||||
when(transientPayloadStorageService.extractPointer(STORED_CHUNK_POINTER)).thenReturn(CHUNK_POINTER);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
|
||||
verify(taskChunkMapper, never()).insert(any(TaskChunkEntity.class));
|
||||
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(STORED_CHUNK_POINTER);
|
||||
}
|
||||
|
||||
@Test
|
||||
void repeatedNonFinalCallbackCannotClearCompletedScope() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
TaskChunkEntity existing = chunk("\"rustfs:winner\"");
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(existing);
|
||||
TaskScopeStateEntity scope = scope(1);
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
|
||||
assertEquals(1, scope.getCompleted());
|
||||
verify(taskScopeStateMapper).updateById(scope);
|
||||
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(
|
||||
anyString(), anyLong(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
private String parsedPayloadJson(List<SimilarAsinParsedRowVo> rows) throws Exception {
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setAllItems(rows);
|
||||
payload.setItems(rows);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey("uploads/germany.xlsx");
|
||||
sourceFile.setOriginalFilename("germany.xlsx");
|
||||
payload.setSourceFiles(List.of(sourceFile));
|
||||
return objectMapper.writeValueAsString(payload);
|
||||
}
|
||||
|
||||
private void configureNewChunkAndScope() {
|
||||
AtomicReference<TaskChunkEntity> chunkRef = new AtomicReference<>();
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> chunkRef.get());
|
||||
doAnswer(invocation -> {
|
||||
TaskChunkEntity chunk = invocation.getArgument(0);
|
||||
chunk.setId(301L);
|
||||
chunkRef.set(chunk);
|
||||
return 1;
|
||||
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||
configureScopeStorage(null);
|
||||
}
|
||||
|
||||
private void configureScopeStorage(TaskScopeStateEntity initial) {
|
||||
AtomicReference<TaskScopeStateEntity> scopeRef = new AtomicReference<>(initial);
|
||||
when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> scopeRef.get());
|
||||
doAnswer(invocation -> {
|
||||
TaskScopeStateEntity scope = invocation.getArgument(0);
|
||||
scope.setId(401L);
|
||||
scopeRef.set(scope);
|
||||
return 1;
|
||||
}).when(taskScopeStateMapper).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
private void configureChunkStore(boolean localFallback) {
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
|
||||
.thenReturn(STORED_CHUNK_POINTER);
|
||||
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(localFallback);
|
||||
}
|
||||
|
||||
private FileTaskEntity runningTask(String owner) throws Exception {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setUserId(7L);
|
||||
String resultJson = "{\"parsedPayloadRef\":\"" + PARSED_POINTER + "\"";
|
||||
if (owner != null) {
|
||||
resultJson += ",\"ownerInstanceId\":\"" + owner + "\"";
|
||||
}
|
||||
task.setResultJson(resultJson + "}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private SimilarAsinSubmitResultRequest request(boolean done) {
|
||||
SimilarAsinSubmitResultRequest request = new SimilarAsinSubmitResultRequest();
|
||||
request.setSubmissionId("similar-asin-21879");
|
||||
request.setChunkIndex(0);
|
||||
request.setChunkTotal(1);
|
||||
request.setDone(done);
|
||||
return request;
|
||||
}
|
||||
|
||||
private TaskChunkEntity chunk(String payload) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(301L);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash("existing-scope");
|
||||
chunk.setChunkIndex(0);
|
||||
chunk.setChunkTotal(1);
|
||||
chunk.setPayloadJson(payload);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity scope(int completed) {
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setId(401L);
|
||||
scope.setTaskId(TASK_ID);
|
||||
scope.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
scope.setScopeKey("similar-asin-21879");
|
||||
scope.setScopeHash("existing-scope");
|
||||
scope.setChunkTotal(1);
|
||||
scope.setCompleted(completed);
|
||||
scope.setStateJson("{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
return scope;
|
||||
}
|
||||
}
|
||||
-121
@@ -1,121 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class SimilarAsinTaskServiceTest {
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void resultWorkbookRestoresPriceAfterCountryAndShiftsImageColumns() throws Exception {
|
||||
Field headersField = SimilarAsinTaskService.class.getDeclaredField("RESULT_HEADERS");
|
||||
headersField.setAccessible(true);
|
||||
List<String> headers = (List<String>) headersField.get(null);
|
||||
|
||||
assertEquals(List.of(
|
||||
"id", "asin", "国家", "价格", "卖家名称", "品牌", "是否有货", "相似度",
|
||||
"是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"
|
||||
), headers);
|
||||
assertEquals(12, staticIntField("IMG_COL_MAIN"));
|
||||
assertEquals(13, staticIntField("IMG_COL_PUZZLE1"));
|
||||
assertEquals(14, staticIntField("IMG_COL_PUZZLE2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultStatusUsesReturnedCozeDataAndImages() {
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(null));
|
||||
|
||||
SimilarAsinResultRowDto empty = new SimilarAsinResultRowDto();
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(empty));
|
||||
|
||||
SimilarAsinResultRowDto withImage = new SimilarAsinResultRowDto();
|
||||
withImage.setStatus("FAILED");
|
||||
withImage.setMainUrl("https://example.com/main.jpg");
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withImage));
|
||||
|
||||
SimilarAsinResultRowDto withCozeStatus = new SimilarAsinResultRowDto();
|
||||
withCozeStatus.setStatus("success");
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withCozeStatus));
|
||||
|
||||
SimilarAsinResultRowDto failedStatus = new SimilarAsinResultRowDto();
|
||||
failedStatus.setStatus("FAILED");
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(failedStatus));
|
||||
|
||||
SimilarAsinResultRowDto withVisibleResultData = new SimilarAsinResultRowDto();
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() {
|
||||
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3));
|
||||
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("FAILED", 3, 3));
|
||||
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("RUNNING", 3, 3));
|
||||
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3));
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedResultRowsWithBlankIsConformEnableCategoryRetry() {
|
||||
SimilarAsinParsedRowVo blankCategory = new SimilarAsinParsedRowVo();
|
||||
blankCategory.setValues(new LinkedHashMap<>());
|
||||
blankCategory.getValues().put("status", "FAILED");
|
||||
blankCategory.getValues().put("is_conform", "");
|
||||
|
||||
assertTrue(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
true));
|
||||
|
||||
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
false));
|
||||
|
||||
blankCategory.getValues().put("is_conform", "符合");
|
||||
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
true));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstPassImageResultWorkbookKeepsAllRowsForSecondParse() {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setValues(new LinkedHashMap<>());
|
||||
row.getValues().put("状态", "成功");
|
||||
row.getValues().put("是否有货", "");
|
||||
row.getValues().put("相似度", "");
|
||||
row.getValues().put("是否符合类目", "");
|
||||
row.getValues().put("不符合理由", "");
|
||||
row.getValues().put("产品类目", "");
|
||||
|
||||
assertTrue(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
|
||||
row.getValues().put("是否符合类目", "符合");
|
||||
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
|
||||
row.getValues().put("是否符合类目", "");
|
||||
row.getValues().put("状态", "失败");
|
||||
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
}
|
||||
|
||||
private int staticIntField(String name) throws Exception {
|
||||
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
return field.getInt(null);
|
||||
}
|
||||
}
|
||||
-269
@@ -1,269 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 8:WorkbookFactory 输入解析改为受控读取。
|
||||
* 解析前先探测 zip 条目数与解压体积,超过配置上限直接拒绝并给出可识别失败提示,
|
||||
* 避免超大/恶意 Excel 直接进入 WorkbookFactory 全量加载导致内存无界增长。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceWorkbookControlTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(50000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-workbook-ctrl-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0WBK%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("workbook.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_default_path() throws Exception {
|
||||
// 正常 xlsx:受控读取通过探测,解析成功且行数不丢失
|
||||
File workbook = buildWorkbook(100);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-default.xlsx");
|
||||
assertEquals(100, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getTotalRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_multiple_items() throws Exception {
|
||||
// 多文件批量:每个文件都走受控读取,汇总不丢行
|
||||
File workbookA = buildWorkbook(25);
|
||||
File workbookB = buildWorkbook(35);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-a.xlsx")).thenReturn(workbookA);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-b.xlsx")).thenReturn(workbookB);
|
||||
SimilarAsinParseRequest request = request("uploads/20260829/wb-a.xlsx");
|
||||
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
|
||||
sourceB.setFileKey("uploads/20260829/wb-b.xlsx");
|
||||
sourceB.setOriginalFilename("workbook-b.xlsx");
|
||||
request.setFiles(List.of(request.getFiles().get(0), sourceB));
|
||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
||||
assertEquals(60, vo.getAcceptedRows());
|
||||
assertEquals(60, vo.getTotalRows());
|
||||
assertNotNull(vo.getTaskId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复解析同一文件:结果一致,不产生重复状态
|
||||
File workbook = buildWorkbook(50);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/wb-idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/wb-idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_empty_input() throws Exception {
|
||||
// 空文件(只有表头无数据行):抛业务异常,不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-empty.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-empty.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
|
||||
// 文件不存在:抛业务异常
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-missing.xlsx")).thenReturn(null);
|
||||
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/wb-missing.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_single_item() throws Exception {
|
||||
// 单行文件:不依赖批量路径,结果正确
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_limit_and_overflow() throws Exception {
|
||||
// 解压体积超限:受控探测阶段拒绝,失败提示可识别,不发生无界内存增长
|
||||
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(1024L);
|
||||
File workbook = buildWorkbook(5);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-huge.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-huge.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && (ex.getMessage().contains("解压") || ex.getMessage().contains("大小")),
|
||||
"超大 Excel 失败提示必须可识别,实际: " + ex.getMessage());
|
||||
// 条目数超限:同样在探测阶段拒绝
|
||||
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
when(properties.getMaxWorkbookZipEntries()).thenReturn(2);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-entries.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex2 = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-entries.xlsx"));
|
||||
assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("条目"),
|
||||
"超条目数失败提示必须可识别,实际: " + ex2.getMessage());
|
||||
// 恢复默认配置:同一文件解析成功(探测不残留状态)
|
||||
when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered.xlsx");
|
||||
assertEquals(5, vo.getAcceptedRows());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_invalid_input_rejected() throws Exception {
|
||||
// 非 xlsx 内容(文本文件):WorkbookFactory 打开失败,抛项目约定异常
|
||||
File fake = Files.createTempFile("similar-asin-not-excel-", ".xlsx").toFile();
|
||||
Files.write(fake.toPath(), "this is not an excel file".getBytes(StandardCharsets.UTF_8));
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fake.xlsx")).thenReturn(fake);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(fake, "uploads/20260829/wb-fake.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"),
|
||||
"损坏文件失败提示必须可识别,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_dependency_failure_releases_resources() throws Exception {
|
||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,受控读取无残留
|
||||
File workbook = buildWorkbook(30);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/50001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/wb-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered2.xlsx");
|
||||
assertEquals(30, vo.getAcceptedRows());
|
||||
// 默认配置值处于有效区间
|
||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
||||
assertNotNull(defaults.getMaxWorkbookZipEntries());
|
||||
assertNotNull(defaults.getMaxWorkbookUncompressedBytes());
|
||||
assertTrue(defaults.getMaxWorkbookZipEntries() >= 1000, "默认条目上限至少 1000");
|
||||
assertTrue(defaults.getMaxWorkbookUncompressedBytes() >= 100L * 1024L * 1024L, "默认解压上限至少 100MB");
|
||||
}
|
||||
}
|
||||
-166
@@ -1,166 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipFile;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ExcelCellImageWriterTest {
|
||||
|
||||
@Test
|
||||
void patchedCellImageValueMetadataUsesOneBasedVmIndexes() throws Exception {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
writeMinimalWorkbook(xlsx);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
session.registerImage(1, 9, new byte[]{1, 2, 3});
|
||||
session.registerImage(1, 10, new byte[]{4, 5, 6});
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
String sheet = read(zip, "xl/worksheets/sheet1.xml");
|
||||
// cell @vm 是 [MS-XLSX] 规范里的 1-based 索引(0 表示"无 metadata")。
|
||||
assertTrue(sheet.contains("<c r=\"J2\" t=\"e\" vm=\"1\"><v>#VALUE!</v></c>"));
|
||||
assertTrue(sheet.contains("<c r=\"K2\" t=\"e\" vm=\"2\"><v>#VALUE!</v></c>"));
|
||||
|
||||
// futureMetadata 内部块仍然是 0-based。
|
||||
String metadata = read(zip, "xl/metadata.xml");
|
||||
assertTrue(metadata.contains("<xlrd:rvb i=\"0\"/>"));
|
||||
assertTrue(metadata.contains("<xlrd:rvb i=\"1\"/>"));
|
||||
assertEquals(2, maxVm(sheet));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void streamsRegisteredImagePathIntoWorkbookMedia() throws Exception {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-path-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
Path image = dir.resolve("thumb.jpeg");
|
||||
byte[] imageBytes = new byte[]{7, 8, 9, 10};
|
||||
writeMinimalWorkbook(xlsx);
|
||||
Files.write(image, imageBytes);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
session.registerImage(1, 9, image);
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
assertArrayEquals(imageBytes,
|
||||
zip.getInputStream(zip.getEntry("xl/media/excelcellimage1.jpeg")).readAllBytes());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void patchesTenThousandImageCellsWithinLinearTimeBudget() {
|
||||
assertTimeoutPreemptively(Duration.ofSeconds(10), () -> {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-linear-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
writeWorkbookWithImageCells(xlsx, 10_000);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
for (int row = 1; row <= 10_000; row++) {
|
||||
session.registerImage(row, 9, new byte[]{1});
|
||||
}
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
String sheet = read(zip, "xl/worksheets/sheet1.xml");
|
||||
assertEquals(10_000, maxVm(sheet));
|
||||
assertTrue(sheet.contains("<c r=\"J10001\" t=\"e\" vm=\"10000\"><v>#VALUE!</v></c>"));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static int maxVm(String sheet) {
|
||||
Matcher matcher = Pattern.compile("vm=\"(\\d+)\"").matcher(sheet);
|
||||
int max = -1;
|
||||
while (matcher.find()) {
|
||||
max = Math.max(max, Integer.parseInt(matcher.group(1)));
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
private static String read(ZipFile zip, String name) throws Exception {
|
||||
return new String(zip.getInputStream(zip.getEntry(name)).readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private static void writeMinimalWorkbook(Path xlsx) throws Exception {
|
||||
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
|
||||
write(out, "[Content_Types].xml", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
|
||||
<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
|
||||
</Types>
|
||||
""");
|
||||
write(out, "_rels/.rels", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
|
||||
</Relationships>
|
||||
""");
|
||||
write(out, "xl/workbook.xml", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<sheets><sheet name="Sheet1" r:id="rId1" sheetId="1"/></sheets>
|
||||
</workbook>
|
||||
""");
|
||||
write(out, "xl/_rels/workbook.xml.rels", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
|
||||
</Relationships>
|
||||
""");
|
||||
write(out, "xl/worksheets/sheet1.xml", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
<row r="2">
|
||||
<c r="J2" t="inlineStr"><is><t>main</t></is></c>
|
||||
<c r="K2" t="inlineStr"><is><t>puzzle</t></is></c>
|
||||
</row>
|
||||
</sheetData>
|
||||
</worksheet>
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeWorkbookWithImageCells(Path xlsx, int imageCount) throws Exception {
|
||||
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
|
||||
write(out, "[Content_Types].xml", "<Types></Types>");
|
||||
write(out, "xl/_rels/workbook.xml.rels", "<Relationships></Relationships>");
|
||||
StringBuilder sheet = new StringBuilder("<worksheet><sheetData>");
|
||||
for (int row = 2; row <= imageCount + 1; row++) {
|
||||
sheet.append("<row r=\"").append(row).append("\"><c r=\"J").append(row)
|
||||
.append("\" t=\"inlineStr\"><is><t>image</t></is></c></row>");
|
||||
}
|
||||
sheet.append("</sheetData></worksheet>");
|
||||
write(out, "xl/worksheets/sheet1.xml", sheet.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private static void write(ZipOutputStream out, String name, String content) throws Exception {
|
||||
out.putNextEntry(new ZipEntry(name));
|
||||
out.write(content.getBytes(StandardCharsets.UTF_8));
|
||||
out.closeEntry();
|
||||
}
|
||||
}
|
||||
-196
@@ -1,196 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizeException;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/**
|
||||
* Task 17:优化图片解码采样、像素上限和 JPEG 质量搜索,降低 CPU 与堆峰值。
|
||||
* - 解码采样:子采样从仅 JPEG 推广到全部格式,按源长边对目标长边取 2 的幂;
|
||||
* - 像素上限:新增子采样后的解码像素上限,格式忽略采样参数时拒绝全量解码,避免爆堆;
|
||||
* - 质量搜索:固定阶梯 {0.75,0.65,0.55} 改为估算搜索(0.75 后按字节比例估算质量,
|
||||
* 最多每长边 2 次编码),最坏编码次数 9 → 6,典型 1-2 次即命中。
|
||||
*/
|
||||
class SimilarAsinImageEmbedderDecodeQualityTest {
|
||||
|
||||
private SimilarAsinImageEmbedder embedder;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
embedder = new SimilarAsinImageEmbedder(new SimilarAsinProperties(), mock(OssStorageService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static byte[] createImage(int width, int height, String format, Color color) throws Exception {
|
||||
BufferedImage img = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = img.createGraphics();
|
||||
try {
|
||||
g.setColor(color);
|
||||
g.fillRect(0, 0, width, height);
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ImageIO.write(img, format, baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_normal_default_path() throws Exception {
|
||||
// 正常输入:JPEG 按源尺寸取 2 的幂子采样,采样后解码像素不超上限,resize 结果长边 = 目标长边。
|
||||
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
|
||||
assertEquals(2, subsampling, "3200x2400 JPEG 子采样应为 2");
|
||||
assertEquals(subsampling, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400),
|
||||
"JPEG 专用子采样应与通用子采样一致");
|
||||
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
|
||||
assertEquals(1600L * 1200L, decodedPixels, "子采样后解码像素 = 1600x1200");
|
||||
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "解码像素不得超上限");
|
||||
|
||||
ResizedImage thumb = embedder.resizeImage("https://example.com/default.jpg",
|
||||
createImage(1200, 1600, "jpg", new Color(0x33, 0x66, 0x99)));
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(thumb.width(), thumb.height()),
|
||||
"长边应缩放到目标 1280");
|
||||
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES, "字节不得超上限");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_normal_multiple_items() throws Exception {
|
||||
// 批量/多格式:非 JPEG(PNG)同样按源尺寸子采样,采样后解码像素受控,resize 结果不丢失。
|
||||
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
|
||||
assertEquals(2, subsampling, "PNG 输入同样应用子采样计算");
|
||||
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
|
||||
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "PNG 解码像素不得超上限");
|
||||
|
||||
ResizedImage png = embedder.resizeImage("https://example.com/multi.png",
|
||||
createImage(3200, 2400, "png", new Color(0x99, 0x33, 0x66)));
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(png.width(), png.height()));
|
||||
assertTrue(png.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行:同一输入两次 resize 字节一致;质量估算纯函数输出确定。
|
||||
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x22, 0x44));
|
||||
ResizedImage first = embedder.resizeImage("https://example.com/idem.jpg", raw);
|
||||
ResizedImage second = embedder.resizeImage("https://example.com/idem.jpg", raw);
|
||||
assertArrayEquals(first.bytes(), second.bytes(), "重复 resize 必须产生相同字节");
|
||||
assertEquals(first.width(), second.width());
|
||||
assertEquals(first.height(), second.height());
|
||||
|
||||
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(120000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
"估算质量不得超过 0.75 上限");
|
||||
assertEquals(0.576f,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(200000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
0.001f, "200KB 超出上限时按字节比例估算质量");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_boundary_empty_input() throws Exception {
|
||||
// 空输入:空字节数组拒绝且抛可识别异常;非法尺寸校验返回可识别异常,不创建资源。
|
||||
byte[] empty = new byte[0];
|
||||
Exception ex = assertThrows(Exception.class,
|
||||
() -> embedder.resizeImage("https://example.com/empty.jpg", empty));
|
||||
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
|
||||
"应为 IOException 或 RuntimeException 兜底,实际=" + ex.getClass().getSimpleName());
|
||||
assertTrue(ex.getMessage() == null || ex.getMessage().toLowerCase().contains("unsupported"),
|
||||
"空输入消息应反映不支持格式");
|
||||
|
||||
ResizeException dimEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/empty.jpg", 0, 100, 1));
|
||||
assertTrue(dimEx.getMessage().contains("invalid image dimensions"), "非法尺寸消息应可识别");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_boundary_single_item() throws Exception {
|
||||
// 单图:小于目标长边的输入不放大,子采样 = 1,结果尺寸保持源尺寸。
|
||||
ResizedImage thumb = embedder.resizeImage("https://example.com/single.jpg",
|
||||
createImage(800, 600, "jpg", new Color(0x55, 0xaa, 0x33)));
|
||||
assertEquals(1, SimilarAsinImageEmbedder.sourceSubsampling(800, 600), "小图子采样应为 1");
|
||||
assertEquals(800, thumb.width(), "小图长边保持源尺寸,不放大");
|
||||
assertEquals(600, thumb.height());
|
||||
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_boundary_limit_and_overflow() throws Exception {
|
||||
// 上限/超限:源像素超上限拒绝;子采样后解码像素超上限拒绝;正常值放行。
|
||||
ResizeException sourceEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage(
|
||||
"https://example.com/huge.jpg", 7000, 7000, 1));
|
||||
assertTrue(sourceEx.getMessage().contains("image too large"), "源像素超限消息应可识别");
|
||||
|
||||
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 6000, 6000, 4);
|
||||
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 1200, 1600, 1);
|
||||
|
||||
ResizeException decodedEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage(
|
||||
"https://example.com/no-subsample.jpg", 3000, 3000, 1));
|
||||
assertTrue(decodedEx.getMessage().contains("decode too large"), "解码像素超限消息应可识别");
|
||||
|
||||
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/subsampled.jpg", 3000, 3000, 2);
|
||||
|
||||
ResizedImage normal = embedder.resizeImage("https://example.com/normal.jpg",
|
||||
createImage(1500, 1500, "jpg", new Color(0x20, 0x40, 0x60)));
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(normal.width(), normal.height()),
|
||||
"正常尺寸图片不得被误拒");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_invalid_input_rejected() throws Exception {
|
||||
// 非法参数:负尺寸拒绝;质量估算越界钳制到上下限。
|
||||
ResizeException negEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/neg.jpg", -1, 100, 1));
|
||||
assertTrue(negEx.getMessage().contains("invalid image dimensions"));
|
||||
|
||||
assertEquals(SimilarAsinImageEmbedder.MIN_JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(300000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
"估算质量低于下限时钳制到 MIN_JPEG_QUALITY");
|
||||
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(0, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
"非法字节数回退默认质量");
|
||||
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(10000, 0),
|
||||
"非法上限回退默认质量");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:截断图片解码失败后抛 IOException,图像处理槽位释放,恢复后重试成功。
|
||||
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x33, 0x77));
|
||||
byte[] truncated = Arrays.copyOf(raw, 64);
|
||||
|
||||
Exception ex = assertThrows(Exception.class,
|
||||
() -> embedder.resizeImage("https://example.com/truncated.jpg", truncated));
|
||||
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
|
||||
"截断图片解码失败应为 IOException 或 RuntimeException 兜底");
|
||||
|
||||
ResizedImage recovered = embedder.resizeImage("https://example.com/recovered.jpg", raw);
|
||||
assertNotNull(recovered, "失败后槽位必须释放,恢复重试成功");
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(recovered.width(), recovered.height()));
|
||||
assertTrue(recovered.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
|
||||
}
|
||||
}
|
||||
-223
@@ -1,223 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ImageSpool;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
/**
|
||||
* Task 16:图片预取改为短预算 best-effort,超时后直接回退 URL。
|
||||
* 新入口 prefetchToDiskBestEffort 在短预算内尽力预取,预算耗尽即停、
|
||||
* 取消在途任务并返回未预取数量;缺图单元格按既有 fallback 直接写 URL,
|
||||
* 不阻塞、不发生无界等待。长预算旧入口 prefetchToDisk 行为不变。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinImageEmbedderPrefetchBudgetTest {
|
||||
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
|
||||
private SimilarAsinImageEmbedder embedder;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.getImageDownloadTimeoutSeconds()).thenReturn(5);
|
||||
lenient().when(properties.getImageDownloadPoolSize()).thenReturn(2);
|
||||
lenient().when(properties.getImagePrefetchTimeoutSeconds()).thenReturn(1800);
|
||||
lenient().when(ossStorageService.normalizeManagedPublicUrl(anyString()))
|
||||
.thenAnswer(invocation -> invocation.getArgument(0));
|
||||
embedder = new SimilarAsinImageEmbedder(properties, ossStorageService);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static ImageSpool newSpool() throws Exception {
|
||||
return new ImageSpool(java.nio.file.Files.createTempDirectory("prefetch-budget-test-"));
|
||||
}
|
||||
|
||||
private static ResizedImage resizedImage(int seed) {
|
||||
return new ResizedImage(new byte[]{(byte) seed}, seed, seed);
|
||||
}
|
||||
|
||||
/** 通过反射调用私有 best-effort 入口,返回 skipped(未预取)数量。 */
|
||||
private static int invokeBestEffort(SimilarAsinImageEmbedder e, List<String> urls,
|
||||
ImageSpool spool, long budgetSeconds) throws Exception {
|
||||
Method m = SimilarAsinImageEmbedder.class.getDeclaredMethod(
|
||||
"prefetchToDiskBestEffort", java.util.Collection.class, ImageSpool.class, long.class);
|
||||
m.setAccessible(true);
|
||||
return (int) m.invoke(e, urls, spool, budgetSeconds);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_normal_default_path() throws Exception {
|
||||
// 正常输入:预算内完成预取,spool 全部填充、无 skipped、无异常。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(1));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
int skipped = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
|
||||
assertEquals(0, skipped, "预算内全部完成,无 skipped");
|
||||
assertNotNull(spool.get(urls.get(0)));
|
||||
assertNotNull(spool.get(urls.get(1)));
|
||||
assertEquals(2, spool.size());
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多个 url 顺序稳定、结果不丢失。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 8; i++) {
|
||||
urls.add("https://img.example.com/multi-" + i + ".jpg");
|
||||
final int idx = i;
|
||||
embedder.registerPrefetchHandler(urls.get(i), () -> resizedImage(idx + 10));
|
||||
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx + 10));
|
||||
}
|
||||
|
||||
int skipped = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
|
||||
assertEquals(0, skipped);
|
||||
for (int i = 0; i < 8; i++) {
|
||||
assertNotNull(spool.get(urls.get(i)), "批量预取结果不丢失: " + urls.get(i));
|
||||
}
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:spool 已缓存的不重复下载,结果一致。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/idem.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(7));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(7));
|
||||
|
||||
int first = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
int second = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
|
||||
assertEquals(0, first);
|
||||
assertEquals(0, second);
|
||||
assertEquals(1, spool.size(), "重复预取不产生重复条目");
|
||||
assertNotNull(spool.get(urls.get(0)));
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空列表安全跳过,不创建任何 spool 条目。
|
||||
ImageSpool spool = newSpool();
|
||||
assertEquals(0, invokeBestEffort(embedder, null, spool, 10L));
|
||||
assertEquals(0, invokeBestEffort(embedder, List.of(), spool, 10L));
|
||||
assertEquals(0, spool.size());
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_boundary_single_item() throws Exception {
|
||||
// 单 url:不依赖批量路径,预算内完成。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/single.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(3));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(3));
|
||||
|
||||
assertEquals(0, invokeBestEffort(embedder, urls, spool, 10L));
|
||||
assertNotNull(spool.get(urls.get(0)));
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_boundary_limit_and_overflow() throws Exception {
|
||||
// 超限/超时:预算不足时提前停止、取消在途任务,返回未预取数量,不发生无界等待。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++) {
|
||||
urls.add("https://img.example.com/slow-" + i + ".jpg");
|
||||
final int idx = i;
|
||||
embedder.registerPrefetchHandler(urls.get(i), () -> {
|
||||
try {
|
||||
Thread.sleep(3000L);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx));
|
||||
}
|
||||
|
||||
int skipped = invokeBestEffort(embedder, urls, spool, 1L);
|
||||
|
||||
assertTrue(skipped > 0, "短预算下必须提前放弃部分 url,实际 skipped=" + skipped);
|
||||
assertTrue(skipped <= 6);
|
||||
long elapsedMs = System.currentTimeMillis();
|
||||
assertTrue(elapsedMs > 0, "预取应在短预算附近结束");
|
||||
assertTrue(spool.size() <= 2, "预算耗尽时只完成已开始的少量任务,不发生无界等待");
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_invalid_input_rejected() throws Exception {
|
||||
// 非法输入:null/空白 url 跳过;spool 为 null 时安全返回 0,不创建资源。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> badUrls = java.util.Arrays.asList(null, " ", "https://img.example.com/ok.jpg");
|
||||
embedder.registerPrefetchHandler("https://img.example.com/ok.jpg", () -> resizedImage(5));
|
||||
embedder.registerPrefetchResult("https://img.example.com/ok.jpg", resizedImage(5));
|
||||
|
||||
assertEquals(0, invokeBestEffort(embedder, badUrls, spool, 10L));
|
||||
assertEquals(1, spool.size(), "空白 url 跳过,有效 url 正常预取");
|
||||
assertEquals(0, invokeBestEffort(embedder, badUrls, null, 10L), "spool 为 null 安全返回");
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:单个 url 预取失败不阻断其余 url;恢复后重试成功。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/fail.jpg", "https://img.example.com/ok.jpg");
|
||||
AtomicInteger failCalls = new AtomicInteger(0);
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> {
|
||||
if (failCalls.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("http down");
|
||||
}
|
||||
});
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
assertEquals(0, invokeBestEffort(embedder, urls, spool, 10L), "失败 url 不阻断其余 url");
|
||||
assertNull(spool.get(urls.get(0)), "失败 url 不落 spool");
|
||||
assertNotNull(spool.get(urls.get(1)), "正常 url 正常落 spool");
|
||||
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
assertEquals(0, invokeBestEffort(embedder, List.of(urls.get(0)), spool, 10L), "恢复后重试成功");
|
||||
assertNotNull(spool.get(urls.get(0)), "恢复后失败 url 预取成功");
|
||||
spool.close();
|
||||
}
|
||||
}
|
||||
-226
@@ -1,226 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ImageSpool;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/**
|
||||
* Task 18:统一图片 spool 生命周期,确保超时、取消和异常路径删除临时文件。
|
||||
* - ImageSpool 增加 closed 状态:close 后 put 拒绝、close 幂等;
|
||||
* - 长预算 prefetchToDisk 在 deadline/中断后仍会把在途任务写入的残留文件清理掉
|
||||
* (ImageSpool.cleanupOrphanFiles 只清理未被索引的文件,已索引文件由 close 兜底);
|
||||
* - 超时路径取消在途任务后,无新文件产生(put 前中断检查),close 后可删除目录。
|
||||
*/
|
||||
class SimilarAsinImageEmbedderSpoolLifecycleTest {
|
||||
|
||||
private SimilarAsinImageEmbedder embedder;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
embedder = new SimilarAsinImageEmbedder(new SimilarAsinProperties(), mock(OssStorageService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static ImageSpool newSpool() throws Exception {
|
||||
return new ImageSpool(Files.createTempDirectory("spool-lifecycle-test-"));
|
||||
}
|
||||
|
||||
private static ResizedImage resizedImage(int seed) {
|
||||
return new ResizedImage(new byte[]{(byte) seed, (byte) seed, (byte) seed, (byte) seed}, seed, seed);
|
||||
}
|
||||
|
||||
/** 通过反射调用私有长预算预取入口。 */
|
||||
private static void invokePrefetchToDisk(SimilarAsinImageEmbedder e, List<String> urls,
|
||||
ImageSpool spool) throws Exception {
|
||||
Method m = SimilarAsinImageEmbedder.class.getDeclaredMethod(
|
||||
"prefetchToDisk", java.util.Collection.class, ImageSpool.class);
|
||||
m.setAccessible(true);
|
||||
m.invoke(e, urls, spool);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_normal_default_path() throws Exception {
|
||||
// 正常输入:预取落盘 → close 删除临时目录与全部文件。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(1));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
invokePrefetchToDisk(embedder, urls, spool);
|
||||
|
||||
assertEquals(2, spool.size(), "预取结果全部落盘");
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "close 后临时目录必须删除");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_normal_multiple_items() throws Exception {
|
||||
// 批量场景:100 个 url 全部落盘,close 后目录清空。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
urls.add("https://img.example.com/multi-" + i + ".jpg");
|
||||
final int idx = i;
|
||||
embedder.registerPrefetchHandler(urls.get(i), () -> resizedImage(idx));
|
||||
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx));
|
||||
}
|
||||
|
||||
invokePrefetchToDisk(embedder, urls, spool);
|
||||
|
||||
assertEquals(100, spool.size());
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行:close 幂等;已索引文件不重复写入(size 不增长)。
|
||||
ImageSpool spool = newSpool();
|
||||
String url = "https://img.example.com/idem.jpg";
|
||||
embedder.registerPrefetchHandler(url, () -> resizedImage(7));
|
||||
embedder.registerPrefetchResult(url, resizedImage(7));
|
||||
|
||||
invokePrefetchToDisk(embedder, List.of(url), spool);
|
||||
assertEquals(1, spool.size());
|
||||
spool.close();
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "close 幂等,二次 close 不抛异常");
|
||||
|
||||
ImageSpool fresh = newSpool();
|
||||
invokePrefetchToDisk(embedder, List.of(url), fresh);
|
||||
invokePrefetchToDisk(embedder, List.of(url), fresh);
|
||||
assertEquals(1, fresh.size(), "重复预取同一 url 不产生重复文件");
|
||||
fresh.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_boundary_empty_input() throws Exception {
|
||||
// 空输入:空列表预取安全跳过;close 对空 spool 幂等删除。
|
||||
ImageSpool spool = newSpool();
|
||||
invokePrefetchToDisk(embedder, List.of(), spool);
|
||||
invokePrefetchToDisk(embedder, null, spool);
|
||||
assertEquals(0, spool.size());
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_boundary_single_item() throws Exception {
|
||||
// 单 url:不依赖批量路径,close 后目录删除。
|
||||
ImageSpool spool = newSpool();
|
||||
String url = "https://img.example.com/single.jpg";
|
||||
embedder.registerPrefetchHandler(url, () -> resizedImage(3));
|
||||
embedder.registerPrefetchResult(url, resizedImage(3));
|
||||
|
||||
invokePrefetchToDisk(embedder, List.of(url), spool);
|
||||
assertEquals(1, spool.size());
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_boundary_limit_and_overflow() throws Exception {
|
||||
// 超时/取消:阻塞 handler 在 deadline 后被取消,取消后不产生新文件;
|
||||
// 取消瞬间已在途的写入文件由 close 兜底清理,目录可完整删除。
|
||||
ImageSpool spool = newSpool();
|
||||
String slow = "https://img.example.com/slow.jpg";
|
||||
embedder.registerPrefetchHandler(slow, () -> {
|
||||
try {
|
||||
TimeUnit.SECONDS.sleep(30L);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
|
||||
Thread caller = new Thread(() -> {
|
||||
try {
|
||||
invokePrefetchToDisk(embedder, List.of(slow), spool);
|
||||
} catch (Exception ignored) {
|
||||
// 中断或超时路径允许异常
|
||||
}
|
||||
});
|
||||
caller.start();
|
||||
caller.join(TimeUnit.SECONDS.toMillis(2));
|
||||
caller.interrupt();
|
||||
caller.join(TimeUnit.SECONDS.toMillis(5));
|
||||
|
||||
try (var paths = Files.walk(spool.directory())) {
|
||||
long files = paths.filter(Files::isRegularFile).count();
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "取消后 close 必须能删除整个目录,残留文件数=" + files);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_invalid_input_rejected() throws Exception {
|
||||
// 非法参数:null url 预取跳过;close 后 put 抛 IOException(已识别消息)。
|
||||
ImageSpool spool = newSpool();
|
||||
invokePrefetchToDisk(embedder, java.util.Arrays.asList(null, " "), spool);
|
||||
assertEquals(0, spool.size());
|
||||
spool.close();
|
||||
|
||||
IOException ioEx = assertThrows(IOException.class,
|
||||
() -> spool.put("https://img.example.com/late.jpg", resizedImage(9)));
|
||||
assertTrue(ioEx.getMessage().contains("closed"), "close 后写入应拒绝,消息含 closed");
|
||||
assertNull(spool.get("https://img.example.com/late.jpg"), "close 后 get 返回 null");
|
||||
assertTrue(spool.size() == 0, "close 后 size 为 0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:预取中单个 url 抛异常不阻断其余;失败路径无残留文件;
|
||||
// 中断后的恢复重试成功;close 删除目录。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/fail.jpg", "https://img.example.com/ok.jpg");
|
||||
AtomicInteger failCalls = new AtomicInteger(0);
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> {
|
||||
if (failCalls.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("http down");
|
||||
}
|
||||
});
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
invokePrefetchToDisk(embedder, urls, spool);
|
||||
|
||||
assertNull(spool.get(urls.get(0)), "失败 url 不落 spool");
|
||||
assertNotNull(spool.get(urls.get(1)), "正常 url 正常落 spool");
|
||||
assertEquals(1, spool.size(), "失败路径不残留文件");
|
||||
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
invokePrefetchToDisk(embedder, List.of(urls.get(0)), spool);
|
||||
assertNotNull(spool.get(urls.get(0)), "恢复后重试成功");
|
||||
assertEquals(2, spool.size());
|
||||
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "失败+恢复后 close 仍能删除目录");
|
||||
}
|
||||
}
|
||||
-542
@@ -1,542 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Protocol;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.ResponseBody;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class SimilarAsinImageEmbedderTest {
|
||||
|
||||
// properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底。
|
||||
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService());
|
||||
|
||||
@AfterEach
|
||||
void shutDownEmbedder() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static OssStorageService createOssStorageService() {
|
||||
OssProperties properties = new OssProperties();
|
||||
properties.setEndpoint("https://oss.aishufu.top");
|
||||
properties.setPublicEndpoint("https://oss.aishufu.top");
|
||||
properties.setBucket("nanri-ai-images");
|
||||
properties.setImageVideoBucket("shufu-video");
|
||||
properties.setDigitalHumanBucket("nanri-ai-digital-human");
|
||||
properties.setAccessKeyId("test-access-key");
|
||||
properties.setAccessKeySecret("test-secret-key");
|
||||
return new OssStorageService(properties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultsImageDownloadPoolToTwoAndCapsItByVisibleCpu() {
|
||||
assertEquals(2, new SimilarAsinProperties().getImageDownloadPoolSize());
|
||||
assertEquals(Math.min(2, SimilarAsinImageEmbedder.cpuBoundPoolLimit()), embedder.downloadPoolSize());
|
||||
}
|
||||
|
||||
@Test
|
||||
void clampsOversizedImagePoolConfigurationByVisibleCpu() {
|
||||
SimilarAsinProperties configured = new SimilarAsinProperties();
|
||||
configured.setImageDownloadPoolSize(Integer.MAX_VALUE);
|
||||
SimilarAsinImageEmbedder limited = new SimilarAsinImageEmbedder(configured, createOssStorageService());
|
||||
try {
|
||||
assertEquals(SimilarAsinImageEmbedder.cpuBoundPoolLimit(), limited.downloadPoolSize());
|
||||
} finally {
|
||||
limited.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrentRequestsForSameUrlDownloadAndResizeOnlyOnce() throws Exception {
|
||||
SimilarAsinImageEmbedder shared = new SimilarAsinImageEmbedder(
|
||||
properties(2, 5, null), createOssStorageService());
|
||||
AtomicInteger networkCalls = new AtomicInteger();
|
||||
CountDownLatch releaseNetwork = new CountDownLatch(1);
|
||||
CountDownLatch firstNetworkCall = new CountDownLatch(1);
|
||||
replaceHttpClient(shared, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
networkCalls.incrementAndGet();
|
||||
firstNetworkCall.countDown();
|
||||
try {
|
||||
if (!releaseNetwork.await(2, TimeUnit.SECONDS)) {
|
||||
throw new IOException("test network release timed out");
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("test network interrupted", ex);
|
||||
}
|
||||
return response(chain.request(), 200, "OK", createJpegBytes());
|
||||
})
|
||||
.build());
|
||||
ExecutorService callers = Executors.newFixedThreadPool(2);
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
try {
|
||||
Future<SimilarAsinImageEmbedder.ResizedImage> first = callers.submit(() -> {
|
||||
start.await();
|
||||
return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
|
||||
});
|
||||
Future<SimilarAsinImageEmbedder.ResizedImage> second = callers.submit(() -> {
|
||||
start.await();
|
||||
return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
|
||||
});
|
||||
|
||||
start.countDown();
|
||||
assertTrue(firstNetworkCall.await(1, TimeUnit.SECONDS));
|
||||
Thread.sleep(100L);
|
||||
releaseNetwork.countDown();
|
||||
|
||||
assertNotNull(first.get(2, TimeUnit.SECONDS));
|
||||
assertNotNull(second.get(2, TimeUnit.SECONDS));
|
||||
assertEquals(1, networkCalls.get());
|
||||
} finally {
|
||||
releaseNetwork.countDown();
|
||||
callers.shutdownNow();
|
||||
shared.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void largeJpegDecodeUsesPowerOfTwoSourceSubsampling() {
|
||||
assertEquals(1, SimilarAsinImageEmbedder.jpegSourceSubsampling(2559, 1200));
|
||||
assertEquals(2, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400));
|
||||
assertEquals(4, SimilarAsinImageEmbedder.jpegSourceSubsampling(6000, 4000));
|
||||
}
|
||||
|
||||
@Test
|
||||
void writesFetchedThumbnailToPersistentCacheForNextEmbedderInstance() throws Exception {
|
||||
Path cacheDir = Files.createTempDirectory("similar-asin-image-cache-test-");
|
||||
String url = "https://images.example.com/product.jpg";
|
||||
byte[] sourceImage = createJpegBytes();
|
||||
AtomicInteger firstNetworkCalls = new AtomicInteger();
|
||||
AtomicInteger secondNetworkCalls = new AtomicInteger();
|
||||
SimilarAsinImageEmbedder first = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, cacheDir), createOssStorageService());
|
||||
SimilarAsinImageEmbedder second = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, cacheDir), createOssStorageService());
|
||||
try {
|
||||
replaceHttpClient(first, respondingClient(firstNetworkCalls, sourceImage));
|
||||
SimilarAsinImageEmbedder.ResizedImage fetched = first.fetchAndResizeForCache(url);
|
||||
|
||||
assertNotNull(fetched);
|
||||
assertEquals(1, firstNetworkCalls.get());
|
||||
try (var cachedFiles = Files.walk(cacheDir)) {
|
||||
assertEquals(1L, cachedFiles.filter(Files::isRegularFile).count());
|
||||
}
|
||||
|
||||
replaceHttpClient(second, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
secondNetworkCalls.incrementAndGet();
|
||||
throw new AssertionError("persistent cache miss triggered a second network request");
|
||||
})
|
||||
.build());
|
||||
SimilarAsinImageEmbedder.ResizedImage cached = second.fetchAndResizeForCache(url);
|
||||
|
||||
assertNotNull(cached);
|
||||
assertArrayEquals(fetched.bytes(), cached.bytes());
|
||||
assertEquals(fetched.width(), cached.width());
|
||||
assertEquals(fetched.height(), cached.height());
|
||||
assertEquals(0, secondNetworkCalls.get());
|
||||
} finally {
|
||||
first.shutdown();
|
||||
second.shutdown();
|
||||
deleteRecursively(cacheDir);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void diskPrefetchDeadlineCancelsInFlightDownload() throws Exception {
|
||||
SimilarAsinImageEmbedder deadlineEmbedder = new SimilarAsinImageEmbedder(
|
||||
properties(1, 1, null), createOssStorageService());
|
||||
CountDownLatch downloadStarted = new CountDownLatch(1);
|
||||
CountDownLatch cancellationObserved = new CountDownLatch(1);
|
||||
replaceHttpClient(deadlineEmbedder, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
downloadStarted.countDown();
|
||||
try {
|
||||
new CountDownLatch(1).await();
|
||||
throw new AssertionError("blocking download unexpectedly completed");
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
cancellationObserved.countDown();
|
||||
throw new IOException("cancelled", ex);
|
||||
}
|
||||
})
|
||||
.build());
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-deadline-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir)) {
|
||||
assertTimeoutPreemptively(Duration.ofSeconds(3), () -> deadlineEmbedder.prefetchToDisk(
|
||||
List.of("https://images.example.com/slow.jpg"), spool));
|
||||
|
||||
assertTrue(downloadStarted.await(100, TimeUnit.MILLISECONDS));
|
||||
assertTrue(cancellationObserved.await(1, TimeUnit.SECONDS),
|
||||
"deadline should interrupt the active image download");
|
||||
assertEquals(0, spool.size());
|
||||
} finally {
|
||||
deadlineEmbedder.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedDiskPrefetchIsNotDownloadedAgainWhileEmbedding() throws Exception {
|
||||
SimilarAsinImageEmbedder failedEmbedder = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, null), createOssStorageService());
|
||||
AtomicInteger networkCalls = new AtomicInteger();
|
||||
replaceHttpClient(failedEmbedder, failingClient(networkCalls));
|
||||
String url = "https://images.example.com/missing.jpg";
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-failed-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
failedEmbedder.prefetchToDisk(List.of(url), spool);
|
||||
int callsAfterPrefetch = networkCalls.get();
|
||||
var row = workbook.createSheet().createRow(1);
|
||||
|
||||
SimilarAsinImageEmbedder.ImageDim result = failedEmbedder.embedAsExcelCellImage(
|
||||
1, 9, url, row, new HashMap<>(), ExcelCellImageWriter.createSession(), spool);
|
||||
|
||||
assertEquals(SimilarAsinImageEmbedder.DOWNLOAD_MAX_RETRY + 1, callsAfterPrefetch);
|
||||
assertEquals(callsAfterPrefetch, networkCalls.get());
|
||||
assertNull(result);
|
||||
assertEquals(url, row.getCell(9).getStringCellValue());
|
||||
} finally {
|
||||
failedEmbedder.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void imageSpoolWritesThumbnailAndDeletesTaskDirectoryOnClose() throws Exception {
|
||||
Path directory = Files.createTempDirectory("similar-asin-image-spool-test-");
|
||||
byte[] bytes = new byte[]{1, 3, 5, 7};
|
||||
SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(directory);
|
||||
try {
|
||||
SimilarAsinImageEmbedder.SpoolImage image = spool.put(
|
||||
"https://example.com/image.jpg",
|
||||
new SimilarAsinImageEmbedder.ResizedImage(bytes, 120, 80));
|
||||
|
||||
assertEquals(1, spool.size());
|
||||
assertEquals(120, image.width());
|
||||
assertEquals(80, image.height());
|
||||
assertArrayEquals(bytes, Files.readAllBytes(image.path()));
|
||||
} finally {
|
||||
spool.close();
|
||||
}
|
||||
|
||||
assertFalse(Files.exists(directory));
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesLegacyMinioUrlBeforeHttpsValidation() {
|
||||
String normalized = embedder.normalizeAndValidateDownloadUrl(
|
||||
"http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg");
|
||||
|
||||
assertEquals("https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg", normalized);
|
||||
}
|
||||
|
||||
@Test
|
||||
void leavesUnmanagedHttpUrlBlocked() {
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> embedder.normalizeAndValidateDownloadUrl("http://example.com/main.jpg"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resizeImageProducesThumbnailUnderHardCap() throws Exception {
|
||||
BufferedImage src = new BufferedImage(1200, 1600, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = src.createGraphics();
|
||||
try {
|
||||
g.setColor(new Color(0x33, 0x66, 0x99));
|
||||
g.fillRect(0, 0, 1200, 1600);
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ImageIO.write(src, "jpg", baos);
|
||||
|
||||
SimilarAsinImageEmbedder.ResizedImage thumb = embedder.resizeImage("https://example.com/big.jpg", baos.toByteArray());
|
||||
|
||||
assertNotNull(thumb);
|
||||
assertNotNull(thumb.bytes());
|
||||
assertTrue(thumb.bytes().length > 0, "thumb 不可为空");
|
||||
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES,
|
||||
"thumb 字节应 <= 300KB 上限,实际=" + thumb.bytes().length);
|
||||
|
||||
// 验证返回的像素尺寸与实际解码一致
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(thumb.width(), thumb.height()),
|
||||
"长边应等于 TARGET_LONG_EDGE_PX=" + SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX);
|
||||
BufferedImage decoded = ImageIO.read(new java.io.ByteArrayInputStream(thumb.bytes()));
|
||||
assertNotNull(decoded, "thumb 应可被 ImageIO 重新解码");
|
||||
assertEquals(thumb.width(), decoded.getWidth(), "返回宽度应与解码宽度一致");
|
||||
assertEquals(thumb.height(), decoded.getHeight(), "返回高度应与解码高度一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void resizeOversizeExceptionCarriesUrlAndSize() {
|
||||
SimilarAsinImageEmbedder.ResizeOversizeException ex =
|
||||
new SimilarAsinImageEmbedder.ResizeOversizeException("https://example.com/huge.jpg", 99999);
|
||||
assertEquals("https://example.com/huge.jpg", ex.url());
|
||||
assertEquals(99999, ex.size());
|
||||
assertTrue(ex.getMessage().contains("oversize"));
|
||||
assertTrue(ex.getMessage().contains("99999"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resizeUnsupportedFormatThrowsIOException() {
|
||||
byte[] notAnImage = "not an image at all, just some bytes".getBytes();
|
||||
Exception ex = assertThrows(Exception.class,
|
||||
() -> embedder.resizeImage("https://example.com/broken.bin", notAnImage));
|
||||
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
|
||||
"应为 IOException 或 RuntimeException 兜底,实际=" + ex.getClass().getSimpleName());
|
||||
assertTrue(ex.getMessage() == null
|
||||
|| ex.getMessage().toLowerCase().contains("unsupported")
|
||||
|| ex.getMessage().toLowerCase().contains("unable"),
|
||||
"异常消息应反映不支持/无法解析");
|
||||
}
|
||||
|
||||
@Test
|
||||
void safeDnsRejectsPrivateIpsOnLookup() {
|
||||
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
|
||||
hostname -> Collections.singletonList(InetAddress.getByName("127.0.0.1")));
|
||||
UnknownHostException ex = assertThrows(UnknownHostException.class,
|
||||
() -> safeDns.lookup("rebound.example.com"));
|
||||
assertTrue(ex.getMessage().contains("blocked private/loopback ip"),
|
||||
"异常消息应说明被阻断,实际=" + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void safeDnsAllowsPublicIpResolution() throws Exception {
|
||||
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
|
||||
hostname -> Collections.singletonList(InetAddress.getByName("8.8.8.8")));
|
||||
List<InetAddress> result = safeDns.lookup("dns.google");
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("8.8.8.8", result.get(0).getHostAddress());
|
||||
}
|
||||
|
||||
@Test
|
||||
void safeDnsRejectsMixedResultsContainingPrivateIp() {
|
||||
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
|
||||
hostname -> List.of(
|
||||
InetAddress.getByName("8.8.8.8"),
|
||||
InetAddress.getByName("10.0.0.5")));
|
||||
// 任一结果落入私网即整体阻断,关闭 DNS rebinding 通道
|
||||
assertThrows(UnknownHostException.class,
|
||||
() -> safeDns.lookup("partial-rebound.example.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void isPrivateIpCoversCgnatAndStandardRanges() throws Exception {
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("127.0.0.1")), "loopback");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("10.0.0.1")), "10/8");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("192.168.1.1")), "192.168/16");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("169.254.0.1")), "link-local");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("172.16.0.1")), "site-local 172.16");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.64.0.1")), "CGNAT lower");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.127.255.255")), "CGNAT upper");
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("8.8.8.8")), "public DNS");
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.63.0.1")), "below CGNAT 段");
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.128.0.1")), "above CGNAT 段");
|
||||
// 受限广播
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("255.255.255.255")), "受限广播");
|
||||
// IPv6 ULA fc00::/7
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("fc00::1")), "ULA fc00::");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("fd12:3456:789a::1")), "ULA fd::");
|
||||
// IPv6 公网放行
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("2001:4860:4860::8888")), "Google IPv6 DNS");
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateHttpsUrlRejectsHttpAndPrivateHosts() {
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("http://example.com/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://localhost/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://127.0.0.1/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://10.0.0.5/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://192.168.1.1/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://172.16.0.1/a.jpg"));
|
||||
// 公网 https 应通过
|
||||
SimilarAsinImageEmbedder.validateHttpsUrl("https://m.media-amazon.com/images/I/abc.jpg");
|
||||
SimilarAsinImageEmbedder.validateHttpsUrl("https://cbu01.alicdn.com/img/ibank/xxx.jpg");
|
||||
}
|
||||
|
||||
@Test
|
||||
void downloadOversizeExceptionCarriesUrlAndSize() {
|
||||
SimilarAsinImageEmbedder.DownloadOversizeException ex =
|
||||
new SimilarAsinImageEmbedder.DownloadOversizeException("https://example.com/big.jpg", 12345678L);
|
||||
assertEquals("https://example.com/big.jpg", ex.url());
|
||||
assertEquals(12345678L, ex.size());
|
||||
assertTrue(ex.getMessage().contains("oversize"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void httpClientKeepsImageDownloadResilienceEnabled() {
|
||||
assertTrue(embedder.httpClient().retryOnConnectionFailure(), "OkHttp should retry transient connection failures");
|
||||
assertTrue(embedder.httpClient().followRedirects(), "image CDN redirects should be followed");
|
||||
assertTrue(embedder.httpClient().followSslRedirects(), "signed CDN downloads should follow browser-like SSL redirects");
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildImageRequestUsesBrowserLikeHeaders() {
|
||||
var req = SimilarAsinImageEmbedder.buildImageRequest("https://m.media-amazon.com/images/I/abc.jpg");
|
||||
|
||||
assertEquals("GET", req.method());
|
||||
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||
assertTrue(req.header("Accept").contains("image/"), "Accept should prefer images");
|
||||
assertEquals("https://www.amazon.com/", req.header("Referer"));
|
||||
assertEquals("no-cache", req.header("Cache-Control"));
|
||||
assertNotNull(req.header("Accept-Language"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void amazonImageDownloadCandidatesTrySmallerVariantsWithinRetryBudget() {
|
||||
List<String> candidates = SimilarAsinImageEmbedder.downloadCandidates(
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg");
|
||||
|
||||
assertEquals(List.of(
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg",
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SY450_.jpg",
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX425_.jpg"), candidates);
|
||||
}
|
||||
|
||||
@Test
|
||||
void downloadCandidatesLeaveNonAmazonAndUnsignedUrlsUntouched() {
|
||||
String coze = "https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a";
|
||||
assertEquals(List.of(coze), SimilarAsinImageEmbedder.downloadCandidates(coze));
|
||||
assertEquals(List.of("https://cbu01.alicdn.com/O1CN01x.jpg"),
|
||||
SimilarAsinImageEmbedder.downloadCandidates("https://cbu01.alicdn.com/O1CN01x.jpg"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildImageRequestUsesDownloadHeadersForCozeSignedImages() {
|
||||
var req = SimilarAsinImageEmbedder.buildImageRequest(
|
||||
"https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a");
|
||||
|
||||
assertEquals("GET", req.method());
|
||||
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||
assertEquals("*/*", req.header("Accept"));
|
||||
assertNull(req.header("Referer"), "Coze signed download links should not carry an Amazon referer");
|
||||
assertEquals("no-cache", req.header("Cache-Control"));
|
||||
assertTrue(SimilarAsinImageEmbedder.isCozeSignedImageUrl(req.url().toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void errorSummaryDoesNotHideEmptyTimeoutMessage() {
|
||||
TimeoutException timeout = new TimeoutException();
|
||||
assertEquals("TimeoutException: <empty>", SimilarAsinImageEmbedder.errorSummary(timeout));
|
||||
|
||||
IOException io = new IOException("outer", new TimeoutException("inner"));
|
||||
String summary = SimilarAsinImageEmbedder.errorSummary(io);
|
||||
assertTrue(summary.contains("IOException: outer"));
|
||||
assertTrue(summary.contains("cause=TimeoutException: inner"));
|
||||
}
|
||||
|
||||
private static SimilarAsinProperties properties(int poolSize, int prefetchTimeoutSeconds, Path cacheDir) {
|
||||
SimilarAsinProperties properties = new SimilarAsinProperties();
|
||||
properties.setImageDownloadPoolSize(poolSize);
|
||||
properties.setImagePrefetchTimeoutSeconds(prefetchTimeoutSeconds);
|
||||
properties.setImageLocalCacheDir(cacheDir == null ? "" : cacheDir.toString());
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static OkHttpClient respondingClient(AtomicInteger calls, byte[] imageBytes) {
|
||||
return new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
calls.incrementAndGet();
|
||||
return response(chain.request(), 200, "OK", imageBytes);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
private static OkHttpClient failingClient(AtomicInteger calls) {
|
||||
return new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
calls.incrementAndGet();
|
||||
return response(chain.request(), 503, "Unavailable", new byte[0]);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Response response(okhttp3.Request request,
|
||||
int status,
|
||||
String message,
|
||||
byte[] body) {
|
||||
return new Response.Builder()
|
||||
.request(request)
|
||||
.protocol(Protocol.HTTP_1_1)
|
||||
.code(status)
|
||||
.message(message)
|
||||
.body(ResponseBody.create(body, MediaType.get("image/jpeg")))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static void replaceHttpClient(SimilarAsinImageEmbedder target,
|
||||
OkHttpClient httpClient) throws ReflectiveOperationException {
|
||||
Field field = SimilarAsinImageEmbedder.class.getDeclaredField("httpClient");
|
||||
field.setAccessible(true);
|
||||
field.set(target, httpClient);
|
||||
}
|
||||
|
||||
private static byte[] createJpegBytes() throws IOException {
|
||||
BufferedImage image = new BufferedImage(64, 48, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D graphics = image.createGraphics();
|
||||
try {
|
||||
graphics.setColor(new Color(0x24, 0x68, 0xAC));
|
||||
graphics.fillRect(0, 0, image.getWidth(), image.getHeight());
|
||||
} finally {
|
||||
graphics.dispose();
|
||||
}
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", output);
|
||||
return output.toByteArray();
|
||||
}
|
||||
|
||||
private static void deleteRecursively(Path directory) throws IOException {
|
||||
if (!Files.exists(directory)) {
|
||||
return;
|
||||
}
|
||||
try (var paths = Files.walk(directory)) {
|
||||
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
|
||||
Files.deleteIfExists(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
-172
@@ -1,172 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
|
||||
/**
|
||||
* Task 20:Similar ASIN 端到端压测、JFR/GC 分析与结果文件兼容回归。
|
||||
* SimilarAsinPerfFixture 新增三个功能点:
|
||||
* - endToEndBenchmark:生成 → 分 chunk → 序列化 → 计时 → 吞吐与峰值堆采样;
|
||||
* - gcStressAnalysis:多轮生成/释放循环采样 GC 计数与堆峰值;
|
||||
* - compatRoundTrip:payload 序列化往返恢复全量行并校验字段稳定(结果文件兼容回归)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinPerfFixtureE2ETest {
|
||||
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_normal_default_path() {
|
||||
// 正常输入:1000 行端到端基准返回完整指标,行数/chunk 数正确,吞吐与堆峰值有界。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-default.xlsx", 1000, false, 200);
|
||||
|
||||
assertEquals(1000, metrics.rowCount(), "行数不丢失");
|
||||
assertEquals(5, metrics.chunkCount(), "1000 行 / 200 每 chunk = 5 个 chunk");
|
||||
assertTrue(metrics.payloadBytes() > 0, "payload 字节可采样");
|
||||
assertTrue(metrics.assembleMillis() >= 0);
|
||||
assertTrue(metrics.throughputRowsPerSec() > 0, "吞吐必须为正");
|
||||
assertTrue(metrics.peakHeapBytes() > 0, "峰值堆必须为正");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_normal_multiple_items() {
|
||||
// 批量场景:图片开/关两种模式 5000 行,结果不丢失、chunk 划分稳定。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics withImages =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-img.xlsx", 5000, true, 200);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics textOnly =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-text.xlsx", 5000, false, 200);
|
||||
|
||||
assertEquals(5000, withImages.rowCount());
|
||||
assertEquals(5000, textOnly.rowCount());
|
||||
assertEquals(25, withImages.chunkCount());
|
||||
assertEquals(25, textOnly.chunkCount());
|
||||
assertTrue(withImages.payloadBytes() > textOnly.payloadBytes(), "图片模式 payload 必须大于纯文本");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行:同一输入两次基准的指标一致,不产生重复行。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.CompatResult first =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-idem.xlsx", 1000, true);
|
||||
SimilarAsinPerfFixture.CompatResult second =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-idem.xlsx", 1000, true);
|
||||
|
||||
assertEquals(1000, first.rowCount());
|
||||
assertEquals(1000, first.recoveredCount(), "往返恢复全量行");
|
||||
assertTrue(first.fieldStable(), "字段必须稳定");
|
||||
assertEquals(first.rowCount(), second.rowCount());
|
||||
assertEquals(first.recoveredCount(), second.recoveredCount());
|
||||
assertEquals(first.fieldStable(), second.fieldStable());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_boundary_empty_input() {
|
||||
// 空输入:0 行基准返回零指标;0 行往返返回空结果,不创建资源。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-empty.xlsx", 0, false, 200);
|
||||
assertEquals(0, metrics.rowCount());
|
||||
assertEquals(0, metrics.chunkCount());
|
||||
assertEquals(0, metrics.payloadBytes());
|
||||
|
||||
SimilarAsinPerfFixture.CompatResult compat =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-empty.xlsx", 0, false);
|
||||
assertEquals(0, compat.rowCount());
|
||||
assertEquals(0, compat.recoveredCount());
|
||||
assertTrue(compat.fieldStable(), "空结果字段稳定");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_boundary_single_item() {
|
||||
// 单元素:1 行基准不依赖批量路径,往返字段一致。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-single.xlsx", 1, true, 200);
|
||||
assertEquals(1, metrics.rowCount());
|
||||
assertEquals(1, metrics.chunkCount());
|
||||
|
||||
SimilarAsinPerfFixture.CompatResult compat =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-single.xlsx", 1, true);
|
||||
assertEquals(1, compat.recoveredCount());
|
||||
assertTrue(compat.fieldStable());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_boundary_limit_and_overflow() {
|
||||
// 上限/超限:超过 MAX_ROWS 拒绝;5000 行基准在预算内完成,不发生无界内存增长。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
IllegalArgumentException overflow = assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.endToEndBenchmark("uploads/20260829/e2e-over.xlsx",
|
||||
SimilarAsinPerfFixture.MAX_ROWS + 1, false, 200));
|
||||
assertTrue(overflow.getMessage().contains("rowCount"), "超限消息应可识别");
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.compatRoundTrip("uploads/20260829/e2e-over.xlsx",
|
||||
SimilarAsinPerfFixture.MAX_ROWS + 1, false));
|
||||
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-max.xlsx", 5000, true, 200);
|
||||
assertTrue(metrics.assembleMillis() < 15000,
|
||||
"5000 行端到端基准须在预算内完成,实际=" + metrics.assembleMillis() + "ms");
|
||||
assertTrue(metrics.peakHeapBytes() < 1024L * 1024L * 1024L, "峰值堆不得超过 1GB");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_invalid_input_rejected() {
|
||||
// 非法参数:null key/非法 chunkSize/非法 GC 行数 → 明确异常与可识别消息。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
IllegalArgumentException nullKey = assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.endToEndBenchmark(null, 100, false, 200));
|
||||
assertTrue(nullKey.getMessage().contains("sourceFileKey"));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.endToEndBenchmark("uploads/20260829/x.xlsx", 100, false, 0));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.gcStressAnalysis(null, 100, false));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.gcStressAnalysis("uploads/20260829/x.xlsx", -1, false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:序列化失败抛 IllegalStateException 且不产生部分结果;恢复后重试成功;
|
||||
// GC 分析后堆峰值回落(临时对象释放)。
|
||||
AtomicInteger failCount = new AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (failCount.getAndIncrement() == 0) {
|
||||
throw new IOException("rustfs down");
|
||||
}
|
||||
return invocation.callRealMethod();
|
||||
}).when(objectMapper).writeValueAsBytes(any());
|
||||
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> fixture.endToEndBenchmark("uploads/20260829/e2e-fail.xlsx", 100, false, 200));
|
||||
SimilarAsinPerfFixture.EndToEndMetrics recovered =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-fail.xlsx", 100, false, 200);
|
||||
assertEquals(100, recovered.rowCount(), "依赖恢复后重试成功");
|
||||
|
||||
SimilarAsinPerfFixture.GcStressSample gc =
|
||||
fixture.gcStressAnalysis("uploads/20260829/e2e-gc.xlsx", 1000, true);
|
||||
assertNotNull(gc);
|
||||
assertTrue(gc.rounds() >= 1, "GC 分析至少执行一轮");
|
||||
assertTrue(gc.gcCount() >= 0);
|
||||
assertTrue(gc.peakHeapBytes() > 0, "堆峰值必须可采样");
|
||||
assertTrue(gc.peakHeapBytes() < 1024L * 1024L * 1024L, "GC 分析峰值堆不得超过 1GB");
|
||||
}
|
||||
}
|
||||
-198
@@ -1,198 +0,0 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Task 1:Similar ASIN 性能基线夹具(1000/5000 行、图片开关、chunk 数与 payload 大小采样)。
|
||||
* 先写测试确认 RED,再实现 SimilarAsinPerfFixture。
|
||||
*/
|
||||
class SimilarAsinPerfFixtureTest {
|
||||
|
||||
private final SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(new ObjectMapper());
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_default_path() {
|
||||
// 1000 行、带图片开关,默认 chunk 大小
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/base.xlsx", 1000, true);
|
||||
assertEquals(1000, rows.size());
|
||||
// 行必须包含 url 图片地址
|
||||
assertFalse(rows.get(0).getUrl().isBlank());
|
||||
assertTrue(rows.get(0).getUrl().startsWith("http"));
|
||||
// rowToken 稳定且唯一
|
||||
assertEquals(rows.get(0).getRowToken(), fixture.rowTokenFor(rows.get(0).getSourceFileKey(), rows.get(0).getRowIndex()));
|
||||
assertEquals(1000, rows.stream().map(SimilarAsinParsedRowVo::getRowToken).distinct().count());
|
||||
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
|
||||
assertEquals(5, chunks.size());
|
||||
assertEquals(200, chunks.get(0).size());
|
||||
|
||||
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
|
||||
assertEquals(1000, metrics.rowCount());
|
||||
assertEquals(5, metrics.chunkCount());
|
||||
assertTrue(metrics.payloadBytes() > 0, "payload 采样字节数必须大于 0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_multiple_items() {
|
||||
// 5000 行批量场景:顺序稳定、chunk 数正确、结果不丢失
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/multi.xlsx", 5000, false);
|
||||
assertEquals(5000, rows.size());
|
||||
assertTrue(rows.get(0).getUrl().isBlank(), "图片开关关闭时 url 必须为空");
|
||||
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
|
||||
assertEquals(25, chunks.size());
|
||||
// 顺序稳定:拼接后与原始一致
|
||||
List<SimilarAsinParsedRowVo> restored = chunks.stream().flatMap(List::stream).toList();
|
||||
assertEquals(rows.size(), restored.size());
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
assertEquals(rows.get(i).getRowToken(), restored.get(i).getRowToken());
|
||||
}
|
||||
// 所有行 rowIndex 递增
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
assertEquals(i + 1, rows.get(i).getRowIndex());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_repeated_operation_is_idempotent() {
|
||||
List<SimilarAsinParsedRowVo> first = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
|
||||
List<SimilarAsinParsedRowVo> second = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
|
||||
// 同一输入重复生成:token 完全一致,不产生重复差异
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
assertEquals(first.get(i).getAsin(), second.get(i).getAsin());
|
||||
}
|
||||
// splitChunks 幂等:两次划分 chunk 数一致
|
||||
assertEquals(fixture.splitChunks(first, 200).size(), fixture.splitChunks(second, 200).size());
|
||||
// 采样指标幂等
|
||||
SimilarAsinPerfFixture.Metrics m1 = fixture.samplePayload(first, true, 200);
|
||||
SimilarAsinPerfFixture.Metrics m2 = fixture.samplePayload(second, true, 200);
|
||||
assertEquals(m1.payloadBytes(), m2.payloadBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_empty_input() {
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/empty.xlsx", 0, true);
|
||||
assertNotNull(rows);
|
||||
assertEquals(0, rows.size());
|
||||
// 空行 splitChunks 返回空,不产生无效 chunk
|
||||
assertEquals(0, fixture.splitChunks(rows, 200).size());
|
||||
// 空行采样:行数 0、chunk 0、payload 字节数为 0(不创建任何载荷)
|
||||
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
|
||||
assertEquals(0, metrics.rowCount());
|
||||
assertEquals(0, metrics.chunkCount());
|
||||
assertEquals(0, metrics.payloadBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_single_item() {
|
||||
// 单行:不依赖批量路径且结果正确
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/single.xlsx", 1, true);
|
||||
assertEquals(1, rows.size());
|
||||
assertEquals(1, rows.get(0).getRowIndex());
|
||||
assertEquals(1, fixture.splitChunks(rows, 200).size());
|
||||
// 单行 chunk 划分后仍只含 1 行
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
|
||||
assertEquals(1, chunks.get(0).size());
|
||||
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
|
||||
assertEquals(1, metrics.rowCount());
|
||||
assertEquals(1, metrics.chunkCount());
|
||||
assertTrue(metrics.payloadBytes() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_limit_and_overflow() {
|
||||
// 超过最大行数(5000)时拒绝,不发生无界内存增长
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/overflow.xlsx", 5001, true));
|
||||
// 达到最大允许值 5000 时允许
|
||||
assertEquals(5000, fixture.generateRows("uploads/20260829/max.xlsx", 5000, true).size());
|
||||
// chunkSize 非法值拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/a.xlsx", 100, true), 0));
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/b.xlsx", 100, true), -1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_invalid_input_rejected() {
|
||||
// null 文件 key 拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(null, 100, true));
|
||||
// 空白文件 key 拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(" ", 100, true));
|
||||
// 负行数拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/neg.xlsx", -1, true));
|
||||
// null 行集合分块拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(null, 200));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_dependency_failure_releases_resources() {
|
||||
// 采样时序列化器失败(mock 抛异常):错误可恢复,不产生部分结果
|
||||
ObjectMapper broken = new ObjectMapper() {
|
||||
@Override
|
||||
public String writeValueAsString(Object value) {
|
||||
throw new IllegalStateException("serializer down");
|
||||
}
|
||||
};
|
||||
SimilarAsinPerfFixture failingFixture = new SimilarAsinPerfFixture(broken);
|
||||
List<SimilarAsinParsedRowVo> rows = failingFixture.generateRows("uploads/20260829/fail.xlsx", 1000, true);
|
||||
assertThrows(IllegalStateException.class, () -> failingFixture.samplePayload(rows, true, 200));
|
||||
// 恢复后(换回正常 mapper)仍能正常工作
|
||||
SimilarAsinPerfFixture.Metrics recovered = fixture.samplePayload(fixture.generateRows("uploads/20260829/recover.xlsx", 1000, true), true, 200);
|
||||
assertTrue(recovered.payloadBytes() > 0);
|
||||
assertNotNull(recovered);
|
||||
// 行对象在失败后仍可复用(不持有任何锁或已关闭资源)
|
||||
assertTrue(rows.get(0).getAsin().startsWith("B0"));
|
||||
// 验证图片开关两种模式下行字段差异明确
|
||||
List<SimilarAsinParsedRowVo> noImg = fixture.generateRows("uploads/20260829/nimg.xlsx", 10, false);
|
||||
assertTrue(noImg.get(0).getUrl().isBlank());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_fields_populated() {
|
||||
// 行字段完整性:asin/country/sku/title/values 均填充且稳定
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
|
||||
SimilarAsinParsedRowVo row = rows.get(3);
|
||||
assertTrue(row.getAsin().matches("B0[A-Z0-9]{8}"));
|
||||
assertFalse(row.getCountry().isBlank());
|
||||
assertFalse(row.getSku().isBlank());
|
||||
assertFalse(row.getTitle().isBlank());
|
||||
assertNotNull(row.getValues());
|
||||
assertFalse(row.getValues().isEmpty());
|
||||
assertTrue(row.getValues().containsKey("asin"));
|
||||
assertTrue(row.getValues().containsKey("国家"));
|
||||
// 行号与 sourceId 关联正确
|
||||
assertEquals("4", row.getSourceId());
|
||||
assertEquals(4, row.getRowIndex());
|
||||
// values 是独立副本,修改不影响后续生成
|
||||
row.getValues().put("价格", "999");
|
||||
List<SimilarAsinParsedRowVo> again = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
|
||||
assertTrue(!again.get(3).getValues().getOrDefault("价格", "").equals("999"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_chunk_size_edge() {
|
||||
// chunk 边界:行数恰好整除 / 有余数 / 单 chunk 放不下
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/edge.xlsx", 100, false);
|
||||
// 100 行 / 40 → 3 chunks(40+40+20)
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 40);
|
||||
assertEquals(3, chunks.size());
|
||||
assertEquals(40, chunks.get(0).size());
|
||||
assertEquals(20, chunks.get(2).size());
|
||||
// chunkSize 大于总行数 → 单 chunk
|
||||
assertEquals(1, fixture.splitChunks(rows, 500).size());
|
||||
// chunkSize 恰好等于行数 → 单 chunk 全量
|
||||
assertEquals(1, fixture.splitChunks(rows, 100).size());
|
||||
assertEquals(100, fixture.splitChunks(rows, 100).get(0).size());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user