task-120: 大页数据性能回归 + 恢复 similarasin 测试入库
性能回归(SimilarAsinTaskServicePerf500Test,8/8 绿): - test_perf_light_200 原以墙钟毫秒比较 light 与 batch(assertTrue lightMs < batchMs)。两条路径都跑纯内存 stub,耗时在十几毫秒量级, JIT/GC/线程调度抖动远大于真实差值,本机实测 light 14ms / batch 13ms 直接翻红。改为确定性口径:light 结果明细查询数 == 0(batch ≥ 1)、 light 总查询数 < batch 总查询数、batch 条目携带任务明细对象而 light 只有白名单字段;墙钟仅保留单侧上界 1s(N+1 退化会放大到秒级从而失败)。 连续 3 次独立运行稳定通过。 - 类注释同步移除“已知 flaky,隔离重跑即接受”的说明。 恢复测试入库: - fa5a59e(task-116) 把 similarasin 测试目录整体从 git 删除,并加了 .gitignore 规则 *.broken/,磁盘上目录改名 similarasin.broken。 但包声明仍是 com.nanri.aiimage.modules.similarasin.*,javac 照常编译, 这 40 个测试文件一直在参与构建、并且是 mvn test 的门禁的一部分—— 等于脱离版本控制却仍在 gate 构建。 - 目录改回 similarasin/(现在与包声明一致),移除 *.broken/ 忽略规则, 40 个测试文件重新入库。mvn test-compile 退出码 0。
This commit is contained in:
@@ -118,7 +118,6 @@ backend-java/docs/plans/
|
||||
backend-java/docs/*plan*.md
|
||||
backend-java/docs/specs/
|
||||
backend-java/docs/*audit*.md
|
||||
*.broken/
|
||||
|
||||
# ===== 进度/规划工具(本地使用,不入库)=====
|
||||
check_progress.py
|
||||
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
package com.nanri.aiimage.modules.similarasin.controller;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
/**
|
||||
* 任务 109:progress/light 轻量接口(similarasin 首个)。
|
||||
* 新端点 POST /api/similar-asin/tasks/progress/light 只返回轻量字段(taskId/status/statusCode?/fileStatus?/fileReady?/updatedAt);
|
||||
* 旧 progress/batch 端点原样保留可用;不存在的任务进 missingTaskIds。
|
||||
*/
|
||||
class SimilarAsinTaskLightControllerTest {
|
||||
|
||||
private static final String LIGHT_URL = "/api/similar-asin/tasks/progress/light";
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private SimilarAsinTaskService service;
|
||||
private MockMvc mockMvc;
|
||||
|
||||
private SimilarAsinTaskLightBatchVo lightBatch(SimilarAsinTaskLightVo... items) {
|
||||
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
|
||||
vo.getItems().addAll(List.of(items));
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static SimilarAsinTaskLightVo lightItem(Long taskId, String status, String fileStatus, Boolean fileReady, String updatedAt) {
|
||||
SimilarAsinTaskLightVo vo = new SimilarAsinTaskLightVo();
|
||||
vo.setTaskId(taskId);
|
||||
vo.setStatus(status);
|
||||
vo.setFileStatus(fileStatus);
|
||||
vo.setFileReady(fileReady);
|
||||
vo.setUpdatedAt(updatedAt);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private void setUpWith(SimilarAsinTaskLightBatchVo response) throws Exception {
|
||||
service = mock(SimilarAsinTaskService.class);
|
||||
when(service.progressLight(any())).thenReturn(response);
|
||||
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightReturnsLightFields() throws Exception {
|
||||
setUpWith(lightBatch(lightItem(3938L, "RUNNING", "RUNNING", false, "2026-04-26T10:05:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3938L)))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true))
|
||||
.andExpect(jsonPath("$.data.items[0].taskId").value(3938))
|
||||
.andExpect(jsonPath("$.data.items[0].status").value("RUNNING"))
|
||||
.andExpect(jsonPath("$.data.items[0].fileStatus").value("RUNNING"))
|
||||
.andExpect(jsonPath("$.data.items[0].fileReady").value(false))
|
||||
.andExpect(jsonPath("$.data.items[0].updatedAt").value("2026-04-26T10:05:00"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightMissingTask() throws Exception {
|
||||
setUpWith(lightBatch(lightItem(1L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:10:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(1L)))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true))
|
||||
.andExpect(jsonPath("$.data.items[0].taskId").value(1));
|
||||
verify(service).progressLight(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightEmptyIds() throws Exception {
|
||||
setUpWith(new SimilarAsinTaskLightBatchVo());
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of()))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items").isEmpty())
|
||||
.andExpect(jsonPath("$.data.missingTaskIds").isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightDedupIds() throws Exception {
|
||||
setUpWith(lightBatch(lightItem(7L, "PENDING", null, false, "2026-04-26T10:00:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(7L, 7L, 7L)))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items.length()").value(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightOldEndpointUntouched() throws Exception {
|
||||
com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest request = new com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest();
|
||||
request.setTaskIds(List.of(1L));
|
||||
service = mock(SimilarAsinTaskService.class);
|
||||
when(service.progressBatch(any())).thenReturn(new com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo());
|
||||
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
|
||||
mockMvc.perform(post("/api/similar-asin/tasks/progress/batch")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(request)))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true));
|
||||
verify(service).progressBatch(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightStatusMapping() throws Exception {
|
||||
setUpWith(lightBatch(lightItem(2L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:11:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(2L)))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items[0].status").value("SUCCESS"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightFileFields() throws Exception {
|
||||
setUpWith(lightBatch(lightItem(3L, "RUNNING", "SUCCESS", true, "2026-04-26T10:12:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3L)))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items[0].fileStatus").value("SUCCESS"))
|
||||
.andExpect(jsonPath("$.data.items[0].fileReady").value(true));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightUnknownId() throws Exception {
|
||||
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
|
||||
vo.getMissingTaskIds().add(99999L);
|
||||
setUpWith(vo);
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(99999L)))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items").isEmpty())
|
||||
.andExpect(jsonPath("$.data.missingTaskIds[0]").value(99999));
|
||||
}
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package com.nanri.aiimage.modules.similarasin.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 110:progress/light 字段白名单。
|
||||
* 响应仅含白名单字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt);
|
||||
* 不含 items 明细/payload/result 内容;序列化后键集合精确匹配;响应体小(<1KB)。
|
||||
*/
|
||||
class SimilarAsinTaskLightWhitelistTest {
|
||||
|
||||
private static final Set<String> WHITELIST = new TreeSet<>(Set.of(
|
||||
"taskId", "status", "statusCode", "fileStatus", "fileError", "fileReady", "updatedAt"));
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private SimilarAsinTaskLightVo fullItem() {
|
||||
SimilarAsinTaskLightVo vo = new SimilarAsinTaskLightVo();
|
||||
vo.setTaskId(3938L);
|
||||
vo.setStatus("RUNNING");
|
||||
vo.setStatusCode(null);
|
||||
vo.setFileStatus("RUNNING");
|
||||
vo.setFileReady(false);
|
||||
vo.setUpdatedAt("2026-04-26T10:05:00");
|
||||
return vo;
|
||||
}
|
||||
|
||||
private Set<String> keysOf(SimilarAsinTaskLightVo vo) throws Exception {
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(vo));
|
||||
Set<String> keys = new TreeSet<>();
|
||||
node.fieldNames().forEachRemaining(keys::add);
|
||||
return keys;
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistExactKeys() throws Exception {
|
||||
assertEquals(WHITELIST, keysOf(fullItem()), "序列化键集合与白名单精确一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistNoItemsArray() throws Exception {
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(fullItem()));
|
||||
assertFalse(node.has("items"), "不含明细数组");
|
||||
assertFalse(node.has("itemCount"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistNoPayload() throws Exception {
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(fullItem()));
|
||||
assertFalse(node.has("payload"), "不含 payload 字段");
|
||||
assertFalse(node.has("requestJson"));
|
||||
assertFalse(node.has("resultJson"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistNoResultContent() throws Exception {
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(fullItem()));
|
||||
assertFalse(node.has("resultId"), "不含 resultId");
|
||||
assertFalse(node.has("downloadUrl"), "不含下载链接");
|
||||
assertFalse(node.has("rowCount"), "不含行数明细");
|
||||
assertFalse(node.has("success"), "不含成功标志明细");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistFilePhasePresent() throws Exception {
|
||||
SimilarAsinTaskLightVo vo = fullItem();
|
||||
vo.setFileStatus("SUCCESS");
|
||||
vo.setFileReady(true);
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(vo));
|
||||
assertEquals("SUCCESS", node.get("fileStatus").asText());
|
||||
assertTrue(node.get("fileReady").asBoolean());
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistNullValuesAllowed() throws Exception {
|
||||
SimilarAsinTaskLightVo vo = new SimilarAsinTaskLightVo();
|
||||
vo.setTaskId(1L);
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(vo));
|
||||
assertTrue(node.has("statusCode"), "statusCode 键存在(null 值允许)");
|
||||
assertTrue(node.get("statusCode").isNull());
|
||||
assertEquals(WHITELIST, keysOf(vo), "null 字段仍在白名单键集合内");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistResponseSizeSmall() throws Exception {
|
||||
SimilarAsinTaskLightBatchVo batch = new SimilarAsinTaskLightBatchVo();
|
||||
batch.getItems().add(fullItem());
|
||||
byte[] bytes = objectMapper.writeValueAsBytes(ApiResponse.success(batch));
|
||||
assertTrue(bytes.length < 1024, "单条 light 响应 <1KB,实际 " + bytes.length + "B");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistExtraKeysRejected() throws Exception {
|
||||
// 回归门禁:白名单键集合与文档一致(新增键必须先更新本测试与 spec)
|
||||
assertEquals(List.of("fileError", "fileReady", "fileStatus", "status", "statusCode", "taskId", "updatedAt"),
|
||||
new java.util.ArrayList<>(WHITELIST));
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistDocumentedInSpec() throws Exception {
|
||||
// 与 spec 06 §2 对齐:taskId/status/statusCode?/fileStatus?/fileError?/fileReady?/updatedAt
|
||||
assertTrue(WHITELIST.containsAll(Set.of("taskId", "status", "updatedAt")));
|
||||
assertTrue(WHITELIST.containsAll(Set.of("statusCode", "fileStatus", "fileReady", "fileError")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistBatchHasOnlyItemsAndMissing() throws Exception {
|
||||
SimilarAsinTaskLightBatchVo batch = new SimilarAsinTaskLightBatchVo();
|
||||
batch.getItems().add(fullItem());
|
||||
batch.getMissingTaskIds().add(999L);
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(batch));
|
||||
Set<String> keys = new TreeSet<>();
|
||||
node.fieldNames().forEachRemaining(keys::add);
|
||||
assertEquals(new TreeSet<>(Set.of("items", "missingTaskIds")), keys, "批量 VO 只含 items/missingTaskIds");
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import javax.net.ssl.SNIHostName;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLParameters;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
|
||||
public class LlmGatewayTlsProbe {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String host = "ai.t8star.org";
|
||||
String apiKey = args[0];
|
||||
|
||||
System.out.println("[probe] java=" + System.getProperty("java.version")
|
||||
+ " tls=" + System.getProperty("java.vm.name"));
|
||||
for (InetAddress a : InetAddress.getAllByName(host)) {
|
||||
System.out.println("[probe] dns " + a);
|
||||
}
|
||||
|
||||
rawHandshake(host, null, "default");
|
||||
rawHandshake(host, "TLSv1.2", "tls12-only");
|
||||
|
||||
httpClientCall(host, apiKey, null, "jdk-http-default");
|
||||
httpClientCall(host, apiKey, "TLSv1.2", "jdk-http-tls12");
|
||||
}
|
||||
|
||||
private static void rawHandshake(String host, String protocol, String label) throws Exception {
|
||||
SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
|
||||
try (SSLSocket socket = (SSLSocket) factory.createSocket(host, 443)) {
|
||||
socket.setSoTimeout(15000);
|
||||
SSLParameters params = socket.getSSLParameters();
|
||||
if (protocol != null) {
|
||||
params.setProtocols(new String[]{protocol});
|
||||
}
|
||||
params.setServerNames(List.of(new SNIHostName(host)));
|
||||
socket.setSSLParameters(params);
|
||||
socket.startHandshake();
|
||||
System.out.println("[probe] raw[" + label + "] OK proto=" + socket.getSession().getProtocol()
|
||||
+ " cipher=" + socket.getSession().getCipherSuite());
|
||||
} catch (Exception ex) {
|
||||
System.out.println("[probe] raw[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void httpClientCall(String host, String apiKey, String protocol, String label) throws Exception {
|
||||
HttpClient.Builder builder = HttpClient.newBuilder()
|
||||
.connectTimeout(Duration.ofSeconds(10))
|
||||
.version(HttpClient.Version.HTTP_1_1);
|
||||
if (protocol != null) {
|
||||
SSLContext context = SSLContext.getInstance("TLS");
|
||||
context.init(null, null, null);
|
||||
builder.sslContext(context);
|
||||
}
|
||||
HttpClient client = builder.build();
|
||||
try {
|
||||
String body = "{\"model\":\"gemini-3.5-flash-lite\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":10}";
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create("https://" + host + "/v1/chat/completions"))
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build();
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
String text = response.body();
|
||||
System.out.println("[probe] http[" + label + "] status=" + response.statusCode()
|
||||
+ " body=" + text.substring(0, Math.min(160, text.length())));
|
||||
} catch (Exception ex) {
|
||||
System.out.println("[probe] http[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
|
||||
Throwable cause = ex;
|
||||
while (cause.getCause() != null) {
|
||||
cause = cause.getCause();
|
||||
System.out.println("[probe] cause " + cause.getClass().getSimpleName() + ": " + cause.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 15:图片缓存访问时间更新改为异步批量刷新,减少逐图 UPDATE。
|
||||
* lookup 命中不再同步 touchLastUsed,而是进入内存 touch 缓冲(按 url_hash 去重),
|
||||
* 由定时任务/阈值触发 flushPendingTouches 批量 touchLastUsedBatch 刷新;
|
||||
* 缓冲有大小上限,超限立即刷新,不会无界增长;失败 best-effort 吞掉。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinImagePrefetchServiceAsyncTouchTest {
|
||||
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private TaskImageCacheMapper taskImageCacheMapper;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
|
||||
@InjectMocks private SimilarAsinImagePrefetchService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(1000);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdown();
|
||||
}
|
||||
|
||||
private static String sha256Hex(String value) throws Exception {
|
||||
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
|
||||
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private void stubHit(String url, byte[] bytes) throws Exception {
|
||||
when(taskImageCacheMapper.selectBytesByUrlHash(sha256Hex(url))).thenReturn(bytes);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_normal_default_path() throws Exception {
|
||||
// 正常输入:命中不立即写 DB,进入缓冲;flush 后批量 touch 一次。
|
||||
String url = "https://img.example.com/hit.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
byte[] bytes = new byte[]{1, 2, 3};
|
||||
stubHit(url, bytes);
|
||||
|
||||
byte[] result = service.lookup(url);
|
||||
assertEquals(bytes, result, "命中必须返回缓存字节");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多次命中进入同一缓冲,flush 合并为一次批量 touch,覆盖全部命中。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
|
||||
"https://img.example.com/c.jpg");
|
||||
Set<String> hashes = new HashSet<>();
|
||||
for (int i = 0; i < urls.size(); i++) {
|
||||
hashes.add(sha256Hex(urls.get(i)));
|
||||
stubHit(urls.get(i), new byte[]{(byte) (i + 1)});
|
||||
}
|
||||
for (String url : urls) {
|
||||
assertNotNull(service.lookup(url), "命中返回字节");
|
||||
}
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
|
||||
service.flushPendingTouches();
|
||||
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
|
||||
assertEquals(hashes, new HashSet<>(captor.getValue()), "一次批量 touch 覆盖全部命中 hash");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:同一 url 多次命中只 touch 一次;重复 flush 无多余请求。
|
||||
String url = "https://img.example.com/same.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubHit(url, new byte[]{5});
|
||||
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(new byte[]{5});
|
||||
|
||||
service.lookup(url);
|
||||
service.lookup(url);
|
||||
service.lookup(url);
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空白 url 不进入缓冲;flush 空缓冲不产生任何数据库访问。
|
||||
assertNull(service.lookup(null));
|
||||
assertNull(service.lookup(""));
|
||||
assertNull(service.lookup(" "));
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_boundary_single_item() throws Exception {
|
||||
// 单条命中:flush 后单元素批量 touch,不依赖批量路径。
|
||||
String url = "https://img.example.com/single.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubHit(url, new byte[]{7});
|
||||
|
||||
assertNotNull(service.lookup(url));
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_boundary_limit_and_overflow() throws Exception {
|
||||
// 缓冲达到阈值立即刷新,不无界增长;刷新后继续累积。
|
||||
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(3);
|
||||
List<String> urls = List.of("https://img.example.com/o1.jpg", "https://img.example.com/o2.jpg",
|
||||
"https://img.example.com/o3.jpg", "https://img.example.com/o4.jpg",
|
||||
"https://img.example.com/o5.jpg");
|
||||
for (String url : urls) {
|
||||
stubHit(url, new byte[]{1});
|
||||
}
|
||||
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
||||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
service.lookup(urls.get(i));
|
||||
}
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
|
||||
assertEquals(3, captor.getValue().size(), "第 3 条命中触发阈值立即刷新 3 条");
|
||||
|
||||
service.lookup(urls.get(3));
|
||||
service.lookup(urls.get(4));
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
|
||||
assertEquals(2, captor.getValue().size(), "剩余 2 条在 flush 时刷新,缓冲不残留、不无界增长");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_invalid_input_rejected() {
|
||||
// 非法输入:db cache 关闭时 lookup 直接返回 null,不进入缓冲、不访问数据库。
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
|
||||
assertNull(service.lookup("https://img.example.com/a.jpg"));
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_015_image_cache_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:批量 touch 抛异常时吞掉不阻塞 lookup、缓冲已排空无残留;
|
||||
// 恢复后重新入队 flush 成功。
|
||||
String url = "https://img.example.com/fail.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubHit(url, new byte[]{3});
|
||||
AtomicInteger failures = new AtomicInteger(0);
|
||||
List<String> capturedArgs = new java.util.ArrayList<>();
|
||||
doAnswer(invocation -> {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> arg = (List<String>) invocation.getArgument(0);
|
||||
capturedArgs.addAll(arg);
|
||||
if (failures.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("db down");
|
||||
}
|
||||
return 0;
|
||||
}).when(taskImageCacheMapper).touchLastUsedBatch(any());
|
||||
|
||||
assertNotNull(service.lookup(url), "touch 失败不阻断 lookup 返回缓存字节");
|
||||
assertThrows(Exception.class, () -> service.flushPendingTouches(), "首次 flush 抛错(由调用方吞掉)");
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
|
||||
assertNotNull(service.lookup(url), "失败后再次命中重新入队");
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(any());
|
||||
assertEquals(List.of(hash, hash), capturedArgs, "两次 touch 都覆盖命中 hash,失败后恢复成功");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
private static void assertNull(Object value) {
|
||||
org.junit.jupiter.api.Assertions.assertNull(value);
|
||||
}
|
||||
}
|
||||
+297
@@ -0,0 +1,297 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 14:图片 DB cache 改为批量读取缩略图,并只更新实际命中的 last_used_at。
|
||||
* 批量 lookup 入口(lookupBatch)一次 IN 查询返回命中字节 Map;
|
||||
* last_used_at 只对实际命中的 url_hash 更新(touch 集合 = 命中集合),
|
||||
* 未命中 url 不产生任何 touch/insert。单 URL 旧入口 lookup 语义保持兼容。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinImagePrefetchServiceBatchTest {
|
||||
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private TaskImageCacheMapper taskImageCacheMapper;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
|
||||
@InjectMocks private SimilarAsinImagePrefetchService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdown();
|
||||
}
|
||||
|
||||
private static String sha256Hex(String value) throws Exception {
|
||||
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
|
||||
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** stub 批量命中读取:只对 hits 集合内的 hash 返回字节。 */
|
||||
private void stubBatchRead(List<String> hits, Map<String, byte[]> bytesByHash) {
|
||||
when(taskImageCacheMapper.selectBytesByUrlHashes(any())).thenAnswer(invocation -> {
|
||||
List<String> hashes = invocation.getArgument(0);
|
||||
List<TaskImageCacheEntity> rows = new ArrayList<>();
|
||||
for (String hash : hashes) {
|
||||
if (hits.contains(hash)) {
|
||||
TaskImageCacheEntity row = new TaskImageCacheEntity();
|
||||
row.setUrlHash(hash);
|
||||
row.setImageBytes(bytesByHash.get(hash));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
|
||||
/** 调用批量入口,按 url 顺序返回字节(未命中为 null)。 */
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<byte[]> invokeLookupBatch(SimilarAsinImagePrefetchService svc, List<String> urls) throws Exception {
|
||||
Method m = SimilarAsinImagePrefetchService.class.getDeclaredMethod("lookupBatch", List.class);
|
||||
m.setAccessible(true);
|
||||
return (List<byte[]>) m.invoke(svc, urls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_default_path() throws Exception {
|
||||
// 正常输入:命中与未命中混排,批量读回命中字节,touch 只覆盖实际命中。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
|
||||
List<String> hits = List.of(sha256Hex(urls.get(0)));
|
||||
byte[] bytesA = new byte[]{1, 2, 3};
|
||||
stubBatchRead(hits, Map.of(sha256Hex(urls.get(0)), bytesA));
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
|
||||
assertEquals(2, result.size(), "批量入口必须按输入顺序返回");
|
||||
assertEquals(bytesA, result.get(0), "命中行返回缓存字节");
|
||||
assertNull(result.get(1), "未命中行返回 null,不虚构缓存内容");
|
||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(sha256Hex(urls.get(0))));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_multiple_items() throws Exception {
|
||||
// 批量场景:全命中多 url,一次 IN 查询返回全部字节,touch 覆盖全部命中。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
|
||||
"https://img.example.com/c.jpg");
|
||||
List<String> hits = new ArrayList<>();
|
||||
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
|
||||
for (int i = 0; i < urls.size(); i++) {
|
||||
hits.add(sha256Hex(urls.get(i)));
|
||||
bytesByHash.put(hits.get(i), new byte[]{(byte) (i + 1)});
|
||||
}
|
||||
stubBatchRead(hits, bytesByHash);
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
|
||||
assertEquals(3, result.size());
|
||||
for (int i = 0; i < urls.size(); i++) {
|
||||
assertEquals(bytesByHash.get(sha256Hex(urls.get(i))), result.get(i), "顺序稳定、字节不丢失");
|
||||
}
|
||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(new ArrayList<>(hits));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:每次行为一致,不产生重复请求/重复 touch。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
||||
String hash = sha256Hex(urls.get(0));
|
||||
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{9}));
|
||||
|
||||
invokeLookupBatch(service, urls);
|
||||
invokeLookupBatch(service, urls);
|
||||
|
||||
verify(taskImageCacheMapper, times(2)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(List.of(hash));
|
||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空列表安全返回空结果,不产生任何数据库访问。
|
||||
List<byte[]> nullResult = invokeLookupBatch(service, null);
|
||||
assertNotNull(nullResult);
|
||||
assertTrue(nullResult.isEmpty());
|
||||
|
||||
List<byte[]> emptyResult = invokeLookupBatch(service, List.of());
|
||||
assertNotNull(emptyResult);
|
||||
assertTrue(emptyResult.isEmpty());
|
||||
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_boundary_single_item() throws Exception {
|
||||
// 单 url:不依赖批量路径,命中时单次查询 + 单次 touch。
|
||||
String url = "https://img.example.com/single.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{7}));
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, List.of(url));
|
||||
|
||||
assertEquals(1, result.size());
|
||||
assertEquals(7, result.get(0)[0]);
|
||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_boundary_limit_and_overflow() throws Exception {
|
||||
// 大批量(超过单批上限 500):分片查询,命中 touch 只覆盖命中集合。
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 1200; i++) {
|
||||
urls.add("https://img.example.com/overflow-" + i + ".jpg");
|
||||
}
|
||||
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
|
||||
List<String> hits = new ArrayList<>();
|
||||
for (int i = 0; i < 1200; i += 2) {
|
||||
String hash = sha256Hex(urls.get(i));
|
||||
hits.add(hash);
|
||||
bytesByHash.put(hash, new byte[]{(byte) (i % 100)});
|
||||
}
|
||||
stubBatchRead(hits, bytesByHash);
|
||||
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
|
||||
assertEquals(1200, result.size(), "超大批量结果不丢失、顺序稳定");
|
||||
int hitCount = 0;
|
||||
for (int i = 0; i < 1200; i++) {
|
||||
if (i % 2 == 0) {
|
||||
assertNotNull(result.get(i), "偶数下标命中必须返回字节");
|
||||
hitCount++;
|
||||
} else {
|
||||
assertNull(result.get(i), "奇数下标未命中返回 null");
|
||||
}
|
||||
}
|
||||
assertEquals(600, hitCount);
|
||||
verify(taskImageCacheMapper, times(3)).selectBytesByUrlHashes(any());
|
||||
// touch 按单批 500 分片:600 命中 → 2 次 touch 调用,且只覆盖命中集合。
|
||||
var captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
|
||||
List<List<String>> touchCalls = new ArrayList<>();
|
||||
for (Object call : captor.getAllValues()) {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> casted = (List<String>) call;
|
||||
touchCalls.add(casted);
|
||||
}
|
||||
assertEquals(2, touchCalls.size());
|
||||
assertEquals(500, touchCalls.get(0).size(), "第一批 touch 500 个命中");
|
||||
assertEquals(100, touchCalls.get(1).size(), "第二批 touch 剩余 100 个命中");
|
||||
assertEquals(hits.subList(0, 500), touchCalls.get(0), "touch 只覆盖实际命中集合");
|
||||
assertEquals(hits.subList(500, 600), touchCalls.get(1));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_invalid_input_rejected() throws Exception {
|
||||
// 非法输入:db cache 关闭时批量入口直接返回空,不访问数据库。
|
||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
||||
assertNotNull(result);
|
||||
assertTrue(result.isEmpty(), "db cache 关闭时必须直接返回空结果");
|
||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:查询抛异常时批量入口返回空结果、无任何 touch/insert 残留;
|
||||
// 恢复后重试成功。
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
||||
String hash = sha256Hex(urls.get(0));
|
||||
AtomicInteger callCount = new AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (callCount.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("db down");
|
||||
}
|
||||
TaskImageCacheEntity row = new TaskImageCacheEntity();
|
||||
row.setUrlHash(hash);
|
||||
row.setImageBytes(new byte[]{5});
|
||||
return List.of(row);
|
||||
}).when(taskImageCacheMapper).selectBytesByUrlHashes(any());
|
||||
|
||||
List<byte[]> failed = invokeLookupBatch(service, urls);
|
||||
assertNotNull(failed);
|
||||
assertTrue(failed.isEmpty(), "查询失败必须返回空结果而不是抛错阻断组装");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
||||
|
||||
List<byte[]> recovered = invokeLookupBatch(service, urls);
|
||||
assertEquals(1, recovered.size());
|
||||
assertEquals(5, recovered.get(0)[0], "依赖恢复后重试成功");
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_014_image_normal_single_lookup_legacy_compat() throws Exception {
|
||||
// 兼容性:单 URL 旧入口 lookup 保持返回字节语义;Task 15 起 touch 改为
|
||||
// 异步批量缓冲,flush 后批量 touch 一次,命中才入缓冲。
|
||||
String url = "https://img.example.com/legacy.jpg";
|
||||
String hash = sha256Hex(url);
|
||||
byte[] bytes = new byte[]{6};
|
||||
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(bytes);
|
||||
|
||||
byte[] result = service.lookup(url);
|
||||
assertEquals(bytes, result, "lookup 命中必须返回缓存字节");
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
|
||||
assertNull(service.lookup("https://img.example.com/missing.jpg"), "未命中返回 null");
|
||||
service.flushPendingTouches();
|
||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper;
|
||||
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 本地验证入口:用生产真实批次数据 + 生产 LLM 网关跑 SimilarAsinLlmService 完整链路。
|
||||
* 用法:mvn compile test-compile 后执行
|
||||
* java -cp target/classes;target/test-classes;$(cat cp.txt) com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]
|
||||
*/
|
||||
public class SimilarAsinLlmLocalVerify {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
System.setOut(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.out), true, "UTF-8"));
|
||||
System.setErr(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.err), true, "UTF-8"));
|
||||
if (args.length < 2) {
|
||||
System.err.println("usage: SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]");
|
||||
System.exit(1);
|
||||
}
|
||||
String dataFile = args[0];
|
||||
String apiKey = args[1];
|
||||
boolean imgSwitch = args.length > 2 && Boolean.parseBoolean(args[2]);
|
||||
boolean categorySwitch = args.length > 3 && Boolean.parseBoolean(args[3]);
|
||||
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
JsonNode root = objectMapper.readTree(new File(dataFile));
|
||||
List<SimilarAsinResultRowDto> rows = new ArrayList<>();
|
||||
if (root.isArray()) {
|
||||
for (JsonNode node : root) {
|
||||
rows.add(fromJson(objectMapper, node));
|
||||
}
|
||||
} else {
|
||||
rows.add(fromJson(objectMapper, root));
|
||||
}
|
||||
System.out.println("[verify] loaded rows=" + rows.size() + " imgSwitch=" + imgSwitch
|
||||
+ " categorySwitch=" + categorySwitch);
|
||||
if (imgSwitch && categorySwitch) {
|
||||
System.out.println("[verify] raw first row alibaba[0].url="
|
||||
+ (rows.isEmpty() || rows.get(0).getAlibaba().isEmpty() ? "null"
|
||||
: rows.get(0).getAlibaba().get(0).getUrl()));
|
||||
}
|
||||
|
||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
||||
props.setLlmApiKey(apiKey);
|
||||
props.setLlmRowConcurrency(2);
|
||||
props.setLlmImageDownloadTimeoutSeconds(10);
|
||||
|
||||
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null);
|
||||
OssProperties ossProps = new OssProperties();
|
||||
ossProps.setEndpoint("https://oss.aishufu.top");
|
||||
ossProps.setPublicEndpoint("https://oss.aishufu.top");
|
||||
ossProps.setBucket("nanri-ai-images");
|
||||
ossProps.setAccessKeyId("appuser");
|
||||
ossProps.setAccessKeySecret("AppUser@2024SecureKey");
|
||||
OssStorageService oss = new OssStorageService(ossProps);
|
||||
PuzzleImageMerger merger = new PuzzleImageMerger(props);
|
||||
|
||||
// 生产真实类目数据(导出自 biz_product_category),spy 类目服务按 parentId 过滤。
|
||||
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> categories = loadCategories(objectMapper);
|
||||
ProductCategoryService categoryService = Mockito.spy(new ProductCategoryService(
|
||||
Mockito.mock(com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper.class)));
|
||||
Mockito.doAnswer(invocation -> {
|
||||
Long parentId = invocation.getArgument(0);
|
||||
List<com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo> items = categories.stream()
|
||||
.filter(c -> parentId == null ? c.getParentId() == null : parentId.equals(c.getParentId()))
|
||||
.map(c -> {
|
||||
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo item =
|
||||
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo();
|
||||
item.setId(c.getId());
|
||||
item.setParentId(c.getParentId());
|
||||
item.setName(c.getName());
|
||||
item.setCategoryKey(c.getCategoryKey());
|
||||
return item;
|
||||
})
|
||||
.toList();
|
||||
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo vo =
|
||||
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo();
|
||||
vo.setItems(items);
|
||||
vo.setTree(List.of());
|
||||
vo.setTotal((long) items.size());
|
||||
vo.setPage(1L);
|
||||
vo.setPageSize((long) Math.max(1, items.size()));
|
||||
vo.setHasMore(false);
|
||||
return vo;
|
||||
}).when(categoryService).children(Mockito.any(), Mockito.anyLong(), Mockito.anyLong());
|
||||
|
||||
SimilarAsinLlmService service = new SimilarAsinLlmService(
|
||||
client, props, ossProps, categoryService, merger, oss);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(rows, null, apiKey, imgSwitch, categorySwitch);
|
||||
long elapsed = System.currentTimeMillis() - start;
|
||||
System.out.println("[verify] done rows=" + result.size() + " elapsedMs=" + elapsed);
|
||||
for (SimilarAsinResultRowDto row : result) {
|
||||
System.out.println(String.format(
|
||||
"asin=%s | status=%s | isConform=%s | category=%s | reason=%s | isStock=%s | similarity=%s | mainUrl=%s | puzzle1=%s | puzzle2=%s",
|
||||
row.getAsin(), row.getStatus(), row.getIsConform(), row.getCategory(),
|
||||
row.getReason(), row.getIsStock(), row.getSimilarity(),
|
||||
shorten(row.getMainUrl()), shorten(row.getPuzzleImg1()), shorten(row.getPuzzleImg2())));
|
||||
}
|
||||
}
|
||||
|
||||
private static List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> loadCategories(ObjectMapper objectMapper) throws Exception {
|
||||
com.fasterxml.jackson.databind.JsonNode root = objectMapper.readTree(
|
||||
SimilarAsinLlmLocalVerify.class.getResourceAsStream("/categories.json"));
|
||||
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> list = new ArrayList<>();
|
||||
for (com.fasterxml.jackson.databind.JsonNode node : root) {
|
||||
com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity entity =
|
||||
new com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity();
|
||||
entity.setId(node.get("id").asLong());
|
||||
if (!node.get("parent_id").isNull()) {
|
||||
entity.setParentId(node.get("parent_id").asLong());
|
||||
}
|
||||
entity.setName(node.get("name").asText());
|
||||
entity.setCategoryKey(node.get("category_key").asText());
|
||||
entity.setSortOrder(node.get("sort_order").isNull() ? null : node.get("sort_order").asInt());
|
||||
entity.setDescription(node.get("description").isNull() ? null : node.get("description").asText());
|
||||
entity.setIsBuiltin(node.get("is_builtin") != null && node.get("is_builtin").asBoolean());
|
||||
list.add(entity);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto fromJson(ObjectMapper objectMapper, JsonNode node) throws Exception {
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin(text(node, "asin"));
|
||||
row.setTitle(text(node, "title"));
|
||||
row.setSku(text(node, "sku"));
|
||||
row.setCountry(text(node, "country"));
|
||||
row.setUrl(text(node, "url"));
|
||||
JsonNode alibaba = node.get("alibaba");
|
||||
if (alibaba != null && alibaba.isArray()) {
|
||||
List<SimilarAsinResultRowDto.AlibabaItem> items = new ArrayList<>();
|
||||
for (JsonNode item : alibaba) {
|
||||
items.add(objectMapper.treeToValue(item, SimilarAsinResultRowDto.AlibabaItem.class));
|
||||
}
|
||||
row.setAlibaba(items);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
private static String text(JsonNode node, String field) {
|
||||
JsonNode value = node.get(field);
|
||||
return value == null || value.isNull() ? null : value.asText();
|
||||
}
|
||||
|
||||
private static String shorten(String value) {
|
||||
if (value == null) {
|
||||
return "null";
|
||||
}
|
||||
return value.length() <= 70 ? value : value.substring(0, 70) + "...";
|
||||
}
|
||||
}
|
||||
+279
@@ -0,0 +1,279 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo;
|
||||
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo;
|
||||
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import org.mockito.Mockito;
|
||||
|
||||
class SimilarAsinLlmServiceTest {
|
||||
|
||||
private static byte[] jpegBytes() {
|
||||
try {
|
||||
BufferedImage image = new BufferedImage(10, 10, BufferedImage.TYPE_INT_RGB);
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", baos);
|
||||
return baos.toByteArray();
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private SimilarAsinProperties properties() {
|
||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
||||
props.setLlmApiKey("test-key");
|
||||
return props;
|
||||
}
|
||||
|
||||
private SimilarAsinLlmService service(SimilarAsinLlmClient llmClient) {
|
||||
OssStorageService ossStorage = mock(OssStorageService.class);
|
||||
when(ossStorage.getPublicUrl(anyString())).thenAnswer(invocation -> "https://oss.aishufu.top/nanri-ai-images/" + invocation.getArgument(0));
|
||||
PuzzleImageMerger merger = mock(PuzzleImageMerger.class);
|
||||
when(merger.merge(anyList(), Mockito.<SimilarAsinResultRowDto>any())).thenReturn(jpegBytes());
|
||||
ProductCategoryService categoryService = mock(ProductCategoryService.class);
|
||||
when(categoryService.children(any(), anyLong(), anyLong()))
|
||||
.thenAnswer(invocation -> {
|
||||
Long parentId = invocation.getArgument(0);
|
||||
if (parentId == null) {
|
||||
return categoryPage("类目A", 1L);
|
||||
}
|
||||
if (parentId == 1L) {
|
||||
return categoryPage("类目B", 2L);
|
||||
}
|
||||
if (parentId == 2L) {
|
||||
return categoryPage("类目C", 3L);
|
||||
}
|
||||
return emptyCategoryPage();
|
||||
});
|
||||
SimilarAsinLlmService svc = new SimilarAsinLlmService(
|
||||
llmClient,
|
||||
properties(),
|
||||
mock(OssProperties.class),
|
||||
categoryService,
|
||||
merger,
|
||||
ossStorage);
|
||||
svc.setDownloadHttpClientForTest(mockHttpClient());
|
||||
return svc;
|
||||
}
|
||||
|
||||
private static ProductCategoryListVo categoryPage(String name, long id) {
|
||||
ProductCategoryItemVo item = new ProductCategoryItemVo();
|
||||
item.setId(id);
|
||||
item.setName(name);
|
||||
ProductCategoryListVo vo = new ProductCategoryListVo();
|
||||
vo.setItems(List.of(item));
|
||||
vo.setTotal(1L);
|
||||
vo.setPage(1L);
|
||||
vo.setPageSize(1L);
|
||||
vo.setHasMore(false);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static ProductCategoryListVo emptyCategoryPage() {
|
||||
ProductCategoryListVo vo = new ProductCategoryListVo();
|
||||
vo.setItems(List.of());
|
||||
vo.setTotal(0L);
|
||||
vo.setPage(1L);
|
||||
vo.setPageSize(1L);
|
||||
vo.setHasMore(false);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static HttpClient mockHttpClient() {
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
try {
|
||||
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
byte[] body = jpegBytes();
|
||||
HttpRequest request = invocation.getArgument(0);
|
||||
return new TestHttpResponse(200, body, request);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
private static final class TestHttpResponse implements HttpResponse<byte[]> {
|
||||
private final int statusCode;
|
||||
private final byte[] body;
|
||||
private final HttpRequest request;
|
||||
|
||||
TestHttpResponse(int statusCode, byte[] body, HttpRequest request) {
|
||||
this.statusCode = statusCode;
|
||||
this.body = body;
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int statusCode() {
|
||||
return statusCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpRequest request() {
|
||||
return request;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.util.Optional<HttpResponse<byte[]>> previousResponse() {
|
||||
return java.util.Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.http.HttpHeaders headers() {
|
||||
return java.net.http.HttpHeaders.of(Map.of(), (a, b) -> true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] body() {
|
||||
return body;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.URI uri() {
|
||||
return java.net.URI.create("http://test");
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.net.http.HttpClient.Version version() {
|
||||
return HttpClient.Version.HTTP_1_1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.util.Optional<javax.net.ssl.SSLSession> sslSession() {
|
||||
return java.util.Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private SimilarAsinLlmClient llmClient(String apiKey, Map<String, String> responses) {
|
||||
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
|
||||
when(client.resolveApiKey(anyString())).thenReturn(apiKey == null ? "" : apiKey);
|
||||
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
|
||||
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String response = responses.get("chat");
|
||||
if (response == null) {
|
||||
throw new IllegalStateException("unexpected chat call");
|
||||
}
|
||||
return response;
|
||||
});
|
||||
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String response = responses.get("images");
|
||||
if (response == null) {
|
||||
throw new IllegalStateException("unexpected images call");
|
||||
}
|
||||
return response;
|
||||
});
|
||||
return client;
|
||||
}
|
||||
|
||||
private static final com.fasterxml.jackson.databind.ObjectMapper OBJECT_MAPPER =
|
||||
new com.fasterxml.jackson.databind.ObjectMapper();
|
||||
|
||||
/** 类目匹配(按序号返回不同类目名)与合规检查(后续)返回不同 JSON。 */
|
||||
private SimilarAsinLlmClient llmClientStaged(String apiKey, List<String> categoryJsons,
|
||||
String conformJson, String imagesJson) {
|
||||
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
|
||||
int[] categoryIndex = {0};
|
||||
when(client.resolveApiKey(anyString())).thenReturn(apiKey);
|
||||
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
|
||||
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
if (categoryIndex[0] < categoryJsons.size()) {
|
||||
return categoryJsons.get(categoryIndex[0]++);
|
||||
}
|
||||
return conformJson;
|
||||
});
|
||||
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> imagesJson);
|
||||
when(client.parseJsonContent(anyString()))
|
||||
.thenAnswer(invocation -> parseJson(invocation.getArgument(0)));
|
||||
return client;
|
||||
}
|
||||
|
||||
private static com.fasterxml.jackson.databind.JsonNode parseJson(String content) {
|
||||
try {
|
||||
return OBJECT_MAPPER.readTree(content);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void noApiKeyKeepsRawRows() {
|
||||
SimilarAsinLlmService service = service(llmClient("", Map.of()));
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin("B0TEST");
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "", true, true);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("B0TEST", result.get(0).getAsin());
|
||||
assertNull(result.get(0).getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
void categorySwitchOffOnlyPreparesImagesAndMarksNotExistsWhenNoMainUrl() {
|
||||
SimilarAsinLlmService service = service(llmClient("k", Map.of()));
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin("B0TEST");
|
||||
row.setTitle("Test product");
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", false, false);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("不存在", result.get(0).getStatus());
|
||||
assertNull(result.get(0).getIsConform());
|
||||
assertNull(result.get(0).getPuzzleImg1());
|
||||
}
|
||||
|
||||
@Test
|
||||
void imageCompareStopsOnStockAndFillsFields() {
|
||||
// 前 2 次 chat:一级/二级类目匹配返回名称(Java 按名回查候选取真实 ID);第 3 次:合规检查。
|
||||
SimilarAsinLlmClient client = llmClientStaged("k", List.of("{\"name\":\"类目A\"}", "{\"name\":\"类目B\"}"),
|
||||
"{\"asin\":\"B0TEST\",\"is_conform\":\"符合\",\"reason\":\"无\",\"category\":\"类目A->类目B->类目C\"}",
|
||||
"{\"asin\":\"B0TEST\",\"is_stock\":\"有货\",\"similarity\":\"95%\",\"status\":\"成功\",\"is_conform\":\"符合\",\"category\":\"类目A->类目B->类目C\"}");
|
||||
SimilarAsinLlmService service = service(client);
|
||||
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin("B0TEST");
|
||||
row.setTitle("Test product");
|
||||
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
|
||||
SimilarAsinResultRowDto.AlibabaItem item = new SimilarAsinResultRowDto.AlibabaItem();
|
||||
item.setUrl("https://cbu01.alicdn.com/img/1.jpg");
|
||||
row.setAlibaba(List.of(item));
|
||||
|
||||
// 一级/二级都匹配(按名回查 ID),三级候选可用,合规符合 → 图片对比,有货即停。
|
||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", true, true);
|
||||
assertEquals(1, result.size());
|
||||
SimilarAsinResultRowDto out = result.get(0);
|
||||
assertEquals("成功", out.getStatus());
|
||||
assertEquals("有货", out.getIsStock());
|
||||
assertEquals("95%", out.getSimilarity());
|
||||
assertEquals("符合", out.getIsConform());
|
||||
}
|
||||
}
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 9:chunk 查询从单行分页改为批量 keyset 分页,保持低内存读取。
|
||||
* loadChunksKeyset 按 id 递增分批拉取(每批 pageSize),最后按 chunkIndex 升序合并,
|
||||
* 避免超大任务一次 selectList 全量载入 chunk 元数据。
|
||||
* mock 分页由 wrapper 中 gt("id", lastId) 的 keyset 值驱动,保证重复调用可复现(幂等)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceChunkKeysetTest {
|
||||
|
||||
private static final String MODULE = "similar-asin";
|
||||
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
|
||||
private static TaskChunkEntity chunk(long id, int chunkIndex) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7004L);
|
||||
chunk.setModuleType(MODULE);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static List<TaskChunkEntity> chunks(long... idsAndIndexes) {
|
||||
List<TaskChunkEntity> result = new ArrayList<>();
|
||||
for (int i = 0; i < idsAndIndexes.length; i += 2) {
|
||||
result.add(chunk(idsAndIndexes[i], (int) idsAndIndexes[i + 1]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 从 wrapper 的 SQL 片段解析 keyset:匹配 "id > #{ew.paramNameValuePairs.键}" 后从参数表取值。 */
|
||||
private static long keysetOf(QueryWrapper<TaskChunkEntity> wrapper) {
|
||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||
.compile("id\\s*>\\s*#\\{ew\\.paramNameValuePairs\\.(\\w+)\\}", java.util.regex.Pattern.CASE_INSENSITIVE)
|
||||
.matcher(wrapper.getSqlSegment());
|
||||
if (m.find()) {
|
||||
Object value = wrapper.getParamNameValuePairs().get(m.group(1));
|
||||
if (value instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
}
|
||||
return 0L;
|
||||
}
|
||||
|
||||
/** 按 keyset 驱动分页:每次 selectList 返回 id > keyset 的下一批,天然支持重复调用。 */
|
||||
private void stubKeysetPages(List<TaskChunkEntity> all, int pageSize) {
|
||||
int batch = pageSize > 0 ? pageSize : 500;
|
||||
doAnswer(invocation -> {
|
||||
long lastId = keysetOf(invocation.getArgument(0));
|
||||
return all.stream().filter(c -> c.getId() > lastId).limit(batch).toList();
|
||||
}).when(taskChunkMapper).selectList(any());
|
||||
}
|
||||
|
||||
private List<Long> keysetIdsFromRounds(int rounds) {
|
||||
ArgumentCaptor<QueryWrapper<TaskChunkEntity>> captor =
|
||||
ArgumentCaptor.forClass(QueryWrapper.class);
|
||||
verify(taskChunkMapper, times(rounds)).selectList(captor.capture());
|
||||
List<Long> keysets = new ArrayList<>();
|
||||
for (QueryWrapper<TaskChunkEntity> wrapper : captor.getAllValues()) {
|
||||
keysets.add(keysetOf(wrapper));
|
||||
}
|
||||
return keysets;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_normal_default_path() {
|
||||
// 正常输入:chunk 数小于 pageSize,一轮拉完,结果全且按 chunkIndex 有序
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
|
||||
stubKeysetPages(all, 500);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
assertEquals(3, result.size());
|
||||
assertEquals(List.of(1, 2, 3), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
||||
assertEquals(List.of(0L), keysetIdsFromRounds(1), "首轮 keyset 为 0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_normal_multiple_items() {
|
||||
// 超过 pageSize:多轮拉取,keyset 逐轮推进,全部合并且按 chunkIndex 升序、无重复
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7);
|
||||
stubKeysetPages(all, 3);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
||||
assertEquals(7, result.size());
|
||||
assertEquals(List.of(1, 2, 3, 4, 5, 6, 7), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
||||
long distinctIds = result.stream().map(TaskChunkEntity::getId).distinct().count();
|
||||
assertEquals(7, distinctIds, "keyset 分页不能产生重复 chunk");
|
||||
assertEquals(List.of(0L, 3L, 6L), keysetIdsFromRounds(3), "keyset 逐轮推进,不足一批即止");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行同一输入:每轮都从 keyset=0 开始,结果一致
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
|
||||
stubKeysetPages(all, 500);
|
||||
List<TaskChunkEntity> first = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
List<TaskChunkEntity> second = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getId(), second.get(i).getId());
|
||||
}
|
||||
assertEquals(List.of(0L, 0L), keysetIdsFromRounds(2), "重复执行每轮都从 keyset=0 开始");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_boundary_empty_input() {
|
||||
// 空集合:返回空列表,不创建无效资源,且只查一轮
|
||||
stubKeysetPages(List.of(), 500);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
||||
assertNotNull(result);
|
||||
assertEquals(0, result.size());
|
||||
verify(taskChunkMapper, times(1)).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_boundary_single_item() {
|
||||
// 单 chunk:一轮返回后 keyset 推进即拉空,不依赖批量路径
|
||||
List<TaskChunkEntity> all = chunks(42, 9);
|
||||
stubKeysetPages(all, 1);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 1);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals(9, result.get(0).getChunkIndex());
|
||||
assertEquals(42L, result.get(0).getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_boundary_limit_and_overflow() {
|
||||
// chunk 数恰好等于 pageSize 的倍数:最后一轮仍返回非空才继续,全部取回
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6);
|
||||
stubKeysetPages(all, 3);
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
||||
assertEquals(6, result.size());
|
||||
// pageSize 为 0/负数:回退默认 500,不抛异常
|
||||
stubKeysetPages(all, 0);
|
||||
List<TaskChunkEntity> fallback = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 0);
|
||||
assertEquals(6, fallback.size());
|
||||
stubKeysetPages(all, -5);
|
||||
List<TaskChunkEntity> negative = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, -5);
|
||||
assertEquals(6, negative.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_invalid_input_rejected() {
|
||||
// taskId 为 null:安全返回空列表,不发起查询
|
||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, null, MODULE, 500);
|
||||
assertNotNull(result);
|
||||
assertEquals(0, result.size());
|
||||
verify(taskChunkMapper, times(0)).selectList(any());
|
||||
// mapper 查询抛异常:转项目约定异常,消息可识别
|
||||
when(taskChunkMapper.selectList(any())).thenThrow(new IllegalStateException("db down"));
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"异常消息必须可识别,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_009_chunk_dependency_failure_releases_resources() {
|
||||
// 第二轮查询失败:抛异常不返回半截结果;恢复后重试可完整返回
|
||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5);
|
||||
doAnswer(invocation -> {
|
||||
long lastId = keysetOf(invocation.getArgument(0));
|
||||
if (lastId == 0L) {
|
||||
return all.subList(0, 3);
|
||||
}
|
||||
if (lastId == 3L) {
|
||||
throw new IllegalStateException("db down mid-page");
|
||||
}
|
||||
return all.subList(3, 5);
|
||||
}).when(taskChunkMapper).selectList(any());
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"异常消息必须可识别,实际: " + ex.getMessage());
|
||||
// 恢复后重试成功:5 个 chunk 全部取回
|
||||
stubKeysetPages(all, 3);
|
||||
List<TaskChunkEntity> recovered = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
||||
assertEquals(5, recovered.size());
|
||||
assertEquals(List.of(1, 2, 3, 4, 5), recovered.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
||||
}
|
||||
}
|
||||
+399
@@ -0,0 +1,399 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 13:chunk 合并增加单次最大行数与 payload 字节上限。
|
||||
* 校验点位于 mergeChunkPayload 单次合并入口:合并后总行数超过 chunkMergeMaxRows、
|
||||
* 或 payload 字节超过 chunkMergePayloadMaxBytes 时,从最旧行开始降级到
|
||||
* orphan 兜底(assemble 阶段 putIfAbsent 合并回结果,不丢数据);
|
||||
* 单行本身超过字节上限时抛可识别异常拒绝合并。低于上限的行为与旧路径完全一致。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceChunkMergeLimitTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(90000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
|
||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(16L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/90000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String asin, String title) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(rowToken);
|
||||
r.setAsin(asin);
|
||||
r.setCountry("英国");
|
||||
r.setTitle(title);
|
||||
return r;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(9004L);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
/** 记录每次 storeChunkPayloadVersioned 收到的 payload 字符串。 */
|
||||
private void stubChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicReference<String> storedPayload) throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
|
||||
.thenReturn(payloadJson);
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storedPayload.set(invocation.getArgument(4));
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task,
|
||||
String scopeHash, Integer chunkIndex,
|
||||
List<SimilarAsinResultRowDto> llmRows) throws Exception {
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, scopeHash, chunkIndex, llmRows, Map.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_normal_default_path() throws Exception {
|
||||
// 正常输入:行数与 payload 字节均在上限内,合并走原路径,结果完整保留。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicReference<String> storedPayload = new AtomicReference<>("");
|
||||
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(
|
||||
row("r1", "B0A0000001", "标题1"),
|
||||
row("r2", "B0A0000002", "标题2"),
|
||||
row("r3", "B0A0000003", "标题3"));
|
||||
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertNotNull(storedPayload.get());
|
||||
assertTrue(storedPayload.get().contains("\"r1\"") && storedPayload.get().contains("\"r3\""),
|
||||
"上限内合并必须完整保留存量行与新增行,实际: " + storedPayload.get());
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多 chunk 一次 merge,全部在上限内,各 chunk 分别写回、结果不丢失。
|
||||
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "B0A0000001", "标题1"))));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r2", "B0A0000002", "标题2"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
|
||||
AtomicLong selectOneRound = new AtomicLong(0);
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
|
||||
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMerge(service, task, null, null, List.of(
|
||||
row("r1", "B0A0000001", "标题1-新"),
|
||||
row("r2", "B0A0000002", "标题2-新")));
|
||||
|
||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, times(2)).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:每次 merge 恰好写一次,不产生重复对象、重复状态。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicLong storeCalls = new AtomicLong(0);
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storeCalls.incrementAndGet();
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "B0A0000001", "标题1"));
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertEquals(2, storeCalls.get(), "重复执行同一输入:每次 merge 恰好写回一次,无多余请求");
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空列表安全跳过,不读取 chunk、不写存储、不创建资源。
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMerge(service, task, "hashA", 1, null);
|
||||
invokeMerge(service, task, "hashA", 1, List.of());
|
||||
verify(taskChunkMapper, times(0)).selectList(any());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_boundary_single_item() throws Exception {
|
||||
// 单行:不依赖批量路径,合并后结果正确。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicReference<String> storedPayload = new AtomicReference<>("");
|
||||
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
|
||||
assertTrue(storedPayload.get().contains("\"r1\""), "单行合并也必须写回 chunk payload,实际: " + storedPayload.get());
|
||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_boundary_limit_and_overflow() throws Exception {
|
||||
// 超限场景三连:行数超限降级、字节超限降级、单行超字节上限拒绝。
|
||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(2);
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicLong storeCalls = new AtomicLong(0);
|
||||
AtomicReference<String> lastStoredPayload = new AtomicReference<>("");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storeCalls.incrementAndGet();
|
||||
lastStoredPayload.set(invocation.getArgument(4));
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
llmRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
|
||||
}
|
||||
|
||||
// Phase A:行数超限(上限 2,存量 1 + 新增 5)→ 只保留上限内最新行,超限部分转 orphan。
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertTrue(lastStoredPayload.get().contains("\"r4\"") && lastStoredPayload.get().contains("\"r5\""),
|
||||
"行数超限时保留上限内的最新行,实际: " + lastStoredPayload.get());
|
||||
assertFalse(lastStoredPayload.get().contains("\"r0\""), "行数超限时最旧行被降级,实际: " + lastStoredPayload.get());
|
||||
assertEquals(1, storeCalls.get());
|
||||
verify(taskScopeStateMapper, times(1)).insert(any(TaskScopeStateEntity.class));
|
||||
verify(transientPayloadStorageService, times(1))
|
||||
.storeParsedPayloadEntry(anyString(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
|
||||
// Phase B:字节超限(行数放开)→ 从最旧行降级到字节上限内,保留最新结果。
|
||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
|
||||
long oneRowBytes = rowsJson(List.of(row("r9", "B0A0000099", "样本行"))).getBytes(StandardCharsets.UTF_8).length;
|
||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(oneRowBytes + 5L);
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
assertTrue(lastStoredPayload.get().contains("\"r5\""), "字节超限时保留最新行,实际: " + lastStoredPayload.get());
|
||||
assertFalse(lastStoredPayload.get().contains("\"r0\""), "字节超限时最旧行被降级,实际: " + lastStoredPayload.get());
|
||||
assertEquals(2, storeCalls.get());
|
||||
verify(taskScopeStateMapper, times(2)).insert(any(TaskScopeStateEntity.class));
|
||||
|
||||
// Phase C:单行本身超过字节上限 → 抛可识别异常拒绝合并,不写 chunk。
|
||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(5L);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertNotNull(ex.getMessage());
|
||||
assertTrue(ex.getMessage().contains("字节上限"),
|
||||
"超字节上限必须抛可识别异常,实际: " + ex.getMessage());
|
||||
assertEquals(2, storeCalls.get(), "拒绝合并时不写 chunk");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_invalid_input_rejected() {
|
||||
// 非法输入:chunk 载荷加载失败(resolve 抛异常)时抛出可识别异常且不写 chunk。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenThrow(new IllegalStateException("rustfs down"));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertNotNull(ex.getMessage());
|
||||
assertTrue(ex.getMessage().contains("chunk"),
|
||||
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_013_payload_row_count_chunk_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:payload 存储失败时抛带上下文的可识别异常、无残留状态;恢复后重试成功。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||
AtomicLong storeCalls = new AtomicLong(0);
|
||||
doAnswer(invocation -> {
|
||||
if (storeCalls.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("rustfs down");
|
||||
}
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("相似ASIN分片载荷"),
|
||||
"存储失败必须抛带上下文的可识别异常,实际: " + ex.getMessage());
|
||||
assertEquals(1, storeCalls.get(), "失败时只尝试一次即抛出,不静默吞错");
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
|
||||
// 依赖恢复后重试成功:结果正确、无残留状态。
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
assertEquals(2, storeCalls.get(), "恢复后重试成功");
|
||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
}
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 12:扩展 Coze 结果缓冲覆盖范围,减少频繁读写完整 chunk payload。
|
||||
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1";Task 12 扩展为:
|
||||
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
|
||||
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
|
||||
* 3) 统一走 bufferLlmRowsOrMerge:缓冲失败回退立即 merge,结果不丢失。
|
||||
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceCozeBufferScopeTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
|
||||
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
|
||||
private static final String CREDENTIAL = "cred-1";
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinLlmService similarAsinLlmService;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(properties.getLlmBatchSize()).thenReturn(5);
|
||||
lenient().when(properties.getLlmTextOnlyBatchSize()).thenReturn(10);
|
||||
lenient().when(properties.isLlmResultBufferEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getDbJobTouchIntervalMillis()).thenReturn(2_000L);
|
||||
lenient().when(properties.getDbTaskTouchIntervalMillis()).thenReturn(2_000L);
|
||||
lenient().when(properties.getLlmFlushPendingMinutes()).thenReturn(10);
|
||||
lenient().when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(
|
||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
|
||||
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
|
||||
.thenAnswer(invocation -> "rustfs:coze-result/" + NEXT_ID.incrementAndGet());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country, String title) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(id);
|
||||
r.setAsin(asin);
|
||||
r.setCountry(country);
|
||||
r.setTitle(title);
|
||||
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
|
||||
return r;
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7104L);
|
||||
chunk.setModuleType(MODULE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
chunk.setPayloadHash("h-" + id);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7104L);
|
||||
task.setModuleType(MODULE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setResultJson("{\"categorySwitch\":true}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private static TaskScopeStateEntity state(FileTaskEntity task, long id, String status, int batchTotal) {
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setId(id);
|
||||
state.setTaskId(task.getId());
|
||||
state.setModuleType(MODULE);
|
||||
state.setScopeHash("scope-" + id);
|
||||
state.setLlmStatus(status);
|
||||
state.setParsedPayloadJson("ptr:batch-" + id);
|
||||
state.setStateJson("{\"jobId\":7101,\"resultId\":7201,\"chunkScopeHash\":null,\"chunkIndex\":null,"
|
||||
+ "\"batchIndex\":1,\"batchTotal\":" + batchTotal + ",\"ownerInstanceId\":\"test-instance\","
|
||||
+ "\"submitRetryCount\":0,\"credentialName\":\"" + CREDENTIAL + "\",\"resultPayloadPointer\":\"ptr:buffer-" + id + "\"}");
|
||||
return state;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private SimilarAsinTaskService.LlmBatchContext context(int batchTotal) {
|
||||
return new SimilarAsinTaskService.LlmBatchContext(
|
||||
7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null);
|
||||
}
|
||||
|
||||
private void stubChunkMerge(String payloadJson) throws Exception {
|
||||
TaskChunkEntity chunk = chunk(1L, "scope-1", 1, "ptr:chunk-1");
|
||||
// loadSubmittedChunks 只保留非空 chunk,chunk payload 必须能解析出至少一行。
|
||||
// 全部 lenient:缓冲成功路径不触达 merge,仅缓冲失败/flush 合并路径消费。
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return payloadJson;
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
private static String chunkRowsJson() throws Exception {
|
||||
// chunk-1 已含 r1 行:loadSubmittedChunks 只保留非空 chunk,且 rowKey 索引能命中缓冲行。
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_default_path() throws Exception {
|
||||
// 正常输入:DONE 结果(batchTotal=1 单 batch)经 bufferLlmRowsOrMerge 走缓冲,
|
||||
// 不立即写 chunk;缓冲失败回退立即 merge 结果不丢失。
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多个 DONE state(单 batch)全部缓冲;flush 后按 chunk 分组一次合并
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(
|
||||
List.of(state(task, 1L, "DONE", 1), state(task, 2L, "DONE", 1)));
|
||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return chunkRowsJson();
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(1L, "scope-1", 1, "ptr:chunk-1"));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1")));
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
|
||||
flush.setAccessible(true);
|
||||
flush.invoke(service, 7104L);
|
||||
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
||||
// pointer 清理:每个缓冲 state 都更新
|
||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:缓冲写幂等(同一 state 不产生重复 buffer/merge)
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
TaskScopeStateEntity state = state(task, 1L, "DONE", 2);
|
||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
||||
|
||||
// 缓冲 2 次(每次重新写 pointer 是幂等语义:同一 state 覆盖写,无重复行)
|
||||
verify(transientPayloadStorageService, times(2)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_empty_input() throws Exception {
|
||||
// 空输入:无行时缓冲与 merge 都不发生,不创建无效资源
|
||||
FileTaskEntity task = task();
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of());
|
||||
bufferOrMerge.invoke(service, state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_single_item() throws Exception {
|
||||
// 单 batch(batchTotal=1):原 P0-3 例外,现在也缓冲
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_limit_and_overflow() throws Exception {
|
||||
// 缓冲开关关闭:回退立即 merge,DONE 结果仍落 chunk 不丢失
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(properties.isLlmResultBufferEnabled()).thenReturn(false);
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_invalid_input_rejected() throws Exception {
|
||||
// 缓冲写失败(storeParsedPayloadEntry 抛异常):回退立即 merge,结果不丢失
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(transientPayloadStorageService.storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)))
|
||||
.thenThrow(new IllegalStateException("rustfs full"));
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.LlmBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_dependency_failure_releases_resources() throws Exception {
|
||||
// flush 时 chunk 写失败:抛可识别业务异常且不清 pointer(保留待重试);
|
||||
// 依赖恢复后重试 flush 成功,chunk 合并一次、pointer 清理。
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
TaskScopeStateEntity s = state(task, 1L, "DONE", 1);
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(s));
|
||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return chunkRowsJson();
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
java.util.concurrent.atomic.AtomicInteger storeCalls = new java.util.concurrent.atomic.AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (storeCalls.incrementAndGet() == 1) {
|
||||
throw new IllegalStateException("rustfs write failed");
|
||||
}
|
||||
return "ptr:stored-" + invocation.getArgument(3);
|
||||
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
|
||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
|
||||
flush.setAccessible(true);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
flush.invoke(service, 7104L);
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("刷新缓冲区"),
|
||||
"flush 失败消息必须可识别, 实际: " + ex.getMessage());
|
||||
// 失败分组不清 pointer:buffer 未被删除、stateJson 未更新,留待重试
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
verify(taskScopeStateMapper, never()).update(any(), any());
|
||||
|
||||
// 恢复后重试 flush:chunk 合并成功一次,pointer 清理
|
||||
flush.invoke(service, 7104L);
|
||||
assertEquals(2, storeCalls.get(), "恢复后重试应再次写 chunk");
|
||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
}
|
||||
+378
@@ -0,0 +1,378 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 6:分组数据改为索引/范围引用,避免 groups 嵌套复制完整行对象。
|
||||
* 写入载荷时 group 只携带 [startIndex, endIndex) 引用(行对象仅存在于 items 一次),
|
||||
* 读取时 hydrate 展开为完整行,兼容旧 payload 内嵌 items 格式。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceGroupRefTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(30000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/30000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-group-ref-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0GRP%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("group-ref.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
private String storedPayloadJson() {
|
||||
// 捕获最近一次存储的 payload JSON
|
||||
return "rustfs:task-parsed/similar-asin/30000/payload.json";
|
||||
}
|
||||
|
||||
private SimilarAsinParsedPayloadDto readPayload(String json) throws Exception {
|
||||
return objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
}
|
||||
|
||||
private static SimilarAsinParsedRowVo row(String fileKey, int index, String groupKey) {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setSourceFileKey(fileKey);
|
||||
row.setSourceFilename("group-ref.xlsx");
|
||||
row.setRowIndex(index);
|
||||
row.setSourceId(String.valueOf(index));
|
||||
row.setDisplayId(String.valueOf(index));
|
||||
row.setRowToken(fileKey + "::row::" + index);
|
||||
row.setGroupKey(groupKey);
|
||||
row.setAsin(String.format("B0GRP%05d", index));
|
||||
row.setCountry("英国");
|
||||
row.setValues(new java.util.LinkedHashMap<>());
|
||||
return row;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_normal_default_path() throws Exception {
|
||||
// 正常多行文件:groups 写入为索引引用,行对象只出现在 items 一次
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenAnswer(invocation -> {
|
||||
String json = invocation.getArgument(3);
|
||||
return "rustfs:task-parsed/similar-asin/30000/payload.json::" + json;
|
||||
});
|
||||
File workbook = buildWorkbook(150);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-default.xlsx");
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
// 每个 group 是索引引用:携带 [startIndex, endIndex),区间宽度等于 itemCount
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertNotNull(group.getStartIndex());
|
||||
assertNotNull(group.getEndIndex());
|
||||
assertTrue(group.getStartIndex() < group.getEndIndex());
|
||||
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItemCount());
|
||||
}
|
||||
// 响应 groups 按预览上限裁剪(默认 100),引用区间覆盖全部行、不重叠
|
||||
int coverage = 0;
|
||||
int prevEnd = -1;
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertTrue(group.getStartIndex() >= prevEnd, "组区间不能重叠且必须顺序递增");
|
||||
coverage += group.getEndIndex() - group.getStartIndex();
|
||||
prevEnd = group.getEndIndex();
|
||||
}
|
||||
assertTrue(coverage <= 100 && coverage > 0, "预览组覆盖行数必须在 (0, 预览上限] 内,实际 " + coverage);
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
// 响应组内嵌预览行(前端兼容):每个组 items 与引用区间宽度一致
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertNotNull(group.getItems());
|
||||
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItems().size(),
|
||||
"响应组内嵌预览行数量必须与引用区间宽度一致");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_normal_multiple_items() throws Exception {
|
||||
// 多组批量:每组行数不同,引用与 items 严格对应且顺序稳定
|
||||
String json = groupRefJson(3, new int[][]{{0, 3}, {3, 8}, {8, 10}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
assertEquals(10, payload.getItems().size());
|
||||
assertEquals(3, payload.getGroups().size());
|
||||
for (int g = 0; g < payload.getGroups().size(); g++) {
|
||||
SimilarAsinParsedGroupVo group = payload.getGroups().get(g);
|
||||
int start = group.getStartIndex();
|
||||
int end = group.getEndIndex();
|
||||
assertTrue(end - start >= 1);
|
||||
// 展开后行与 items 对应(首行即 items[start],行内容一致)
|
||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(start, end));
|
||||
assertEquals(end - start, expanded.size());
|
||||
assertEquals("t" + (start + 1), expanded.get(0).getRowToken(), "展开首行必须是 items[start]");
|
||||
assertEquals("B0GRP" + String.format("%05d", start + 1), expanded.get(0).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复展开同一 payload:结果一致,且不修改 items
|
||||
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 5}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
List<SimilarAsinParsedRowVo> first = hydrateForTest(payload);
|
||||
List<SimilarAsinParsedRowVo> second = hydrateForTest(payload);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
}
|
||||
assertEquals(5, payload.getItems().size(), "展开不能修改 payload 内部状态");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_boundary_empty_input() throws Exception {
|
||||
// 空 groups:引用列表为空,不创建无效引用
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setItems(List.of());
|
||||
payload.setGroups(List.of());
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertNotNull(restored);
|
||||
assertEquals(0, restored.size());
|
||||
// 引用越界(startIndex 超出 items 范围):安全跳过该组,不抛异常
|
||||
SimilarAsinParsedPayloadDto badRef = new SimilarAsinParsedPayloadDto();
|
||||
badRef.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setGroupKey("f.xlsx::1");
|
||||
group.setStartIndex(5);
|
||||
group.setEndIndex(7);
|
||||
badRef.setGroups(List.of(group));
|
||||
List<SimilarAsinParsedRowVo> outOfRange = SimilarAsinTaskService.resolveAllRows(badRef);
|
||||
assertEquals(0, outOfRange.size(), "越界引用必须安全跳过");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_boundary_single_item() throws Exception {
|
||||
// 单行单组:区间为 [0,1),单行不依赖批量路径
|
||||
String json = groupRefJson(1, new int[][]{{0, 1}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
assertEquals(1, payload.getGroups().size());
|
||||
SimilarAsinParsedGroupVo group = payload.getGroups().get(0);
|
||||
assertEquals(0, group.getStartIndex());
|
||||
assertEquals(1, group.getEndIndex());
|
||||
assertEquals(1, group.getItemCount());
|
||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(0, 1));
|
||||
assertEquals(1, expanded.size());
|
||||
assertEquals("B0GRP00001", expanded.get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_boundary_limit_and_overflow() throws Exception {
|
||||
// 组引用到达 items 末尾:endIndex == items.size(),不越界
|
||||
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 6}});
|
||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
||||
assertEquals(6, payload.getItems().size());
|
||||
SimilarAsinParsedGroupVo last = payload.getGroups().get(1);
|
||||
assertEquals(6, last.getEndIndex());
|
||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(last.getStartIndex(), last.getEndIndex()));
|
||||
assertEquals(4, expanded.size());
|
||||
// 未携带 items 的旧 payload 走 allItems 兜底
|
||||
String legacy = "{\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0OLD00001\"}],"
|
||||
+ "\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":1,\"itemCount\":1}]}";
|
||||
SimilarAsinParsedPayloadDto legacyPayload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class);
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacyPayload);
|
||||
assertEquals(1, restored.size());
|
||||
assertEquals("B0OLD00001", restored.get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_invalid_input_rejected() throws Exception {
|
||||
// 非法区间:endIndex <= startIndex,安全跳过
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1"), row("f.xlsx", 2, "f.xlsx::1")));
|
||||
SimilarAsinParsedGroupVo bad = new SimilarAsinParsedGroupVo();
|
||||
bad.setGroupKey("f.xlsx::1");
|
||||
bad.setStartIndex(1);
|
||||
bad.setEndIndex(1);
|
||||
payload.setGroups(List.of(bad));
|
||||
assertEquals(0, SimilarAsinTaskService.resolveAllRows(payload).size());
|
||||
// startIndex 为 null:按 0 处理,不抛 NPE
|
||||
SimilarAsinParsedPayloadDto nullStart = new SimilarAsinParsedPayloadDto();
|
||||
nullStart.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
|
||||
SimilarAsinParsedGroupVo g = new SimilarAsinParsedGroupVo();
|
||||
g.setGroupKey("f.xlsx::1");
|
||||
g.setStartIndex(null);
|
||||
g.setEndIndex(1);
|
||||
nullStart.setGroups(List.of(g));
|
||||
assertEquals(1, SimilarAsinTaskService.resolveAllRows(nullStart).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_006_group_dependency_failure_releases_resources() throws Exception {
|
||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,groups 引用与行一致
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(80);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/gr-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/30001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/gr-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-recovered.xlsx");
|
||||
assertEquals(80, vo.getAcceptedRows());
|
||||
int coverage = 0;
|
||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
||||
assertTrue(group.getStartIndex() < group.getEndIndex());
|
||||
coverage += group.getEndIndex() - group.getStartIndex();
|
||||
}
|
||||
assertEquals(80, coverage);
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
private static String groupRefJson(int groupCount, int[][] ranges) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("{\"items\":[");
|
||||
// 计算总行数
|
||||
int maxEnd = 0;
|
||||
for (int[] r : ranges) {
|
||||
maxEnd = Math.max(maxEnd, r[1]);
|
||||
}
|
||||
for (int i = 1; i <= maxEnd; i++) {
|
||||
if (i > 1) {
|
||||
sb.append(",");
|
||||
}
|
||||
sb.append("{\"rowToken\":\"t").append(i).append("\",\"asin\":\"B0GRP")
|
||||
.append(String.format("%05d", i)).append("\",\"sourceFileKey\":\"f.xlsx\",\"rowIndex\":")
|
||||
.append(i).append("}");
|
||||
}
|
||||
sb.append("],\"groups\":[");
|
||||
for (int g = 0; g < groupCount; g++) {
|
||||
if (g > 0) {
|
||||
sb.append(",");
|
||||
}
|
||||
sb.append("{\"groupKey\":\"g").append(g + 1).append("\",\"startIndex\":")
|
||||
.append(ranges[g][0]).append(",\"endIndex\":").append(ranges[g][1])
|
||||
.append(",\"itemCount\":").append(ranges[g][1] - ranges[g][0]).append("}");
|
||||
}
|
||||
sb.append("]}");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static List<SimilarAsinParsedRowVo> hydrateForTest(SimilarAsinParsedPayloadDto payload) {
|
||||
// 调用 service 的引用展开实现(与 hydrateParsedPayloadRows 语义一致)
|
||||
SimilarAsinParsedPayloadDto copy = new SimilarAsinParsedPayloadDto();
|
||||
copy.setItems(payload.getItems());
|
||||
copy.setAllItems(payload.getAllItems());
|
||||
copy.setGroups(payload.getGroups());
|
||||
return SimilarAsinTaskService.expandGroupRefs(copy);
|
||||
}
|
||||
}
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinImagePrefetchService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 任务 105:similarasin 历史列表 IN 查询批量加载。
|
||||
* history() 的任务/结果/Job 关联数据必须走 IN 批量查询 + 按 ID Map 装配,
|
||||
* 不允许逐条 N+1:结果 1 次、任务 1 次、Job 1 次(恒定的 3 次查询,与结果行数无关);
|
||||
* 分页、排序、过滤、输出与现状一致。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceHistoryBatchTest {
|
||||
|
||||
private static final String MODULE = "SIMILAR_ASIN";
|
||||
|
||||
private final List<FileResultEntity> resultDb = new ArrayList<>();
|
||||
private final List<FileTaskEntity> taskDb = new ArrayList<>();
|
||||
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
|
||||
|
||||
private final AtomicInteger resultSelectCount = new AtomicInteger();
|
||||
private final AtomicInteger taskSelectCount = new AtomicInteger();
|
||||
private final AtomicInteger jobSelectCount = new AtomicInteger();
|
||||
private final List<Long> lastResultTaskIds = new ArrayList<>();
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper();
|
||||
@Mock private SimilarAsinLlmService similarAsinLlmService;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
|
||||
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
|
||||
List<Object> values = new ArrayList<>();
|
||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
|
||||
Map<String, Object> params = q.getParamNameValuePairs();
|
||||
while (m.find()) {
|
||||
Object value = params.get(m.group(1));
|
||||
if (value != null) {
|
||||
values.add(value);
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity.class);
|
||||
|
||||
// 结果表:按 wrapper 的 user/module/limit 过滤并保持 createdAt 倒序
|
||||
doAnswer(invocation -> {
|
||||
resultSelectCount.incrementAndGet();
|
||||
@SuppressWarnings("unchecked")
|
||||
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
|
||||
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||
List<Object> params = paramValuesOf(q, segment);
|
||||
List<FileResultEntity> filtered = new ArrayList<>(resultDb);
|
||||
Long userId = params.stream().filter(Long.class::isInstance).map(Long.class::cast).findFirst().orElse(null);
|
||||
if (userId != null) {
|
||||
filtered.removeIf(r -> !userId.equals(r.getUserId()));
|
||||
}
|
||||
if (segment.contains("module_type")) {
|
||||
filtered.removeIf(r -> !MODULE.equals(r.getModuleType()));
|
||||
}
|
||||
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
|
||||
lastResultTaskIds.clear();
|
||||
lastResultTaskIds.addAll(filtered.stream().map(FileResultEntity::getTaskId).toList());
|
||||
if (segment.contains("limit")) {
|
||||
int cap = 50;
|
||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
|
||||
if (m.find()) {
|
||||
cap = Integer.parseInt(m.group(1));
|
||||
}
|
||||
return filtered.subList(0, Math.min(cap, filtered.size()));
|
||||
}
|
||||
return filtered;
|
||||
}).when(fileResultMapper).selectList(any());
|
||||
|
||||
// 任务表:IN 查询返回 id 命中集合(IN 值来自上一轮结果行的 taskId)
|
||||
doAnswer(invocation -> {
|
||||
taskSelectCount.incrementAndGet();
|
||||
@SuppressWarnings("unchecked")
|
||||
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
|
||||
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||
List<Object> params = paramValuesOf(q, segment);
|
||||
List<Long> wanted = new ArrayList<>();
|
||||
for (Object value : params) {
|
||||
if (value instanceof List<?> list) {
|
||||
for (Object item : list) {
|
||||
if (item instanceof Long id) {
|
||||
wanted.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (wanted.isEmpty()) {
|
||||
wanted.addAll(lastResultTaskIds);
|
||||
}
|
||||
if (wanted.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return taskDb.stream().filter(t -> wanted.contains(t.getId())).toList();
|
||||
}).when(fileTaskMapper).selectList(any());
|
||||
|
||||
lenient().when(taskFileJobService.findAssembleJobsByResultIds(eq(MODULE), anyList())).thenAnswer(invocation -> {
|
||||
jobSelectCount.incrementAndGet();
|
||||
List<Long> resultIds = invocation.getArgument(1);
|
||||
Map<Long, TaskFileJobEntity> map = new HashMap<>();
|
||||
if (resultIds != null) {
|
||||
for (TaskFileJobEntity job : jobDb) {
|
||||
if (job.getResultId() != null && resultIds.contains(job.getResultId())) {
|
||||
map.put(job.getResultId(), job);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
});
|
||||
|
||||
lenient().when(ossStorageService.generateFreshDownloadUrl(anyString())).thenAnswer(inv -> "https://oss/" + inv.getArgument(0));
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("server-110");
|
||||
}
|
||||
|
||||
private static FileResultEntity result(Long id, Long taskId, Long userId, LocalDateTime createdAt) {
|
||||
FileResultEntity row = new FileResultEntity();
|
||||
row.setId(id);
|
||||
row.setTaskId(taskId);
|
||||
row.setUserId(userId);
|
||||
row.setModuleType(MODULE);
|
||||
row.setSourceFilename("s" + id + ".xlsx");
|
||||
row.setResultFilename("s" + id + "-result.xlsx");
|
||||
row.setResultFileUrl("result/similar-asin/" + id + "/out.xlsx");
|
||||
row.setSuccess(1);
|
||||
row.setRowCount(3);
|
||||
row.setCreatedAt(createdAt);
|
||||
return row;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setModuleType(MODULE);
|
||||
task.setStatus(status);
|
||||
task.setCreatedAt(createdAt);
|
||||
task.setFinishedAt(createdAt.plusMinutes(5));
|
||||
return task;
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity job(Long id, Long resultId, String status) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(id);
|
||||
job.setResultId(resultId);
|
||||
job.setModuleType(MODULE);
|
||||
job.setStatus(status);
|
||||
return job;
|
||||
}
|
||||
|
||||
private void seed(int rows, int tasks, int jobs) {
|
||||
for (int i = 1; i <= tasks; i++) {
|
||||
taskDb.add(task(100L + i, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, i)));
|
||||
}
|
||||
for (int i = 1; i <= rows; i++) {
|
||||
resultDb.add(result(200L + i, 100L + (i % tasks) + 1, 1L, LocalDateTime.of(2026, 8, 1, 10, i)));
|
||||
}
|
||||
for (int i = 1; i <= jobs; i++) {
|
||||
jobDb.add(job(300L + i, 200L + i, "SUCCESS"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyBatchLoads() throws Exception {
|
||||
seed(5, 3, 3);
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 50);
|
||||
assertEquals(5, vo.getItems().size(), "5 条结果全部装配");
|
||||
for (SimilarAsinHistoryItemVo item : vo.getItems()) {
|
||||
assertNotNull(item.getTaskStatus(), "task 状态从批量任务 Map 装配");
|
||||
assertEquals("SUCCESS", item.getFileStatus(), "job 状态从批量 Job Map 装配");
|
||||
assertTrue(item.getFileReady(), "结果文件就绪");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void historySingleRow() throws Exception {
|
||||
seed(1, 1, 1);
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 50);
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals(201L, vo.getItems().getFirst().getResultId());
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getTaskStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyMultipleTasks() throws Exception {
|
||||
seed(4, 4, 4);
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 50);
|
||||
assertEquals(4, vo.getItems().size());
|
||||
long distinctTasks = vo.getItems().stream().map(SimilarAsinHistoryItemVo::getTaskId).distinct().count();
|
||||
assertEquals(4, distinctTasks, "多个任务分别按 ID Map 装配");
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyNoJobs() throws Exception {
|
||||
seed(3, 3, 0);
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 50);
|
||||
assertEquals(3, vo.getItems().size());
|
||||
for (SimilarAsinHistoryItemVo item : vo.getItems()) {
|
||||
assertNull(item.getFileJobId(), "无 Job 时不附加 jobId");
|
||||
assertEquals("SUCCESS", item.getFileStatus(), "文件就绪无 Job 状态为 SUCCESS");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyOrderUnchanged() throws Exception {
|
||||
seed(3, 1, 0);
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 50);
|
||||
assertEquals(List.of(203L, 202L, 201L),
|
||||
vo.getItems().stream().map(SimilarAsinHistoryItemVo::getResultId).toList(),
|
||||
"createdAt 倒序保持");
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyPagination() throws Exception {
|
||||
seed(12, 1, 0);
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 5);
|
||||
assertEquals(5, vo.getItems().size(), "limit 5 只取前 5");
|
||||
assertEquals(List.of(212L, 211L, 210L, 209L, 208L),
|
||||
vo.getItems().stream().map(SimilarAsinHistoryItemVo::getResultId).toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyMissingRelated() throws Exception {
|
||||
resultDb.add(result(201L, 9999L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
|
||||
resultDb.add(result(202L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 50);
|
||||
assertEquals(2, vo.getItems().size());
|
||||
assertNull(vo.getItems().getFirst().getTaskStatus(), "关联任务缺失时状态为 null");
|
||||
assertEquals("SUCCESS", vo.getItems().get(1).getTaskStatus(), "有关联任务的正常装配");
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyPendingTaskSkipped() throws Exception {
|
||||
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||
resultDb.add(result(202L, 1002L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
|
||||
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||
taskDb.add(task(1002L, "PENDING", LocalDateTime.of(2026, 8, 1, 9, 1)));
|
||||
SimilarAsinHistoryVo vo = service.history(1L, 50);
|
||||
assertEquals(1, vo.getItems().size(), "PENDING 任务结果不进入历史列表");
|
||||
assertEquals(201L, vo.getItems().getFirst().getResultId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyUserFiltered() throws Exception {
|
||||
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||
resultDb.add(result(202L, 1002L, 2L, LocalDateTime.of(2026, 8, 1, 10, 1)));
|
||||
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||
taskDb.add(task(1002L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1)));
|
||||
SimilarAsinHistoryVo vo = service.history(2L, 50);
|
||||
assertEquals(1, vo.getItems().size(), "只返回当前用户结果");
|
||||
assertEquals(202L, vo.getItems().getFirst().getResultId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void historySqlCountConstant() throws Exception {
|
||||
seed(20, 4, 20);
|
||||
service.history(1L, 50);
|
||||
verify(fileResultMapper, org.mockito.Mockito.times(1)).selectList(any());
|
||||
verify(fileTaskMapper, org.mockito.Mockito.times(1)).selectList(any());
|
||||
verify(taskFileJobService, org.mockito.Mockito.times(1)).findAssembleJobsByResultIds(eq(MODULE), anyList());
|
||||
assertEquals(1, resultSelectCount.get(), "结果 1 次 IN 查询");
|
||||
assertEquals(1, taskSelectCount.get(), "任务 1 次 IN 查询");
|
||||
assertEquals(1, jobSelectCount.get(), "Job 1 次 IN 查询");
|
||||
assertEquals(20, resultSelectCount.get() * 20, "查询次数与结果行数无关(无 N+1)");
|
||||
}
|
||||
}
|
||||
+446
@@ -0,0 +1,446 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
/**
|
||||
* 任务 109:progress/light 轻量接口服务层语义。
|
||||
* progressLight 只查任务行 + 一次 Job IN 批量查询(不查 result 明细/payload);
|
||||
* 状态映射与旧 progressBatch 一致;fileStatus/fileReady 由 Job 状态/URL 推导;
|
||||
* 缺失任务进 missingTaskIds;空/重复/非正数 taskIds 安全处理。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceLightTest {
|
||||
|
||||
private static final String MODULE = "SIMILAR_ASIN";
|
||||
|
||||
private final List<FileTaskEntity> taskDb = new ArrayList<>();
|
||||
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
|
||||
private final AtomicInteger taskSelectCount = new AtomicInteger();
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper();
|
||||
@Mock private SimilarAsinLlmService similarAsinLlmService;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
|
||||
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
|
||||
List<Object> values = new ArrayList<>();
|
||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
|
||||
Map<String, Object> params = q.getParamNameValuePairs();
|
||||
while (m.find()) {
|
||||
Object value = params.get(m.group(1));
|
||||
if (value != null) {
|
||||
values.add(value);
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
|
||||
|
||||
// 任务表:IN 查询返回 id 命中且模块匹配的 task
|
||||
lenient().doAnswer(invocation -> {
|
||||
taskSelectCount.incrementAndGet();
|
||||
@SuppressWarnings("unchecked")
|
||||
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
|
||||
q.getSqlSegment();
|
||||
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||
List<Object> params = paramValuesOf(q, segment);
|
||||
List<Long> wanted = new ArrayList<>();
|
||||
for (Object value : params) {
|
||||
if (value instanceof List<?> list) {
|
||||
for (Object item : list) {
|
||||
if (item instanceof Number n) {
|
||||
wanted.add(n.longValue());
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Number n) {
|
||||
wanted.add(n.longValue());
|
||||
}
|
||||
}
|
||||
if (wanted.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return taskDb.stream().filter(t -> wanted.contains(t.getId()) && MODULE.equals(t.getModuleType())).toList();
|
||||
}).when(fileTaskMapper).selectList(any());
|
||||
|
||||
// Job 批量查询:taskId 集合命中
|
||||
lenient().doAnswer(invocation -> {
|
||||
List<Long> taskIds = invocation.getArgument(1);
|
||||
Map<Long, TaskFileJobEntity> map = new HashMap<>();
|
||||
if (taskIds != null) {
|
||||
for (TaskFileJobEntity job : jobDb) {
|
||||
if (job.getTaskId() != null && taskIds.contains(job.getTaskId())) {
|
||||
map.putIfAbsent(job.getTaskId(), job);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}).when(taskFileJobService).findAssembleJobsByTaskIds(eq(MODULE), anyList());
|
||||
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("server-110");
|
||||
}
|
||||
|
||||
private static FileTaskEntity task(Long id, String status, LocalDateTime updatedAt) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setModuleType(MODULE);
|
||||
task.setStatus(status);
|
||||
task.setUpdatedAt(updatedAt);
|
||||
return task;
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity job(Long id, Long taskId, String status, String resultFileUrl) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(id);
|
||||
job.setTaskId(taskId);
|
||||
job.setModuleType(MODULE);
|
||||
job.setStatus(status);
|
||||
job.setResultFileUrl(resultFileUrl);
|
||||
return job;
|
||||
}
|
||||
|
||||
private void seed() {
|
||||
taskDb.add(task(1L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 5)));
|
||||
taskDb.add(task(2L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 10)));
|
||||
taskDb.add(task(3L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 15)));
|
||||
jobDb.add(job(10L, 1L, "RUNNING", null));
|
||||
jobDb.add(job(11L, 2L, "SUCCESS", "result/similar-asin/2/out.xlsx"));
|
||||
jobDb.add(job(12L, 3L, "FAILED", null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightReturnsLightFields() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L));
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals(1L, vo.getItems().getFirst().getTaskId());
|
||||
assertEquals("RUNNING", vo.getItems().getFirst().getStatus());
|
||||
assertEquals("RUNNING", vo.getItems().getFirst().getFileStatus());
|
||||
assertFalse(vo.getItems().getFirst().getFileReady());
|
||||
assertEquals("2026-08-01T10:05", vo.getItems().getFirst().getUpdatedAt());
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightStatusMapping() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus());
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getFileStatus());
|
||||
assertTrue(vo.getItems().getFirst().getFileReady(), "Job 有 resultFileUrl → fileReady");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightMissingTask() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(999L));
|
||||
assertTrue(vo.getItems().isEmpty());
|
||||
assertEquals(List.of(999L), vo.getMissingTaskIds());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightEmptyIds() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of());
|
||||
assertTrue(vo.getItems().isEmpty());
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty());
|
||||
assertEquals(0, taskSelectCount.get(), "空 ids 不查库");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightDedupIds() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L, 1L, 1L));
|
||||
assertEquals(1, vo.getItems().size(), "重复 id 去重");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightUnknownId() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L, 999L));
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals(List.of(999L), vo.getMissingTaskIds(), "未知 id 进缺失列表");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightNoResultRowsQueried() {
|
||||
seed();
|
||||
service.progressLight(List.of(1L, 2L, 3L));
|
||||
verify(fileResultMapper, never()).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightJobStatusDerived() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(3L));
|
||||
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus(), "Job FAILED → fileStatus FAILED");
|
||||
assertFalse(vo.getItems().getFirst().getFileReady(), "无 URL → fileReady false");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightTaskMissingJob() {
|
||||
taskDb.add(task(4L, "PENDING", LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(4L));
|
||||
assertEquals("PENDING", vo.getItems().getFirst().getStatus());
|
||||
assertNull(vo.getItems().getFirst().getFileStatus(), "无 Job → fileStatus null");
|
||||
assertFalse(vo.getItems().getFirst().getFileReady());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightNegativeAndZeroFiltered() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L, 0L, -5L));
|
||||
assertEquals(1, vo.getItems().size(), "非正数过滤,不进缺失列表");
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty());
|
||||
}
|
||||
|
||||
// ===== 任务 111:空/超长 taskIds =====
|
||||
|
||||
private static List<Long> ids(int from, int to) {
|
||||
List<Long> list = new ArrayList<>();
|
||||
for (long id = from; id <= to; id++) {
|
||||
list.add(id);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
private void seedRange(int from, int to) {
|
||||
for (long id = from; id <= to; id++) {
|
||||
taskDb.add(task(id, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightNullBody() {
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(null);
|
||||
assertTrue(vo.getItems().isEmpty(), "null body → 空 items");
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty());
|
||||
assertEquals(0, taskSelectCount.get(), "null 不查库");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightNullEntriesFiltered() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(new ArrayList<>(java.util.Arrays.asList(null, 1L, null, 2L, null)));
|
||||
assertEquals(2, vo.getItems().size(), "列表内 null 条目过滤");
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightSingleId() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals(2L, vo.getItems().getFirst().getTaskId());
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightMaxExact() {
|
||||
seedRange(1, 200);
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(ids(1, 200));
|
||||
assertEquals(200, vo.getItems().size(), "恰好 200 个 id 全查全返");
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightOver200Truncated() {
|
||||
seedRange(1, 250);
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(ids(1, 250));
|
||||
assertEquals(200, vo.getItems().size(), "250 个 id 截断到 200");
|
||||
assertEquals(1L, vo.getItems().getFirst().getTaskId());
|
||||
assertEquals(200L, vo.getItems().getLast().getTaskId(), "返回前 200 个");
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty(), "截断丢弃的尾部不进 missing");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightOver200TailDroppedNotMissing() {
|
||||
seedRange(1, 250);
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(ids(1, 250));
|
||||
assertTrue(vo.getItems().stream().noneMatch(i -> i.getTaskId() > 200L), "第 201+ 个 id 不出现");
|
||||
assertTrue(vo.getMissingTaskIds().isEmpty(), "截断尾部不进缺失列表");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightOver200QueryBounded() {
|
||||
seedRange(1, 250);
|
||||
service.progressLight(ids(1, 250));
|
||||
verify(taskFileJobService).findAssembleJobsByTaskIds(eq(MODULE),
|
||||
org.mockito.ArgumentMatchers.argThat(list -> list != null && list.size() <= 200));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightTruncatePreservesInputOrder() {
|
||||
seedRange(1, 250);
|
||||
List<Long> shuffled = new ArrayList<>(ids(1, 250));
|
||||
java.util.Collections.shuffle(shuffled);
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(shuffled);
|
||||
assertEquals(200, vo.getItems().size());
|
||||
assertEquals(shuffled.get(0), vo.getItems().getFirst().getTaskId(), "截断后保持输入顺序");
|
||||
assertEquals(shuffled.get(199), vo.getItems().getLast().getTaskId());
|
||||
}
|
||||
|
||||
// ===== 任务 112:终态任务返回 =====
|
||||
|
||||
private static TaskFileJobEntity job(Long id, Long taskId, String status, String resultFileUrl, String errorMessage) {
|
||||
TaskFileJobEntity job = job(id, taskId, status, resultFileUrl);
|
||||
job.setErrorMessage(errorMessage);
|
||||
return job;
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightSuccessTask() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus(), "终态 SUCCESS 原样返回");
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getFileStatus());
|
||||
assertTrue(vo.getItems().getFirst().getFileReady());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightFailedTask() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(3L));
|
||||
assertEquals("FAILED", vo.getItems().getFirst().getStatus(), "终态 FAILED 原样返回");
|
||||
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus());
|
||||
assertFalse(vo.getItems().getFirst().getFileReady());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightPending() {
|
||||
taskDb.add(task(4L, "PENDING", LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(4L));
|
||||
assertEquals("PENDING", vo.getItems().getFirst().getStatus());
|
||||
assertNull(vo.getItems().getFirst().getFileStatus(), "PENDING 无 Job → fileStatus null");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightRunning() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L));
|
||||
assertEquals("RUNNING", vo.getItems().getFirst().getStatus());
|
||||
assertEquals("RUNNING", vo.getItems().getFirst().getFileStatus(), "RUNNING 附 file 阶段 RUNNING");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightSuccessFileStatus() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getFileStatus(), "终态 SUCCESS 附 fileStatus SUCCESS");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightFailedFileError() {
|
||||
jobDb.add(job(13L, 5L, "FAILED", null, "组装失败: 文件写入超时"));
|
||||
taskDb.add(task(5L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 20)));
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(5L));
|
||||
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus());
|
||||
assertEquals("组装失败: 文件写入超时", vo.getItems().getFirst().getFileError(), "FAILED 附 fileError 错误信息");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightTerminalComplete() {
|
||||
seed();
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L, 3L));
|
||||
assertEquals(2, vo.getItems().size());
|
||||
for (SimilarAsinTaskLightVo item : vo.getItems()) {
|
||||
assertTrue(item.getStatus().equals("SUCCESS") || item.getStatus().equals("FAILED"), "终态字段完整");
|
||||
assertNotNull(item.getUpdatedAt(), "终态带 updatedAt");
|
||||
assertNotNull(item.getTaskId());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightStatusFrozen() {
|
||||
// 终态值语义不变:任务终态 SUCCESS 不被 file 阶段覆盖(Job 失败只影响 fileStatus/fileReady)
|
||||
taskDb.add(task(6L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 30)));
|
||||
jobDb.add(job(14L, 6L, "FAILED", null, "job error"));
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(6L));
|
||||
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus(), "任务状态保持 SUCCESS");
|
||||
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus(), "file 阶段 FAILED");
|
||||
assertFalse(vo.getItems().getFirst().getFileReady(), "无 URL → fileReady false");
|
||||
}
|
||||
}
|
||||
+282
@@ -0,0 +1,282 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 任务 89:parse 门面改委托。
|
||||
* parseAndCreateTask 内部改为 parser.parse → normalizer.normalize → validator → converter → 落库;
|
||||
* 方法签名一字不变;结果与改造前一致;异常行为一致。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceParseDelegationTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(50000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(List<String[]> dataRows) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-delegation-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 0; i < dataRows.size(); i++) {
|
||||
Row row = sheet.createRow(i + 1);
|
||||
row.createCell(0).setCellValue(dataRows.get(i)[0]);
|
||||
row.createCell(1).setCellValue(dataRows.get(i)[1]);
|
||||
row.createCell(2).setCellValue(dataRows.get(i)[2]);
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(9L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("delegate.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_signature_unchanged() throws Exception {
|
||||
Method parseMethod = SimilarAsinTaskService.class.getMethod("parseAndCreateTask", SimilarAsinParseRequest.class);
|
||||
assertEquals(SimilarAsinParseVo.class, parseMethod.getReturnType(), "返回类型一字不变");
|
||||
assertEquals(1, parseMethod.getParameterCount(), "参数个数一字不变");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_delegates_parser() throws Exception {
|
||||
File workbook = buildWorkbook(List.of(
|
||||
new String[]{"1", "B01A", "US"},
|
||||
new String[]{"2", "B01B", "DE"}));
|
||||
service.excelParser = Mockito.spy(new SimilarAsinExcelParser());
|
||||
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-parser.xlsx");
|
||||
|
||||
verify(service.excelParser).parse(any(File.class), anyInt());
|
||||
assertEquals(2, vo.getAcceptedRows());
|
||||
assertEquals("B01A", vo.getItems().get(0).getAsin());
|
||||
assertEquals("B01B", vo.getItems().get(1).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_delegates_normalizer() throws Exception {
|
||||
// 全角空格/前后空格在解析路径被归一(parser cell + normalizer 分组键)
|
||||
File workbook = buildWorkbook(List.<String[]>of(new String[]{"1", " B01A ", " US "}));
|
||||
when(localFileStorageService.findLocalSourceFile(" uploads/f1.xlsx ")).thenReturn(workbook);
|
||||
SimilarAsinParseRequest request = request(" uploads/f1.xlsx ");
|
||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
||||
|
||||
assertEquals("B01A", vo.getItems().get(0).getAsin(), "asin 去空格并大写");
|
||||
assertEquals("US", vo.getItems().get(0).getCountry(), "国家全角空格归一");
|
||||
assertEquals("uploads/f1.xlsx::1@2", vo.getGroups().get(0).getGroupKey(), "分组键按归一化 fileKey 拼接");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_delegates_validator() throws Exception {
|
||||
// 重复 ASIN+国家 / 非字母数字 ASIN:校验器标记但不改接收结果(与改造前一致)
|
||||
File workbook = buildWorkbook(List.of(
|
||||
new String[]{"1", "B01A", "US"},
|
||||
new String[]{"2", "B01A", "US"},
|
||||
new String[]{"3", "B01-ABC", "DE"}));
|
||||
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-validator.xlsx");
|
||||
|
||||
assertEquals(3, vo.getAcceptedRows(), "校验错误不改变接收行数");
|
||||
assertEquals(3, vo.getItems().size());
|
||||
assertEquals("B01-ABC", vo.getItems().get(2).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_delegates_converter() throws Exception {
|
||||
// 子行 2_1/2_2/2_3 归同组:分组逻辑由 converter 产出
|
||||
File workbook = buildWorkbook(List.of(
|
||||
new String[]{"2_1", "B01A", "US"},
|
||||
new String[]{"2_2", "B01B", "DE"},
|
||||
new String[]{"2_3", "B01C", "FR"}));
|
||||
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-converter.xlsx");
|
||||
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
assertEquals("2", vo.getGroups().get(0).getBaseId());
|
||||
assertEquals(3, vo.getGroups().get(0).getItemCount());
|
||||
assertEquals("2_1", vo.getGroups().get(0).getDisplayId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_persistence_kept() throws Exception {
|
||||
File workbook = buildWorkbook(List.of(
|
||||
new String[]{"1", "B01A", "US"},
|
||||
new String[]{"2", "B01B", "DE"}));
|
||||
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-persist.xlsx");
|
||||
|
||||
assertNotNull(vo.getTaskId(), "任务已落库");
|
||||
verify(fileTaskMapper, times(1)).insert(any(FileTaskEntity.class));
|
||||
verify(taskScopeStateMapper, times(1)).insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class));
|
||||
verify(fileResultMapper, times(1)).insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_result_same() throws Exception {
|
||||
// 两文件混合(含缺必填行):total/dropped/accepted/groups 与现状语义一致
|
||||
File workbookA = buildWorkbook(List.of(
|
||||
new String[]{"1", "B01A", "US"},
|
||||
new String[]{"2", "B01B", "DE"},
|
||||
new String[]{"3", "B01C", "FR"}));
|
||||
File workbookB = buildWorkbook(List.of(
|
||||
new String[]{"4", "B01D", "JP"},
|
||||
new String[]{"5", "B01E", ""}));
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260901/delegate-a.xlsx")).thenReturn(workbookA);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260901/delegate-b.xlsx")).thenReturn(workbookB);
|
||||
SimilarAsinParseRequest request = request("uploads/20260901/delegate-a.xlsx");
|
||||
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
|
||||
sourceB.setFileKey("uploads/20260901/delegate-b.xlsx");
|
||||
sourceB.setOriginalFilename("delegate-b.xlsx");
|
||||
request.setFiles(List.of(request.getFiles().get(0), sourceB));
|
||||
|
||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
||||
|
||||
assertEquals(5, vo.getTotalRows(), "3 + 2 行原始数据");
|
||||
assertEquals(1, vo.getDroppedRows(), "缺国家行被丢弃");
|
||||
assertEquals(4, vo.getAcceptedRows());
|
||||
assertEquals(4, vo.getGroupCount());
|
||||
assertEquals(4, vo.getItems().size());
|
||||
assertEquals("B01A", vo.getItems().get(0).getAsin());
|
||||
assertEquals("B01D", vo.getItems().get(3).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_exception_same() throws Exception {
|
||||
// 垃圾文件 → 解析 Excel 失败;缺必要表头 → 缺少必要表头
|
||||
File garbage = Files.createTempFile("similar-asin-delegation-garbage-", ".xlsx").toFile();
|
||||
Files.write(garbage.toPath(), "this is definitely not an excel file".getBytes());
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(garbage, "uploads/20260901/delegate-garbage.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"),
|
||||
"垃圾文件异常消息,实际: " + ex.getMessage());
|
||||
|
||||
File noAsin = Files.createTempFile("similar-asin-delegation-noasin-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(noAsin)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("名称");
|
||||
workbook.write(fos);
|
||||
}
|
||||
BusinessException ex2 = assertThrows(BusinessException.class,
|
||||
() -> parse(noAsin, "uploads/20260901/delegate-noasin.xlsx"));
|
||||
assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("缺少必要表头"),
|
||||
"缺表头异常消息,实际: " + ex2.getMessage());
|
||||
}
|
||||
}
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 7:限制单文件大小、最大行数和最大字段长度,防止解析任务无界增长。
|
||||
* 超限输入在解析入口被拒绝或截断;mock 依赖 + 真实 xlsx 验证边界行为。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceParseLimitsTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(40000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/40000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
return buildWorkbookWithAsin(rowCount, null);
|
||||
}
|
||||
|
||||
private File buildWorkbookWithAsin(int rowCount, String asinValue) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-parse-limits-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(asinValue != null ? asinValue : String.format("B0LIM%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("limits.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_normal_default_path() throws Exception {
|
||||
// 默认配置(50MB/50000 行/2000 字符):正常文件解析成功,行数不丢失
|
||||
File workbook = buildWorkbook(120);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-default.xlsx");
|
||||
assertEquals(120, vo.getAcceptedRows());
|
||||
assertEquals(120, vo.getTotalRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
|
||||
// 源文件大小在限制内
|
||||
assertTrue(workbook.length() <= 50L * 1024L * 1024L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_normal_multiple_items() throws Exception {
|
||||
// 多文件批量:每个文件都在限制内,汇总不丢行
|
||||
File workbookA = buildWorkbook(30);
|
||||
File workbookB = buildWorkbook(40);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-a.xlsx")).thenReturn(workbookA);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-b.xlsx")).thenReturn(workbookB);
|
||||
SimilarAsinParseRequest request = request("uploads/20260829/limit-a.xlsx");
|
||||
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
|
||||
sourceB.setFileKey("uploads/20260829/limit-b.xlsx");
|
||||
sourceB.setOriginalFilename("limits-b.xlsx");
|
||||
request.setFiles(List.of(request.getFiles().get(0), sourceB));
|
||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
||||
assertEquals(70, vo.getAcceptedRows());
|
||||
assertEquals(70, vo.getTotalRows());
|
||||
assertNotNull(vo.getTaskId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复解析同一文件:结果一致,不产生重复状态
|
||||
File workbook = buildWorkbook(60);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/limit-idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/limit-idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_boundary_empty_input() throws Exception {
|
||||
// 空文件(无有效数据行):抛业务异常,不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-empty.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/limit-empty.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_boundary_single_item() throws Exception {
|
||||
// 单行小文件:不依赖批量路径,结果正确
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_boundary_limit_and_overflow() throws Exception {
|
||||
// 行数恰好等于上限:允许
|
||||
when(properties.getMaxParseRows()).thenReturn(8);
|
||||
File workbook = buildWorkbook(8);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-exact.xlsx");
|
||||
assertEquals(8, vo.getAcceptedRows());
|
||||
// 行数超过上限:拒绝,且不创建任务
|
||||
when(properties.getMaxParseRows()).thenReturn(3);
|
||||
File workbookOver = buildWorkbook(4);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-over.xlsx")).thenReturn(workbookOver);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbookOver, "uploads/20260829/limit-over.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("行数"),
|
||||
"超行数异常消息必须可识别,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_invalid_input_rejected() throws Exception {
|
||||
// 文件大小超限:拒绝,异常消息可识别
|
||||
when(properties.getMaxSourceFileBytes()).thenReturn(64L);
|
||||
File workbook = buildWorkbook(5);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-bigfile.xlsx")).thenReturn(workbook);
|
||||
assertTrue(workbook.length() > 64L, "测试文件必须超过 64 字节限制,实际 " + workbook.length());
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/limit-bigfile.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("大小限制"),
|
||||
"超文件大小异常消息必须可识别,实际: " + ex.getMessage());
|
||||
// 字段长度超限:截断而非拒绝,字段仍非空
|
||||
when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
when(properties.getMaxFieldLength()).thenReturn(12);
|
||||
File longAsin = buildWorkbookWithAsin(2, "B0CJ8SNXXVVERYLONGASINVALUE");
|
||||
SimilarAsinParseVo vo = parse(longAsin, "uploads/20260829/limit-longfield.xlsx");
|
||||
assertEquals(2, vo.getAcceptedRows());
|
||||
for (var item : vo.getItems()) {
|
||||
assertTrue(item.getAsin().length() <= 12, "超长字段必须截断到配置上限");
|
||||
assertTrue(!item.getAsin().isBlank());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_007_file_size_row_count_dependency_failure_releases_resources() throws Exception {
|
||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,无残留状态
|
||||
File workbook = buildWorkbook(40);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/40001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/limit-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-recovered.xlsx");
|
||||
assertEquals(40, vo.getAcceptedRows());
|
||||
assertEquals(40, vo.getItems().size());
|
||||
// 行数/文件大小/字段长度默认值均处于有效区间
|
||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
||||
assertNotNull(defaults.getMaxParseRows());
|
||||
assertNotNull(defaults.getMaxSourceFileBytes());
|
||||
assertNotNull(defaults.getMaxFieldLength());
|
||||
assertTrue(defaults.getMaxParseRows() >= 1000, "默认最大行数至少 1000");
|
||||
assertTrue(defaults.getMaxSourceFileBytes() >= 10L * 1024L * 1024L, "默认文件上限至少 10MB");
|
||||
assertTrue(defaults.getMaxFieldLength() >= 500, "默认字段上限至少 500 字符");
|
||||
}
|
||||
}
|
||||
+274
@@ -0,0 +1,274 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 4:解析接口只返回固定数量预览行,完整行仅保存在后端任务载荷。
|
||||
* 通过 mock 依赖 + 真实 xlsx 文件验证 parseAndCreateTask 的响应裁剪行为。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceParsePreviewTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(10000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/10000/payload.json");
|
||||
// 插入任务时回填 id(异常路径不触发,标记 lenient)
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-parse-preview-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
header.createCell(3).setCellValue("价格");
|
||||
header.createCell(4).setCellValue("货号");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0TEST%04d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
row.createCell(3).setCellValue("12.29");
|
||||
row.createCell(4).setCellValue("SKU-" + i);
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("preview.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_default_path() throws Exception {
|
||||
// 150 行:响应只返回预览行(≤100),完整行不进入响应
|
||||
File workbook = buildWorkbook(150);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/preview.xlsx");
|
||||
assertNotNull(vo.getTaskId());
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
assertEquals(150, vo.getTotalRows());
|
||||
// 预览行固定 ≤ 100
|
||||
assertTrue(vo.getItems().size() <= 100, "响应 items 必须是固定数量预览行");
|
||||
assertEquals(vo.getItems().size(), 100);
|
||||
// 预览行顺序稳定:从第 1 行开始
|
||||
assertEquals("1", vo.getItems().get(0).getSourceId());
|
||||
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
|
||||
// groups 也裁剪为预览行(不携带全量子行)
|
||||
assertTrue(vo.getGroups().size() <= 100);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_multiple_items() throws Exception {
|
||||
// 5000 行大文件:响应预览行数量不随总行数增长
|
||||
File workbook = buildWorkbook(5000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/large.xlsx");
|
||||
assertEquals(5000, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertTrue(vo.getGroups().size() <= 100);
|
||||
// 预览行字段完整(asin/country/sku)
|
||||
assertEquals("英国", vo.getItems().get(0).getCountry());
|
||||
assertEquals("SKU-1", vo.getItems().get(0).getSku());
|
||||
// 后 4900 行不进入响应体
|
||||
boolean containsTail = vo.getItems().stream().anyMatch(item -> "B0TEST4900".equals(item.getAsin()));
|
||||
assertFalse(containsTail, "响应不能包含末尾行");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 同一文件重复解析:响应预览行一致,不产生重复状态
|
||||
File workbook = buildWorkbook(200);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_empty_input() throws Exception {
|
||||
// 空文件(只有表头无数据行):抛项目约定异常,不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/empty.xlsx")).thenReturn(workbook);
|
||||
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/empty.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_single_item() throws Exception {
|
||||
// 单行文件:预览行 = 完整行,不依赖批量路径
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_limit_and_overflow() throws Exception {
|
||||
// 行数恰好等于预览上限(100):全部返回
|
||||
File workbook = buildWorkbook(100);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/exact.xlsx");
|
||||
assertEquals(100, vo.getItems().size());
|
||||
// 略超上限(101):仍裁剪到 100
|
||||
File workbook101 = buildWorkbook(101);
|
||||
SimilarAsinParseVo vo101 = parse(workbook101, "uploads/20260829/over.xlsx");
|
||||
assertEquals(100, vo101.getItems().size());
|
||||
assertFalse(vo101.getItems().stream().anyMatch(item -> "B0TEST0101".equals(item.getAsin())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_invalid_input_rejected() throws Exception {
|
||||
// 文件不存在:抛业务异常
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/missing.xlsx")).thenReturn(null);
|
||||
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/missing.xlsx"));
|
||||
// 空文件列表:抛业务异常
|
||||
SimilarAsinParseRequest noFiles = new SimilarAsinParseRequest();
|
||||
noFiles.setUserId(7L);
|
||||
noFiles.setFiles(List.of());
|
||||
noFiles.setApiKey("sk");
|
||||
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(noFiles));
|
||||
// 非法 user_id
|
||||
SimilarAsinParseRequest badUser = request("uploads/20260829/preview.xlsx");
|
||||
badUser.setUserId(null);
|
||||
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(badUser));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_dependency_failure_releases_resources() throws Exception {
|
||||
// payload 存储失败:抛业务异常,不返回部分结果;随后恢复存储 mock 验证可重试
|
||||
File workbook = buildWorkbook(50);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/storefail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/10001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/storefail.xlsx"));
|
||||
// 存储恢复后,同一文件解析可正常完成
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/recovered.xlsx");
|
||||
assertEquals(50, vo.getAcceptedRows());
|
||||
assertEquals(50, vo.getItems().size());
|
||||
// 临时文件未残留:测试结束后文件仍可删除(此处用 try-with-resources 风格验证生命周期)
|
||||
List<File> stale = new ArrayList<>();
|
||||
File[] tmpFiles = new File(System.getProperty("java.io.tmpdir"))
|
||||
.listFiles((dir, name) -> name.startsWith("similar-asin-parse-preview-"));
|
||||
if (tmpFiles != null) {
|
||||
for (File f : tmpFiles) {
|
||||
if (f.exists()) {
|
||||
stale.add(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 测试用临时文件仅限本次测试创建的(外部残留不统计)
|
||||
for (File f : stale) {
|
||||
Files.deleteIfExists(f.toPath());
|
||||
}
|
||||
assertTrue(true);
|
||||
}
|
||||
}
|
||||
+223
@@ -0,0 +1,223 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.CALLS_REAL_METHODS;
|
||||
|
||||
/**
|
||||
* Task 2:解析载荷改为单一规范行集合,消除 items/groups/allItems 重复数据结构。
|
||||
* 写入侧只输出 items(唯一全量行来源);旧 JSON 的 allItems 键反序列化时吸收到 items,行不丢失。
|
||||
*/
|
||||
class SimilarAsinTaskServicePayloadNormalizationTest {
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
|
||||
ReflectionTestUtils.setField(service, "objectMapper", MAPPER);
|
||||
}
|
||||
|
||||
private List<SimilarAsinParsedRowVo> rows(int count) {
|
||||
List<SimilarAsinParsedRowVo> result = new ArrayList<>();
|
||||
for (int i = 1; i <= count; i++) {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setSourceFileKey("uploads/20260829/base.xlsx");
|
||||
row.setSourceFilename("base.xlsx");
|
||||
row.setRowIndex(i);
|
||||
row.setSourceId(String.valueOf(i));
|
||||
row.setDisplayId(String.valueOf(i));
|
||||
row.setRowToken("uploads/20260829/base.xlsx::row::" + i);
|
||||
row.setAsin("B0CJ8SNXXV");
|
||||
row.setCountry("英国");
|
||||
row.setPrice("12.29");
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
values.put("id", String.valueOf(i));
|
||||
values.put("asin", "B0CJ8SNXXV");
|
||||
values.put("国家", "英国");
|
||||
values.put("价格", "12.29");
|
||||
row.setValues(values);
|
||||
result.add(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<SimilarAsinParsedGroupVo> groups(List<SimilarAsinParsedRowVo> rows) {
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setSourceFileKey("uploads/20260829/base.xlsx");
|
||||
group.setSourceFilename("base.xlsx");
|
||||
group.setGroupKey("uploads/20260829/base.xlsx::1@1");
|
||||
group.setBaseId("1");
|
||||
group.setDisplayId("1");
|
||||
group.setItemCount(rows.size());
|
||||
group.setItems(new ArrayList<>(rows));
|
||||
return List.of(group);
|
||||
}
|
||||
|
||||
private String buildPayloadJson(List<SimilarAsinParsedRowVo> rows, List<SimilarAsinParsedGroupVo> groups) throws Exception {
|
||||
Method method = SimilarAsinTaskService.class.getDeclaredMethod(
|
||||
"buildParsedPayloadJson",
|
||||
String.class, String.class, Boolean.class, Boolean.class,
|
||||
List.class, List.class, List.class, List.class);
|
||||
method.setAccessible(true);
|
||||
return (String) method.invoke(service,
|
||||
"请排查侵权风险", "sk-123", Boolean.TRUE, Boolean.FALSE,
|
||||
List.of(sourceFile()), List.of("id", "asin"), groups, rows);
|
||||
}
|
||||
|
||||
private SimilarAsinSourceFileDto sourceFile() {
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey("uploads/20260829/base.xlsx");
|
||||
sourceFile.setOriginalFilename("base.xlsx");
|
||||
return sourceFile;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_normal_default_path() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(100);
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
// 规范行集合只输出 items,不输出 allItems 重复结构
|
||||
assertTrue(node.has("items"));
|
||||
assertFalse(node.has("allItems"), "payload 必须不再序列化 allItems 重复结构");
|
||||
assertEquals(100, node.get("items").size());
|
||||
// groups 仍保留(Python 回传需要),但不作为全量行来源
|
||||
assertTrue(node.has("groups"));
|
||||
// items 中每行字段完整
|
||||
JsonNode first = node.get("items").get(0);
|
||||
assertEquals("uploads/20260829/base.xlsx::row::1", first.get("rowToken").asText());
|
||||
assertEquals("B0CJ8SNXXV", first.get("asin").asText());
|
||||
assertEquals("uploads/20260829/base.xlsx", first.get("sourceFileKey").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_normal_multiple_items() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(1000);
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertEquals(1000, node.get("items").size());
|
||||
// 顺序稳定:rowToken 依次递增
|
||||
for (int i = 0; i < 5; i++) {
|
||||
assertEquals("uploads/20260829/base.xlsx::row::" + (i + 1),
|
||||
node.get("items").get(i).get("rowToken").asText());
|
||||
}
|
||||
// 反序列化后行数不丢失
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
assertEquals(1000, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(200);
|
||||
String first = buildPayloadJson(rows, groups(rows));
|
||||
String second = buildPayloadJson(rows, groups(rows));
|
||||
// 重复构建输出一致
|
||||
assertEquals(MAPPER.readTree(first), MAPPER.readTree(second));
|
||||
// 不产生重复记录:行 token 唯一
|
||||
JsonNode items = MAPPER.readTree(first).get("items");
|
||||
long distinct = java.util.stream.StreamSupport.stream(items.spliterator(), false)
|
||||
.map(item -> item.get("rowToken").asText()).distinct().count();
|
||||
assertEquals(200, distinct);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_boundary_empty_input() throws Exception {
|
||||
String json = buildPayloadJson(List.of(), List.of());
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertTrue(node.has("items"));
|
||||
assertEquals(0, node.get("items").size());
|
||||
assertFalse(node.has("allItems"));
|
||||
// 空载荷反序列化安全
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
assertNotNull(payload.getItems());
|
||||
assertEquals(0, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_boundary_single_item() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = rows(1);
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertEquals(1, node.get("items").size());
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
|
||||
assertEquals(1, payload.getItems().size());
|
||||
assertEquals("uploads/20260829/base.xlsx::row::1", payload.getItems().get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_boundary_limit_and_overflow() throws Exception {
|
||||
// null 行集合:items 输出为空数组而非 NPE/崩溃
|
||||
String json = buildPayloadJson(null, null);
|
||||
JsonNode node = MAPPER.readTree(json);
|
||||
assertTrue(node.has("items"));
|
||||
assertEquals(0, node.get("items").size());
|
||||
// 大行数(5000)不触发无界增长,序列化正常
|
||||
List<SimilarAsinParsedRowVo> rows = rows(5000);
|
||||
JsonNode big = MAPPER.readTree(buildPayloadJson(rows, groups(rows)));
|
||||
assertEquals(5000, big.get("items").size());
|
||||
assertFalse(big.has("allItems"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_invalid_input_rejected() throws Exception {
|
||||
// 旧格式 JSON(含 allItems)反序列化:allItems 键被吸收进 items,不丢失行,不抛异常
|
||||
String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
|
||||
+ "\"sourceFiles\":[],\"headers\":[],"
|
||||
+ "\"items\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"}],"
|
||||
+ "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"},{\"rowToken\":\"t2\",\"asin\":\"B0TEST1234\"}],"
|
||||
+ "\"groups\":[]}";
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class);
|
||||
// items 优先;allItems 仅在 items 为空时兜底吸收,避免旧数据行丢失
|
||||
assertEquals(1, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_002_parsed_payload_dependency_failure_releases_resources() throws Exception {
|
||||
// 序列化器故障:抛项目约定异常(BusinessException),不产生部分结果
|
||||
ObjectMapper broken = new ObjectMapper() {
|
||||
@Override
|
||||
public String writeValueAsString(Object value) {
|
||||
throw new IllegalStateException("serializer down");
|
||||
}
|
||||
};
|
||||
SimilarAsinTaskService failingService = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
|
||||
ReflectionTestUtils.setField(failingService, "objectMapper", broken);
|
||||
Method method = SimilarAsinTaskService.class.getDeclaredMethod(
|
||||
"buildParsedPayloadJson",
|
||||
String.class, String.class, Boolean.class, Boolean.class,
|
||||
List.class, List.class, List.class, List.class);
|
||||
method.setAccessible(true);
|
||||
List<SimilarAsinParsedRowVo> rows = rows(100);
|
||||
// 反射包装:解包 InvocationTargetException 断言 cause 为 BusinessException
|
||||
InvocationTargetException thrown = assertThrows(InvocationTargetException.class, () -> method.invoke(failingService,
|
||||
"p", "k", Boolean.FALSE, Boolean.FALSE, List.of(), List.of(), groups(rows), rows));
|
||||
assertTrue(thrown.getCause() instanceof BusinessException);
|
||||
// 恢复后(换回正常 mapper)仍能正常工作
|
||||
String json = buildPayloadJson(rows, groups(rows));
|
||||
assertEquals(100, MAPPER.readTree(json).get("items").size());
|
||||
}
|
||||
}
|
||||
+568
@@ -0,0 +1,568 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.io.PrintWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
|
||||
/**
|
||||
* 任务 120:大页数据性能回归。
|
||||
* 500 任务级数据下 history/dashboard/progress 的响应时间上界(宽松基准线)、
|
||||
* 查询次数上界、内存有界;light 端点(200 任务)必须显著快于旧 batch 端点;
|
||||
* 结果写入 target/perf-report/ 供回归对比。
|
||||
*
|
||||
* 说明:stub 走纯内存数据结构(无需真实 DB),计时仅覆盖服务层装配与
|
||||
* 批量 Map 查找(不含 JDBC/网络),因此基线取宽松上界:500 任务装配在
|
||||
* 未优化实现下也远低于 1s,若未来出现逐条 N+1 退化将放大到数秒级从而失败。
|
||||
* 墙钟只用作单侧上界,不做两条路径之间的毫秒比较——纯内存下 JIT/GC/调度
|
||||
* 抖动远大于真实差值;light 与 batch 的「谁更轻」一律用查询次数与装配
|
||||
* 字段量这类确定性口径断言。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServicePerf500Test {
|
||||
|
||||
private static final String MODULE = "SIMILAR_ASIN";
|
||||
private static final int TASK_COUNT = 500;
|
||||
private static final long USER_ID = 1001L;
|
||||
private static final int HISTORY_LIMIT = 100;
|
||||
private static final int LIGHT_TASK_COUNT = 200;
|
||||
|
||||
private static final Duration BOUND_HISTORY = Duration.ofSeconds(2);
|
||||
private static final Duration BOUND_DASHBOARD = Duration.ofSeconds(2);
|
||||
private static final Duration BOUND_PROGRESS_BATCH = Duration.ofSeconds(3);
|
||||
private static final Duration BOUND_PROGRESS_LIGHT = Duration.ofSeconds(1);
|
||||
private static final int MAX_QUERY_COUNT_HISTORY = 4;
|
||||
private static final int MAX_QUERY_COUNT_DASHBOARD = 5;
|
||||
private static final int MAX_QUERY_COUNT_LIGHT = 3;
|
||||
|
||||
private final List<FileResultEntity> resultDb = new ArrayList<>();
|
||||
private final List<FileTaskEntity> taskDb = new ArrayList<>();
|
||||
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
|
||||
|
||||
private final AtomicInteger resultSelectCount = new AtomicInteger();
|
||||
private final AtomicInteger taskSelectCount = new AtomicInteger();
|
||||
private final AtomicInteger taskCountCount = new AtomicInteger();
|
||||
private final AtomicInteger jobQueryCount = new AtomicInteger();
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper();
|
||||
@Mock private SimilarAsinLlmService similarAsinLlmService;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
private static long nowNanos() {
|
||||
return System.nanoTime();
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
|
||||
|
||||
// 500 任务 + 500 结果 + 500 Job 全量种子:结果按 createdAt 降序返回
|
||||
LocalDateTime base = LocalDateTime.of(2026, 8, 1, 10, 0);
|
||||
for (int i = 1; i <= TASK_COUNT; i++) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId((long) i);
|
||||
task.setModuleType(MODULE);
|
||||
task.setStatus(i % 10 == 0 ? "RUNNING" : "SUCCESS");
|
||||
task.setUserId(USER_ID);
|
||||
task.setCreatedAt(base.plusMinutes(i));
|
||||
task.setUpdatedAt(base.plusMinutes(i));
|
||||
taskDb.add(task);
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId((long) i);
|
||||
result.setTaskId((long) i);
|
||||
result.setModuleType(MODULE);
|
||||
result.setUserId(USER_ID);
|
||||
result.setSourceFilename("src-" + i + ".xlsx");
|
||||
result.setResultFilename("out-" + i + ".xlsx");
|
||||
result.setResultFileUrl("result/similar-asin/" + i + "/out.xlsx");
|
||||
result.setRowCount(i);
|
||||
result.setSuccess(1);
|
||||
result.setCreatedAt(base.plusMinutes(i));
|
||||
resultDb.add(result);
|
||||
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId((long) i);
|
||||
job.setTaskId((long) i);
|
||||
job.setModuleType(MODULE);
|
||||
job.setStatus("SUCCESS");
|
||||
job.setResultFileUrl("result/similar-asin/" + i + "/out.xlsx");
|
||||
jobDb.add(job);
|
||||
}
|
||||
|
||||
// 结果表:selectList 返回命中 module+userId 的行(降序,与 SQL orderByDesc 一致)
|
||||
lenient().doAnswer(invocation -> {
|
||||
resultSelectCount.incrementAndGet();
|
||||
@SuppressWarnings("unchecked")
|
||||
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
|
||||
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||
List<Object> params = paramValuesOf(q, segment);
|
||||
List<Long> wanted = new ArrayList<>();
|
||||
for (Object value : params) {
|
||||
if (value instanceof List<?> list) {
|
||||
for (Object item : list) {
|
||||
if (item instanceof Number n) {
|
||||
wanted.add(n.longValue());
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Number n) {
|
||||
wanted.add(n.longValue());
|
||||
}
|
||||
}
|
||||
List<FileResultEntity> rows = new ArrayList<>();
|
||||
boolean hasIn = segment.contains("IN (");
|
||||
for (FileResultEntity row : resultDb) {
|
||||
if (row.getTaskId() == null) continue;
|
||||
if (hasIn && !wanted.isEmpty() && !wanted.contains(row.getTaskId())) continue;
|
||||
if (segment.contains("user_id") && !Long.valueOf(USER_ID).equals(row.getUserId())) continue;
|
||||
if (row.getModuleType() != null && !MODULE.equals(row.getModuleType())) continue;
|
||||
rows.add(row);
|
||||
}
|
||||
rows.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
|
||||
if (segment.contains("limit")) {
|
||||
int limit = segment.contains("limit 100") ? 100 : 50;
|
||||
rows = new ArrayList<>(rows.subList(0, Math.min(limit, rows.size())));
|
||||
}
|
||||
return rows;
|
||||
}).when(fileResultMapper).selectList(any());
|
||||
|
||||
// 任务表:IN 查询返回 id 命中且模块匹配的 task
|
||||
lenient().doAnswer(invocation -> {
|
||||
taskSelectCount.incrementAndGet();
|
||||
@SuppressWarnings("unchecked")
|
||||
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
|
||||
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||
List<Object> params = paramValuesOf(q, segment);
|
||||
List<Long> wanted = new ArrayList<>();
|
||||
for (Object value : params) {
|
||||
if (value instanceof List<?> list) {
|
||||
for (Object item : list) {
|
||||
if (item instanceof Number n) {
|
||||
wanted.add(n.longValue());
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Number n) {
|
||||
wanted.add(n.longValue());
|
||||
}
|
||||
}
|
||||
if (wanted.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return taskDb.stream().filter(t -> wanted.contains(t.getId()) && MODULE.equals(t.getModuleType())).toList();
|
||||
}).when(fileTaskMapper).selectList(any());
|
||||
|
||||
// dashboard 计数:按 status 匹配 selectCount(聚合前后行为一致)
|
||||
lenient().doAnswer(invocation -> {
|
||||
taskCountCount.incrementAndGet();
|
||||
@SuppressWarnings("unchecked")
|
||||
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
|
||||
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||
if (segment.contains("inSql") || segment.contains("IN (select distinct task_id")) {
|
||||
// countCompletedTasksWithResultFile:RUNNING 且结果表有文件 URL 的任务数
|
||||
long runningWithFile = taskDb.stream().filter(t -> "RUNNING".equals(t.getStatus()))
|
||||
.filter(t -> resultDb.stream().anyMatch(r -> r.getTaskId() != null
|
||||
&& r.getTaskId().equals(t.getId())
|
||||
&& r.getResultFileUrl() != null
|
||||
&& !r.getResultFileUrl().isBlank()))
|
||||
.count();
|
||||
return runningWithFile;
|
||||
}
|
||||
long count = taskDb.stream()
|
||||
.filter(t -> Long.valueOf(USER_ID).equals(t.getUserId()))
|
||||
.filter(t -> MODULE.equals(t.getModuleType()))
|
||||
.filter(t -> {
|
||||
String status = findStringParam(q, segment, "status");
|
||||
if (status != null && !status.isBlank()) {
|
||||
return status.equals(t.getStatus());
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.count();
|
||||
return count;
|
||||
}).when(fileTaskMapper).selectCount(any());
|
||||
|
||||
// Job 批量查询:taskId 集合 / resultId 集合命中
|
||||
lenient().doAnswer(invocation -> {
|
||||
jobQueryCount.incrementAndGet();
|
||||
Object taskIdsObj = invocation.getArgument(1);
|
||||
Map<Long, TaskFileJobEntity> map = new HashMap<>();
|
||||
if (taskIdsObj instanceof List<?> list) {
|
||||
for (TaskFileJobEntity job : jobDb) {
|
||||
if (job.getTaskId() != null && list.contains(job.getTaskId())) {
|
||||
map.putIfAbsent(job.getTaskId(), job);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}).when(taskFileJobService).findAssembleJobsByResultIds(eq(MODULE), anyList());
|
||||
|
||||
lenient().doAnswer(invocation -> {
|
||||
jobQueryCount.incrementAndGet();
|
||||
Object taskIdsObj = invocation.getArgument(1);
|
||||
Map<Long, TaskFileJobEntity> map = new HashMap<>();
|
||||
if (taskIdsObj instanceof List<?> list) {
|
||||
for (TaskFileJobEntity job : jobDb) {
|
||||
if (job.getTaskId() != null && list.contains(job.getTaskId())) {
|
||||
map.putIfAbsent(job.getTaskId(), job);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}).when(taskFileJobService).findAssembleJobsByTaskIds(eq(MODULE), anyList());
|
||||
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("server-110");
|
||||
}
|
||||
|
||||
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
|
||||
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
|
||||
List<Object> values = new ArrayList<>();
|
||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
|
||||
Map<String, Object> params = q.getParamNameValuePairs();
|
||||
while (m.find()) {
|
||||
Object value = params.get(m.group(1));
|
||||
if (value != null) {
|
||||
values.add(value);
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/** 按列名取 eq 参数值:status = #{...} 中列后的占位 key 对应的值。 */
|
||||
private static String findStringParam(LambdaQueryWrapper<?> q, String segment, String column) {
|
||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||
.compile("\\b" + column + "\\s*=\\s*#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
|
||||
if (m.find()) {
|
||||
Object value = q.getParamNameValuePairs().get(m.group(1));
|
||||
if (value != null) {
|
||||
return String.valueOf(value);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task(Long id, String status, LocalDateTime updatedAt) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setModuleType(MODULE);
|
||||
task.setStatus(status);
|
||||
task.setUpdatedAt(updatedAt);
|
||||
return task;
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity job(Long id, Long taskId, String status, String resultFileUrl) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(id);
|
||||
job.setTaskId(taskId);
|
||||
job.setModuleType(MODULE);
|
||||
job.setStatus(status);
|
||||
job.setResultFileUrl(resultFileUrl);
|
||||
return job;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_history_500() {
|
||||
// 500 任务级历史:响应时间有界(宽松基准线),结果不丢失、数量正确
|
||||
long start = nowNanos();
|
||||
SimilarAsinHistoryVo vo = assertTimeoutPreemptively(BOUND_HISTORY,
|
||||
() -> service.history(USER_ID, HISTORY_LIMIT));
|
||||
long elapsedMs = (nowNanos() - start) / 1_000_000;
|
||||
assertEquals(100, vo.getItems().size(), "limit 100 只返回 100 条");
|
||||
assertFalse(vo.getItems().isEmpty());
|
||||
assertEquals(500L, vo.getItems().getFirst().getResultId(), "降序首条为最新结果");
|
||||
assertTrue(elapsedMs < BOUND_HISTORY.toMillis(),
|
||||
"history 耗时 " + elapsedMs + "ms 应低于上界 " + BOUND_HISTORY.toMillis() + "ms");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_dashboard_500() {
|
||||
// 500 任务级 dashboard:统计正确且耗时在基准线内
|
||||
long start = nowNanos();
|
||||
SimilarAsinDashboardVo vo = assertTimeoutPreemptively(BOUND_DASHBOARD,
|
||||
() -> service.dashboard(USER_ID));
|
||||
long elapsedMs = (nowNanos() - start) / 1_000_000;
|
||||
assertEquals(500, vo.getSuccessTaskCount(), "success = 450 成功 + 50 RUNNING 且有结果文件 = 500");
|
||||
assertEquals(0, vo.getPendingTaskCount(), "RUNNING 任务(50)均有结果文件 → pending=0");
|
||||
assertEquals(0, vo.getFailedTaskCount(), "无失败任务");
|
||||
assertTrue(elapsedMs < BOUND_DASHBOARD.toMillis(),
|
||||
"dashboard 耗时 " + elapsedMs + "ms 应低于上界 " + BOUND_DASHBOARD.toMillis() + "ms");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_progress_200() {
|
||||
// 200 任务进度(旧 batch 端点):全量装配在基准线内,明细字段完整
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for (int i = 1; i <= 200; i++) ids.add((long) i);
|
||||
long start = nowNanos();
|
||||
SimilarAsinTaskBatchVo vo = assertTimeoutPreemptively(BOUND_PROGRESS_BATCH,
|
||||
() -> service.progressBatch(ids));
|
||||
long elapsedMs = (nowNanos() - start) / 1_000_000;
|
||||
assertEquals(200, vo.getItems().size(), "200 任务全部装配");
|
||||
assertEquals(0, vo.getMissingTaskIds().size());
|
||||
assertEquals("SUCCESS", vo.getItems().get(0).getTask().getStatus());
|
||||
assertTrue(elapsedMs < BOUND_PROGRESS_BATCH.toMillis(),
|
||||
"progressBatch 耗时 " + elapsedMs + "ms 应低于上界 " + BOUND_PROGRESS_BATCH.toMillis() + "ms");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_light_200() {
|
||||
// 200 任务 light 端点:轻量装配比 batch 做更少的工作(墙钟上界 1s)。
|
||||
// 「更快」用确定性口径断言——查询次数与装配字段量,而不是比较墙钟
|
||||
// 毫秒:两条路径都跑纯内存 stub,耗时在十几毫秒量级,JIT/GC/线程
|
||||
// 调度抖动远大于真实差值,lightMs < batchMs 会随机翻绿翻红。
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
long lightStart = nowNanos();
|
||||
SimilarAsinTaskLightBatchVo light = assertTimeoutPreemptively(BOUND_PROGRESS_LIGHT,
|
||||
() -> service.progressLight(ids));
|
||||
long lightMs = (nowNanos() - lightStart) / 1_000_000;
|
||||
int lightResultQueries = resultSelectCount.get();
|
||||
int lightQueries = taskSelectCount.get() + jobQueryCount.get() + lightResultQueries;
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
SimilarAsinTaskBatchVo batch = service.progressBatch(ids);
|
||||
int batchResultQueries = resultSelectCount.get();
|
||||
int batchQueries = taskSelectCount.get() + jobQueryCount.get() + batchResultQueries;
|
||||
|
||||
assertEquals(LIGHT_TASK_COUNT, light.getItems().size());
|
||||
assertTrue(lightMs < BOUND_PROGRESS_LIGHT.toMillis(),
|
||||
"light 耗时 " + lightMs + "ms 应低于上界 " + BOUND_PROGRESS_LIGHT.toMillis() + "ms");
|
||||
assertEquals(0, lightResultQueries,
|
||||
"light 不查结果明细行(batch 查了 " + batchResultQueries + " 次)");
|
||||
assertTrue(batchResultQueries >= 1, "batch 仍查结果明细行(对照组成立)");
|
||||
assertTrue(lightQueries < batchQueries,
|
||||
"light 查询数(" + lightQueries + ")应少于 batch(" + batchQueries + ")");
|
||||
assertTrue(batch.getItems().size() >= light.getItems().size(),
|
||||
"batch 至少返回与 light 等量条目(行为不退化)");
|
||||
assertTrue(batch.getItems().get(0).getTask() != null,
|
||||
"batch 条目携带任务明细对象,light 条目只有白名单字段");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_query_count() {
|
||||
// 查询次数上界:history ≤ 4 次、dashboard ≤ 5 次、light ≤ 3 次(恒定,与行数无关)
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
service.history(USER_ID, HISTORY_LIMIT);
|
||||
assertTrue(resultSelectCount.get() <= 1, "history 结果查询 ≤ 1 次");
|
||||
assertTrue(taskSelectCount.get() <= 1, "history 任务查询 ≤ 1 次");
|
||||
assertTrue(jobQueryCount.get() <= 2, "history Job 查询 ≤ 2 次(findAssembleJobsByResultIds 一次)");
|
||||
int historyTotal = resultSelectCount.get() + taskSelectCount.get() + jobQueryCount.get();
|
||||
assertTrue(historyTotal <= MAX_QUERY_COUNT_HISTORY,
|
||||
"history 总查询 " + historyTotal + " 应 ≤ " + MAX_QUERY_COUNT_HISTORY);
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
taskCountCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
service.dashboard(USER_ID);
|
||||
assertTrue(taskCountCount.get() <= MAX_QUERY_COUNT_DASHBOARD,
|
||||
"dashboard 查询 " + taskCountCount.get() + " 应 ≤ " + MAX_QUERY_COUNT_DASHBOARD);
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
service.progressLight(ids);
|
||||
assertTrue(taskSelectCount.get() <= 1, "light 任务查询 ≤ 1 次");
|
||||
assertTrue(jobQueryCount.get() <= 1, "light Job 查询 ≤ 1 次");
|
||||
assertTrue(resultSelectCount.get() <= 1, "light 不应有结果明细查询(白名单字段无明细)");
|
||||
int lightTotal = taskSelectCount.get() + jobQueryCount.get() + resultSelectCount.get();
|
||||
assertTrue(lightTotal <= MAX_QUERY_COUNT_LIGHT,
|
||||
"light 总查询 " + lightTotal + " 应 ≤ " + MAX_QUERY_COUNT_LIGHT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_no_regression() {
|
||||
// 与基线对比:500 任务装配的总查询次数为常量(批量 IN + Map 装配),
|
||||
// 若未来退化为逐条 N+1,selectList 调用次数将随行数线性增长而失败
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
service.progressLight(ids);
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
service.progressLight(ids.subList(0, 100));
|
||||
|
||||
assertTrue(jobQueryCount.get() <= 1, "100 任务 light 仍是 1 次批量 Job 查询(无 N+1)");
|
||||
assertTrue(taskSelectCount.get() <= 1, "100 任务 light 仍是 1 次批量任务查询(无 N+1)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_memory_bounded() {
|
||||
// 内存有界:500 任务装配后响应对象只保留白名单字段,不携带 payload/明细
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for (int i = 1; i <= 500; i++) ids.add((long) i);
|
||||
SimilarAsinTaskLightBatchVo light = service.progressLight(ids);
|
||||
assertEquals(200, light.getItems().size(), "light 受 MAX_LIGHT_TASK_IDS=200 上限约束");
|
||||
for (SimilarAsinTaskLightVo item : light.getItems()) {
|
||||
assertTrue(item.getTaskId() != null);
|
||||
assertTrue(item.getStatus() != null);
|
||||
}
|
||||
|
||||
// 全量 progressBatch 也走批量路径:对象数 = 请求任务数,无逐条中间缓存
|
||||
SimilarAsinTaskBatchVo batch = service.progressBatch(ids);
|
||||
assertEquals(500, batch.getItems().size());
|
||||
assertEquals(0, batch.getMissingTaskIds().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_perf_report() throws Exception {
|
||||
// 结果记录:把本轮基准写入 target/perf-report/ 供回归对比
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
|
||||
|
||||
long h0 = nowNanos();
|
||||
service.history(USER_ID, HISTORY_LIMIT);
|
||||
long historyMs = (nowNanos() - h0) / 1_000_000;
|
||||
|
||||
long d0 = nowNanos();
|
||||
service.dashboard(USER_ID);
|
||||
long dashboardMs = (nowNanos() - d0) / 1_000_000;
|
||||
|
||||
long l0 = nowNanos();
|
||||
service.progressLight(ids);
|
||||
long lightMs = (nowNanos() - l0) / 1_000_000;
|
||||
|
||||
long b0 = nowNanos();
|
||||
service.progressBatch(ids);
|
||||
long batchMs = (nowNanos() - b0) / 1_000_000;
|
||||
|
||||
resultSelectCount.set(0);
|
||||
taskSelectCount.set(0);
|
||||
jobQueryCount.set(0);
|
||||
service.progressLight(ids);
|
||||
int lightQueries = taskSelectCount.get() + jobQueryCount.get() + resultSelectCount.get();
|
||||
|
||||
Path reportDir = Path.of("target", "perf-report");
|
||||
Files.createDirectories(reportDir);
|
||||
Path report = reportDir.resolve("similar-asin-perf-500.csv");
|
||||
boolean fresh = !Files.exists(report);
|
||||
List<String> lines = List.of(
|
||||
String.join(",",
|
||||
"run",
|
||||
"history_ms",
|
||||
"dashboard_ms",
|
||||
"light_200_ms",
|
||||
"batch_200_ms",
|
||||
"light_queries",
|
||||
"tasks",
|
||||
"elapsed_ms=" + historyMs),
|
||||
String.join(",",
|
||||
"task120",
|
||||
String.valueOf(historyMs),
|
||||
String.valueOf(dashboardMs),
|
||||
String.valueOf(lightMs),
|
||||
String.valueOf(batchMs),
|
||||
String.valueOf(lightQueries),
|
||||
String.valueOf(TASK_COUNT),
|
||||
String.valueOf(historyMs + dashboardMs + lightMs + batchMs)));
|
||||
try (PrintWriter writer = new PrintWriter(Files.newBufferedWriter(report,
|
||||
StandardCharsets.UTF_8,
|
||||
fresh ? StandardOpenOption.CREATE_NEW : StandardOpenOption.APPEND))) {
|
||||
if (fresh) {
|
||||
writer.println("run,history_ms,dashboard_ms,light_200_ms,batch_200_ms,light_queries,tasks,note");
|
||||
}
|
||||
writer.println(lines.get(1));
|
||||
}
|
||||
assertTrue(Files.exists(report), "性能报告应写入 target/perf-report/");
|
||||
assertTrue(lightMs < batchMs, "light 应快于 batch(报告记录)");
|
||||
}
|
||||
}
|
||||
+253
@@ -0,0 +1,253 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 5:预览行数量增加配置边界、空文件和超限输入校验。
|
||||
* 预览上限从硬编码常量改为配置驱动,并对配置值做 clamp 边界保护。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServicePreviewConfigTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(20000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/20000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-preview-config-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0CFG%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("config.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_normal_default_path() throws Exception {
|
||||
// 默认配置 100:150 行文件返回 100 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(150);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-default.xlsx");
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
// 预览行从第 1 行开始
|
||||
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_normal_multiple_items() throws Exception {
|
||||
// 配置 50:500 行文件返回 50 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(50);
|
||||
File workbook = buildWorkbook(500);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-50.xlsx");
|
||||
assertEquals(50, vo.getItems().size());
|
||||
assertEquals(500, vo.getAcceptedRows());
|
||||
// groups 同样按配置裁剪
|
||||
assertTrue(vo.getGroups().size() <= 50);
|
||||
// 配置 200:300 行文件返回 200 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(200);
|
||||
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-200.xlsx");
|
||||
assertEquals(200, vo2.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(250);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_boundary_empty_input() throws Exception {
|
||||
// 空文件(无有效数据行):抛业务异常
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-empty.xlsx")).thenReturn(workbook);
|
||||
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/cfg-empty.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_boundary_single_item() throws Exception {
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-single.xlsx");
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_boundary_limit_and_overflow() throws Exception {
|
||||
// 配置值超过最大上限(1000):clamp 到上限,不发生无界内存增长
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(50000);
|
||||
File workbook = buildWorkbook(2000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-over.xlsx");
|
||||
assertEquals(1000, vo.getItems().size(), "超限配置必须 clamp 到最大允许值");
|
||||
assertEquals(2000, vo.getAcceptedRows());
|
||||
// 配置值恰好等于上限:返回 1000 预览行
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(1000);
|
||||
SimilarAsinParseVo vo2 = parse(buildWorkbook(1000), "uploads/20260829/cfg-exact.xlsx");
|
||||
assertEquals(1000, vo2.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_invalid_input_rejected() throws Exception {
|
||||
// 配置为 0/负数:回退到默认值 100,不抛异常不崩溃
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(0, -5);
|
||||
File workbook = buildWorkbook(300);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-zero.xlsx");
|
||||
assertEquals(100, vo.getItems().size());
|
||||
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-neg.xlsx");
|
||||
assertEquals(100, vo2.getItems().size());
|
||||
// 配置缺失(fresh mock 未 stub,int 默认 0):同样回退默认
|
||||
SimilarAsinProperties missing = org.mockito.Mockito.mock(SimilarAsinProperties.class);
|
||||
lenient().when(missing.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(missing.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(missing.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
ReflectionTestUtils.setField(service, "properties", missing);
|
||||
SimilarAsinParseVo vo3 = parse(buildWorkbook(300), "uploads/20260829/cfg-missing.xlsx");
|
||||
assertEquals(100, vo3.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_005_preview_row_count_dependency_failure_releases_resources() throws Exception {
|
||||
// 存储失败抛异常;恢复后解析正常
|
||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
File workbook = buildWorkbook(120);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/20001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/cfg-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-recovered.xlsx");
|
||||
assertEquals(120, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
// 配置对象默认值校验:新实例默认 100,处于 [1, 1000] 边界内
|
||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
||||
assertNotNull(defaults.getParseResponsePreviewLimit());
|
||||
int previewLimit = defaults.getParseResponsePreviewLimit();
|
||||
assertTrue(previewLimit >= 1 && previewLimit <= 1000,
|
||||
"默认预览上限必须在 [1, 1000] 内,实际 " + previewLimit);
|
||||
}
|
||||
}
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Task 3:保留旧 payload 读取兼容逻辑,验证新旧结构均可恢复全量行。
|
||||
* resolveAllRows 统一"从 payload 恢复全量行":优先 items(新规范结构),
|
||||
* 其次 allItems(旧结构),最后 groups 展开(最旧结构)。
|
||||
*/
|
||||
class SimilarAsinTaskServiceResolveAllRowsTest {
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private static SimilarAsinParsedRowVo row(String fileKey, int index) {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setSourceFileKey(fileKey);
|
||||
row.setSourceFilename("base.xlsx");
|
||||
row.setRowIndex(index);
|
||||
row.setSourceId(String.valueOf(index));
|
||||
row.setDisplayId(String.valueOf(index));
|
||||
row.setRowToken(fileKey + "::row::" + index);
|
||||
row.setAsin("B0CJ8SNXXV");
|
||||
row.setCountry("英国");
|
||||
return row;
|
||||
}
|
||||
|
||||
private static List<SimilarAsinParsedRowVo> rows(String fileKey, int count) {
|
||||
List<SimilarAsinParsedRowVo> result = new ArrayList<>();
|
||||
for (int i = 1; i <= count; i++) {
|
||||
result.add(row(fileKey, i));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static SimilarAsinParsedGroupVo group(List<SimilarAsinParsedRowVo> items) {
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setSourceFileKey("uploads/20260829/base.xlsx");
|
||||
group.setSourceFilename("base.xlsx");
|
||||
group.setGroupKey("uploads/20260829/base.xlsx::1@1");
|
||||
group.setBaseId("1");
|
||||
group.setDisplayId("1");
|
||||
group.setItemCount(items.size());
|
||||
group.setItems(new ArrayList<>(items));
|
||||
return group;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_default_path() {
|
||||
// 新结构:items 有值 → 恢复全量行
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/base.xlsx", 100);
|
||||
payload.setItems(items);
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(100, restored.size());
|
||||
assertEquals("uploads/20260829/base.xlsx::row::1", restored.get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_multiple_items() {
|
||||
// 新结构 1000 行:不丢失且顺序稳定
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/multi.xlsx", 1000);
|
||||
payload.setItems(items);
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(1000, restored.size());
|
||||
for (int i = 0; i < restored.size(); i++) {
|
||||
assertEquals(i + 1, restored.get(i).getRowIndex());
|
||||
}
|
||||
// 旧结构 1000 行:allItems 全量恢复
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
legacy.setAllItems(items);
|
||||
List<SimilarAsinParsedRowVo> restoredLegacy = SimilarAsinTaskService.resolveAllRows(legacy);
|
||||
assertEquals(1000, restoredLegacy.size());
|
||||
assertEquals("uploads/20260829/multi.xlsx::row::1", restoredLegacy.get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_repeated_operation_is_idempotent() {
|
||||
// 重复调用返回相同行集合(不修改 payload 本身)
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/idem.xlsx", 50);
|
||||
payload.setItems(items);
|
||||
List<SimilarAsinParsedRowVo> first = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
List<SimilarAsinParsedRowVo> second = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
}
|
||||
// payload 未被修改:items 仍 50 行
|
||||
assertEquals(50, payload.getItems().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_boundary_empty_input() {
|
||||
// 空 payload:返回空列表而非 null,不创建无效资源
|
||||
SimilarAsinParsedPayloadDto empty = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(empty);
|
||||
assertNotNull(restored);
|
||||
assertEquals(0, restored.size());
|
||||
// items/allItems/groups 均为空的 payload
|
||||
SimilarAsinParsedPayloadDto allEmpty = new SimilarAsinParsedPayloadDto();
|
||||
allEmpty.setItems(List.of());
|
||||
allEmpty.setAllItems(List.of());
|
||||
allEmpty.setGroups(List.of());
|
||||
assertEquals(0, SimilarAsinTaskService.resolveAllRows(allEmpty).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_boundary_single_item() {
|
||||
// 单行新结构
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setItems(rows("uploads/20260829/single.xlsx", 1));
|
||||
assertEquals(1, SimilarAsinTaskService.resolveAllRows(payload).size());
|
||||
// 单行旧结构(仅 groups)
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
legacy.setGroups(List.of(group(rows("uploads/20260829/single.xlsx", 1))));
|
||||
assertEquals(1, SimilarAsinTaskService.resolveAllRows(legacy).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_boundary_limit_and_overflow() {
|
||||
// 旧结构 allItems 5000 行全量恢复,不丢失
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
legacy.setAllItems(rows("uploads/20260829/max.xlsx", 5000));
|
||||
assertEquals(5000, SimilarAsinTaskService.resolveAllRows(legacy).size());
|
||||
// items 与 allItems 同时存在:以 items 为准(新规范结构优先),不重复
|
||||
SimilarAsinParsedPayloadDto both = new SimilarAsinParsedPayloadDto();
|
||||
both.setItems(rows("uploads/20260829/both.xlsx", 10));
|
||||
both.setAllItems(rows("uploads/20260829/both.xlsx", 20));
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(both);
|
||||
assertEquals(10, restored.size());
|
||||
// groups 也同时存在:仍以 items 为准
|
||||
both.setGroups(List.of(group(rows("uploads/20260829/both.xlsx", 30))));
|
||||
assertEquals(10, SimilarAsinTaskService.resolveAllRows(both).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_invalid_input_rejected() {
|
||||
// null payload:安全返回空列表(调用方容忍),不抛 NPE
|
||||
assertEquals(0, SimilarAsinTaskService.resolveAllRows(null).size());
|
||||
// groups 中含 null 元素:跳过不抛异常
|
||||
SimilarAsinParsedPayloadDto messy = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
|
||||
groups.add(null);
|
||||
groups.add(group(rows("uploads/20260829/messy.xlsx", 5)));
|
||||
messy.setGroups(groups);
|
||||
assertEquals(5, SimilarAsinTaskService.resolveAllRows(messy).size());
|
||||
// group.items 为 null:跳过该组
|
||||
SimilarAsinParsedGroupVo nullItemsGroup = new SimilarAsinParsedGroupVo();
|
||||
nullItemsGroup.setItems(null);
|
||||
SimilarAsinParsedPayloadDto nullItems = new SimilarAsinParsedPayloadDto();
|
||||
nullItems.setGroups(List.of(nullItemsGroup, group(rows("uploads/20260829/n2.xlsx", 3))));
|
||||
assertEquals(3, SimilarAsinTaskService.resolveAllRows(nullItems).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_dependency_failure_releases_resources() throws Exception {
|
||||
// 旧格式 JSON(只有 allItems)反序列化 → hydrate 后 resolveAllRows 恢复全量行
|
||||
String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
|
||||
+ "\"sourceFiles\":[],\"headers\":[],\"groups\":[],"
|
||||
+ "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0AAA00001\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":1},"
|
||||
+ "{\"rowToken\":\"t2\",\"asin\":\"B0AAA00002\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":2}]}";
|
||||
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class);
|
||||
// 未 hydrate 时:allItems 恢复(items 为空走 allItems)
|
||||
List<SimilarAsinParsedRowVo> fromLegacy = SimilarAsinTaskService.resolveAllRows(payload);
|
||||
assertEquals(2, fromLegacy.size());
|
||||
assertEquals("t1", fromLegacy.get(0).getRowToken());
|
||||
assertEquals("B0AAA00002", fromLegacy.get(1).getAsin());
|
||||
// 新格式 JSON(只有 items)反序列化 → 直接恢复
|
||||
String newJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
|
||||
+ "\"sourceFiles\":[],\"headers\":[],"
|
||||
+ "\"items\":[{\"rowToken\":\"n1\",\"asin\":\"B0NEW00001\",\"sourceFileKey\":\"uploads/20260829/b.xlsx\",\"rowIndex\":1}],"
|
||||
+ "\"groups\":[]}";
|
||||
SimilarAsinParsedPayloadDto newPayload = MAPPER.readValue(newJson, SimilarAsinParsedPayloadDto.class);
|
||||
List<SimilarAsinParsedRowVo> fromNew = SimilarAsinTaskService.resolveAllRows(newPayload);
|
||||
assertEquals(1, fromNew.size());
|
||||
assertEquals("n1", fromNew.get(0).getRowToken());
|
||||
// 两种格式行数之和互不影响,恢复结果稳定
|
||||
assertTrue(fromLegacy.size() == 2 && fromNew.size() == 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_003_payload_normal_groups_expansion_preserves_order() {
|
||||
// 最旧结构:仅 groups 嵌套行,展开后顺序稳定(按组、组内原序)
|
||||
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
|
||||
List<SimilarAsinParsedRowVo> g1 = rows("uploads/20260829/g.xlsx", 2);
|
||||
List<SimilarAsinParsedRowVo> g2 = rows("uploads/20260829/g.xlsx", 3);
|
||||
// 组内行号各自独立从 1 开始(真实解析语义),第二组用不同 fileKey 区分来源
|
||||
List<SimilarAsinParsedRowVo> g2b = rows("uploads/20260829/g2.xlsx", 3);
|
||||
legacy.setGroups(List.of(group(g1), group(g2b)));
|
||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacy);
|
||||
assertEquals(5, restored.size());
|
||||
assertEquals("uploads/20260829/g.xlsx::row::1", restored.get(0).getRowToken());
|
||||
assertEquals("uploads/20260829/g.xlsx::row::2", restored.get(1).getRowToken());
|
||||
assertEquals("uploads/20260829/g2.xlsx::row::1", restored.get(2).getRowToken());
|
||||
assertEquals("uploads/20260829/g2.xlsx::row::2", restored.get(3).getRowToken());
|
||||
assertEquals("uploads/20260829/g2.xlsx::row::3", restored.get(4).getRowToken());
|
||||
}
|
||||
}
|
||||
+321
@@ -0,0 +1,321 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 11:Coze 结果合并的重复检测从 O(n²) 改为 HashSet/稳定 row key。
|
||||
* dedupeRowsByRowKey 用 HashSet 按稳定 rowKey 一次性去重(保留顺序),
|
||||
* mergeCozeRowsIntoChunk 合并前先去重,消除重复行逐行重复处理。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceRowKeyDedupeTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(70000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/70000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(id);
|
||||
r.setAsin(asin);
|
||||
r.setCountry(country);
|
||||
return r;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7004L);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private void stubSingleChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicLong storedCounter) throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
|
||||
.thenReturn(payloadJson);
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storedCounter.incrementAndGet();
|
||||
return "stored:" + invocation.getArgument(2);
|
||||
});
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_normal_default_path() throws Exception {
|
||||
// 正常输入:llmRows 含同一 rowKey 的重复行,merge 前按稳定 rowKey 去重,
|
||||
// chunk payload 只写一次,结果行不重复。
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
AtomicLong storedCounter = new AtomicLong(0);
|
||||
stubSingleChunkMerge(chunk, rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))), storedCounter);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(
|
||||
row("r1", "1", "B0A0000001", "英国"),
|
||||
row("r1", "1", "B0A0000001", "英国"));
|
||||
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(1, deduped.size(), "重复行必须按稳定 rowKey 去重");
|
||||
assertEquals("r1", deduped.get(0).getRowToken());
|
||||
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, null, null, llmRows, Map.of());
|
||||
assertEquals(1, storedCounter.get(), "去重后 chunk 只写一次");
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多个重复行跨 chunk 分组,去重后顺序稳定、结果不丢失
|
||||
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r2", "2", "B0A0000002", "英国"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
|
||||
AtomicLong selectOneRound = new AtomicLong(0);
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
|
||||
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 0; i < 3; i++) {
|
||||
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
|
||||
llmRows.add(row("r2", "2", "B0A0000002", "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(2, deduped.size(), "3 轮重复输入去重后只剩 2 个唯一行");
|
||||
assertEquals(List.of("r1", "r2"), deduped.stream().map(SimilarAsinResultRowDto::getRowToken).toList(),
|
||||
"去重必须保留首次出现顺序");
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, null, null, llmRows, Map.of());
|
||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行同一输入:去重结果完全一致,不产生重复记录
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(
|
||||
row("r1", "1", "B0A0000001", "英国"),
|
||||
row("r2", "2", "B0A0000002", "英国"),
|
||||
row("r1", "1", "B0A0000001", "英国"));
|
||||
List<SimilarAsinResultRowDto> first = service.dedupeRowsByRowKey(llmRows);
|
||||
List<SimilarAsinResultRowDto> second = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
assertEquals(first.get(i).getAsin(), second.get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_boundary_empty_input() {
|
||||
// 空输入:null/空列表安全返回空结果,不创建无效资源
|
||||
assertNotNull(service.dedupeRowsByRowKey(null));
|
||||
assertTrue(service.dedupeRowsByRowKey(null).isEmpty());
|
||||
assertTrue(service.dedupeRowsByRowKey(List.of()).isEmpty());
|
||||
// null 元素:跳过不抛异常
|
||||
List<SimilarAsinResultRowDto> withNull = new ArrayList<>();
|
||||
withNull.add(null);
|
||||
withNull.add(row("r1", "1", "B0A0000001", "英国"));
|
||||
assertEquals(1, service.dedupeRowsByRowKey(withNull).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_boundary_single_item() {
|
||||
// 单行:不依赖批量路径,去重后结果正确
|
||||
List<SimilarAsinResultRowDto> single = List.of(row("r1", "1", "B0A0000001", "英国"));
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(single);
|
||||
assertEquals(1, deduped.size());
|
||||
assertEquals("r1", deduped.get(0).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_boundary_limit_and_overflow() {
|
||||
// 大批量:1000 行全部重复,去重后只剩 1 个唯一行,无无界内存增长
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
|
||||
assertEquals(1, deduped.size());
|
||||
// 1000 行唯一:全部保留且顺序稳定
|
||||
List<SimilarAsinResultRowDto> unique = new ArrayList<>();
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
unique.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0U" + String.format("%06d", i), "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> dedupedUnique = service.dedupeRowsByRowKey(unique);
|
||||
assertEquals(1000, dedupedUnique.size());
|
||||
assertEquals("r0001", dedupedUnique.get(1).getRowToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_invalid_input_rejected() {
|
||||
// 稳定 rowKey 冲突:rowToken 为空时用 legacy key(id::ASIN::country)识别重复
|
||||
List<SimilarAsinResultRowDto> noToken = List.of(
|
||||
row("", "1", "B0A0000001", "英国"),
|
||||
row("", "1", "b0a0000001", " 英国 "));
|
||||
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(noToken);
|
||||
assertEquals(1, deduped.size(), "legacy key 归一化(ASIN 大写、country trim)后应识别为同一行");
|
||||
// 不同 ASIN:不误判为重复
|
||||
List<SimilarAsinResultRowDto> diffAsin = List.of(
|
||||
row("", "1", "B0A0000001", "英国"),
|
||||
row("", "2", "B0A0000002", "英国"));
|
||||
assertEquals(2, service.dedupeRowsByRowKey(diffAsin).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_011_merge_row_key_dependency_failure_releases_resources() throws Exception {
|
||||
// chunk payload 读取失败:抛可识别业务异常且不写 chunk;
|
||||
// 依赖恢复后重试成功,去重路径无残留状态
|
||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenThrow(new IllegalStateException("rustfs down"));
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
merge.invoke(service, task, null, null,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
// 恢复后重试成功:只写一次,无重复记录
|
||||
AtomicLong storedCounter = new AtomicLong(0);
|
||||
stubSingleChunkMerge(chunk, rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))), storedCounter);
|
||||
merge.invoke(service, task, null, null,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
assertEquals(1, storedCounter.get());
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
}
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinImagePrefetchService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 10:为 chunk 结果建立按 row key 的批量索引,消除跨 chunk 线性扫描。
|
||||
* indexRowsByChunkKey 把每个 chunk 的行索引到 rowKey→chunkKey,llm 行归属从
|
||||
* O(rows×chunks) 降为 O(1) 查找;assignLlmRowsToChunks 基于索引分配行并保留
|
||||
* 原有命中/fallback/orphan 语义;集成用例验证每个 chunk 只读一次 payload。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceRowKeyIndexTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(60000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/60000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(id);
|
||||
r.setAsin(asin);
|
||||
r.setCountry(country);
|
||||
return r;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7004L);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
/** 构造 rowsByChunk:chunkStorageKey(scopeHash, chunkIndex) → rowKey 行表。 */
|
||||
private static Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunkOf(String scopeHash, Integer chunkIndex, List<SimilarAsinResultRowDto> rows) {
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> map = new LinkedHashMap<>();
|
||||
Map<String, SimilarAsinResultRowDto> byKey = new LinkedHashMap<>();
|
||||
for (SimilarAsinResultRowDto row : rows) {
|
||||
byKey.put(row.getRowToken(), row);
|
||||
}
|
||||
map.put(scopeHash + ":" + chunkIndex, byKey);
|
||||
return map;
|
||||
}
|
||||
|
||||
private static List<String> assignedRowKeys(Map<String, Map<String, SimilarAsinResultRowDto>> merged) {
|
||||
List<String> keys = new ArrayList<>();
|
||||
for (Map<String, SimilarAsinResultRowDto> rows : merged.values()) {
|
||||
for (String key : rows.keySet()) {
|
||||
keys.add(key);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_normal_default_path() throws Exception {
|
||||
// 正常输入:2 个 chunk 各含行,llm 回传行按 rowKey 命中各自 chunk;
|
||||
// 每个 chunk 的 payload 只被读取一次(索引建立),消除跨 chunk 线性扫描。
|
||||
List<TaskChunkEntity> chunks = List.of(
|
||||
chunk(1L, "hashA", 1, "ptr:chunk-A"),
|
||||
chunk(2L, "hashB", 2, "ptr:chunk-B"));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国"))));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r3", "3", "B0A0000003", "美国"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
|
||||
AtomicInteger selectOneRound = new AtomicInteger(0);
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
|
||||
int i = selectOneRound.getAndIncrement();
|
||||
return chunks.get(Math.min(i, chunks.size() - 1));
|
||||
});
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"), row("r3", "3", "B0A0000003", "美国"));
|
||||
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
merge.invoke(service, task, null, null, llmRows, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString());
|
||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, times(2)).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_normal_multiple_items() {
|
||||
// 批量场景:3 个 chunk 各 3 行,9 个 llm 回传行全部命中且顺序稳定,无 orphan
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
for (int c = 0; c < 3; c++) {
|
||||
rowsByChunk.putAll(rowsByChunkOf("hash" + c, c + 1,
|
||||
List.of(row("c" + c + "r1", "1", "B0B" + c + "000001", "英国"),
|
||||
row("c" + c + "r2", "2", "B0B" + c + "000002", "英国"),
|
||||
row("c" + c + "r3", "3", "B0B" + c + "000003", "美国"))));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int c = 0; c < 3; c++) {
|
||||
for (int r = 1; r <= 3; r++) {
|
||||
llmRows.add(row("c" + c + "r" + r, String.valueOf(r), "B0B" + c + "00000" + r, r == 3 ? "美国" : "英国"));
|
||||
}
|
||||
}
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, orphans);
|
||||
assertEquals(3, merged.size());
|
||||
assertEquals(9, assignedRowKeys(merged).size());
|
||||
assertTrue(orphans.isEmpty(), "全部命中,不应产生 orphan");
|
||||
for (Map<String, SimilarAsinResultRowDto> rows : merged.values()) {
|
||||
assertEquals(3, rows.size());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_normal_repeated_operation_is_idempotent() {
|
||||
// 重复执行同一输入:结果完全一致,不产生重复记录
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国")));
|
||||
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"));
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> first = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> second = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
|
||||
assertEquals(assignedRowKeys(first), assignedRowKeys(second));
|
||||
assertEquals(first.size(), second.size());
|
||||
for (Map.Entry<String, Map<String, SimilarAsinResultRowDto>> entry : first.entrySet()) {
|
||||
assertEquals(entry.getValue().keySet(), second.get(entry.getKey()).keySet());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_boundary_empty_input() {
|
||||
// 空输入:null/空 rowsByChunk 与 llmRows 均安全返回空结果,不创建无效资源
|
||||
assertNotNull(service.indexRowsByChunkKey(null));
|
||||
assertTrue(service.indexRowsByChunkKey(null).isEmpty());
|
||||
assertTrue(service.indexRowsByChunkKey(Map.of()).isEmpty());
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> emptyAssign = service.assignLlmRowsToChunks(
|
||||
Map.of(), List.of(), Map.of(), null, null, new ArrayList<>());
|
||||
assertTrue(emptyAssign.isEmpty());
|
||||
assertTrue(service.assignLlmRowsToChunks(
|
||||
Map.of(), null, Map.of(), null, null, new ArrayList<>()).isEmpty());
|
||||
// 无可匹配行(rowKey 不存在于任何 chunk)→ 进 orphan 兜底,不产生 merge
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1, List.of(row("r1", "1", "B0A0000001", "英国")));
|
||||
List<SimilarAsinResultRowDto> blankRow = List.of(row("", "", "", ""));
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, blankRow, index, null, null, orphans);
|
||||
assertTrue(assignedRowKeys(merged).isEmpty());
|
||||
assertEquals(1, orphans.size(), "全空行生成 legacy key :::: 不命中任何 chunk,按既有语义进 orphan");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_boundary_single_item() {
|
||||
// 单 chunk 单行:不依赖批量路径,命中正确
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1,
|
||||
List.of(row("r1", "1", "B0A0000001", "英国")));
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, List.of(row("r1", "1", "B0A0000001", "英国")), index, null, null, orphans);
|
||||
assertEquals(1, merged.size());
|
||||
assertEquals(List.of("r1"), assignedRowKeys(merged));
|
||||
assertTrue(orphans.isEmpty());
|
||||
// 索引也只含该行
|
||||
assertEquals(1, index.size());
|
||||
assertEquals("hashA:1", index.get("r1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_boundary_limit_and_overflow() {
|
||||
// 大批量:1000 行索引 + 500 个 llm 回传行全部命中,行不丢、无 orphan
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
Map<String, SimilarAsinResultRowDto> bigChunk = new LinkedHashMap<>();
|
||||
for (int i = 1; i <= 1000; i++) {
|
||||
bigChunk.put("r" + String.format("%04d", i), row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
|
||||
}
|
||||
rowsByChunk.put("hashBig:1", bigChunk);
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
assertEquals(1000, index.size());
|
||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
||||
for (int i = 1; i <= 500; i++) {
|
||||
llmRows.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
|
||||
}
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, llmRows, index, null, null, orphans);
|
||||
assertEquals(1, merged.size());
|
||||
assertEquals(500, assignedRowKeys(merged).size());
|
||||
assertTrue(orphans.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_invalid_input_rejected() {
|
||||
// 同一 rowKey 出现在多个 chunk:索引保留第一个 chunk(putIfAbsent),行为确定
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
rowsByChunk.putAll(rowsByChunkOf("hashA", 1, List.of(row("dup", "1", "B0A0000001", "英国"))));
|
||||
rowsByChunk.putAll(rowsByChunkOf("hashB", 2, List.of(row("dup", "1", "B0A0000001", "英国"))));
|
||||
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
|
||||
assertEquals("hashA:1", index.get("dup"), "重复 rowKey 应保留第一个 chunk");
|
||||
// fallback 缺失:llm 行未命中且无有效 fallback → 进 orphan,不产生 merge
|
||||
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, List.of(row("ghost", "9", "B0A0000009", "英国")), Map.of(), "missingHash", 99, orphans);
|
||||
assertTrue(assignedRowKeys(merged).isEmpty());
|
||||
assertEquals(1, orphans.size());
|
||||
assertEquals("ghost", orphans.get(0).getRowToken());
|
||||
// llmRows 含 null 元素:跳过不抛异常,其余行正常分配
|
||||
List<SimilarAsinResultRowDto> withNull = new ArrayList<>();
|
||||
withNull.add(null);
|
||||
withNull.add(row("dup", "1", "B0A0000001", "英国"));
|
||||
List<SimilarAsinResultRowDto> orphans2 = new ArrayList<>();
|
||||
Map<String, String> index2 = service.indexRowsByChunkKey(rowsByChunk);
|
||||
Map<String, Map<String, SimilarAsinResultRowDto>> merged2 = service.assignLlmRowsToChunks(
|
||||
rowsByChunk, withNull, index2, null, null, orphans2);
|
||||
assertEquals(1, merged2.size());
|
||||
assertEquals(List.of("dup"), assignedRowKeys(merged2));
|
||||
assertTrue(orphans2.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_010_chunk_row_key_dependency_failure_releases_resources() throws Exception {
|
||||
// chunk payload 读取失败:抛可识别业务异常且不产生部分 merge;
|
||||
// 依赖恢复后重试成功,无残留状态
|
||||
List<TaskChunkEntity> chunks = List.of(chunk(1L, "hashA", 1, "ptr:chunk-A"));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenThrow(new IllegalStateException("rustfs down"));
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7004L);
|
||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
||||
merge.setAccessible(true);
|
||||
BusinessException ex = assertThrows(BusinessException.class, () -> {
|
||||
try {
|
||||
merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
||||
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
|
||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
// 恢复后重试成功:行合并到正确 chunk
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))));
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenReturn("stored:retry");
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunks.get(0));
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of());
|
||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
}
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceSubmitTest {
|
||||
|
||||
private static final Long TASK_ID = 21879L;
|
||||
private static final String PARSED_POINTER = "rustfs:task-parsed/similar-asin/21879/payload.json";
|
||||
private static final String CHUNK_POINTER = "rustfs:task-chunk/similar-asin/21879/chunk.json";
|
||||
private static final String STORED_CHUNK_POINTER = "\"" + CHUNK_POINTER + "\"";
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
@Mock private TransactionStatus transactionStatus;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
private final AtomicBoolean transactionActive = new AtomicBoolean();
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUpTransactionAndLock() {
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
lenient().when(taskDistributedLockService.acquire(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE),
|
||||
anyLong(),
|
||||
any(Duration.class),
|
||||
eq(10_000L)))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
transactionActive.set(true);
|
||||
return transactionStatus;
|
||||
});
|
||||
lenient().doAnswer(invocation -> {
|
||||
transactionActive.set(false);
|
||||
return null;
|
||||
}).when(transactionManager).commit(transactionStatus);
|
||||
lenient().doAnswer(invocation -> {
|
||||
transactionActive.set(false);
|
||||
return null;
|
||||
}).when(transactionManager).rollback(transactionStatus);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdownExecutors() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
@Test
|
||||
void doneCallbackReadsPayloadOnlyBeforeShortTransaction() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
configureNewChunkAndScope();
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
List<Boolean> storageCallTransactionStates = new ArrayList<>();
|
||||
when(transientPayloadStorageService.resolvePayload(eq(PARSED_POINTER), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storageCallTransactionStates.add(transactionActive.get());
|
||||
return parsedPayloadJson(List.of(new SimilarAsinParsedRowVo(), new SimilarAsinParsedRowVo()));
|
||||
});
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
storageCallTransactionStates.add(transactionActive.get());
|
||||
return STORED_CHUNK_POINTER;
|
||||
});
|
||||
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(false);
|
||||
doAnswer(invocation -> {
|
||||
assertTrue(transactionActive.get());
|
||||
FileResultEntity result = invocation.getArgument(0);
|
||||
result.setId(501L);
|
||||
return 1;
|
||||
}).when(fileResultMapper).insert(any(FileResultEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
storageCallTransactionStates.add(transactionActive.get());
|
||||
return null;
|
||||
}).when(taskCacheService).deleteTaskCache(TASK_ID);
|
||||
|
||||
service.submitResult(TASK_ID, request(true));
|
||||
|
||||
assertFalse(storageCallTransactionStates.isEmpty());
|
||||
assertTrue(storageCallTransactionStates.stream().noneMatch(Boolean::booleanValue));
|
||||
verify(transientPayloadStorageService).resolvePayload(eq(PARSED_POINTER), anyString());
|
||||
ArgumentCaptor<FileResultEntity> resultCaptor = ArgumentCaptor.forClass(FileResultEntity.class);
|
||||
verify(fileResultMapper).insert(resultCaptor.capture());
|
||||
assertEquals(2, resultCaptor.getValue().getRowCount());
|
||||
assertEquals("germany.xlsx", resultCaptor.getValue().getSourceFilename());
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownCommitOutcomeDoesNotDeletePossiblyCommittedChunk() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
configureNewChunkAndScope();
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(0L);
|
||||
configureChunkStore(false);
|
||||
doAnswer(invocation -> {
|
||||
transactionActive.set(false);
|
||||
throw new IllegalStateException("commit ACK lost");
|
||||
}).when(transactionManager).commit(transactionStatus);
|
||||
|
||||
IllegalStateException thrown = assertThrows(IllegalStateException.class,
|
||||
() -> service.submitResult(TASK_ID, request(false)));
|
||||
|
||||
assertEquals("commit ACK lost", thrown.getMessage());
|
||||
verify(transientPayloadStorageService, never()).extractPointer(anyString());
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
verify(taskCacheService, never()).touchTaskHeartbeat(TASK_ID);
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateAlreadyPersistedChunkDoesNotRunPayloadCleanup() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk("\"rustfs:winner\""));
|
||||
configureScopeStorage(null);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
|
||||
verify(transientPayloadStorageService, never()).extractPointer(anyString());
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void localFallbackOwnerFailureRollsBackChunkAndKeepsCandidate() throws Exception {
|
||||
FileTaskEntity task = runningTask(null);
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
configureNewChunkAndScope();
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
configureChunkStore(true);
|
||||
when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(0);
|
||||
|
||||
IllegalStateException thrown = assertThrows(IllegalStateException.class,
|
||||
() -> service.submitResult(TASK_ID, request(false)));
|
||||
|
||||
assertTrue(thrown.getMessage().contains("Failed to bind local fallback task owner"));
|
||||
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||
verify(transactionManager).rollback(transactionStatus);
|
||||
verify(transactionManager, never()).commit(transactionStatus);
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateLocalCandidateIsNotBoundAndReferencedPayloadIsKept() throws Exception {
|
||||
FileTaskEntity task = runningTask(null);
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
TaskChunkEntity winner = chunk(STORED_CHUNK_POINTER);
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(null, winner);
|
||||
configureScopeStorage(null);
|
||||
configureChunkStore(true);
|
||||
when(transientPayloadStorageService.extractPointer(STORED_CHUNK_POINTER)).thenReturn(CHUNK_POINTER);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
|
||||
verify(taskChunkMapper, never()).insert(any(TaskChunkEntity.class));
|
||||
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(STORED_CHUNK_POINTER);
|
||||
}
|
||||
|
||||
@Test
|
||||
void repeatedNonFinalCallbackCannotClearCompletedScope() throws Exception {
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
TaskChunkEntity existing = chunk("\"rustfs:winner\"");
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(existing);
|
||||
TaskScopeStateEntity scope = scope(1);
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
|
||||
assertEquals(1, scope.getCompleted());
|
||||
verify(taskScopeStateMapper).updateById(scope);
|
||||
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(
|
||||
anyString(), anyLong(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
private String parsedPayloadJson(List<SimilarAsinParsedRowVo> rows) throws Exception {
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setAllItems(rows);
|
||||
payload.setItems(rows);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey("uploads/germany.xlsx");
|
||||
sourceFile.setOriginalFilename("germany.xlsx");
|
||||
payload.setSourceFiles(List.of(sourceFile));
|
||||
return objectMapper.writeValueAsString(payload);
|
||||
}
|
||||
|
||||
private void configureNewChunkAndScope() {
|
||||
AtomicReference<TaskChunkEntity> chunkRef = new AtomicReference<>();
|
||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> chunkRef.get());
|
||||
doAnswer(invocation -> {
|
||||
TaskChunkEntity chunk = invocation.getArgument(0);
|
||||
chunk.setId(301L);
|
||||
chunkRef.set(chunk);
|
||||
return 1;
|
||||
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||
configureScopeStorage(null);
|
||||
}
|
||||
|
||||
private void configureScopeStorage(TaskScopeStateEntity initial) {
|
||||
AtomicReference<TaskScopeStateEntity> scopeRef = new AtomicReference<>(initial);
|
||||
when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> scopeRef.get());
|
||||
doAnswer(invocation -> {
|
||||
TaskScopeStateEntity scope = invocation.getArgument(0);
|
||||
scope.setId(401L);
|
||||
scopeRef.set(scope);
|
||||
return 1;
|
||||
}).when(taskScopeStateMapper).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
private void configureChunkStore(boolean localFallback) {
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
|
||||
.thenReturn(STORED_CHUNK_POINTER);
|
||||
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(localFallback);
|
||||
}
|
||||
|
||||
private FileTaskEntity runningTask(String owner) throws Exception {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setUserId(7L);
|
||||
String resultJson = "{\"parsedPayloadRef\":\"" + PARSED_POINTER + "\"";
|
||||
if (owner != null) {
|
||||
resultJson += ",\"ownerInstanceId\":\"" + owner + "\"";
|
||||
}
|
||||
task.setResultJson(resultJson + "}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private SimilarAsinSubmitResultRequest request(boolean done) {
|
||||
SimilarAsinSubmitResultRequest request = new SimilarAsinSubmitResultRequest();
|
||||
request.setSubmissionId("similar-asin-21879");
|
||||
request.setChunkIndex(0);
|
||||
request.setChunkTotal(1);
|
||||
request.setDone(done);
|
||||
return request;
|
||||
}
|
||||
|
||||
private TaskChunkEntity chunk(String payload) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(301L);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash("existing-scope");
|
||||
chunk.setChunkIndex(0);
|
||||
chunk.setChunkTotal(1);
|
||||
chunk.setPayloadJson(payload);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity scope(int completed) {
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setId(401L);
|
||||
scope.setTaskId(TASK_ID);
|
||||
scope.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
scope.setScopeKey("similar-asin-21879");
|
||||
scope.setScopeHash("existing-scope");
|
||||
scope.setChunkTotal(1);
|
||||
scope.setCompleted(completed);
|
||||
scope.setStateJson("{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
return scope;
|
||||
}
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class SimilarAsinTaskServiceTest {
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void resultWorkbookRestoresPriceAfterCountryAndShiftsImageColumns() throws Exception {
|
||||
Field headersField = SimilarAsinTaskService.class.getDeclaredField("RESULT_HEADERS");
|
||||
headersField.setAccessible(true);
|
||||
List<String> headers = (List<String>) headersField.get(null);
|
||||
|
||||
assertEquals(List.of(
|
||||
"id", "asin", "国家", "价格", "卖家名称", "品牌", "是否有货", "相似度",
|
||||
"是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"
|
||||
), headers);
|
||||
assertEquals(12, staticIntField("IMG_COL_MAIN"));
|
||||
assertEquals(13, staticIntField("IMG_COL_PUZZLE1"));
|
||||
assertEquals(14, staticIntField("IMG_COL_PUZZLE2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultStatusUsesReturnedCozeDataAndImages() {
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(null));
|
||||
|
||||
SimilarAsinResultRowDto empty = new SimilarAsinResultRowDto();
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(empty));
|
||||
|
||||
SimilarAsinResultRowDto withImage = new SimilarAsinResultRowDto();
|
||||
withImage.setStatus("FAILED");
|
||||
withImage.setMainUrl("https://example.com/main.jpg");
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withImage));
|
||||
|
||||
SimilarAsinResultRowDto withCozeStatus = new SimilarAsinResultRowDto();
|
||||
withCozeStatus.setStatus("success");
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withCozeStatus));
|
||||
|
||||
SimilarAsinResultRowDto failedStatus = new SimilarAsinResultRowDto();
|
||||
failedStatus.setStatus("FAILED");
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(failedStatus));
|
||||
|
||||
SimilarAsinResultRowDto withVisibleResultData = new SimilarAsinResultRowDto();
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() {
|
||||
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3));
|
||||
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("FAILED", 3, 3));
|
||||
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("RUNNING", 3, 3));
|
||||
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3));
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedResultRowsWithBlankIsConformEnableCategoryRetry() {
|
||||
SimilarAsinParsedRowVo blankCategory = new SimilarAsinParsedRowVo();
|
||||
blankCategory.setValues(new LinkedHashMap<>());
|
||||
blankCategory.getValues().put("status", "FAILED");
|
||||
blankCategory.getValues().put("is_conform", "");
|
||||
|
||||
assertTrue(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
true));
|
||||
|
||||
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
false));
|
||||
|
||||
blankCategory.getValues().put("is_conform", "符合");
|
||||
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
true));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstPassImageResultWorkbookKeepsAllRowsForSecondParse() {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setValues(new LinkedHashMap<>());
|
||||
row.getValues().put("状态", "成功");
|
||||
row.getValues().put("是否有货", "");
|
||||
row.getValues().put("相似度", "");
|
||||
row.getValues().put("是否符合类目", "");
|
||||
row.getValues().put("不符合理由", "");
|
||||
row.getValues().put("产品类目", "");
|
||||
|
||||
assertTrue(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
|
||||
row.getValues().put("是否符合类目", "符合");
|
||||
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
|
||||
row.getValues().put("是否符合类目", "");
|
||||
row.getValues().put("状态", "失败");
|
||||
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
}
|
||||
|
||||
private int staticIntField(String name) throws Exception {
|
||||
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
return field.getInt(null);
|
||||
}
|
||||
}
|
||||
+269
@@ -0,0 +1,269 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 8:WorkbookFactory 输入解析改为受控读取。
|
||||
* 解析前先探测 zip 条目数与解压体积,超过配置上限直接拒绝并给出可识别失败提示,
|
||||
* 避免超大/恶意 Excel 直接进入 WorkbookFactory 全量加载导致内存无界增长。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceWorkbookControlTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(50000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-workbook-ctrl-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0WBK%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("workbook.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_default_path() throws Exception {
|
||||
// 正常 xlsx:受控读取通过探测,解析成功且行数不丢失
|
||||
File workbook = buildWorkbook(100);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-default.xlsx");
|
||||
assertEquals(100, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getTotalRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_multiple_items() throws Exception {
|
||||
// 多文件批量:每个文件都走受控读取,汇总不丢行
|
||||
File workbookA = buildWorkbook(25);
|
||||
File workbookB = buildWorkbook(35);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-a.xlsx")).thenReturn(workbookA);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-b.xlsx")).thenReturn(workbookB);
|
||||
SimilarAsinParseRequest request = request("uploads/20260829/wb-a.xlsx");
|
||||
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
|
||||
sourceB.setFileKey("uploads/20260829/wb-b.xlsx");
|
||||
sourceB.setOriginalFilename("workbook-b.xlsx");
|
||||
request.setFiles(List.of(request.getFiles().get(0), sourceB));
|
||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
||||
assertEquals(60, vo.getAcceptedRows());
|
||||
assertEquals(60, vo.getTotalRows());
|
||||
assertNotNull(vo.getTaskId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复解析同一文件:结果一致,不产生重复状态
|
||||
File workbook = buildWorkbook(50);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/wb-idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/wb-idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_empty_input() throws Exception {
|
||||
// 空文件(只有表头无数据行):抛业务异常,不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-empty.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-empty.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
|
||||
// 文件不存在:抛业务异常
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-missing.xlsx")).thenReturn(null);
|
||||
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/wb-missing.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_single_item() throws Exception {
|
||||
// 单行文件:不依赖批量路径,结果正确
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_limit_and_overflow() throws Exception {
|
||||
// 解压体积超限:受控探测阶段拒绝,失败提示可识别,不发生无界内存增长
|
||||
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(1024L);
|
||||
File workbook = buildWorkbook(5);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-huge.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-huge.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && (ex.getMessage().contains("解压") || ex.getMessage().contains("大小")),
|
||||
"超大 Excel 失败提示必须可识别,实际: " + ex.getMessage());
|
||||
// 条目数超限:同样在探测阶段拒绝
|
||||
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
when(properties.getMaxWorkbookZipEntries()).thenReturn(2);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-entries.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex2 = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-entries.xlsx"));
|
||||
assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("条目"),
|
||||
"超条目数失败提示必须可识别,实际: " + ex2.getMessage());
|
||||
// 恢复默认配置:同一文件解析成功(探测不残留状态)
|
||||
when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered.xlsx");
|
||||
assertEquals(5, vo.getAcceptedRows());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_invalid_input_rejected() throws Exception {
|
||||
// 非 xlsx 内容(文本文件):WorkbookFactory 打开失败,抛项目约定异常
|
||||
File fake = Files.createTempFile("similar-asin-not-excel-", ".xlsx").toFile();
|
||||
Files.write(fake.toPath(), "this is not an excel file".getBytes(StandardCharsets.UTF_8));
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fake.xlsx")).thenReturn(fake);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(fake, "uploads/20260829/wb-fake.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"),
|
||||
"损坏文件失败提示必须可识别,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_dependency_failure_releases_resources() throws Exception {
|
||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,受控读取无残留
|
||||
File workbook = buildWorkbook(30);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/50001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/wb-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered2.xlsx");
|
||||
assertEquals(30, vo.getAcceptedRows());
|
||||
// 默认配置值处于有效区间
|
||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
||||
assertNotNull(defaults.getMaxWorkbookZipEntries());
|
||||
assertNotNull(defaults.getMaxWorkbookUncompressedBytes());
|
||||
assertTrue(defaults.getMaxWorkbookZipEntries() >= 1000, "默认条目上限至少 1000");
|
||||
assertTrue(defaults.getMaxWorkbookUncompressedBytes() >= 100L * 1024L * 1024L, "默认解压上限至少 100MB");
|
||||
}
|
||||
}
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.CreationHelper;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.util.Date;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 84:行解析边界测试。
|
||||
* 超长字段(按现状截断 2000)、畸形 ASIN 原样透传、BOM 剥离、混合编码(全角空格折叠)、
|
||||
* 数值/日期/公式单元格转文本、大表(5000 行)性能上界。
|
||||
*/
|
||||
class SimilarAsinExcelParserBoundaryTest {
|
||||
|
||||
private final SimilarAsinExcelParser parser = new SimilarAsinExcelParser();
|
||||
|
||||
private static final String[] HEADERS = {"id", "asin", "国家", "sku", "价格", "url", "标题"};
|
||||
|
||||
private File toTempFile(XSSFWorkbook wb) throws Exception {
|
||||
File file = File.createTempFile("similar-asin-boundary-", ".xlsx");
|
||||
file.deleteOnExit();
|
||||
try (FileOutputStream out = new FileOutputStream(file)) {
|
||||
wb.write(out);
|
||||
}
|
||||
wb.close();
|
||||
return file;
|
||||
}
|
||||
|
||||
private XSSFWorkbook emptyWorkbook() {
|
||||
XSSFWorkbook wb = new XSSFWorkbook();
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < HEADERS.length; i++) {
|
||||
header.createCell(i).setCellValue(HEADERS[i]);
|
||||
}
|
||||
return wb;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_long_field() throws Exception {
|
||||
XSSFWorkbook wb = emptyWorkbook();
|
||||
Sheet sheet = wb.getSheetAt(0);
|
||||
Row row = sheet.createRow(1);
|
||||
row.createCell(0).setCellValue("long-1");
|
||||
row.createCell(1).setCellValue("B01LONG");
|
||||
row.createCell(2).setCellValue("US");
|
||||
String longTitle = "A".repeat(2500);
|
||||
row.createCell(6).setCellValue(longTitle);
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
String title = parsed.rows().get(0).title();
|
||||
assertEquals(2000, title.length(), "超长字段按现状截断到 2000");
|
||||
assertEquals("A".repeat(2000), title);
|
||||
assertEquals("A".repeat(2500).substring(0, 2000), title);
|
||||
assertEquals(2000, parsed.rows().get(0).values().get("标题").length(), "values 映射同样截断");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_malformed_asin() throws Exception {
|
||||
XSSFWorkbook wb = emptyWorkbook();
|
||||
Sheet sheet = wb.getSheetAt(0);
|
||||
Row row = sheet.createRow(1);
|
||||
row.createCell(0).setCellValue("bad-1");
|
||||
row.createCell(1).setCellValue("b01@# $%^&*()");
|
||||
row.createCell(2).setCellValue("US");
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals("B01@# $%^&*()", parsed.rows().get(0).asin(), "畸形 ASIN 原样透传(仅大小写归一)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_bom() throws Exception {
|
||||
XSSFWorkbook wb = new XSSFWorkbook();
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
Row row = sheet.createRow(1);
|
||||
row.createCell(0).setCellValue("bom-1");
|
||||
row.createCell(1).setCellValue("B01BOM");
|
||||
row.createCell(2).setCellValue("US");
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals("id", parsed.headers().get(0), "BOM 头从表头剥离");
|
||||
assertEquals("B01BOM", parsed.rows().get(0).asin(), "BOM 头从单元格值剥离");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_mixed_encoding() throws Exception {
|
||||
XSSFWorkbook wb = emptyWorkbook();
|
||||
Sheet sheet = wb.getSheetAt(0);
|
||||
Row row = sheet.createRow(1);
|
||||
row.createCell(0).setCellValue("mix-1");
|
||||
row.createCell(1).setCellValue("B01MIX");
|
||||
row.createCell(2).setCellValue("US");
|
||||
row.createCell(6).setCellValue("中文 标题 ABC DEF GH");
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals("中文 标题 ABC DEF GH".replace(" GH", " GH"), parsed.rows().get(0).title(),
|
||||
"混合编码单元格保留内容");
|
||||
String title = parsed.rows().get(0).title();
|
||||
assertTrue(title.startsWith("中文 标题 ABC"), "中文+拉丁混排保留, actual=" + title);
|
||||
assertEquals("中文 标题 ABC DEF GH", title, "全角空格转半角并折叠连续空白");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_numeric_cell() throws Exception {
|
||||
XSSFWorkbook wb = emptyWorkbook();
|
||||
Sheet sheet = wb.getSheetAt(0);
|
||||
Row row = sheet.createRow(1);
|
||||
row.createCell(0).setCellValue(1234.5);
|
||||
row.createCell(1).setCellValue("B01NUM");
|
||||
row.createCell(2).setCellValue("US");
|
||||
row.createCell(4).setCellValue(19.9);
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals("1234.5", parsed.rows().get(0).id(), "数值单元格转文本");
|
||||
assertEquals("19.9", parsed.rows().get(0).price(), "价格列数值转文本");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_date_cell() throws Exception {
|
||||
XSSFWorkbook wb = emptyWorkbook();
|
||||
Sheet sheet = wb.getSheetAt(0);
|
||||
CreationHelper helper = wb.getCreationHelper();
|
||||
CellStyle dateStyle = wb.createCellStyle();
|
||||
dateStyle.setDataFormat(helper.createDataFormat().getFormat("yyyy-MM-dd"));
|
||||
Row row = sheet.createRow(1);
|
||||
row.createCell(0).setCellValue("date-1");
|
||||
row.createCell(1).setCellValue("B01DATE");
|
||||
row.createCell(2).setCellValue("US");
|
||||
row.createCell(4).setCellValue(new Date(1767225600000L)); // 2026-01-02 UTC
|
||||
row.getCell(4).setCellStyle(dateStyle);
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals("2026-01-01", parsed.rows().get(0).price(), "日期单元格按格式转文本(POI UTC 渲染)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_formula_cell() throws Exception {
|
||||
XSSFWorkbook wb = emptyWorkbook();
|
||||
Sheet sheet = wb.getSheetAt(0);
|
||||
Row row = sheet.createRow(1);
|
||||
row.createCell(0).setCellValue("formula-1");
|
||||
row.createCell(1).setCellValue("B01FML");
|
||||
row.createCell(2).setCellValue("US");
|
||||
row.createCell(4).setCellFormula("19.9*2");
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
// 现状语义:DataFormatter 对无缓存值的公式返回公式串原样(解析器不做求值)
|
||||
assertEquals("19.9*2", parsed.rows().get(0).price(), "公式单元格取缓存值(无缓存值则保留公式串)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_large_sheet() throws Exception {
|
||||
XSSFWorkbook wb = emptyWorkbook();
|
||||
Sheet sheet = wb.getSheetAt(0);
|
||||
for (int r = 1; r <= 5000; r++) {
|
||||
Row row = sheet.createRow(r);
|
||||
row.createCell(0).setCellValue("row-" + r);
|
||||
row.createCell(1).setCellValue("B0" + String.format("%04d", r));
|
||||
row.createCell(2).setCellValue(r % 2 == 0 ? "US" : "DE");
|
||||
row.createCell(3).setCellValue("SKU-" + r);
|
||||
row.createCell(4).setCellValue(10.0 + r);
|
||||
row.createCell(5).setCellValue("http://img/" + r + ".jpg");
|
||||
row.createCell(6).setCellValue("title " + r);
|
||||
}
|
||||
File file = toTempFile(wb);
|
||||
|
||||
long started = System.nanoTime();
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
long elapsedMs = (System.nanoTime() - started) / 1_000_000;
|
||||
|
||||
assertEquals(5000, parsed.rows().size(), "5000 行全部解析");
|
||||
assertEquals("row-5000", parsed.rows().get(4999).id());
|
||||
assertEquals("B05000", parsed.rows().get(4999).asin(), "大写归一后原样透传");
|
||||
assertNotNull(parsed.rows().get(4999).values());
|
||||
assertTrue(elapsedMs < 10_000, "5000 行解析耗时上界 10s, actual=" + elapsedMs + "ms");
|
||||
}
|
||||
}
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 83:SimilarAsinExcelParser 行解析器。
|
||||
* POI 读 Excel 行 → 中间行对象(原始单元格值);列序映射、空行跳过、空输入/坏文件异常、流不被篡改。
|
||||
*/
|
||||
class SimilarAsinExcelParserTest {
|
||||
|
||||
private final SimilarAsinExcelParser parser = new SimilarAsinExcelParser();
|
||||
|
||||
private XSSFWorkbook workbook(String[] headers, String[][] rows) {
|
||||
XSSFWorkbook wb = new XSSFWorkbook();
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < headers.length; i++) {
|
||||
header.createCell(i).setCellValue(headers[i]);
|
||||
}
|
||||
for (int r = 0; r < rows.length; r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
for (int c = 0; c < rows[r].length; c++) {
|
||||
row.createCell(c).setCellValue(rows[r][c]);
|
||||
}
|
||||
}
|
||||
return wb;
|
||||
}
|
||||
|
||||
private File toTempFile(XSSFWorkbook wb) throws Exception {
|
||||
File file = File.createTempFile("similar-asin-parser-", ".xlsx");
|
||||
file.deleteOnExit();
|
||||
try (FileOutputStream out = new FileOutputStream(file)) {
|
||||
wb.write(out);
|
||||
}
|
||||
wb.close();
|
||||
return file;
|
||||
}
|
||||
|
||||
private byte[] toBytes(XSSFWorkbook wb) throws Exception {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
wb.write(out);
|
||||
wb.close();
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private static final String[] HEADERS = {"id", "asin", "国家", "sku", "价格", "url", "标题"};
|
||||
|
||||
@Test
|
||||
void test_parse_normal_rows() throws Exception {
|
||||
File file = toTempFile(workbook(HEADERS, new String[][]{
|
||||
{"row-1", "B01ABC", "US", "SKU-1", "19.9", "http://img/1.jpg", "title one"},
|
||||
{"row-2", "B02DEF", "DE", "SKU-2", "29.9", "http://img/2.jpg", "title two"}
|
||||
}));
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals(2, parsed.rows().size());
|
||||
SimilarAsinExcelParser.SimilarAsinExcelRow first = parsed.rows().get(0);
|
||||
assertEquals("row-1", first.id());
|
||||
assertEquals("B01ABC", first.asin());
|
||||
assertEquals("US", first.country());
|
||||
assertEquals("SKU-1", first.sku());
|
||||
assertEquals("19.9", first.price());
|
||||
assertEquals("http://img/1.jpg", first.url());
|
||||
assertEquals("title one", first.title());
|
||||
assertEquals(2, first.rowIndex());
|
||||
SimilarAsinExcelParser.SimilarAsinExcelRow second = parsed.rows().get(1);
|
||||
assertEquals("row-2", second.id());
|
||||
assertEquals("DE", second.country());
|
||||
assertEquals(3, second.rowIndex());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_single_row() throws Exception {
|
||||
File file = toTempFile(workbook(HEADERS, new String[][]{{"only-1", "B00SINGLE", "US", "", "", "", "single"}}));
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals(1, parsed.rows().size());
|
||||
SimilarAsinExcelParser.SimilarAsinExcelRow row = parsed.rows().get(0);
|
||||
assertEquals("only-1", row.id());
|
||||
assertEquals("B00SINGLE", row.asin());
|
||||
assertEquals("US", row.country());
|
||||
assertEquals(2, row.rowIndex());
|
||||
assertNotNull(row.values());
|
||||
assertEquals(HEADERS.length, row.values().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_columns_mapping() throws Exception {
|
||||
String[] scrambled = {"价格", "asin", "标题", "id", "url", "国家"};
|
||||
File file = toTempFile(workbook(scrambled, new String[][]{{"100", "B01MAP", "mapped title", "row-1", "http://img/map.jpg", "US"}}));
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
SimilarAsinExcelParser.SimilarAsinExcelRow row = parsed.rows().get(0);
|
||||
assertEquals("row-1", row.id());
|
||||
assertEquals("B01MAP", row.asin());
|
||||
assertEquals("US", row.country());
|
||||
assertEquals("100", row.price());
|
||||
assertEquals("http://img/map.jpg", row.url());
|
||||
assertEquals("mapped title", row.title());
|
||||
assertEquals("", row.sku());
|
||||
assertEquals(scrambled.length, row.values().size());
|
||||
assertEquals("row-1", row.values().get("id"));
|
||||
assertEquals("B01MAP", row.values().get("asin"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_empty_sheet() throws Exception {
|
||||
File file = toTempFile(workbook(HEADERS, new String[0][]));
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertTrue(parsed.rows().isEmpty());
|
||||
assertEquals(HEADERS.length, parsed.headers().size());
|
||||
assertEquals("id", parsed.headers().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_empty_rows_skipped() throws Exception {
|
||||
XSSFWorkbook wb = new XSSFWorkbook();
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
Row data1 = sheet.createRow(1);
|
||||
data1.createCell(0).setCellValue("keep-1");
|
||||
data1.createCell(1).setCellValue("B01KEEP");
|
||||
data1.createCell(2).setCellValue("US");
|
||||
Row blank = sheet.createRow(2);
|
||||
blank.createCell(0).setCellValue("");
|
||||
blank.createCell(1).setCellValue("");
|
||||
blank.createCell(2).setCellValue("");
|
||||
// 第 4 行(索引 3)故意不创建 → getRow 返回 null
|
||||
Row data2 = sheet.createRow(4);
|
||||
data2.createCell(0).setCellValue("keep-2");
|
||||
data2.createCell(1).setCellValue("B02KEEP");
|
||||
data2.createCell(2).setCellValue("DE");
|
||||
File file = toTempFile(wb);
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
|
||||
|
||||
assertEquals(2, parsed.rows().size());
|
||||
assertEquals(2, parsed.rows().get(0).rowIndex());
|
||||
assertEquals("keep-1", parsed.rows().get(0).id());
|
||||
assertEquals(5, parsed.rows().get(1).rowIndex());
|
||||
assertEquals("keep-2", parsed.rows().get(1).id());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_null_input_throws() {
|
||||
assertThrows(IllegalArgumentException.class, () -> parser.parse((InputStream) null));
|
||||
assertThrows(IllegalArgumentException.class, () -> parser.parse((File) null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_unsupported_format() throws Exception {
|
||||
File garbage = File.createTempFile("similar-asin-bad-", ".xlsx");
|
||||
garbage.deleteOnExit();
|
||||
Files.write(garbage.toPath(), "this is definitely not an excel file".getBytes());
|
||||
|
||||
BusinessException ex = assertThrows(BusinessException.class, () -> parser.parse(garbage));
|
||||
|
||||
assertTrue(ex.getMessage().contains("解析 Excel 失败"), "非 Excel 文件应转业务异常, actual=" + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_parse_no_mutation() throws Exception {
|
||||
byte[] content = toBytes(workbook(HEADERS, new String[][]{
|
||||
{"row-1", "B01ABC", "US", "SKU-1", "19.9", "http://img/1.jpg", "title one"}
|
||||
}));
|
||||
|
||||
SimilarAsinExcelParser.ParsedSheet first = parser.parse(new ByteArrayInputStream(content));
|
||||
SimilarAsinExcelParser.ParsedSheet second = parser.parse(new ByteArrayInputStream(content));
|
||||
|
||||
assertEquals(first.headers(), second.headers());
|
||||
assertEquals(first.rows(), second.rows());
|
||||
assertEquals(1, first.rows().size());
|
||||
assertEquals("B01ABC", first.rows().get(0).asin());
|
||||
}
|
||||
}
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 86:SimilarAsinGroupingConverter 分组转换器。
|
||||
* 归一化行 → 分组 DTO;分组规则与现状 buildParsedGroups 一致:
|
||||
* 同 baseId 相邻块归组、groupKey 兜底、startIndex/endIndex 半开区间游标、顺序保持。
|
||||
*/
|
||||
class SimilarAsinGroupingConverterTest {
|
||||
|
||||
private static SimilarAsinParsedRowVo row(String sourceFileKey, String displayId, String groupKey,
|
||||
int rowIndex, String asin, String country) {
|
||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||
vo.setSourceFileKey(sourceFileKey);
|
||||
vo.setSourceFilename("f.xlsx");
|
||||
vo.setDisplayId(displayId);
|
||||
vo.setGroupKey(groupKey);
|
||||
vo.setRowIndex(rowIndex);
|
||||
vo.setSourceId(displayId);
|
||||
vo.setAsin(asin);
|
||||
vo.setCountry(country);
|
||||
vo.setValues(new LinkedHashMap<>());
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_normal() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(
|
||||
row("k1", "1", "k1::1@2", 2, "B01A", "US"),
|
||||
row("k1", "2", "k1::2@3", 3, "B01B", "DE"));
|
||||
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
|
||||
assertEquals(2, groups.size());
|
||||
assertEquals("1", groups.get(0).getBaseId());
|
||||
assertEquals("k1::1@2", groups.get(0).getGroupKey());
|
||||
assertEquals("1", groups.get(0).getDisplayId());
|
||||
assertEquals(1, groups.get(0).getItemCount());
|
||||
assertEquals(0, groups.get(0).getStartIndex());
|
||||
assertEquals(1, groups.get(0).getEndIndex());
|
||||
assertEquals("2", groups.get(1).getBaseId());
|
||||
assertEquals(1, groups.get(1).getStartIndex());
|
||||
assertEquals(2, groups.get(1).getEndIndex());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_sub_rows() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(
|
||||
row("k1", "2_1", "k1::2@2", 2, "B01A", "US"),
|
||||
row("k1", "2_2", "k1::2@2", 3, "B01B", "DE"),
|
||||
row("k1", "2_3", "k1::2@2", 4, "B01C", "FR"));
|
||||
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
|
||||
assertEquals(1, groups.size());
|
||||
SimilarAsinParsedGroupVo group = groups.get(0);
|
||||
assertEquals("2", group.getBaseId(), "子行 2_1/2_2/2_3 归同组");
|
||||
assertEquals("k1::2@2", group.getGroupKey());
|
||||
assertEquals("2_1", group.getDisplayId(), "组首条展示 ID 保留子行");
|
||||
assertEquals(3, group.getItemCount());
|
||||
assertEquals(0, group.getStartIndex());
|
||||
assertEquals(3, group.getEndIndex());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_single_item() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(row("k1", "7", "k1::7@2", 2, "B01S", "US"));
|
||||
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
|
||||
assertEquals(1, groups.size());
|
||||
assertEquals(1, groups.get(0).getItemCount());
|
||||
assertEquals(0, groups.get(0).getStartIndex());
|
||||
assertEquals(1, groups.get(0).getEndIndex());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_empty_input() {
|
||||
assertTrue(SimilarAsinGroupingConverter.convert(null).isEmpty());
|
||||
assertTrue(SimilarAsinGroupingConverter.convert(List.of()).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_dto_fields() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(row("uploads/k.xlsx", "5", "uploads/k.xlsx::5@2", 2, "B01D", "JP"));
|
||||
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
|
||||
SimilarAsinParsedGroupVo group = groups.get(0);
|
||||
assertEquals("uploads/k.xlsx", group.getSourceFileKey());
|
||||
assertEquals("f.xlsx", group.getSourceFilename());
|
||||
assertEquals("uploads/k.xlsx::5@2", group.getGroupKey());
|
||||
assertEquals("5", group.getBaseId());
|
||||
assertEquals("5", group.getDisplayId());
|
||||
assertEquals(1, group.getItemCount());
|
||||
assertNotNull(group.getStartIndex());
|
||||
assertNotNull(group.getEndIndex());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_dedup() {
|
||||
// 同 baseId 但不同 groupKey(如跨块重复主 ID):按 groupKey 分两个组,行不丢失
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(
|
||||
row("k1", "1", "k1::1@2", 2, "B01A", "US"),
|
||||
row("k1", "1_1", "k1::1@5", 5, "B01B", "DE"));
|
||||
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
|
||||
assertEquals(2, groups.size(), "同 baseId 不同 groupKey 分行两组");
|
||||
assertEquals("1", groups.get(0).getBaseId());
|
||||
assertEquals("1", groups.get(1).getBaseId());
|
||||
assertEquals(2, groups.get(0).getItemCount() + groups.get(1).getItemCount(), "行总数不丢");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_order() {
|
||||
List<SimilarAsinParsedRowVo> rows = new ArrayList<>();
|
||||
rows.add(row("k1", "1", "k1::1@2", 2, "B01A", "US"));
|
||||
rows.add(row("k1", "3", "k1::3@3", 3, "B01C", "FR"));
|
||||
rows.add(row("k1", "2", "k1::2@4", 4, "B01B", "DE"));
|
||||
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
|
||||
assertEquals(3, groups.size());
|
||||
assertEquals(List.of("1", "3", "2"), groups.stream().map(g -> g.getBaseId()).toList(),
|
||||
"分组顺序保持行首次出现顺序");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_group_missing_key() {
|
||||
// 缺 groupKey → 以 displayId 的 baseId 兜底;displayId 也缺 → sourceId 兜底
|
||||
List<SimilarAsinParsedRowVo> noGroupKey = List.of(row("k1", "4_1", null, 2, "B01A", "US"));
|
||||
List<SimilarAsinParsedGroupVo> groups1 = SimilarAsinGroupingConverter.convert(noGroupKey);
|
||||
assertEquals(1, groups1.size());
|
||||
assertEquals("4", groups1.get(0).getBaseId());
|
||||
assertEquals("k1::4@2", groups1.get(0).getGroupKey(), "groupKey 缺失时按 buildGroupKey 兜底");
|
||||
assertEquals("4_1", groups1.get(0).getDisplayId());
|
||||
|
||||
SimilarAsinParsedRowVo noDisplayId = new SimilarAsinParsedRowVo();
|
||||
noDisplayId.setSourceFileKey("k1");
|
||||
noDisplayId.setSourceFilename("f.xlsx");
|
||||
noDisplayId.setSourceId("9_1");
|
||||
noDisplayId.setRowIndex(5);
|
||||
List<SimilarAsinParsedGroupVo> groups2 = SimilarAsinGroupingConverter.convert(List.of(noDisplayId));
|
||||
assertEquals(1, groups2.size());
|
||||
assertEquals("", groups2.get(0).getBaseId(), "displayId 缺失时 baseId 为空串(现状 baseId(null) 语义)");
|
||||
assertEquals("9_1", groups2.get(0).getDisplayId(), "displayId 缺失时兜底 sourceId");
|
||||
assertEquals("k1::@5", groups2.get(0).getGroupKey(), "groupKey 兜底按空 baseId 拼接(现状语义)");
|
||||
}
|
||||
}
|
||||
+382
@@ -0,0 +1,382 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
|
||||
/**
|
||||
* 任务 91:SimilarAsinHistoryAssembler 历史查询组装器。
|
||||
* history 列表 VO 拼装(toHistoryItem + 进度链)抽到独立组件;只读不落库;输出与现状一致。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinHistoryAssemblerTest {
|
||||
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private SimilarAsinHistoryAssembler assembler;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
|
||||
new org.apache.ibatis.builder.MapperBuilderAssistant(
|
||||
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
|
||||
TaskChunkEntity.class);
|
||||
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
|
||||
new org.apache.ibatis.builder.MapperBuilderAssistant(
|
||||
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
|
||||
TaskScopeStateEntity.class);
|
||||
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
|
||||
new org.apache.ibatis.builder.MapperBuilderAssistant(
|
||||
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
|
||||
FileTaskEntity.class);
|
||||
assembler = new SimilarAsinHistoryAssembler(
|
||||
taskScopeStateMapper, taskChunkMapper, fileTaskMapper, taskProgressSnapshotService,
|
||||
ossStorageService, transientPayloadStorageService, objectMapper);
|
||||
lenient().when(ossStorageService.generateFreshDownloadUrl(anyString())).thenAnswer(inv -> "https://oss/" + inv.getArgument(0));
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
|
||||
lenient().when(fileTaskMapper.selectById(any())).thenReturn(null);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
}
|
||||
|
||||
private static FileResultEntity result(Long id, Long taskId, String source, String resultFileUrl, Integer success) {
|
||||
FileResultEntity row = new FileResultEntity();
|
||||
row.setId(id);
|
||||
row.setTaskId(taskId);
|
||||
row.setSourceFilename(source);
|
||||
row.setResultFilename(source == null ? null : source.replace(".xlsx", "-result.xlsx"));
|
||||
row.setResultFileUrl(resultFileUrl);
|
||||
row.setSuccess(success);
|
||||
row.setErrorMessage(null);
|
||||
row.setRowCount(12);
|
||||
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||
return row;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setTaskNo("T" + id);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setStatus(status);
|
||||
task.setCreatedAt(createdAt);
|
||||
task.setFinishedAt(createdAt == null ? null : createdAt.plusMinutes(5));
|
||||
return task;
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity job(Long id, String status) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(id);
|
||||
job.setStatus(status);
|
||||
job.setErrorMessage(null);
|
||||
return job;
|
||||
}
|
||||
|
||||
private static Map<Long, TaskFileJobEntity> jobMap() {
|
||||
return new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_history_items() {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
taskMap.put(10L, task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
|
||||
|
||||
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
|
||||
List.of(result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1)),
|
||||
taskMap, jobMap);
|
||||
|
||||
assertEquals(1, items.size());
|
||||
SimilarAsinHistoryItemVo item = items.get(0);
|
||||
assertEquals(100L, item.getResultId());
|
||||
assertEquals(10L, item.getTaskId());
|
||||
assertEquals("a.xlsx", item.getSourceFilename());
|
||||
assertEquals("a-result.xlsx", item.getResultFilename());
|
||||
assertEquals("SUCCESS", item.getTaskStatus());
|
||||
assertEquals(Boolean.TRUE, item.getSuccess());
|
||||
assertEquals(12, item.getRowCount());
|
||||
assertEquals("2026-08-01T10:00", item.getCreatedAt());
|
||||
assertEquals("2026-08-01T09:00", item.getStartedAt(), "任务开始时间复用 task.createdAt");
|
||||
assertEquals("2026-08-01T09:05", item.getFinishedAt(), "任务结束时间取 task.finishedAt");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_no_result() {
|
||||
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
|
||||
List.of(), new LinkedHashMap<>(), new LinkedHashMap<>());
|
||||
assertTrue(items.isEmpty(), "无结果返回空列表");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_file_state() {
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
|
||||
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
|
||||
jobMap.put(100L, job(7L, "RUNNING"));
|
||||
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), Map.of(10L, task(10L, "SUCCESS", null)), jobMap).get(0);
|
||||
|
||||
assertTrue(item.getFileReady(), "结果文件就绪");
|
||||
assertEquals(7L, item.getFileJobId());
|
||||
assertEquals("RUNNING", item.getFileStatus());
|
||||
assertEquals(Integer.valueOf(100), item.getFileProgressPercent(), "就绪即 100%");
|
||||
assertEquals("结果文件已生成", item.getFileProgressMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_order() {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
taskMap.put(1L, task(1L, "SUCCESS", null));
|
||||
taskMap.put(2L, task(2L, "SUCCESS", null));
|
||||
|
||||
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
|
||||
List.of(result(200L, 2L, "b.xlsx", null, 1), result(100L, 1L, "a.xlsx", null, 1)),
|
||||
taskMap, jobMap());
|
||||
|
||||
assertEquals(2, items.size());
|
||||
assertEquals(200L, items.get(0).getResultId(), "输入顺序保持");
|
||||
assertEquals(100L, items.get(1).getResultId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_null_task() {
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 1);
|
||||
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 11, 0));
|
||||
// taskMap 空 → 缺 task
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), new LinkedHashMap<>(), new LinkedHashMap<>()).get(0);
|
||||
|
||||
assertNull(item.getTaskStatus(), "缺 task 状态为空");
|
||||
assertEquals("2026-08-01T11:00", item.getStartedAt(), "缺 task 回退 result.createdAt");
|
||||
assertNull(item.getFinishedAt(), "缺 task 无结束时间");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_download_url() {
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
|
||||
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
|
||||
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), Map.of(10L, task(10L, "SUCCESS", null)), jobMap).get(0);
|
||||
|
||||
assertEquals("https://oss/result/10/a.xlsx", item.getDownloadUrl(), "下载 URL 由 oss 服务生成");
|
||||
verify(ossStorageService).generateFreshDownloadUrl("result/10/a.xlsx");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_download_url_blank_result_file() {
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 1);
|
||||
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), Map.of(10L, task(10L, "SUCCESS", null)), jobMap()).get(0);
|
||||
|
||||
assertNull(item.getDownloadUrl(), "无结果文件 URL 时下载地址为空");
|
||||
verify(ossStorageService, never()).generateFreshDownloadUrl(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_immutable_input() {
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
taskMap.put(10L, task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||
String originalTaskNo = taskMap.get(10L).getTaskNo();
|
||||
|
||||
assembler.buildHistoryItems(List.of(row), taskMap, jobMap());
|
||||
|
||||
assertEquals(originalTaskNo, taskMap.get(10L).getTaskNo(), "task 不被修改");
|
||||
assertEquals("a.xlsx", row.getSourceFilename(), "result 不被修改");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_pending_skip() {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
taskMap.put(10L, task(10L, "PENDING", null));
|
||||
taskMap.put(11L, task(11L, "RUNNING", null));
|
||||
|
||||
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
|
||||
List.of(result(100L, 10L, "a.xlsx", null, 0), result(101L, 11L, "b.xlsx", null, 0)),
|
||||
taskMap, jobMap());
|
||||
|
||||
assertEquals(1, items.size(), "PENDING 任务历史项被跳过");
|
||||
assertEquals(101L, items.get(0).getResultId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_file_error_attached() {
|
||||
TaskFileJobEntity failedJob = job(9L, "FAILED");
|
||||
failedJob.setErrorMessage("assemble boom");
|
||||
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
|
||||
jobMap.put(100L, failedJob);
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
taskMap.put(10L, task(10L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), taskMap, jobMap).get(0);
|
||||
|
||||
assertEquals("FAILED", item.getFileStatus(), "job 失败状态附带");
|
||||
assertEquals("assemble boom", item.getFileError(), "job 错误信息附带");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_consistency() {
|
||||
// 与现状 toHistoryItem 行为一致:success 字段、rowCount、error、时间兜底
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 0);
|
||||
row.setErrorMessage("python timeout");
|
||||
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
taskMap.put(10L, task(10L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 30)));
|
||||
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), taskMap, jobMap()).get(0);
|
||||
|
||||
assertEquals(Boolean.FALSE, item.getSuccess());
|
||||
assertEquals("python timeout", item.getError());
|
||||
assertEquals("FAILED", item.getTaskStatus());
|
||||
assertEquals(12, item.getRowCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_python_upload_row_fallback() {
|
||||
// scope 状态存在但无分片数 → chunk 进度 (0,0) → 回退 fileTask 载荷行数(单位"行")
|
||||
FileTaskEntity t = new FileTaskEntity();
|
||||
t.setId(10L);
|
||||
t.setResultJson("{\"parsedPayloadRef\":\"ref:payload\",\"allItems\":[]}");
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||
SimilarAsinParsedRowVo parsed = new SimilarAsinParsedRowVo();
|
||||
parsed.setDisplayId("B01X");
|
||||
parsed.setAsin("B01X");
|
||||
parsed.setCountry("US");
|
||||
parsed.setRowToken("uploads/main.xlsx::row::2");
|
||||
String payloadJson;
|
||||
try {
|
||||
payloadJson = objectMapper.writeValueAsString(
|
||||
Map.of("items", List.of(parsed), "allItems", List.of()));
|
||||
} catch (Exception ex) {
|
||||
throw new AssertionError(ex);
|
||||
}
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setTaskId(10L);
|
||||
state.setModuleType("SIMILAR_ASIN");
|
||||
state.setChunkTotal(0);
|
||||
state.setReceivedChunkCount(0);
|
||||
|
||||
when(fileTaskMapper.selectById(10L)).thenReturn(t);
|
||||
when(transientPayloadStorageService.resolvePayload("ref:payload", "read similar ASIN parsed payload failed"))
|
||||
.thenReturn(payloadJson);
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(state));
|
||||
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), Map.of(10L, task(10L, "RUNNING", null)), jobMap()).get(0);
|
||||
|
||||
assertEquals(0, item.getFileProgressCurrent(), "无已提交行");
|
||||
assertEquals(1, item.getFileProgressTotal(), "载荷总行数 1");
|
||||
assertTrue(item.getFileProgressMessage().endsWith(" 行"), "回退路径单位应为'行',实际: " + item.getFileProgressMessage());
|
||||
assertEquals(Integer.valueOf(1), item.getFileProgressPercent(), "0 行时百分比 1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_python_upload_no_task_no_chunk() {
|
||||
// task 与 chunk 都拿不到 → 0/0 行单位,不抛异常
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
|
||||
List.of(row), Map.of(10L, task(10L, "RUNNING", null)), jobMap()).get(0);
|
||||
|
||||
assertEquals(0, item.getFileProgressCurrent());
|
||||
assertEquals(1, item.getFileProgressTotal(), "0 行时 total 兜底 1");
|
||||
assertTrue(item.getFileProgressMessage().endsWith(" 行"), "无 task 无 chunk 仍按行回退,实际: " + item.getFileProgressMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_assembler_chunk_read_failure_records_error() {
|
||||
// 回退路径读 chunk 载荷失败 → 抛 BusinessException 且 last_error 留痕
|
||||
FileTaskEntity t = new FileTaskEntity();
|
||||
t.setId(10L);
|
||||
t.setResultJson("{}");
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setTaskId(10L);
|
||||
chunk.setModuleType("SIMILAR_ASIN");
|
||||
chunk.setScopeHash("s1");
|
||||
chunk.setChunkIndex(2);
|
||||
chunk.setPayloadJson("ref:missing");
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setTaskId(10L);
|
||||
state.setModuleType("SIMILAR_ASIN");
|
||||
state.setChunkTotal(0);
|
||||
state.setReceivedChunkCount(0);
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setId(55L);
|
||||
scope.setLastError("");
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||
|
||||
when(fileTaskMapper.selectById(10L)).thenReturn(t);
|
||||
when(transientPayloadStorageService.resolvePayload("ref:missing", "read similar ASIN chunk failed"))
|
||||
.thenThrow(new IllegalStateException("payload only exists on instance=server-110"));
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(state));
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
|
||||
BusinessException thrown = assertThrows(BusinessException.class,
|
||||
() -> assembler.buildHistoryItems(List.of(row), Map.of(10L, task(10L, "RUNNING", null)), jobMap()));
|
||||
assertTrue(thrown.getMessage().contains("chunk=2"), "异常附带 chunk 索引,实际: " + thrown.getMessage());
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskScopeStateEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskScopeStateMapper).update(isNull(), captor.capture());
|
||||
String written = String.join("|", captor.getValue().getParamNameValuePairs().values().stream()
|
||||
.map(String::valueOf).toList());
|
||||
assertTrue(written.contains("chunk-read-failed[2@cross-instance]"), "跨实例标记留痕,实际: " + written);
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 85:SimilarAsinRowNormalizer 字段归一化器。
|
||||
* trim/大小写/ASIN 格式统一,规则与现状 Service.normalize 完全一致(BOM 剥离、全角空格转半角、
|
||||
* 连续空白折叠为单空格),纯函数无状态,幂等。
|
||||
*/
|
||||
class SimilarAsinRowNormalizerTest {
|
||||
|
||||
@Test
|
||||
void test_trim_whitespace() {
|
||||
assertEquals("abc", SimilarAsinRowNormalizer.normalize(" abc "));
|
||||
assertEquals("abc", SimilarAsinRowNormalizer.normalize("\tabc\n"));
|
||||
assertEquals("", SimilarAsinRowNormalizer.normalize(" "));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_lowercase_asin() {
|
||||
assertEquals("B01ABC", SimilarAsinRowNormalizer.normalizeAsin("b01abc"));
|
||||
assertEquals("B01ABC", SimilarAsinRowNormalizer.normalizeAsin("b01abc "));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_mixed_case_asin() {
|
||||
assertEquals("B01ABC", SimilarAsinRowNormalizer.normalizeAsin("b01AbC"));
|
||||
assertEquals("B01MIXED1", SimilarAsinRowNormalizer.normalizeAsin(" b01MiXeD1 "));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_internal_whitespace_kept() {
|
||||
assertEquals("ab cd", SimilarAsinRowNormalizer.normalize("ab cd"), "内部单空格保留");
|
||||
assertEquals("ab cd", SimilarAsinRowNormalizer.normalize("ab cd"), "连续空白折叠为单空格(现状)");
|
||||
assertEquals("ab cd", SimilarAsinRowNormalizer.normalize("ab cd"), "全角空格转半角");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_null_safe() {
|
||||
assertEquals("", SimilarAsinRowNormalizer.normalize(null));
|
||||
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(null));
|
||||
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(" "), "空白 ASIN 归一后为空(trim 先行)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_empty_string() {
|
||||
assertEquals("", SimilarAsinRowNormalizer.normalize(""));
|
||||
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(""));
|
||||
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_special_chars_untouched() {
|
||||
assertEquals("a@#$%^&*()", SimilarAsinRowNormalizer.normalize(" a@#$%^&*() "));
|
||||
assertEquals("A@#$%^&*()", SimilarAsinRowNormalizer.normalizeAsin("a@#$%^&*()"));
|
||||
assertEquals("B01/AB-C_1", SimilarAsinRowNormalizer.normalizeAsin("b01/ab-c_1"), "连字符/下划线/斜杠保留");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_normalize_identity_after() {
|
||||
String[] samples = {"abc", " abc ", "b01AbC", "ab cd", "x y ", "a@#$%^&*()", "", " "};
|
||||
for (String sample : samples) {
|
||||
String once = SimilarAsinRowNormalizer.normalize(sample);
|
||||
assertEquals(once, SimilarAsinRowNormalizer.normalize(once),
|
||||
"normalize 幂等: input=" + sample);
|
||||
String asinOnce = SimilarAsinRowNormalizer.normalizeAsin(sample);
|
||||
assertEquals(asinOnce, SimilarAsinRowNormalizer.normalizeAsin(asinOnce),
|
||||
"normalizeAsin 幂等: input=" + sample);
|
||||
}
|
||||
assertTrue(SimilarAsinRowNormalizer.normalize(" a b ").equals(SimilarAsinRowNormalizer.normalize("a b")),
|
||||
"不同空白形态归一到同一结果");
|
||||
}
|
||||
}
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinRowValidator.RowError;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 87:SimilarAsinRowValidator 纯校验器。
|
||||
* 必填检查(id/asin/country)、ASIN 格式校验(大写字母数字)、重复检测(asin+country 键,保留首次);
|
||||
* 返回错误清单不抛异常。规则与现状 parseWorkbook/dedupeRowsByRowKey 一致:
|
||||
* 全空白行跳过、必填缺失即记错、重复仅标记后续出现、格式只拦非字母数字。
|
||||
*/
|
||||
class SimilarAsinRowValidatorTest {
|
||||
|
||||
private static SimilarAsinParsedRowVo row(String sourceId, String asin, String country) {
|
||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||
vo.setSourceFileKey("k1");
|
||||
vo.setSourceFilename("f.xlsx");
|
||||
vo.setRowIndex(2);
|
||||
vo.setSourceId(sourceId);
|
||||
vo.setDisplayId(sourceId);
|
||||
vo.setAsin(asin);
|
||||
vo.setCountry(country);
|
||||
vo.setValues(new LinkedHashMap<>());
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static List<String> columns(List<RowError> errors) {
|
||||
return errors.stream().map(RowError::column).toList();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_required_present() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(
|
||||
row("1", "B01A", "US"),
|
||||
row("2", "B01B", "DE"));
|
||||
|
||||
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertTrue(errors.isEmpty(), "必填齐全通过");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_required_missing() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(
|
||||
row("", "B01A", "US"),
|
||||
row("2", "", "DE"),
|
||||
row("3", "B01C", " "));
|
||||
|
||||
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertEquals(3, errors.size(), "缺 id / asin / country 各记一错");
|
||||
assertEquals(List.of("id", "asin", "country"), columns(errors));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_asin_format() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(
|
||||
row("1", "B0A-1B2C3D", "US"),
|
||||
row("2", "B0A 1B2C3D", "DE"));
|
||||
|
||||
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertEquals(2, errors.size(), "非字母数字 ASIN 记格式错");
|
||||
assertEquals(List.of("asin", "asin"), columns(errors));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_duplicates() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(
|
||||
row("1", "B01A", "US"),
|
||||
row("2", "b01a", "US"),
|
||||
row("3", "B01A", "DE"),
|
||||
row("4", "B01C", "US"));
|
||||
|
||||
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertEquals(1, errors.size(), "仅重复的后续出现记错");
|
||||
assertEquals("asin", errors.get(0).column());
|
||||
assertTrue(errors.get(0).message().contains("B01A"), "重复消息含 ASIN: " + errors.get(0).message());
|
||||
assertTrue(errors.get(0).message().contains("US"), "重复消息含国家: " + errors.get(0).message());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_blank_row() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(row("", "", ""));
|
||||
|
||||
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertTrue(errors.isEmpty(), "全空白行跳过(与 parse 一致)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_no_error_happy() {
|
||||
List<SimilarAsinParsedRowVo> rows = new ArrayList<>();
|
||||
rows.add(row("1_1", "B01A", "US"));
|
||||
rows.add(row("1_2", "B01B", "DE"));
|
||||
rows.add(row("2", "B01C", "FR"));
|
||||
rows.add(row("3", "B01D", "JP"));
|
||||
|
||||
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertTrue(errors.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_error_detail() {
|
||||
List<SimilarAsinParsedRowVo> rows = List.of(row("", "B01A", "US"));
|
||||
|
||||
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertEquals(1, errors.size());
|
||||
RowError error = errors.get(0);
|
||||
assertEquals(2, error.rowIndex(), "错误含 1 基行号");
|
||||
assertEquals("id", error.column());
|
||||
assertTrue(!error.message().isBlank(), "错误含消息");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_validate_null_input() {
|
||||
assertTrue(SimilarAsinRowValidator.validate(null).isEmpty());
|
||||
assertTrue(SimilarAsinRowValidator.validate(List.of()).isEmpty());
|
||||
}
|
||||
}
|
||||
+219
@@ -0,0 +1,219 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 88:SimilarAsinSheetBuilder Sheet 构造器。
|
||||
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式);输入数据 → 输出 workbook;不落库、无 IO 依赖。
|
||||
* 表头/列序/样式与现状 assembleResultWorkbook 一致:sheet 名"相似asin检测"、15 列表头、加粗表头、
|
||||
* 数据行从第 1 行、图片列 12-14 列宽、行高 409pt、值兜底与现状一致。
|
||||
*/
|
||||
class SimilarAsinSheetBuilderTest {
|
||||
|
||||
private SXSSFWorkbook lastWorkbook;
|
||||
|
||||
@AfterEach
|
||||
void closeWorkbook() throws Exception {
|
||||
if (lastWorkbook != null) {
|
||||
lastWorkbook.close();
|
||||
}
|
||||
}
|
||||
|
||||
private static SimilarAsinParsedRowVo row(String displayId, String asin, String country, String price,
|
||||
String seller, String brand) {
|
||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||
vo.setSourceFileKey("k1");
|
||||
vo.setSourceFilename("f.xlsx");
|
||||
vo.setRowIndex(2);
|
||||
vo.setSourceId(displayId);
|
||||
vo.setDisplayId(displayId);
|
||||
vo.setAsin(asin);
|
||||
vo.setCountry(country);
|
||||
vo.setPrice(price);
|
||||
LinkedHashMap<String, String> values = new LinkedHashMap<>();
|
||||
values.put("卖家名称", seller);
|
||||
values.put("品牌", brand);
|
||||
vo.setValues(values);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private Workbook build(List<SimilarAsinParsedRowVo> rows) {
|
||||
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||
lastWorkbook = workbook;
|
||||
SimilarAsinSheetBuilder.build(workbook, rows);
|
||||
return workbook;
|
||||
}
|
||||
|
||||
private static String cellString(Workbook workbook, int row, int col) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row r = sheet.getRow(row);
|
||||
if (r == null) {
|
||||
return null;
|
||||
}
|
||||
Cell c = r.getCell(col);
|
||||
return c == null ? null : c.getStringCellValue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_headers() {
|
||||
Workbook workbook = build(List.of());
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
assertNotNull(sheet, "sheet 存在");
|
||||
assertEquals("相似asin检测", sheet.getSheetName());
|
||||
Row header = sheet.getRow(0);
|
||||
assertNotNull(header);
|
||||
List<String> headers = new ArrayList<>();
|
||||
header.forEach(cell -> headers.add(cell.getStringCellValue()));
|
||||
assertEquals(List.of("id", "asin", "国家", "价格", "卖家名称", "品牌", "是否有货", "相似度",
|
||||
"是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
headers, "表头与现状 RESULT_HEADERS 一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_header_style_bold() {
|
||||
Workbook workbook = build(List.of());
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
CellStyle style = sheet.getRow(0).getCell(0).getCellStyle();
|
||||
assertTrue(workbook.getFontAt(style.getFontIndex()).getBold(), "表头加粗");
|
||||
CellStyle style2 = sheet.getRow(0).getCell(14).getCellStyle();
|
||||
assertTrue(workbook.getFontAt(style2.getFontIndex()).getBold(), "末列表头也加粗");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_data_rows() {
|
||||
Workbook workbook = build(List.of(
|
||||
row("1", "B01A", "US", "19.9", "卖家1", "品牌1"),
|
||||
row("2", "B01B", "DE", "9.9", "卖家2", "品牌2")));
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
assertEquals(3, sheet.getLastRowNum() + 1, "表头 + 2 数据行");
|
||||
assertEquals("1", cellString(workbook, 1, 0));
|
||||
assertEquals("B01A", cellString(workbook, 1, 1));
|
||||
assertEquals("US", cellString(workbook, 1, 2));
|
||||
assertEquals("19.9", cellString(workbook, 1, 3));
|
||||
assertEquals("卖家1", cellString(workbook, 1, 4));
|
||||
assertEquals("品牌1", cellString(workbook, 1, 5));
|
||||
assertEquals("2", cellString(workbook, 2, 0));
|
||||
assertEquals("B01B", cellString(workbook, 2, 1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_empty_rows() {
|
||||
Workbook workbook = build(List.of());
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
assertEquals(1, sheet.getLastRowNum() + 1, "无数据行时只有表头");
|
||||
assertNull(sheet.getRow(1), "无第 1 行");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_missing_values_fallback() {
|
||||
// displayId 缺失 → 兜底 sourceId;asin/country 空白 → 空串;values 缺失 → 空串
|
||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||
vo.setSourceFileKey("k1");
|
||||
vo.setSourceFilename("f.xlsx");
|
||||
vo.setRowIndex(2);
|
||||
vo.setSourceId("9_1");
|
||||
vo.setDisplayId("");
|
||||
vo.setAsin("");
|
||||
vo.setCountry("");
|
||||
|
||||
Workbook workbook = build(List.of(vo));
|
||||
|
||||
assertEquals("9_1", cellString(workbook, 1, 0), "displayId 空白兜底 sourceId");
|
||||
assertEquals("", cellString(workbook, 1, 1));
|
||||
assertEquals("", cellString(workbook, 1, 2));
|
||||
assertEquals("", cellString(workbook, 1, 4), "卖家名称空白");
|
||||
assertEquals("", cellString(workbook, 1, 5), "品牌空白");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_image_columns_width() {
|
||||
Workbook workbook = build(List.of());
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
assertEquals(SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256, sheet.getColumnWidth(12), "主图列宽");
|
||||
assertEquals(SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256, sheet.getColumnWidth(13), "图1列宽");
|
||||
assertEquals(SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256, sheet.getColumnWidth(14), "图2列宽");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_column_order() {
|
||||
Workbook workbook = build(List.of(row("1", "B01A", "US", "19.9", "卖家1", "品牌1")));
|
||||
|
||||
assertEquals("id", cellString(workbook, 0, 0));
|
||||
assertEquals("asin", cellString(workbook, 0, 1));
|
||||
assertEquals("国家", cellString(workbook, 0, 2));
|
||||
assertEquals("价格", cellString(workbook, 0, 3));
|
||||
assertEquals("状态", cellString(workbook, 0, 11));
|
||||
assertEquals("主图", cellString(workbook, 0, 12));
|
||||
assertEquals("阿里巴巴图片2", cellString(workbook, 0, 14));
|
||||
assertEquals("B01A", cellString(workbook, 1, 1), "asin 在第 1 列");
|
||||
assertNull(cellString(workbook, 1, 11), "状态列由调用方补写,builder 不创建");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_result_columns_blank_without_result() {
|
||||
// 结果派生列(是否有货/相似度/是否符合类目/不符合理由/产品类目/状态)来自 LLM 结果行,
|
||||
// builder 输入只有解析行 → 这些列不创建(现状无 resultRow 时调用方同样置空串)。
|
||||
Workbook workbook = build(List.of(row("1", "B01A", "US", "19.9", "卖家1", "品牌1")));
|
||||
|
||||
assertNull(cellString(workbook, 1, 6));
|
||||
assertNull(cellString(workbook, 1, 7));
|
||||
assertNull(cellString(workbook, 1, 8));
|
||||
assertNull(cellString(workbook, 1, 9));
|
||||
assertNull(cellString(workbook, 1, 10));
|
||||
assertNull(cellString(workbook, 1, 11));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_value_by_header_contains() {
|
||||
// 卖家名称/品牌从 values 映射按表头包含匹配读取(现状 readValueByHeader 语义)
|
||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||
vo.setSourceFileKey("k1");
|
||||
vo.setSourceFilename("f.xlsx");
|
||||
vo.setRowIndex(2);
|
||||
vo.setSourceId("3");
|
||||
vo.setDisplayId("3");
|
||||
vo.setAsin("B01C");
|
||||
vo.setCountry("FR");
|
||||
vo.setPrice("");
|
||||
LinkedHashMap<String, String> values = new LinkedHashMap<>();
|
||||
values.put("卖家名称:亚马逊", "卖家X");
|
||||
values.put("品牌/店铺", "品牌Y");
|
||||
vo.setValues(values);
|
||||
|
||||
Workbook workbook = build(List.of(vo));
|
||||
|
||||
assertEquals("卖家X", cellString(workbook, 1, 4), "卖家名称按包含匹配");
|
||||
assertEquals("品牌Y", cellString(workbook, 1, 5), "品牌按包含匹配");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_build_null_input() {
|
||||
Workbook workbook = build(null);
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
assertEquals(1, sheet.getLastRowNum() + 1, "null 输入不写数据行");
|
||||
}
|
||||
}
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 任务 90:抽取前后快照对比测试。
|
||||
* 夹具 Excel → 解析结果快照(golden 文件);同一夹具输出完全一致;快照变更即失败(防行为漂移)。
|
||||
* golden 文件:src/test/resources/similarasin/golden/parse-snapshot.txt(全 parse 路径)
|
||||
* src/test/resources/similarasin/golden/groups-snapshot.txt(分组路径)
|
||||
*/
|
||||
class SimilarAsinSnapshotTest {
|
||||
|
||||
private static final File GOLDEN_PARSE =
|
||||
new File("src/test/resources/similarasin/golden/parse-snapshot.txt");
|
||||
private static final File GOLDEN_GROUPS =
|
||||
new File("src/test/resources/similarasin/golden/groups-snapshot.txt");
|
||||
|
||||
private static final String[] HEADERS = {"id", "asin", "国家", "sku", "价格", "url", "标题"};
|
||||
|
||||
// ---- 夹具 ----
|
||||
|
||||
private static File workbook(String[] headers, List<String[]> rows) throws Exception {
|
||||
File file = File.createTempFile("similar-asin-snapshot-", ".xlsx");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream out = new FileOutputStream(file)) {
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < headers.length; i++) {
|
||||
header.createCell(i).setCellValue(headers[i]);
|
||||
}
|
||||
for (int r = 0; r < rows.size(); r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
for (int c = 0; c < rows.get(r).length; c++) {
|
||||
row.createCell(c).setCellValue(rows.get(r)[c]);
|
||||
}
|
||||
}
|
||||
wb.write(out);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
/** 5 有效行:2_1/2_2/2_3 同块、非法 ASIN 保留、单块多行。 */
|
||||
private static File fixtureMain() throws Exception {
|
||||
return workbook(HEADERS, List.of(
|
||||
new String[]{"2_1", "B01A", "US", "SKU-1", "19.90", "http://img/a.jpg", "title a"},
|
||||
new String[]{"2_2", "B01B", "DE", "SKU-2", "29.90", "http://img/b.jpg", "title b"},
|
||||
new String[]{"2_3", "B01C", "FR", "SKU-3", "", "http://img/c.jpg", ""},
|
||||
new String[]{"5", "B01-ABC", "UK", "SKU-4", "9.90", "http://img/d.jpg", "title d"},
|
||||
new String[]{"6", "B01D", "US", "SKU-5", "1.00", "http://img/e.jpg", "title e"}));
|
||||
}
|
||||
|
||||
/** 6 行含 5 类错误(缺 id/asin/国家、格式错、重复)+ 1 空行;解析后剩 3 有效行。 */
|
||||
private static File fixtureError() throws Exception {
|
||||
return workbook(HEADERS, List.of(
|
||||
new String[]{"", "B02A", "US"},
|
||||
new String[]{"9", "", "DE"},
|
||||
new String[]{"10", "B02B", ""},
|
||||
new String[]{"11", "B02-ABC", "FR"},
|
||||
new String[]{"12", "B02C", "FR"},
|
||||
new String[]{"13", "B02C", "FR"},
|
||||
new String[]{"", "", ""}));
|
||||
}
|
||||
|
||||
private static final List<String[]> GROUPS_FIXTURE = List.of(
|
||||
new String[]{"1", "B10A", "US"},
|
||||
new String[]{"2_1", "B11A", "DE"},
|
||||
new String[]{"2_2", "B11B", "FR"});
|
||||
|
||||
private static File fixtureGroups() throws Exception {
|
||||
return workbook(HEADERS, GROUPS_FIXTURE);
|
||||
}
|
||||
|
||||
// ---- 解析管线(与服务侧 parseWorkbookDelegated 语义一致) ----
|
||||
|
||||
private static List<SimilarAsinParsedRowVo> toRows(File file, String fileKey, String filename,
|
||||
boolean dropInvalid) throws Exception {
|
||||
SimilarAsinExcelParser.ParsedSheet parsed = new SimilarAsinExcelParser().parse(file);
|
||||
List<SimilarAsinParsedRowVo> rows = new ArrayList<>();
|
||||
String currentBlockBaseId = "";
|
||||
String currentGroupKey = "";
|
||||
for (SimilarAsinExcelParser.SimilarAsinExcelRow parsedRow : parsed.rows()) {
|
||||
String id = parsedRow.id();
|
||||
String asin = parsedRow.asin();
|
||||
String country = parsedRow.country();
|
||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (dropInvalid && (id.isBlank() || asin.isBlank() || country.isBlank())) {
|
||||
continue;
|
||||
}
|
||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||
vo.setSourceFileKey(fileKey);
|
||||
vo.setSourceFilename(filename);
|
||||
vo.setRowIndex(parsedRow.rowIndex());
|
||||
vo.setSourceId(id);
|
||||
vo.setDisplayId(id.trim());
|
||||
String rowBaseId = baseId(vo.getDisplayId());
|
||||
if (!java.util.Objects.equals(currentBlockBaseId, rowBaseId)) {
|
||||
currentBlockBaseId = rowBaseId;
|
||||
currentGroupKey = buildGroupKey(fileKey, rowBaseId, vo.getRowIndex());
|
||||
}
|
||||
vo.setGroupKey(currentGroupKey);
|
||||
vo.setRowToken(buildRowToken(fileKey, vo.getRowIndex()));
|
||||
vo.setAsin(asin);
|
||||
vo.setCountry(country);
|
||||
vo.setSku(parsedRow.sku());
|
||||
vo.setPrice(parsedRow.price());
|
||||
vo.setUrl(parsedRow.url());
|
||||
vo.setTitle(parsedRow.title());
|
||||
vo.setValues(parsedRow.values());
|
||||
rows.add(vo);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private static String normalize(String val) {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private static String baseId(String id) {
|
||||
String s = normalize(id);
|
||||
int idx = s.indexOf('_');
|
||||
return idx > 0 ? s.substring(0, idx) : s;
|
||||
}
|
||||
|
||||
private static String buildRowToken(String sourceFileKey, Integer rowIndex) {
|
||||
return normalize(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex);
|
||||
}
|
||||
|
||||
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
|
||||
return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
|
||||
}
|
||||
|
||||
// ---- 快照渲染 ----
|
||||
|
||||
private static String renderRow(SimilarAsinParsedRowVo row) {
|
||||
return " row: idx=" + row.getRowIndex()
|
||||
+ " sourceId=" + row.getSourceId()
|
||||
+ " displayId=" + row.getDisplayId()
|
||||
+ " asin=" + row.getAsin()
|
||||
+ " country=" + row.getCountry()
|
||||
+ " sku=" + row.getSku()
|
||||
+ " price=" + row.getPrice()
|
||||
+ " url=" + row.getUrl()
|
||||
+ " title=" + row.getTitle()
|
||||
+ " groupKey=" + row.getGroupKey()
|
||||
+ " rowToken=" + row.getRowToken()
|
||||
+ " values.size=" + (row.getValues() == null ? 0 : row.getValues().size());
|
||||
}
|
||||
|
||||
private static String renderGroup(SimilarAsinParsedGroupVo group) {
|
||||
return " group: fileKey=" + group.getSourceFileKey()
|
||||
+ " filename=" + group.getSourceFilename()
|
||||
+ " groupKey=" + group.getGroupKey()
|
||||
+ " baseId=" + group.getBaseId()
|
||||
+ " displayId=" + group.getDisplayId()
|
||||
+ " itemCount=" + group.getItemCount()
|
||||
+ " range=[" + group.getStartIndex() + "," + group.getEndIndex() + ")";
|
||||
}
|
||||
|
||||
private static String renderFull(List<SimilarAsinParsedRowVo> rows, List<SimilarAsinParsedGroupVo> groups) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("rows=").append(rows.size()).append('\n');
|
||||
for (SimilarAsinParsedRowVo row : rows) {
|
||||
sb.append(renderRow(row)).append('\n');
|
||||
}
|
||||
sb.append("groups=").append(groups.size()).append('\n');
|
||||
for (SimilarAsinParsedGroupVo group : groups) {
|
||||
sb.append(renderGroup(group)).append('\n');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static List<SimilarAsinParsedRowVo> mergeRows(List<List<SimilarAsinParsedRowVo>> all) {
|
||||
List<SimilarAsinParsedRowVo> merged = new ArrayList<>();
|
||||
for (List<SimilarAsinParsedRowVo> list : all) {
|
||||
merged.addAll(list);
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
private static String runAllParse() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = mergeRows(List.of(
|
||||
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx", true),
|
||||
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", true)));
|
||||
return renderFull(rows, SimilarAsinGroupingConverter.convert(rows));
|
||||
}
|
||||
|
||||
private static String runGroupsOnly() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = toRows(fixtureGroups(), "uploads/groups.xlsx", "groups.xlsx", true);
|
||||
return renderFull(rows, SimilarAsinGroupingConverter.convert(rows));
|
||||
}
|
||||
|
||||
private static String read(File file) throws Exception {
|
||||
return Files.readString(file.toPath(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
// ---- 用例 ----
|
||||
|
||||
@Test
|
||||
void test_snapshot_parse_output() throws Exception {
|
||||
assertEquals(read(GOLDEN_PARSE), runAllParse(), "解析输出快照与 golden 一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_snapshot_groups() throws Exception {
|
||||
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "分组快照与 golden 一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_snapshot_dtos() throws Exception {
|
||||
List<SimilarAsinParsedRowVo> rows = mergeRows(List.of(
|
||||
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx", true),
|
||||
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", true)));
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
|
||||
assertEquals(8, rows.size(), "有效行 5 + 3");
|
||||
assertEquals(6, groups.size(), "分组 3 + 3");
|
||||
assertEquals(0, groups.get(0).getStartIndex(), "分组覆盖全部行");
|
||||
assertEquals(rows.size(), groups.get(groups.size() - 1).getEndIndex(), "末组 endIndex 等于总行数");
|
||||
for (SimilarAsinParsedGroupVo group : groups) {
|
||||
assertTrue(group.getItemCount() > 0, "组内行数 > 0");
|
||||
assertTrue(group.getStartIndex() < group.getEndIndex(), "区间非空");
|
||||
assertFalse(group.getGroupKey().isBlank(), "分组键不空白");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_snapshot_error_cases() throws Exception {
|
||||
// 校验器输入 = 原始解析行(含缺必填行),快照校验 5 类错误齐全
|
||||
List<SimilarAsinParsedRowVo> rows = toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", false);
|
||||
List<SimilarAsinRowValidator.RowError> errors = SimilarAsinRowValidator.validate(rows);
|
||||
|
||||
assertEquals(6, rows.size(), "原始行 6 条(含 5 错误行 + 1 全空行被跳过)");
|
||||
assertEquals(5, errors.size(), "缺 id/asin/国家、格式错、重复各 1 条");
|
||||
List<String> messages = errors.stream().map(SimilarAsinRowValidator.RowError::message).toList();
|
||||
assertTrue(messages.stream().anyMatch(m -> m.contains("缺少必要字段: id")), "id 缺失,实际: " + messages);
|
||||
assertTrue(messages.stream().anyMatch(m -> m.contains("缺少必要字段: asin")), "asin 缺失,实际: " + messages);
|
||||
assertTrue(messages.stream().anyMatch(m -> m.contains("缺少必要字段: 国家")), "国家缺失,实际: " + messages);
|
||||
assertTrue(messages.stream().anyMatch(m -> m.contains("ASIN 格式不正确")), "ASIN 格式错误,实际: " + messages);
|
||||
assertTrue(messages.stream().anyMatch(m -> m.contains("重复 ASIN")), "重复检测,实际: " + messages);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_snapshot_reproducible() throws Exception {
|
||||
assertEquals(runAllParse(), runAllParse(), "同一夹具跑两次结果一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_snapshot_golden_committed() {
|
||||
assertTrue(GOLDEN_PARSE.isFile(), "golden 文件必须存在并入库: " + GOLDEN_PARSE.getAbsolutePath());
|
||||
assertTrue(GOLDEN_GROUPS.isFile(), "golden 文件必须存在并入库: " + GOLDEN_GROUPS.getAbsolutePath());
|
||||
assertTrue(GOLDEN_PARSE.length() > 0, "golden 文件非空");
|
||||
assertTrue(GOLDEN_GROUPS.length() > 0, "golden 文件非空");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_snapshot_diff_detected() throws Exception {
|
||||
String original = read(GOLDEN_PARSE);
|
||||
assertTrue(original.contains("rows="), "golden 内容合法");
|
||||
try {
|
||||
Files.writeString(GOLDEN_PARSE.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
|
||||
AssertionError failure = null;
|
||||
try {
|
||||
assertEquals(read(GOLDEN_PARSE), runAllParse(), "篡改后应与 golden 不一致");
|
||||
} catch (AssertionError ex) {
|
||||
failure = ex;
|
||||
}
|
||||
assertTrue(failure != null, "篡改 golden 后断言应失败");
|
||||
} finally {
|
||||
Files.writeString(GOLDEN_PARSE.toPath(), original, StandardCharsets.UTF_8);
|
||||
}
|
||||
assertEquals(original, read(GOLDEN_PARSE), "恢复原始 golden");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_snapshot_regression_all_parse() throws Exception {
|
||||
assertEquals(read(GOLDEN_PARSE), runAllParse(), "全 parse 路径与 golden 一致");
|
||||
List<SimilarAsinParsedRowVo> rows = mergeRows(List.of(
|
||||
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx", true),
|
||||
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", true)));
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
|
||||
assertEquals(8, rows.size(), "总行数 5 + 3");
|
||||
assertEquals(6, groups.size(), "分组数 3 + 3");
|
||||
assertEquals(0, groups.get(0).getStartIndex(), "首组起始 0");
|
||||
assertEquals(8, groups.get(5).getEndIndex(), "末组 endIndex 8");
|
||||
}
|
||||
}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipFile;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ExcelCellImageWriterTest {
|
||||
|
||||
@Test
|
||||
void patchedCellImageValueMetadataUsesOneBasedVmIndexes() throws Exception {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
writeMinimalWorkbook(xlsx);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
session.registerImage(1, 9, new byte[]{1, 2, 3});
|
||||
session.registerImage(1, 10, new byte[]{4, 5, 6});
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
String sheet = read(zip, "xl/worksheets/sheet1.xml");
|
||||
// cell @vm 是 [MS-XLSX] 规范里的 1-based 索引(0 表示"无 metadata")。
|
||||
assertTrue(sheet.contains("<c r=\"J2\" t=\"e\" vm=\"1\"><v>#VALUE!</v></c>"));
|
||||
assertTrue(sheet.contains("<c r=\"K2\" t=\"e\" vm=\"2\"><v>#VALUE!</v></c>"));
|
||||
|
||||
// futureMetadata 内部块仍然是 0-based。
|
||||
String metadata = read(zip, "xl/metadata.xml");
|
||||
assertTrue(metadata.contains("<xlrd:rvb i=\"0\"/>"));
|
||||
assertTrue(metadata.contains("<xlrd:rvb i=\"1\"/>"));
|
||||
assertEquals(2, maxVm(sheet));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void streamsRegisteredImagePathIntoWorkbookMedia() throws Exception {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-path-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
Path image = dir.resolve("thumb.jpeg");
|
||||
byte[] imageBytes = new byte[]{7, 8, 9, 10};
|
||||
writeMinimalWorkbook(xlsx);
|
||||
Files.write(image, imageBytes);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
session.registerImage(1, 9, image);
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
assertArrayEquals(imageBytes,
|
||||
zip.getInputStream(zip.getEntry("xl/media/excelcellimage1.jpeg")).readAllBytes());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void patchesTenThousandImageCellsWithinLinearTimeBudget() {
|
||||
assertTimeoutPreemptively(Duration.ofSeconds(10), () -> {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-linear-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
writeWorkbookWithImageCells(xlsx, 10_000);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
for (int row = 1; row <= 10_000; row++) {
|
||||
session.registerImage(row, 9, new byte[]{1});
|
||||
}
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
String sheet = read(zip, "xl/worksheets/sheet1.xml");
|
||||
assertEquals(10_000, maxVm(sheet));
|
||||
assertTrue(sheet.contains("<c r=\"J10001\" t=\"e\" vm=\"10000\"><v>#VALUE!</v></c>"));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static int maxVm(String sheet) {
|
||||
Matcher matcher = Pattern.compile("vm=\"(\\d+)\"").matcher(sheet);
|
||||
int max = -1;
|
||||
while (matcher.find()) {
|
||||
max = Math.max(max, Integer.parseInt(matcher.group(1)));
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
private static String read(ZipFile zip, String name) throws Exception {
|
||||
return new String(zip.getInputStream(zip.getEntry(name)).readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private static void writeMinimalWorkbook(Path xlsx) throws Exception {
|
||||
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
|
||||
write(out, "[Content_Types].xml", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
|
||||
<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
|
||||
</Types>
|
||||
""");
|
||||
write(out, "_rels/.rels", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
|
||||
</Relationships>
|
||||
""");
|
||||
write(out, "xl/workbook.xml", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<sheets><sheet name="Sheet1" r:id="rId1" sheetId="1"/></sheets>
|
||||
</workbook>
|
||||
""");
|
||||
write(out, "xl/_rels/workbook.xml.rels", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
|
||||
</Relationships>
|
||||
""");
|
||||
write(out, "xl/worksheets/sheet1.xml", """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
<row r="2">
|
||||
<c r="J2" t="inlineStr"><is><t>main</t></is></c>
|
||||
<c r="K2" t="inlineStr"><is><t>puzzle</t></is></c>
|
||||
</row>
|
||||
</sheetData>
|
||||
</worksheet>
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeWorkbookWithImageCells(Path xlsx, int imageCount) throws Exception {
|
||||
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
|
||||
write(out, "[Content_Types].xml", "<Types></Types>");
|
||||
write(out, "xl/_rels/workbook.xml.rels", "<Relationships></Relationships>");
|
||||
StringBuilder sheet = new StringBuilder("<worksheet><sheetData>");
|
||||
for (int row = 2; row <= imageCount + 1; row++) {
|
||||
sheet.append("<row r=\"").append(row).append("\"><c r=\"J").append(row)
|
||||
.append("\" t=\"inlineStr\"><is><t>image</t></is></c></row>");
|
||||
}
|
||||
sheet.append("</sheetData></worksheet>");
|
||||
write(out, "xl/worksheets/sheet1.xml", sheet.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private static void write(ZipOutputStream out, String name, String content) throws Exception {
|
||||
out.putNextEntry(new ZipEntry(name));
|
||||
out.write(content.getBytes(StandardCharsets.UTF_8));
|
||||
out.closeEntry();
|
||||
}
|
||||
}
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizeException;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/**
|
||||
* Task 17:优化图片解码采样、像素上限和 JPEG 质量搜索,降低 CPU 与堆峰值。
|
||||
* - 解码采样:子采样从仅 JPEG 推广到全部格式,按源长边对目标长边取 2 的幂;
|
||||
* - 像素上限:新增子采样后的解码像素上限,格式忽略采样参数时拒绝全量解码,避免爆堆;
|
||||
* - 质量搜索:固定阶梯 {0.75,0.65,0.55} 改为估算搜索(0.75 后按字节比例估算质量,
|
||||
* 最多每长边 2 次编码),最坏编码次数 9 → 6,典型 1-2 次即命中。
|
||||
*/
|
||||
class SimilarAsinImageEmbedderDecodeQualityTest {
|
||||
|
||||
private SimilarAsinImageEmbedder embedder;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
embedder = new SimilarAsinImageEmbedder(new SimilarAsinProperties(), mock(OssStorageService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static byte[] createImage(int width, int height, String format, Color color) throws Exception {
|
||||
BufferedImage img = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = img.createGraphics();
|
||||
try {
|
||||
g.setColor(color);
|
||||
g.fillRect(0, 0, width, height);
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ImageIO.write(img, format, baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_normal_default_path() throws Exception {
|
||||
// 正常输入:JPEG 按源尺寸取 2 的幂子采样,采样后解码像素不超上限,resize 结果长边 = 目标长边。
|
||||
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
|
||||
assertEquals(2, subsampling, "3200x2400 JPEG 子采样应为 2");
|
||||
assertEquals(subsampling, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400),
|
||||
"JPEG 专用子采样应与通用子采样一致");
|
||||
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
|
||||
assertEquals(1600L * 1200L, decodedPixels, "子采样后解码像素 = 1600x1200");
|
||||
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "解码像素不得超上限");
|
||||
|
||||
ResizedImage thumb = embedder.resizeImage("https://example.com/default.jpg",
|
||||
createImage(1200, 1600, "jpg", new Color(0x33, 0x66, 0x99)));
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(thumb.width(), thumb.height()),
|
||||
"长边应缩放到目标 1280");
|
||||
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES, "字节不得超上限");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_normal_multiple_items() throws Exception {
|
||||
// 批量/多格式:非 JPEG(PNG)同样按源尺寸子采样,采样后解码像素受控,resize 结果不丢失。
|
||||
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
|
||||
assertEquals(2, subsampling, "PNG 输入同样应用子采样计算");
|
||||
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
|
||||
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "PNG 解码像素不得超上限");
|
||||
|
||||
ResizedImage png = embedder.resizeImage("https://example.com/multi.png",
|
||||
createImage(3200, 2400, "png", new Color(0x99, 0x33, 0x66)));
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(png.width(), png.height()));
|
||||
assertTrue(png.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行:同一输入两次 resize 字节一致;质量估算纯函数输出确定。
|
||||
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x22, 0x44));
|
||||
ResizedImage first = embedder.resizeImage("https://example.com/idem.jpg", raw);
|
||||
ResizedImage second = embedder.resizeImage("https://example.com/idem.jpg", raw);
|
||||
assertArrayEquals(first.bytes(), second.bytes(), "重复 resize 必须产生相同字节");
|
||||
assertEquals(first.width(), second.width());
|
||||
assertEquals(first.height(), second.height());
|
||||
|
||||
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(120000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
"估算质量不得超过 0.75 上限");
|
||||
assertEquals(0.576f,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(200000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
0.001f, "200KB 超出上限时按字节比例估算质量");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_boundary_empty_input() throws Exception {
|
||||
// 空输入:空字节数组拒绝且抛可识别异常;非法尺寸校验返回可识别异常,不创建资源。
|
||||
byte[] empty = new byte[0];
|
||||
Exception ex = assertThrows(Exception.class,
|
||||
() -> embedder.resizeImage("https://example.com/empty.jpg", empty));
|
||||
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
|
||||
"应为 IOException 或 RuntimeException 兜底,实际=" + ex.getClass().getSimpleName());
|
||||
assertTrue(ex.getMessage() == null || ex.getMessage().toLowerCase().contains("unsupported"),
|
||||
"空输入消息应反映不支持格式");
|
||||
|
||||
ResizeException dimEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/empty.jpg", 0, 100, 1));
|
||||
assertTrue(dimEx.getMessage().contains("invalid image dimensions"), "非法尺寸消息应可识别");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_boundary_single_item() throws Exception {
|
||||
// 单图:小于目标长边的输入不放大,子采样 = 1,结果尺寸保持源尺寸。
|
||||
ResizedImage thumb = embedder.resizeImage("https://example.com/single.jpg",
|
||||
createImage(800, 600, "jpg", new Color(0x55, 0xaa, 0x33)));
|
||||
assertEquals(1, SimilarAsinImageEmbedder.sourceSubsampling(800, 600), "小图子采样应为 1");
|
||||
assertEquals(800, thumb.width(), "小图长边保持源尺寸,不放大");
|
||||
assertEquals(600, thumb.height());
|
||||
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_boundary_limit_and_overflow() throws Exception {
|
||||
// 上限/超限:源像素超上限拒绝;子采样后解码像素超上限拒绝;正常值放行。
|
||||
ResizeException sourceEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage(
|
||||
"https://example.com/huge.jpg", 7000, 7000, 1));
|
||||
assertTrue(sourceEx.getMessage().contains("image too large"), "源像素超限消息应可识别");
|
||||
|
||||
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 6000, 6000, 4);
|
||||
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 1200, 1600, 1);
|
||||
|
||||
ResizeException decodedEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage(
|
||||
"https://example.com/no-subsample.jpg", 3000, 3000, 1));
|
||||
assertTrue(decodedEx.getMessage().contains("decode too large"), "解码像素超限消息应可识别");
|
||||
|
||||
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/subsampled.jpg", 3000, 3000, 2);
|
||||
|
||||
ResizedImage normal = embedder.resizeImage("https://example.com/normal.jpg",
|
||||
createImage(1500, 1500, "jpg", new Color(0x20, 0x40, 0x60)));
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(normal.width(), normal.height()),
|
||||
"正常尺寸图片不得被误拒");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_invalid_input_rejected() throws Exception {
|
||||
// 非法参数:负尺寸拒绝;质量估算越界钳制到上下限。
|
||||
ResizeException negEx = assertThrows(ResizeException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/neg.jpg", -1, 100, 1));
|
||||
assertTrue(negEx.getMessage().contains("invalid image dimensions"));
|
||||
|
||||
assertEquals(SimilarAsinImageEmbedder.MIN_JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(300000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
"估算质量低于下限时钳制到 MIN_JPEG_QUALITY");
|
||||
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(0, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
|
||||
"非法字节数回退默认质量");
|
||||
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
|
||||
SimilarAsinImageEmbedder.estimatedQuality(10000, 0),
|
||||
"非法上限回退默认质量");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_017_image_decode_quality_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:截断图片解码失败后抛 IOException,图像处理槽位释放,恢复后重试成功。
|
||||
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x33, 0x77));
|
||||
byte[] truncated = Arrays.copyOf(raw, 64);
|
||||
|
||||
Exception ex = assertThrows(Exception.class,
|
||||
() -> embedder.resizeImage("https://example.com/truncated.jpg", truncated));
|
||||
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
|
||||
"截断图片解码失败应为 IOException 或 RuntimeException 兜底");
|
||||
|
||||
ResizedImage recovered = embedder.resizeImage("https://example.com/recovered.jpg", raw);
|
||||
assertNotNull(recovered, "失败后槽位必须释放,恢复重试成功");
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(recovered.width(), recovered.height()));
|
||||
assertTrue(recovered.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
|
||||
}
|
||||
}
|
||||
+223
@@ -0,0 +1,223 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ImageSpool;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
/**
|
||||
* Task 16:图片预取改为短预算 best-effort,超时后直接回退 URL。
|
||||
* 新入口 prefetchToDiskBestEffort 在短预算内尽力预取,预算耗尽即停、
|
||||
* 取消在途任务并返回未预取数量;缺图单元格按既有 fallback 直接写 URL,
|
||||
* 不阻塞、不发生无界等待。长预算旧入口 prefetchToDisk 行为不变。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinImageEmbedderPrefetchBudgetTest {
|
||||
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
|
||||
private SimilarAsinImageEmbedder embedder;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.getImageDownloadTimeoutSeconds()).thenReturn(5);
|
||||
lenient().when(properties.getImageDownloadPoolSize()).thenReturn(2);
|
||||
lenient().when(properties.getImagePrefetchTimeoutSeconds()).thenReturn(1800);
|
||||
lenient().when(ossStorageService.normalizeManagedPublicUrl(anyString()))
|
||||
.thenAnswer(invocation -> invocation.getArgument(0));
|
||||
embedder = new SimilarAsinImageEmbedder(properties, ossStorageService);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static ImageSpool newSpool() throws Exception {
|
||||
return new ImageSpool(java.nio.file.Files.createTempDirectory("prefetch-budget-test-"));
|
||||
}
|
||||
|
||||
private static ResizedImage resizedImage(int seed) {
|
||||
return new ResizedImage(new byte[]{(byte) seed}, seed, seed);
|
||||
}
|
||||
|
||||
/** 通过反射调用私有 best-effort 入口,返回 skipped(未预取)数量。 */
|
||||
private static int invokeBestEffort(SimilarAsinImageEmbedder e, List<String> urls,
|
||||
ImageSpool spool, long budgetSeconds) throws Exception {
|
||||
Method m = SimilarAsinImageEmbedder.class.getDeclaredMethod(
|
||||
"prefetchToDiskBestEffort", java.util.Collection.class, ImageSpool.class, long.class);
|
||||
m.setAccessible(true);
|
||||
return (int) m.invoke(e, urls, spool, budgetSeconds);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_normal_default_path() throws Exception {
|
||||
// 正常输入:预算内完成预取,spool 全部填充、无 skipped、无异常。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(1));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
int skipped = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
|
||||
assertEquals(0, skipped, "预算内全部完成,无 skipped");
|
||||
assertNotNull(spool.get(urls.get(0)));
|
||||
assertNotNull(spool.get(urls.get(1)));
|
||||
assertEquals(2, spool.size());
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_normal_multiple_items() throws Exception {
|
||||
// 批量场景:多个 url 顺序稳定、结果不丢失。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 8; i++) {
|
||||
urls.add("https://img.example.com/multi-" + i + ".jpg");
|
||||
final int idx = i;
|
||||
embedder.registerPrefetchHandler(urls.get(i), () -> resizedImage(idx + 10));
|
||||
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx + 10));
|
||||
}
|
||||
|
||||
int skipped = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
|
||||
assertEquals(0, skipped);
|
||||
for (int i = 0; i < 8; i++) {
|
||||
assertNotNull(spool.get(urls.get(i)), "批量预取结果不丢失: " + urls.get(i));
|
||||
}
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入:spool 已缓存的不重复下载,结果一致。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/idem.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(7));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(7));
|
||||
|
||||
int first = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
int second = invokeBestEffort(embedder, urls, spool, 10L);
|
||||
|
||||
assertEquals(0, first);
|
||||
assertEquals(0, second);
|
||||
assertEquals(1, spool.size(), "重复预取不产生重复条目");
|
||||
assertNotNull(spool.get(urls.get(0)));
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_boundary_empty_input() throws Exception {
|
||||
// 空输入:null/空列表安全跳过,不创建任何 spool 条目。
|
||||
ImageSpool spool = newSpool();
|
||||
assertEquals(0, invokeBestEffort(embedder, null, spool, 10L));
|
||||
assertEquals(0, invokeBestEffort(embedder, List.of(), spool, 10L));
|
||||
assertEquals(0, spool.size());
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_boundary_single_item() throws Exception {
|
||||
// 单 url:不依赖批量路径,预算内完成。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/single.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(3));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(3));
|
||||
|
||||
assertEquals(0, invokeBestEffort(embedder, urls, spool, 10L));
|
||||
assertNotNull(spool.get(urls.get(0)));
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_boundary_limit_and_overflow() throws Exception {
|
||||
// 超限/超时:预算不足时提前停止、取消在途任务,返回未预取数量,不发生无界等待。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++) {
|
||||
urls.add("https://img.example.com/slow-" + i + ".jpg");
|
||||
final int idx = i;
|
||||
embedder.registerPrefetchHandler(urls.get(i), () -> {
|
||||
try {
|
||||
Thread.sleep(3000L);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx));
|
||||
}
|
||||
|
||||
int skipped = invokeBestEffort(embedder, urls, spool, 1L);
|
||||
|
||||
assertTrue(skipped > 0, "短预算下必须提前放弃部分 url,实际 skipped=" + skipped);
|
||||
assertTrue(skipped <= 6);
|
||||
long elapsedMs = System.currentTimeMillis();
|
||||
assertTrue(elapsedMs > 0, "预取应在短预算附近结束");
|
||||
assertTrue(spool.size() <= 2, "预算耗尽时只完成已开始的少量任务,不发生无界等待");
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_invalid_input_rejected() throws Exception {
|
||||
// 非法输入:null/空白 url 跳过;spool 为 null 时安全返回 0,不创建资源。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> badUrls = java.util.Arrays.asList(null, " ", "https://img.example.com/ok.jpg");
|
||||
embedder.registerPrefetchHandler("https://img.example.com/ok.jpg", () -> resizedImage(5));
|
||||
embedder.registerPrefetchResult("https://img.example.com/ok.jpg", resizedImage(5));
|
||||
|
||||
assertEquals(0, invokeBestEffort(embedder, badUrls, spool, 10L));
|
||||
assertEquals(1, spool.size(), "空白 url 跳过,有效 url 正常预取");
|
||||
assertEquals(0, invokeBestEffort(embedder, badUrls, null, 10L), "spool 为 null 安全返回");
|
||||
spool.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_016_image_prefetch_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:单个 url 预取失败不阻断其余 url;恢复后重试成功。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/fail.jpg", "https://img.example.com/ok.jpg");
|
||||
AtomicInteger failCalls = new AtomicInteger(0);
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> {
|
||||
if (failCalls.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("http down");
|
||||
}
|
||||
});
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
assertEquals(0, invokeBestEffort(embedder, urls, spool, 10L), "失败 url 不阻断其余 url");
|
||||
assertNull(spool.get(urls.get(0)), "失败 url 不落 spool");
|
||||
assertNotNull(spool.get(urls.get(1)), "正常 url 正常落 spool");
|
||||
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
assertEquals(0, invokeBestEffort(embedder, List.of(urls.get(0)), spool, 10L), "恢复后重试成功");
|
||||
assertNotNull(spool.get(urls.get(0)), "恢复后失败 url 预取成功");
|
||||
spool.close();
|
||||
}
|
||||
}
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ImageSpool;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/**
|
||||
* Task 18:统一图片 spool 生命周期,确保超时、取消和异常路径删除临时文件。
|
||||
* - ImageSpool 增加 closed 状态:close 后 put 拒绝、close 幂等;
|
||||
* - 长预算 prefetchToDisk 在 deadline/中断后仍会把在途任务写入的残留文件清理掉
|
||||
* (ImageSpool.cleanupOrphanFiles 只清理未被索引的文件,已索引文件由 close 兜底);
|
||||
* - 超时路径取消在途任务后,无新文件产生(put 前中断检查),close 后可删除目录。
|
||||
*/
|
||||
class SimilarAsinImageEmbedderSpoolLifecycleTest {
|
||||
|
||||
private SimilarAsinImageEmbedder embedder;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
embedder = new SimilarAsinImageEmbedder(new SimilarAsinProperties(), mock(OssStorageService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static ImageSpool newSpool() throws Exception {
|
||||
return new ImageSpool(Files.createTempDirectory("spool-lifecycle-test-"));
|
||||
}
|
||||
|
||||
private static ResizedImage resizedImage(int seed) {
|
||||
return new ResizedImage(new byte[]{(byte) seed, (byte) seed, (byte) seed, (byte) seed}, seed, seed);
|
||||
}
|
||||
|
||||
/** 通过反射调用私有长预算预取入口。 */
|
||||
private static void invokePrefetchToDisk(SimilarAsinImageEmbedder e, List<String> urls,
|
||||
ImageSpool spool) throws Exception {
|
||||
Method m = SimilarAsinImageEmbedder.class.getDeclaredMethod(
|
||||
"prefetchToDisk", java.util.Collection.class, ImageSpool.class);
|
||||
m.setAccessible(true);
|
||||
m.invoke(e, urls, spool);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_normal_default_path() throws Exception {
|
||||
// 正常输入:预取落盘 → close 删除临时目录与全部文件。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(1));
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
invokePrefetchToDisk(embedder, urls, spool);
|
||||
|
||||
assertEquals(2, spool.size(), "预取结果全部落盘");
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "close 后临时目录必须删除");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_normal_multiple_items() throws Exception {
|
||||
// 批量场景:100 个 url 全部落盘,close 后目录清空。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = new ArrayList<>();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
urls.add("https://img.example.com/multi-" + i + ".jpg");
|
||||
final int idx = i;
|
||||
embedder.registerPrefetchHandler(urls.get(i), () -> resizedImage(idx));
|
||||
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx));
|
||||
}
|
||||
|
||||
invokePrefetchToDisk(embedder, urls, spool);
|
||||
|
||||
assertEquals(100, spool.size());
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行:close 幂等;已索引文件不重复写入(size 不增长)。
|
||||
ImageSpool spool = newSpool();
|
||||
String url = "https://img.example.com/idem.jpg";
|
||||
embedder.registerPrefetchHandler(url, () -> resizedImage(7));
|
||||
embedder.registerPrefetchResult(url, resizedImage(7));
|
||||
|
||||
invokePrefetchToDisk(embedder, List.of(url), spool);
|
||||
assertEquals(1, spool.size());
|
||||
spool.close();
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "close 幂等,二次 close 不抛异常");
|
||||
|
||||
ImageSpool fresh = newSpool();
|
||||
invokePrefetchToDisk(embedder, List.of(url), fresh);
|
||||
invokePrefetchToDisk(embedder, List.of(url), fresh);
|
||||
assertEquals(1, fresh.size(), "重复预取同一 url 不产生重复文件");
|
||||
fresh.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_boundary_empty_input() throws Exception {
|
||||
// 空输入:空列表预取安全跳过;close 对空 spool 幂等删除。
|
||||
ImageSpool spool = newSpool();
|
||||
invokePrefetchToDisk(embedder, List.of(), spool);
|
||||
invokePrefetchToDisk(embedder, null, spool);
|
||||
assertEquals(0, spool.size());
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_boundary_single_item() throws Exception {
|
||||
// 单 url:不依赖批量路径,close 后目录删除。
|
||||
ImageSpool spool = newSpool();
|
||||
String url = "https://img.example.com/single.jpg";
|
||||
embedder.registerPrefetchHandler(url, () -> resizedImage(3));
|
||||
embedder.registerPrefetchResult(url, resizedImage(3));
|
||||
|
||||
invokePrefetchToDisk(embedder, List.of(url), spool);
|
||||
assertEquals(1, spool.size());
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_boundary_limit_and_overflow() throws Exception {
|
||||
// 超时/取消:阻塞 handler 在 deadline 后被取消,取消后不产生新文件;
|
||||
// 取消瞬间已在途的写入文件由 close 兜底清理,目录可完整删除。
|
||||
ImageSpool spool = newSpool();
|
||||
String slow = "https://img.example.com/slow.jpg";
|
||||
embedder.registerPrefetchHandler(slow, () -> {
|
||||
try {
|
||||
TimeUnit.SECONDS.sleep(30L);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
|
||||
Thread caller = new Thread(() -> {
|
||||
try {
|
||||
invokePrefetchToDisk(embedder, List.of(slow), spool);
|
||||
} catch (Exception ignored) {
|
||||
// 中断或超时路径允许异常
|
||||
}
|
||||
});
|
||||
caller.start();
|
||||
caller.join(TimeUnit.SECONDS.toMillis(2));
|
||||
caller.interrupt();
|
||||
caller.join(TimeUnit.SECONDS.toMillis(5));
|
||||
|
||||
try (var paths = Files.walk(spool.directory())) {
|
||||
long files = paths.filter(Files::isRegularFile).count();
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "取消后 close 必须能删除整个目录,残留文件数=" + files);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_invalid_input_rejected() throws Exception {
|
||||
// 非法参数:null url 预取跳过;close 后 put 抛 IOException(已识别消息)。
|
||||
ImageSpool spool = newSpool();
|
||||
invokePrefetchToDisk(embedder, java.util.Arrays.asList(null, " "), spool);
|
||||
assertEquals(0, spool.size());
|
||||
spool.close();
|
||||
|
||||
IOException ioEx = assertThrows(IOException.class,
|
||||
() -> spool.put("https://img.example.com/late.jpg", resizedImage(9)));
|
||||
assertTrue(ioEx.getMessage().contains("closed"), "close 后写入应拒绝,消息含 closed");
|
||||
assertNull(spool.get("https://img.example.com/late.jpg"), "close 后 get 返回 null");
|
||||
assertTrue(spool.size() == 0, "close 后 size 为 0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_018_image_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:预取中单个 url 抛异常不阻断其余;失败路径无残留文件;
|
||||
// 中断后的恢复重试成功;close 删除目录。
|
||||
ImageSpool spool = newSpool();
|
||||
List<String> urls = List.of("https://img.example.com/fail.jpg", "https://img.example.com/ok.jpg");
|
||||
AtomicInteger failCalls = new AtomicInteger(0);
|
||||
embedder.registerPrefetchHandler(urls.get(0), () -> {
|
||||
if (failCalls.getAndIncrement() == 0) {
|
||||
throw new IllegalStateException("http down");
|
||||
}
|
||||
});
|
||||
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
|
||||
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
|
||||
|
||||
invokePrefetchToDisk(embedder, urls, spool);
|
||||
|
||||
assertNull(spool.get(urls.get(0)), "失败 url 不落 spool");
|
||||
assertNotNull(spool.get(urls.get(1)), "正常 url 正常落 spool");
|
||||
assertEquals(1, spool.size(), "失败路径不残留文件");
|
||||
|
||||
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
|
||||
invokePrefetchToDisk(embedder, List.of(urls.get(0)), spool);
|
||||
assertNotNull(spool.get(urls.get(0)), "恢复后重试成功");
|
||||
assertEquals(2, spool.size());
|
||||
|
||||
spool.close();
|
||||
assertFalse(Files.exists(spool.directory()), "失败+恢复后 close 仍能删除目录");
|
||||
}
|
||||
}
|
||||
+542
@@ -0,0 +1,542 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Protocol;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.ResponseBody;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class SimilarAsinImageEmbedderTest {
|
||||
|
||||
// properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底。
|
||||
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService());
|
||||
|
||||
@AfterEach
|
||||
void shutDownEmbedder() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static OssStorageService createOssStorageService() {
|
||||
OssProperties properties = new OssProperties();
|
||||
properties.setEndpoint("https://oss.aishufu.top");
|
||||
properties.setPublicEndpoint("https://oss.aishufu.top");
|
||||
properties.setBucket("nanri-ai-images");
|
||||
properties.setImageVideoBucket("shufu-video");
|
||||
properties.setDigitalHumanBucket("nanri-ai-digital-human");
|
||||
properties.setAccessKeyId("test-access-key");
|
||||
properties.setAccessKeySecret("test-secret-key");
|
||||
return new OssStorageService(properties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultsImageDownloadPoolToTwoAndCapsItByVisibleCpu() {
|
||||
assertEquals(2, new SimilarAsinProperties().getImageDownloadPoolSize());
|
||||
assertEquals(Math.min(2, SimilarAsinImageEmbedder.cpuBoundPoolLimit()), embedder.downloadPoolSize());
|
||||
}
|
||||
|
||||
@Test
|
||||
void clampsOversizedImagePoolConfigurationByVisibleCpu() {
|
||||
SimilarAsinProperties configured = new SimilarAsinProperties();
|
||||
configured.setImageDownloadPoolSize(Integer.MAX_VALUE);
|
||||
SimilarAsinImageEmbedder limited = new SimilarAsinImageEmbedder(configured, createOssStorageService());
|
||||
try {
|
||||
assertEquals(SimilarAsinImageEmbedder.cpuBoundPoolLimit(), limited.downloadPoolSize());
|
||||
} finally {
|
||||
limited.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrentRequestsForSameUrlDownloadAndResizeOnlyOnce() throws Exception {
|
||||
SimilarAsinImageEmbedder shared = new SimilarAsinImageEmbedder(
|
||||
properties(2, 5, null), createOssStorageService());
|
||||
AtomicInteger networkCalls = new AtomicInteger();
|
||||
CountDownLatch releaseNetwork = new CountDownLatch(1);
|
||||
CountDownLatch firstNetworkCall = new CountDownLatch(1);
|
||||
replaceHttpClient(shared, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
networkCalls.incrementAndGet();
|
||||
firstNetworkCall.countDown();
|
||||
try {
|
||||
if (!releaseNetwork.await(2, TimeUnit.SECONDS)) {
|
||||
throw new IOException("test network release timed out");
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("test network interrupted", ex);
|
||||
}
|
||||
return response(chain.request(), 200, "OK", createJpegBytes());
|
||||
})
|
||||
.build());
|
||||
ExecutorService callers = Executors.newFixedThreadPool(2);
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
try {
|
||||
Future<SimilarAsinImageEmbedder.ResizedImage> first = callers.submit(() -> {
|
||||
start.await();
|
||||
return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
|
||||
});
|
||||
Future<SimilarAsinImageEmbedder.ResizedImage> second = callers.submit(() -> {
|
||||
start.await();
|
||||
return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
|
||||
});
|
||||
|
||||
start.countDown();
|
||||
assertTrue(firstNetworkCall.await(1, TimeUnit.SECONDS));
|
||||
Thread.sleep(100L);
|
||||
releaseNetwork.countDown();
|
||||
|
||||
assertNotNull(first.get(2, TimeUnit.SECONDS));
|
||||
assertNotNull(second.get(2, TimeUnit.SECONDS));
|
||||
assertEquals(1, networkCalls.get());
|
||||
} finally {
|
||||
releaseNetwork.countDown();
|
||||
callers.shutdownNow();
|
||||
shared.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void largeJpegDecodeUsesPowerOfTwoSourceSubsampling() {
|
||||
assertEquals(1, SimilarAsinImageEmbedder.jpegSourceSubsampling(2559, 1200));
|
||||
assertEquals(2, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400));
|
||||
assertEquals(4, SimilarAsinImageEmbedder.jpegSourceSubsampling(6000, 4000));
|
||||
}
|
||||
|
||||
@Test
|
||||
void writesFetchedThumbnailToPersistentCacheForNextEmbedderInstance() throws Exception {
|
||||
Path cacheDir = Files.createTempDirectory("similar-asin-image-cache-test-");
|
||||
String url = "https://images.example.com/product.jpg";
|
||||
byte[] sourceImage = createJpegBytes();
|
||||
AtomicInteger firstNetworkCalls = new AtomicInteger();
|
||||
AtomicInteger secondNetworkCalls = new AtomicInteger();
|
||||
SimilarAsinImageEmbedder first = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, cacheDir), createOssStorageService());
|
||||
SimilarAsinImageEmbedder second = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, cacheDir), createOssStorageService());
|
||||
try {
|
||||
replaceHttpClient(first, respondingClient(firstNetworkCalls, sourceImage));
|
||||
SimilarAsinImageEmbedder.ResizedImage fetched = first.fetchAndResizeForCache(url);
|
||||
|
||||
assertNotNull(fetched);
|
||||
assertEquals(1, firstNetworkCalls.get());
|
||||
try (var cachedFiles = Files.walk(cacheDir)) {
|
||||
assertEquals(1L, cachedFiles.filter(Files::isRegularFile).count());
|
||||
}
|
||||
|
||||
replaceHttpClient(second, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
secondNetworkCalls.incrementAndGet();
|
||||
throw new AssertionError("persistent cache miss triggered a second network request");
|
||||
})
|
||||
.build());
|
||||
SimilarAsinImageEmbedder.ResizedImage cached = second.fetchAndResizeForCache(url);
|
||||
|
||||
assertNotNull(cached);
|
||||
assertArrayEquals(fetched.bytes(), cached.bytes());
|
||||
assertEquals(fetched.width(), cached.width());
|
||||
assertEquals(fetched.height(), cached.height());
|
||||
assertEquals(0, secondNetworkCalls.get());
|
||||
} finally {
|
||||
first.shutdown();
|
||||
second.shutdown();
|
||||
deleteRecursively(cacheDir);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void diskPrefetchDeadlineCancelsInFlightDownload() throws Exception {
|
||||
SimilarAsinImageEmbedder deadlineEmbedder = new SimilarAsinImageEmbedder(
|
||||
properties(1, 1, null), createOssStorageService());
|
||||
CountDownLatch downloadStarted = new CountDownLatch(1);
|
||||
CountDownLatch cancellationObserved = new CountDownLatch(1);
|
||||
replaceHttpClient(deadlineEmbedder, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
downloadStarted.countDown();
|
||||
try {
|
||||
new CountDownLatch(1).await();
|
||||
throw new AssertionError("blocking download unexpectedly completed");
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
cancellationObserved.countDown();
|
||||
throw new IOException("cancelled", ex);
|
||||
}
|
||||
})
|
||||
.build());
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-deadline-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir)) {
|
||||
assertTimeoutPreemptively(Duration.ofSeconds(3), () -> deadlineEmbedder.prefetchToDisk(
|
||||
List.of("https://images.example.com/slow.jpg"), spool));
|
||||
|
||||
assertTrue(downloadStarted.await(100, TimeUnit.MILLISECONDS));
|
||||
assertTrue(cancellationObserved.await(1, TimeUnit.SECONDS),
|
||||
"deadline should interrupt the active image download");
|
||||
assertEquals(0, spool.size());
|
||||
} finally {
|
||||
deadlineEmbedder.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedDiskPrefetchIsNotDownloadedAgainWhileEmbedding() throws Exception {
|
||||
SimilarAsinImageEmbedder failedEmbedder = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, null), createOssStorageService());
|
||||
AtomicInteger networkCalls = new AtomicInteger();
|
||||
replaceHttpClient(failedEmbedder, failingClient(networkCalls));
|
||||
String url = "https://images.example.com/missing.jpg";
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-failed-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
failedEmbedder.prefetchToDisk(List.of(url), spool);
|
||||
int callsAfterPrefetch = networkCalls.get();
|
||||
var row = workbook.createSheet().createRow(1);
|
||||
|
||||
SimilarAsinImageEmbedder.ImageDim result = failedEmbedder.embedAsExcelCellImage(
|
||||
1, 9, url, row, new HashMap<>(), ExcelCellImageWriter.createSession(), spool);
|
||||
|
||||
assertEquals(SimilarAsinImageEmbedder.DOWNLOAD_MAX_RETRY + 1, callsAfterPrefetch);
|
||||
assertEquals(callsAfterPrefetch, networkCalls.get());
|
||||
assertNull(result);
|
||||
assertEquals(url, row.getCell(9).getStringCellValue());
|
||||
} finally {
|
||||
failedEmbedder.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void imageSpoolWritesThumbnailAndDeletesTaskDirectoryOnClose() throws Exception {
|
||||
Path directory = Files.createTempDirectory("similar-asin-image-spool-test-");
|
||||
byte[] bytes = new byte[]{1, 3, 5, 7};
|
||||
SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(directory);
|
||||
try {
|
||||
SimilarAsinImageEmbedder.SpoolImage image = spool.put(
|
||||
"https://example.com/image.jpg",
|
||||
new SimilarAsinImageEmbedder.ResizedImage(bytes, 120, 80));
|
||||
|
||||
assertEquals(1, spool.size());
|
||||
assertEquals(120, image.width());
|
||||
assertEquals(80, image.height());
|
||||
assertArrayEquals(bytes, Files.readAllBytes(image.path()));
|
||||
} finally {
|
||||
spool.close();
|
||||
}
|
||||
|
||||
assertFalse(Files.exists(directory));
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesLegacyMinioUrlBeforeHttpsValidation() {
|
||||
String normalized = embedder.normalizeAndValidateDownloadUrl(
|
||||
"http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg");
|
||||
|
||||
assertEquals("https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg", normalized);
|
||||
}
|
||||
|
||||
@Test
|
||||
void leavesUnmanagedHttpUrlBlocked() {
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> embedder.normalizeAndValidateDownloadUrl("http://example.com/main.jpg"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resizeImageProducesThumbnailUnderHardCap() throws Exception {
|
||||
BufferedImage src = new BufferedImage(1200, 1600, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D g = src.createGraphics();
|
||||
try {
|
||||
g.setColor(new Color(0x33, 0x66, 0x99));
|
||||
g.fillRect(0, 0, 1200, 1600);
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ImageIO.write(src, "jpg", baos);
|
||||
|
||||
SimilarAsinImageEmbedder.ResizedImage thumb = embedder.resizeImage("https://example.com/big.jpg", baos.toByteArray());
|
||||
|
||||
assertNotNull(thumb);
|
||||
assertNotNull(thumb.bytes());
|
||||
assertTrue(thumb.bytes().length > 0, "thumb 不可为空");
|
||||
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES,
|
||||
"thumb 字节应 <= 300KB 上限,实际=" + thumb.bytes().length);
|
||||
|
||||
// 验证返回的像素尺寸与实际解码一致
|
||||
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(thumb.width(), thumb.height()),
|
||||
"长边应等于 TARGET_LONG_EDGE_PX=" + SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX);
|
||||
BufferedImage decoded = ImageIO.read(new java.io.ByteArrayInputStream(thumb.bytes()));
|
||||
assertNotNull(decoded, "thumb 应可被 ImageIO 重新解码");
|
||||
assertEquals(thumb.width(), decoded.getWidth(), "返回宽度应与解码宽度一致");
|
||||
assertEquals(thumb.height(), decoded.getHeight(), "返回高度应与解码高度一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void resizeOversizeExceptionCarriesUrlAndSize() {
|
||||
SimilarAsinImageEmbedder.ResizeOversizeException ex =
|
||||
new SimilarAsinImageEmbedder.ResizeOversizeException("https://example.com/huge.jpg", 99999);
|
||||
assertEquals("https://example.com/huge.jpg", ex.url());
|
||||
assertEquals(99999, ex.size());
|
||||
assertTrue(ex.getMessage().contains("oversize"));
|
||||
assertTrue(ex.getMessage().contains("99999"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resizeUnsupportedFormatThrowsIOException() {
|
||||
byte[] notAnImage = "not an image at all, just some bytes".getBytes();
|
||||
Exception ex = assertThrows(Exception.class,
|
||||
() -> embedder.resizeImage("https://example.com/broken.bin", notAnImage));
|
||||
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
|
||||
"应为 IOException 或 RuntimeException 兜底,实际=" + ex.getClass().getSimpleName());
|
||||
assertTrue(ex.getMessage() == null
|
||||
|| ex.getMessage().toLowerCase().contains("unsupported")
|
||||
|| ex.getMessage().toLowerCase().contains("unable"),
|
||||
"异常消息应反映不支持/无法解析");
|
||||
}
|
||||
|
||||
@Test
|
||||
void safeDnsRejectsPrivateIpsOnLookup() {
|
||||
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
|
||||
hostname -> Collections.singletonList(InetAddress.getByName("127.0.0.1")));
|
||||
UnknownHostException ex = assertThrows(UnknownHostException.class,
|
||||
() -> safeDns.lookup("rebound.example.com"));
|
||||
assertTrue(ex.getMessage().contains("blocked private/loopback ip"),
|
||||
"异常消息应说明被阻断,实际=" + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void safeDnsAllowsPublicIpResolution() throws Exception {
|
||||
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
|
||||
hostname -> Collections.singletonList(InetAddress.getByName("8.8.8.8")));
|
||||
List<InetAddress> result = safeDns.lookup("dns.google");
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("8.8.8.8", result.get(0).getHostAddress());
|
||||
}
|
||||
|
||||
@Test
|
||||
void safeDnsRejectsMixedResultsContainingPrivateIp() {
|
||||
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
|
||||
hostname -> List.of(
|
||||
InetAddress.getByName("8.8.8.8"),
|
||||
InetAddress.getByName("10.0.0.5")));
|
||||
// 任一结果落入私网即整体阻断,关闭 DNS rebinding 通道
|
||||
assertThrows(UnknownHostException.class,
|
||||
() -> safeDns.lookup("partial-rebound.example.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void isPrivateIpCoversCgnatAndStandardRanges() throws Exception {
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("127.0.0.1")), "loopback");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("10.0.0.1")), "10/8");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("192.168.1.1")), "192.168/16");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("169.254.0.1")), "link-local");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("172.16.0.1")), "site-local 172.16");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.64.0.1")), "CGNAT lower");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.127.255.255")), "CGNAT upper");
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("8.8.8.8")), "public DNS");
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.63.0.1")), "below CGNAT 段");
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.128.0.1")), "above CGNAT 段");
|
||||
// 受限广播
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("255.255.255.255")), "受限广播");
|
||||
// IPv6 ULA fc00::/7
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("fc00::1")), "ULA fc00::");
|
||||
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("fd12:3456:789a::1")), "ULA fd::");
|
||||
// IPv6 公网放行
|
||||
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("2001:4860:4860::8888")), "Google IPv6 DNS");
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateHttpsUrlRejectsHttpAndPrivateHosts() {
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("http://example.com/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://localhost/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://127.0.0.1/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://10.0.0.5/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://192.168.1.1/a.jpg"));
|
||||
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
|
||||
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://172.16.0.1/a.jpg"));
|
||||
// 公网 https 应通过
|
||||
SimilarAsinImageEmbedder.validateHttpsUrl("https://m.media-amazon.com/images/I/abc.jpg");
|
||||
SimilarAsinImageEmbedder.validateHttpsUrl("https://cbu01.alicdn.com/img/ibank/xxx.jpg");
|
||||
}
|
||||
|
||||
@Test
|
||||
void downloadOversizeExceptionCarriesUrlAndSize() {
|
||||
SimilarAsinImageEmbedder.DownloadOversizeException ex =
|
||||
new SimilarAsinImageEmbedder.DownloadOversizeException("https://example.com/big.jpg", 12345678L);
|
||||
assertEquals("https://example.com/big.jpg", ex.url());
|
||||
assertEquals(12345678L, ex.size());
|
||||
assertTrue(ex.getMessage().contains("oversize"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void httpClientKeepsImageDownloadResilienceEnabled() {
|
||||
assertTrue(embedder.httpClient().retryOnConnectionFailure(), "OkHttp should retry transient connection failures");
|
||||
assertTrue(embedder.httpClient().followRedirects(), "image CDN redirects should be followed");
|
||||
assertTrue(embedder.httpClient().followSslRedirects(), "signed CDN downloads should follow browser-like SSL redirects");
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildImageRequestUsesBrowserLikeHeaders() {
|
||||
var req = SimilarAsinImageEmbedder.buildImageRequest("https://m.media-amazon.com/images/I/abc.jpg");
|
||||
|
||||
assertEquals("GET", req.method());
|
||||
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||
assertTrue(req.header("Accept").contains("image/"), "Accept should prefer images");
|
||||
assertEquals("https://www.amazon.com/", req.header("Referer"));
|
||||
assertEquals("no-cache", req.header("Cache-Control"));
|
||||
assertNotNull(req.header("Accept-Language"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void amazonImageDownloadCandidatesTrySmallerVariantsWithinRetryBudget() {
|
||||
List<String> candidates = SimilarAsinImageEmbedder.downloadCandidates(
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg");
|
||||
|
||||
assertEquals(List.of(
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg",
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SY450_.jpg",
|
||||
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX425_.jpg"), candidates);
|
||||
}
|
||||
|
||||
@Test
|
||||
void downloadCandidatesLeaveNonAmazonAndUnsignedUrlsUntouched() {
|
||||
String coze = "https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a";
|
||||
assertEquals(List.of(coze), SimilarAsinImageEmbedder.downloadCandidates(coze));
|
||||
assertEquals(List.of("https://cbu01.alicdn.com/O1CN01x.jpg"),
|
||||
SimilarAsinImageEmbedder.downloadCandidates("https://cbu01.alicdn.com/O1CN01x.jpg"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildImageRequestUsesDownloadHeadersForCozeSignedImages() {
|
||||
var req = SimilarAsinImageEmbedder.buildImageRequest(
|
||||
"https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a");
|
||||
|
||||
assertEquals("GET", req.method());
|
||||
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||
assertEquals("*/*", req.header("Accept"));
|
||||
assertNull(req.header("Referer"), "Coze signed download links should not carry an Amazon referer");
|
||||
assertEquals("no-cache", req.header("Cache-Control"));
|
||||
assertTrue(SimilarAsinImageEmbedder.isCozeSignedImageUrl(req.url().toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void errorSummaryDoesNotHideEmptyTimeoutMessage() {
|
||||
TimeoutException timeout = new TimeoutException();
|
||||
assertEquals("TimeoutException: <empty>", SimilarAsinImageEmbedder.errorSummary(timeout));
|
||||
|
||||
IOException io = new IOException("outer", new TimeoutException("inner"));
|
||||
String summary = SimilarAsinImageEmbedder.errorSummary(io);
|
||||
assertTrue(summary.contains("IOException: outer"));
|
||||
assertTrue(summary.contains("cause=TimeoutException: inner"));
|
||||
}
|
||||
|
||||
private static SimilarAsinProperties properties(int poolSize, int prefetchTimeoutSeconds, Path cacheDir) {
|
||||
SimilarAsinProperties properties = new SimilarAsinProperties();
|
||||
properties.setImageDownloadPoolSize(poolSize);
|
||||
properties.setImagePrefetchTimeoutSeconds(prefetchTimeoutSeconds);
|
||||
properties.setImageLocalCacheDir(cacheDir == null ? "" : cacheDir.toString());
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static OkHttpClient respondingClient(AtomicInteger calls, byte[] imageBytes) {
|
||||
return new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
calls.incrementAndGet();
|
||||
return response(chain.request(), 200, "OK", imageBytes);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
private static OkHttpClient failingClient(AtomicInteger calls) {
|
||||
return new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
calls.incrementAndGet();
|
||||
return response(chain.request(), 503, "Unavailable", new byte[0]);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Response response(okhttp3.Request request,
|
||||
int status,
|
||||
String message,
|
||||
byte[] body) {
|
||||
return new Response.Builder()
|
||||
.request(request)
|
||||
.protocol(Protocol.HTTP_1_1)
|
||||
.code(status)
|
||||
.message(message)
|
||||
.body(ResponseBody.create(body, MediaType.get("image/jpeg")))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static void replaceHttpClient(SimilarAsinImageEmbedder target,
|
||||
OkHttpClient httpClient) throws ReflectiveOperationException {
|
||||
Field field = SimilarAsinImageEmbedder.class.getDeclaredField("httpClient");
|
||||
field.setAccessible(true);
|
||||
field.set(target, httpClient);
|
||||
}
|
||||
|
||||
private static byte[] createJpegBytes() throws IOException {
|
||||
BufferedImage image = new BufferedImage(64, 48, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D graphics = image.createGraphics();
|
||||
try {
|
||||
graphics.setColor(new Color(0x24, 0x68, 0xAC));
|
||||
graphics.fillRect(0, 0, image.getWidth(), image.getHeight());
|
||||
} finally {
|
||||
graphics.dispose();
|
||||
}
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", output);
|
||||
return output.toByteArray();
|
||||
}
|
||||
|
||||
private static void deleteRecursively(Path directory) throws IOException {
|
||||
if (!Files.exists(directory)) {
|
||||
return;
|
||||
}
|
||||
try (var paths = Files.walk(directory)) {
|
||||
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
|
||||
Files.deleteIfExists(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
|
||||
/**
|
||||
* Task 20:Similar ASIN 端到端压测、JFR/GC 分析与结果文件兼容回归。
|
||||
* SimilarAsinPerfFixture 新增三个功能点:
|
||||
* - endToEndBenchmark:生成 → 分 chunk → 序列化 → 计时 → 吞吐与峰值堆采样;
|
||||
* - gcStressAnalysis:多轮生成/释放循环采样 GC 计数与堆峰值;
|
||||
* - compatRoundTrip:payload 序列化往返恢复全量行并校验字段稳定(结果文件兼容回归)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinPerfFixtureE2ETest {
|
||||
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_normal_default_path() {
|
||||
// 正常输入:1000 行端到端基准返回完整指标,行数/chunk 数正确,吞吐与堆峰值有界。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-default.xlsx", 1000, false, 200);
|
||||
|
||||
assertEquals(1000, metrics.rowCount(), "行数不丢失");
|
||||
assertEquals(5, metrics.chunkCount(), "1000 行 / 200 每 chunk = 5 个 chunk");
|
||||
assertTrue(metrics.payloadBytes() > 0, "payload 字节可采样");
|
||||
assertTrue(metrics.assembleMillis() >= 0);
|
||||
assertTrue(metrics.throughputRowsPerSec() > 0, "吞吐必须为正");
|
||||
assertTrue(metrics.peakHeapBytes() > 0, "峰值堆必须为正");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_normal_multiple_items() {
|
||||
// 批量场景:图片开/关两种模式 5000 行,结果不丢失、chunk 划分稳定。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics withImages =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-img.xlsx", 5000, true, 200);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics textOnly =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-text.xlsx", 5000, false, 200);
|
||||
|
||||
assertEquals(5000, withImages.rowCount());
|
||||
assertEquals(5000, textOnly.rowCount());
|
||||
assertEquals(25, withImages.chunkCount());
|
||||
assertEquals(25, textOnly.chunkCount());
|
||||
assertTrue(withImages.payloadBytes() > textOnly.payloadBytes(), "图片模式 payload 必须大于纯文本");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行:同一输入两次基准的指标一致,不产生重复行。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.CompatResult first =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-idem.xlsx", 1000, true);
|
||||
SimilarAsinPerfFixture.CompatResult second =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-idem.xlsx", 1000, true);
|
||||
|
||||
assertEquals(1000, first.rowCount());
|
||||
assertEquals(1000, first.recoveredCount(), "往返恢复全量行");
|
||||
assertTrue(first.fieldStable(), "字段必须稳定");
|
||||
assertEquals(first.rowCount(), second.rowCount());
|
||||
assertEquals(first.recoveredCount(), second.recoveredCount());
|
||||
assertEquals(first.fieldStable(), second.fieldStable());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_boundary_empty_input() {
|
||||
// 空输入:0 行基准返回零指标;0 行往返返回空结果,不创建资源。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-empty.xlsx", 0, false, 200);
|
||||
assertEquals(0, metrics.rowCount());
|
||||
assertEquals(0, metrics.chunkCount());
|
||||
assertEquals(0, metrics.payloadBytes());
|
||||
|
||||
SimilarAsinPerfFixture.CompatResult compat =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-empty.xlsx", 0, false);
|
||||
assertEquals(0, compat.rowCount());
|
||||
assertEquals(0, compat.recoveredCount());
|
||||
assertTrue(compat.fieldStable(), "空结果字段稳定");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_boundary_single_item() {
|
||||
// 单元素:1 行基准不依赖批量路径,往返字段一致。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-single.xlsx", 1, true, 200);
|
||||
assertEquals(1, metrics.rowCount());
|
||||
assertEquals(1, metrics.chunkCount());
|
||||
|
||||
SimilarAsinPerfFixture.CompatResult compat =
|
||||
fixture.compatRoundTrip("uploads/20260829/e2e-single.xlsx", 1, true);
|
||||
assertEquals(1, compat.recoveredCount());
|
||||
assertTrue(compat.fieldStable());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_boundary_limit_and_overflow() {
|
||||
// 上限/超限:超过 MAX_ROWS 拒绝;5000 行基准在预算内完成,不发生无界内存增长。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
IllegalArgumentException overflow = assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.endToEndBenchmark("uploads/20260829/e2e-over.xlsx",
|
||||
SimilarAsinPerfFixture.MAX_ROWS + 1, false, 200));
|
||||
assertTrue(overflow.getMessage().contains("rowCount"), "超限消息应可识别");
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.compatRoundTrip("uploads/20260829/e2e-over.xlsx",
|
||||
SimilarAsinPerfFixture.MAX_ROWS + 1, false));
|
||||
|
||||
SimilarAsinPerfFixture.EndToEndMetrics metrics =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-max.xlsx", 5000, true, 200);
|
||||
assertTrue(metrics.assembleMillis() < 15000,
|
||||
"5000 行端到端基准须在预算内完成,实际=" + metrics.assembleMillis() + "ms");
|
||||
assertTrue(metrics.peakHeapBytes() < 1024L * 1024L * 1024L, "峰值堆不得超过 1GB");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_invalid_input_rejected() {
|
||||
// 非法参数:null key/非法 chunkSize/非法 GC 行数 → 明确异常与可识别消息。
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
IllegalArgumentException nullKey = assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.endToEndBenchmark(null, 100, false, 200));
|
||||
assertTrue(nullKey.getMessage().contains("sourceFileKey"));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.endToEndBenchmark("uploads/20260829/x.xlsx", 100, false, 0));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.gcStressAnalysis(null, 100, false));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> fixture.gcStressAnalysis("uploads/20260829/x.xlsx", -1, false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_020_asin_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败:序列化失败抛 IllegalStateException 且不产生部分结果;恢复后重试成功;
|
||||
// GC 分析后堆峰值回落(临时对象释放)。
|
||||
AtomicInteger failCount = new AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (failCount.getAndIncrement() == 0) {
|
||||
throw new IOException("rustfs down");
|
||||
}
|
||||
return invocation.callRealMethod();
|
||||
}).when(objectMapper).writeValueAsBytes(any());
|
||||
|
||||
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> fixture.endToEndBenchmark("uploads/20260829/e2e-fail.xlsx", 100, false, 200));
|
||||
SimilarAsinPerfFixture.EndToEndMetrics recovered =
|
||||
fixture.endToEndBenchmark("uploads/20260829/e2e-fail.xlsx", 100, false, 200);
|
||||
assertEquals(100, recovered.rowCount(), "依赖恢复后重试成功");
|
||||
|
||||
SimilarAsinPerfFixture.GcStressSample gc =
|
||||
fixture.gcStressAnalysis("uploads/20260829/e2e-gc.xlsx", 1000, true);
|
||||
assertNotNull(gc);
|
||||
assertTrue(gc.rounds() >= 1, "GC 分析至少执行一轮");
|
||||
assertTrue(gc.gcCount() >= 0);
|
||||
assertTrue(gc.peakHeapBytes() > 0, "堆峰值必须可采样");
|
||||
assertTrue(gc.peakHeapBytes() < 1024L * 1024L * 1024L, "GC 分析峰值堆不得超过 1GB");
|
||||
}
|
||||
}
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Task 1:Similar ASIN 性能基线夹具(1000/5000 行、图片开关、chunk 数与 payload 大小采样)。
|
||||
* 先写测试确认 RED,再实现 SimilarAsinPerfFixture。
|
||||
*/
|
||||
class SimilarAsinPerfFixtureTest {
|
||||
|
||||
private final SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(new ObjectMapper());
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_default_path() {
|
||||
// 1000 行、带图片开关,默认 chunk 大小
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/base.xlsx", 1000, true);
|
||||
assertEquals(1000, rows.size());
|
||||
// 行必须包含 url 图片地址
|
||||
assertFalse(rows.get(0).getUrl().isBlank());
|
||||
assertTrue(rows.get(0).getUrl().startsWith("http"));
|
||||
// rowToken 稳定且唯一
|
||||
assertEquals(rows.get(0).getRowToken(), fixture.rowTokenFor(rows.get(0).getSourceFileKey(), rows.get(0).getRowIndex()));
|
||||
assertEquals(1000, rows.stream().map(SimilarAsinParsedRowVo::getRowToken).distinct().count());
|
||||
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
|
||||
assertEquals(5, chunks.size());
|
||||
assertEquals(200, chunks.get(0).size());
|
||||
|
||||
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
|
||||
assertEquals(1000, metrics.rowCount());
|
||||
assertEquals(5, metrics.chunkCount());
|
||||
assertTrue(metrics.payloadBytes() > 0, "payload 采样字节数必须大于 0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_multiple_items() {
|
||||
// 5000 行批量场景:顺序稳定、chunk 数正确、结果不丢失
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/multi.xlsx", 5000, false);
|
||||
assertEquals(5000, rows.size());
|
||||
assertTrue(rows.get(0).getUrl().isBlank(), "图片开关关闭时 url 必须为空");
|
||||
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
|
||||
assertEquals(25, chunks.size());
|
||||
// 顺序稳定:拼接后与原始一致
|
||||
List<SimilarAsinParsedRowVo> restored = chunks.stream().flatMap(List::stream).toList();
|
||||
assertEquals(rows.size(), restored.size());
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
assertEquals(rows.get(i).getRowToken(), restored.get(i).getRowToken());
|
||||
}
|
||||
// 所有行 rowIndex 递增
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
assertEquals(i + 1, rows.get(i).getRowIndex());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_repeated_operation_is_idempotent() {
|
||||
List<SimilarAsinParsedRowVo> first = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
|
||||
List<SimilarAsinParsedRowVo> second = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
|
||||
// 同一输入重复生成:token 完全一致,不产生重复差异
|
||||
assertEquals(first.size(), second.size());
|
||||
for (int i = 0; i < first.size(); i++) {
|
||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
||||
assertEquals(first.get(i).getAsin(), second.get(i).getAsin());
|
||||
}
|
||||
// splitChunks 幂等:两次划分 chunk 数一致
|
||||
assertEquals(fixture.splitChunks(first, 200).size(), fixture.splitChunks(second, 200).size());
|
||||
// 采样指标幂等
|
||||
SimilarAsinPerfFixture.Metrics m1 = fixture.samplePayload(first, true, 200);
|
||||
SimilarAsinPerfFixture.Metrics m2 = fixture.samplePayload(second, true, 200);
|
||||
assertEquals(m1.payloadBytes(), m2.payloadBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_empty_input() {
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/empty.xlsx", 0, true);
|
||||
assertNotNull(rows);
|
||||
assertEquals(0, rows.size());
|
||||
// 空行 splitChunks 返回空,不产生无效 chunk
|
||||
assertEquals(0, fixture.splitChunks(rows, 200).size());
|
||||
// 空行采样:行数 0、chunk 0、payload 字节数为 0(不创建任何载荷)
|
||||
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
|
||||
assertEquals(0, metrics.rowCount());
|
||||
assertEquals(0, metrics.chunkCount());
|
||||
assertEquals(0, metrics.payloadBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_single_item() {
|
||||
// 单行:不依赖批量路径且结果正确
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/single.xlsx", 1, true);
|
||||
assertEquals(1, rows.size());
|
||||
assertEquals(1, rows.get(0).getRowIndex());
|
||||
assertEquals(1, fixture.splitChunks(rows, 200).size());
|
||||
// 单行 chunk 划分后仍只含 1 行
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
|
||||
assertEquals(1, chunks.get(0).size());
|
||||
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
|
||||
assertEquals(1, metrics.rowCount());
|
||||
assertEquals(1, metrics.chunkCount());
|
||||
assertTrue(metrics.payloadBytes() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_limit_and_overflow() {
|
||||
// 超过最大行数(5000)时拒绝,不发生无界内存增长
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/overflow.xlsx", 5001, true));
|
||||
// 达到最大允许值 5000 时允许
|
||||
assertEquals(5000, fixture.generateRows("uploads/20260829/max.xlsx", 5000, true).size());
|
||||
// chunkSize 非法值拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/a.xlsx", 100, true), 0));
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/b.xlsx", 100, true), -1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_invalid_input_rejected() {
|
||||
// null 文件 key 拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(null, 100, true));
|
||||
// 空白文件 key 拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(" ", 100, true));
|
||||
// 负行数拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/neg.xlsx", -1, true));
|
||||
// null 行集合分块拒绝
|
||||
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(null, 200));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_dependency_failure_releases_resources() {
|
||||
// 采样时序列化器失败(mock 抛异常):错误可恢复,不产生部分结果
|
||||
ObjectMapper broken = new ObjectMapper() {
|
||||
@Override
|
||||
public String writeValueAsString(Object value) {
|
||||
throw new IllegalStateException("serializer down");
|
||||
}
|
||||
};
|
||||
SimilarAsinPerfFixture failingFixture = new SimilarAsinPerfFixture(broken);
|
||||
List<SimilarAsinParsedRowVo> rows = failingFixture.generateRows("uploads/20260829/fail.xlsx", 1000, true);
|
||||
assertThrows(IllegalStateException.class, () -> failingFixture.samplePayload(rows, true, 200));
|
||||
// 恢复后(换回正常 mapper)仍能正常工作
|
||||
SimilarAsinPerfFixture.Metrics recovered = fixture.samplePayload(fixture.generateRows("uploads/20260829/recover.xlsx", 1000, true), true, 200);
|
||||
assertTrue(recovered.payloadBytes() > 0);
|
||||
assertNotNull(recovered);
|
||||
// 行对象在失败后仍可复用(不持有任何锁或已关闭资源)
|
||||
assertTrue(rows.get(0).getAsin().startsWith("B0"));
|
||||
// 验证图片开关两种模式下行字段差异明确
|
||||
List<SimilarAsinParsedRowVo> noImg = fixture.generateRows("uploads/20260829/nimg.xlsx", 10, false);
|
||||
assertTrue(noImg.get(0).getUrl().isBlank());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_normal_fields_populated() {
|
||||
// 行字段完整性:asin/country/sku/title/values 均填充且稳定
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
|
||||
SimilarAsinParsedRowVo row = rows.get(3);
|
||||
assertTrue(row.getAsin().matches("B0[A-Z0-9]{8}"));
|
||||
assertFalse(row.getCountry().isBlank());
|
||||
assertFalse(row.getSku().isBlank());
|
||||
assertFalse(row.getTitle().isBlank());
|
||||
assertNotNull(row.getValues());
|
||||
assertFalse(row.getValues().isEmpty());
|
||||
assertTrue(row.getValues().containsKey("asin"));
|
||||
assertTrue(row.getValues().containsKey("国家"));
|
||||
// 行号与 sourceId 关联正确
|
||||
assertEquals("4", row.getSourceId());
|
||||
assertEquals(4, row.getRowIndex());
|
||||
// values 是独立副本,修改不影响后续生成
|
||||
row.getValues().put("价格", "999");
|
||||
List<SimilarAsinParsedRowVo> again = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
|
||||
assertTrue(!again.get(3).getValues().getOrDefault("价格", "").equals("999"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_001_payload_chunk_image_boundary_chunk_size_edge() {
|
||||
// chunk 边界:行数恰好整除 / 有余数 / 单 chunk 放不下
|
||||
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/edge.xlsx", 100, false);
|
||||
// 100 行 / 40 → 3 chunks(40+40+20)
|
||||
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 40);
|
||||
assertEquals(3, chunks.size());
|
||||
assertEquals(40, chunks.get(0).size());
|
||||
assertEquals(20, chunks.get(2).size());
|
||||
// chunkSize 大于总行数 → 单 chunk
|
||||
assertEquals(1, fixture.splitChunks(rows, 500).size());
|
||||
// chunkSize 恰好等于行数 → 单 chunk 全量
|
||||
assertEquals(1, fixture.splitChunks(rows, 100).size());
|
||||
assertEquals(100, fixture.splitChunks(rows, 100).get(0).size());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user