task-98: 移除 similar-asin/appearance-patent 模块 Coze,状态机与共享组件改名 LLM

- similarasin/appearancepatent 模块全部 Coze 工作流调用改走 direct-LLM(已确认唯一运行路径)
- 共享组件改名:CozeTaskQueueGate→TaskQueueGate、CozeGroupResultPropagator→GroupResultPropagator
- 状态机改名:biz_task_scope_state 的 coze_* 列→llm_*、stateJson coze 键→llm(V100 迁移已应用生产)
- 删除 biz_coze_credential 表、CozeCredential* 类、SimilarAsinCozeClient、AppearancePatentCozeClient→LlmClient
- 前端 brand 页 Coze 文案→LLM;Python 后端删除 cozepy 依赖与死配置
- 修复 TaskResultFileJobWorker 启动失败:TaskFileJobConfig 注册 ResultFileJobHandlerRegistry 与 13 个 handler bean(含 validateCoverage 启动校验)
This commit is contained in:
2026-09-01 02:19:45 +08:00
parent acdb1dd030
commit da0eaccf3b
109 changed files with 6903 additions and 4582 deletions
@@ -1,156 +0,0 @@
package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinLogSupport;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Task 19:Coze 请求/响应及 Python 回传日志改为采样、截断和 DEBUG 级别。
* 新工具 SimilarAsinLogSupport 提供两条纯函数:
* - truncate:正文超限截断为 maxChars + 后缀,长文本不占满日志;
* - shouldLog:每 everyN 次采样一次(counter % everyN == 0),控制轮询/逐行日志量。
* Coze 客户端正文日志与 Python 回传逐行日志经该工具后输出有界、可识别。
*/
class SimilarAsinCozeClientLoggingTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void test_task_019_logging_normal_default_path() {
// 正常输入:短文本不截断;每次采样(everyN=1)恒记录。
assertEquals("hello", SimilarAsinLogSupport.truncate("hello", 100));
assertEquals("", SimilarAsinLogSupport.truncate(null, 100), "null 文本返回空串");
assertTrue(SimilarAsinLogSupport.shouldLog(0, 1), "everyN=1 恒采样");
assertTrue(SimilarAsinLogSupport.shouldLog(7, 1), "everyN=1 不抑制任何计数");
}
@Test
void test_task_019_logging_normal_multiple_items() throws Exception {
// 批量场景:多个长文本各自截断、结果互不影响;采样按每 everyN 次一次。
String longA = "A".repeat(3000);
String longB = "B".repeat(5000);
String truncatedA = SimilarAsinLogSupport.truncate(longA, 100);
String truncatedB = SimilarAsinLogSupport.truncate(longB, 100);
assertTrue(truncatedA.startsWith("A".repeat(100)));
assertTrue(truncatedB.startsWith("B".repeat(100)));
assertTrue(truncatedA.length() < longA.length(), "截断后必须短于原文");
int sampled = 0;
for (int i = 0; i < 30; i++) {
if (SimilarAsinLogSupport.shouldLog(i, 10)) {
sampled++;
}
}
assertEquals(3, sampled, "everyN=10 在 0..29 内应采样 0/10/20 共 3 次");
}
@Test
void test_task_019_logging_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行:同一文本多次截断结果一致;同一计数采样判定一致。
String text = "x".repeat(1234);
String first = SimilarAsinLogSupport.truncate(text, 500);
String second = SimilarAsinLogSupport.truncate(text, 500);
assertEquals(first, second, "重复截断必须产生相同输出");
assertEquals(first, SimilarAsinLogSupport.truncate(text, 500), "截断幂等");
assertEquals(SimilarAsinLogSupport.shouldLog(20, 10), SimilarAsinLogSupport.shouldLog(20, 10));
}
@Test
void test_task_019_logging_boundary_empty_input() throws Exception {
// 空输入:null/空串安全返回空串;空白串按长度截断语义原样保留。
assertEquals("", SimilarAsinLogSupport.truncate(null, 100));
assertEquals("", SimilarAsinLogSupport.truncate("", 100));
assertEquals(" ", SimilarAsinLogSupport.truncate(" ", 100), "空白串不做 trim,按原样返回");
assertTrue(SimilarAsinLogSupport.shouldLog(0, 10), "计数 0 必须采样(首条不丢)");
}
@Test
void test_task_019_logging_boundary_single_item() throws Exception {
// 单元素边界:恰好等于上限不截断;超 1 字符截断并带长度后缀。
String exact = "y".repeat(100);
assertEquals(exact, SimilarAsinLogSupport.truncate(exact, 100), "恰好等于上限不截断");
String over = "y".repeat(101);
String truncated = SimilarAsinLogSupport.truncate(over, 100);
assertEquals(over.substring(0, 100), truncated.substring(0, 100), "截断保留前缀");
assertTrue(truncated.contains("101"), "截断输出应携带原文长度");
}
@Test
void test_task_019_logging_boundary_limit_and_overflow() throws Exception {
// 上限/超限:10 万字符文本截断后有界、不再无界增长;采样 everyN 超限不抑制。
String huge = "z".repeat(100_000);
String truncated = SimilarAsinLogSupport.truncate(huge, 2000);
assertTrue(truncated.length() < 2200, "截断输出必须有界,实际=" + truncated.length());
assertTrue(truncated.length() > 2000, "应保留 2000 前缀 + 后缀");
assertTrue(truncated.endsWith("]"), "截断输出带可识别后缀");
assertTrue(SimilarAsinLogSupport.shouldLog(0, Integer.MAX_VALUE), "计数 0 在超大 everyN 下仍采样");
assertFalse(SimilarAsinLogSupport.shouldLog(1, Integer.MAX_VALUE), "非零计数在超大 everyN 下抑制");
assertFalse(SimilarAsinLogSupport.shouldLog(31, 10), "非采样点必须被抑制");
}
@Test
void test_task_019_logging_invalid_input_rejected() throws Exception {
// 非法参数:maxChars ≤ 0 时原样返回(不截断);null 文本始终空串。
String text = "invalid-max";
assertEquals(text, SimilarAsinLogSupport.truncate(text, 0), "maxChars=0 不截断");
assertEquals(text, SimilarAsinLogSupport.truncate(text, -1), "负上限不截断");
assertEquals("", SimilarAsinLogSupport.truncate(null, -5));
assertTrue(SimilarAsinLogSupport.shouldLog(5, 0), "everyN=0 视为恒采样");
assertTrue(SimilarAsinLogSupport.shouldLog(5, -3), "负 everyN 视为恒采样");
}
@Test
void test_task_019_logging_dependency_failure_releases_resources() throws Exception {
// 依赖失败:含代理对(emoji)的长文本截断不抛异常、不产生孤立代理项;
// 计数接近 Long.MAX_VALUE 不溢出;掩码后的请求体经截断管线输出有界且不泄漏密钥。
String emoji = "🚀".repeat(3000);
String truncatedEmoji = SimilarAsinLogSupport.truncate(emoji, 100);
assertNotNull(truncatedEmoji);
assertTrue(truncatedEmoji.length() < emoji.length(), "代理对文本必须被截断");
assertFalse(SimilarAsinLogSupport.shouldLog(Long.MAX_VALUE, 10), "极大计数采样判定不抛异常");
List<SimilarAsinResultRowDto> rows = new java.util.ArrayList<>();
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0SECRET1");
row.setUrl("https://m.media-amazon.com/images/I/" + "U".repeat(500) + ".jpg");
row.setTitle("T".repeat(5000));
row.setSku("SKU-SECRET");
rows.add(row);
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
Method maskMethod = SimilarAsinCozeClient.class.getDeclaredMethod(
"maskCozeRequestBody", Map.class);
maskMethod.setAccessible(true);
Method buildMethod = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
buildMethod.setAccessible(true);
@SuppressWarnings("unchecked")
Map<String, Object> parameters = (Map<String, Object>) buildMethod.invoke(client, rows, "", "supersecretkey", true);
Map<String, Object> body = new java.util.LinkedHashMap<>();
body.put("workflow_id", "wf-1");
body.put("parameters", parameters);
body.put("api_key", "supersecretkey");
@SuppressWarnings("unchecked")
Map<String, Object> masked = (Map<String, Object>) maskMethod.invoke(client, body);
Method writeMethod = SimilarAsinCozeClient.class.getDeclaredMethod("writeJson", Object.class);
writeMethod.setAccessible(true);
String maskedJson = (String) writeMethod.invoke(client, masked);
String logged = SimilarAsinLogSupport.truncate(maskedJson, 2000);
assertTrue(logged.length() < maskedJson.length(), "超长掩码 body 必须截断");
assertFalse(logged.contains("supersecretkey"), "日志不得泄漏完整 api_key");
assertTrue(logged.contains("B0SECRET1"), "截断保留正文关键字段");
assertTrue(logged.length() < 2500, "截断输出必须有界");
}
}
@@ -1,240 +0,0 @@
package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SimilarAsinCozeClientTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
@SuppressWarnings("unchecked")
void buildParametersIncludesAlibabaPriceFromPythonPayload() throws Exception {
List<SimilarAsinResultRowDto> rows = objectMapper.readValue("""
[
{
"asin": "B0TEST123",
"url": "https://m.media-amazon.com/images/I/main.jpg",
"alibaba": [
{"url": "https://cbu01.alicdn.com/img/ibank/a.jpg", "price": 12.80},
{"url": "https://cbu01.alicdn.com/img/ibank/b.jpg", "price": 19.99}
],
"title": "Test title",
"sku": "SKU-1"
}
]
""", new TypeReference<>() {
});
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, rows, "", "", true);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
Map<String, Object> item = items.getFirst();
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) item.get("alibaba");
assertEquals("B0TEST123", item.get("asin"));
assertEquals("https://m.media-amazon.com/images/I/main.jpg", item.get("url"));
assertEquals("Test title", item.get("title"));
assertEquals("SKU-1", item.get("sku"));
assertEquals(2, alibaba.size());
assertEquals("https://cbu01.alicdn.com/img/ibank/a.jpg", alibaba.get(0).get("url"));
assertEquals(0, new BigDecimal("12.8").compareTo((BigDecimal) alibaba.get(0).get("price")));
assertEquals("https://cbu01.alicdn.com/img/ibank/b.jpg", alibaba.get(1).get("url"));
assertEquals(0, new BigDecimal("19.99").compareTo((BigDecimal) alibaba.get(1).get("price")));
assertTrue(rows.getFirst().hasImageUrl());
}
@Test
@SuppressWarnings("unchecked")
void buildParametersFallsBackAlibabaFromLegacyUrlsAndTopLevelPrice() throws Exception {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0LEGACY1");
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
row.setUrls(List.of("https://cbu01.alicdn.com/img/ibank/legacy-a.jpg"));
row.setPrice("8.50");
row.setTitle("Legacy title");
row.setSku("SKU-LEGACY");
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, List.of(row), "", "", false);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
assertEquals(1, alibaba.size());
assertEquals("https://cbu01.alicdn.com/img/ibank/legacy-a.jpg", alibaba.getFirst().get("url"));
assertEquals(0, new BigDecimal("8.5").compareTo((BigDecimal) alibaba.getFirst().get("price")));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersConvertsBlankAlibabaPricesToZero() throws Exception {
List<SimilarAsinResultRowDto> rows = objectMapper.readValue("""
[
{
"asin": "B0BLANK123",
"url": "https://m.media-amazon.com/images/I/main.jpg",
"alibaba": [
{"url": "https://cbu01.alicdn.com/img/ibank/blank.jpg", "price": ""},
{"url": "https://cbu01.alicdn.com/img/ibank/null.jpg", "price": null}
]
}
]
""", new TypeReference<>() {
});
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, rows, "", "", true);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
String json = objectMapper.writeValueAsString(parameters);
assertEquals(2, alibaba.size());
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) alibaba.get(0).get("price")));
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) alibaba.get(1).get("price")));
assertTrue(json.contains("\"price\":0"));
assertFalse(json.contains("\"price\":\"\""));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersConvertsBlankFallbackRowPriceToZero() throws Exception {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0BLANKROW");
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
row.setUrls(List.of("https://cbu01.alicdn.com/img/ibank/fallback.jpg"));
row.setPrice("");
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, List.of(row), "", "", true);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
String json = objectMapper.writeValueAsString(parameters);
assertEquals(1, alibaba.size());
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) alibaba.getFirst().get("price")));
assertTrue(json.contains("\"price\":0"));
assertFalse(json.contains("\"price\":\"\""));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersSerializesWholeNumberPricesWithoutScientificNotation() throws Exception {
List<SimilarAsinResultRowDto> rows = objectMapper.readValue("""
[
{
"asin": "B0SCI123",
"url": "https://m.media-amazon.com/images/I/main.jpg",
"alibaba": [
{"url": "https://cbu01.alicdn.com/img/ibank/ten.jpg", "price": 10.0},
{"url": "https://cbu01.alicdn.com/img/ibank/eighty.jpg", "price": 80.0},
{"url": "https://cbu01.alicdn.com/img/ibank/fraction.jpg", "price": 7.5800}
]
}
]
""", new TypeReference<>() {
});
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, rows, "", "", true);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
String json = objectMapper.writeValueAsString(parameters);
assertEquals("10", ((BigDecimal) alibaba.get(0).get("price")).toPlainString());
assertEquals("80", ((BigDecimal) alibaba.get(1).get("price")).toPlainString());
assertEquals("7.58", ((BigDecimal) alibaba.get(2).get("price")).toPlainString());
assertFalse(json.contains("1E+1"));
assertFalse(json.contains("8E+1"));
assertTrue(json.contains("\"price\":10"));
assertTrue(json.contains("\"price\":80"));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersIncludesCategorySwitch() throws Exception {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0CATEGORY1");
row.setTitle("Category test");
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, List.of(row), "", "", false, true);
assertEquals(Boolean.TRUE, parameters.get("category_switch"));
assertEquals(Boolean.FALSE, parameters.get("img_switch"));
}
@Test
void imageOnlyWorkflowOutputIsExtractedAndMergedByAsin() throws Exception {
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null, null);
String imageData = """
{"data":[{
"asin":"B0BQNHDP2F",
"main_url":"https://example.com/main.jpg",
"puzzle_img1":"https://example.com/puzzle-1.jpg",
"puzzle_img2":"https://example.com/puzzle-2.jpg"
}]}
""";
String workflowOutput = objectMapper.writeValueAsString(Map.of(
"node_status", "{}",
"Output", imageData));
var historyResponse = objectMapper.createObjectNode();
historyResponse.put("code", 0);
historyResponse.putArray("data")
.addObject()
.put("execute_status", "Success")
.put("output", workflowOutput);
Method extract = SimilarAsinCozeClient.class.getDeclaredMethod("extractResultDataText", JsonNode.class);
extract.setAccessible(true);
String dataText = (String) extract.invoke(client, historyResponse);
SimilarAsinResultRowDto source = new SimilarAsinResultRowDto();
source.setAsin("B0BQNHDP2F");
List<SimilarAsinResultRowDto> merged = client.mergeRowsFromDataText(List.of(source), dataText);
assertFalse(dataText.isBlank());
assertEquals(1, merged.size());
assertEquals("https://example.com/main.jpg", merged.getFirst().getMainUrl());
assertEquals("https://example.com/puzzle-1.jpg", merged.getFirst().getPuzzleImg1());
assertEquals("https://example.com/puzzle-2.jpg", merged.getFirst().getPuzzleImg2());
Method resolvedCount = SimilarAsinCozeClient.class.getDeclaredMethod("resolvedCount", List.class);
resolvedCount.setAccessible(true);
assertEquals(1, resolvedCount.invoke(client, merged));
}
}
@@ -2,7 +2,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
@@ -71,7 +70,6 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@@ -81,7 +79,6 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService cozeCredentialPoolService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@@ -168,11 +165,11 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task,
String scopeHash, Integer chunkIndex,
List<SimilarAsinResultRowDto> cozeRows) throws Exception {
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeCozeRowsIntoChunk",
List<SimilarAsinResultRowDto> llmRows) throws Exception {
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, scopeHash, chunkIndex, cozeRows, Map.of());
merge.invoke(service, task, scopeHash, chunkIndex, llmRows, Map.of());
}
@Test
@@ -184,12 +181,12 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> cozeRows = List.of(
List<SimilarAsinResultRowDto> llmRows = List.of(
row("r1", "B0A0000001", "标题1"),
row("r2", "B0A0000002", "标题2"),
row("r3", "B0A0000003", "标题3"));
invokeMerge(service, task, "hashA", 1, cozeRows);
invokeMerge(service, task, "hashA", 1, llmRows);
assertNotNull(storedPayload.get());
assertTrue(storedPayload.get().contains("\"r1\"") && storedPayload.get().contains("\"r3\""),
"上限内合并必须完整保留存量行与新增行,实际: " + storedPayload.get());
@@ -243,9 +240,9 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> cozeRows = List.of(row("r1", "B0A0000001", "标题1"));
invokeMerge(service, task, "hashA", 1, cozeRows);
invokeMerge(service, task, "hashA", 1, cozeRows);
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "B0A0000001", "标题1"));
invokeMerge(service, task, "hashA", 1, llmRows);
invokeMerge(service, task, "hashA", 1, llmRows);
assertEquals(2, storeCalls.get(), "重复执行同一输入:每次 merge 恰好写回一次,无多余请求");
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@@ -298,13 +295,13 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> cozeRows = new ArrayList<>();
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 0; i < 5; i++) {
cozeRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
llmRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
}
// Phase A:行数超限(上限 2,存量 1 + 新增 5)→ 只保留上限内最新行,超限部分转 orphan。
invokeMerge(service, task, "hashA", 1, cozeRows);
invokeMerge(service, task, "hashA", 1, llmRows);
assertTrue(lastStoredPayload.get().contains("\"r4\"") && lastStoredPayload.get().contains("\"r5\""),
"行数超限时保留上限内的最新行,实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "行数超限时最旧行被降级,实际: " + lastStoredPayload.get());
@@ -317,7 +314,7 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
long oneRowBytes = rowsJson(List.of(row("r9", "B0A0000099", "样本行"))).getBytes(StandardCharsets.UTF_8).length;
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(oneRowBytes + 5L);
invokeMerge(service, task, "hashA", 1, cozeRows);
invokeMerge(service, task, "hashA", 1, llmRows);
assertTrue(lastStoredPayload.get().contains("\"r5\""), "字节超限时保留最新行,实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "字节超限时最旧行被降级,实际: " + lastStoredPayload.get());
assertEquals(2, storeCalls.get());
@@ -327,7 +324,7 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(5L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, cozeRows);
invokeMerge(service, task, "hashA", 1, llmRows);
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
@@ -2,8 +2,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
@@ -59,8 +57,8 @@ import static org.mockito.Mockito.when;
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1";Task 12 扩展为:
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
* 3) 统一走 bufferCozeRowsOrMerge:缓冲失败回退立即 merge,结果不丢失。
* flushBufferedCozeResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
* 3) 统一走 bufferLlmRowsOrMerge:缓冲失败回退立即 merge,结果不丢失。
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceCozeBufferScopeTest {
@@ -78,7 +76,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinLlmService similarAsinLlmService;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@@ -88,7 +86,6 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService cozeCredentialPoolService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@@ -113,24 +110,12 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(properties.getCozeBatchSize()).thenReturn(5);
lenient().when(properties.getCozeTextOnlyBatchSize()).thenReturn(10);
lenient().when(properties.isCozeResultBufferEnabled()).thenReturn(true);
lenient().when(properties.getCozePollTimeoutMillis()).thenReturn(30_000);
lenient().when(properties.getLlmBatchSize()).thenReturn(5);
lenient().when(properties.getLlmTextOnlyBatchSize()).thenReturn(10);
lenient().when(properties.isLlmResultBufferEnabled()).thenReturn(true);
lenient().when(properties.getDbJobTouchIntervalMillis()).thenReturn(2_000L);
lenient().when(properties.getDbTaskTouchIntervalMillis()).thenReturn(2_000L);
lenient().when(properties.getCozeSubmitLockWaitMillis()).thenReturn(1_000L);
lenient().when(properties.getCozeSubmitLockRetryDelayMillis()).thenReturn(100L);
lenient().when(properties.getCozeSubmitMinIntervalMillis()).thenReturn(0L);
lenient().when(properties.getCozeSubmitMaxRetryCount()).thenReturn(3);
lenient().when(properties.getCozeFlushPendingMinutes()).thenReturn(10);
lenient().when(cozeClient.configuredCredentialCount()).thenReturn(1);
lenient().when(cozeClient.nextCredential()).thenReturn(new SimilarAsinCozeClient.CozeCredentialRef(
CREDENTIAL, "wf-1", "token-1", 4));
lenient().when(cozeClient.credentialByName(CREDENTIAL)).thenReturn(new SimilarAsinCozeClient.CozeCredentialRef(
CREDENTIAL, "wf-1", "token-1", 4));
lenient().when(cozeCredentialPoolService.borrow(eq(MODULE), any())).thenReturn(
mock(CozeCredentialPoolService.BorrowedCredential.class));
lenient().when(properties.getLlmFlushPendingMinutes()).thenReturn(10);
lenient().when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(
mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
lenient().doAnswer(invocation -> {
@@ -192,7 +177,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
state.setTaskId(task.getId());
state.setModuleType(MODULE);
state.setScopeHash("scope-" + id);
state.setCozeStatus(status);
state.setLlmStatus(status);
state.setParsedPayloadJson("ptr:batch-" + id);
state.setStateJson("{\"jobId\":7101,\"resultId\":7201,\"chunkScopeHash\":null,\"chunkIndex\":null,"
+ "\"batchIndex\":1,\"batchTotal\":" + batchTotal + ",\"ownerInstanceId\":\"test-instance\","
@@ -204,8 +189,8 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
return new ObjectMapper().writeValueAsString(rows);
}
private SimilarAsinTaskService.CozeBatchContext context(int batchTotal) {
return new SimilarAsinTaskService.CozeBatchContext(
private SimilarAsinTaskService.LlmBatchContext context(int batchTotal) {
return new SimilarAsinTaskService.LlmBatchContext(
7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null);
}
@@ -235,14 +220,14 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
@Test
void test_task_012_payload_chunk_normal_default_path() throws Exception {
// 正常输入:DONE 结果(batchTotal=1 单 batch)经 bufferCozeRowsOrMerge 走缓冲,
// 正常输入:DONE 结果(batchTotal=1 单 batch)经 bufferLlmRowsOrMerge 走缓冲,
// 不立即写 chunk;缓冲失败回退立即 merge 结果不丢失。
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.CozeBatchContext.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
@@ -278,7 +263,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1")));
when(taskChunkMapper.update(any(), any())).thenReturn(1);
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushBufferedCozeResults", Long.class);
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
flush.setAccessible(true);
flush.invoke(service, 7104L);
@@ -294,9 +279,9 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.CozeBatchContext.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
TaskScopeStateEntity state = state(task, 1L, "DONE", 2);
@@ -313,9 +298,9 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
void test_task_012_payload_chunk_boundary_empty_input() throws Exception {
// 空输入:无行时缓冲与 merge 都不发生,不创建无效资源
FileTaskEntity task = task();
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.CozeBatchContext.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of());
@@ -330,9 +315,9 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
// 单 batch(batchTotal=1):原 P0-3 例外,现在也缓冲
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.CozeBatchContext.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
@@ -347,11 +332,11 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
// 缓冲开关关闭:回退立即 merge,DONE 结果仍落 chunk 不丢失
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
when(properties.isCozeResultBufferEnabled()).thenReturn(false);
when(properties.isLlmResultBufferEnabled()).thenReturn(false);
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.CozeBatchContext.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
@@ -369,9 +354,9 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)))
.thenThrow(new IllegalStateException("rustfs full"));
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.CozeBatchContext.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
@@ -408,7 +393,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
return "ptr:stored-" + invocation.getArgument(3);
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushBufferedCozeResults", Long.class);
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
flush.setAccessible(true);
Exception ex = assertThrows(Exception.class, () -> {
try {
@@ -3,7 +3,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
@@ -69,7 +68,6 @@ class SimilarAsinTaskServiceGroupRefTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@@ -3,7 +3,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
@@ -64,7 +63,6 @@ class SimilarAsinTaskServiceParseLimitsTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@@ -3,7 +3,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
@@ -66,7 +65,6 @@ class SimilarAsinTaskServiceParsePreviewTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@@ -3,7 +3,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
@@ -65,7 +64,6 @@ class SimilarAsinTaskServicePreviewConfigTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@@ -2,7 +2,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
@@ -65,7 +64,6 @@ class SimilarAsinTaskServiceRowKeyDedupeTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@@ -75,7 +73,6 @@ class SimilarAsinTaskServiceRowKeyDedupeTest {
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService cozeCredentialPoolService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@@ -157,7 +154,7 @@ class SimilarAsinTaskServiceRowKeyDedupeTest {
@Test
void test_task_011_merge_row_key_normal_default_path() throws Exception {
// 正常输入:cozeRows 含同一 rowKey 的重复行,merge 前按稳定 rowKey 去重,
// 正常输入:llmRows 含同一 rowKey 的重复行,merge 前按稳定 rowKey 去重,
// chunk payload 只写一次,结果行不重复。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storedCounter = new AtomicLong(0);
@@ -165,18 +162,18 @@ class SimilarAsinTaskServiceRowKeyDedupeTest {
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
List<SimilarAsinResultRowDto> cozeRows = List.of(
List<SimilarAsinResultRowDto> llmRows = List.of(
row("r1", "1", "B0A0000001", "英国"),
row("r1", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(cozeRows);
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
assertEquals(1, deduped.size(), "重复行必须按稳定 rowKey 去重");
assertEquals("r1", deduped.get(0).getRowToken());
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeCozeRowsIntoChunk",
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, null, null, cozeRows, Map.of());
merge.invoke(service, task, null, null, llmRows, Map.of());
assertEquals(1, storedCounter.get(), "去重后 chunk 只写一次");
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
@@ -198,34 +195,34 @@ class SimilarAsinTaskServiceRowKeyDedupeTest {
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
List<SimilarAsinResultRowDto> cozeRows = new ArrayList<>();
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 0; i < 3; i++) {
cozeRows.add(row("r1", "1", "B0A0000001", "英国"));
cozeRows.add(row("r2", "2", "B0A0000002", "英国"));
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
llmRows.add(row("r2", "2", "B0A0000002", "英国"));
}
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(cozeRows);
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
assertEquals(2, deduped.size(), "3 轮重复输入去重后只剩 2 个唯一行");
assertEquals(List.of("r1", "r2"), deduped.stream().map(SimilarAsinResultRowDto::getRowToken).toList(),
"去重必须保留首次出现顺序");
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeCozeRowsIntoChunk",
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, null, null, cozeRows, Map.of());
merge.invoke(service, task, null, null, llmRows, Map.of());
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
@Test
void test_task_011_merge_row_key_normal_repeated_operation_is_idempotent() {
// 重复执行同一输入:去重结果完全一致,不产生重复记录
List<SimilarAsinResultRowDto> cozeRows = List.of(
List<SimilarAsinResultRowDto> llmRows = List.of(
row("r1", "1", "B0A0000001", "英国"),
row("r2", "2", "B0A0000002", "英国"),
row("r1", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> first = service.dedupeRowsByRowKey(cozeRows);
List<SimilarAsinResultRowDto> second = service.dedupeRowsByRowKey(cozeRows);
List<SimilarAsinResultRowDto> first = service.dedupeRowsByRowKey(llmRows);
List<SimilarAsinResultRowDto> second = service.dedupeRowsByRowKey(llmRows);
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
@@ -258,11 +255,11 @@ class SimilarAsinTaskServiceRowKeyDedupeTest {
@Test
void test_task_011_merge_row_key_boundary_limit_and_overflow() {
// 大批量:1000 行全部重复,去重后只剩 1 个唯一行,无无界内存增长
List<SimilarAsinResultRowDto> cozeRows = new ArrayList<>();
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 0; i < 1000; i++) {
cozeRows.add(row("r1", "1", "B0A0000001", "英国"));
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
}
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(cozeRows);
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
assertEquals(1, deduped.size());
// 1000 行唯一:全部保留且顺序稳定
List<SimilarAsinResultRowDto> unique = new ArrayList<>();
@@ -299,7 +296,7 @@ class SimilarAsinTaskServiceRowKeyDedupeTest {
.thenThrow(new IllegalStateException("rustfs down"));
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeCozeRowsIntoChunk",
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
Exception ex = assertThrows(Exception.class, () -> {
@@ -3,7 +3,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
@@ -52,8 +51,8 @@ import static org.mockito.Mockito.when;
/**
* Task 10:为 chunk 结果建立按 row key 的批量索引,消除跨 chunk 线性扫描。
* indexRowsByChunkKey 把每个 chunk 的行索引到 rowKey→chunkKey,coze 行归属从
* O(rows×chunks) 降为 O(1) 查找;assignCozeRowsToChunks 基于索引分配行并保留
* indexRowsByChunkKey 把每个 chunk 的行索引到 rowKey→chunkKey,llm 行归属从
* O(rows×chunks) 降为 O(1) 查找;assignLlmRowsToChunks 基于索引分配行并保留
* 原有命中/fallback/orphan 语义;集成用例验证每个 chunk 只读一次 payload。
*/
@ExtendWith(MockitoExtension.class)
@@ -70,7 +69,6 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@@ -80,7 +78,6 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService cozeCredentialPoolService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@@ -170,7 +167,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Test
void test_task_010_chunk_row_key_normal_default_path() throws Exception {
// 正常输入:2 个 chunk 各含行,coze 回传行按 rowKey 命中各自 chunk;
// 正常输入:2 个 chunk 各含行,llm 回传行按 rowKey 命中各自 chunk;
// 每个 chunk 的 payload 只被读取一次(索引建立),消除跨 chunk 线性扫描。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "hashA", 1, "ptr:chunk-A"),
@@ -191,12 +188,12 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
List<SimilarAsinResultRowDto> cozeRows = List.of(row("r1", "1", "B0A0000001", "英国"), row("r3", "3", "B0A0000003", "美国"));
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"), row("r3", "3", "B0A0000003", "美国"));
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeCozeRowsIntoChunk",
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, null, null, cozeRows, Map.of());
merge.invoke(service, task, null, null, llmRows, Map.of());
verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString());
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
@@ -205,7 +202,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Test
void test_task_010_chunk_row_key_normal_multiple_items() {
// 批量场景:3 个 chunk 各 3 行,9 个 coze 回传行全部命中且顺序稳定,无 orphan
// 批量场景:3 个 chunk 各 3 行,9 个 llm 回传行全部命中且顺序稳定,无 orphan
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
for (int c = 0; c < 3; c++) {
rowsByChunk.putAll(rowsByChunkOf("hash" + c, c + 1,
@@ -213,16 +210,16 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
row("c" + c + "r2", "2", "B0B" + c + "000002", "英国"),
row("c" + c + "r3", "3", "B0B" + c + "000003", "美国"))));
}
List<SimilarAsinResultRowDto> cozeRows = new ArrayList<>();
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int c = 0; c < 3; c++) {
for (int r = 1; r <= 3; r++) {
cozeRows.add(row("c" + c + "r" + r, String.valueOf(r), "B0B" + c + "00000" + r, r == 3 ? "美国" : "英国"));
llmRows.add(row("c" + c + "r" + r, String.valueOf(r), "B0B" + c + "00000" + r, r == 3 ? "美国" : "英国"));
}
}
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignCozeRowsToChunks(
rowsByChunk, cozeRows, index, null, null, orphans);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, orphans);
assertEquals(3, merged.size());
assertEquals(9, assignedRowKeys(merged).size());
assertTrue(orphans.isEmpty(), "全部命中,不应产生 orphan");
@@ -236,12 +233,12 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
// 重复执行同一输入:结果完全一致,不产生重复记录
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1,
List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国")));
List<SimilarAsinResultRowDto> cozeRows = List.of(row("r1", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"));
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> first = service.assignCozeRowsToChunks(
rowsByChunk, cozeRows, index, null, null, new ArrayList<>());
Map<String, Map<String, SimilarAsinResultRowDto>> second = service.assignCozeRowsToChunks(
rowsByChunk, cozeRows, index, null, null, new ArrayList<>());
Map<String, Map<String, SimilarAsinResultRowDto>> first = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
Map<String, Map<String, SimilarAsinResultRowDto>> second = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
assertEquals(assignedRowKeys(first), assignedRowKeys(second));
assertEquals(first.size(), second.size());
for (Map.Entry<String, Map<String, SimilarAsinResultRowDto>> entry : first.entrySet()) {
@@ -251,21 +248,21 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Test
void test_task_010_chunk_row_key_boundary_empty_input() {
// 空输入:null/空 rowsByChunk 与 cozeRows 均安全返回空结果,不创建无效资源
// 空输入:null/空 rowsByChunk 与 llmRows 均安全返回空结果,不创建无效资源
assertNotNull(service.indexRowsByChunkKey(null));
assertTrue(service.indexRowsByChunkKey(null).isEmpty());
assertTrue(service.indexRowsByChunkKey(Map.of()).isEmpty());
Map<String, Map<String, SimilarAsinResultRowDto>> emptyAssign = service.assignCozeRowsToChunks(
Map<String, Map<String, SimilarAsinResultRowDto>> emptyAssign = service.assignLlmRowsToChunks(
Map.of(), List.of(), Map.of(), null, null, new ArrayList<>());
assertTrue(emptyAssign.isEmpty());
assertTrue(service.assignCozeRowsToChunks(
assertTrue(service.assignLlmRowsToChunks(
Map.of(), null, Map.of(), null, null, new ArrayList<>()).isEmpty());
// 无可匹配行(rowKey 不存在于任何 chunk)→ 进 orphan 兜底,不产生 merge
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1, List.of(row("r1", "1", "B0A0000001", "英国")));
List<SimilarAsinResultRowDto> blankRow = List.of(row("", "", "", ""));
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignCozeRowsToChunks(
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, blankRow, index, null, null, orphans);
assertTrue(assignedRowKeys(merged).isEmpty());
assertEquals(1, orphans.size(), "全空行生成 legacy key :::: 不命中任何 chunk,按既有语义进 orphan");
@@ -278,7 +275,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
List.of(row("r1", "1", "B0A0000001", "英国")));
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignCozeRowsToChunks(
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, List.of(row("r1", "1", "B0A0000001", "英国")), index, null, null, orphans);
assertEquals(1, merged.size());
assertEquals(List.of("r1"), assignedRowKeys(merged));
@@ -290,7 +287,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Test
void test_task_010_chunk_row_key_boundary_limit_and_overflow() {
// 大批量:1000 行索引 + 500 个 coze 回传行全部命中,行不丢、无 orphan
// 大批量:1000 行索引 + 500 个 llm 回传行全部命中,行不丢、无 orphan
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
Map<String, SimilarAsinResultRowDto> bigChunk = new LinkedHashMap<>();
for (int i = 1; i <= 1000; i++) {
@@ -299,13 +296,13 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
rowsByChunk.put("hashBig:1", bigChunk);
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
assertEquals(1000, index.size());
List<SimilarAsinResultRowDto> cozeRows = new ArrayList<>();
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 1; i <= 500; i++) {
cozeRows.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
llmRows.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
}
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignCozeRowsToChunks(
rowsByChunk, cozeRows, index, null, null, orphans);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, orphans);
assertEquals(1, merged.size());
assertEquals(500, assignedRowKeys(merged).size());
assertTrue(orphans.isEmpty());
@@ -319,20 +316,20 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
rowsByChunk.putAll(rowsByChunkOf("hashB", 2, List.of(row("dup", "1", "B0A0000001", "英国"))));
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
assertEquals("hashA:1", index.get("dup"), "重复 rowKey 应保留第一个 chunk");
// fallback 缺失:coze 行未命中且无有效 fallback → 进 orphan,不产生 merge
// fallback 缺失:llm 行未命中且无有效 fallback → 进 orphan,不产生 merge
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignCozeRowsToChunks(
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, List.of(row("ghost", "9", "B0A0000009", "英国")), Map.of(), "missingHash", 99, orphans);
assertTrue(assignedRowKeys(merged).isEmpty());
assertEquals(1, orphans.size());
assertEquals("ghost", orphans.get(0).getRowToken());
// cozeRows 含 null 元素:跳过不抛异常,其余行正常分配
// llmRows 含 null 元素:跳过不抛异常,其余行正常分配
List<SimilarAsinResultRowDto> withNull = new ArrayList<>();
withNull.add(null);
withNull.add(row("dup", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> orphans2 = new ArrayList<>();
Map<String, String> index2 = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged2 = service.assignCozeRowsToChunks(
Map<String, Map<String, SimilarAsinResultRowDto>> merged2 = service.assignLlmRowsToChunks(
rowsByChunk, withNull, index2, null, null, orphans2);
assertEquals(1, merged2.size());
assertEquals(List.of("dup"), assignedRowKeys(merged2));
@@ -349,7 +346,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
.thenThrow(new IllegalStateException("rustfs down"));
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeCozeRowsIntoChunk",
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
BusinessException ex = assertThrows(BusinessException.class, () -> {
@@ -5,10 +5,8 @@ import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
@@ -82,7 +80,6 @@ class SimilarAsinTaskServiceSubmitTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@@ -92,7 +89,6 @@ class SimilarAsinTaskServiceSubmitTest {
@Mock private PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private CozeCredentialPoolService cozeCredentialPoolService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@Mock private TransactionStatus transactionStatus;
@@ -360,7 +356,7 @@ class SimilarAsinTaskServiceSubmitTest {
scope.setScopeHash("existing-scope");
scope.setChunkTotal(1);
scope.setCompleted(completed);
scope.setStateJson("{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}");
scope.setStateJson("{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
return scope;
}
}
@@ -3,7 +3,6 @@ package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
@@ -66,7 +65,6 @@ class SimilarAsinTaskServiceWorkbookControlTest {
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;