refactor(队列): coze 残留改名到 task-queue / LLM,直连改造后不再出现第三方平台名
货源查询与外观专利早已改成直连 LLM,但配置键、指标名、类名与注释里的 "coze" 还留着,
排查线上问题时按平台名 grep 到的是已经不参与调用的历史命名。
对外可见的两处改名(运维需同步):
- 配置键 aiimage.coze-task.{max-concurrent,max-waiting} -> aiimage.task-queue.*
- 指标 aiimage.coze-task.submit.rejected.total / .execution.duration / .queue.wait.duration
-> aiimage.task-queue.* (Grafana 面板与告警表达式要一起改,否则曲线断流)
SimilarAsinTaskServiceCozeBufferScopeTest 随之改名(git 识别为 R)。
OSS 签名域名测试改用 example-cdn.com 占位,不再依赖真实 coze.cn 域名形态。
SimilarAsinTaskServiceTest 补上 result.notice 两条口径的测试:缺必填字段的行显示「未检测」
而不是留白(28551:790/1000 行空白仍报成功),404 行优先显示「ASIN未找到」。
This commit is contained in:
@@ -4,7 +4,7 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* LLM 客户端统一配置解析(task-168,Coze 已改直连 LLM)。
|
* LLM 客户端统一配置解析(task-168)。
|
||||||
*
|
*
|
||||||
* 从 aiimage.http-client.* 命名空间读取 LLM 客户端超时/重试,默认值与现状一致
|
* 从 aiimage.http-client.* 命名空间读取 LLM 客户端超时/重试,默认值与现状一致
|
||||||
* (connect 10s / read 60s / call 90s / retry 3);非法值钳制。模块级显式配置
|
* (connect 10s / read 60s / call 90s / retry 3);非法值钳制。模块级显式配置
|
||||||
|
|||||||
@@ -106,8 +106,8 @@ public class TaskFileJobConfig {
|
|||||||
@Bean("taskQueueExecutor")
|
@Bean("taskQueueExecutor")
|
||||||
public TaskExecutor taskQueueExecutor(
|
public TaskExecutor taskQueueExecutor(
|
||||||
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
|
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
|
||||||
@Value("${aiimage.coze-task.max-concurrent:12}") int maxConcurrent,
|
@Value("${aiimage.task-queue.max-concurrent:12}") int maxConcurrent,
|
||||||
@Value("${aiimage.coze-task.max-waiting:1000}") int maxWaiting,
|
@Value("${aiimage.task-queue.max-waiting:1000}") int maxWaiting,
|
||||||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||||||
Semaphore semaphore = new Semaphore(Math.max(1, maxConcurrent));
|
Semaphore semaphore = new Semaphore(Math.max(1, maxConcurrent));
|
||||||
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
|
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
|
||||||
|
|||||||
@@ -101,14 +101,14 @@ public class TaskQueueGate implements TaskExecutor {
|
|||||||
private void recordRejected(String reason) {
|
private void recordRejected(String reason) {
|
||||||
MeterRegistry registry = meterRegistry();
|
MeterRegistry registry = meterRegistry();
|
||||||
if (registry != null) {
|
if (registry != null) {
|
||||||
registry.counter("aiimage.coze-task.submit.rejected.total", "reason", reason).increment();
|
registry.counter("aiimage.task-queue.submit.rejected.total", "reason", reason).increment();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void recordQueueWait(long waitNanos) {
|
private void recordQueueWait(long waitNanos) {
|
||||||
MeterRegistry registry = meterRegistry();
|
MeterRegistry registry = meterRegistry();
|
||||||
if (registry != null && waitNanos >= 0L) {
|
if (registry != null && waitNanos >= 0L) {
|
||||||
Timer.builder("aiimage.coze-task.queue.wait.duration")
|
Timer.builder("aiimage.task-queue.queue.wait.duration")
|
||||||
.register(registry)
|
.register(registry)
|
||||||
.record(waitNanos, TimeUnit.NANOSECONDS);
|
.record(waitNanos, TimeUnit.NANOSECONDS);
|
||||||
}
|
}
|
||||||
@@ -117,7 +117,7 @@ public class TaskQueueGate implements TaskExecutor {
|
|||||||
private void recordExecution(long durationNanos) {
|
private void recordExecution(long durationNanos) {
|
||||||
MeterRegistry registry = meterRegistry();
|
MeterRegistry registry = meterRegistry();
|
||||||
if (registry != null && durationNanos >= 0L) {
|
if (registry != null && durationNanos >= 0L) {
|
||||||
Timer.builder("aiimage.coze-task.execution.duration")
|
Timer.builder("aiimage.task-queue.execution.duration")
|
||||||
.register(registry)
|
.register(registry)
|
||||||
.record(durationNanos, TimeUnit.NANOSECONDS);
|
.record(durationNanos, TimeUnit.NANOSECONDS);
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -107,7 +107,7 @@ class HttpClientConnectionReuseTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void test_task_077_brand_normal_multiple_items() throws Exception {
|
void test_task_077_brand_normal_multiple_items() throws Exception {
|
||||||
// 批量场景:Coze/品牌/紫鸟三个客户端各自持有独立 RestClient,
|
// 批量场景:LLM/品牌/紫鸟三个客户端各自持有独立 RestClient,
|
||||||
// 但底层连接池共用同一 HttpClient 实例,不重复创建。
|
// 但底层连接池共用同一 HttpClient 实例,不重复创建。
|
||||||
SimilarAsinLlmClient llm = new SimilarAsinLlmClient(new SimilarAsinProperties(), new ObjectMapper(), null, null);
|
SimilarAsinLlmClient llm = new SimilarAsinLlmClient(new SimilarAsinProperties(), new ObjectMapper(), null, null);
|
||||||
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null);
|
BrandCheckClient brand = new BrandCheckClient(new BrandCheckProperties(), null);
|
||||||
|
|||||||
@@ -49,18 +49,18 @@ class TaskQueueGateTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private long rejectedCount(String reason) {
|
private long rejectedCount(String reason) {
|
||||||
Counter counter = registry.find("aiimage.coze-task.submit.rejected.total")
|
Counter counter = registry.find("aiimage.task-queue.submit.rejected.total")
|
||||||
.tag("reason", reason).counter();
|
.tag("reason", reason).counter();
|
||||||
return counter == null ? 0 : (long) counter.count();
|
return counter == null ? 0 : (long) counter.count();
|
||||||
}
|
}
|
||||||
|
|
||||||
private long executionCount() {
|
private long executionCount() {
|
||||||
Timer timer = registry.find("aiimage.coze-task.execution.duration").timer();
|
Timer timer = registry.find("aiimage.task-queue.execution.duration").timer();
|
||||||
return timer == null ? 0 : timer.count();
|
return timer == null ? 0 : timer.count();
|
||||||
}
|
}
|
||||||
|
|
||||||
private long waitCount() {
|
private long waitCount() {
|
||||||
Timer timer = registry.find("aiimage.coze-task.queue.wait.duration").timer();
|
Timer timer = registry.find("aiimage.task-queue.queue.wait.duration").timer();
|
||||||
return timer == null ? 0 : timer.count();
|
return timer == null ? 0 : timer.count();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -38,7 +38,7 @@ import static org.mockito.Mockito.when;
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Task 79:为对象存储(RustFS/MinIO)、数据库(任务文件作业表)和队列
|
* Task 79:为对象存储(RustFS/MinIO)、数据库(任务文件作业表)和队列
|
||||||
* (Coze 执行队列)增加故障注入测试。全部通过 mock 依赖注入故障,验证
|
* (任务执行队列)增加故障注入测试。全部通过 mock 依赖注入故障,验证
|
||||||
* 错误可恢复、重试有界、信号量/等待槽释放、非法输入不创建无效资源。
|
* 错误可恢复、重试有界、信号量/等待槽释放、非法输入不创建无效资源。
|
||||||
*/
|
*/
|
||||||
class FaultInjectionTest {
|
class FaultInjectionTest {
|
||||||
@@ -236,7 +236,7 @@ class FaultInjectionTest {
|
|||||||
// 第一次提交被执行器拒绝:等待槽释放、拒绝指标记录
|
// 第一次提交被执行器拒绝:等待槽释放、拒绝指标记录
|
||||||
assertThrows(TaskRejectedException.class, () -> gate.execute(task));
|
assertThrows(TaskRejectedException.class, () -> gate.execute(task));
|
||||||
assertEquals(0, gate.waiting(), "执行器拒绝后等待槽必须释放");
|
assertEquals(0, gate.waiting(), "执行器拒绝后等待槽必须释放");
|
||||||
assertTrue(registry.counter("aiimage.coze-task.submit.rejected.total", "reason", "delegate-rejected").count() > 0);
|
assertTrue(registry.counter("aiimage.task-queue.submit.rejected.total", "reason", "delegate-rejected").count() > 0);
|
||||||
|
|
||||||
// 第二次提交恢复成功:等待槽正常占用并释放
|
// 第二次提交恢复成功:等待槽正常占用并释放
|
||||||
gate.execute(task);
|
gate.execute(task);
|
||||||
|
|||||||
+1
-1
@@ -78,7 +78,7 @@ class OssStorageServiceTest {
|
|||||||
List<String> externalUrls = List.of(
|
List<String> externalUrls = List.of(
|
||||||
"https://m.media-amazon.com/nanri-ai-images/main.jpg?size=large",
|
"https://m.media-amazon.com/nanri-ai-images/main.jpg?size=large",
|
||||||
"https://cbu01.alicdn.com/img/ibank/main.jpg",
|
"https://cbu01.alicdn.com/img/ibank/main.jpg",
|
||||||
"https://lf9-bot-platform-tos-sign.coze.cn/file.jpg?x-expires=1&x-signature=abc",
|
"https://lf9-tos-sign.example-cdn.com/file.jpg?x-expires=1&x-signature=abc",
|
||||||
"https://nanri-ai-images.evil.example/supply_images/main.jpg");
|
"https://nanri-ai-images.evil.example/supply_images/main.jpg");
|
||||||
|
|
||||||
for (String url : externalUrls) {
|
for (String url : externalUrls) {
|
||||||
|
|||||||
+3
-3
@@ -55,7 +55,7 @@ import static org.mockito.Mockito.verify;
|
|||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Task 12:扩展 Coze 结果缓冲覆盖范围,减少频繁读写完整 chunk payload。
|
* Task 12:扩展 LLM 结果缓冲覆盖范围,减少频繁读写完整 chunk payload。
|
||||||
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1";Task 12 扩展为:
|
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1";Task 12 扩展为:
|
||||||
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
|
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
|
||||||
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
|
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
|
||||||
@@ -63,7 +63,7 @@ import static org.mockito.Mockito.when;
|
|||||||
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
|
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
|
||||||
*/
|
*/
|
||||||
@ExtendWith(MockitoExtension.class)
|
@ExtendWith(MockitoExtension.class)
|
||||||
class SimilarAsinTaskServiceCozeBufferScopeTest {
|
class SimilarAsinTaskServiceLlmBufferScopeTest {
|
||||||
|
|
||||||
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
|
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
|
||||||
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
|
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
|
||||||
@@ -133,7 +133,7 @@ class SimilarAsinTaskServiceCozeBufferScopeTest {
|
|||||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||||
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
|
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
|
||||||
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
|
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
|
||||||
.thenAnswer(invocation -> "rustfs:coze-result/" + NEXT_ID.incrementAndGet());
|
.thenAnswer(invocation -> "rustfs:llm-result/" + NEXT_ID.incrementAndGet());
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterEach
|
@AfterEach
|
||||||
+2
-2
@@ -48,9 +48,9 @@ import static org.mockito.Mockito.verify;
|
|||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Task 11:Coze 结果合并的重复检测从 O(n²) 改为 HashSet/稳定 row key。
|
* Task 11:LLM 结果合并的重复检测从 O(n²) 改为 HashSet/稳定 row key。
|
||||||
* dedupeRowsByRowKey 用 HashSet 按稳定 rowKey 一次性去重(保留顺序),
|
* dedupeRowsByRowKey 用 HashSet 按稳定 rowKey 一次性去重(保留顺序),
|
||||||
* mergeCozeRowsIntoChunk 合并前先去重,消除重复行逐行重复处理。
|
* mergeLlmRowsIntoChunk 合并前先去重,消除重复行逐行重复处理。
|
||||||
*/
|
*/
|
||||||
@ExtendWith(MockitoExtension.class)
|
@ExtendWith(MockitoExtension.class)
|
||||||
class SimilarAsinTaskServiceRowKeyDedupeTest {
|
class SimilarAsinTaskServiceRowKeyDedupeTest {
|
||||||
|
|||||||
+47
-4
@@ -13,6 +13,7 @@ import java.util.List;
|
|||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
class SimilarAsinTaskServiceTest {
|
class SimilarAsinTaskServiceTest {
|
||||||
@@ -34,7 +35,7 @@ class SimilarAsinTaskServiceTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void resultStatusUsesReturnedCozeDataAndImages() {
|
void resultStatusUsesReturnedLlmDataAndImages() {
|
||||||
assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(null));
|
assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(null));
|
||||||
|
|
||||||
SimilarAsinResultRowDto empty = new SimilarAsinResultRowDto();
|
SimilarAsinResultRowDto empty = new SimilarAsinResultRowDto();
|
||||||
@@ -45,9 +46,9 @@ class SimilarAsinTaskServiceTest {
|
|||||||
withImage.setMainUrl("https://example.com/main.jpg");
|
withImage.setMainUrl("https://example.com/main.jpg");
|
||||||
assertEquals("\u6210\u529f", SimilarAsinResultTextSupport.resolveResultStatus(withImage));
|
assertEquals("\u6210\u529f", SimilarAsinResultTextSupport.resolveResultStatus(withImage));
|
||||||
|
|
||||||
SimilarAsinResultRowDto withCozeStatus = new SimilarAsinResultRowDto();
|
SimilarAsinResultRowDto withLlmStatus = new SimilarAsinResultRowDto();
|
||||||
withCozeStatus.setStatus("success");
|
withLlmStatus.setStatus("success");
|
||||||
assertEquals("\u6210\u529f", SimilarAsinResultTextSupport.resolveResultStatus(withCozeStatus));
|
assertEquals("\u6210\u529f", SimilarAsinResultTextSupport.resolveResultStatus(withLlmStatus));
|
||||||
|
|
||||||
SimilarAsinResultRowDto failedStatus = new SimilarAsinResultRowDto();
|
SimilarAsinResultRowDto failedStatus = new SimilarAsinResultRowDto();
|
||||||
failedStatus.setStatus("FAILED");
|
failedStatus.setStatus("FAILED");
|
||||||
@@ -79,6 +80,48 @@ class SimilarAsinTaskServiceTest {
|
|||||||
assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(llmFailure));
|
assertEquals("\u5931\u8d25", SimilarAsinResultTextSupport.resolveResultStatus(llmFailure));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void resultStatusMarksMissingRequiredFieldRowsAsNotDetected() {
|
||||||
|
// 缺必填字段(无图/无 ASIN/无标题)的行:Python 端根本没采到货源数据,
|
||||||
|
// 状态列必须显示"未检测",不能整格留白—— 2026-09-20 任务 28551 的
|
||||||
|
// 790/1000 行空白就是这样来的,任务还报成功。
|
||||||
|
SimilarAsinResultRowDto missing = new SimilarAsinResultRowDto();
|
||||||
|
missing.setError("required field missing (asin/title/url), skip LLM");
|
||||||
|
missing.setReason("required field missing (asin/title/url), skip LLM");
|
||||||
|
missing.setConclusion("required field missing (asin/title/url), skip LLM");
|
||||||
|
missing.setStatus("FAILED");
|
||||||
|
assertEquals("未检测",
|
||||||
|
SimilarAsinResultTextSupport.resolveResultStatus(missing));
|
||||||
|
|
||||||
|
// 404 行优先报"ASIN未找到":它的 reason 里带的也是同一句缺字段文案,
|
||||||
|
// 不能被"未检测"抢走(判定顺序敏感)
|
||||||
|
SimilarAsinResultRowDto notFound = new SimilarAsinResultRowDto();
|
||||||
|
notFound.setError("404 Not Found");
|
||||||
|
notFound.setReason("required field missing (asin/title/url), skip LLM");
|
||||||
|
notFound.setStatus("FAILED");
|
||||||
|
assertEquals("ASIN未找到",
|
||||||
|
SimilarAsinResultTextSupport.resolveResultStatus(notFound));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void incompleteRowsMessageOnlyWhenRowsAreMissing() {
|
||||||
|
// 行数齐了必须返回 null:正常任务不能被挂上"仅完成 N/M 行"
|
||||||
|
assertNull(SimilarAsinResultTextSupport.incompleteRowsMessage(1000, 1000));
|
||||||
|
assertNull(SimilarAsinResultTextSupport.incompleteRowsMessage(1001, 1000));
|
||||||
|
|
||||||
|
// 少一行也要提示(宁可多提示,不能让用户以为跑全了)
|
||||||
|
assertEquals("仅完成 999/1000 行,其余 1 行未采到数据(未检测)",
|
||||||
|
SimilarAsinResultTextSupport.incompleteRowsMessage(999, 1000));
|
||||||
|
|
||||||
|
// 28551 的实际比例:1000 行只跑出 210 行
|
||||||
|
assertEquals("仅完成 210/1000 行,其余 790 行未采到数据(未检测)",
|
||||||
|
SimilarAsinResultTextSupport.incompleteRowsMessage(210, 1000));
|
||||||
|
|
||||||
|
// 一行没跑成同样要提示,不能静默
|
||||||
|
assertEquals("仅完成 0/1000 行,其余 1000 行未采到数据(未检测)",
|
||||||
|
SimilarAsinResultTextSupport.incompleteRowsMessage(0, 1000));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void asinNotFoundDetectionMatchesPythonErrors() {
|
void asinNotFoundDetectionMatchesPythonErrors() {
|
||||||
assertTrue(SimilarAsinResultTextSupport.isAsinNotFound("404 Not Found"));
|
assertTrue(SimilarAsinResultTextSupport.isAsinNotFound("404 Not Found"));
|
||||||
|
|||||||
+1
-1
@@ -53,7 +53,7 @@ import static org.mockito.Mockito.verify;
|
|||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* TaskResultFileJobWorker.process coze offload 判定注册表改造测试(任务 76)。
|
* TaskResultFileJobWorker.process LLM offload 判定注册表改造测试(任务 76)。
|
||||||
* 行为等价基线:改造前对现有实现断言 GREEN;改造后(handler.supportsAsyncOffload 委托)仍 GREEN;
|
* 行为等价基线:改造前对现有实现断言 GREEN;改造后(handler.supportsAsyncOffload 委托)仍 GREEN;
|
||||||
* 其中"无 offload 模块硬编码"为静态检查,改造前 RED、改造后 GREEN。
|
* 其中"无 offload 模块硬编码"为静态检查,改造前 RED、改造后 GREEN。
|
||||||
*/
|
*/
|
||||||
|
|||||||
Reference in New Issue
Block a user