Files
crawler-plugin/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java
T
huangzd1997 67f28c5b4f refactor(队列): coze 残留改名到 task-queue / LLM,直连改造后不再出现第三方平台名
货源查询与外观专利早已改成直连 LLM,但配置键、指标名、类名与注释里的 "coze" 还留着,
排查线上问题时按平台名 grep 到的是已经不参与调用的历史命名。

对外可见的两处改名(运维需同步):
- 配置键 aiimage.coze-task.{max-concurrent,max-waiting} -> aiimage.task-queue.*
- 指标 aiimage.coze-task.submit.rejected.total / .execution.duration / .queue.wait.duration
  -> aiimage.task-queue.*   (Grafana 面板与告警表达式要一起改,否则曲线断流)
SimilarAsinTaskServiceCozeBufferScopeTest 随之改名(git 识别为 R)。
OSS 签名域名测试改用 example-cdn.com 占位,不再依赖真实 coze.cn 域名形态。
SimilarAsinTaskServiceTest 补上 result.notice 两条口径的测试:缺必填字段的行显示「未检测」
而不是留白(28551:790/1000 行空白仍报成功),404 行优先显示「ASIN未找到」。
2026-09-27 11:27:27 +08:00

216 lines
11 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package com.nanri.aiimage.config;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentResultFileJobHandler;
import com.nanri.aiimage.modules.brand.service.BrandResultFileJobHandler;
import com.nanri.aiimage.modules.collectdata.service.CollectDataResultFileJobHandler;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandResultFileJobHandler;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteResultFileJobHandler;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackResultFileJobHandler;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskResultFileJobHandler;
import com.nanri.aiimage.modules.publish.service.PublishResultFileJobHandler;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinResultFileJobHandler;
import com.nanri.aiimage.modules.task.service.ResultFileJobHandler;
import com.nanri.aiimage.modules.task.service.ResultFileJobHandlerRegistry;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResultFileJobHandler;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchResultFileJobHandler;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinResultFileJobHandler;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.withdraw.service.WithdrawResultFileJobHandler;
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
import io.micrometer.core.instrument.MeterRegistry;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import com.nanri.aiimage.common.module.TaskModuleRegistry;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
@Configuration
public class TaskFileJobConfig {
/** 结果文件 Job 支持的全部 moduleType(启动校验枚举源,见 ResultFileJobHandlerRegistry.validateCoverage) */
/**
* 结果文件 Job 支持的 moduleType:取自模块注册表(G6),
* 与 {@code ResultFileJobHandlerRegistry.validateCoverage} 的启动自检配合使用。
*/
public static final Set<String> RESULT_FILE_JOB_MODULE_TYPES = TaskModuleRegistry.resultFileJobModuleTypes();
@Bean("taskFileJobDispatchExecutor")
public TaskExecutor taskFileJobDispatchExecutor(
@Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize,
@Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
int normalizedPoolSize = Math.max(1, poolSize);
executor.setCorePoolSize(normalizedPoolSize);
executor.setMaxPoolSize(normalizedPoolSize);
executor.setQueueCapacity(Math.max(10, queueCapacity));
executor.setThreadNamePrefix("task-file-job-dispatch-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(30);
executor.initialize();
return executor;
}
@Bean(destroyMethod = "shutdown")
public ExecutorService taskQueueVirtualThreadExecutor() {
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("task-queue-", 0)
.factory());
}
/**
* transient payload 物理删除使用独立有界线程池,不能与业务虚拟线程执行器共用,
* 避免清理洪峰占满业务任务线程并形成无界在途删除。
*/
@Bean(name = "transientPayloadDeleteExecutor", destroyMethod = "shutdown")
public ExecutorService transientPayloadDeleteExecutor(
@Value("${aiimage.transient-storage.delete-dispatch-pool-size:2}") int poolSize,
@Value("${aiimage.transient-storage.delete-dispatch-queue-capacity:100}") int queueCapacity) {
int workers = Math.max(1, Math.min(poolSize, 16));
int queue = Math.max(1, Math.min(queueCapacity, 10_000));
return new ThreadPoolExecutor(
workers, workers, 0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(queue),
runnable -> {
Thread thread = new Thread(runnable, "transient-payload-delete");
thread.setDaemon(true);
return thread;
},
new ThreadPoolExecutor.AbortPolicy());
}
@Bean("taskQueueExecutor")
public TaskExecutor taskQueueExecutor(
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
@Value("${aiimage.task-queue.max-concurrent:12}") int maxConcurrent,
@Value("${aiimage.task-queue.max-waiting:1000}") int maxWaiting,
ObjectProvider<MeterRegistry> meterRegistryProvider) {
Semaphore semaphore = new Semaphore(Math.max(1, maxConcurrent));
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
if (command == null) {
throw new IllegalArgumentException("task 不能为 null");
}
taskQueueVirtualThreadExecutor.execute(() -> {
boolean acquired = false;
try {
semaphore.acquire();
acquired = true;
command.run();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
} finally {
if (acquired) {
semaphore.release();
}
}
});
});
return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider);
}
@Bean
public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List<ResultFileJobHandler> handlers) {
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
registry.validateCoverage(RESULT_FILE_JOB_MODULE_TYPES);
return registry;
}
@Bean
public ResultFileJobHandler appearancePatentResultFileJobHandler(
AppearancePatentTaskService appearancePatentTaskService) {
return new AppearancePatentResultFileJobHandler(appearancePatentTaskService);
}
@Bean
public ResultFileJobHandler brandResultFileJobHandler(
BrandTaskService brandTaskService, TaskResultPayloadService taskResultPayloadService) {
return new BrandResultFileJobHandler(brandTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler collectDataResultFileJobHandler(CollectDataService collectDataService) {
return new CollectDataResultFileJobHandler(collectDataService);
}
@Bean
public ResultFileJobHandler deleteBrandResultFileJobHandler(DeleteBrandRunService deleteBrandRunService) {
return new DeleteBrandResultFileJobHandler(deleteBrandRunService);
}
@Bean
public ResultFileJobHandler patrolDeleteResultFileJobHandler(
PatrolDeleteTaskService patrolDeleteTaskService, TaskResultPayloadService taskResultPayloadService) {
return new PatrolDeleteResultFileJobHandler(patrolDeleteTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler priceTrackResultFileJobHandler(
PriceTrackTaskService priceTrackTaskService, TaskResultPayloadService taskResultPayloadService) {
return new PriceTrackResultFileJobHandler(priceTrackTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler productRiskResultFileJobHandler(
ProductRiskTaskService productRiskTaskService, TaskResultPayloadService taskResultPayloadService) {
return new ProductRiskResultFileJobHandler(productRiskTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler publishResultFileJobHandler(PublishTaskService publishTaskService) {
return new PublishResultFileJobHandler(publishTaskService);
}
@Bean
public ResultFileJobHandler queryAsinResultFileJobHandler(
QueryAsinTaskService queryAsinTaskService, TaskResultPayloadService taskResultPayloadService) {
return new QueryAsinResultFileJobHandler(queryAsinTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler shopDataCrawlResultFileJobHandler(
ShopDataCrawlTaskService shopDataCrawlTaskService, TaskResultPayloadService taskResultPayloadService) {
return new ShopDataCrawlResultFileJobHandler(shopDataCrawlTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler shopMatchResultFileJobHandler(
ShopMatchTaskService shopMatchTaskService, TaskResultPayloadService taskResultPayloadService) {
return new ShopMatchResultFileJobHandler(shopMatchTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler similarAsinResultFileJobHandler(SimilarAsinTaskService similarAsinTaskService) {
return new SimilarAsinResultFileJobHandler(similarAsinTaskService);
}
@Bean
public ResultFileJobHandler withdrawResultFileJobHandler(
WithdrawTaskService withdrawTaskService, TaskResultPayloadService taskResultPayloadService) {
return new WithdrawResultFileJobHandler(withdrawTaskService, taskResultPayloadService);
}
}