67f28c5b4f
货源查询与外观专利早已改成直连 LLM,但配置键、指标名、类名与注释里的 "coze" 还留着,
排查线上问题时按平台名 grep 到的是已经不参与调用的历史命名。
对外可见的两处改名(运维需同步):
- 配置键 aiimage.coze-task.{max-concurrent,max-waiting} -> aiimage.task-queue.*
- 指标 aiimage.coze-task.submit.rejected.total / .execution.duration / .queue.wait.duration
-> aiimage.task-queue.* (Grafana 面板与告警表达式要一起改,否则曲线断流)
SimilarAsinTaskServiceCozeBufferScopeTest 随之改名(git 识别为 R)。
OSS 签名域名测试改用 example-cdn.com 占位,不再依赖真实 coze.cn 域名形态。
SimilarAsinTaskServiceTest 补上 result.notice 两条口径的测试:缺必填字段的行显示「未检测」
而不是留白(28551:790/1000 行空白仍报成功),404 行优先显示「ASIN未找到」。
216 lines
11 KiB
Java
216 lines
11 KiB
Java
package com.nanri.aiimage.config;
|
||
|
||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
|
||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
|
||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
|
||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
|
||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
|
||
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
|
||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
|
||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.brand.service.BrandResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.collectdata.service.CollectDataResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.publish.service.PublishResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.task.service.ResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.task.service.ResultFileJobHandlerRegistry;
|
||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
|
||
import com.nanri.aiimage.modules.withdraw.service.WithdrawResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||
import io.micrometer.core.instrument.MeterRegistry;
|
||
import org.springframework.beans.factory.ObjectProvider;
|
||
import org.springframework.beans.factory.annotation.Qualifier;
|
||
import org.springframework.beans.factory.annotation.Value;
|
||
import com.nanri.aiimage.common.module.TaskModuleRegistry;
|
||
import org.springframework.context.annotation.Bean;
|
||
import org.springframework.context.annotation.Configuration;
|
||
import org.springframework.core.task.TaskExecutor;
|
||
import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor;
|
||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||
|
||
import java.util.List;
|
||
import java.util.Set;
|
||
import java.util.concurrent.ArrayBlockingQueue;
|
||
import java.util.concurrent.ExecutorService;
|
||
import java.util.concurrent.ThreadPoolExecutor;
|
||
import java.util.concurrent.TimeUnit;
|
||
import java.util.concurrent.Executors;
|
||
import java.util.concurrent.Semaphore;
|
||
|
||
@Configuration
|
||
public class TaskFileJobConfig {
|
||
|
||
/** 结果文件 Job 支持的全部 moduleType(启动校验枚举源,见 ResultFileJobHandlerRegistry.validateCoverage) */
|
||
/**
|
||
* 结果文件 Job 支持的 moduleType:取自模块注册表(G6),
|
||
* 与 {@code ResultFileJobHandlerRegistry.validateCoverage} 的启动自检配合使用。
|
||
*/
|
||
public static final Set<String> RESULT_FILE_JOB_MODULE_TYPES = TaskModuleRegistry.resultFileJobModuleTypes();
|
||
|
||
@Bean("taskFileJobDispatchExecutor")
|
||
public TaskExecutor taskFileJobDispatchExecutor(
|
||
@Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize,
|
||
@Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) {
|
||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||
int normalizedPoolSize = Math.max(1, poolSize);
|
||
executor.setCorePoolSize(normalizedPoolSize);
|
||
executor.setMaxPoolSize(normalizedPoolSize);
|
||
executor.setQueueCapacity(Math.max(10, queueCapacity));
|
||
executor.setThreadNamePrefix("task-file-job-dispatch-");
|
||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||
executor.setAwaitTerminationSeconds(30);
|
||
executor.initialize();
|
||
return executor;
|
||
}
|
||
|
||
@Bean(destroyMethod = "shutdown")
|
||
public ExecutorService taskQueueVirtualThreadExecutor() {
|
||
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||
.name("task-queue-", 0)
|
||
.factory());
|
||
}
|
||
|
||
/**
|
||
* transient payload 物理删除使用独立有界线程池,不能与业务虚拟线程执行器共用,
|
||
* 避免清理洪峰占满业务任务线程并形成无界在途删除。
|
||
*/
|
||
@Bean(name = "transientPayloadDeleteExecutor", destroyMethod = "shutdown")
|
||
public ExecutorService transientPayloadDeleteExecutor(
|
||
@Value("${aiimage.transient-storage.delete-dispatch-pool-size:2}") int poolSize,
|
||
@Value("${aiimage.transient-storage.delete-dispatch-queue-capacity:100}") int queueCapacity) {
|
||
int workers = Math.max(1, Math.min(poolSize, 16));
|
||
int queue = Math.max(1, Math.min(queueCapacity, 10_000));
|
||
return new ThreadPoolExecutor(
|
||
workers, workers, 0L, TimeUnit.MILLISECONDS,
|
||
new ArrayBlockingQueue<>(queue),
|
||
runnable -> {
|
||
Thread thread = new Thread(runnable, "transient-payload-delete");
|
||
thread.setDaemon(true);
|
||
return thread;
|
||
},
|
||
new ThreadPoolExecutor.AbortPolicy());
|
||
}
|
||
|
||
@Bean("taskQueueExecutor")
|
||
public TaskExecutor taskQueueExecutor(
|
||
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
|
||
@Value("${aiimage.task-queue.max-concurrent:12}") int maxConcurrent,
|
||
@Value("${aiimage.task-queue.max-waiting:1000}") int maxWaiting,
|
||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||
Semaphore semaphore = new Semaphore(Math.max(1, maxConcurrent));
|
||
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
|
||
if (command == null) {
|
||
throw new IllegalArgumentException("task 不能为 null");
|
||
}
|
||
taskQueueVirtualThreadExecutor.execute(() -> {
|
||
boolean acquired = false;
|
||
try {
|
||
semaphore.acquire();
|
||
acquired = true;
|
||
command.run();
|
||
} catch (InterruptedException ex) {
|
||
Thread.currentThread().interrupt();
|
||
} finally {
|
||
if (acquired) {
|
||
semaphore.release();
|
||
}
|
||
}
|
||
});
|
||
});
|
||
return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List<ResultFileJobHandler> handlers) {
|
||
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
|
||
registry.validateCoverage(RESULT_FILE_JOB_MODULE_TYPES);
|
||
return registry;
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler appearancePatentResultFileJobHandler(
|
||
AppearancePatentTaskService appearancePatentTaskService) {
|
||
return new AppearancePatentResultFileJobHandler(appearancePatentTaskService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler brandResultFileJobHandler(
|
||
BrandTaskService brandTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new BrandResultFileJobHandler(brandTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler collectDataResultFileJobHandler(CollectDataService collectDataService) {
|
||
return new CollectDataResultFileJobHandler(collectDataService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler deleteBrandResultFileJobHandler(DeleteBrandRunService deleteBrandRunService) {
|
||
return new DeleteBrandResultFileJobHandler(deleteBrandRunService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler patrolDeleteResultFileJobHandler(
|
||
PatrolDeleteTaskService patrolDeleteTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new PatrolDeleteResultFileJobHandler(patrolDeleteTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler priceTrackResultFileJobHandler(
|
||
PriceTrackTaskService priceTrackTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new PriceTrackResultFileJobHandler(priceTrackTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler productRiskResultFileJobHandler(
|
||
ProductRiskTaskService productRiskTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new ProductRiskResultFileJobHandler(productRiskTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler publishResultFileJobHandler(PublishTaskService publishTaskService) {
|
||
return new PublishResultFileJobHandler(publishTaskService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler queryAsinResultFileJobHandler(
|
||
QueryAsinTaskService queryAsinTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new QueryAsinResultFileJobHandler(queryAsinTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler shopDataCrawlResultFileJobHandler(
|
||
ShopDataCrawlTaskService shopDataCrawlTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new ShopDataCrawlResultFileJobHandler(shopDataCrawlTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler shopMatchResultFileJobHandler(
|
||
ShopMatchTaskService shopMatchTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new ShopMatchResultFileJobHandler(shopMatchTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler similarAsinResultFileJobHandler(SimilarAsinTaskService similarAsinTaskService) {
|
||
return new SimilarAsinResultFileJobHandler(similarAsinTaskService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler withdrawResultFileJobHandler(
|
||
WithdrawTaskService withdrawTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new WithdrawResultFileJobHandler(withdrawTaskService, taskResultPayloadService);
|
||
}
|
||
}
|