215 lines
10 KiB
Java
215 lines
10 KiB
Java
package com.nanri.aiimage.config;
|
||
|
||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
|
||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
|
||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
|
||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
|
||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
|
||
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
|
||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
|
||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.brand.service.BrandResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.collectdata.service.CollectDataResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.publish.service.PublishResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.task.service.ResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.task.service.ResultFileJobHandlerRegistry;
|
||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
|
||
import com.nanri.aiimage.modules.withdraw.service.WithdrawResultFileJobHandler;
|
||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||
import io.micrometer.core.instrument.MeterRegistry;
|
||
import org.springframework.beans.factory.ObjectProvider;
|
||
import org.springframework.beans.factory.annotation.Qualifier;
|
||
import org.springframework.beans.factory.annotation.Value;
|
||
import org.springframework.context.annotation.Bean;
|
||
import org.springframework.context.annotation.Configuration;
|
||
import org.springframework.core.task.TaskExecutor;
|
||
import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor;
|
||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||
|
||
import java.util.List;
|
||
import java.util.Set;
|
||
import java.util.concurrent.ArrayBlockingQueue;
|
||
import java.util.concurrent.ExecutorService;
|
||
import java.util.concurrent.ThreadPoolExecutor;
|
||
import java.util.concurrent.TimeUnit;
|
||
import java.util.concurrent.Executors;
|
||
import java.util.concurrent.Semaphore;
|
||
|
||
@Configuration
|
||
public class TaskFileJobConfig {
|
||
|
||
/** 结果文件 Job 支持的全部 moduleType(启动校验枚举源,见 ResultFileJobHandlerRegistry.validateCoverage) */
|
||
public static final Set<String> RESULT_FILE_JOB_MODULE_TYPES = Set.of(
|
||
"SHOP_MATCH", "PRICE_TRACK", "PRODUCT_RISK_RESOLVE",
|
||
"PUBLISH", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW",
|
||
"PATROL_DELETE", "APPEARANCE_PATENT", "SIMILAR_ASIN",
|
||
"DELETE_BRAND", "BRAND", "COLLECT_DATA");
|
||
|
||
@Bean("taskFileJobDispatchExecutor")
|
||
public TaskExecutor taskFileJobDispatchExecutor(
|
||
@Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize,
|
||
@Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) {
|
||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||
int normalizedPoolSize = Math.max(1, poolSize);
|
||
executor.setCorePoolSize(normalizedPoolSize);
|
||
executor.setMaxPoolSize(normalizedPoolSize);
|
||
executor.setQueueCapacity(Math.max(10, queueCapacity));
|
||
executor.setThreadNamePrefix("task-file-job-dispatch-");
|
||
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||
executor.setAwaitTerminationSeconds(30);
|
||
executor.initialize();
|
||
return executor;
|
||
}
|
||
|
||
@Bean(destroyMethod = "shutdown")
|
||
public ExecutorService taskQueueVirtualThreadExecutor() {
|
||
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||
.name("task-queue-", 0)
|
||
.factory());
|
||
}
|
||
|
||
/**
|
||
* transient payload 物理删除使用独立有界线程池,不能与业务虚拟线程执行器共用,
|
||
* 避免清理洪峰占满业务任务线程并形成无界在途删除。
|
||
*/
|
||
@Bean(name = "transientPayloadDeleteExecutor", destroyMethod = "shutdown")
|
||
public ExecutorService transientPayloadDeleteExecutor(
|
||
@Value("${aiimage.transient-storage.delete-dispatch-pool-size:2}") int poolSize,
|
||
@Value("${aiimage.transient-storage.delete-dispatch-queue-capacity:100}") int queueCapacity) {
|
||
int workers = Math.max(1, Math.min(poolSize, 16));
|
||
int queue = Math.max(1, Math.min(queueCapacity, 10_000));
|
||
return new ThreadPoolExecutor(
|
||
workers, workers, 0L, TimeUnit.MILLISECONDS,
|
||
new ArrayBlockingQueue<>(queue),
|
||
runnable -> {
|
||
Thread thread = new Thread(runnable, "transient-payload-delete");
|
||
thread.setDaemon(true);
|
||
return thread;
|
||
},
|
||
new ThreadPoolExecutor.AbortPolicy());
|
||
}
|
||
|
||
@Bean("taskQueueExecutor")
|
||
public TaskExecutor taskQueueExecutor(
|
||
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
|
||
@Value("${aiimage.coze-task.max-concurrent:12}") int maxConcurrent,
|
||
@Value("${aiimage.coze-task.max-waiting:1000}") int maxWaiting,
|
||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||
Semaphore semaphore = new Semaphore(Math.max(1, maxConcurrent));
|
||
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
|
||
if (command == null) {
|
||
throw new IllegalArgumentException("task 不能为 null");
|
||
}
|
||
taskQueueVirtualThreadExecutor.execute(() -> {
|
||
boolean acquired = false;
|
||
try {
|
||
semaphore.acquire();
|
||
acquired = true;
|
||
command.run();
|
||
} catch (InterruptedException ex) {
|
||
Thread.currentThread().interrupt();
|
||
} finally {
|
||
if (acquired) {
|
||
semaphore.release();
|
||
}
|
||
}
|
||
});
|
||
});
|
||
return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List<ResultFileJobHandler> handlers) {
|
||
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
|
||
registry.validateCoverage(RESULT_FILE_JOB_MODULE_TYPES);
|
||
return registry;
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler appearancePatentResultFileJobHandler(
|
||
AppearancePatentTaskService appearancePatentTaskService) {
|
||
return new AppearancePatentResultFileJobHandler(appearancePatentTaskService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler brandResultFileJobHandler(
|
||
BrandTaskService brandTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new BrandResultFileJobHandler(brandTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler collectDataResultFileJobHandler(CollectDataService collectDataService) {
|
||
return new CollectDataResultFileJobHandler(collectDataService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler deleteBrandResultFileJobHandler(DeleteBrandRunService deleteBrandRunService) {
|
||
return new DeleteBrandResultFileJobHandler(deleteBrandRunService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler patrolDeleteResultFileJobHandler(
|
||
PatrolDeleteTaskService patrolDeleteTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new PatrolDeleteResultFileJobHandler(patrolDeleteTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler priceTrackResultFileJobHandler(
|
||
PriceTrackTaskService priceTrackTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new PriceTrackResultFileJobHandler(priceTrackTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler productRiskResultFileJobHandler(
|
||
ProductRiskTaskService productRiskTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new ProductRiskResultFileJobHandler(productRiskTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler publishResultFileJobHandler(PublishTaskService publishTaskService) {
|
||
return new PublishResultFileJobHandler(publishTaskService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler queryAsinResultFileJobHandler(
|
||
QueryAsinTaskService queryAsinTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new QueryAsinResultFileJobHandler(queryAsinTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler shopDataCrawlResultFileJobHandler(
|
||
ShopDataCrawlTaskService shopDataCrawlTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new ShopDataCrawlResultFileJobHandler(shopDataCrawlTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler shopMatchResultFileJobHandler(
|
||
ShopMatchTaskService shopMatchTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new ShopMatchResultFileJobHandler(shopMatchTaskService, taskResultPayloadService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler similarAsinResultFileJobHandler(SimilarAsinTaskService similarAsinTaskService) {
|
||
return new SimilarAsinResultFileJobHandler(similarAsinTaskService);
|
||
}
|
||
|
||
@Bean
|
||
public ResultFileJobHandler withdrawResultFileJobHandler(
|
||
WithdrawTaskService withdrawTaskService, TaskResultPayloadService taskResultPayloadService) {
|
||
return new WithdrawResultFileJobHandler(withdrawTaskService, taskResultPayloadService);
|
||
}
|
||
}
|