Files
crawler-plugin/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java
T

215 lines
10 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package com.nanri.aiimage.config;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentResultFileJobHandler;
import com.nanri.aiimage.modules.brand.service.BrandResultFileJobHandler;
import com.nanri.aiimage.modules.collectdata.service.CollectDataResultFileJobHandler;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandResultFileJobHandler;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteResultFileJobHandler;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackResultFileJobHandler;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskResultFileJobHandler;
import com.nanri.aiimage.modules.publish.service.PublishResultFileJobHandler;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinResultFileJobHandler;
import com.nanri.aiimage.modules.task.service.ResultFileJobHandler;
import com.nanri.aiimage.modules.task.service.ResultFileJobHandlerRegistry;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResultFileJobHandler;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchResultFileJobHandler;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinResultFileJobHandler;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.withdraw.service.WithdrawResultFileJobHandler;
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
import io.micrometer.core.instrument.MeterRegistry;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
@Configuration
public class TaskFileJobConfig {
/** 结果文件 Job 支持的全部 moduleType(启动校验枚举源,见 ResultFileJobHandlerRegistry.validateCoverage) */
public static final Set<String> RESULT_FILE_JOB_MODULE_TYPES = Set.of(
"SHOP_MATCH", "PRICE_TRACK", "PRODUCT_RISK_RESOLVE",
"PUBLISH", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW",
"PATROL_DELETE", "APPEARANCE_PATENT", "SIMILAR_ASIN",
"DELETE_BRAND", "BRAND", "COLLECT_DATA");
@Bean("taskFileJobDispatchExecutor")
public TaskExecutor taskFileJobDispatchExecutor(
@Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize,
@Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
int normalizedPoolSize = Math.max(1, poolSize);
executor.setCorePoolSize(normalizedPoolSize);
executor.setMaxPoolSize(normalizedPoolSize);
executor.setQueueCapacity(Math.max(10, queueCapacity));
executor.setThreadNamePrefix("task-file-job-dispatch-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(30);
executor.initialize();
return executor;
}
@Bean(destroyMethod = "shutdown")
public ExecutorService taskQueueVirtualThreadExecutor() {
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("task-queue-", 0)
.factory());
}
/**
* transient payload 物理删除使用独立有界线程池,不能与业务虚拟线程执行器共用,
* 避免清理洪峰占满业务任务线程并形成无界在途删除。
*/
@Bean(name = "transientPayloadDeleteExecutor", destroyMethod = "shutdown")
public ExecutorService transientPayloadDeleteExecutor(
@Value("${aiimage.transient-storage.delete-dispatch-pool-size:2}") int poolSize,
@Value("${aiimage.transient-storage.delete-dispatch-queue-capacity:100}") int queueCapacity) {
int workers = Math.max(1, Math.min(poolSize, 16));
int queue = Math.max(1, Math.min(queueCapacity, 10_000));
return new ThreadPoolExecutor(
workers, workers, 0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(queue),
runnable -> {
Thread thread = new Thread(runnable, "transient-payload-delete");
thread.setDaemon(true);
return thread;
},
new ThreadPoolExecutor.AbortPolicy());
}
@Bean("taskQueueExecutor")
public TaskExecutor taskQueueExecutor(
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
@Value("${aiimage.coze-task.max-concurrent:12}") int maxConcurrent,
@Value("${aiimage.coze-task.max-waiting:1000}") int maxWaiting,
ObjectProvider<MeterRegistry> meterRegistryProvider) {
Semaphore semaphore = new Semaphore(Math.max(1, maxConcurrent));
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
if (command == null) {
throw new IllegalArgumentException("task 不能为 null");
}
taskQueueVirtualThreadExecutor.execute(() -> {
boolean acquired = false;
try {
semaphore.acquire();
acquired = true;
command.run();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
} finally {
if (acquired) {
semaphore.release();
}
}
});
});
return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider);
}
@Bean
public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List<ResultFileJobHandler> handlers) {
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
registry.validateCoverage(RESULT_FILE_JOB_MODULE_TYPES);
return registry;
}
@Bean
public ResultFileJobHandler appearancePatentResultFileJobHandler(
AppearancePatentTaskService appearancePatentTaskService) {
return new AppearancePatentResultFileJobHandler(appearancePatentTaskService);
}
@Bean
public ResultFileJobHandler brandResultFileJobHandler(
BrandTaskService brandTaskService, TaskResultPayloadService taskResultPayloadService) {
return new BrandResultFileJobHandler(brandTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler collectDataResultFileJobHandler(CollectDataService collectDataService) {
return new CollectDataResultFileJobHandler(collectDataService);
}
@Bean
public ResultFileJobHandler deleteBrandResultFileJobHandler(DeleteBrandRunService deleteBrandRunService) {
return new DeleteBrandResultFileJobHandler(deleteBrandRunService);
}
@Bean
public ResultFileJobHandler patrolDeleteResultFileJobHandler(
PatrolDeleteTaskService patrolDeleteTaskService, TaskResultPayloadService taskResultPayloadService) {
return new PatrolDeleteResultFileJobHandler(patrolDeleteTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler priceTrackResultFileJobHandler(
PriceTrackTaskService priceTrackTaskService, TaskResultPayloadService taskResultPayloadService) {
return new PriceTrackResultFileJobHandler(priceTrackTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler productRiskResultFileJobHandler(
ProductRiskTaskService productRiskTaskService, TaskResultPayloadService taskResultPayloadService) {
return new ProductRiskResultFileJobHandler(productRiskTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler publishResultFileJobHandler(PublishTaskService publishTaskService) {
return new PublishResultFileJobHandler(publishTaskService);
}
@Bean
public ResultFileJobHandler queryAsinResultFileJobHandler(
QueryAsinTaskService queryAsinTaskService, TaskResultPayloadService taskResultPayloadService) {
return new QueryAsinResultFileJobHandler(queryAsinTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler shopDataCrawlResultFileJobHandler(
ShopDataCrawlTaskService shopDataCrawlTaskService, TaskResultPayloadService taskResultPayloadService) {
return new ShopDataCrawlResultFileJobHandler(shopDataCrawlTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler shopMatchResultFileJobHandler(
ShopMatchTaskService shopMatchTaskService, TaskResultPayloadService taskResultPayloadService) {
return new ShopMatchResultFileJobHandler(shopMatchTaskService, taskResultPayloadService);
}
@Bean
public ResultFileJobHandler similarAsinResultFileJobHandler(SimilarAsinTaskService similarAsinTaskService) {
return new SimilarAsinResultFileJobHandler(similarAsinTaskService);
}
@Bean
public ResultFileJobHandler withdrawResultFileJobHandler(
WithdrawTaskService withdrawTaskService, TaskResultPayloadService taskResultPayloadService) {
return new WithdrawResultFileJobHandler(withdrawTaskService, taskResultPayloadService);
}
}