super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update task routing and coze handling

Browse Source
This commit is contained in:
super
2026-06-03 17:14:50 +08:00
parent ea37d82d73
commit e98a1a1207
37 changed files with 1883 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
backend-java/src/main/resources/application.yml
View File

@@ -75,6 +75,14 @@ aiimage:
# For IDEA/local startup, set -Daiimage.instance-id=<stable-id> or configure the AIIMAGE_INSTANCE_ID env var.
# Avoid relying on container hostname/container ID, otherwise task owner continuity may break after restart/redeploy.
instance-id: ${AIIMAGE_INSTANCE_ID:local-dev}
instance-routing:
routes:
server-110: ${AIIMAGE_INSTANCE_ROUTE_SERVER_110:http://192.168.0.171:18080}
server-111: ${AIIMAGE_INSTANCE_ROUTE_SERVER_111:http://47.111.163.154:18080}
server-121: ${AIIMAGE_INSTANCE_ROUTE_SERVER_121:http://192.168.0.170:18080}
connect-timeout-millis: ${AIIMAGE_INSTANCE_ROUTE_CONNECT_TIMEOUT_MILLIS:3000}
read-timeout-millis: ${AIIMAGE_INSTANCE_ROUTE_READ_TIMEOUT_MILLIS:300000}
request-body-cache-limit-bytes: ${AIIMAGE_INSTANCE_ROUTE_REQUEST_BODY_CACHE_LIMIT_BYTES:104857600}
oss:
region: ${AIIMAGE_OSS_REGION:cn-hangzhou}
endpoint: ${AIIMAGE_OSS_ENDPOINT:oss-cn-hangzhou.aliyuncs.com}
@@ -104,21 +112,17 @@ aiimage:
brand-progress:
ttl-hours: ${AIIMAGE_BRAND_PROGRESS_TTL_HOURS:24}
failed-ttl-hours: ${AIIMAGE_BRAND_PROGRESS_FAILED_TTL_HOURS:2}
heartbeat-timeout-minutes: ${AIIMAGE_BRAND_PROGRESS_HEARTBEAT_TIMEOUT_MINUTES:15}
heartbeat-timeout-minutes: ${AIIMAGE_BRAND_PROGRESS_HEARTBEAT_TIMEOUT_MINUTES:30}
stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *}
delete-brand-progress:
heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:15}
delete-brand-initial-timeout-minutes: ${AIIMAGE_DELETE_BRAND_INITIAL_TIMEOUT_MINUTES:20}
heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:30}
stale-check-cron: ${AIIMAGE_DELETE_BRAND_STALE_CHECK_CRON:*/30 * * * * *}
finalize-check-cron: ${AIIMAGE_DELETE_BRAND_FINALIZE_CHECK_CRON:30 */2 * * * *}
product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:20}
product-risk-initial-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_INITIAL_TIMEOUT_MINUTES:20}
price-track-stale-timeout-minutes: ${AIIMAGE_PRICE_TRACK_STALE_TIMEOUT_MINUTES:20}
price-track-initial-timeout-minutes: ${AIIMAGE_PRICE_TRACK_INITIAL_TIMEOUT_MINUTES:20}
shop-match-stale-timeout-minutes: ${AIIMAGE_SHOP_MATCH_STALE_TIMEOUT_MINUTES:20}
shop-match-initial-timeout-minutes: ${AIIMAGE_SHOP_MATCH_INITIAL_TIMEOUT_MINUTES:20}
patrol-delete-stale-timeout-minutes: ${AIIMAGE_PATROL_DELETE_STALE_TIMEOUT_MINUTES:20}
patrol-delete-initial-timeout-minutes: ${AIIMAGE_PATROL_DELETE_INITIAL_TIMEOUT_MINUTES:20}
product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:30}
price-track-stale-timeout-minutes: ${AIIMAGE_PRICE_TRACK_STALE_TIMEOUT_MINUTES:30}
shop-match-stale-timeout-minutes: ${AIIMAGE_SHOP_MATCH_STALE_TIMEOUT_MINUTES:30}
patrol-delete-stale-timeout-minutes: ${AIIMAGE_PATROL_DELETE_STALE_TIMEOUT_MINUTES:30}
query-asin-stale-timeout-minutes: ${AIIMAGE_QUERY_ASIN_STALE_TIMEOUT_MINUTES:30}
module-cleanup:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
@@ -132,6 +136,12 @@ aiimage:
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
task-image-cache-cleanup:
enabled: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_CRON:0 30 2 * * *}
retention-days: ${AIIMAGE_TASK_IMAGE_CACHE_RETENTION_DAYS:3}
batch-size: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_BATCH_SIZE:5000}
max-batches-per-run: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_MAX_BATCHES_PER_RUN:200}
result-file-job:
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:true}
topic: ${AIIMAGE_RESULT_FILE_JOB_TOPIC:aiimage-result-file-job}
@@ -149,6 +159,13 @@ aiimage:
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
coze-task:
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
brand-check:
base-url: ${AIIMAGE_BRAND_CHECK_BASE_URL:http://47.110.241.161:16890}
path: ${AIIMAGE_BRAND_CHECK_PATH:/brand_check}
token: ${AIIMAGE_BRAND_CHECK_TOKEN:eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9}
default-strategy: ${AIIMAGE_BRAND_CHECK_DEFAULT_STRATEGY:Terms}
connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000}
read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000}
appearance-patent:
coze-base-url: ${AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL:https://api.coze.cn}
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
@@ -159,6 +176,7 @@ aiimage:
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
coze-poll-interval-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_INTERVAL_MILLIS:30000}
coze-poll-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_TIMEOUT_MILLIS:600000}
coze-flush-pending-minutes: ${AIIMAGE_APPEARANCE_PATENT_COZE_FLUSH_PENDING_MINUTES:1}
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:30}
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
similar-asin: