super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交货源流程优化

Browse Source
This commit is contained in:
super
2026-05-25 22:16:50 +08:00
parent 7503e3fa8b
commit 73ac9187a6
14 changed files with 1490 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend-java/src/main/resources/application-local.example.yml
View File

@@ -43,7 +43,7 @@ AIIMAGE_SIMILAR_ASIN_COZE_BASE_URL=https://api.coze.cn
AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH=/v1/workflow/run
AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID=7635328462404583478
AIIMAGE_SIMILAR_ASIN_COZE_TOKEN=
AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE=50
AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE=3
AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS=60000
AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES=30

9
backend-java/src/main/resources/application.yml
View File

@@ -166,17 +166,18 @@ aiimage:
coze-workflow-path: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH:/v1/workflow/run}
coze-workflow-id: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID:7639708860686024756}
coze-token: ${AIIMAGE_SIMILAR_ASIN_COZE_TOKEN:}
coze-batch-size: ${AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE:10}
coze-batch-size: ${AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE:3}
coze-credential-stripe-size: ${AIIMAGE_SIMILAR_ASIN_COZE_CREDENTIAL_STRIPE_SIZE:0}
coze-connect-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_CONNECT_TIMEOUT_MILLIS:10000}
coze-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS:60000}
coze-poll-interval-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_POLL_INTERVAL_MILLIS:30000}
coze-poll-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_POLL_TIMEOUT_MILLIS:1800000}
coze-submit-min-interval-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MIN_INTERVAL_MILLIS:5000}
coze-flush-pending-minutes: ${AIIMAGE_SIMILAR_ASIN_COZE_FLUSH_PENDING_MINUTES:15}
coze-flush-pending-minutes: ${AIIMAGE_SIMILAR_ASIN_COZE_FLUSH_PENDING_MINUTES:1}
coze-submit-max-retry-count: ${AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MAX_RETRY_COUNT:5}
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:16}
image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:8}
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:32}
image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:5}
image-cache-max-bytes: ${AIIMAGE_SIMILAR_ASIN_IMAGE_CACHE_MAX_BYTES:268435456}
stale-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES:30}
stale-finalize-cron: ${AIIMAGE_SIMILAR_ASIN_STALE_FINALIZE_CRON:0 */2 * * * *}
coze-include-legacy-api-key: ${AIIMAGE_SIMILAR_ASIN_COZE_INCLUDE_LEGACY_API_KEY:true}

18
backend-java/src/main/resources/db/V53__task_image_cache.sql Normal file
View File

@@ -0,0 +1,18 @@
-- P2-12：相似ASIN/通用 图片缩略图缓存表。
-- 用于跨任务复用 Coze 回包中的 main_url / puzzle_img1 / puzzle_img2 等图片，
-- 与 SimilarAsinImagePrefetchService 协作，避免 assemble 阶段每次都重新下载远程图片。
-- url_hash 走 sha256(lowercase hex)，避免 1024 长 URL 作为唯一键命中索引长度限制。
CREATE TABLE IF NOT EXISTS biz_task_image_cache (
id BIGINT NOT NULL AUTO_INCREMENT,
url_hash CHAR(64) NOT NULL COMMENT 'sha256 lowercase hex of url',
url VARCHAR(1024) NOT NULL,
image_bytes MEDIUMBLOB NOT NULL COMMENT '已 resize 缩略图字节，最大约 300KB',
byte_size INT NOT NULL DEFAULT 0,
width INT NOT NULL DEFAULT 0,
height INT NOT NULL DEFAULT 0,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
last_used_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
PRIMARY KEY (id),
UNIQUE KEY uk_task_image_cache_url_hash (url_hash),
KEY idx_task_image_cache_last_used_at (last_used_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='相似ASIN/通用 图片缩略图缓存，跨任务复用';