super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善任务存储、权限及货源查询流程
Build Backend JAR / build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
supernijia
2026-08-14 22:49:15 +08:00
parent 5b1ccad40e
commit 7a7f1dfa21
21 changed files with 2584 additions and 566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/resources/application.yml
+7 -2
View File
@@ -103,6 +103,8 @@ aiimage:
connect-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_CONNECT_TIMEOUT_SECONDS:10}
read-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_READ_TIMEOUT_SECONDS:60}
write-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_WRITE_TIMEOUT_SECONDS:60}
call-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_CALL_TIMEOUT_SECONDS:90}
operation-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_OPERATION_TIMEOUT_SECONDS:120}
upload-max-retries: ${AIIMAGE_TRANSIENT_STORAGE_UPLOAD_MAX_RETRIES:3}
read-max-retries: ${AIIMAGE_TRANSIENT_STORAGE_READ_MAX_RETRIES:3}
delete-max-retries: ${AIIMAGE_TRANSIENT_STORAGE_DELETE_MAX_RETRIES:3}
@@ -116,8 +118,10 @@ aiimage:
failure-window-seconds: ${AIIMAGE_TRANSIENT_STORAGE_FAILURE_WINDOW_SECONDS:60}
failure-window-threshold: ${AIIMAGE_TRANSIENT_STORAGE_FAILURE_WINDOW_THRESHOLD:20}
failure-cooldown-millis: ${AIIMAGE_TRANSIENT_STORAGE_FAILURE_COOLDOWN_MILLIS:10000}
connection-pool-max-idle: ${AIIMAGE_TRANSIENT_STORAGE_CONNECTION_POOL_MAX_IDLE:0}
connection-pool-keep-alive-millis: ${AIIMAGE_TRANSIENT_STORAGE_CONNECTION_POOL_KEEP_ALIVE_MILLIS:1}
dispatcher-max-requests: ${AIIMAGE_TRANSIENT_STORAGE_DISPATCHER_MAX_REQUESTS:56}
dispatcher-max-requests-per-host: ${AIIMAGE_TRANSIENT_STORAGE_DISPATCHER_MAX_REQUESTS_PER_HOST:56}
connection-pool-max-idle: ${AIIMAGE_TRANSIENT_STORAGE_CONNECTION_POOL_MAX_IDLE:5}
connection-pool-keep-alive-millis: ${AIIMAGE_TRANSIENT_STORAGE_CONNECTION_POOL_KEEP_ALIVE_MILLIS:300000}
warn-payload-bytes: ${AIIMAGE_TRANSIENT_STORAGE_WARN_PAYLOAD_BYTES:5242880}
max-payload-bytes: ${AIIMAGE_TRANSIENT_STORAGE_MAX_PAYLOAD_BYTES:52428800}
max-stored-payload-bytes: ${AIIMAGE_TRANSIENT_STORAGE_MAX_STORED_PAYLOAD_BYTES:52428800}
@@ -126,6 +130,7 @@ aiimage:
delete-retry-cron: ${AIIMAGE_TRANSIENT_STORAGE_DELETE_RETRY_CRON:0 */5 * * * *}
delete-retry-queue-capacity: ${AIIMAGE_TRANSIENT_STORAGE_DELETE_RETRY_QUEUE_CAPACITY:10000}
delete-retry-batch-size: ${AIIMAGE_TRANSIENT_STORAGE_DELETE_RETRY_BATCH_SIZE:200}
delete-retry-batch-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_DELETE_RETRY_BATCH_TIMEOUT_SECONDS:60}
storage:
local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}
cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true}
backend-java/src/main/resources/db/V88__expand_invalid_asin_brand.sql
+36
View File
@@ -0,0 +1,36 @@
SET @invalid_asin_brand_index_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND INDEX_NAME = 'idx_brand'
);
SET @sql_drop_invalid_asin_brand_index := IF(
@invalid_asin_brand_index_exists > 0,
'ALTER TABLE biz_invalid_asin_data DROP INDEX idx_brand',
'SELECT 1'
);
PREPARE stmt_drop_invalid_asin_brand_index FROM @sql_drop_invalid_asin_brand_index;
EXECUTE stmt_drop_invalid_asin_brand_index;
DEALLOCATE PREPARE stmt_drop_invalid_asin_brand_index;
ALTER TABLE biz_invalid_asin_data
MODIFY COLUMN brand TEXT NULL COMMENT 'brand name';
SET @invalid_asin_brand_prefix_index_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND INDEX_NAME = 'idx_brand_prefix'
);
SET @sql_add_invalid_asin_brand_prefix_index := IF(
@invalid_asin_brand_prefix_index_exists = 0,
'ALTER TABLE biz_invalid_asin_data ADD INDEX idx_brand_prefix (brand(191))',
'SELECT 1'
);
PREPARE stmt_add_invalid_asin_brand_prefix_index FROM @sql_add_invalid_asin_brand_prefix_index;
EXECUTE stmt_add_invalid_asin_brand_prefix_index;
DEALLOCATE PREPARE stmt_add_invalid_asin_brand_prefix_index;