super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成后端架构重构等

Browse Source
This commit is contained in:
super
2026-04-23 15:25:41 +08:00
parent 6894f9cc57
commit 0391cb223f
86 changed files with 10843 additions and 1757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend-java/src/main/resources/application.yml
View File

@@ -99,6 +99,10 @@ aiimage:
task-pressure:
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
scope-payload-flush-interval-millis: ${AIIMAGE_TASK_SCOPE_PAYLOAD_FLUSH_INTERVAL_MILLIS:15000}
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}

76
backend-java/src/main/resources/db/V29__skip_price_asin_minimum_price.sql Normal file
View File

@@ -0,0 +1,76 @@
SET @schema_name = DATABASE();
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_de'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_de DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''DE minimum price'' AFTER asin_de'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_uk'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_uk DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''UK minimum price'' AFTER asin_uk'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_fr'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_fr DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''FR minimum price'' AFTER asin_fr'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_it'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_it DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''IT minimum price'' AFTER asin_it'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_es'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_es DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''ES minimum price'' AFTER asin_es'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

36
backend-java/src/main/resources/db/V30__task_chunk_storage.sql Normal file
View File

@@ -0,0 +1,36 @@
CREATE TABLE IF NOT EXISTS biz_task_scope_state (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
parsed_payload_json JSON NULL COMMENT 'parsed source payload',
state_json JSON NULL COMMENT 'merged intermediate state',
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'expected chunk count',
received_chunk_count INT NOT NULL DEFAULT 0 COMMENT 'received chunk count',
completed TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'whether scope is complete',
last_chunk_at DATETIME NULL COMMENT 'last chunk received time',
last_error VARCHAR(1000) NULL COMMENT 'last error message',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope (task_id, module_type, scope_hash),
KEY idx_task_completed (task_id, module_type, completed),
KEY idx_scope_hash (scope_hash)
) COMMENT='task scope state';
CREATE TABLE IF NOT EXISTS biz_task_chunk (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
chunk_index INT NOT NULL COMMENT 'chunk index starting from 1',
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'total chunk count',
payload_json JSON NOT NULL COMMENT 'chunk payload',
payload_hash CHAR(64) NOT NULL COMMENT 'hash of payload json',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_chunk (task_id, module_type, scope_hash, chunk_index),
KEY idx_task_scope (task_id, module_type, scope_hash),
KEY idx_task_module (task_id, module_type)
) COMMENT='task chunk detail';