super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

后端架构更新

Browse Source
This commit is contained in:
super
2026-04-27 09:18:10 +08:00
parent 0caf62c3d2
commit 995e2ee65a
141 changed files with 6957 additions and 772 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
backend-java/src/main/resources/db/V34__appearance_patent.sql Normal file
View File

@@ -0,0 +1,85 @@
-- 外观专利检测模块复用通用任务表：
-- biz_file_task 保存任务主记录，module_type = APPEARANCE_PATENT
-- biz_file_result 保存最终 xlsx 结果文件和历史记录
-- biz_task_scope_state 保存解析载荷 OSS 指针、scope 状态和心跳辅助状态
-- biz_task_chunk 保存 Python 回传分片和 Java/Coze 处理后的分片结果
-- 兼容老库：如果通用任务表缺少 user_id，则补齐用户归属字段。
SET @biz_file_task_user_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND COLUMN_NAME = 'user_id'
);
SET @sql_add_biz_file_task_user_id := IF(
@biz_file_task_user_id_exists = 0,
'ALTER TABLE biz_file_task ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER created_by',
'SELECT 1'
);
PREPARE stmt_add_biz_file_task_user_id FROM @sql_add_biz_file_task_user_id;
EXECUTE stmt_add_biz_file_task_user_id;
DEALLOCATE PREPARE stmt_add_biz_file_task_user_id;
SET @biz_file_result_user_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_result'
AND COLUMN_NAME = 'user_id'
);
SET @sql_add_biz_file_result_user_id := IF(
@biz_file_result_user_id_exists = 0,
'ALTER TABLE biz_file_result ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER error_message',
'SELECT 1'
);
PREPARE stmt_add_biz_file_result_user_id FROM @sql_add_biz_file_result_user_id;
EXECUTE stmt_add_biz_file_result_user_id;
DEALLOCATE PREPARE stmt_add_biz_file_result_user_id;
-- 外观专利检测常用查询索引：按用户查任务、按用户查历史。
SET @idx_file_task_module_user_created_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND INDEX_NAME = 'idx_biz_file_task_module_user_created'
);
SET @sql_add_idx_file_task_module_user_created := IF(
@idx_file_task_module_user_created_exists = 0,
'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_module_user_created (module_type, user_id, created_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_task_module_user_created FROM @sql_add_idx_file_task_module_user_created;
EXECUTE stmt_add_idx_file_task_module_user_created;
DEALLOCATE PREPARE stmt_add_idx_file_task_module_user_created;
SET @idx_file_result_module_user_created_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_result'
AND INDEX_NAME = 'idx_biz_file_result_module_user_created'
);
SET @sql_add_idx_file_result_module_user_created := IF(
@idx_file_result_module_user_created_exists = 0,
'ALTER TABLE biz_file_result ADD INDEX idx_biz_file_result_module_user_created (module_type, user_id, created_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_result_module_user_created FROM @sql_add_idx_file_result_module_user_created;
EXECUTE stmt_add_idx_file_result_module_user_created;
DEALLOCATE PREPARE stmt_add_idx_file_result_module_user_created;
-- 前端栏目权限兜底：外观专利检测属于前端工具下的页面。
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '外观专利检测', 'appearance_patent', 'app', 'appearance-patent', 111
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'appearance_patent'
);
UPDATE `columns`
SET `name` = '外观专利检测',
`menu_type` = 'app',
`route_path` = 'appearance-patent',
`sort_order` = 111
WHERE `column_key` = 'appearance_patent';

74
backend-java/src/main/resources/db/V35__task_result_payload_and_file_job.sql Normal file
View File

@@ -0,0 +1,74 @@
CREATE TABLE IF NOT EXISTS biz_task_result_payload (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
submission_id VARCHAR(128) NULL COMMENT 'idempotency key from submitter when available',
payload_json JSON NOT NULL COMMENT 'raw or normalized callback payload',
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
version BIGINT NOT NULL DEFAULT 1 COMMENT 'scope payload version',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_submission (task_id, module_type, scope_hash, submission_id),
KEY idx_task_module_scope (task_id, module_type, scope_hash),
KEY idx_task_module_updated (task_id, module_type, updated_at)
) COMMENT='task result callback payload detail';
CREATE TABLE IF NOT EXISTS biz_task_file_job (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
result_id BIGINT NULL COMMENT 'biz_file_result id when job is scoped to one result row',
scope_key VARCHAR(1000) NULL COMMENT 'scope identifier',
job_type VARCHAR(32) NOT NULL DEFAULT 'ASSEMBLE_RESULT' COMMENT 'job type',
status VARCHAR(32) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING/RUNNING/SUCCESS/FAILED',
retry_count INT NOT NULL DEFAULT 0 COMMENT 'retry count',
error_message VARCHAR(1000) NULL COMMENT 'last error',
result_file_url VARCHAR(1000) NULL COMMENT 'generated file object key or url',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
finished_at DATETIME NULL COMMENT 'finished time',
UNIQUE KEY uk_file_job_scope (task_id, module_type, result_id, job_type),
KEY idx_file_job_status (status, updated_at),
KEY idx_file_job_task (task_id, module_type)
) COMMENT='async result file assembly job';
CREATE TABLE IF NOT EXISTS biz_task_result_item (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
result_id BIGINT NULL COMMENT 'biz_file_result id when available',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
item_key VARCHAR(512) NOT NULL COMMENT 'dedupe key, such as country + asin',
country_code VARCHAR(32) NULL COMMENT 'country code',
asin VARCHAR(128) NULL COMMENT 'asin or item id',
status VARCHAR(64) NULL COMMENT 'business status',
payload_json JSON NOT NULL COMMENT 'normalized row payload',
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_item (task_id, module_type, scope_hash, item_key),
KEY idx_result_item_result (result_id),
KEY idx_result_item_task (task_id, module_type),
KEY idx_result_item_country_asin (task_id, module_type, country_code, asin)
) COMMENT='normalized task result item detail';
CREATE TABLE IF NOT EXISTS biz_task_progress_snapshot (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
status VARCHAR(32) NOT NULL COMMENT 'task status',
total_count INT NOT NULL DEFAULT 0 COMMENT 'total item count',
success_count INT NOT NULL DEFAULT 0 COMMENT 'success count',
failed_count INT NOT NULL DEFAULT 0 COMMENT 'failed count',
pending_count INT NOT NULL DEFAULT 0 COMMENT 'pending count',
current_scope_key VARCHAR(1000) NULL COMMENT 'current scope identifier',
message VARCHAR(1000) NULL COMMENT 'progress message',
snapshot_json JSON NULL COMMENT 'small progress snapshot',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_progress_snapshot (task_id, module_type),
KEY idx_progress_module_status (module_type, status, updated_at)
) COMMENT='lightweight task progress snapshot';