super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

后端架构更新

Browse Source
This commit is contained in:
super
2026-04-27 09:18:10 +08:00
parent 0caf62c3d2
commit 995e2ee65a
141 changed files with 6957 additions and 772 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
backend-java/src/main/resources/application-local.example.yml
View File

@@ -3,15 +3,22 @@
AIIMAGE_SERVER_PORT=18080
AIIMAGE_DB_URL=jdbc:mysql://159.75.121.33:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
AIIMAGE_DB_USERNAME=AIimage
AIIMAGE_DB_PASSWORD=WTFrb5y6hNLz6hNy
AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
AIIMAGE_DB_USERNAME=change-me
AIIMAGE_DB_PASSWORD=change-me
AIIMAGE_OSS_REGION=cn-hangzhou
AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
AIIMAGE_OSS_BUCKET=nanri-ai-images
AIIMAGE_OSS_ACCESS_KEY_ID=LTAI5tNpyvzMNz9f2dHarsm8
AIIMAGE_OSS_ACCESS_KEY_SECRET=bQSZnFH455i8tzyOgeahJmUzwmhynz
AIIMAGE_OSS_BUCKET=change-me
AIIMAGE_OSS_ACCESS_KEY_ID=change-me
AIIMAGE_OSS_ACCESS_KEY_SECRET=change-me
AIIMAGE_TRANSIENT_STORAGE_ENABLED=true
AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://change-me
AIIMAGE_TRANSIENT_STORAGE_BUCKET=change-me
AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID=change-me
AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET=change-me
AIIMAGE_TRANSIENT_STORAGE_REGION=us-east-1
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=F:/project/crawler-plugin/backend-java/data/tmp
@@ -19,6 +26,22 @@ AIIMAGE_MODULE_CLEANUP_ENABLED=true
AIIMAGE_MODULE_CLEANUP_CRON=0 0 0 * * *
AIIMAGE_MODULE_CLEANUP_MODULE_TYPES=DEDUPE,SPLIT,CONVERT,DELETE_BRAND
AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL=https://api.coze.cn
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH=/v1/workflow/run
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID=7632683471312355338
AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN=
AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10
AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE=2
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY=200
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED=false
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS=10000
AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=30
AIIMAGE_ZINIAO_ENABLED=false
AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router
AIIMAGE_ZINIAO_API_KEY=change-me

35
backend-java/src/main/resources/application.yml
View File

@@ -28,6 +28,12 @@ spring:
database: ${AIIMAGE_REDIS_DATABASE:0}
timeout: ${AIIMAGE_REDIS_TIMEOUT:5s}
rocketmq:
name-server: ${AIIMAGE_ROCKETMQ_NAME_SERVER:121.196.149.225:9876}
producer:
group: ${AIIMAGE_ROCKETMQ_PRODUCER_GROUP:aiimage-backend-producer}
send-message-timeout: ${AIIMAGE_ROCKETMQ_SEND_TIMEOUT_MS:3000}
management:
health:
db:
@@ -67,6 +73,13 @@ aiimage:
bucket: ${AIIMAGE_OSS_BUCKET:change-me}
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:change-me}
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me}
transient-storage:
enabled: ${AIIMAGE_TRANSIENT_STORAGE_ENABLED:false}
endpoint: ${AIIMAGE_TRANSIENT_STORAGE_ENDPOINT:}
bucket: ${AIIMAGE_TRANSIENT_STORAGE_BUCKET:}
access-key-id: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID:}
access-key-secret: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET:}
region: ${AIIMAGE_TRANSIENT_STORAGE_REGION:us-east-1}
storage:
local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}
cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true}
@@ -96,6 +109,8 @@ aiimage:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE}
permission-schema-init:
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
task-pressure:
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
@@ -103,6 +118,26 @@ aiimage:
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
result-file-job:
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:false}
topic: ${AIIMAGE_RESULT_FILE_JOB_TOPIC:aiimage-result-file-job}
consumer-group: ${AIIMAGE_RESULT_FILE_JOB_CONSUMER_GROUP:aiimage-result-file-job-consumer}
local-dispatch-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED:true}
local-dispatch-pool-size: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE:2}
local-dispatch-queue-capacity: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY:200}
local-worker-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED:true}
local-worker-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS:10000}
stuck-scan-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_STUCK_SCAN_DELAY_MS:60000}
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:30}
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
appearance-patent:
coze-base-url: ${AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL:https://api.coze.cn}
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
coze-workflow-id: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID:7632683471312355338}
coze-token: ${AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN:}
coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:10}
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20}
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}

85
backend-java/src/main/resources/db/V34__appearance_patent.sql Normal file
View File

@@ -0,0 +1,85 @@
-- 外观专利检测模块复用通用任务表：
-- biz_file_task 保存任务主记录，module_type = APPEARANCE_PATENT
-- biz_file_result 保存最终 xlsx 结果文件和历史记录
-- biz_task_scope_state 保存解析载荷 OSS 指针、scope 状态和心跳辅助状态
-- biz_task_chunk 保存 Python 回传分片和 Java/Coze 处理后的分片结果
-- 兼容老库：如果通用任务表缺少 user_id，则补齐用户归属字段。
SET @biz_file_task_user_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND COLUMN_NAME = 'user_id'
);
SET @sql_add_biz_file_task_user_id := IF(
@biz_file_task_user_id_exists = 0,
'ALTER TABLE biz_file_task ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER created_by',
'SELECT 1'
);
PREPARE stmt_add_biz_file_task_user_id FROM @sql_add_biz_file_task_user_id;
EXECUTE stmt_add_biz_file_task_user_id;
DEALLOCATE PREPARE stmt_add_biz_file_task_user_id;
SET @biz_file_result_user_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_result'
AND COLUMN_NAME = 'user_id'
);
SET @sql_add_biz_file_result_user_id := IF(
@biz_file_result_user_id_exists = 0,
'ALTER TABLE biz_file_result ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER error_message',
'SELECT 1'
);
PREPARE stmt_add_biz_file_result_user_id FROM @sql_add_biz_file_result_user_id;
EXECUTE stmt_add_biz_file_result_user_id;
DEALLOCATE PREPARE stmt_add_biz_file_result_user_id;
-- 外观专利检测常用查询索引：按用户查任务、按用户查历史。
SET @idx_file_task_module_user_created_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND INDEX_NAME = 'idx_biz_file_task_module_user_created'
);
SET @sql_add_idx_file_task_module_user_created := IF(
@idx_file_task_module_user_created_exists = 0,
'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_module_user_created (module_type, user_id, created_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_task_module_user_created FROM @sql_add_idx_file_task_module_user_created;
EXECUTE stmt_add_idx_file_task_module_user_created;
DEALLOCATE PREPARE stmt_add_idx_file_task_module_user_created;
SET @idx_file_result_module_user_created_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_result'
AND INDEX_NAME = 'idx_biz_file_result_module_user_created'
);
SET @sql_add_idx_file_result_module_user_created := IF(
@idx_file_result_module_user_created_exists = 0,
'ALTER TABLE biz_file_result ADD INDEX idx_biz_file_result_module_user_created (module_type, user_id, created_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_result_module_user_created FROM @sql_add_idx_file_result_module_user_created;
EXECUTE stmt_add_idx_file_result_module_user_created;
DEALLOCATE PREPARE stmt_add_idx_file_result_module_user_created;
-- 前端栏目权限兜底：外观专利检测属于前端工具下的页面。
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '外观专利检测', 'appearance_patent', 'app', 'appearance-patent', 111
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'appearance_patent'
);
UPDATE `columns`
SET `name` = '外观专利检测',
`menu_type` = 'app',
`route_path` = 'appearance-patent',
`sort_order` = 111
WHERE `column_key` = 'appearance_patent';

74
backend-java/src/main/resources/db/V35__task_result_payload_and_file_job.sql Normal file
View File

@@ -0,0 +1,74 @@
CREATE TABLE IF NOT EXISTS biz_task_result_payload (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
submission_id VARCHAR(128) NULL COMMENT 'idempotency key from submitter when available',
payload_json JSON NOT NULL COMMENT 'raw or normalized callback payload',
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
version BIGINT NOT NULL DEFAULT 1 COMMENT 'scope payload version',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_submission (task_id, module_type, scope_hash, submission_id),
KEY idx_task_module_scope (task_id, module_type, scope_hash),
KEY idx_task_module_updated (task_id, module_type, updated_at)
) COMMENT='task result callback payload detail';
CREATE TABLE IF NOT EXISTS biz_task_file_job (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
result_id BIGINT NULL COMMENT 'biz_file_result id when job is scoped to one result row',
scope_key VARCHAR(1000) NULL COMMENT 'scope identifier',
job_type VARCHAR(32) NOT NULL DEFAULT 'ASSEMBLE_RESULT' COMMENT 'job type',
status VARCHAR(32) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING/RUNNING/SUCCESS/FAILED',
retry_count INT NOT NULL DEFAULT 0 COMMENT 'retry count',
error_message VARCHAR(1000) NULL COMMENT 'last error',
result_file_url VARCHAR(1000) NULL COMMENT 'generated file object key or url',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
finished_at DATETIME NULL COMMENT 'finished time',
UNIQUE KEY uk_file_job_scope (task_id, module_type, result_id, job_type),
KEY idx_file_job_status (status, updated_at),
KEY idx_file_job_task (task_id, module_type)
) COMMENT='async result file assembly job';
CREATE TABLE IF NOT EXISTS biz_task_result_item (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
result_id BIGINT NULL COMMENT 'biz_file_result id when available',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
item_key VARCHAR(512) NOT NULL COMMENT 'dedupe key, such as country + asin',
country_code VARCHAR(32) NULL COMMENT 'country code',
asin VARCHAR(128) NULL COMMENT 'asin or item id',
status VARCHAR(64) NULL COMMENT 'business status',
payload_json JSON NOT NULL COMMENT 'normalized row payload',
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_item (task_id, module_type, scope_hash, item_key),
KEY idx_result_item_result (result_id),
KEY idx_result_item_task (task_id, module_type),
KEY idx_result_item_country_asin (task_id, module_type, country_code, asin)
) COMMENT='normalized task result item detail';
CREATE TABLE IF NOT EXISTS biz_task_progress_snapshot (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
status VARCHAR(32) NOT NULL COMMENT 'task status',
total_count INT NOT NULL DEFAULT 0 COMMENT 'total item count',
success_count INT NOT NULL DEFAULT 0 COMMENT 'success count',
failed_count INT NOT NULL DEFAULT 0 COMMENT 'failed count',
pending_count INT NOT NULL DEFAULT 0 COMMENT 'pending count',
current_scope_key VARCHAR(1000) NULL COMMENT 'current scope identifier',
message VARCHAR(1000) NULL COMMENT 'progress message',
snapshot_json JSON NULL COMMENT 'small progress snapshot',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_progress_snapshot (task_id, module_type),
KEY idx_progress_module_status (module_type, status, updated_at)
) COMMENT='lightweight task progress snapshot';