super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

异常记录检测修改

Browse Source
This commit is contained in:
super
2026-05-04 19:04:55 +08:00
parent f46c231323
commit 111f30d150
70 changed files with 7182 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend-java/src/main/resources/application-local.example.yml
View File

@@ -34,6 +34,14 @@ AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10
AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000
AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20
AIIMAGE_SIMILAR_ASIN_COZE_BASE_URL=https://api.coze.cn
AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH=/v1/workflow/run
AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID=7632683471312355338
AIIMAGE_SIMILAR_ASIN_COZE_TOKEN=
AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE=10
AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS=60000
AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES=20
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true

12
backend-java/src/main/resources/application.yml
View File

@@ -94,6 +94,7 @@ aiimage:
cleanup-cron: ${AIIMAGE_STORAGE_CLEANUP_CRON:0 0 * * * *}
source-retention-hours: ${AIIMAGE_STORAGE_SOURCE_RETENTION_HOURS:24}
result-retention-hours: ${AIIMAGE_STORAGE_RESULT_RETENTION_HOURS:24}
transient-payload-retention-hours: ${AIIMAGE_STORAGE_TRANSIENT_PAYLOAD_RETENTION_HOURS:72}
temp-dir:
retention-hours: ${AIIMAGE_TEMP_DIR_RETENTION_HOURS:24}
brand-progress:
@@ -103,6 +104,7 @@ aiimage:
stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *}
delete-brand-progress:
heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:15}
delete-brand-initial-timeout-minutes: ${AIIMAGE_DELETE_BRAND_INITIAL_TIMEOUT_MINUTES:20}
stale-check-cron: ${AIIMAGE_DELETE_BRAND_STALE_CHECK_CRON:*/30 * * * * *}
finalize-check-cron: ${AIIMAGE_DELETE_BRAND_FINALIZE_CHECK_CRON:30 */2 * * * *}
product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:20}
@@ -148,6 +150,16 @@ aiimage:
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20}
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
similar-asin:
coze-base-url: ${AIIMAGE_SIMILAR_ASIN_COZE_BASE_URL:https://api.coze.cn}
coze-workflow-path: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH:/v1/workflow/run}
coze-workflow-id: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID:7632683471312355338}
coze-token: ${AIIMAGE_SIMILAR_ASIN_COZE_TOKEN:Bearer sat_gsInckdsN0qqhnZSK7K4Zevw6neJCHmYHyIbm85oGgJIGEmSdkH5OFctSKLsAHvT}
coze-batch-size: ${AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE:10}
coze-connect-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_CONNECT_TIMEOUT_MILLIS:10000}
coze-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS:60000}
stale-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES:20}
stale-finalize-cron: ${AIIMAGE_SIMILAR_ASIN_STALE_FINALIZE_CRON:0 */2 * * * *}
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}

127
backend-java/src/main/resources/db/V43__appearance_patent_coze_async.sql Normal file
View File

@@ -0,0 +1,127 @@
SET @coze_execute_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_execute_id'
);
SET @sql_add_coze_execute_id := IF(
@coze_execute_id_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_execute_id VARCHAR(128) NULL COMMENT ''Coze execute id'' AFTER state_json',
'SELECT 1'
);
PREPARE stmt_add_coze_execute_id FROM @sql_add_coze_execute_id;
EXECUTE stmt_add_coze_execute_id;
DEALLOCATE PREPARE stmt_add_coze_execute_id;
SET @coze_status_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_status'
);
SET @sql_add_coze_status := IF(
@coze_status_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_status VARCHAR(32) NULL COMMENT ''Coze workflow status'' AFTER coze_execute_id',
'SELECT 1'
);
PREPARE stmt_add_coze_status FROM @sql_add_coze_status;
EXECUTE stmt_add_coze_status;
DEALLOCATE PREPARE stmt_add_coze_status;
SET @coze_submitted_at_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_submitted_at'
);
SET @sql_add_coze_submitted_at := IF(
@coze_submitted_at_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_submitted_at DATETIME NULL COMMENT ''Coze submitted time'' AFTER coze_status',
'SELECT 1'
);
PREPARE stmt_add_coze_submitted_at FROM @sql_add_coze_submitted_at;
EXECUTE stmt_add_coze_submitted_at;
DEALLOCATE PREPARE stmt_add_coze_submitted_at;
SET @coze_last_polled_at_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_last_polled_at'
);
SET @sql_add_coze_last_polled_at := IF(
@coze_last_polled_at_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_last_polled_at DATETIME NULL COMMENT ''Coze last polled time'' AFTER coze_submitted_at',
'SELECT 1'
);
PREPARE stmt_add_coze_last_polled_at FROM @sql_add_coze_last_polled_at;
EXECUTE stmt_add_coze_last_polled_at;
DEALLOCATE PREPARE stmt_add_coze_last_polled_at;
SET @coze_completed_at_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_completed_at'
);
SET @sql_add_coze_completed_at := IF(
@coze_completed_at_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_completed_at DATETIME NULL COMMENT ''Coze completed time'' AFTER coze_last_polled_at',
'SELECT 1'
);
PREPARE stmt_add_coze_completed_at FROM @sql_add_coze_completed_at;
EXECUTE stmt_add_coze_completed_at;
DEALLOCATE PREPARE stmt_add_coze_completed_at;
SET @coze_attempt_count_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_attempt_count'
);
SET @sql_add_coze_attempt_count := IF(
@coze_attempt_count_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_attempt_count INT NOT NULL DEFAULT 0 COMMENT ''Coze poll attempt count'' AFTER coze_completed_at',
'SELECT 1'
);
PREPARE stmt_add_coze_attempt_count FROM @sql_add_coze_attempt_count;
EXECUTE stmt_add_coze_attempt_count;
DEALLOCATE PREPARE stmt_add_coze_attempt_count;
SET @coze_error_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_error'
);
SET @sql_add_coze_error := IF(
@coze_error_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_error VARCHAR(1000) NULL COMMENT ''Coze error message'' AFTER coze_attempt_count',
'SELECT 1'
);
PREPARE stmt_add_coze_error FROM @sql_add_coze_error;
EXECUTE stmt_add_coze_error;
DEALLOCATE PREPARE stmt_add_coze_error;
SET @idx_task_coze_status_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND INDEX_NAME = 'idx_task_coze_status'
);
SET @sql_add_idx_task_coze_status := IF(
@idx_task_coze_status_exists = 0,
'ALTER TABLE biz_task_scope_state ADD INDEX idx_task_coze_status (module_type, coze_status, updated_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_task_coze_status FROM @sql_add_idx_task_coze_status;
EXECUTE stmt_add_idx_task_coze_status;
DEALLOCATE PREPARE stmt_add_idx_task_coze_status;

15
backend-java/src/main/resources/db/V44__delete_brand_history_order_indexes.sql Normal file
View File

@@ -0,0 +1,15 @@
SET @idx_file_task_delete_brand_history_updated_exists := (
SELECT COUNT(1)
FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND INDEX_NAME = 'idx_file_task_delete_brand_history_updated'
);
SET @sql_add_idx_file_task_delete_brand_history_updated := IF(
@idx_file_task_delete_brand_history_updated_exists = 0,
'ALTER TABLE biz_file_task ADD INDEX idx_file_task_delete_brand_history_updated (module_type, user_id, updated_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_task_delete_brand_history_updated FROM @sql_add_idx_file_task_delete_brand_history_updated;
EXECUTE stmt_add_idx_file_task_delete_brand_history_updated;
DEALLOCATE PREPARE stmt_add_idx_file_task_delete_brand_history_updated;

14
backend-java/src/main/resources/db/V45__similar_asin.sql Normal file
View File

@@ -0,0 +1,14 @@
-- Similar ASIN detection reuses the generic file task/result/chunk tables.
-- Register the frontend app menu column so permission queries can expose the page.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '相似ASIN检测', 'similar_asin', 'app', 'similar-asin', 112
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'similar_asin'
);
UPDATE `columns`
SET `name` = '相似ASIN检测',
`menu_type` = 'app',
`route_path` = 'similar-asin',
`sort_order` = 112
WHERE `column_key` = 'similar_asin';