task-48: invalid ASIN 记录改为批量 INSERT IGNORE/upsert,唯一键扩为 (data_value, brand)
This commit is contained in:
@@ -263,6 +263,7 @@ aiimage:
|
||||
max-chunk-rows: ${AIIMAGE_COLLECT_DATA_MAX_CHUNK_ROWS:0}
|
||||
brand-check-batch-size: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_BATCH_SIZE:10}
|
||||
brand-check-cache-capacity: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CACHE_CAPACITY:512}
|
||||
invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100}
|
||||
image-video:
|
||||
coze-base-url: ${AIIMAGE_IMAGE_VIDEO_COZE_BASE_URL:https://api.coze.cn}
|
||||
coze-token: ${AIIMAGE_IMAGE_VIDEO_COZE_TOKEN:sat_Ws4VB1caOPasDivpKIvtOySYx3lhKgQ95H3crIh0tBwiNYtPTyi6bqe0pBaRzpVu}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
-- invalid ASIN 记录批量 INSERT IGNORE:唯一键从 data_value 扩展为
|
||||
-- (data_value, brand),与记录语义键(ASIN + 品牌)一致,保证批量幂等。
|
||||
-- 历史冲突行仅保留最新一条(id 最大),再按冲突行重建唯一键。
|
||||
CREATE TABLE IF NOT EXISTS biz_invalid_asin_data (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键',
|
||||
data_value VARCHAR(128) NOT NULL COMMENT '不符合 ASIN 值',
|
||||
brand TEXT NULL COMMENT 'brand name',
|
||||
group_id BIGINT NULL COMMENT '分组ID',
|
||||
record_source VARCHAR(32) NOT NULL DEFAULT 'AUTO' COMMENT '记录来源',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
KEY idx_brand_prefix (brand(191)),
|
||||
KEY idx_created_at (created_at)
|
||||
) COMMENT='不符合ASIN数据表';
|
||||
|
||||
-- 1. 清理 (data_value, brand) 完全重复的行,仅保留 id 最大的一条
|
||||
DELETE t1 FROM biz_invalid_asin_data t1
|
||||
JOIN biz_invalid_asin_data t2
|
||||
ON t1.data_value = t2.data_value
|
||||
AND ((t1.brand <=> t2.brand) OR (t1.brand IS NULL AND t2.brand IS NULL))
|
||||
AND t1.id < t2.id;
|
||||
|
||||
-- 2. 删除既有单列唯一键 uk_data_value(存在时)
|
||||
SET @uk_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'biz_invalid_asin_data'
|
||||
AND INDEX_NAME = 'uk_data_value'
|
||||
);
|
||||
SET @sql_drop_uk := IF(
|
||||
@uk_exists > 0,
|
||||
'ALTER TABLE biz_invalid_asin_data DROP INDEX uk_data_value',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt_drop_uk FROM @sql_drop_uk;
|
||||
EXECUTE stmt_drop_uk;
|
||||
DEALLOCATE PREPARE stmt_drop_uk;
|
||||
|
||||
-- 3. 重建为 (data_value, brand) 复合唯一键(不存在时)
|
||||
SET @uk_dup_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'biz_invalid_asin_data'
|
||||
AND INDEX_NAME = 'uk_data_value_brand'
|
||||
);
|
||||
SET @sql_add_uk := IF(
|
||||
@uk_dup_exists = 0,
|
||||
'ALTER TABLE biz_invalid_asin_data ADD UNIQUE KEY uk_data_value_brand (data_value, brand(191))',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt_add_uk FROM @sql_add_uk;
|
||||
EXECUTE stmt_add_uk;
|
||||
DEALLOCATE PREPARE stmt_add_uk;
|
||||
Reference in New Issue
Block a user