task-48: invalid ASIN 记录改为批量 INSERT IGNORE/upsert,唯一键扩为 (data_value, brand)

This commit is contained in:
2026-08-30 14:11:59 +08:00
parent c609ffb279
commit 98c45486fa
6 changed files with 380 additions and 27 deletions
@@ -263,6 +263,7 @@ aiimage:
max-chunk-rows: ${AIIMAGE_COLLECT_DATA_MAX_CHUNK_ROWS:0}
brand-check-batch-size: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_BATCH_SIZE:10}
brand-check-cache-capacity: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CACHE_CAPACITY:512}
invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100}
image-video:
coze-base-url: ${AIIMAGE_IMAGE_VIDEO_COZE_BASE_URL:https://api.coze.cn}
coze-token: ${AIIMAGE_IMAGE_VIDEO_COZE_TOKEN:sat_Ws4VB1caOPasDivpKIvtOySYx3lhKgQ95H3crIh0tBwiNYtPTyi6bqe0pBaRzpVu}
@@ -0,0 +1,53 @@
-- invalid ASIN 记录批量 INSERT IGNORE:唯一键从 data_value 扩展为
-- (data_value, brand),与记录语义键(ASIN + 品牌)一致,保证批量幂等。
-- 历史冲突行仅保留最新一条(id 最大),再按冲突行重建唯一键。
CREATE TABLE IF NOT EXISTS biz_invalid_asin_data (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键',
data_value VARCHAR(128) NOT NULL COMMENT '不符合 ASIN 值',
brand TEXT NULL COMMENT 'brand name',
group_id BIGINT NULL COMMENT '分组ID',
record_source VARCHAR(32) NOT NULL DEFAULT 'AUTO' COMMENT '记录来源',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
KEY idx_brand_prefix (brand(191)),
KEY idx_created_at (created_at)
) COMMENT='不符合ASIN数据表';
-- 1. 清理 (data_value, brand) 完全重复的行,仅保留 id 最大的一条
DELETE t1 FROM biz_invalid_asin_data t1
JOIN biz_invalid_asin_data t2
ON t1.data_value = t2.data_value
AND ((t1.brand <=> t2.brand) OR (t1.brand IS NULL AND t2.brand IS NULL))
AND t1.id < t2.id;
-- 2. 删除既有单列唯一键 uk_data_value(存在时)
SET @uk_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND INDEX_NAME = 'uk_data_value'
);
SET @sql_drop_uk := IF(
@uk_exists > 0,
'ALTER TABLE biz_invalid_asin_data DROP INDEX uk_data_value',
'SELECT 1'
);
PREPARE stmt_drop_uk FROM @sql_drop_uk;
EXECUTE stmt_drop_uk;
DEALLOCATE PREPARE stmt_drop_uk;
-- 3. 重建为 (data_value, brand) 复合唯一键(不存在时)
SET @uk_dup_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_invalid_asin_data'
AND INDEX_NAME = 'uk_data_value_brand'
);
SET @sql_add_uk := IF(
@uk_dup_exists = 0,
'ALTER TABLE biz_invalid_asin_data ADD UNIQUE KEY uk_data_value_brand (data_value, brand(191))',
'SELECT 1'
);
PREPARE stmt_add_uk FROM @sql_add_uk;
EXECUTE stmt_add_uk;
DEALLOCATE PREPARE stmt_add_uk;