From 00f45cc20f7f64e2d0249a0277a7d3a0c8690a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 4 Nov 2026 02:56:50 +0800 Subject: [PATCH] =?UTF-8?q?feat(phase-B75):=20=E8=AE=A1=E6=AC=A1=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E5=8A=A0=20source=20=E7=BB=B4=E5=BA=A6=EF=BC=88V188?= =?UTF-8?q?=EF=BC=89=E2=80=94=E2=80=94=E3=80=8C=E8=B0=81=E5=9C=A8=E7=83=A7?= =?UTF-8?q?=E3=80=8D=E4=B8=80=E5=88=97=E8=AF=BB=E6=95=B0=EF=BC=8C=E5=BD=92?= =?UTF-8?q?=E5=9B=A0=E4=B8=8D=E5=86=8D=E6=89=AB=E8=AE=BE=E5=A4=87=E6=97=A5?= =?UTF-8?q?=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - V188:biz_user_secret_usage_daily 加 source varchar(24) NOT NULL DEFAULT 'legacy'(存量行诚实落 legacy 不回填),唯一键 uk_user_module_date 扩成四元组(user/module/date/source),加列与建索引均 information_schema 判存可重跑(列按 V157 款、索引按 V184 款),八段式含验证 SQL 与人工回滚 --- .../InternalUserSecretUsageController.java | 6 +- .../mapper/UserSecretUsageMapper.java | 17 ++- .../dto/UserSecretUsageReportRequest.java | 4 + .../model/entity/UserSecretUsageEntity.java | 4 +- .../service/UserSecretUsageService.java | 43 +++++-- .../db/V188__user_secret_usage_source.sql | 83 +++++++++++++ .../usersecret/UserSecretUsageMapper.xml | 9 +- .../aiimage/MigratedSqlEquivalenceTest.java | 9 +- ...serSecretUsageSourceMigrationV188Test.java | 117 ++++++++++++++++++ .../service/UserSecretUsageServiceTest.java | 88 ++++++++++--- .../sql-layering/migrated-statements.tsv | 1 - 11 files changed, 340 insertions(+), 41 deletions(-) create mode 100644 backend-java/src/main/resources/db/V188__user_secret_usage_source.sql create mode 100644 backend-java/src/test/java/com/nanri/aiimage/config/UserSecretUsageSourceMigrationV188Test.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/InternalUserSecretUsageController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/InternalUserSecretUsageController.java index 0bdafd77e..08f52aeba 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/InternalUserSecretUsageController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/controller/InternalUserSecretUsageController.java @@ -46,9 +46,9 @@ public class InternalUserSecretUsageController { return ApiResponse.fail("上报参数不完整"); } boolean accepted = userSecretUsageService.recordReported( - body.getUserId(), body.getModule(), body.getCount()); - log.info("[internal-usage] 计次上报 userId={} module={} count={} accepted={}", - body.getUserId(), body.getModule(), body.getCount(), accepted); + body.getUserId(), body.getModule(), body.getCount(), body.getSource()); + log.info("[internal-usage] 计次上报 userId={} module={} count={} source={} accepted={}", + body.getUserId(), body.getModule(), body.getCount(), body.getSource(), accepted); if (!accepted) { return ApiResponse.fail("模块或参数非法,未记录"); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java index 04bda69eb..492e9ec67 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/mapper/UserSecretUsageMapper.java @@ -13,27 +13,34 @@ import java.util.List; public interface UserSecretUsageMapper extends BaseMapper { /** - * 按唯一键 uk_user_module_date 原子累加计次:命中则 call_count 自增,否则插入。 - * 并发下无需应用层加锁(引擎层唯一键冲突转更新)。 + * 按唯一键 uk_user_module_date_source(V188:user_id+module_key+business_date+source) + * 原子累加计次:命中则 call_count 自增,否则插入。并发下无需应用层加锁 + * (引擎层唯一键冲突转更新)。source=上报方自报身份(client/brand16890/brand15126/ + * pt_worker),Java 自产行=backend,缺省/非法=legacy——同一天多路各占一行, + * 读侧按模块聚合对多出的行透明。 */ int increment(@Param("userId") Long userId, @Param("moduleKey") String moduleKey, @Param("businessDate") LocalDate businessDate, - @Param("count") int count); + @Param("count") int count, + @Param("source") String source); /** - * 按唯一键 uk_user_module_date 累加 token 读数。 + * 按唯一键 uk_user_module_date_source 累加 token 读数。 * *

刻意与 {@link #increment} 分成两条:计次发生在**发请求之前**(口径含重试,请求没回来也要算一次), * 而 token 只有响应回来才知道。合并成一条 upsert 就会把"已发出但超时"的那些请求从计次里漏掉。 *

这里是 UPDATE 而不是 upsert:行由前一条 increment 保证存在;真不存在就是 0 行受影响, * token 读数丢掉也不该反过来影响计次。 + *

WHERE 必须带 source(V188):唯一键扩成四元组后,同一 (user, module, date) 可能 + * 有 legacy 与 backend 多行,不带 source 会把 token 记到历史行上造成双计。 */ int addTokens(@Param("userId") Long userId, @Param("moduleKey") String moduleKey, @Param("businessDate") LocalDate businessDate, @Param("promptTokens") long promptTokens, - @Param("completionTokens") long completionTokens); + @Param("completionTokens") long completionTokens, + @Param("source") String source); /** * 当日用量按用户 × 模块聚合(损耗扫描每分钟读一次)。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageReportRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageReportRequest.java index cf0f0b690..1094a8f9e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageReportRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/dto/UserSecretUsageReportRequest.java @@ -16,4 +16,8 @@ public class UserSecretUsageReportRequest { @Schema(description = "本次新增调用次数(真实对外请求)", requiredMode = Schema.RequiredMode.REQUIRED) private Integer count; + + @Schema(description = "上报方来源(V188):client/brand16890/brand15126/pt_worker;" + + "缺省或非法落 legacy 桶。旧上报方不带该字段,兼容。", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + private String source; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserSecretUsageEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserSecretUsageEntity.java index 75d7130ed..d29225d99 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserSecretUsageEntity.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/model/entity/UserSecretUsageEntity.java @@ -9,7 +9,7 @@ import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; -/** 用户密钥调用计次(按天聚合,一行 = 用户 × 模块 × 日期)。 */ +/** 用户密钥调用计次(按天聚合,一行 = 用户 × 模块 × 日期 × 来源(V188))。 */ @Data @TableName("biz_user_secret_usage_daily") public class UserSecretUsageEntity { @@ -18,6 +18,8 @@ public class UserSecretUsageEntity { private Long id; private Long userId; private String moduleKey; + /** 计次来源(V188):client/brand16890/brand15126/pt_worker/backend;存量行=legacy。 */ + private String source; private LocalDate businessDate; private Integer callCount; /** 输入 token 累计(网关 usage.prompt_tokens)。0 = 该天没有 token 证据,不等于没消耗。 */ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserSecretUsageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserSecretUsageService.java index bbcf4f6f3..dadfe7ec1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserSecretUsageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/usersecret/service/UserSecretUsageService.java @@ -49,6 +49,11 @@ public class UserSecretUsageService { public static final long PLATFORM_USER_ID = 0L; static final String PLATFORM_USER_LABEL = "平台自费"; + /** 计次来源(V188):Java 自产行(LLM record/recordTokens)落 backend。 */ + static final String SOURCE_BACKEND = "backend"; + /** 存量行与缺省/非法报文的落桶:历史行无法追溯是哪一路报的,不猜测。 */ + public static final String SOURCE_LEGACY = "legacy"; + private final UserSecretUsageMapper usageMapper; private final AdminUserMapper adminUserMapper; private final AdminAuthSupport adminAuthSupport; @@ -68,11 +73,16 @@ public class UserSecretUsageService { log.warn("[user-secret-usage] 忽略未知模块计次 userId={} module={}", userId, moduleKey); return; } - recordModule(userId, module, count, 0L, 0L); + recordModule(userId, module, count, 0L, 0L, SOURCE_BACKEND); } - /** 内部通道上报入口:模块非法返回 false(供接口回执),写库失败只告警。 */ + /** 内部通道上报入口(3 参旧形状):source 缺省落 legacy 桶。模块非法返回 false。 */ public boolean recordReported(Long userId, String moduleKey, int count) { + return recordReported(userId, moduleKey, count, null); + } + + /** 内部通道上报入口(V188 带 source):模块非法返回 false,写库失败只告警。 */ + public boolean recordReported(Long userId, String moduleKey, int count, String source) { if (userId == null || userId < 0 || count <= 0) { return false; } @@ -85,7 +95,8 @@ public class UserSecretUsageService { log.warn("[user-secret-usage] 平台自费桶只收 proxy 计次 userId=0 module={}", moduleKey); return false; } - return recordModule(userId, module, Math.min(count, MAX_REPORT_COUNT), 0L, 0L); + return recordModule(userId, module, Math.min(count, MAX_REPORT_COUNT), 0L, 0L, + normalizeSource(source)); } /** @@ -106,18 +117,36 @@ public class UserSecretUsageService { if (module == null) { return; } - recordModule(userId, module, 0, promptTokens, completionTokens); + recordModule(userId, module, 0, promptTokens, completionTokens, SOURCE_BACKEND); + } + + /** + * source 规整(V188):小写、^[a-z0-9][a-z0-9_-]{0,23}$ 以外的一律落 legacy—— + * 上报方是可信内部调用方,但字符串仍可能写错;落 legacy 而不是拒收, + * 计次比归因更贵(宁可归因模糊也不丢账)。 + */ + static String normalizeSource(String raw) { + if (raw == null) { + return SOURCE_LEGACY; + } + String trimmed = raw.trim().toLowerCase(); + if (trimmed.isEmpty() || trimmed.length() > 24 + || !trimmed.matches("[a-z0-9][a-z0-9_-]*")) { + return SOURCE_LEGACY; + } + return trimmed; } private boolean recordModule(Long userId, UserSecretModule module, int count, - long promptTokens, long completionTokens) { + long promptTokens, long completionTokens, String source) { LocalDate today = LocalDate.now(); try { if (count > 0) { - usageMapper.increment(userId, module.key(), today, count); + usageMapper.increment(userId, module.key(), today, count, source); } if (promptTokens > 0 || completionTokens > 0) { - usageMapper.addTokens(userId, module.key(), today, promptTokens, completionTokens); + usageMapper.addTokens(userId, module.key(), today, promptTokens, completionTokens, + source); } return true; } catch (Exception ex) { diff --git a/backend-java/src/main/resources/db/V188__user_secret_usage_source.sql b/backend-java/src/main/resources/db/V188__user_secret_usage_source.sql new file mode 100644 index 000000000..748a1d1bf --- /dev/null +++ b/backend-java/src/main/resources/db/V188__user_secret_usage_source.sql @@ -0,0 +1,83 @@ +-- V188: 给 `biz_user_secret_usage_daily` 加 `source` 列,并把唯一键扩到四元组 +-- ============================================================= +-- 1. 变更目的:把「谁在烧」从猜测变成一列读数。四路上报方(客户端 90s flush/16890 +-- 30s 批量/15126 60s 周期/pt_worker 逐次)此前都只报 {userId, module, count}, +-- 同一用户同一天被多个面消费时在表里揉成一行,归因只能去扫 MinIO 设备日志与两台 +-- 机器(2026-11-04 客户反馈「代理IP用的很猛」的归因实测就是这么做的)。加 source +-- 后一条 `GROUP BY source` 直接出账。 +-- +-- 2. 前提取证(2026-11-04 只读探针):该表 7 天 proxy 19,040 次/95 用户,行数 +-- =用户×模块×天 量级,加列存储代价可忽略;现键 `uk_user_module_date(user_id, +-- module_key, business_date)`(V117)是 increment 的 ON DUPLICATE KEY 幂等键。 +-- +-- 3. 形状与数据:`source varchar(24) NOT NULL DEFAULT 'legacy'`——存量行统一落 +-- 'legacy'(历史行无法追溯是哪一路报的,不猜测);新行由四路上报方自报 +-- (client/brand16890/brand15126/pt_worker),Java 自产 LLM 行=backend,缺省/非法 +-- 报文=legacy。唯一键扩成 `(user_id, module_key, business_date, source)`: +-- 同一天多路各占一行,读侧 adminPage/SUM 按模块聚合,对多出的行天然透明。 +-- +-- 4. 锁表风险:极低。日聚合表、万行量级,MySQL 8 ADD COLUMN NOT NULL DEFAULT +-- 走 INSTANT;索引重建在当前行数下毫秒级。 +-- +-- 5. 验证 SQL(迁移后跑,不许只看启动日志的 Successfully applied): +-- SHOW COLUMNS FROM `biz_user_secret_usage_daily` LIKE 'source'; +-- -> 期望 1 行:source,Type=varchar(24),Null=NO,Default=legacy +-- SHOW INDEX FROM `biz_user_secret_usage_daily` WHERE Key_name='uk_user_module_date_source'; +-- -> 期望 4 行(user_id/module_key/business_date/source),Non_unique=0 +-- 应用侧:UserSecretUsageServiceTest 全绿(increment/addTokens 带来源、 +-- 缺省/非法报文落 legacy、Java 自产 LLM 行=backend)。 +-- +-- 6. 上线窗口:必须与读写的 jar 同版同窗——旧 jar 的 increment 不带 source, +-- INSERT 会落进 DEFAULT 'legacy'(幂等键还是旧三元组形状时列不参与键,行为不变); +-- 新 jar 写四元组前 Flyway 已保证新键存在。只发 jar 不发迁移会在第一次 upsert +-- 时撞 1054 Unknown column,只发迁移不发 jar 则所有行落 legacy(无损失但无归因)。 +-- +-- 7. 可重跑性:加列与建索引都先查 information_schema(列按 V157 款,索引按 V184 款)—— +-- 迁移中途失败时 flyway repair + 重启会从第一条语句重跑整个文件,裸 ALTER 会撞 +-- 1060/1061。 +-- +-- 8. 回滚(人工补丁,写成注释以免落在 Flyway 扫描路径里被当迁移执行): +-- 触发场景是"这列不该加"。顺序:停两台应用 → 跑下面三条 → +-- 删 flyway_schema_history 的 version='188' 行 → 回滚 jar → 起应用。 +-- ALTER TABLE `biz_user_secret_usage_daily` DROP INDEX `uk_user_module_date_source`; +-- ALTER TABLE `biz_user_secret_usage_daily` ADD UNIQUE KEY `uk_user_module_date` +-- (`user_id`, `module_key`, `business_date`); +-- ALTER TABLE `biz_user_secret_usage_daily` DROP COLUMN `source`; +-- 注意回滚前须先把四元组里的重行并回(同键多行会让 ADD UNIQUE 失败): +-- (按 user_id+module_key+business_date 求 SUM 后重插——正常不会发生, +-- 除非回滚前已有多来源同行;真发生时按该日逐行手工核对再并。) +-- 仅回滚 jar(代码问题、与列无关)不需要 DROP:旧 jar 不写 source,行形状保持 +-- DEFAULT 'legacy',读侧聚合不变。 +-- ============================================================= + +SET @col := ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_user_secret_usage_daily' + AND COLUMN_NAME = 'source' +); +SET @ddl := IF(@col = 0, + 'ALTER TABLE `biz_user_secret_usage_daily` ADD COLUMN `source` varchar(24) NOT NULL DEFAULT ''legacy'' COMMENT ''计次来源(client/brand16890/brand15126/pt_worker/backend;存量行=legacy)'' AFTER `module_key`', + 'SELECT ''biz_user_secret_usage_daily.source already exists, skipped'' AS note'); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @old_key := ( + SELECT COUNT(DISTINCT INDEX_NAME) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_user_secret_usage_daily' + AND INDEX_NAME = 'uk_user_module_date' +); +SET @new_key := ( + SELECT COUNT(DISTINCT INDEX_NAME) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_user_secret_usage_daily' + AND INDEX_NAME = 'uk_user_module_date_source' +); +SET @ddl2 := IF(@new_key = 0 AND @old_key > 0, + 'ALTER TABLE `biz_user_secret_usage_daily` DROP INDEX `uk_user_module_date`, ADD UNIQUE KEY `uk_user_module_date_source` (`user_id`, `module_key`, `business_date`, `source`)', + 'SELECT ''uk_user_module_date_source already exists (or old key missing), skipped'' AS note'); +PREPARE stmt2 FROM @ddl2; +EXECUTE stmt2; +DEALLOCATE PREPARE stmt2; diff --git a/backend-java/src/main/resources/mapper/usersecret/UserSecretUsageMapper.xml b/backend-java/src/main/resources/mapper/usersecret/UserSecretUsageMapper.xml index 1014f2209..1ad37a569 100644 --- a/backend-java/src/main/resources/mapper/usersecret/UserSecretUsageMapper.xml +++ b/backend-java/src/main/resources/mapper/usersecret/UserSecretUsageMapper.xml @@ -9,18 +9,21 @@ LIMIT #{batchSize} - INSERT INTO biz_user_secret_usage_daily (user_id, module_key, business_date, call_count, created_at, updated_at) - VALUES (#{userId}, #{moduleKey}, #{businessDate}, #{count}, NOW(), NOW()) + INSERT INTO biz_user_secret_usage_daily (user_id, module_key, business_date, source, call_count, created_at, updated_at) + VALUES (#{userId}, #{moduleKey}, #{businessDate}, #{source}, #{count}, NOW(), NOW()) ON DUPLICATE KEY UPDATE call_count = call_count + #{count}, updated_at = NOW() + 与上面的 increment 分两条是口径要求:计次先于响应,token 只在响应回来后有读数。 + WHERE 带 source(V188):唯一键四元组后同一 (user,module,date) 可能有多行, + 不钉 source 会把 token 记到 legacy 历史行上造成双计。 --> UPDATE biz_user_secret_usage_daily SET prompt_tokens = prompt_tokens + #{promptTokens}, completion_tokens = completion_tokens + #{completionTokens}, updated_at = NOW() WHERE user_id = #{userId} AND module_key = #{moduleKey} AND business_date = #{businessDate} + AND source = #{source}