b0829db81e
取证基线(库侧 biz_user_secret_usage_daily,10-01~10-11 实测 12,704 / 4,134 / 15,135 次): 外观 24h 全量 831 条请求逐条 json 解析,564 条带图共投 2,693 张、去重只剩 1,465 张 ⇒ 45.6% 是同一张图在同一次请求里的副本;重试占比只有 0.24%(attempt 829/2/0), 所以省钱的口子在图片副本,不在重试。货源 4,134 次里 4,096 次集中在 10-10/10-11 两天, 类目已合批(10-02)而合规与图比仍逐行 ⇒ 合规是该模块最后一步逐行 LLM 调用。 1) 外观:每张不同的图只随请求发一份,正文附「图j=序号1、2…」对照表;每个序号仍各自 出一份 status/reason(省副本,不省判定,与 10-10「同组兄弟行也要都检测」同口径)。 归一化 URL 只当去重的键,发出去的还是行里那条原始串。应急回退: llm-appearance-batch-size=1 即退回逐行,无需改码。 2) 货源:新键 llm-conform-batch-size=10(设 1 退回逐行)。前置条件是分类预计算存在, 否则整条不生效——避免同一行的类目被算两遍(两遍=多扣用户一次);缺编号行与整块 失败都逐行补检,并有一支 A/B 差分管住合批与逐行写回行的三字段+状态相等。 3) V157 给 biz_user_secret_usage_daily 加 prompt_tokens/completion_tokens,两个 client 收下 网关 usage。计次与 token 分两条语句:计次发生在发请求之前(含重试,请求没回来也算 一次),token 只在响应回来后有读数;网关没回 usage 就一条不写——0 会被读成"实测为 0"。 护栏记账:新 mapper 语句 addTokens 两处一起点名(SqlLayeringConventionTest 分母 97→98 + MigratedSqlEquivalenceTest.NOT_MIGRATED);批量路径的 catch 不新增一处——把分类那一路的 "整块失败→逐行补检"抽成 batchAttempt/fallbackSingleRows 共用,静默 catch 棘轮四格一格没放宽。 顺带修掉 AppearancePatentResultRowDto 上一句已经过期的 javadoc(还写着兄弟行复用代表行结论)。 证据:后端全量 4288 例 0 红(本批 +24 例:去重 7 / token 4 / 合规合批 7 / 用量服务 6)。 变异自证 3/3 全杀:取消去重、摘掉 usage 落库、断掉合规合批,分别让对应用例判红; 两个受试文件按注入前字节还原,sha 复算一致。
425 lines
20 KiB
Java
425 lines
20 KiB
Java
package com.nanri.aiimage;
|
||
|
||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||
import org.apache.ibatis.builder.xml.XMLMapperBuilder;
|
||
import org.apache.ibatis.io.Resources;
|
||
import org.apache.ibatis.mapping.BoundSql;
|
||
import org.apache.ibatis.mapping.MappedStatement;
|
||
import org.apache.ibatis.mapping.SqlSource;
|
||
import org.junit.jupiter.api.Test;
|
||
|
||
import java.io.IOException;
|
||
import java.io.InputStream;
|
||
import java.lang.reflect.GenericArrayType;
|
||
import java.lang.reflect.Method;
|
||
import java.lang.reflect.ParameterizedType;
|
||
import java.lang.reflect.Type;
|
||
import java.nio.charset.StandardCharsets;
|
||
import java.nio.file.Files;
|
||
import java.nio.file.Path;
|
||
import java.util.ArrayList;
|
||
import java.util.HashMap;
|
||
import java.util.LinkedHashMap;
|
||
import java.util.List;
|
||
import java.util.Map;
|
||
import java.util.Set;
|
||
import java.util.regex.Matcher;
|
||
import java.util.regex.Pattern;
|
||
import java.util.stream.Collectors;
|
||
import java.util.stream.Stream;
|
||
|
||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||
import static org.junit.jupiter.api.Assertions.fail;
|
||
|
||
/**
|
||
* 注解 SQL 平移到 mapper XML 的等价性守卫。
|
||
*
|
||
* <p>基准是<b>搬迁前注解的原文</b>({@code src/test/resources/sql-layering/migrated-statements.tsv},
|
||
* 85 条),而且比对发生在 <b>MyBatis 解析之后</b>:把注解原文现场交给
|
||
* {@code LanguageDriver.createSqlSource}(注解当年就是走这条路进 MyBatis 的),
|
||
* 与 XML 解析出来的 {@code MappedStatement} 各渲染一遍 {@code BoundSql} 再对撞。
|
||
* 这样"元素体该怎么转义"由 MyBatis 判定,不由生成脚本判定——
|
||
* 上一轮正是拿生成器的输出当基准,同一个解析器错两遍,85 条"逐字相等"全绿而 SQL 已经残缺。
|
||
*
|
||
* <p>渲染用两组参数各对撞一次:空参数(所有 {@code <if>} 走 false 支)与把
|
||
* {@code <foreach collection>} 喂满 2 个元素(动态支展开成 {@code ?,?})。
|
||
* 两边同抛异常也算一致,但必须是同一异常类型与同一消息,不许一边过一边炸。
|
||
*
|
||
* <p>第二支管 {@code resultType}:期望类型由反射从接口签名反推,不由生成脚本算,
|
||
* 避免"生成时解错、断言时照着错的对"。
|
||
*/
|
||
class MigratedSqlEquivalenceTest {
|
||
|
||
private static final String FIXTURE = "sql-layering/migrated-statements.tsv";
|
||
private static final int LEDGER_ROWS = 85;
|
||
private static final int LEDGER_SELECTS = 60;
|
||
/** 全部 mapper XML 里的 select 总数:账本 60 + TaskImageCache 5 + AsinResultCache 1。 */
|
||
/** 66 → 67:会话容忍新增的 {@code AdminUserMapper#countLiveWorkFor} 也是一条 select。 */
|
||
private static final int XML_SELECT_TOTAL = 67;
|
||
|
||
/**
|
||
* 不是「从注解搬过来」的语句:注解时代没有原文可对撞,等价性这支只能跳过它们。
|
||
* 登记在此而不是让它们红,是为了让「谁新写了语句」这件事留痕;新增一条不记名的,照样红。
|
||
*/
|
||
private static final java.util.Set<String> NOT_MIGRATED =
|
||
java.util.Set.of("AdminUserMapper#countLiveWorkFor",
|
||
// V157 加 token 两列时新写的累加语句;注解时代没有原文可对撞,等价性这支只能跳过它。
|
||
// 与 increment 分两条是口径要求(计次先于响应、token 只在响应后有读数),不是随手加的第二条 SQL。
|
||
"UserSecretUsageMapper#addTokens");
|
||
|
||
/** 账本 5 列:接口 / 方法 / @Select 种类 / 是否含常量插值 / 注解原文(Java 反转义后那条串)。 */
|
||
private static final int LEDGER_COLUMNS = 5;
|
||
private static final int BASELINE_COL = 4;
|
||
|
||
private static final Pattern STATEMENT = Pattern.compile(
|
||
"(?is)<(select|insert|update|delete)\\s+id=\"([A-Za-z0-9_]+)\"([^>]*)>(.*?)</\\1>");
|
||
private static final Pattern RESULT_TYPE = Pattern.compile("resultType=\"([^\"]+)\"");
|
||
private static final Pattern FOREACH_COLLECTION = Pattern.compile("collection=[\"'](\\w+)[\"']");
|
||
private static final Pattern TEST_NAME = Pattern.compile("test=[\"']([^\"']*)[\"']");
|
||
private static final Pattern HASH_PARAM = Pattern.compile("#\\{\\s*([A-Za-z_]\\w*)");
|
||
/** OGNL 里的字面量与方法名不是参数名,喂值进去会让两边同时以错误告终。 */
|
||
private static final Set<String> OGNL_WORDS =
|
||
Set.of("null", "true", "false", "and", "or", "not", "size", "isEmpty", "equals", "length");
|
||
|
||
@Test
|
||
void ledgerItselfIsTheExpectedShape() throws IOException {
|
||
List<String[]> rows = readLedger();
|
||
assertEquals(LEDGER_ROWS, rows.size(), "fixture 行数变了——它是搬迁前的快照,不该跟着代码漂");
|
||
assertEquals(LEDGER_SELECTS, rows.stream().filter(r -> "Select".equals(r[2])).count(),
|
||
"fixture 里的 Select 条数与基线不符(读侧账本 = 60)");
|
||
assertEquals(LEDGER_ROWS - LEDGER_SELECTS,
|
||
rows.stream().filter(r -> !"Select".equals(r[2])).count(),
|
||
"写侧条数对不上(25 = Insert+Update+Delete)");
|
||
assertTrue(rows.stream().allMatch(r -> !r[1].isEmpty() && !r[BASELINE_COL].isEmpty()),
|
||
"fixture 有空方法名或空 SQL,说明抽取器漏了形状");
|
||
// 基准得是「整条注解串」:半截 <script> 外壳说明抽取被截断(上一轮就是把常量与标签吃了)
|
||
assertTrue(rows.stream().allMatch(r ->
|
||
r[BASELINE_COL].startsWith("<script>") == r[BASELINE_COL].contains("</script>")),
|
||
"有基准行的 <script> 外壳不成对——抽取时注解表达式被截断了");
|
||
assertTrue(rows.stream().allMatch(r ->
|
||
r[BASELINE_COL].contains("<if") == r[BASELINE_COL].contains("</if>")),
|
||
"有基准行的 <if> 不成对——常量插值没展开");
|
||
assertTrue(rows.stream().noneMatch(r -> r[BASELINE_COL].contains("IN ()")),
|
||
"有基准行出现空的 IN (),说明 foreach/常量被吞了");
|
||
}
|
||
|
||
@Test
|
||
void everyMigratedStatementRendersTheSameSqlAsTheAnnotationDid() throws Exception {
|
||
Map<String, String> baselines = new LinkedHashMap<>();
|
||
Map<String, String> kinds = new LinkedHashMap<>();
|
||
for (String[] row : readLedger()) {
|
||
baselines.put(row[0] + "#" + row[1], unescapeJava(row[4]));
|
||
kinds.put(row[0] + "#" + row[1], row[2].toLowerCase());
|
||
}
|
||
List<String> problems = new ArrayList<>();
|
||
int checked = 0;
|
||
for (Path xml : mapperXmls()) {
|
||
String iface = stem(xml);
|
||
if (!baselines.keySet().stream().anyMatch(k -> k.startsWith(iface + "#"))) {
|
||
continue; // TaskImageCache / AsinResultCache 由各自用例钉,不在这本账里
|
||
}
|
||
Class<?> mapperClass = Class.forName(namespaceOf(xml));
|
||
MybatisConfiguration configuration = parseInto(xml, new MybatisConfiguration());
|
||
Matcher m = STATEMENT.matcher(read(xml));
|
||
while (m.find()) {
|
||
String key = iface + "#" + m.group(2);
|
||
String baseline = baselines.get(key);
|
||
if (baseline == null) {
|
||
if (NOT_MIGRATED.contains(key)) {
|
||
continue;
|
||
}
|
||
problems.add(key + " 在基准账本里没有对应注解(XML 多出一条语句)");
|
||
continue;
|
||
}
|
||
checked++;
|
||
if (!kinds.get(key).equals(m.group(1))) {
|
||
problems.add(key + " 注解是 @" + kinds.get(key) + ",XML 用了 <" + m.group(1) + ">");
|
||
continue;
|
||
}
|
||
MappedStatement fromXml =
|
||
configuration.getMappedStatement(mapperClass.getName() + "." + m.group(2));
|
||
// 注解当年就是走这条驱动进 MyBatis 的(MapperAnnotationBuilder 用的正是默认脚本语言),
|
||
// 所以这里拿它现场编译基准串,等价性由 MyBatis 判定而不是由生成脚本判定。
|
||
SqlSource fromAnnotation = configuration.getDefaultScriptingLanguageInstance()
|
||
.createSqlSource(configuration, baseline, Object.class);
|
||
for (Map<String, Object> params : renderParams(baseline)) {
|
||
String where = key + "(" + params.get("_case") + ")";
|
||
compare(fromXml.getBoundSql(params), fromAnnotation.getBoundSql(params), where, problems);
|
||
}
|
||
}
|
||
}
|
||
int checkedRows = checked;
|
||
assertEquals(LEDGER_ROWS, checkedRows,
|
||
() -> "只比对了 " + checkedRows + " 条,其余没在 XML 里落地(应为 " + LEDGER_ROWS + " 条)");
|
||
assertTrue(problems.isEmpty(),
|
||
() -> "这些语句搬完之后 MyBatis 渲染出的 SQL 与注解时不同(" + problems.size() + " 处):"
|
||
+ System.lineSeparator() + String.join(System.lineSeparator(), problems));
|
||
}
|
||
|
||
/** 两边各渲染一次:都出 SQL 就逐字比,都抛就比异常,一边过一边炸必须算问题。 */
|
||
private static void compare(BoundSql a, BoundSql b, String where, List<String> problems) {
|
||
String sqlA, sqlB;
|
||
try {
|
||
sqlA = normalize(a.getSql());
|
||
} catch (RuntimeException e) {
|
||
problems.add(where + " XML 侧渲染就抛:" + e);
|
||
return;
|
||
}
|
||
try {
|
||
sqlB = normalize(b.getSql());
|
||
} catch (RuntimeException e) {
|
||
problems.add(where + " 注解侧渲染抛 " + e.getClass().getSimpleName()
|
||
+ " 而 XML 侧正常出 SQL: " + sqlA);
|
||
return;
|
||
}
|
||
if (!sqlA.equals(sqlB)) {
|
||
problems.add(where + System.lineSeparator()
|
||
+ " 注解渲染: " + sqlB + System.lineSeparator()
|
||
+ " XML 渲染 : " + sqlA);
|
||
return;
|
||
}
|
||
if (a.getParameterMappings().size() != b.getParameterMappings().size()) {
|
||
problems.add(where + " 占位符数不同:注解 " + b.getParameterMappings().size()
|
||
+ " vs XML " + a.getParameterMappings().size() + " —— " + sqlA);
|
||
}
|
||
}
|
||
|
||
/** 空参数走所有 <if> 的 false 支;再喂满 foreach 走动态支。两支都要两边一致。 */
|
||
private static List<Map<String, Object>> renderParams(String baseline) {
|
||
Map<String, Object> full = new HashMap<>();
|
||
Set<String> collections = new java.util.LinkedHashSet<>();
|
||
Matcher fc = FOREACH_COLLECTION.matcher(baseline);
|
||
while (fc.find()) {
|
||
collections.add(fc.group(1));
|
||
}
|
||
Set<String> names = new java.util.LinkedHashSet<>(collections);
|
||
Matcher t = TEST_NAME.matcher(baseline);
|
||
while (t.find()) {
|
||
Matcher word = Pattern.compile("\\w+").matcher(t.group(1));
|
||
while (word.find()) {
|
||
String w = word.group();
|
||
if (!OGNL_WORDS.contains(w)) {
|
||
names.add(w);
|
||
}
|
||
}
|
||
}
|
||
Matcher h = HASH_PARAM.matcher(baseline);
|
||
while (h.find()) {
|
||
names.add(h.group(1));
|
||
}
|
||
for (String name : names) {
|
||
full.put(name, collections.contains(name) ? List.of("a", "b") : "v");
|
||
}
|
||
return List.of(Map.of("_case", "空参数"), Map.of("_case", "带参数"))
|
||
.stream().map(marker -> {
|
||
Map<String, Object> params =
|
||
new HashMap<>("_case".equals(marker.get("_case")) ? Map.of() : full);
|
||
params.putAll(marker);
|
||
return params;
|
||
}).collect(Collectors.toList());
|
||
}
|
||
|
||
/** resultType 必须与接口方法签名一致;期望值由反射算,不用生成脚本的结论。 */
|
||
@Test
|
||
void everySelectKeepsTheReturnTypeItsAnnotationHad() throws Exception {
|
||
List<String> problems = new ArrayList<>();
|
||
int checked = 0;
|
||
for (Path xml : mapperXmls()) {
|
||
String text = read(xml);
|
||
Matcher ns = Pattern.compile("namespace=\"([^\"]+)\"").matcher(text);
|
||
if (!ns.find()) {
|
||
problems.add(xml + " 缺 namespace");
|
||
continue;
|
||
}
|
||
Class<?> iface = Class.forName(ns.group(1));
|
||
MybatisConfiguration configuration = parseInto(xml, new MybatisConfiguration());
|
||
Matcher m = STATEMENT.matcher(text);
|
||
while (m.find()) {
|
||
if (!"select".equals(m.group(1))) {
|
||
continue;
|
||
}
|
||
String method = m.group(2);
|
||
checked++;
|
||
MappedStatement statement =
|
||
configuration.getMappedStatement(iface.getName() + "." + method);
|
||
Class<?> resolved = statement.getResultMaps().get(0).getType();
|
||
Matcher attrs = RESULT_TYPE.matcher(m.group(3));
|
||
String declared = attrs.find() ? attrs.group(1) : null;
|
||
if (declared == null) {
|
||
problems.add(iface.getSimpleName() + "#" + method + " 的 <select> 没写 resultType");
|
||
continue;
|
||
}
|
||
Class<?> expected = expectedType(iface, method);
|
||
if (expected == null) {
|
||
problems.add(iface.getSimpleName() + "#" + method + " 找不到接口方法签名");
|
||
} else if (!compatible(expected, resolved)) {
|
||
problems.add(iface.getSimpleName() + "#" + method
|
||
+ " resultType=" + declared + " 解析成 " + resolved.getName()
|
||
+ ",而方法签名要的是 " + expected.getName());
|
||
}
|
||
}
|
||
}
|
||
int checkedSelects = checked;
|
||
assertTrue(problems.isEmpty(), () -> "resultType 与接口签名不一致(" + problems.size() + " 条):"
|
||
+ System.lineSeparator() + String.join(System.lineSeparator(), problems));
|
||
// 这支扫的是全部 mapper XML,所以分母比账本大:本轮 60 + TaskImageCache 5 + AsinResultCache 1
|
||
assertEquals(XML_SELECT_TOTAL, checkedSelects,
|
||
() -> "只检查了 " + checkedSelects + " 条 select,应为 " + XML_SELECT_TOTAL
|
||
+ " 条(正则或 XML 漂了)");
|
||
}
|
||
|
||
/** 只把 XML 解析进配置:XMLMapperBuilder 会顺带按 namespace 绑上接口。 */
|
||
private static MybatisConfiguration parseInto(Path xml, MybatisConfiguration configuration)
|
||
throws IOException {
|
||
try (InputStream in = Files.newInputStream(xml)) {
|
||
new XMLMapperBuilder(in, configuration, xml.toString(),
|
||
configuration.getSqlFragments()).parse();
|
||
}
|
||
return configuration;
|
||
}
|
||
|
||
private static Class<?> expectedType(Class<?> iface, String method) {
|
||
for (Method m : iface.getDeclaredMethods()) {
|
||
if (!m.getName().equals(method)) {
|
||
continue;
|
||
}
|
||
return unwrap(m.getGenericReturnType());
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/** List<X> / Set<X> / Collection<X> 取 X;byte[] 原样;泛型数组元素再解一层。 */
|
||
private static Class<?> unwrap(Type type) {
|
||
if (type instanceof ParameterizedType pt) {
|
||
Type raw = pt.getRawType();
|
||
if (raw instanceof Class<?> c && (c == List.class || c == java.util.Set.class
|
||
|| c == java.util.Collection.class || c == Iterable.class)) {
|
||
return unwrap(pt.getActualTypeArguments()[0]);
|
||
}
|
||
return (Class<?>) raw;
|
||
}
|
||
if (type instanceof GenericArrayType ga) {
|
||
Class<?> component = unwrap(ga.getGenericComponentType());
|
||
if (component == byte.class) {
|
||
return byte[].class;
|
||
}
|
||
return java.lang.reflect.Array.newInstance(component, 0).getClass();
|
||
}
|
||
if (type instanceof Class<?> c) {
|
||
if (c == long.class) {
|
||
return Long.class;
|
||
}
|
||
if (c == int.class) {
|
||
return Integer.class;
|
||
}
|
||
if (c == boolean.class) {
|
||
return Boolean.class;
|
||
}
|
||
if (c == double.class) {
|
||
return Double.class;
|
||
}
|
||
return c;
|
||
}
|
||
return Object.class;
|
||
}
|
||
|
||
private static boolean compatible(Class<?> expected, Class<?> resolved) {
|
||
if (expected == resolved) {
|
||
return true;
|
||
}
|
||
if (Map.class.isAssignableFrom(expected)) {
|
||
return Map.class.isAssignableFrom(resolved);
|
||
}
|
||
return resolved != null && expected.isAssignableFrom(resolved);
|
||
}
|
||
|
||
private static List<String[]> readLedger() throws IOException {
|
||
try (InputStream in = Resources.getResourceAsStream(FIXTURE)) {
|
||
String text = new String(in.readAllBytes(), StandardCharsets.UTF_8);
|
||
List<String[]> rows = new ArrayList<>();
|
||
for (String line : text.split("\n")) {
|
||
if (line.isBlank()) {
|
||
continue;
|
||
}
|
||
String[] cols = line.split("\t", -1);
|
||
if (cols.length != 5) {
|
||
throw new IllegalStateException("fixture 列数不对: " + line.substring(0,
|
||
Math.min(80, line.length())));
|
||
}
|
||
rows.add(cols);
|
||
}
|
||
return rows;
|
||
}
|
||
}
|
||
|
||
private static List<Path> mapperXmls() throws IOException {
|
||
Path dir = moduleRoot().resolve("src/main/resources/mapper");
|
||
if (!Files.isDirectory(dir)) {
|
||
fail("找不到 " + dir + " —— 本用例依赖以 backend-java 为工作目录运行");
|
||
}
|
||
try (Stream<Path> walk = Files.walk(dir)) {
|
||
return walk.filter(p -> p.getFileName().toString().endsWith(".xml"))
|
||
.sorted().collect(Collectors.toList());
|
||
}
|
||
}
|
||
|
||
private static String namespaceOf(Path xml) {
|
||
Matcher m = Pattern.compile("namespace=\"([^\"]+)\"").matcher(read(xml));
|
||
if (!m.find()) {
|
||
throw new IllegalStateException(xml + " 缺 namespace");
|
||
}
|
||
return m.group(1);
|
||
}
|
||
|
||
/** tsv 里的基准行是 Java 转义过的(一行一条),还原成注解当年真正的那个字符串。 */
|
||
private static String unescapeJava(String s) {
|
||
StringBuilder out = new StringBuilder(s.length());
|
||
for (int i = 0; i < s.length(); i++) {
|
||
char c = s.charAt(i);
|
||
if (c == '\\' && i + 1 < s.length()) {
|
||
char n = s.charAt(++i);
|
||
switch (n) {
|
||
case 'n' -> out.append('\n');
|
||
case 't' -> out.append('\t');
|
||
case 'r' -> out.append('\r');
|
||
case '\\' -> out.append('\\');
|
||
default -> out.append(n);
|
||
}
|
||
continue;
|
||
}
|
||
out.append(c);
|
||
}
|
||
return out.toString();
|
||
}
|
||
|
||
private static String normalize(String text) {
|
||
return text.replaceAll("\\s+", " ").trim();
|
||
}
|
||
|
||
private static String read(Path path) {
|
||
try {
|
||
return Files.readString(path, StandardCharsets.UTF_8);
|
||
} catch (IOException e) {
|
||
throw new IllegalStateException("读取失败: " + path, e);
|
||
}
|
||
}
|
||
|
||
private static String stem(Path path) {
|
||
String name = path.getFileName().toString();
|
||
return name.substring(0, name.length() - 4);
|
||
}
|
||
|
||
private static Path moduleRoot() {
|
||
Path here = Path.of("").toAbsolutePath();
|
||
if (Files.isDirectory(here.resolve("src/main/java"))) {
|
||
return here;
|
||
}
|
||
Path nested = here.resolve("backend-java");
|
||
if (Files.isDirectory(nested.resolve("src/main/java"))) {
|
||
return nested;
|
||
}
|
||
throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")");
|
||
}
|
||
}
|