Files
crawler-plugin/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java
T
huangzd1997 b0829db81e 降本三条(10-11 现网取证):外观同请求图片去重 + 货源合规合批 + LLM token 计量落库
取证基线(库侧 biz_user_secret_usage_daily,10-01~10-11 实测 12,704 / 4,134 / 15,135 次):
外观 24h 全量 831 条请求逐条 json 解析,564 条带图共投 2,693 张、去重只剩 1,465 张
⇒ 45.6% 是同一张图在同一次请求里的副本;重试占比只有 0.24%(attempt 829/2/0),
所以省钱的口子在图片副本,不在重试。货源 4,134 次里 4,096 次集中在 10-10/10-11 两天,
类目已合批(10-02)而合规与图比仍逐行 ⇒ 合规是该模块最后一步逐行 LLM 调用。

1) 外观:每张不同的图只随请求发一份,正文附「图j=序号1、2…」对照表;每个序号仍各自
   出一份 status/reason(省副本,不省判定,与 10-10「同组兄弟行也要都检测」同口径)。
   归一化 URL 只当去重的键,发出去的还是行里那条原始串。应急回退:
   llm-appearance-batch-size=1 即退回逐行,无需改码。
2) 货源:新键 llm-conform-batch-size=10(设 1 退回逐行)。前置条件是分类预计算存在,
   否则整条不生效——避免同一行的类目被算两遍(两遍=多扣用户一次);缺编号行与整块
   失败都逐行补检,并有一支 A/B 差分管住合批与逐行写回行的三字段+状态相等。
3) V157 给 biz_user_secret_usage_daily 加 prompt_tokens/completion_tokens,两个 client 收下
   网关 usage。计次与 token 分两条语句:计次发生在发请求之前(含重试,请求没回来也算
   一次),token 只在响应回来后有读数;网关没回 usage 就一条不写——0 会被读成"实测为 0"。

护栏记账:新 mapper 语句 addTokens 两处一起点名(SqlLayeringConventionTest 分母 97→98 +
MigratedSqlEquivalenceTest.NOT_MIGRATED);批量路径的 catch 不新增一处——把分类那一路的
"整块失败→逐行补检"抽成 batchAttempt/fallbackSingleRows 共用,静默 catch 棘轮四格一格没放宽。
顺带修掉 AppearancePatentResultRowDto 上一句已经过期的 javadoc(还写着兄弟行复用代表行结论)。

证据:后端全量 4288 例 0 红(本批 +24 例:去重 7 / token 4 / 合规合批 7 / 用量服务 6)。
变异自证 3/3 全杀:取消去重、摘掉 usage 落库、断掉合规合批,分别让对应用例判红;
两个受试文件按注入前字节还原,sha 复算一致。
2026-10-11 23:52:19 +08:00

425 lines
20 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package com.nanri.aiimage;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import org.apache.ibatis.builder.xml.XMLMapperBuilder;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.mapping.BoundSql;
import org.apache.ibatis.mapping.MappedStatement;
import org.apache.ibatis.mapping.SqlSource;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.GenericArrayType;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
/**
* 注解 SQL 平移到 mapper XML 的等价性守卫。
*
* <p>基准是<b>搬迁前注解的原文</b>({@code src/test/resources/sql-layering/migrated-statements.tsv},
* 85 条),而且比对发生在 <b>MyBatis 解析之后</b>:把注解原文现场交给
* {@code LanguageDriver.createSqlSource}(注解当年就是走这条路进 MyBatis 的),
* 与 XML 解析出来的 {@code MappedStatement} 各渲染一遍 {@code BoundSql} 再对撞。
* 这样"元素体该怎么转义"由 MyBatis 判定,不由生成脚本判定——
* 上一轮正是拿生成器的输出当基准,同一个解析器错两遍,85 条"逐字相等"全绿而 SQL 已经残缺。
*
* <p>渲染用两组参数各对撞一次:空参数(所有 {@code <if>} 走 false 支)与把
* {@code <foreach collection>} 喂满 2 个元素(动态支展开成 {@code ?,?})。
* 两边同抛异常也算一致,但必须是同一异常类型与同一消息,不许一边过一边炸。
*
* <p>第二支管 {@code resultType}:期望类型由反射从接口签名反推,不由生成脚本算,
* 避免"生成时解错、断言时照着错的对"。
*/
class MigratedSqlEquivalenceTest {
private static final String FIXTURE = "sql-layering/migrated-statements.tsv";
private static final int LEDGER_ROWS = 85;
private static final int LEDGER_SELECTS = 60;
/** 全部 mapper XML 里的 select 总数:账本 60 + TaskImageCache 5 + AsinResultCache 1。 */
/** 66 → 67:会话容忍新增的 {@code AdminUserMapper#countLiveWorkFor} 也是一条 select。 */
private static final int XML_SELECT_TOTAL = 67;
/**
* 不是「从注解搬过来」的语句:注解时代没有原文可对撞,等价性这支只能跳过它们。
* 登记在此而不是让它们红,是为了让「谁新写了语句」这件事留痕;新增一条不记名的,照样红。
*/
private static final java.util.Set<String> NOT_MIGRATED =
java.util.Set.of("AdminUserMapper#countLiveWorkFor",
// V157 加 token 两列时新写的累加语句;注解时代没有原文可对撞,等价性这支只能跳过它。
// 与 increment 分两条是口径要求(计次先于响应、token 只在响应后有读数),不是随手加的第二条 SQL。
"UserSecretUsageMapper#addTokens");
/** 账本 5 列:接口 / 方法 / @Select 种类 / 是否含常量插值 / 注解原文(Java 反转义后那条串)。 */
private static final int LEDGER_COLUMNS = 5;
private static final int BASELINE_COL = 4;
private static final Pattern STATEMENT = Pattern.compile(
"(?is)<(select|insert|update|delete)\\s+id=\"([A-Za-z0-9_]+)\"([^>]*)>(.*?)</\\1>");
private static final Pattern RESULT_TYPE = Pattern.compile("resultType=\"([^\"]+)\"");
private static final Pattern FOREACH_COLLECTION = Pattern.compile("collection=[\"'](\\w+)[\"']");
private static final Pattern TEST_NAME = Pattern.compile("test=[\"']([^\"']*)[\"']");
private static final Pattern HASH_PARAM = Pattern.compile("#\\{\\s*([A-Za-z_]\\w*)");
/** OGNL 里的字面量与方法名不是参数名,喂值进去会让两边同时以错误告终。 */
private static final Set<String> OGNL_WORDS =
Set.of("null", "true", "false", "and", "or", "not", "size", "isEmpty", "equals", "length");
@Test
void ledgerItselfIsTheExpectedShape() throws IOException {
List<String[]> rows = readLedger();
assertEquals(LEDGER_ROWS, rows.size(), "fixture 行数变了——它是搬迁前的快照,不该跟着代码漂");
assertEquals(LEDGER_SELECTS, rows.stream().filter(r -> "Select".equals(r[2])).count(),
"fixture 里的 Select 条数与基线不符(读侧账本 = 60)");
assertEquals(LEDGER_ROWS - LEDGER_SELECTS,
rows.stream().filter(r -> !"Select".equals(r[2])).count(),
"写侧条数对不上(25 = Insert+Update+Delete)");
assertTrue(rows.stream().allMatch(r -> !r[1].isEmpty() && !r[BASELINE_COL].isEmpty()),
"fixture 有空方法名或空 SQL,说明抽取器漏了形状");
// 基准得是「整条注解串」:半截 <script> 外壳说明抽取被截断(上一轮就是把常量与标签吃了)
assertTrue(rows.stream().allMatch(r ->
r[BASELINE_COL].startsWith("<script>") == r[BASELINE_COL].contains("</script>")),
"有基准行的 <script> 外壳不成对——抽取时注解表达式被截断了");
assertTrue(rows.stream().allMatch(r ->
r[BASELINE_COL].contains("<if") == r[BASELINE_COL].contains("</if>")),
"有基准行的 <if> 不成对——常量插值没展开");
assertTrue(rows.stream().noneMatch(r -> r[BASELINE_COL].contains("IN ()")),
"有基准行出现空的 IN (),说明 foreach/常量被吞了");
}
@Test
void everyMigratedStatementRendersTheSameSqlAsTheAnnotationDid() throws Exception {
Map<String, String> baselines = new LinkedHashMap<>();
Map<String, String> kinds = new LinkedHashMap<>();
for (String[] row : readLedger()) {
baselines.put(row[0] + "#" + row[1], unescapeJava(row[4]));
kinds.put(row[0] + "#" + row[1], row[2].toLowerCase());
}
List<String> problems = new ArrayList<>();
int checked = 0;
for (Path xml : mapperXmls()) {
String iface = stem(xml);
if (!baselines.keySet().stream().anyMatch(k -> k.startsWith(iface + "#"))) {
continue; // TaskImageCache / AsinResultCache 由各自用例钉,不在这本账里
}
Class<?> mapperClass = Class.forName(namespaceOf(xml));
MybatisConfiguration configuration = parseInto(xml, new MybatisConfiguration());
Matcher m = STATEMENT.matcher(read(xml));
while (m.find()) {
String key = iface + "#" + m.group(2);
String baseline = baselines.get(key);
if (baseline == null) {
if (NOT_MIGRATED.contains(key)) {
continue;
}
problems.add(key + " 在基准账本里没有对应注解(XML 多出一条语句)");
continue;
}
checked++;
if (!kinds.get(key).equals(m.group(1))) {
problems.add(key + " 注解是 @" + kinds.get(key) + ",XML 用了 <" + m.group(1) + ">");
continue;
}
MappedStatement fromXml =
configuration.getMappedStatement(mapperClass.getName() + "." + m.group(2));
// 注解当年就是走这条驱动进 MyBatis 的(MapperAnnotationBuilder 用的正是默认脚本语言),
// 所以这里拿它现场编译基准串,等价性由 MyBatis 判定而不是由生成脚本判定。
SqlSource fromAnnotation = configuration.getDefaultScriptingLanguageInstance()
.createSqlSource(configuration, baseline, Object.class);
for (Map<String, Object> params : renderParams(baseline)) {
String where = key + "(" + params.get("_case") + ")";
compare(fromXml.getBoundSql(params), fromAnnotation.getBoundSql(params), where, problems);
}
}
}
int checkedRows = checked;
assertEquals(LEDGER_ROWS, checkedRows,
() -> "只比对了 " + checkedRows + " 条,其余没在 XML 里落地(应为 " + LEDGER_ROWS + " 条)");
assertTrue(problems.isEmpty(),
() -> "这些语句搬完之后 MyBatis 渲染出的 SQL 与注解时不同(" + problems.size() + " 处):"
+ System.lineSeparator() + String.join(System.lineSeparator(), problems));
}
/** 两边各渲染一次:都出 SQL 就逐字比,都抛就比异常,一边过一边炸必须算问题。 */
private static void compare(BoundSql a, BoundSql b, String where, List<String> problems) {
String sqlA, sqlB;
try {
sqlA = normalize(a.getSql());
} catch (RuntimeException e) {
problems.add(where + " XML 侧渲染就抛:" + e);
return;
}
try {
sqlB = normalize(b.getSql());
} catch (RuntimeException e) {
problems.add(where + " 注解侧渲染抛 " + e.getClass().getSimpleName()
+ " 而 XML 侧正常出 SQL: " + sqlA);
return;
}
if (!sqlA.equals(sqlB)) {
problems.add(where + System.lineSeparator()
+ " 注解渲染: " + sqlB + System.lineSeparator()
+ " XML 渲染 : " + sqlA);
return;
}
if (a.getParameterMappings().size() != b.getParameterMappings().size()) {
problems.add(where + " 占位符数不同:注解 " + b.getParameterMappings().size()
+ " vs XML " + a.getParameterMappings().size() + " —— " + sqlA);
}
}
/** 空参数走所有 <if> 的 false 支;再喂满 foreach 走动态支。两支都要两边一致。 */
private static List<Map<String, Object>> renderParams(String baseline) {
Map<String, Object> full = new HashMap<>();
Set<String> collections = new java.util.LinkedHashSet<>();
Matcher fc = FOREACH_COLLECTION.matcher(baseline);
while (fc.find()) {
collections.add(fc.group(1));
}
Set<String> names = new java.util.LinkedHashSet<>(collections);
Matcher t = TEST_NAME.matcher(baseline);
while (t.find()) {
Matcher word = Pattern.compile("\\w+").matcher(t.group(1));
while (word.find()) {
String w = word.group();
if (!OGNL_WORDS.contains(w)) {
names.add(w);
}
}
}
Matcher h = HASH_PARAM.matcher(baseline);
while (h.find()) {
names.add(h.group(1));
}
for (String name : names) {
full.put(name, collections.contains(name) ? List.of("a", "b") : "v");
}
return List.of(Map.of("_case", "空参数"), Map.of("_case", "带参数"))
.stream().map(marker -> {
Map<String, Object> params =
new HashMap<>("_case".equals(marker.get("_case")) ? Map.of() : full);
params.putAll(marker);
return params;
}).collect(Collectors.toList());
}
/** resultType 必须与接口方法签名一致;期望值由反射算,不用生成脚本的结论。 */
@Test
void everySelectKeepsTheReturnTypeItsAnnotationHad() throws Exception {
List<String> problems = new ArrayList<>();
int checked = 0;
for (Path xml : mapperXmls()) {
String text = read(xml);
Matcher ns = Pattern.compile("namespace=\"([^\"]+)\"").matcher(text);
if (!ns.find()) {
problems.add(xml + " 缺 namespace");
continue;
}
Class<?> iface = Class.forName(ns.group(1));
MybatisConfiguration configuration = parseInto(xml, new MybatisConfiguration());
Matcher m = STATEMENT.matcher(text);
while (m.find()) {
if (!"select".equals(m.group(1))) {
continue;
}
String method = m.group(2);
checked++;
MappedStatement statement =
configuration.getMappedStatement(iface.getName() + "." + method);
Class<?> resolved = statement.getResultMaps().get(0).getType();
Matcher attrs = RESULT_TYPE.matcher(m.group(3));
String declared = attrs.find() ? attrs.group(1) : null;
if (declared == null) {
problems.add(iface.getSimpleName() + "#" + method + " 的 <select> 没写 resultType");
continue;
}
Class<?> expected = expectedType(iface, method);
if (expected == null) {
problems.add(iface.getSimpleName() + "#" + method + " 找不到接口方法签名");
} else if (!compatible(expected, resolved)) {
problems.add(iface.getSimpleName() + "#" + method
+ " resultType=" + declared + " 解析成 " + resolved.getName()
+ ",而方法签名要的是 " + expected.getName());
}
}
}
int checkedSelects = checked;
assertTrue(problems.isEmpty(), () -> "resultType 与接口签名不一致(" + problems.size() + " 条):"
+ System.lineSeparator() + String.join(System.lineSeparator(), problems));
// 这支扫的是全部 mapper XML,所以分母比账本大:本轮 60 + TaskImageCache 5 + AsinResultCache 1
assertEquals(XML_SELECT_TOTAL, checkedSelects,
() -> "只检查了 " + checkedSelects + " 条 select,应为 " + XML_SELECT_TOTAL
+ " 条(正则或 XML 漂了)");
}
/** 只把 XML 解析进配置:XMLMapperBuilder 会顺带按 namespace 绑上接口。 */
private static MybatisConfiguration parseInto(Path xml, MybatisConfiguration configuration)
throws IOException {
try (InputStream in = Files.newInputStream(xml)) {
new XMLMapperBuilder(in, configuration, xml.toString(),
configuration.getSqlFragments()).parse();
}
return configuration;
}
private static Class<?> expectedType(Class<?> iface, String method) {
for (Method m : iface.getDeclaredMethods()) {
if (!m.getName().equals(method)) {
continue;
}
return unwrap(m.getGenericReturnType());
}
return null;
}
/** List<X> / Set<X> / Collection<X> 取 X;byte[] 原样;泛型数组元素再解一层。 */
private static Class<?> unwrap(Type type) {
if (type instanceof ParameterizedType pt) {
Type raw = pt.getRawType();
if (raw instanceof Class<?> c && (c == List.class || c == java.util.Set.class
|| c == java.util.Collection.class || c == Iterable.class)) {
return unwrap(pt.getActualTypeArguments()[0]);
}
return (Class<?>) raw;
}
if (type instanceof GenericArrayType ga) {
Class<?> component = unwrap(ga.getGenericComponentType());
if (component == byte.class) {
return byte[].class;
}
return java.lang.reflect.Array.newInstance(component, 0).getClass();
}
if (type instanceof Class<?> c) {
if (c == long.class) {
return Long.class;
}
if (c == int.class) {
return Integer.class;
}
if (c == boolean.class) {
return Boolean.class;
}
if (c == double.class) {
return Double.class;
}
return c;
}
return Object.class;
}
private static boolean compatible(Class<?> expected, Class<?> resolved) {
if (expected == resolved) {
return true;
}
if (Map.class.isAssignableFrom(expected)) {
return Map.class.isAssignableFrom(resolved);
}
return resolved != null && expected.isAssignableFrom(resolved);
}
private static List<String[]> readLedger() throws IOException {
try (InputStream in = Resources.getResourceAsStream(FIXTURE)) {
String text = new String(in.readAllBytes(), StandardCharsets.UTF_8);
List<String[]> rows = new ArrayList<>();
for (String line : text.split("\n")) {
if (line.isBlank()) {
continue;
}
String[] cols = line.split("\t", -1);
if (cols.length != 5) {
throw new IllegalStateException("fixture 列数不对: " + line.substring(0,
Math.min(80, line.length())));
}
rows.add(cols);
}
return rows;
}
}
private static List<Path> mapperXmls() throws IOException {
Path dir = moduleRoot().resolve("src/main/resources/mapper");
if (!Files.isDirectory(dir)) {
fail("找不到 " + dir + " —— 本用例依赖以 backend-java 为工作目录运行");
}
try (Stream<Path> walk = Files.walk(dir)) {
return walk.filter(p -> p.getFileName().toString().endsWith(".xml"))
.sorted().collect(Collectors.toList());
}
}
private static String namespaceOf(Path xml) {
Matcher m = Pattern.compile("namespace=\"([^\"]+)\"").matcher(read(xml));
if (!m.find()) {
throw new IllegalStateException(xml + " 缺 namespace");
}
return m.group(1);
}
/** tsv 里的基准行是 Java 转义过的(一行一条),还原成注解当年真正的那个字符串。 */
private static String unescapeJava(String s) {
StringBuilder out = new StringBuilder(s.length());
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c == '\\' && i + 1 < s.length()) {
char n = s.charAt(++i);
switch (n) {
case 'n' -> out.append('\n');
case 't' -> out.append('\t');
case 'r' -> out.append('\r');
case '\\' -> out.append('\\');
default -> out.append(n);
}
continue;
}
out.append(c);
}
return out.toString();
}
private static String normalize(String text) {
return text.replaceAll("\\s+", " ").trim();
}
private static String read(Path path) {
try {
return Files.readString(path, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("读取失败: " + path, e);
}
}
private static String stem(Path path) {
String name = path.getFileName().toString();
return name.substring(0, name.length() - 4);
}
private static Path moduleRoot() {
Path here = Path.of("").toAbsolutePath();
if (Files.isDirectory(here.resolve("src/main/java"))) {
return here;
}
Path nested = here.resolve("backend-java");
if (Files.isDirectory(nested.resolve("src/main/java"))) {
return nested;
}
throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")");
}
}