Compare commits

...

5 Commits

Author SHA1 Message Date
huangzd1997 3fd58ec197 feat(安全): 店铺凭据解密支持历史密钥,为轮换留出双密钥窗口
换 shop-credential-key 会让 435 行历史密文全部解不开,而调用方拿到解不开的串
是当明文口令往外发(紫鸟账号、1688 店铺、第三方 api_key 都这样用)。
现在新增 aiimage.security.shop-credential-key-previous(逗号分隔),解密按
当前→历史的顺序回试,加密只用当前密钥;全部不中时 WARN 写明"这串密文会被
当成明文口令发出去",不再静默回退。与当前密钥相同的历史项自动丢弃。
previous 留空时行为与改动前逐字节一致(6 例单测盯着这一条)。
轮换设计与取证见 docs/ops/credential-rotation-design-2026-10-02.md(只含长度与
sha8 指纹,无明文值);轮换本身与存量重加密脚本仍未做。
2026-10-02 11:01:18 +08:00
huangzd1997 7ffb404a36 docs(运维): A1 外观批间全量重读已改码 91d85ec6,记下未做的改法2与货源同形态 2026-10-02 10:36:18 +08:00
huangzd1997 91d85ec63e perf(外观专利): LLM 批循环按 payload_hash 复用分片行索引,不再每批重读对象存储
一次 submitLlmBatches 有 N 批,每批落库前都要知道「哪些行属于哪个分片」,此前这份索引
是每批现做:把该任务全部分片的载荷重读重解析一遍,读的量是 (批数+1)×分片数,全在 LLM
请求的串行路径上。改成收集候选那一遍就把行键记进一次性的索引,后续批次只对 payload_hash
变过的分片重读;载荷哈希与载荷内容在所有写入点都是同一条 UPDATE 落的,可作新鲜度锚。

落库路径一行未改:mergeChunkPayload 仍自己 selectOne + 重读最新载荷再做 CAS,索引只用于
「行属于哪一片」的匹配,不作为合并基底。

测试 AppearancePatentChunkRowIndexTest 三例,逐个做过变异验证:去掉哈希比较只咬住
「变过必须重读」,命中缓存返回空集只咬住「复用后行仍要匹配得上分片」;
全量 3413 例通过。
2026-10-02 10:35:23 +08:00
huangzd1997 4015563ecc fix(货源): 品牌检测部分结果文案只写可观测事实,不再指认猜测的故障原因
通知里那句「(品牌检测服务超时或被限流)」是本层代码推断出来的,真实原因
只有 BrandCheckClient 的 brand-check warn 日志知道(本轮实际是代理取不到 IP)。
把猜测写进用户通知,会把排查方向从「代理/账号余额」带到「品牌服务」上去。
新增 CollectDataPartialResultNoticeTest 3 例:驱动真 processResultFileJob,
断言文案含可核对的语义、不含「超时/限流」,且任务已被判失败时不推这条。
2026-10-02 10:35:17 +08:00
huangzd1997 0e4a854169 fix(通知): 聚合去重键从小时桶改自然日,刷新不再把已读点亮回未读
后台「通知」被刷屏的两个乘数:一条告警一天最多重发 24 次(小时桶换键),
以及刷新时 set read_at=null 把用户已读过的红点重新点亮。现在 task_failed /
service_down / maixiang_anomaly 五类键统一按「用户×模块×自然日」聚合,
pushOrRefresh 只更新文案不动 read_at;桶的日期取事件自身的 updatedAt,
跨零点轮询不会把昨天的失败挪进今天的桶。
2026-10-02 10:35:06 +08:00
18 changed files with 930 additions and 86 deletions
@@ -10,7 +10,9 @@ import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
@Service
@Slf4j
@@ -21,7 +23,12 @@ public class ShopCredentialCryptoService {
@Value("${aiimage.security.shop-credential-key:change-me-shop-credential-key}")
private String rawKey;
/** 轮换过渡期用:逗号分隔的旧密钥,解密时按顺序回试,加密只用当前密钥。 */
@Value("${aiimage.security.shop-credential-key-previous:}")
private String previousKeys;
private SecretKeySpec keySpec;
private List<SecretKeySpec> previousKeySpecs = List.of();
@PostConstruct
public void init() {
@@ -30,16 +37,31 @@ public class ShopCredentialCryptoService {
log.warn("[security] AIIMAGE_SHOP_CREDENTIAL_KEY 未配置或仍为默认值,店铺凭据加密强度不足,"
+ "请通过环境变量 AIIMAGE_SHOP_CREDENTIAL_KEY 配置固定密钥");
}
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
byte[] full = sha256.digest(rawKey.getBytes(StandardCharsets.UTF_8));
byte[] key16 = new byte[16];
System.arraycopy(full, 0, key16, 0, 16);
this.keySpec = new SecretKeySpec(key16, "AES");
this.keySpec = deriveKey(rawKey);
List<SecretKeySpec> previous = new ArrayList<>();
for (String old : (previousKeys == null ? "" : previousKeys).split(",")) {
if (old.isBlank()) {
continue;
}
SecretKeySpec spec = deriveKey(old.trim());
if (!spec.equals(keySpec)) {
previous.add(spec);
}
}
this.previousKeySpecs = List.copyOf(previous);
} catch (Exception ex) {
throw new BusinessException("初始化店铺凭据加密器失败");
}
}
private SecretKeySpec deriveKey(String key) throws Exception {
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
byte[] full = sha256.digest(key.getBytes(StandardCharsets.UTF_8));
byte[] key16 = new byte[16];
System.arraycopy(full, 0, key16, 0, 16);
return new SecretKeySpec(key16, "AES");
}
public String encrypt(String plainText) {
try {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
@@ -52,17 +74,26 @@ public class ShopCredentialCryptoService {
}
public String decrypt(String cipherText) {
try {
if (cipherText == null || cipherText.isBlank()) {
return "";
}
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, keySpec);
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(cipherText));
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception ex) {
// 兼容历史数据:旧记录可能是明文或使用旧密钥加密,回退原文避免中断自动化流程。
return cipherText == null ? "" : cipherText;
if (cipherText == null || cipherText.isBlank()) {
return "";
}
List<SecretKeySpec> candidates = new ArrayList<>();
candidates.add(keySpec);
candidates.addAll(previousKeySpecs);
for (SecretKeySpec key : candidates) {
try {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, key);
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(cipherText));
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception ex) {
// 换下一个密钥再试
}
}
// 兼容历史数据:旧记录可能是明文;回退原文会让调用方把密文当口令用,所以要出声
log.warn("[security] 凭据解密失败(当前密钥 + {} 个历史密钥都试过),回退原文;"
+ "刚轮换过 shop-credential-key 的话这条还没重加密,这串密文会被当成明文口令发出去",
previousKeySpecs.size());
return cipherText;
}
}
@@ -14,7 +14,7 @@ public class NotificationProperties {
/** 扫描总开关:关闭后任务失败扫描与服务探测都不执行(应急降噪)。 */
private boolean scanEnabled = true;
/** 扫描间隔(毫秒),默认 5 分钟(任务失败聚合按小时去重,高频扫描不会重复提醒)。 */
/** 扫描间隔(毫秒),默认 5 分钟(聚合通知按自然日去重,高频扫描只会刷新同一条、不会重复提醒)。 */
private long scanIntervalMs = 5 * 60 * 1000L;
/** 首次执行延迟(毫秒),默认 2 分钟,避开启动阶段的数据库压力。 */
@@ -913,8 +913,9 @@ public class AppearancePatentTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
int batchSize = Math.max(1, properties.getLlmBatchSize());
List<AppearancePatentResultRowDto> candidates =
terminalizeRoundExhaustedRows(task, allRowsByBaseId, collectPendingLlmCandidates(chunks));
ChunkRowKeyIndex rowKeyIndex = new ChunkRowKeyIndex();
List<AppearancePatentResultRowDto> candidates = terminalizeRoundExhaustedRows(
task, allRowsByBaseId, collectPendingLlmCandidates(chunks, rowKeyIndex), rowKeyIndex);
boolean flushRemainder = isResultSubmissionComplete(task.getId());
if (!flushRemainder && !candidates.isEmpty()) {
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
@@ -967,7 +968,7 @@ public class AppearancePatentTaskService {
}
List<String> keysToRetry = new ArrayList<>(batchRowKeys);
try {
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows, rowKeyIndex);
// 完成标记只能打在真正落库的行上——这是「发版重启会不会丢行」的分界线
if (batchAnswered) {
List<String> landed = batchRowKeys.stream().filter(landedKeys::contains).toList();
@@ -1007,7 +1008,8 @@ public class AppearancePatentTaskService {
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
List<AppearancePatentResultRowDto> candidates) {
List<AppearancePatentResultRowDto> candidates,
ChunkRowKeyIndex rowKeyIndex) {
int maxRounds = properties.getLlmRowMaxRounds();
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
return candidates == null ? List.of() : candidates;
@@ -1031,7 +1033,8 @@ public class AppearancePatentTaskService {
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
boolean rowsTerminated = false;
try {
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason), exhausted);
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason),
exhausted, rowKeyIndex);
rowsTerminated = true;
} finally {
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
@@ -1047,7 +1050,8 @@ public class AppearancePatentTaskService {
private Set<String> mergeLlmRowsIntoChunks(FileTaskEntity task,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
List<AppearancePatentResultRowDto> llmRows,
List<AppearancePatentResultRowDto> batchRows) {
List<AppearancePatentResultRowDto> batchRows,
ChunkRowKeyIndex rowKeyIndex) {
if (llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
@@ -1065,7 +1069,8 @@ public class AppearancePatentTaskService {
deduped.put(key, row);
}
}
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId);
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId,
null, null, rowKeyIndex);
}
/**
@@ -1084,7 +1089,8 @@ public class AppearancePatentTaskService {
}
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks) {
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
ChunkRowKeyIndex rowKeyIndex) {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
@@ -1094,6 +1100,9 @@ public class AppearancePatentTaskService {
if (persistedRows.isEmpty()) {
continue;
}
// 这一遍已经把所有分片的载荷读进来了,顺手把「这片里有哪些行」记进索引:
// 下面每批落库前都要这份索引,不记的话就得每批把全部分片重读重解析一遍
rememberChunkRowKeys(rowKeyIndex, chunk, persistedRows.keySet());
for (AppearancePatentResultRowDto row : pickGroupRepresentativesForLlm(persistedRows.values())) {
String key = rowKey(row);
if (key.isBlank() || hasResolvedLlmFields(row)) {
@@ -1908,18 +1917,13 @@ public class AppearancePatentTaskService {
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成");
}
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
return mergeLlmRowsIntoSubmittedChunks(task, llmRows, allRowsByBaseId, null, null);
}
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
String fallbackScopeHash,
Integer fallbackChunkIndex) {
Integer fallbackChunkIndex,
ChunkRowKeyIndex rowKeyIndex) {
if (task == null || llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
@@ -1927,11 +1931,11 @@ public class AppearancePatentTaskService {
if (chunks.isEmpty()) {
return Set.of();
}
Map<String, Map<String, AppearancePatentResultRowDto>> rowsByChunk = new LinkedHashMap<>();
Map<String, Set<String>> rowKeysByChunk = new LinkedHashMap<>();
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
String chunkKey = chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
rowsByChunk.put(chunkKey, readChunkRows(chunk));
rowKeysByChunk.put(chunkKey, chunkRowKeySet(rowKeyIndex, chunk));
chunkByKey.put(chunkKey, chunk);
}
Map<String, Map<String, AppearancePatentResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
@@ -1942,8 +1946,8 @@ public class AppearancePatentTaskService {
continue;
}
boolean matched = false;
for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : rowsByChunk.entrySet()) {
if (entry.getValue().containsKey(rowKey)) {
for (Map.Entry<String, Set<String>> entry : rowKeysByChunk.entrySet()) {
if (entry.getValue().contains(rowKey)) {
mergeRowsByChunk.computeIfAbsent(entry.getKey(), ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
@@ -1979,6 +1983,37 @@ public class AppearancePatentTaskService {
return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex);
}
/**
* 一次 {@link #submitLlmBatches} 内共享的「哪个分片里有哪些行」索引,按 payload_hash 判新鲜。
*
* <p>只存 rowKey 不存整行,索引本身不会把载荷钉在内存里。作用域必须是单次调用(局部变量,随栈丢弃):
* 本服务是单例、多个任务同时在跑批,做成字段就是多线程共享一个裸 LinkedHashMap,而且分片键永不过期、只增不减。
* 数据正确性反过来不依赖局部性——收集候选那一遍每次都会重读并重记全部分片。
*/
static final class ChunkRowKeyIndex {
private final Map<String, Snapshot> snapshots = new LinkedHashMap<>();
}
/** 把刚解析出来的分片行键记进索引(自己拷一份,不握住整行载荷),返回索引里那份。 */
private Set<String> rememberChunkRowKeys(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk, Set<String> rowKeys) {
Set<String> copied = new LinkedHashSet<>(rowKeys);
rowKeyIndex.snapshots.put(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()),
new Snapshot(chunk.getPayloadHash(), copied));
return copied;
}
/** 分片当前载荷里的全部行键;载荷哈希与索引里那次解析一致时直接复用,不重读对象存储。 */
private Set<String> chunkRowKeySet(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk) {
Snapshot cached = rowKeyIndex.snapshots.get(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()));
if (cached != null && Objects.equals(cached.payloadHash, chunk.getPayloadHash())) {
return cached.rowKeys;
}
return rememberChunkRowKeys(rowKeyIndex, chunk, readChunkRows(chunk).keySet());
}
private record Snapshot(String payloadHash, Set<String> rowKeys) {
}
private boolean isResultSubmissionComplete(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
@@ -1156,10 +1156,12 @@ public class CollectDataService {
}
// 品牌查询失败的行不进结果、也不落无效品牌表(见上面 rejected/queryFailed 的区分),
// 用户端因此完全没有出口:只在任务确实翻成成功时补一条通知。
// 文案只写这一层观测得到的事实(未返回结果);真实原因(代理取不到 IP / 上游 503 / 被限流)
// 只有 BrandCheckClient 的 brand-check warn 日志知道,写进通知会把排查方向带偏。
if (updated > 0 && stats.brandQueryFailedCount > 0) {
notificationDispatchService.pushPartialResult(task.getUserId(), task.getId(), MODULE_TYPE,
task.getTaskNo(), stats.brandQueryFailedCount
+ " 行商品的品牌检测未返回结果(品牌检测服务超时或被限流),这些行不计入结果条数,"
+ " 行商品的品牌检测未返回结果,这些行不计入结果条数,"
+ "也没有被判定为无效品牌,可稍后单独重跑这些 ASIN");
}
} finally {
@@ -41,9 +41,9 @@ import java.util.stream.Collectors;
* "采集机掉线"(2026-09-29 停摆 6 小时即如此,且当时 210 条 pending 没到 300 的阈值,
* 连积压检查都没触发)。
*
* <p>全部为管理员全局事件(与「服务探测」一致,不按数据权限过滤);去重键:
* 停滞类按天、失败/积压按小时、空结果按任务——同一异常窗口内只提醒一次,
* 情况变化(新增停滞任务/失败数增长)时刷新内容并重新提醒。
* <p>全部为管理员全局事件(与「服务探测」一致,不按数据权限过滤);去重键按自然日
* (空结果按任务):同一异常当天只一条,情况变化(新增停滞任务/失败数增长)时刷新内容,
* 但不把已读的重新点亮 —— 一台掉线的采集机会被扫几十轮,逐轮点亮等于刷屏。
*
* <p>由 {@link NotificationScanScheduler} 在分布式锁内调用;单项检查异常只记日志不中断其余检查。
*/
@@ -66,7 +66,6 @@ public class MaixiangAnomalyScanner {
private static final int REASON_MAX_LENGTH = 120;
private static final DateTimeFormatter DAY_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd");
private static final DateTimeFormatter HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH");
private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
/** 麦象任务类型 → 中文名(未登记的类型回退原始值)。 */
@@ -146,7 +145,7 @@ public class MaixiangAnomalyScanner {
String content = "麦象采集机疑似掉线:所有打点信号已静默 " + (machineIdle / 60) + " 分钟(" + detail
+ ")。该机器无开机自启,重启/升配后必须人工拉起;请检查 5 个 worker 是否在运行"
+ "(start_task.bat 或 更新并重启.bat)。";
push(audience, "采集机掉线", content, "maixiang_silent:" + now.format(HOUR_FORMAT));
push(audience, "采集机掉线", content, "maixiang_silent:" + now.format(DAY_FORMAT));
}
/** 打点信号的中文标签:接口没给 label 时回退信号名。 */
@@ -214,7 +213,7 @@ public class MaixiangAnomalyScanner {
push(audience, "麦象任务滞留未消费", content, "maixiang_stuck_single:" + now.format(DAY_FORMAT));
}
/** 任务失败:近窗口内 status=3 的单任务达到最小条数(按小时桶聚合,原因取最新一条)。 */
/** 任务失败:近窗口内 status=3 的单任务达到最小条数(按自然日聚合,原因取最新一条)。 */
void checkFailedSingleTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
SingleTaskPage page = consoleClient.singleTasks(3, null, FAIL_PAGE_SIZE);
LocalDateTime windowStart = now.minusMinutes(FAIL_WINDOW_MINUTES);
@@ -234,7 +233,7 @@ public class MaixiangAnomalyScanner {
+ typeSummary(failed.stream().map(SingleTaskItem::taskType).toList()) + ")"
+ (reason.isEmpty() ? "" : ";最近失败原因:" + reason)
+ "。请排查采集端配置(如代理余额、任务类型注册)。";
push(audience, "麦象任务失败", content, "maixiang_fail:" + now.format(HOUR_FORMAT));
push(audience, "麦象任务失败", content, "maixiang_fail:" + now.format(DAY_FORMAT));
}
/** 批量任务空结果:最近完成但结果 0 条(worker 报错时批次仍会被标为"已完成")。 */
@@ -289,7 +288,7 @@ public class MaixiangAnomalyScanner {
}
String content = "麦象单任务队列积压:" + detail + "(阈值 待处理 " + pendingThreshold
+ " 且无消费 / 处理中 " + processingThreshold + ")。消费端可能掉线,请尽快排查采集机 worker。";
push(audience, "麦象队列积压", content, "maixiang_queue_backlog:" + now.format(HOUR_FORMAT));
push(audience, "麦象队列积压", content, "maixiang_queue_backlog:" + now.format(DAY_FORMAT));
}
// ======================== 文案辅助 ========================
@@ -29,11 +29,11 @@ import java.util.Map;
* 站内通知扫描:任务失败聚合提醒 + 下游服务健康探测。
*
* <p>任务失败:回看窗口内进入失败终态的任务(biz_file_task / brand_crawl_tasks),
* 按「用户 × 模块 × 小时」聚合,同一小时桶内增量刷新同一条通知(不刷屏);
* 按「用户 × 模块 × 自然日」聚合,同一天内增量刷新同一条通知(不刷屏);
* 用户侧推给任务创建者,管理员侧按数据权限推给管辖该用户的管理员。
*
* <p>服务探测:品牌检测服务(15126) / 跟价任务 API(18960) / jikip 代理接口,
* 失败立即重试一次(过滤瞬抖),两次都失败才告警;同服务每小时最多一条。
* 失败立即重试一次(过滤瞬抖),两次都失败才告警;同服务每天最多一条。
*
* <p>麦象异常扫描:见 {@link MaixiangAnomalyScanner}(批量/单任务停滞、任务失败、
* 批量任务空结果、队列积压 → 管理员通知),与任务失败扫描共用本调度与分布式锁。
@@ -55,7 +55,7 @@ public class NotificationScanScheduler {
private static final int REASON_MAX_LENGTH = 120;
private static final int PROBE_READ_TIMEOUT_MILLIS = 8_000;
private static final DateTimeFormatter HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH");
private static final DateTimeFormatter DAY_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd");
/**
* 任务模块类型 → 中文名:直接取模块注册表(G6),新增模块只需在
@@ -102,7 +102,7 @@ public class NotificationScanScheduler {
}
}
/** 任务失败聚合扫描:窗口内失败终态任务按「用户 × 模块 × 小时」聚合成一条通知。 */
/** 任务失败聚合扫描:窗口内失败终态任务按「用户 × 模块 × 自然日」聚合成一条通知。 */
void scanFailedTasks() {
LocalDateTime cutoff = LocalDateTime.now()
.minusMinutes(Math.max(1, properties.getTaskFailedWindowMinutes()));
@@ -121,7 +121,7 @@ public class NotificationScanScheduler {
continue;
}
failed.add(new FailedTask(task.getId(), displayTaskNo(task.getTaskNo(), task.getId()),
task.getModuleType(), task.getUserId(), task.getErrorMessage()));
task.getModuleType(), task.getUserId(), task.getErrorMessage(), task.getUpdatedAt()));
}
List<BrandCrawlTaskEntity> brandTasks = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper<BrandCrawlTaskEntity>()
.select(BrandCrawlTaskEntity::getId, BrandCrawlTaskEntity::getUserId,
@@ -135,19 +135,18 @@ public class NotificationScanScheduler {
continue;
}
failed.add(new FailedTask(task.getId(), "#" + task.getId(),
BRAND_MODULE_TYPE, task.getUserId(), task.getErrorMessage()));
BRAND_MODULE_TYPE, task.getUserId(), task.getErrorMessage(), task.getUpdatedAt()));
}
if (failed.isEmpty()) {
log.info("[notification-scan] 近 {} 分钟无失败任务", properties.getTaskFailedWindowMinutes());
return;
}
String hour = LocalDateTime.now().format(HOUR_FORMAT);
Map<BucketKey, List<FailedTask>> buckets = new LinkedHashMap<>();
for (FailedTask task : failed) {
String moduleType = normalizeModuleType(task.moduleType());
buckets.computeIfAbsent(new BucketKey(task.userId(), moduleType, hour), key -> new ArrayList<>())
.add(task);
buckets.computeIfAbsent(new BucketKey(task.userId(), moduleType, dayOf(task.updatedAt())),
key -> new ArrayList<>()).add(task);
}
NotificationDispatchService.AdminAudience audience = notificationDispatchService.prepareAdminAudience();
@@ -179,7 +178,7 @@ public class NotificationScanScheduler {
adminPushed += notificationDispatchService.pushToAdmins(audience,
NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
"用户任务失败:" + username, adminContent,
"task_failed_admin:" + key.userId() + ":" + key.moduleType() + ":" + key.hour(),
"task_failed_admin:" + key.userId() + ":" + key.moduleType() + ":" + key.day(),
key.userId());
}
log.info("[notification-scan] 任务失败扫描完成 窗口={}分钟 失败任务={} 聚合桶={} 用户通知={} 管理员通知={}",
@@ -255,7 +254,7 @@ public class NotificationScanScheduler {
}
}
/** 探测结果上报:正常记 info;故障按小时去重推管理员通知;outcome=null 表示跳过。 */
/** 探测结果上报:正常记 info;故障按自然日去重推管理员通知;outcome=null 表示跳过。 */
private void reportProbe(String serviceKey, String serviceLabel, ProbeOutcome outcome) {
if (outcome == null) {
return;
@@ -265,12 +264,11 @@ public class NotificationScanScheduler {
return;
}
log.warn("[notification-scan] 服务探测失败 service={} detail={}", serviceLabel, outcome.detail());
String hour = LocalDateTime.now().format(HOUR_FORMAT);
notificationDispatchService.pushToAdmins(NotificationService.SCENE_SERVICE_DOWN,
NotificationService.LEVEL_ERROR,
serviceLabel + "不可用",
serviceLabel + "探测失败:" + outcome.detail() + "。该服务相关任务可能大面积失败,请尽快排查。",
"service_down:" + serviceKey + ":" + hour, null);
"service_down:" + serviceKey + ":" + dayOf(LocalDateTime.now()), null);
}
/** 通知保留期清理:每天最多一次;已读与未读各按自己的保留天数。 */
@@ -287,7 +285,12 @@ public class NotificationScanScheduler {
}
private String dedupeKeyOf(BucketKey key) {
return "task_failed:" + key.userId() + ":" + key.moduleType() + ":" + key.hour();
return "task_failed:" + key.userId() + ":" + key.moduleType() + ":" + key.day();
}
/** 聚合桶的日期标签:取事件自身所属自然日,跨零点轮询时不会把昨天的失败挪进今天的桶。 */
private String dayOf(LocalDateTime at) {
return (at == null ? LocalDateTime.now() : at).format(DAY_FORMAT);
}
private String taskNoPreview(List<FailedTask> tasks) {
@@ -355,12 +358,13 @@ public class NotificationScanScheduler {
return current.getClass().getSimpleName() + ": " + message;
}
/** 聚合桶键:用户 × 模块 × 小时。 */
record BucketKey(Long userId, String moduleType, String hour) {
/** 聚合桶键:用户 × 模块 × 自然日。 */
record BucketKey(Long userId, String moduleType, String day) {
}
/** 统一视图的失败任务(两类表归一)。 */
record FailedTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage) {
/** 统一视图的失败任务(两类表归一);updatedAt 即失败终态时间,决定落入哪一天的桶。 */
record FailedTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage,
LocalDateTime updatedAt) {
}
/** 探测结果。 */
@@ -107,10 +107,11 @@ public class NotificationService {
}
/**
* 同键推送或刷新:不存在则新建;已存在且内容有变化时更新内容并重置为未读
* (用于小时桶聚合的增量更新:同小时内新增失败任务时刷新计数与明细)。
* 同键推送或刷新:不存在则新建;已存在且内容有变化时只更新文案,
* <b>不动 read_at</b>(聚合通知的去重键按自然日,同一条一天内会被刷新很多次,
* 重新点亮已读等于把红点做成闹钟)。
*
* @return true=新建或内容有更新(用户有新信息),false=无变化或去重未命中变化
* @return true=新建或内容有更新,false=无变化
*/
public boolean pushOrRefresh(Long userId, String audience, String scene, String level,
String title, String content, String dedupeKey) {
@@ -131,8 +132,7 @@ public class NotificationService {
.eq(UserNotificationEntity::getId, existing.getId())
.set(UserNotificationEntity::getTitle, nextTitle)
.set(UserNotificationEntity::getContent, nextContent)
.set(UserNotificationEntity::getLevel, normalize(level).isEmpty() ? LEVEL_WARNING : level)
.set(UserNotificationEntity::getReadAt, null));
.set(UserNotificationEntity::getLevel, normalize(level).isEmpty() ? LEVEL_WARNING : level));
log.info("[notification] 同键通知已刷新 id={} userId={} scene={} dedupeKey={}",
existing.getId(), userId, scene, key);
return true;
@@ -464,6 +464,8 @@ aiimage:
unread-retention-days: ${AIIMAGE_NOTIFICATION_UNREAD_RETENTION_DAYS:180}
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
# 轮换过渡期用:逗号分隔的旧密钥,解密按顺序回试,加密只用上面那个;迁完就清空
shop-credential-key-previous: ${AIIMAGE_SHOP_CREDENTIAL_KEY_PREVIOUS:}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
internal-token-file: ${AIIMAGE_INTERNAL_TOKEN_FILE:}
auth:
@@ -0,0 +1,96 @@
package com.nanri.aiimage.common.security;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 店铺凭据/用户密钥加密器的轮换语义测试。
* 关键契约:轮换后旧密文必须还能解(按 previous 回试),且全部解不开时是「回退原文」而不是抛错——
* 这条回退会把密文当明文口令发出去,所以轮换必须配合重加密,见 crawler-plugin/docs/ops/credential-rotation-design-2026-10-02.md。
*/
class ShopCredentialCryptoServiceTest {
private static final String OLD_KEY = "old-key-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
private static final String NEW_KEY = "new-key-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
private static final String THIRD_KEY = "third-key-cccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
private static ShopCredentialCryptoService newService(String rawKey, String previousKeys) throws Exception {
ShopCredentialCryptoService service = new ShopCredentialCryptoService();
set(service, "rawKey", rawKey);
set(service, "previousKeys", previousKeys);
service.init();
return service;
}
private static void set(Object target, String name, String value) throws Exception {
Field field = target.getClass().getDeclaredField(name);
field.setAccessible(true);
field.set(target, value);
}
@SuppressWarnings("unchecked")
private static List<Object> previousOf(ShopCredentialCryptoService service) throws Exception {
Field field = service.getClass().getDeclaredField("previousKeySpecs");
field.setAccessible(true);
return (List<Object>) field.get(service);
}
@Test
void rotates_key_but_still_decrypts_legacy_ciphertext() throws Exception {
ShopCredentialCryptoService old = newService(OLD_KEY, "");
ShopCredentialCryptoService rotated = newService(NEW_KEY, OLD_KEY);
String legacy = old.encrypt("P@ssw0rd-shopA");
assertEquals("P@ssw0rd-shopA", rotated.decrypt(legacy));
assertNotEquals(old.encrypt("same-input"), rotated.encrypt("same-input"), "加密只用当前密钥");
assertEquals("same-input", rotated.decrypt(rotated.encrypt("same-input")));
}
@Test
void falls_back_to_the_input_when_no_key_matches() throws Exception {
String legacy = newService(OLD_KEY, "").encrypt("P@ssw0rd-shopA");
ShopCredentialCryptoService wrong = newService(NEW_KEY, THIRD_KEY);
assertEquals(legacy, wrong.decrypt(legacy), "回退原文是既有兼容行为,不能借轮换悄悄改成抛错");
}
@Test
void legacy_plaintext_values_pass_through_unchanged() throws Exception {
ShopCredentialCryptoService service = newService(NEW_KEY, "");
assertEquals("not-a-ciphertext!!", service.decrypt("not-a-ciphertext!!"));
assertEquals("", service.decrypt(null));
assertEquals("", service.decrypt(" "));
}
@Test
void behaves_exactly_as_before_when_previous_keys_is_empty() throws Exception {
ShopCredentialCryptoService service = newService(NEW_KEY, "");
String cipher = service.encrypt("v");
assertEquals("v", service.decrypt(cipher));
assertTrue(previousOf(service).isEmpty());
}
@Test
void drops_previous_keys_that_equal_the_current_key() throws Exception {
ShopCredentialCryptoService service = newService(NEW_KEY, NEW_KEY + " , ," + NEW_KEY);
assertTrue(previousOf(service).isEmpty(), "重复密钥白跑一遍解密,还会掩盖 previous 其实没配");
}
@Test
void tries_several_previous_keys_in_order() throws Exception {
String fromThird = newService(THIRD_KEY, "").encrypt("v");
ShopCredentialCryptoService service = newService(NEW_KEY, OLD_KEY + "," + THIRD_KEY);
assertEquals("v", service.decrypt(fromThird));
assertEquals(2, previousOf(service).size());
}
}
@@ -0,0 +1,265 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.service.ResultDownloadResolver;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 「没变过的分片,一次检测里只解析一遍」。
*
* <p>合批后一次 {@code submitLlmBatches} 要跑 N 批,而每批落库前都得先知道「这些行属于哪个分片」。
* 这份索引此前是每批现做:把该任务**全部**分片的载荷从对象存储重读一遍、JSON 重解析一遍。
* 线上一个 211 片的任务光索引就每批 211 次读,批数一多就是批数×分片数的平方级浪费,
* 而这些读全都在 LLM 请求的串行路径上——检测阶段的墙钟时间有一半花在这里。
*
* <p>这里钉住三件事:载荷哈希没变过的分片不再解析(省下来的那部分)、
* 变过的一定要从头解析(不然索引会指向已经不存在的行,跨节点并发回传时就会把结果写错片)、
* 命中复用时行照常能匹配上自己的分片(省读省成返回空集合,就是"扣次数没结论")。
*
* <p>索引的作用域是单次调用(局部变量,随栈丢弃),这一条不做断言:收集候选那一遍本来就会
* 重读并重记每个分片,就算有人把它做成字段也不会有可见行为差别,能咬住它的只有内存占用和并发。
*/
@ExtendWith(MockitoExtension.class)
class AppearancePatentChunkRowIndexTest {
private static final Long TASK_ID = 29501L;
private static final Long JOB_ID = 20801L;
private static final String SCOPE_A = "scope-a";
private static final String SCOPE_B = "scope-b";
private static final String SCOPE_C = "scope-c";
private static final String POINTER_A = "ptr-chunk-a.json";
private static final String POINTER_B = "ptr-chunk-b.json";
private static final String POINTER_C = "ptr-chunk-c.json";
private static final String HASH_A = "hash-a-1";
private static final String HASH_B = "hash-b-1";
private static final String HASH_C = "hash-c-1";
/** 本批写入后分片 A 在库里的新哈希(模拟 mergeChunkPayload 的 CAS 生效)。 */
private static final String HASH_A_WRITTEN = "hash-a-2";
/** 6 行未检测:batchSize=2 时正好 3 批,批数大于 1 才看得见「每批重读全部分片」。 */
private static final String PENDING_ROWS = rows(6, "r", false);
/** 已检测完的分片:不产生候选、也不会被本次调用写入,是「纯粹没变过」的那两片。 */
private static final String DONE_ROWS_B = rows(3, "b", true);
private static final String DONE_ROWS_C = rows(3, "c", true);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private ResultDownloadResolver resultDownloadResolver;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private AppearancePatentLlmClient llmClient;
@Mock private AppearancePatentTaskCacheService taskCacheService;
@Mock private AppearancePatentProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
@Mock private UserApiSecretService userApiSecretService;
@InjectMocks private AppearancePatentTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.getLlmBatchSize()).thenReturn(2);
lenient().when(properties.getFlushPendingMinutes()).thenReturn(1);
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(userApiSecretService.findPlainValue(any(), anyString())).thenReturn("");
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_A), anyString())).thenReturn(PENDING_ROWS);
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString())).thenReturn(DONE_ROWS_B);
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_C), anyString())).thenReturn(DONE_ROWS_C);
// 库里每批都重新查一遍:只有 A 因为刚落库换了哈希,B/C 一个字节都没动
lenient().when(taskChunkMapper.selectList(any()))
.thenReturn(List.of(chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, HASH_C)));
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk(SCOPE_A, POINTER_A, HASH_A));
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
anyString(), anyLong(), anyString(), any(), anyString())).thenReturn("sibling-1");
lenient().when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
lenient().when(llmClient.inspectRows(any(), any(), any())).thenAnswer(invocation -> {
List<AppearancePatentResultRowDto> batch = invocation.getArgument(0);
for (AppearancePatentResultRowDto row : batch) {
row.setTitleRisk("无侵权");
row.setAppearanceRisk("无侵权");
row.setConclusion("无侵权");
}
return new ArrayList<>(batch);
});
}
/**
* 一次调用 3 批,从头到尾没被改过的分片 C 只应该被解析一次(收集候选那一遍)。
*
* <p>钉住的是「省下来的那部分」:现在每批都要为 C 重读重解析一遍,3 批 + 收集 = 4 次。
*/
@Test
void chunkThatNeverChangedIsParsedOncePerRun() {
submit();
verify(transientPayloadStorageService, times(1)).resolvePayload(eq(POINTER_C), anyString());
}
/**
* 复用索引不能只省读、还得照常认得别的分片的行:第 4 批的结果照样要落进 B,而不是变成孤儿。
*
* <p>省读的实现有个致命的顺手写法——命中缓存时返回空集合或只返回「本批候选所在片」,
* 表现是行匹配不上分片、被撤销标记反复重发,用户侧就是扣了次数没结论。
*/
@Test
void reusedIndexStillMatchesRowsOfEveryChunk() {
when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString()))
.thenReturn(rows(2, "b", false));
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, HASH_C)));
submit();
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("r1", "r2"));
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("b1", "b2"));
verify(taskCacheService, never()).releaseRowInspection(eq(TASK_ID), any());
}
/**
* 载荷哈希一变就必须重新解析——省读不能省成「拿旧行集合匹配新载荷」。
*
* <p>收集候选用的是入口那份 chunk 快照(哈希还是 hash-c-1),批次里查库拿到的是别的节点
* 回传之后的 hash-c-2。此时 C 的行集合已经不可信,必须重读。
*/
@Test
void chunkWhosePayloadHashChangedIsParsedAgain() {
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
task(), new FileResultEntity(), job(),
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, "hash-c-stale")),
Map.of());
verify(transientPayloadStorageService, times(2)).resolvePayload(eq(POINTER_C), anyString());
}
// ===== 辅助 =====
private void submit() {
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
task(), new FileResultEntity(), job(),
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, HASH_C)),
Map.of());
}
private FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
task.setUserId(7L);
return task;
}
private com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job() {
com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job =
new com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity();
job.setId(JOB_ID);
job.setTaskId(TASK_ID);
return job;
}
private TaskChunkEntity chunk(String scopeHash, String pointer, String payloadHash) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(11L);
chunk.setTaskId(TASK_ID);
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(1);
chunk.setPayloadJson(pointer);
chunk.setPayloadHash(payloadHash);
return chunk;
}
private static String rows(int count, String prefix, boolean resolved) {
StringBuilder json = new StringBuilder("[");
for (int i = 1; i <= count; i++) {
if (i > 1) {
json.append(',');
}
json.append("{\"rowToken\":\"").append(prefix).append(i)
.append("\",\"asin\":\"B0A000000").append(i % 10)
.append("\",\"title\":\"PRODUCT ").append(i)
.append("\",\"url\":\"http://img/").append(i).append(".jpg\",")
.append("\"groupKey\":\"").append(prefix).append("-g").append(i).append("\"");
if (resolved) {
json.append(",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"");
}
json.append('}');
}
return json.append(']').toString();
}
}
@@ -135,7 +135,7 @@ class AppearancePatentRowMarkTimingTest {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
assertEquals(2, candidates.size());
verify(taskCacheService).beginRowInspection(TASK_ID, "r1");
@@ -151,7 +151,7 @@ class AppearancePatentRowMarkTimingTest {
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r2"))).thenReturn(false);
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
assertEquals(1, candidates.size());
assertEquals("r1", candidates.get(0).getRowToken());
@@ -0,0 +1,149 @@
package com.nanri.aiimage.modules.collectdata.service;
import cn.hutool.core.io.FileUtil;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.File;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 「任务成功但有一部分行没跑出来」通知文案(2026-10-02)。
*
* <p>背景:后台通知里这条写死了「品牌检测服务超时或被限流」,而 10-02 取证出的真实根因是品牌链
* 上游的采集代理取不到 IP(余额不足)与 503 —— 代码在这个位置只观测到「品牌检测没返回结果」,
* 观测不到原因。写死一个猜测的原因会把排查方向从代理/上游带到自身服务上。
*
* <p>锁定契约:文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效、可重跑),
* 不指认代码没有依据的具体故障原因;真实原因留在 BrandCheckClient 的 warn 日志里
* (brand-check 的 lastErr 字段),由运维按日志排查。
*/
@ExtendWith(MockitoExtension.class)
class CollectDataPartialResultNoticeTest {
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private CollectDataExcelAssemblyService excelAssemblyService;
@Mock private OssStorageService ossStorageService;
@Mock private NotificationDispatchService notificationDispatchService;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks private CollectDataService service;
private static final long TASK_ID = 29991L;
/** LambdaUpdateWrapper 需要实体的列缓存,纯单测里没有 Spring 容器要手工初始化。 */
@BeforeAll
static void initTableInfo() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
}
@AfterEach
void cleanWorkDir() {
FileUtil.del(new File(System.getProperty("java.io.tmpdir"), "collect-data-result/" + TASK_ID));
}
private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows) {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType("COLLECT_DATA");
task.setStatus("RUNNING");
task.setTaskNo("CD-29991");
task.setUserId(7L);
task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows + "}");
return task;
}
private TaskFileJobEntity job() {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setTaskId(TASK_ID);
job.setResultId(5001L);
return job;
}
private FileResultEntity result() {
FileResultEntity result = new FileResultEntity();
result.setId(5001L);
result.setModuleType("COLLECT_DATA");
result.setSourceFilename("keywords.xlsx");
return result;
}
private String stubHappyPathAndCaptureDetail() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
// 条件更新返回 1:任务翻成成功,这条「部分缺失」通知才会推给用户
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
service.processResultFileJob(job());
ArgumentCaptor<String> detail = ArgumentCaptor.forClass(String.class);
verify(notificationDispatchService).pushPartialResult(eq(7L), eq(TASK_ID), eq("COLLECT_DATA"),
eq("CD-29991"), detail.capture());
return detail.getValue();
}
@Test
void 部分结果通知只说可观测事实_不指认猜测的故障原因() {
String detail = stubHappyPathAndCaptureDetail();
assertThat(detail).contains("7 行");
// 「超时」「被限流」都是这一层观测不到的原因,写进文案会把排查方向带偏
assertThat(detail).doesNotContain("超时");
assertThat(detail).doesNotContain("限流");
}
@Test
void 部分结果通知保留用户能自己核对的语义() {
String detail = stubHappyPathAndCaptureDetail();
assertThat(detail).contains("不计入结果条数");
assertThat(detail).contains("未返回结果");
assertThat(detail).contains("重跑");
}
@Test
void 任务已被判失败时不推部分结果通知() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
when(fileTaskMapper.update(isNull(), any())).thenReturn(0);
service.processResultFileJob(job());
verify(notificationDispatchService, org.mockito.Mockito.never())
.pushPartialResult(anyLong(), anyLong(), anyString(), anyString(), anyString());
}
}
@@ -162,7 +162,7 @@ class MaixiangAnomalyScannerTest {
content.capture(), key.capture(), isNull());
assertThat(content.getValue()).contains("1 条任务失败");
assertThat(content.getValue()).contains("未注册的任务类型: price-track-run");
assertThat(key.getValue()).isEqualTo("maixiang_fail:2026091512");
assertThat(key.getValue()).isEqualTo("maixiang_fail:20260915");
}
@Test
@@ -238,7 +238,7 @@ class MaixiangAnomalyScannerTest {
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象队列积压"),
content.capture(), eq("maixiang_queue_backlog:2026091512"), isNull());
content.capture(), eq("maixiang_queue_backlog:20260915"), isNull());
assertThat(content.getValue()).contains("L1 待处理 210 / 处理中 0").contains("堆积且无人在消费");
}
@@ -263,7 +263,7 @@ class MaixiangAnomalyScannerTest {
newScanner().checkQueueBacklog(audience, NOW);
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(),
eq("maixiang_queue_backlog:2026091512"), any());
eq("maixiang_queue_backlog:20260915"), any());
}
@Test
@@ -292,7 +292,7 @@ class MaixiangAnomalyScannerTest {
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:采集机掉线"),
content.capture(), eq("maixiang_silent:2026091512"), isNull());
content.capture(), eq("maixiang_silent:20260915"), isNull());
assertThat(content.getValue())
.contains("静默 11 分钟")
.contains("跟价 静默 11 分钟")
@@ -15,6 +15,7 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -64,13 +65,13 @@ class NotificationDispatchServiceTest {
NotificationDispatchService.AdminAudience audience = service.prepareAdminAudience();
int pushed = service.pushToAdmins(audience, NotificationService.SCENE_TASK_FAILED,
NotificationService.LEVEL_WARNING, "标题", "内容", "task_failed_admin:20:PRICE_TRACK:2026091310", 20L);
NotificationService.LEVEL_WARNING, "标题", "内容", "task_failed_admin:20:PRICE_TRACK:20260913", 20L);
// 主管乙不可见用户 20 → 只有超管收到
assertThat(pushed).isEqualTo(1);
org.mockito.Mockito.verify(notificationService).pushOrRefresh(eq(1L), eq(NotificationService.AUDIENCE_ADMIN),
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:2026091310:1"));
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:20260913:1"));
}
@Test
@@ -89,21 +90,54 @@ class NotificationDispatchServiceTest {
int pushed = service.pushToAdmins(NotificationService.SCENE_TASK_FAILED,
NotificationService.LEVEL_WARNING, "标题", "内容",
"task_failed_admin:20:PRICE_TRACK:2026091310", 20L);
"task_failed_admin:20:PRICE_TRACK:20260913", 20L);
// 超管全量 + 本组主管各落一条,各带自己的 dedupe 后缀(已读状态互不影响)
assertThat(pushed).isEqualTo(2);
verify(notificationService).pushOrRefresh(eq(1L), eq(NotificationService.AUDIENCE_ADMIN),
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:2026091310:1"));
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:20260913:1"));
verify(notificationService).pushOrRefresh(eq(2L), eq(NotificationService.AUDIENCE_ADMIN),
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:2026091310:2"));
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:20260913:2"));
// 非本组主管一条都不落
verify(notificationService, never()).pushOrRefresh(eq(3L), anyString(), anyString(),
anyString(), anyString(), anyString(), anyString());
}
/**
* 合并告警的权限口径:能看到组内任一号的管理员收 1 条,一个都看不到的收不到 —— 每人最多一条。
* 逐个主体调用 pushToAdmins 会让同一管理员为 N 个账号收到 N 条同内容通知。
*/
@Test
void pushToAdminsAnyOfDeliversAtMostOnePerAdminSeeingAnySubject() {
AdminUserEntity superAdmin = admin(1L, "超管甲");
AdminUserEntity inScope = admin(2L, "主管乙");
AdminUserEntity outOfScope = admin(3L, "主管丙");
when(adminUserMapper.selectList(any())).thenReturn(List.of(superAdmin, inScope, outOfScope));
when(adminAuthSupport.currentRole(superAdmin)).thenReturn("super_admin");
when(adminAuthSupport.currentRole(inScope)).thenReturn("admin");
when(adminAuthSupport.currentRole(outOfScope)).thenReturn("admin");
when(userDataScopeSupport.resolveVisibleUserIds(2L)).thenReturn(List.of(2L, 20L));
when(userDataScopeSupport.resolveVisibleUserIds(3L)).thenReturn(List.of(3L, 30L));
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
.thenReturn(true);
int pushed = service.pushToAdminsAnyOf(service.prepareAdminAudience(),
NotificationService.SCENE_SECRET_BALANCE, NotificationService.LEVEL_ERROR,
"标题", "内容", "secret_balance_admin:group:hash:20261002", List.of(11L, 12L, 20L));
// 3 个受影响账号 × 2 个可见管理员 = 只落 2 条;非本组主管一条都不落
assertThat(pushed).isEqualTo(2);
verify(notificationService, times(2)).pushOrRefresh(anyLong(), anyString(), anyString(),
anyString(), anyString(), anyString(), anyString());
verify(notificationService).pushOrRefresh(eq(2L), eq(NotificationService.AUDIENCE_ADMIN),
eq(NotificationService.SCENE_SECRET_BALANCE), eq(NotificationService.LEVEL_ERROR),
eq("标题"), eq("内容"), eq("secret_balance_admin:group:hash:20261002:2"));
verify(notificationService, never()).pushOrRefresh(eq(3L), anyString(), anyString(),
anyString(), anyString(), anyString(), anyString());
}
@Test
void globalEventReachesEveryAdminByDesign() {
AdminUserEntity superAdmin = admin(1L, "超管甲");
@@ -120,7 +154,7 @@ class NotificationDispatchServiceTest {
// 服务探测失败这类全局事件(subjectUserId=null)按设计推给所有管理员,不受分组限制
int pushed = service.pushToAdmins(NotificationService.SCENE_SERVICE_DOWN,
NotificationService.LEVEL_ERROR, "品牌检测服务不可用", "探测失败",
"service_down:brand-service:2026091310", null);
"service_down:brand-service:20260913", null);
assertThat(pushed).isEqualTo(3);
}
@@ -117,7 +117,89 @@ class NotificationScanSchedulerTest {
anyString(), anyString(), anyString(), anyString());
}
/**
* 同一天同模块的失败必须共用一条通知:按「扫描时刻的小时」分桶时,同一个失败任务
* 只要还落在 60 分钟回看窗口里,跨过整点就会被塞进新桶,于是每小时重发一条
* (2026-10-02 取证:task_failed 管理员侧因此 24 倍放大)。
*/
@Test
void failedTasksOfSameDayShareOneDedupeKeyAcrossHours() {
when(fileTaskMapper.selectList(any())).thenReturn(List.of(
fileTask(101L, "T-101", "PRICE_TRACK", 7L, "浏览器启动失败", LocalDateTime.of(2026, 10, 1, 9, 12)),
fileTask(102L, "T-102", "PRICE_TRACK", 7L, "cookie 失效", LocalDateTime.of(2026, 10, 1, 18, 36))));
when(brandCrawlTaskMapper.selectList(any())).thenReturn(List.of());
stubAdminAudience();
newScheduler().scanFailedTasks();
ArgumentCaptor<String> userKey = ArgumentCaptor.forClass(String.class);
verify(notificationService, times(1)).pushOrRefresh(eq(7L), eq(NotificationService.AUDIENCE_USER),
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
anyString(), anyString(), userKey.capture());
assertThat(userKey.getValue()).isEqualTo("task_failed:7:PRICE_TRACK:20261001");
ArgumentCaptor<String> adminKey = ArgumentCaptor.forClass(String.class);
verify(dispatch, times(1)).pushToAdmins(any(NotificationDispatchService.AdminAudience.class),
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
anyString(), anyString(), adminKey.capture(), eq(7L));
assertThat(adminKey.getValue()).isEqualTo("task_failed_admin:7:PRICE_TRACK:20261001");
}
/** 桶按「任务自身所属自然日」而非扫描时钟:跨零点轮询时,昨晚的失败仍归昨晚那条通知。 */
@Test
void failedTasksOnDifferentDaysGetSeparateDedupeKeys() {
when(fileTaskMapper.selectList(any())).thenReturn(List.of(
fileTask(101L, "T-101", "PRICE_TRACK", 7L, "浏览器启动失败", LocalDateTime.of(2026, 10, 1, 23, 50)),
fileTask(102L, "T-102", "PRICE_TRACK", 7L, "cookie 失效", LocalDateTime.of(2026, 10, 2, 0, 10))));
when(brandCrawlTaskMapper.selectList(any())).thenReturn(List.of());
stubAdminAudience();
newScheduler().scanFailedTasks();
ArgumentCaptor<String> userKey = ArgumentCaptor.forClass(String.class);
verify(notificationService, times(2)).pushOrRefresh(anyLong(), anyString(), anyString(), anyString(),
anyString(), anyString(), userKey.capture());
assertThat(userKey.getAllValues()).containsExactlyInAnyOrder(
"task_failed:7:PRICE_TRACK:20261001", "task_failed:7:PRICE_TRACK:20261002");
}
/**
* 服务探测故障:去重键按天,同一天同服务只提醒一次(小时桶时一个持续 503 的上游
* 每小时给每位管理员各塞一条)。取精确 8 位日期,多一位小时都不行。
*/
@Test
void serviceDownAlertIsDedupedPerCalendarDay() {
properties.setBrandServiceUrl("http://127.0.0.1:1");
properties.setPriceTrackApiUrl("");
properties.setJikipProbeEnabled(false);
when(dispatch.pushToAdmins(anyString(), anyString(), anyString(), anyString(), anyString(), any()))
.thenReturn(1);
newScheduler().probeServices();
ArgumentCaptor<String> keyBase = ArgumentCaptor.forClass(String.class);
verify(dispatch).pushToAdmins(eq(NotificationService.SCENE_SERVICE_DOWN),
eq(NotificationService.LEVEL_ERROR), eq("品牌检测服务不可用"), anyString(),
keyBase.capture(), org.mockito.ArgumentMatchers.<Long>isNull());
assertThat(keyBase.getValue()).matches("service_down:brand-service:\\d{8}");
}
private void stubAdminAudience() {
when(dispatch.prepareAdminAudience()).thenReturn(
new NotificationDispatchService.AdminAudience(List.of(), Set.of(), Map.of()));
when(dispatch.displayNameOf(anyLong())).thenReturn("张三");
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
.thenReturn(true);
when(dispatch.pushToAdmins(any(NotificationDispatchService.AdminAudience.class), anyString(), anyString(),
anyString(), anyString(), anyString(), any())).thenReturn(1);
}
private FileTaskEntity fileTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage) {
return fileTask(id, taskNo, moduleType, userId, errorMessage, LocalDateTime.of(2026, 10, 1, 18, 36));
}
private FileTaskEntity fileTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage,
LocalDateTime updatedAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setTaskNo(taskNo);
@@ -125,7 +207,7 @@ class NotificationScanSchedulerTest {
task.setUserId(userId);
task.setErrorMessage(errorMessage);
task.setStatus("FAILED");
task.setUpdatedAt(LocalDateTime.now());
task.setUpdatedAt(updatedAt);
return task;
}
}
@@ -84,23 +84,34 @@ class NotificationServiceTest {
verify(mapper, never()).insert(any(UserNotificationEntity.class));
}
/**
* 聚合通知按自然日只有一条,一整天会被反复刷新:内容有变化时只改文案,
* 绝不能把 read_at 置空——否则管理员刚读完红点又亮,一天点亮几十次
* (2026-10-02 后台超管 582 条未读的主放大器之一)。
*/
@Test
void pushOrRefreshUpdatesContentAndResetsUnread() {
@SuppressWarnings("unchecked")
void pushOrRefreshUpdatesContentButKeepsReadState() {
UserNotificationEntity existing = new UserNotificationEntity();
existing.setId(9L);
existing.setTitle("跟价任务失败");
existing.setContent("最近 60 分钟内有 2 个跟价任务失败");
existing.setLevel("warning");
existing.setReadAt(LocalDateTime.now());
existing.setReadAt(LocalDateTime.of(2026, 9, 13, 10, 0));
when(mapper.selectOne(any())).thenReturn(existing);
boolean refreshed = service.pushOrRefresh(7L, NotificationService.AUDIENCE_ADMIN,
NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
"跟价任务失败", "最近 60 分钟内有 5 个跟价任务失败", "task_failed:7:PRICE_TRACK:2026091310");
"跟价任务失败", "最近 60 分钟内有 5 个跟价任务失败", "task_failed:7:PRICE_TRACK:20260913");
assertThat(refreshed).isTrue();
verify(mapper, never()).insert(any(UserNotificationEntity.class));
verify(mapper, times(1)).update(any(), any());
ArgumentCaptor<LambdaUpdateWrapper<UserNotificationEntity>> captor =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(mapper, times(1)).update(any(), captor.capture());
String sqlSet = captor.getValue().getSqlSet();
assertThat(sqlSet).contains("title").contains("content").contains("level");
assertThat(sqlSet).doesNotContain("read_at");
}
@Test
@@ -113,7 +124,7 @@ class NotificationServiceTest {
boolean refreshed = service.pushOrRefresh(7L, NotificationService.AUDIENCE_USER,
NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
"跟价任务失败", "最近 60 分钟内有 2 个跟价任务失败", "task_failed:7:PRICE_TRACK:2026091310");
"跟价任务失败", "最近 60 分钟内有 2 个跟价任务失败", "task_failed:7:PRICE_TRACK:20260913");
assertThat(refreshed).isFalse();
verify(mapper, never()).update(any(), any());
@@ -0,0 +1,125 @@
# 店铺凭据密钥轮换设计(2026-10-02)
范围:`aiimage.security.shop-credential-key` 这一把密钥的轮换,以及轮换时三张库里凭据表怎么处理。
本文只给设计、取证结果和待办,**没有执行过任何轮换或重启**。
## 一、为什么现在不能直接换密钥
`ShopCredentialCryptoService.decrypt()` 解不出来时回退返回原文(`ShopCredentialCryptoService.java:76-97`)。
调用方拿到这串原文就当明文口令往外发——紫鸟账号、1688 店铺、第三方 api_key 都是这样被使用的。
所以「改了密钥、没重加密存量」这段时间里的表现不是报错,而是**拿着密文当口令去登录**,
失败点离原因很远,最难查。
本次改动(只做防护,未动线上):
- `ShopCredentialCryptoService` 支持 `shop-credential-key-previous`(逗号分隔的旧密钥),解密时当前密钥失败后按顺序回试旧密钥,加密只用当前密钥。
- 回退不再静默:打 WARN,写明「当前密钥 + N 个历史密钥都试过」,并直接说出后果(这串密文会被当成明文口令发出去)。
## 二、生效值到底配在哪一层(今天的实测)
| 键 | 位置 | 读数 |
| --- | --- | --- |
| `shop-credential-key` | 容器 `/app/application-server.yml`(宿主 `/app/java/`) | `configured=yes len=64 sha8=190cc85c occurrences=1`,**两台一致** |
| `jwt-secret` | 同上 | `len=64 sha8=2d40fe8a`,**两台一致** |
| `AIIMAGE_SHOP_CREDENTIAL_KEY` | 宿主 `/.env`、java 进程 environ | **两处都没有** |
| `AIIMAGE_JWT_SECRET` | 同上 | **两处都没有** |
| `AIIMAGE_DB_PASSWORD` | 宿主 `/.env`(= java 进程 environ) | `len=16 sha8=83809acd`,两台一致 |
**结论:这两把密钥的生效值只有 yml 一个来源,没有环境变量覆盖它。** 之前按「改 `/.env` 追加 export」的
配方去轮换这两把是无效的——那条配方只对 `/.env` 里真实存在的那些键有效。改密钥就改
`/app/java/application-server.yml`,两台各一份,改完 `sha8` 应当同步变化。
复现命令(只读,值不出生产):
```
MSYS2_ARG_CONV_EXCL='*' python scripts/_prod_config.py all --container java-server \
--file /app/application-server.yml --key shop-credential-key
```
## 三、存量行的形态:没有明文(本次实测)
`scripts/_probe_credential_decrypt_audit.sh` 在主机A 上用**当前密钥真解了一遍**三张表的七个凭据列,
只输出计数。判据与 Java 一致:AES/ECB/PKCS5Padding,密钥 = SHA-256(原文密钥) 前 16 字节;
并且必须过 base64 往返、16 字节对齐、填充合法、**解出来是合法 UTF-8** 四道关
——只看长度和对齐是不够的,换错密钥也能解出「看着像样」的字节。
| 表.列 | 行数 | 空 | 解得出 | b64 非串 | 长度不对 | 填充不对 | 非 UTF-8 |
| --- | --- | --- | --- | --- | --- | --- | --- |
| `biz_shop_manage.password` | 86 | 0 | 86 | 0 | 0 | 0 | 0 |
| `biz_user_api_secret.secret_value` | 314 | 0 | 314 | 0 | 0 | 0 | 0 |
| `biz_image_video_secret.copy_api_key` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
| `biz_image_video_secret.t8_key` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
| `biz_image_video_secret.t8_video_key` | 7 | 2 | 5 | 0 | 0 | 0 | 0 |
| `biz_image_video_secret.voice_api_key` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
| `biz_image_video_secret.voice_group_id` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
三点用处:
1. **没有明文行**,435 行里 433 行是当前密钥解得出的密文,另 2 行是空值。所以重加密可以走纯 Java
一条路,不需要「遇到解不出的行就跳过」的分支——那个分支的存在本身就等于把密文当明文写回去。
2. 轮换前跑一次、轮换后跑一次,**`解得出` 的计数应当一模一样**,这是这批改动的验收标准。
3. 写入路径本来就全过加密(`ShopManageService.java:100,124`、`UserApiSecretService.java:843`,
两处 `setSecretValue`/`setPassword` 都包着 `encrypt(...)`),没有发现明文字段入口。
## 四、轮换顺序(每一步都能单独发版)
1. **先发版**:带双密钥回试的版本。此时 `previous` 为空,行为与旧版逐字节相同
(`behaves_exactly_as_before_when_previous_keys_is_empty` 钉住这条)。
2. **改配置**:两台 `/app/java/application-server.yml` 同时改——`shop-credential-key` 写新值、
`shop-credential-key-previous` 写旧值。**这一步起新数据用新密钥加密,旧数据仍解得出。**
回滚 = 把两行改回原值重启,比重加密便宜得多。
3. **重加密存量**:跑一次迁移(下面第 5 节),把这 433 行用当前密钥重新加密。
4. **验证**:再跑 `scripts/_probe_credential_decrypt_audit.sh`,`解得出` 仍是 433、其余四类仍为 0。
5. **收尾**:清掉 `shop-credential-key-previous`,下次发版生效。留着不影响正确性,只是每次解密多一轮试。
顺序不能反:先改配置再发版的话,旧 jar 不认 `previous` 这个键,存量行立刻进回退分支。
## 五、重加密迁移的关键约束
- **只改密文,不改业务时间戳。** 这批表上有 `updated_at` 会被心跳/看板读,走实体更新会顺带刷新它。
- **不要走业务 Service 的写入路径**(会带缓存刷新、索引重算、`ziniaoShopIndexRefreshTrigger` 之类的副作用)。
用一条独立的取行→解密→用当前密钥加密→按主键回写的脚本,跑在 Java 侧。
- 分表分批、每批独立事务;失败要能重跑(重加密天然幂等,重复跑只是再解一次)。
- 迁移脚本**本次没写**,等你定了窗口再动。
## 六、不该走这条路的几把密钥
| 密钥 | 为什么另算 |
| --- | --- |
| `auth.jwt-secret`(sha8 `2d40fe8a`) | 换它 = 现有登录态全废,所有人被踢;必须单独定维护窗口,和重加密无关 |
| `AIIMAGE_DB_PASSWORD`(sha8 `83809acd`) | 库口令,改它要同时动 `/.env` 与 MySQL 账号,属另一套流程 |
| 极客图 / jikip / LLM 代理 key | 生效值在 yml 或 `biz_user_api_secret`,前者按普通配置轮换、后者随第五节的迁移一起走 |
| MinIO(B 机设备日志、A 机业务桶) | 走各自的 `.env`/服务重启,与店铺凭据无关 |
## 七、验证做到哪一步
- 新增 `ShopCredentialCryptoServiceTest` 6 条,`mvn -o -Dtest=ShopCredentialCryptoServiceTest test`
→ `Tests run: 6, Failures: 0, Errors: 0` + `BUILD SUCCESS`。
- 变异检验:把 `candidates.addAll(previousKeySpecs)` 删掉后重跑 → `Tests run: 6, Failures: 2`
(`rotates_key_but_still_decrypts_legacy_ciphertext`、`tries_several_previous_keys_in_order` 变红),
说明这两条真的在钉双密钥回试,不是恒绿断言。
(Python 侧的等价做法见 `scripts/tests/`:`umask 077` 挪到 `base64 -d >` 之后、或绕过 `upload_and_run_command` 直接拼命令,各有 1 条用例变红,改完的文件按 md5 原样还原。)
- 这批测试走的是标准生命周期(`mvn -o -Dtest=ShopCredentialCryptoServiceTest test`)。
上午有一段时间 `test-compile` 被并发会话新加的未跟踪测试挡住,当时只能用「javac 单编我的测试类 +
`surefire:test` 单跑」绕过;09:5x 复跑时那条阻塞已经自己清了,没有动过别人的文件。
- Python 侧:`scripts/tests` 共 78 例(`PYTHONUTF8=1 python -m pytest scripts/tests -q` 全绿)。
掩码层那 8 个护栏点也各打坏过一次(出口不掩码、三层各自失效、`verdict` 把读不到当一致、裸标记不记录、MSYS 路径不拦、`atexit` 没注册),**8/8 都有用例变红**;其中「`atexit.register(_report)` 删掉」原本是漏网之鱼——直接调 `_report` 的用例看不见「没注册」这个状态,补了一条开子进程实跑的用例才咬住。
## 八、还欠什么
1. **轮换本身没执行**,新密钥由你定、yml 由你改(生产写操作按约定只交付命令,不代跑)。
2. **重加密迁移脚本没写**——第五节的约束已经定好,等窗口。
3. `ShopManageService.java:260` 往 VO 写的是 `masked`,读侧掩码是有的;**日志侧**这几把密钥有没有
别的出口,本次没审。
4. 外观/货源的 LLM key 分组那部分结论来自之前的实测记录(一把 key 平均挂 9.5 个账号),
本次没复核,引它做决策前先重跑一遍当时的脚本。
5. **两台 `/tmp` 里躺着 48 个 644 权限的历史探针副本**(主机A 36 个、主机B 12 个,其中 20 个含
`password|secret|token|api_key` 形式的行)——那是 `_ssh_bash.py` 加 `rm -f` 收尾之前几次会话留下的。
上传命令现在带 `umask 077`,新副本落成 600,但**旧的这 48 个还在**,删除算生产改动,命令给你:
```
python scripts/_ssh_bash.py a scripts/清理用临时脚本 # 或直接
MSYS2_ARG_CONV_EXCL='*' python scripts/_ssh_probe.py all 'rm -f /tmp/_run__*.sh; ls /tmp/_run__*.sh 2>/dev/null | wc -l'
```
后面那条回读应当是 `0`。
+9
View File
@@ -38,6 +38,15 @@ B 组是 DB / 对象存储的往返浪费(心跳整行读、删除路径的全
**风险**:分片在批中被别的入口改写时读到旧态。现状靠 `mergeChunkPayload` 的 CAS 兜住,改后仍要走同一 CAS,只省读不省写。行数与判定结果不变。
**状态(2026-10-02 10:2x):改法 1 已提交 `91d85ec6`,未发版。**
索引改为「分片键 → (payload_hash, rowKey 集合)」,在 `collectPendingLlmCandidates` 那一遍就建好,
批内 merge 只对 hash 变过的分片重读(`chunkRowKeySet`);载荷哈希与内容在所有写入点都是同一条 UPDATE 落的,可直接作新鲜度锚。
落库路径一行未动,`mergeChunkPayload` 仍自己 `selectOne` + 重读最新载荷再 CAS,索引不参与合并基底。
每批的 `loadSubmittedChunks`(DB SELECT 全部分片)保留——它就是锚,不能省。
读的次数从 `(批数+1)×分片数` 降到 `分片数 + 本批写过的片数`。
改法 2(`rowKey→chunkKey` 索引去掉逐片线性试探)**未做**:那是内存里的 O(C) 扫描,不是这次的瓶颈。
测试 `AppearancePatentChunkRowIndexTest` 3 例,逐条做过变异检验;**货源侧 `SimilarAsinPipelineSupport.java:183-187` 是同一形态、还没改**。
### A2 外观专利:品牌校验合批 + 复用已有缓存 `[P0]`
**证据链**