Compare commits
5 Commits
17cc775176
...
3fd58ec197
| Author | SHA1 | Date | |
|---|---|---|---|
| 3fd58ec197 | |||
| 7ffb404a36 | |||
| 91d85ec63e | |||
| 4015563ecc | |||
| 0e4a854169 |
+47
-16
@@ -10,7 +10,9 @@ import javax.crypto.Cipher;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -21,7 +23,12 @@ public class ShopCredentialCryptoService {
|
||||
@Value("${aiimage.security.shop-credential-key:change-me-shop-credential-key}")
|
||||
private String rawKey;
|
||||
|
||||
/** 轮换过渡期用:逗号分隔的旧密钥,解密时按顺序回试,加密只用当前密钥。 */
|
||||
@Value("${aiimage.security.shop-credential-key-previous:}")
|
||||
private String previousKeys;
|
||||
|
||||
private SecretKeySpec keySpec;
|
||||
private List<SecretKeySpec> previousKeySpecs = List.of();
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
@@ -30,16 +37,31 @@ public class ShopCredentialCryptoService {
|
||||
log.warn("[security] AIIMAGE_SHOP_CREDENTIAL_KEY 未配置或仍为默认值,店铺凭据加密强度不足,"
|
||||
+ "请通过环境变量 AIIMAGE_SHOP_CREDENTIAL_KEY 配置固定密钥");
|
||||
}
|
||||
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
|
||||
byte[] full = sha256.digest(rawKey.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] key16 = new byte[16];
|
||||
System.arraycopy(full, 0, key16, 0, 16);
|
||||
this.keySpec = new SecretKeySpec(key16, "AES");
|
||||
this.keySpec = deriveKey(rawKey);
|
||||
List<SecretKeySpec> previous = new ArrayList<>();
|
||||
for (String old : (previousKeys == null ? "" : previousKeys).split(",")) {
|
||||
if (old.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
SecretKeySpec spec = deriveKey(old.trim());
|
||||
if (!spec.equals(keySpec)) {
|
||||
previous.add(spec);
|
||||
}
|
||||
}
|
||||
this.previousKeySpecs = List.copyOf(previous);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("初始化店铺凭据加密器失败");
|
||||
}
|
||||
}
|
||||
|
||||
private SecretKeySpec deriveKey(String key) throws Exception {
|
||||
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
|
||||
byte[] full = sha256.digest(key.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] key16 = new byte[16];
|
||||
System.arraycopy(full, 0, key16, 0, 16);
|
||||
return new SecretKeySpec(key16, "AES");
|
||||
}
|
||||
|
||||
public String encrypt(String plainText) {
|
||||
try {
|
||||
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
|
||||
@@ -52,17 +74,26 @@ public class ShopCredentialCryptoService {
|
||||
}
|
||||
|
||||
public String decrypt(String cipherText) {
|
||||
try {
|
||||
if (cipherText == null || cipherText.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
|
||||
cipher.init(Cipher.DECRYPT_MODE, keySpec);
|
||||
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(cipherText));
|
||||
return new String(decrypted, StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
// 兼容历史数据:旧记录可能是明文或使用旧密钥加密,回退原文避免中断自动化流程。
|
||||
return cipherText == null ? "" : cipherText;
|
||||
if (cipherText == null || cipherText.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
List<SecretKeySpec> candidates = new ArrayList<>();
|
||||
candidates.add(keySpec);
|
||||
candidates.addAll(previousKeySpecs);
|
||||
for (SecretKeySpec key : candidates) {
|
||||
try {
|
||||
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
|
||||
cipher.init(Cipher.DECRYPT_MODE, key);
|
||||
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(cipherText));
|
||||
return new String(decrypted, StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
// 换下一个密钥再试
|
||||
}
|
||||
}
|
||||
// 兼容历史数据:旧记录可能是明文;回退原文会让调用方把密文当口令用,所以要出声
|
||||
log.warn("[security] 凭据解密失败(当前密钥 + {} 个历史密钥都试过),回退原文;"
|
||||
+ "刚轮换过 shop-credential-key 的话这条还没重加密,这串密文会被当成明文口令发出去",
|
||||
previousKeySpecs.size());
|
||||
return cipherText;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ public class NotificationProperties {
|
||||
/** 扫描总开关:关闭后任务失败扫描与服务探测都不执行(应急降噪)。 */
|
||||
private boolean scanEnabled = true;
|
||||
|
||||
/** 扫描间隔(毫秒),默认 5 分钟(任务失败聚合按小时去重,高频扫描不会重复提醒)。 */
|
||||
/** 扫描间隔(毫秒),默认 5 分钟(聚合通知按自然日去重,高频扫描只会刷新同一条、不会重复提醒)。 */
|
||||
private long scanIntervalMs = 5 * 60 * 1000L;
|
||||
|
||||
/** 首次执行延迟(毫秒),默认 2 分钟,避开启动阶段的数据库压力。 */
|
||||
|
||||
+54
-19
@@ -913,8 +913,9 @@ public class AppearancePatentTaskService {
|
||||
String prompt = readAiPrompt(task);
|
||||
String apiKey = readApiKey(task);
|
||||
int batchSize = Math.max(1, properties.getLlmBatchSize());
|
||||
List<AppearancePatentResultRowDto> candidates =
|
||||
terminalizeRoundExhaustedRows(task, allRowsByBaseId, collectPendingLlmCandidates(chunks));
|
||||
ChunkRowKeyIndex rowKeyIndex = new ChunkRowKeyIndex();
|
||||
List<AppearancePatentResultRowDto> candidates = terminalizeRoundExhaustedRows(
|
||||
task, allRowsByBaseId, collectPendingLlmCandidates(chunks, rowKeyIndex), rowKeyIndex);
|
||||
boolean flushRemainder = isResultSubmissionComplete(task.getId());
|
||||
if (!flushRemainder && !candidates.isEmpty()) {
|
||||
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
|
||||
@@ -967,7 +968,7 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
List<String> keysToRetry = new ArrayList<>(batchRowKeys);
|
||||
try {
|
||||
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows);
|
||||
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmRows, batchRows, rowKeyIndex);
|
||||
// 完成标记只能打在真正落库的行上——这是「发版重启会不会丢行」的分界线
|
||||
if (batchAnswered) {
|
||||
List<String> landed = batchRowKeys.stream().filter(landedKeys::contains).toList();
|
||||
@@ -1007,7 +1008,8 @@ public class AppearancePatentTaskService {
|
||||
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
|
||||
FileTaskEntity task,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
List<AppearancePatentResultRowDto> candidates) {
|
||||
List<AppearancePatentResultRowDto> candidates,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
int maxRounds = properties.getLlmRowMaxRounds();
|
||||
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
|
||||
return candidates == null ? List.of() : candidates;
|
||||
@@ -1031,7 +1033,8 @@ public class AppearancePatentTaskService {
|
||||
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
boolean rowsTerminated = false;
|
||||
try {
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason), exhausted);
|
||||
mergeLlmRowsIntoChunks(task, allRowsByBaseId, llmClient.markRowsFailed(exhausted, reason),
|
||||
exhausted, rowKeyIndex);
|
||||
rowsTerminated = true;
|
||||
} finally {
|
||||
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
|
||||
@@ -1047,7 +1050,8 @@ public class AppearancePatentTaskService {
|
||||
private Set<String> mergeLlmRowsIntoChunks(FileTaskEntity task,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
List<AppearancePatentResultRowDto> batchRows) {
|
||||
List<AppearancePatentResultRowDto> batchRows,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
if (llmRows == null || llmRows.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
@@ -1065,7 +1069,8 @@ public class AppearancePatentTaskService {
|
||||
deduped.put(key, row);
|
||||
}
|
||||
}
|
||||
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId);
|
||||
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), allRowsByBaseId,
|
||||
null, null, rowKeyIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1084,7 +1089,8 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
|
||||
|
||||
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks) {
|
||||
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
@@ -1094,6 +1100,9 @@ public class AppearancePatentTaskService {
|
||||
if (persistedRows.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
// 这一遍已经把所有分片的载荷读进来了,顺手把「这片里有哪些行」记进索引:
|
||||
// 下面每批落库前都要这份索引,不记的话就得每批把全部分片重读重解析一遍
|
||||
rememberChunkRowKeys(rowKeyIndex, chunk, persistedRows.keySet());
|
||||
for (AppearancePatentResultRowDto row : pickGroupRepresentativesForLlm(persistedRows.values())) {
|
||||
String key = rowKey(row);
|
||||
if (key.isBlank() || hasResolvedLlmFields(row)) {
|
||||
@@ -1908,18 +1917,13 @@ public class AppearancePatentTaskService {
|
||||
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成");
|
||||
}
|
||||
|
||||
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
|
||||
return mergeLlmRowsIntoSubmittedChunks(task, llmRows, allRowsByBaseId, null, null);
|
||||
}
|
||||
|
||||
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
|
||||
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId,
|
||||
String fallbackScopeHash,
|
||||
Integer fallbackChunkIndex) {
|
||||
Integer fallbackChunkIndex,
|
||||
ChunkRowKeyIndex rowKeyIndex) {
|
||||
if (task == null || llmRows == null || llmRows.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
@@ -1927,11 +1931,11 @@ public class AppearancePatentTaskService {
|
||||
if (chunks.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Map<String, Map<String, AppearancePatentResultRowDto>> rowsByChunk = new LinkedHashMap<>();
|
||||
Map<String, Set<String>> rowKeysByChunk = new LinkedHashMap<>();
|
||||
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
String chunkKey = chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
|
||||
rowsByChunk.put(chunkKey, readChunkRows(chunk));
|
||||
rowKeysByChunk.put(chunkKey, chunkRowKeySet(rowKeyIndex, chunk));
|
||||
chunkByKey.put(chunkKey, chunk);
|
||||
}
|
||||
Map<String, Map<String, AppearancePatentResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
|
||||
@@ -1942,8 +1946,8 @@ public class AppearancePatentTaskService {
|
||||
continue;
|
||||
}
|
||||
boolean matched = false;
|
||||
for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : rowsByChunk.entrySet()) {
|
||||
if (entry.getValue().containsKey(rowKey)) {
|
||||
for (Map.Entry<String, Set<String>> entry : rowKeysByChunk.entrySet()) {
|
||||
if (entry.getValue().contains(rowKey)) {
|
||||
mergeRowsByChunk.computeIfAbsent(entry.getKey(), ignored -> new LinkedHashMap<>())
|
||||
.put(rowKey, expandedRow);
|
||||
matched = true;
|
||||
@@ -1979,6 +1983,37 @@ public class AppearancePatentTaskService {
|
||||
return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次 {@link #submitLlmBatches} 内共享的「哪个分片里有哪些行」索引,按 payload_hash 判新鲜。
|
||||
*
|
||||
* <p>只存 rowKey 不存整行,索引本身不会把载荷钉在内存里。作用域必须是单次调用(局部变量,随栈丢弃):
|
||||
* 本服务是单例、多个任务同时在跑批,做成字段就是多线程共享一个裸 LinkedHashMap,而且分片键永不过期、只增不减。
|
||||
* 数据正确性反过来不依赖局部性——收集候选那一遍每次都会重读并重记全部分片。
|
||||
*/
|
||||
static final class ChunkRowKeyIndex {
|
||||
private final Map<String, Snapshot> snapshots = new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
/** 把刚解析出来的分片行键记进索引(自己拷一份,不握住整行载荷),返回索引里那份。 */
|
||||
private Set<String> rememberChunkRowKeys(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk, Set<String> rowKeys) {
|
||||
Set<String> copied = new LinkedHashSet<>(rowKeys);
|
||||
rowKeyIndex.snapshots.put(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()),
|
||||
new Snapshot(chunk.getPayloadHash(), copied));
|
||||
return copied;
|
||||
}
|
||||
|
||||
/** 分片当前载荷里的全部行键;载荷哈希与索引里那次解析一致时直接复用,不重读对象存储。 */
|
||||
private Set<String> chunkRowKeySet(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk) {
|
||||
Snapshot cached = rowKeyIndex.snapshots.get(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()));
|
||||
if (cached != null && Objects.equals(cached.payloadHash, chunk.getPayloadHash())) {
|
||||
return cached.rowKeys;
|
||||
}
|
||||
return rememberChunkRowKeys(rowKeyIndex, chunk, readChunkRows(chunk).keySet());
|
||||
}
|
||||
|
||||
private record Snapshot(String payloadHash, Set<String> rowKeys) {
|
||||
}
|
||||
|
||||
private boolean isResultSubmissionComplete(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return false;
|
||||
|
||||
+3
-1
@@ -1156,10 +1156,12 @@ public class CollectDataService {
|
||||
}
|
||||
// 品牌查询失败的行不进结果、也不落无效品牌表(见上面 rejected/queryFailed 的区分),
|
||||
// 用户端因此完全没有出口:只在任务确实翻成成功时补一条通知。
|
||||
// 文案只写这一层观测得到的事实(未返回结果);真实原因(代理取不到 IP / 上游 503 / 被限流)
|
||||
// 只有 BrandCheckClient 的 brand-check warn 日志知道,写进通知会把排查方向带偏。
|
||||
if (updated > 0 && stats.brandQueryFailedCount > 0) {
|
||||
notificationDispatchService.pushPartialResult(task.getUserId(), task.getId(), MODULE_TYPE,
|
||||
task.getTaskNo(), stats.brandQueryFailedCount
|
||||
+ " 行商品的品牌检测未返回结果(品牌检测服务超时或被限流),这些行不计入结果条数,"
|
||||
+ " 行商品的品牌检测未返回结果,这些行不计入结果条数,"
|
||||
+ "也没有被判定为无效品牌,可稍后单独重跑这些 ASIN");
|
||||
}
|
||||
} finally {
|
||||
|
||||
+7
-8
@@ -41,9 +41,9 @@ import java.util.stream.Collectors;
|
||||
* "采集机掉线"(2026-09-29 停摆 6 小时即如此,且当时 210 条 pending 没到 300 的阈值,
|
||||
* 连积压检查都没触发)。
|
||||
*
|
||||
* <p>全部为管理员全局事件(与「服务探测」一致,不按数据权限过滤);去重键:
|
||||
* 停滞类按天、失败/积压按小时、空结果按任务——同一异常窗口内只提醒一次,
|
||||
* 情况变化(新增停滞任务/失败数增长)时刷新内容并重新提醒。
|
||||
* <p>全部为管理员全局事件(与「服务探测」一致,不按数据权限过滤);去重键按自然日
|
||||
* (空结果按任务):同一异常当天只一条,情况变化(新增停滞任务/失败数增长)时刷新内容,
|
||||
* 但不把已读的重新点亮 —— 一台掉线的采集机会被扫几十轮,逐轮点亮等于刷屏。
|
||||
*
|
||||
* <p>由 {@link NotificationScanScheduler} 在分布式锁内调用;单项检查异常只记日志不中断其余检查。
|
||||
*/
|
||||
@@ -66,7 +66,6 @@ public class MaixiangAnomalyScanner {
|
||||
private static final int REASON_MAX_LENGTH = 120;
|
||||
|
||||
private static final DateTimeFormatter DAY_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
private static final DateTimeFormatter HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH");
|
||||
private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
/** 麦象任务类型 → 中文名(未登记的类型回退原始值)。 */
|
||||
@@ -146,7 +145,7 @@ public class MaixiangAnomalyScanner {
|
||||
String content = "麦象采集机疑似掉线:所有打点信号已静默 " + (machineIdle / 60) + " 分钟(" + detail
|
||||
+ ")。该机器无开机自启,重启/升配后必须人工拉起;请检查 5 个 worker 是否在运行"
|
||||
+ "(start_task.bat 或 更新并重启.bat)。";
|
||||
push(audience, "采集机掉线", content, "maixiang_silent:" + now.format(HOUR_FORMAT));
|
||||
push(audience, "采集机掉线", content, "maixiang_silent:" + now.format(DAY_FORMAT));
|
||||
}
|
||||
|
||||
/** 打点信号的中文标签:接口没给 label 时回退信号名。 */
|
||||
@@ -214,7 +213,7 @@ public class MaixiangAnomalyScanner {
|
||||
push(audience, "麦象任务滞留未消费", content, "maixiang_stuck_single:" + now.format(DAY_FORMAT));
|
||||
}
|
||||
|
||||
/** 任务失败:近窗口内 status=3 的单任务达到最小条数(按小时桶聚合,原因取最新一条)。 */
|
||||
/** 任务失败:近窗口内 status=3 的单任务达到最小条数(按自然日聚合,原因取最新一条)。 */
|
||||
void checkFailedSingleTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
||||
SingleTaskPage page = consoleClient.singleTasks(3, null, FAIL_PAGE_SIZE);
|
||||
LocalDateTime windowStart = now.minusMinutes(FAIL_WINDOW_MINUTES);
|
||||
@@ -234,7 +233,7 @@ public class MaixiangAnomalyScanner {
|
||||
+ typeSummary(failed.stream().map(SingleTaskItem::taskType).toList()) + ")"
|
||||
+ (reason.isEmpty() ? "" : ";最近失败原因:" + reason)
|
||||
+ "。请排查采集端配置(如代理余额、任务类型注册)。";
|
||||
push(audience, "麦象任务失败", content, "maixiang_fail:" + now.format(HOUR_FORMAT));
|
||||
push(audience, "麦象任务失败", content, "maixiang_fail:" + now.format(DAY_FORMAT));
|
||||
}
|
||||
|
||||
/** 批量任务空结果:最近完成但结果 0 条(worker 报错时批次仍会被标为"已完成")。 */
|
||||
@@ -289,7 +288,7 @@ public class MaixiangAnomalyScanner {
|
||||
}
|
||||
String content = "麦象单任务队列积压:" + detail + "(阈值 待处理 " + pendingThreshold
|
||||
+ " 且无消费 / 处理中 " + processingThreshold + ")。消费端可能掉线,请尽快排查采集机 worker。";
|
||||
push(audience, "麦象队列积压", content, "maixiang_queue_backlog:" + now.format(HOUR_FORMAT));
|
||||
push(audience, "麦象队列积压", content, "maixiang_queue_backlog:" + now.format(DAY_FORMAT));
|
||||
}
|
||||
|
||||
// ======================== 文案辅助 ========================
|
||||
|
||||
+22
-18
@@ -29,11 +29,11 @@ import java.util.Map;
|
||||
* 站内通知扫描:任务失败聚合提醒 + 下游服务健康探测。
|
||||
*
|
||||
* <p>任务失败:回看窗口内进入失败终态的任务(biz_file_task / brand_crawl_tasks),
|
||||
* 按「用户 × 模块 × 小时」聚合,同一小时桶内增量刷新同一条通知(不刷屏);
|
||||
* 按「用户 × 模块 × 自然日」聚合,同一天内增量刷新同一条通知(不刷屏);
|
||||
* 用户侧推给任务创建者,管理员侧按数据权限推给管辖该用户的管理员。
|
||||
*
|
||||
* <p>服务探测:品牌检测服务(15126) / 跟价任务 API(18960) / jikip 代理接口,
|
||||
* 失败立即重试一次(过滤瞬抖),两次都失败才告警;同服务每小时最多一条。
|
||||
* 失败立即重试一次(过滤瞬抖),两次都失败才告警;同服务每天最多一条。
|
||||
*
|
||||
* <p>麦象异常扫描:见 {@link MaixiangAnomalyScanner}(批量/单任务停滞、任务失败、
|
||||
* 批量任务空结果、队列积压 → 管理员通知),与任务失败扫描共用本调度与分布式锁。
|
||||
@@ -55,7 +55,7 @@ public class NotificationScanScheduler {
|
||||
private static final int REASON_MAX_LENGTH = 120;
|
||||
private static final int PROBE_READ_TIMEOUT_MILLIS = 8_000;
|
||||
|
||||
private static final DateTimeFormatter HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH");
|
||||
private static final DateTimeFormatter DAY_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
|
||||
/**
|
||||
* 任务模块类型 → 中文名:直接取模块注册表(G6),新增模块只需在
|
||||
@@ -102,7 +102,7 @@ public class NotificationScanScheduler {
|
||||
}
|
||||
}
|
||||
|
||||
/** 任务失败聚合扫描:窗口内失败终态任务按「用户 × 模块 × 小时」聚合成一条通知。 */
|
||||
/** 任务失败聚合扫描:窗口内失败终态任务按「用户 × 模块 × 自然日」聚合成一条通知。 */
|
||||
void scanFailedTasks() {
|
||||
LocalDateTime cutoff = LocalDateTime.now()
|
||||
.minusMinutes(Math.max(1, properties.getTaskFailedWindowMinutes()));
|
||||
@@ -121,7 +121,7 @@ public class NotificationScanScheduler {
|
||||
continue;
|
||||
}
|
||||
failed.add(new FailedTask(task.getId(), displayTaskNo(task.getTaskNo(), task.getId()),
|
||||
task.getModuleType(), task.getUserId(), task.getErrorMessage()));
|
||||
task.getModuleType(), task.getUserId(), task.getErrorMessage(), task.getUpdatedAt()));
|
||||
}
|
||||
List<BrandCrawlTaskEntity> brandTasks = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||
.select(BrandCrawlTaskEntity::getId, BrandCrawlTaskEntity::getUserId,
|
||||
@@ -135,19 +135,18 @@ public class NotificationScanScheduler {
|
||||
continue;
|
||||
}
|
||||
failed.add(new FailedTask(task.getId(), "#" + task.getId(),
|
||||
BRAND_MODULE_TYPE, task.getUserId(), task.getErrorMessage()));
|
||||
BRAND_MODULE_TYPE, task.getUserId(), task.getErrorMessage(), task.getUpdatedAt()));
|
||||
}
|
||||
if (failed.isEmpty()) {
|
||||
log.info("[notification-scan] 近 {} 分钟无失败任务", properties.getTaskFailedWindowMinutes());
|
||||
return;
|
||||
}
|
||||
|
||||
String hour = LocalDateTime.now().format(HOUR_FORMAT);
|
||||
Map<BucketKey, List<FailedTask>> buckets = new LinkedHashMap<>();
|
||||
for (FailedTask task : failed) {
|
||||
String moduleType = normalizeModuleType(task.moduleType());
|
||||
buckets.computeIfAbsent(new BucketKey(task.userId(), moduleType, hour), key -> new ArrayList<>())
|
||||
.add(task);
|
||||
buckets.computeIfAbsent(new BucketKey(task.userId(), moduleType, dayOf(task.updatedAt())),
|
||||
key -> new ArrayList<>()).add(task);
|
||||
}
|
||||
|
||||
NotificationDispatchService.AdminAudience audience = notificationDispatchService.prepareAdminAudience();
|
||||
@@ -179,7 +178,7 @@ public class NotificationScanScheduler {
|
||||
adminPushed += notificationDispatchService.pushToAdmins(audience,
|
||||
NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
|
||||
"用户任务失败:" + username, adminContent,
|
||||
"task_failed_admin:" + key.userId() + ":" + key.moduleType() + ":" + key.hour(),
|
||||
"task_failed_admin:" + key.userId() + ":" + key.moduleType() + ":" + key.day(),
|
||||
key.userId());
|
||||
}
|
||||
log.info("[notification-scan] 任务失败扫描完成 窗口={}分钟 失败任务={} 聚合桶={} 用户通知={} 管理员通知={}",
|
||||
@@ -255,7 +254,7 @@ public class NotificationScanScheduler {
|
||||
}
|
||||
}
|
||||
|
||||
/** 探测结果上报:正常记 info;故障按小时去重推管理员通知;outcome=null 表示跳过。 */
|
||||
/** 探测结果上报:正常记 info;故障按自然日去重推管理员通知;outcome=null 表示跳过。 */
|
||||
private void reportProbe(String serviceKey, String serviceLabel, ProbeOutcome outcome) {
|
||||
if (outcome == null) {
|
||||
return;
|
||||
@@ -265,12 +264,11 @@ public class NotificationScanScheduler {
|
||||
return;
|
||||
}
|
||||
log.warn("[notification-scan] 服务探测失败 service={} detail={}", serviceLabel, outcome.detail());
|
||||
String hour = LocalDateTime.now().format(HOUR_FORMAT);
|
||||
notificationDispatchService.pushToAdmins(NotificationService.SCENE_SERVICE_DOWN,
|
||||
NotificationService.LEVEL_ERROR,
|
||||
serviceLabel + "不可用",
|
||||
serviceLabel + "探测失败:" + outcome.detail() + "。该服务相关任务可能大面积失败,请尽快排查。",
|
||||
"service_down:" + serviceKey + ":" + hour, null);
|
||||
"service_down:" + serviceKey + ":" + dayOf(LocalDateTime.now()), null);
|
||||
}
|
||||
|
||||
/** 通知保留期清理:每天最多一次;已读与未读各按自己的保留天数。 */
|
||||
@@ -287,7 +285,12 @@ public class NotificationScanScheduler {
|
||||
}
|
||||
|
||||
private String dedupeKeyOf(BucketKey key) {
|
||||
return "task_failed:" + key.userId() + ":" + key.moduleType() + ":" + key.hour();
|
||||
return "task_failed:" + key.userId() + ":" + key.moduleType() + ":" + key.day();
|
||||
}
|
||||
|
||||
/** 聚合桶的日期标签:取事件自身所属自然日,跨零点轮询时不会把昨天的失败挪进今天的桶。 */
|
||||
private String dayOf(LocalDateTime at) {
|
||||
return (at == null ? LocalDateTime.now() : at).format(DAY_FORMAT);
|
||||
}
|
||||
|
||||
private String taskNoPreview(List<FailedTask> tasks) {
|
||||
@@ -355,12 +358,13 @@ public class NotificationScanScheduler {
|
||||
return current.getClass().getSimpleName() + ": " + message;
|
||||
}
|
||||
|
||||
/** 聚合桶键:用户 × 模块 × 小时。 */
|
||||
record BucketKey(Long userId, String moduleType, String hour) {
|
||||
/** 聚合桶键:用户 × 模块 × 自然日。 */
|
||||
record BucketKey(Long userId, String moduleType, String day) {
|
||||
}
|
||||
|
||||
/** 统一视图的失败任务(两类表归一)。 */
|
||||
record FailedTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage) {
|
||||
/** 统一视图的失败任务(两类表归一);updatedAt 即失败终态时间,决定落入哪一天的桶。 */
|
||||
record FailedTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage,
|
||||
LocalDateTime updatedAt) {
|
||||
}
|
||||
|
||||
/** 探测结果。 */
|
||||
|
||||
+5
-5
@@ -107,10 +107,11 @@ public class NotificationService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 同键推送或刷新:不存在则新建;已存在且内容有变化时更新内容并重置为未读
|
||||
* (用于小时桶聚合的增量更新:同小时内新增失败任务时刷新计数与明细)。
|
||||
* 同键推送或刷新:不存在则新建;已存在且内容有变化时只更新文案,
|
||||
* <b>不动 read_at</b>(聚合通知的去重键按自然日,同一条一天内会被刷新很多次,
|
||||
* 重新点亮已读等于把红点做成闹钟)。
|
||||
*
|
||||
* @return true=新建或内容有更新(用户有新信息),false=无变化或去重未命中变化
|
||||
* @return true=新建或内容有更新,false=无变化
|
||||
*/
|
||||
public boolean pushOrRefresh(Long userId, String audience, String scene, String level,
|
||||
String title, String content, String dedupeKey) {
|
||||
@@ -131,8 +132,7 @@ public class NotificationService {
|
||||
.eq(UserNotificationEntity::getId, existing.getId())
|
||||
.set(UserNotificationEntity::getTitle, nextTitle)
|
||||
.set(UserNotificationEntity::getContent, nextContent)
|
||||
.set(UserNotificationEntity::getLevel, normalize(level).isEmpty() ? LEVEL_WARNING : level)
|
||||
.set(UserNotificationEntity::getReadAt, null));
|
||||
.set(UserNotificationEntity::getLevel, normalize(level).isEmpty() ? LEVEL_WARNING : level));
|
||||
log.info("[notification] 同键通知已刷新 id={} userId={} scene={} dedupeKey={}",
|
||||
existing.getId(), userId, scene, key);
|
||||
return true;
|
||||
|
||||
@@ -464,6 +464,8 @@ aiimage:
|
||||
unread-retention-days: ${AIIMAGE_NOTIFICATION_UNREAD_RETENTION_DAYS:180}
|
||||
security:
|
||||
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
||||
# 轮换过渡期用:逗号分隔的旧密钥,解密按顺序回试,加密只用上面那个;迁完就清空
|
||||
shop-credential-key-previous: ${AIIMAGE_SHOP_CREDENTIAL_KEY_PREVIOUS:}
|
||||
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
||||
internal-token-file: ${AIIMAGE_INTERNAL_TOKEN_FILE:}
|
||||
auth:
|
||||
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
package com.nanri.aiimage.common.security;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 店铺凭据/用户密钥加密器的轮换语义测试。
|
||||
* 关键契约:轮换后旧密文必须还能解(按 previous 回试),且全部解不开时是「回退原文」而不是抛错——
|
||||
* 这条回退会把密文当明文口令发出去,所以轮换必须配合重加密,见 crawler-plugin/docs/ops/credential-rotation-design-2026-10-02.md。
|
||||
*/
|
||||
class ShopCredentialCryptoServiceTest {
|
||||
|
||||
private static final String OLD_KEY = "old-key-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
private static final String NEW_KEY = "new-key-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
|
||||
private static final String THIRD_KEY = "third-key-cccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
|
||||
|
||||
private static ShopCredentialCryptoService newService(String rawKey, String previousKeys) throws Exception {
|
||||
ShopCredentialCryptoService service = new ShopCredentialCryptoService();
|
||||
set(service, "rawKey", rawKey);
|
||||
set(service, "previousKeys", previousKeys);
|
||||
service.init();
|
||||
return service;
|
||||
}
|
||||
|
||||
private static void set(Object target, String name, String value) throws Exception {
|
||||
Field field = target.getClass().getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<Object> previousOf(ShopCredentialCryptoService service) throws Exception {
|
||||
Field field = service.getClass().getDeclaredField("previousKeySpecs");
|
||||
field.setAccessible(true);
|
||||
return (List<Object>) field.get(service);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rotates_key_but_still_decrypts_legacy_ciphertext() throws Exception {
|
||||
ShopCredentialCryptoService old = newService(OLD_KEY, "");
|
||||
ShopCredentialCryptoService rotated = newService(NEW_KEY, OLD_KEY);
|
||||
String legacy = old.encrypt("P@ssw0rd-shopA");
|
||||
|
||||
assertEquals("P@ssw0rd-shopA", rotated.decrypt(legacy));
|
||||
assertNotEquals(old.encrypt("same-input"), rotated.encrypt("same-input"), "加密只用当前密钥");
|
||||
assertEquals("same-input", rotated.decrypt(rotated.encrypt("same-input")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void falls_back_to_the_input_when_no_key_matches() throws Exception {
|
||||
String legacy = newService(OLD_KEY, "").encrypt("P@ssw0rd-shopA");
|
||||
ShopCredentialCryptoService wrong = newService(NEW_KEY, THIRD_KEY);
|
||||
|
||||
assertEquals(legacy, wrong.decrypt(legacy), "回退原文是既有兼容行为,不能借轮换悄悄改成抛错");
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacy_plaintext_values_pass_through_unchanged() throws Exception {
|
||||
ShopCredentialCryptoService service = newService(NEW_KEY, "");
|
||||
|
||||
assertEquals("not-a-ciphertext!!", service.decrypt("not-a-ciphertext!!"));
|
||||
assertEquals("", service.decrypt(null));
|
||||
assertEquals("", service.decrypt(" "));
|
||||
}
|
||||
|
||||
@Test
|
||||
void behaves_exactly_as_before_when_previous_keys_is_empty() throws Exception {
|
||||
ShopCredentialCryptoService service = newService(NEW_KEY, "");
|
||||
String cipher = service.encrypt("v");
|
||||
|
||||
assertEquals("v", service.decrypt(cipher));
|
||||
assertTrue(previousOf(service).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void drops_previous_keys_that_equal_the_current_key() throws Exception {
|
||||
ShopCredentialCryptoService service = newService(NEW_KEY, NEW_KEY + " , ," + NEW_KEY);
|
||||
|
||||
assertTrue(previousOf(service).isEmpty(), "重复密钥白跑一遍解密,还会掩盖 previous 其实没配");
|
||||
}
|
||||
|
||||
@Test
|
||||
void tries_several_previous_keys_in_order() throws Exception {
|
||||
String fromThird = newService(THIRD_KEY, "").encrypt("v");
|
||||
ShopCredentialCryptoService service = newService(NEW_KEY, OLD_KEY + "," + THIRD_KEY);
|
||||
|
||||
assertEquals("v", service.decrypt(fromThird));
|
||||
assertEquals(2, previousOf(service).size());
|
||||
}
|
||||
}
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.service.ResultDownloadResolver;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 「没变过的分片,一次检测里只解析一遍」。
|
||||
*
|
||||
* <p>合批后一次 {@code submitLlmBatches} 要跑 N 批,而每批落库前都得先知道「这些行属于哪个分片」。
|
||||
* 这份索引此前是每批现做:把该任务**全部**分片的载荷从对象存储重读一遍、JSON 重解析一遍。
|
||||
* 线上一个 211 片的任务光索引就每批 211 次读,批数一多就是批数×分片数的平方级浪费,
|
||||
* 而这些读全都在 LLM 请求的串行路径上——检测阶段的墙钟时间有一半花在这里。
|
||||
*
|
||||
* <p>这里钉住三件事:载荷哈希没变过的分片不再解析(省下来的那部分)、
|
||||
* 变过的一定要从头解析(不然索引会指向已经不存在的行,跨节点并发回传时就会把结果写错片)、
|
||||
* 命中复用时行照常能匹配上自己的分片(省读省成返回空集合,就是"扣次数没结论")。
|
||||
*
|
||||
* <p>索引的作用域是单次调用(局部变量,随栈丢弃),这一条不做断言:收集候选那一遍本来就会
|
||||
* 重读并重记每个分片,就算有人把它做成字段也不会有可见行为差别,能咬住它的只有内存占用和并发。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AppearancePatentChunkRowIndexTest {
|
||||
|
||||
private static final Long TASK_ID = 29501L;
|
||||
private static final Long JOB_ID = 20801L;
|
||||
private static final String SCOPE_A = "scope-a";
|
||||
private static final String SCOPE_B = "scope-b";
|
||||
private static final String SCOPE_C = "scope-c";
|
||||
private static final String POINTER_A = "ptr-chunk-a.json";
|
||||
private static final String POINTER_B = "ptr-chunk-b.json";
|
||||
private static final String POINTER_C = "ptr-chunk-c.json";
|
||||
private static final String HASH_A = "hash-a-1";
|
||||
private static final String HASH_B = "hash-b-1";
|
||||
private static final String HASH_C = "hash-c-1";
|
||||
/** 本批写入后分片 A 在库里的新哈希(模拟 mergeChunkPayload 的 CAS 生效)。 */
|
||||
private static final String HASH_A_WRITTEN = "hash-a-2";
|
||||
|
||||
/** 6 行未检测:batchSize=2 时正好 3 批,批数大于 1 才看得见「每批重读全部分片」。 */
|
||||
private static final String PENDING_ROWS = rows(6, "r", false);
|
||||
/** 已检测完的分片:不产生候选、也不会被本次调用写入,是「纯粹没变过」的那两片。 */
|
||||
private static final String DONE_ROWS_B = rows(3, "b", true);
|
||||
private static final String DONE_ROWS_C = rows(3, "c", true);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private ResultDownloadResolver resultDownloadResolver;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private AppearancePatentLlmClient llmClient;
|
||||
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||
@Mock private AppearancePatentProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
@Mock private UserApiSecretService userApiSecretService;
|
||||
|
||||
@InjectMocks private AppearancePatentTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.getLlmBatchSize()).thenReturn(2);
|
||||
lenient().when(properties.getFlushPendingMinutes()).thenReturn(1);
|
||||
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||
lenient().when(userApiSecretService.findPlainValue(any(), anyString())).thenReturn("");
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_A), anyString())).thenReturn(PENDING_ROWS);
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString())).thenReturn(DONE_ROWS_B);
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_C), anyString())).thenReturn(DONE_ROWS_C);
|
||||
// 库里每批都重新查一遍:只有 A 因为刚落库换了哈希,B/C 一个字节都没动
|
||||
lenient().when(taskChunkMapper.selectList(any()))
|
||||
.thenReturn(List.of(chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
|
||||
chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, HASH_C)));
|
||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk(SCOPE_A, POINTER_A, HASH_A));
|
||||
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
anyString(), anyLong(), anyString(), any(), anyString())).thenReturn("sibling-1");
|
||||
lenient().when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
lenient().when(llmClient.inspectRows(any(), any(), any())).thenAnswer(invocation -> {
|
||||
List<AppearancePatentResultRowDto> batch = invocation.getArgument(0);
|
||||
for (AppearancePatentResultRowDto row : batch) {
|
||||
row.setTitleRisk("无侵权");
|
||||
row.setAppearanceRisk("无侵权");
|
||||
row.setConclusion("无侵权");
|
||||
}
|
||||
return new ArrayList<>(batch);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次调用 3 批,从头到尾没被改过的分片 C 只应该被解析一次(收集候选那一遍)。
|
||||
*
|
||||
* <p>钉住的是「省下来的那部分」:现在每批都要为 C 重读重解析一遍,3 批 + 收集 = 4 次。
|
||||
*/
|
||||
@Test
|
||||
void chunkThatNeverChangedIsParsedOncePerRun() {
|
||||
submit();
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).resolvePayload(eq(POINTER_C), anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 复用索引不能只省读、还得照常认得别的分片的行:第 4 批的结果照样要落进 B,而不是变成孤儿。
|
||||
*
|
||||
* <p>省读的实现有个致命的顺手写法——命中缓存时返回空集合或只返回「本批候选所在片」,
|
||||
* 表现是行匹配不上分片、被撤销标记反复重发,用户侧就是扣了次数没结论。
|
||||
*/
|
||||
@Test
|
||||
void reusedIndexStillMatchesRowsOfEveryChunk() {
|
||||
when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString()))
|
||||
.thenReturn(rows(2, "b", false));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||
chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
|
||||
chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, HASH_C)));
|
||||
|
||||
submit();
|
||||
|
||||
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("r1", "r2"));
|
||||
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("b1", "b2"));
|
||||
verify(taskCacheService, never()).releaseRowInspection(eq(TASK_ID), any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 载荷哈希一变就必须重新解析——省读不能省成「拿旧行集合匹配新载荷」。
|
||||
*
|
||||
* <p>收集候选用的是入口那份 chunk 快照(哈希还是 hash-c-1),批次里查库拿到的是别的节点
|
||||
* 回传之后的 hash-c-2。此时 C 的行集合已经不可信,必须重读。
|
||||
*/
|
||||
@Test
|
||||
void chunkWhosePayloadHashChangedIsParsedAgain() {
|
||||
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
|
||||
task(), new FileResultEntity(), job(),
|
||||
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, "hash-c-stale")),
|
||||
Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(2)).resolvePayload(eq(POINTER_C), anyString());
|
||||
}
|
||||
|
||||
// ===== 辅助 =====
|
||||
|
||||
private void submit() {
|
||||
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
|
||||
task(), new FileResultEntity(), job(),
|
||||
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, HASH_C)),
|
||||
Map.of());
|
||||
}
|
||||
|
||||
private FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
task.setUserId(7L);
|
||||
return task;
|
||||
}
|
||||
|
||||
private com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job() {
|
||||
com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job =
|
||||
new com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity();
|
||||
job.setId(JOB_ID);
|
||||
job.setTaskId(TASK_ID);
|
||||
return job;
|
||||
}
|
||||
|
||||
private TaskChunkEntity chunk(String scopeHash, String pointer, String payloadHash) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(11L);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(1);
|
||||
chunk.setPayloadJson(pointer);
|
||||
chunk.setPayloadHash(payloadHash);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static String rows(int count, String prefix, boolean resolved) {
|
||||
StringBuilder json = new StringBuilder("[");
|
||||
for (int i = 1; i <= count; i++) {
|
||||
if (i > 1) {
|
||||
json.append(',');
|
||||
}
|
||||
json.append("{\"rowToken\":\"").append(prefix).append(i)
|
||||
.append("\",\"asin\":\"B0A000000").append(i % 10)
|
||||
.append("\",\"title\":\"PRODUCT ").append(i)
|
||||
.append("\",\"url\":\"http://img/").append(i).append(".jpg\",")
|
||||
.append("\"groupKey\":\"").append(prefix).append("-g").append(i).append("\"");
|
||||
if (resolved) {
|
||||
json.append(",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"");
|
||||
}
|
||||
json.append('}');
|
||||
}
|
||||
return json.append(']').toString();
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -135,7 +135,7 @@ class AppearancePatentRowMarkTimingTest {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
|
||||
|
||||
assertEquals(2, candidates.size());
|
||||
verify(taskCacheService).beginRowInspection(TASK_ID, "r1");
|
||||
@@ -151,7 +151,7 @@ class AppearancePatentRowMarkTimingTest {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r2"))).thenReturn(false);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
|
||||
|
||||
assertEquals(1, candidates.size());
|
||||
assertEquals("r1", candidates.get(0).getRowToken());
|
||||
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 「任务成功但有一部分行没跑出来」通知文案(2026-10-02)。
|
||||
*
|
||||
* <p>背景:后台通知里这条写死了「品牌检测服务超时或被限流」,而 10-02 取证出的真实根因是品牌链
|
||||
* 上游的采集代理取不到 IP(余额不足)与 503 —— 代码在这个位置只观测到「品牌检测没返回结果」,
|
||||
* 观测不到原因。写死一个猜测的原因会把排查方向从代理/上游带到自身服务上。
|
||||
*
|
||||
* <p>锁定契约:文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效、可重跑),
|
||||
* 不指认代码没有依据的具体故障原因;真实原因留在 BrandCheckClient 的 warn 日志里
|
||||
* (brand-check 的 lastErr 字段),由运维按日志排查。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class CollectDataPartialResultNoticeTest {
|
||||
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private CollectDataExcelAssemblyService excelAssemblyService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private NotificationDispatchService notificationDispatchService;
|
||||
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@InjectMocks private CollectDataService service;
|
||||
|
||||
private static final long TASK_ID = 29991L;
|
||||
|
||||
/** LambdaUpdateWrapper 需要实体的列缓存,纯单测里没有 Spring 容器要手工初始化。 */
|
||||
@BeforeAll
|
||||
static void initTableInfo() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void cleanWorkDir() {
|
||||
FileUtil.del(new File(System.getProperty("java.io.tmpdir"), "collect-data-result/" + TASK_ID));
|
||||
}
|
||||
|
||||
private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("COLLECT_DATA");
|
||||
task.setStatus("RUNNING");
|
||||
task.setTaskNo("CD-29991");
|
||||
task.setUserId(7L);
|
||||
task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows + "}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private TaskFileJobEntity job() {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setTaskId(TASK_ID);
|
||||
job.setResultId(5001L);
|
||||
return job;
|
||||
}
|
||||
|
||||
private FileResultEntity result() {
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(5001L);
|
||||
result.setModuleType("COLLECT_DATA");
|
||||
result.setSourceFilename("keywords.xlsx");
|
||||
return result;
|
||||
}
|
||||
|
||||
private String stubHappyPathAndCaptureDetail() {
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
|
||||
when(fileResultMapper.selectById(5001L)).thenReturn(result());
|
||||
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
|
||||
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
|
||||
// 条件更新返回 1:任务翻成成功,这条「部分缺失」通知才会推给用户
|
||||
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
|
||||
|
||||
service.processResultFileJob(job());
|
||||
|
||||
ArgumentCaptor<String> detail = ArgumentCaptor.forClass(String.class);
|
||||
verify(notificationDispatchService).pushPartialResult(eq(7L), eq(TASK_ID), eq("COLLECT_DATA"),
|
||||
eq("CD-29991"), detail.capture());
|
||||
return detail.getValue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void 部分结果通知只说可观测事实_不指认猜测的故障原因() {
|
||||
String detail = stubHappyPathAndCaptureDetail();
|
||||
|
||||
assertThat(detail).contains("7 行");
|
||||
// 「超时」「被限流」都是这一层观测不到的原因,写进文案会把排查方向带偏
|
||||
assertThat(detail).doesNotContain("超时");
|
||||
assertThat(detail).doesNotContain("限流");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 部分结果通知保留用户能自己核对的语义() {
|
||||
String detail = stubHappyPathAndCaptureDetail();
|
||||
|
||||
assertThat(detail).contains("不计入结果条数");
|
||||
assertThat(detail).contains("未返回结果");
|
||||
assertThat(detail).contains("重跑");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 任务已被判失败时不推部分结果通知() {
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
|
||||
when(fileResultMapper.selectById(5001L)).thenReturn(result());
|
||||
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
|
||||
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
|
||||
when(fileTaskMapper.update(isNull(), any())).thenReturn(0);
|
||||
|
||||
service.processResultFileJob(job());
|
||||
|
||||
verify(notificationDispatchService, org.mockito.Mockito.never())
|
||||
.pushPartialResult(anyLong(), anyLong(), anyString(), anyString(), anyString());
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -162,7 +162,7 @@ class MaixiangAnomalyScannerTest {
|
||||
content.capture(), key.capture(), isNull());
|
||||
assertThat(content.getValue()).contains("1 条任务失败");
|
||||
assertThat(content.getValue()).contains("未注册的任务类型: price-track-run");
|
||||
assertThat(key.getValue()).isEqualTo("maixiang_fail:2026091512");
|
||||
assertThat(key.getValue()).isEqualTo("maixiang_fail:20260915");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -238,7 +238,7 @@ class MaixiangAnomalyScannerTest {
|
||||
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
|
||||
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:麦象队列积压"),
|
||||
content.capture(), eq("maixiang_queue_backlog:2026091512"), isNull());
|
||||
content.capture(), eq("maixiang_queue_backlog:20260915"), isNull());
|
||||
assertThat(content.getValue()).contains("L1 待处理 210 / 处理中 0").contains("堆积且无人在消费");
|
||||
}
|
||||
|
||||
@@ -263,7 +263,7 @@ class MaixiangAnomalyScannerTest {
|
||||
newScanner().checkQueueBacklog(audience, NOW);
|
||||
|
||||
verify(dispatch).pushToAdmins(any(), anyString(), anyString(), anyString(), anyString(),
|
||||
eq("maixiang_queue_backlog:2026091512"), any());
|
||||
eq("maixiang_queue_backlog:20260915"), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -292,7 +292,7 @@ class MaixiangAnomalyScannerTest {
|
||||
ArgumentCaptor<String> content = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(eq(audience), eq(NotificationService.SCENE_MAIXIANG_ANOMALY),
|
||||
eq(NotificationService.LEVEL_WARNING), eq("麦象异常:采集机掉线"),
|
||||
content.capture(), eq("maixiang_silent:2026091512"), isNull());
|
||||
content.capture(), eq("maixiang_silent:20260915"), isNull());
|
||||
assertThat(content.getValue())
|
||||
.contains("静默 11 分钟")
|
||||
.contains("跟价 静默 11 分钟")
|
||||
|
||||
+40
-6
@@ -15,6 +15,7 @@ import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@@ -64,13 +65,13 @@ class NotificationDispatchServiceTest {
|
||||
|
||||
NotificationDispatchService.AdminAudience audience = service.prepareAdminAudience();
|
||||
int pushed = service.pushToAdmins(audience, NotificationService.SCENE_TASK_FAILED,
|
||||
NotificationService.LEVEL_WARNING, "标题", "内容", "task_failed_admin:20:PRICE_TRACK:2026091310", 20L);
|
||||
NotificationService.LEVEL_WARNING, "标题", "内容", "task_failed_admin:20:PRICE_TRACK:20260913", 20L);
|
||||
|
||||
// 主管乙不可见用户 20 → 只有超管收到
|
||||
assertThat(pushed).isEqualTo(1);
|
||||
org.mockito.Mockito.verify(notificationService).pushOrRefresh(eq(1L), eq(NotificationService.AUDIENCE_ADMIN),
|
||||
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
|
||||
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:2026091310:1"));
|
||||
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:20260913:1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -89,21 +90,54 @@ class NotificationDispatchServiceTest {
|
||||
|
||||
int pushed = service.pushToAdmins(NotificationService.SCENE_TASK_FAILED,
|
||||
NotificationService.LEVEL_WARNING, "标题", "内容",
|
||||
"task_failed_admin:20:PRICE_TRACK:2026091310", 20L);
|
||||
"task_failed_admin:20:PRICE_TRACK:20260913", 20L);
|
||||
|
||||
// 超管全量 + 本组主管各落一条,各带自己的 dedupe 后缀(已读状态互不影响)
|
||||
assertThat(pushed).isEqualTo(2);
|
||||
verify(notificationService).pushOrRefresh(eq(1L), eq(NotificationService.AUDIENCE_ADMIN),
|
||||
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
|
||||
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:2026091310:1"));
|
||||
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:20260913:1"));
|
||||
verify(notificationService).pushOrRefresh(eq(2L), eq(NotificationService.AUDIENCE_ADMIN),
|
||||
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
|
||||
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:2026091310:2"));
|
||||
eq("标题"), eq("内容"), eq("task_failed_admin:20:PRICE_TRACK:20260913:2"));
|
||||
// 非本组主管一条都不落
|
||||
verify(notificationService, never()).pushOrRefresh(eq(3L), anyString(), anyString(),
|
||||
anyString(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 合并告警的权限口径:能看到组内任一号的管理员收 1 条,一个都看不到的收不到 —— 每人最多一条。
|
||||
* 逐个主体调用 pushToAdmins 会让同一管理员为 N 个账号收到 N 条同内容通知。
|
||||
*/
|
||||
@Test
|
||||
void pushToAdminsAnyOfDeliversAtMostOnePerAdminSeeingAnySubject() {
|
||||
AdminUserEntity superAdmin = admin(1L, "超管甲");
|
||||
AdminUserEntity inScope = admin(2L, "主管乙");
|
||||
AdminUserEntity outOfScope = admin(3L, "主管丙");
|
||||
when(adminUserMapper.selectList(any())).thenReturn(List.of(superAdmin, inScope, outOfScope));
|
||||
when(adminAuthSupport.currentRole(superAdmin)).thenReturn("super_admin");
|
||||
when(adminAuthSupport.currentRole(inScope)).thenReturn("admin");
|
||||
when(adminAuthSupport.currentRole(outOfScope)).thenReturn("admin");
|
||||
when(userDataScopeSupport.resolveVisibleUserIds(2L)).thenReturn(List.of(2L, 20L));
|
||||
when(userDataScopeSupport.resolveVisibleUserIds(3L)).thenReturn(List.of(3L, 30L));
|
||||
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||
.thenReturn(true);
|
||||
|
||||
int pushed = service.pushToAdminsAnyOf(service.prepareAdminAudience(),
|
||||
NotificationService.SCENE_SECRET_BALANCE, NotificationService.LEVEL_ERROR,
|
||||
"标题", "内容", "secret_balance_admin:group:hash:20261002", List.of(11L, 12L, 20L));
|
||||
|
||||
// 3 个受影响账号 × 2 个可见管理员 = 只落 2 条;非本组主管一条都不落
|
||||
assertThat(pushed).isEqualTo(2);
|
||||
verify(notificationService, times(2)).pushOrRefresh(anyLong(), anyString(), anyString(),
|
||||
anyString(), anyString(), anyString(), anyString());
|
||||
verify(notificationService).pushOrRefresh(eq(2L), eq(NotificationService.AUDIENCE_ADMIN),
|
||||
eq(NotificationService.SCENE_SECRET_BALANCE), eq(NotificationService.LEVEL_ERROR),
|
||||
eq("标题"), eq("内容"), eq("secret_balance_admin:group:hash:20261002:2"));
|
||||
verify(notificationService, never()).pushOrRefresh(eq(3L), anyString(), anyString(),
|
||||
anyString(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void globalEventReachesEveryAdminByDesign() {
|
||||
AdminUserEntity superAdmin = admin(1L, "超管甲");
|
||||
@@ -120,7 +154,7 @@ class NotificationDispatchServiceTest {
|
||||
// 服务探测失败这类全局事件(subjectUserId=null)按设计推给所有管理员,不受分组限制
|
||||
int pushed = service.pushToAdmins(NotificationService.SCENE_SERVICE_DOWN,
|
||||
NotificationService.LEVEL_ERROR, "品牌检测服务不可用", "探测失败",
|
||||
"service_down:brand-service:2026091310", null);
|
||||
"service_down:brand-service:20260913", null);
|
||||
|
||||
assertThat(pushed).isEqualTo(3);
|
||||
}
|
||||
|
||||
+83
-1
@@ -117,7 +117,89 @@ class NotificationScanSchedulerTest {
|
||||
anyString(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 同一天同模块的失败必须共用一条通知:按「扫描时刻的小时」分桶时,同一个失败任务
|
||||
* 只要还落在 60 分钟回看窗口里,跨过整点就会被塞进新桶,于是每小时重发一条
|
||||
* (2026-10-02 取证:task_failed 管理员侧因此 24 倍放大)。
|
||||
*/
|
||||
@Test
|
||||
void failedTasksOfSameDayShareOneDedupeKeyAcrossHours() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(
|
||||
fileTask(101L, "T-101", "PRICE_TRACK", 7L, "浏览器启动失败", LocalDateTime.of(2026, 10, 1, 9, 12)),
|
||||
fileTask(102L, "T-102", "PRICE_TRACK", 7L, "cookie 失效", LocalDateTime.of(2026, 10, 1, 18, 36))));
|
||||
when(brandCrawlTaskMapper.selectList(any())).thenReturn(List.of());
|
||||
stubAdminAudience();
|
||||
|
||||
newScheduler().scanFailedTasks();
|
||||
|
||||
ArgumentCaptor<String> userKey = ArgumentCaptor.forClass(String.class);
|
||||
verify(notificationService, times(1)).pushOrRefresh(eq(7L), eq(NotificationService.AUDIENCE_USER),
|
||||
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
|
||||
anyString(), anyString(), userKey.capture());
|
||||
assertThat(userKey.getValue()).isEqualTo("task_failed:7:PRICE_TRACK:20261001");
|
||||
|
||||
ArgumentCaptor<String> adminKey = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch, times(1)).pushToAdmins(any(NotificationDispatchService.AdminAudience.class),
|
||||
eq(NotificationService.SCENE_TASK_FAILED), eq(NotificationService.LEVEL_WARNING),
|
||||
anyString(), anyString(), adminKey.capture(), eq(7L));
|
||||
assertThat(adminKey.getValue()).isEqualTo("task_failed_admin:7:PRICE_TRACK:20261001");
|
||||
}
|
||||
|
||||
/** 桶按「任务自身所属自然日」而非扫描时钟:跨零点轮询时,昨晚的失败仍归昨晚那条通知。 */
|
||||
@Test
|
||||
void failedTasksOnDifferentDaysGetSeparateDedupeKeys() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(
|
||||
fileTask(101L, "T-101", "PRICE_TRACK", 7L, "浏览器启动失败", LocalDateTime.of(2026, 10, 1, 23, 50)),
|
||||
fileTask(102L, "T-102", "PRICE_TRACK", 7L, "cookie 失效", LocalDateTime.of(2026, 10, 2, 0, 10))));
|
||||
when(brandCrawlTaskMapper.selectList(any())).thenReturn(List.of());
|
||||
stubAdminAudience();
|
||||
|
||||
newScheduler().scanFailedTasks();
|
||||
|
||||
ArgumentCaptor<String> userKey = ArgumentCaptor.forClass(String.class);
|
||||
verify(notificationService, times(2)).pushOrRefresh(anyLong(), anyString(), anyString(), anyString(),
|
||||
anyString(), anyString(), userKey.capture());
|
||||
assertThat(userKey.getAllValues()).containsExactlyInAnyOrder(
|
||||
"task_failed:7:PRICE_TRACK:20261001", "task_failed:7:PRICE_TRACK:20261002");
|
||||
}
|
||||
|
||||
/**
|
||||
* 服务探测故障:去重键按天,同一天同服务只提醒一次(小时桶时一个持续 503 的上游
|
||||
* 每小时给每位管理员各塞一条)。取精确 8 位日期,多一位小时都不行。
|
||||
*/
|
||||
@Test
|
||||
void serviceDownAlertIsDedupedPerCalendarDay() {
|
||||
properties.setBrandServiceUrl("http://127.0.0.1:1");
|
||||
properties.setPriceTrackApiUrl("");
|
||||
properties.setJikipProbeEnabled(false);
|
||||
when(dispatch.pushToAdmins(anyString(), anyString(), anyString(), anyString(), anyString(), any()))
|
||||
.thenReturn(1);
|
||||
|
||||
newScheduler().probeServices();
|
||||
|
||||
ArgumentCaptor<String> keyBase = ArgumentCaptor.forClass(String.class);
|
||||
verify(dispatch).pushToAdmins(eq(NotificationService.SCENE_SERVICE_DOWN),
|
||||
eq(NotificationService.LEVEL_ERROR), eq("品牌检测服务不可用"), anyString(),
|
||||
keyBase.capture(), org.mockito.ArgumentMatchers.<Long>isNull());
|
||||
assertThat(keyBase.getValue()).matches("service_down:brand-service:\\d{8}");
|
||||
}
|
||||
|
||||
private void stubAdminAudience() {
|
||||
when(dispatch.prepareAdminAudience()).thenReturn(
|
||||
new NotificationDispatchService.AdminAudience(List.of(), Set.of(), Map.of()));
|
||||
when(dispatch.displayNameOf(anyLong())).thenReturn("张三");
|
||||
when(notificationService.pushOrRefresh(anyLong(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString()))
|
||||
.thenReturn(true);
|
||||
when(dispatch.pushToAdmins(any(NotificationDispatchService.AdminAudience.class), anyString(), anyString(),
|
||||
anyString(), anyString(), anyString(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
private FileTaskEntity fileTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage) {
|
||||
return fileTask(id, taskNo, moduleType, userId, errorMessage, LocalDateTime.of(2026, 10, 1, 18, 36));
|
||||
}
|
||||
|
||||
private FileTaskEntity fileTask(Long id, String taskNo, String moduleType, Long userId, String errorMessage,
|
||||
LocalDateTime updatedAt) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setTaskNo(taskNo);
|
||||
@@ -125,7 +207,7 @@ class NotificationScanSchedulerTest {
|
||||
task.setUserId(userId);
|
||||
task.setErrorMessage(errorMessage);
|
||||
task.setStatus("FAILED");
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(updatedAt);
|
||||
return task;
|
||||
}
|
||||
}
|
||||
|
||||
+16
-5
@@ -84,23 +84,34 @@ class NotificationServiceTest {
|
||||
verify(mapper, never()).insert(any(UserNotificationEntity.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 聚合通知按自然日只有一条,一整天会被反复刷新:内容有变化时只改文案,
|
||||
* 绝不能把 read_at 置空——否则管理员刚读完红点又亮,一天点亮几十次
|
||||
* (2026-10-02 后台超管 582 条未读的主放大器之一)。
|
||||
*/
|
||||
@Test
|
||||
void pushOrRefreshUpdatesContentAndResetsUnread() {
|
||||
@SuppressWarnings("unchecked")
|
||||
void pushOrRefreshUpdatesContentButKeepsReadState() {
|
||||
UserNotificationEntity existing = new UserNotificationEntity();
|
||||
existing.setId(9L);
|
||||
existing.setTitle("跟价任务失败");
|
||||
existing.setContent("最近 60 分钟内有 2 个跟价任务失败");
|
||||
existing.setLevel("warning");
|
||||
existing.setReadAt(LocalDateTime.now());
|
||||
existing.setReadAt(LocalDateTime.of(2026, 9, 13, 10, 0));
|
||||
when(mapper.selectOne(any())).thenReturn(existing);
|
||||
|
||||
boolean refreshed = service.pushOrRefresh(7L, NotificationService.AUDIENCE_ADMIN,
|
||||
NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
|
||||
"跟价任务失败", "最近 60 分钟内有 5 个跟价任务失败", "task_failed:7:PRICE_TRACK:2026091310");
|
||||
"跟价任务失败", "最近 60 分钟内有 5 个跟价任务失败", "task_failed:7:PRICE_TRACK:20260913");
|
||||
|
||||
assertThat(refreshed).isTrue();
|
||||
verify(mapper, never()).insert(any(UserNotificationEntity.class));
|
||||
verify(mapper, times(1)).update(any(), any());
|
||||
ArgumentCaptor<LambdaUpdateWrapper<UserNotificationEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(mapper, times(1)).update(any(), captor.capture());
|
||||
String sqlSet = captor.getValue().getSqlSet();
|
||||
assertThat(sqlSet).contains("title").contains("content").contains("level");
|
||||
assertThat(sqlSet).doesNotContain("read_at");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -113,7 +124,7 @@ class NotificationServiceTest {
|
||||
|
||||
boolean refreshed = service.pushOrRefresh(7L, NotificationService.AUDIENCE_USER,
|
||||
NotificationService.SCENE_TASK_FAILED, NotificationService.LEVEL_WARNING,
|
||||
"跟价任务失败", "最近 60 分钟内有 2 个跟价任务失败", "task_failed:7:PRICE_TRACK:2026091310");
|
||||
"跟价任务失败", "最近 60 分钟内有 2 个跟价任务失败", "task_failed:7:PRICE_TRACK:20260913");
|
||||
|
||||
assertThat(refreshed).isFalse();
|
||||
verify(mapper, never()).update(any(), any());
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
# 店铺凭据密钥轮换设计(2026-10-02)
|
||||
|
||||
范围:`aiimage.security.shop-credential-key` 这一把密钥的轮换,以及轮换时三张库里凭据表怎么处理。
|
||||
本文只给设计、取证结果和待办,**没有执行过任何轮换或重启**。
|
||||
|
||||
## 一、为什么现在不能直接换密钥
|
||||
|
||||
`ShopCredentialCryptoService.decrypt()` 解不出来时回退返回原文(`ShopCredentialCryptoService.java:76-97`)。
|
||||
调用方拿到这串原文就当明文口令往外发——紫鸟账号、1688 店铺、第三方 api_key 都是这样被使用的。
|
||||
所以「改了密钥、没重加密存量」这段时间里的表现不是报错,而是**拿着密文当口令去登录**,
|
||||
失败点离原因很远,最难查。
|
||||
|
||||
本次改动(只做防护,未动线上):
|
||||
|
||||
- `ShopCredentialCryptoService` 支持 `shop-credential-key-previous`(逗号分隔的旧密钥),解密时当前密钥失败后按顺序回试旧密钥,加密只用当前密钥。
|
||||
- 回退不再静默:打 WARN,写明「当前密钥 + N 个历史密钥都试过」,并直接说出后果(这串密文会被当成明文口令发出去)。
|
||||
|
||||
## 二、生效值到底配在哪一层(今天的实测)
|
||||
|
||||
| 键 | 位置 | 读数 |
|
||||
| --- | --- | --- |
|
||||
| `shop-credential-key` | 容器 `/app/application-server.yml`(宿主 `/app/java/`) | `configured=yes len=64 sha8=190cc85c occurrences=1`,**两台一致** |
|
||||
| `jwt-secret` | 同上 | `len=64 sha8=2d40fe8a`,**两台一致** |
|
||||
| `AIIMAGE_SHOP_CREDENTIAL_KEY` | 宿主 `/.env`、java 进程 environ | **两处都没有** |
|
||||
| `AIIMAGE_JWT_SECRET` | 同上 | **两处都没有** |
|
||||
| `AIIMAGE_DB_PASSWORD` | 宿主 `/.env`(= java 进程 environ) | `len=16 sha8=83809acd`,两台一致 |
|
||||
|
||||
**结论:这两把密钥的生效值只有 yml 一个来源,没有环境变量覆盖它。** 之前按「改 `/.env` 追加 export」的
|
||||
配方去轮换这两把是无效的——那条配方只对 `/.env` 里真实存在的那些键有效。改密钥就改
|
||||
`/app/java/application-server.yml`,两台各一份,改完 `sha8` 应当同步变化。
|
||||
|
||||
复现命令(只读,值不出生产):
|
||||
|
||||
```
|
||||
MSYS2_ARG_CONV_EXCL='*' python scripts/_prod_config.py all --container java-server \
|
||||
--file /app/application-server.yml --key shop-credential-key
|
||||
```
|
||||
|
||||
## 三、存量行的形态:没有明文(本次实测)
|
||||
|
||||
`scripts/_probe_credential_decrypt_audit.sh` 在主机A 上用**当前密钥真解了一遍**三张表的七个凭据列,
|
||||
只输出计数。判据与 Java 一致:AES/ECB/PKCS5Padding,密钥 = SHA-256(原文密钥) 前 16 字节;
|
||||
并且必须过 base64 往返、16 字节对齐、填充合法、**解出来是合法 UTF-8** 四道关
|
||||
——只看长度和对齐是不够的,换错密钥也能解出「看着像样」的字节。
|
||||
|
||||
| 表.列 | 行数 | 空 | 解得出 | b64 非串 | 长度不对 | 填充不对 | 非 UTF-8 |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| `biz_shop_manage.password` | 86 | 0 | 86 | 0 | 0 | 0 | 0 |
|
||||
| `biz_user_api_secret.secret_value` | 314 | 0 | 314 | 0 | 0 | 0 | 0 |
|
||||
| `biz_image_video_secret.copy_api_key` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
|
||||
| `biz_image_video_secret.t8_key` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
|
||||
| `biz_image_video_secret.t8_video_key` | 7 | 2 | 5 | 0 | 0 | 0 | 0 |
|
||||
| `biz_image_video_secret.voice_api_key` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
|
||||
| `biz_image_video_secret.voice_group_id` | 7 | 0 | 7 | 0 | 0 | 0 | 0 |
|
||||
|
||||
三点用处:
|
||||
|
||||
1. **没有明文行**,435 行里 433 行是当前密钥解得出的密文,另 2 行是空值。所以重加密可以走纯 Java
|
||||
一条路,不需要「遇到解不出的行就跳过」的分支——那个分支的存在本身就等于把密文当明文写回去。
|
||||
2. 轮换前跑一次、轮换后跑一次,**`解得出` 的计数应当一模一样**,这是这批改动的验收标准。
|
||||
3. 写入路径本来就全过加密(`ShopManageService.java:100,124`、`UserApiSecretService.java:843`,
|
||||
两处 `setSecretValue`/`setPassword` 都包着 `encrypt(...)`),没有发现明文字段入口。
|
||||
|
||||
## 四、轮换顺序(每一步都能单独发版)
|
||||
|
||||
1. **先发版**:带双密钥回试的版本。此时 `previous` 为空,行为与旧版逐字节相同
|
||||
(`behaves_exactly_as_before_when_previous_keys_is_empty` 钉住这条)。
|
||||
2. **改配置**:两台 `/app/java/application-server.yml` 同时改——`shop-credential-key` 写新值、
|
||||
`shop-credential-key-previous` 写旧值。**这一步起新数据用新密钥加密,旧数据仍解得出。**
|
||||
回滚 = 把两行改回原值重启,比重加密便宜得多。
|
||||
3. **重加密存量**:跑一次迁移(下面第 5 节),把这 433 行用当前密钥重新加密。
|
||||
4. **验证**:再跑 `scripts/_probe_credential_decrypt_audit.sh`,`解得出` 仍是 433、其余四类仍为 0。
|
||||
5. **收尾**:清掉 `shop-credential-key-previous`,下次发版生效。留着不影响正确性,只是每次解密多一轮试。
|
||||
|
||||
顺序不能反:先改配置再发版的话,旧 jar 不认 `previous` 这个键,存量行立刻进回退分支。
|
||||
|
||||
## 五、重加密迁移的关键约束
|
||||
|
||||
- **只改密文,不改业务时间戳。** 这批表上有 `updated_at` 会被心跳/看板读,走实体更新会顺带刷新它。
|
||||
- **不要走业务 Service 的写入路径**(会带缓存刷新、索引重算、`ziniaoShopIndexRefreshTrigger` 之类的副作用)。
|
||||
用一条独立的取行→解密→用当前密钥加密→按主键回写的脚本,跑在 Java 侧。
|
||||
- 分表分批、每批独立事务;失败要能重跑(重加密天然幂等,重复跑只是再解一次)。
|
||||
- 迁移脚本**本次没写**,等你定了窗口再动。
|
||||
|
||||
## 六、不该走这条路的几把密钥
|
||||
|
||||
| 密钥 | 为什么另算 |
|
||||
| --- | --- |
|
||||
| `auth.jwt-secret`(sha8 `2d40fe8a`) | 换它 = 现有登录态全废,所有人被踢;必须单独定维护窗口,和重加密无关 |
|
||||
| `AIIMAGE_DB_PASSWORD`(sha8 `83809acd`) | 库口令,改它要同时动 `/.env` 与 MySQL 账号,属另一套流程 |
|
||||
| 极客图 / jikip / LLM 代理 key | 生效值在 yml 或 `biz_user_api_secret`,前者按普通配置轮换、后者随第五节的迁移一起走 |
|
||||
| MinIO(B 机设备日志、A 机业务桶) | 走各自的 `.env`/服务重启,与店铺凭据无关 |
|
||||
|
||||
## 七、验证做到哪一步
|
||||
|
||||
- 新增 `ShopCredentialCryptoServiceTest` 6 条,`mvn -o -Dtest=ShopCredentialCryptoServiceTest test`
|
||||
→ `Tests run: 6, Failures: 0, Errors: 0` + `BUILD SUCCESS`。
|
||||
- 变异检验:把 `candidates.addAll(previousKeySpecs)` 删掉后重跑 → `Tests run: 6, Failures: 2`
|
||||
(`rotates_key_but_still_decrypts_legacy_ciphertext`、`tries_several_previous_keys_in_order` 变红),
|
||||
说明这两条真的在钉双密钥回试,不是恒绿断言。
|
||||
(Python 侧的等价做法见 `scripts/tests/`:`umask 077` 挪到 `base64 -d >` 之后、或绕过 `upload_and_run_command` 直接拼命令,各有 1 条用例变红,改完的文件按 md5 原样还原。)
|
||||
- 这批测试走的是标准生命周期(`mvn -o -Dtest=ShopCredentialCryptoServiceTest test`)。
|
||||
上午有一段时间 `test-compile` 被并发会话新加的未跟踪测试挡住,当时只能用「javac 单编我的测试类 +
|
||||
`surefire:test` 单跑」绕过;09:5x 复跑时那条阻塞已经自己清了,没有动过别人的文件。
|
||||
- Python 侧:`scripts/tests` 共 78 例(`PYTHONUTF8=1 python -m pytest scripts/tests -q` 全绿)。
|
||||
掩码层那 8 个护栏点也各打坏过一次(出口不掩码、三层各自失效、`verdict` 把读不到当一致、裸标记不记录、MSYS 路径不拦、`atexit` 没注册),**8/8 都有用例变红**;其中「`atexit.register(_report)` 删掉」原本是漏网之鱼——直接调 `_report` 的用例看不见「没注册」这个状态,补了一条开子进程实跑的用例才咬住。
|
||||
|
||||
## 八、还欠什么
|
||||
|
||||
1. **轮换本身没执行**,新密钥由你定、yml 由你改(生产写操作按约定只交付命令,不代跑)。
|
||||
2. **重加密迁移脚本没写**——第五节的约束已经定好,等窗口。
|
||||
3. `ShopManageService.java:260` 往 VO 写的是 `masked`,读侧掩码是有的;**日志侧**这几把密钥有没有
|
||||
别的出口,本次没审。
|
||||
4. 外观/货源的 LLM key 分组那部分结论来自之前的实测记录(一把 key 平均挂 9.5 个账号),
|
||||
本次没复核,引它做决策前先重跑一遍当时的脚本。
|
||||
5. **两台 `/tmp` 里躺着 48 个 644 权限的历史探针副本**(主机A 36 个、主机B 12 个,其中 20 个含
|
||||
`password|secret|token|api_key` 形式的行)——那是 `_ssh_bash.py` 加 `rm -f` 收尾之前几次会话留下的。
|
||||
上传命令现在带 `umask 077`,新副本落成 600,但**旧的这 48 个还在**,删除算生产改动,命令给你:
|
||||
|
||||
```
|
||||
python scripts/_ssh_bash.py a scripts/清理用临时脚本 # 或直接
|
||||
MSYS2_ARG_CONV_EXCL='*' python scripts/_ssh_probe.py all 'rm -f /tmp/_run__*.sh; ls /tmp/_run__*.sh 2>/dev/null | wc -l'
|
||||
```
|
||||
|
||||
后面那条回读应当是 `0`。
|
||||
@@ -38,6 +38,15 @@ B 组是 DB / 对象存储的往返浪费(心跳整行读、删除路径的全
|
||||
|
||||
**风险**:分片在批中被别的入口改写时读到旧态。现状靠 `mergeChunkPayload` 的 CAS 兜住,改后仍要走同一 CAS,只省读不省写。行数与判定结果不变。
|
||||
|
||||
**状态(2026-10-02 10:2x):改法 1 已提交 `91d85ec6`,未发版。**
|
||||
索引改为「分片键 → (payload_hash, rowKey 集合)」,在 `collectPendingLlmCandidates` 那一遍就建好,
|
||||
批内 merge 只对 hash 变过的分片重读(`chunkRowKeySet`);载荷哈希与内容在所有写入点都是同一条 UPDATE 落的,可直接作新鲜度锚。
|
||||
落库路径一行未动,`mergeChunkPayload` 仍自己 `selectOne` + 重读最新载荷再 CAS,索引不参与合并基底。
|
||||
每批的 `loadSubmittedChunks`(DB SELECT 全部分片)保留——它就是锚,不能省。
|
||||
读的次数从 `(批数+1)×分片数` 降到 `分片数 + 本批写过的片数`。
|
||||
改法 2(`rowKey→chunkKey` 索引去掉逐片线性试探)**未做**:那是内存里的 O(C) 扫描,不是这次的瓶颈。
|
||||
测试 `AppearancePatentChunkRowIndexTest` 3 例,逐条做过变异检验;**货源侧 `SimilarAsinPipelineSupport.java:183-187` 是同一形态、还没改**。
|
||||
|
||||
### A2 外观专利:品牌校验合批 + 复用已有缓存 `[P0]`
|
||||
|
||||
**证据链**
|
||||
|
||||
Reference in New Issue
Block a user