feat(brand-check): 品牌检测按任务归属用户取代理并计用量——三处调用接入
- BrandCheckClient 契约扩展:构造器收 UserApiSecretService,check/checkAll/ checkTitleText(s) 全系带 userId,请求体新增 user_id + proxy_url(proxy 取自 用户密钥库 module=proxy);响应新增 reason/retryable,retryable=false(供应商 拒付、用户没配代理)当场停重试阶梯并短路整批 - 三处调用方接入:CollectDataService 品牌过滤、CollectDataBrandRecheckScheduler 补测(markAttempted 带真实原因)、AppearancePatentLlmClient 标题查标(取 SecretUsageContext 归属用户) - 用户未配置代理:整批本地定论「用户未配置代理」,零 HTTP,不再有平台钥匙兜底 - 测试迁移:15 支测试文件按新契约迁移(3 参构造、userId 实参、BrandCheckBatchResult 第 4 位 reason、Mockito 匹配器加宽);HTTP 类测试 stub findPlainValue 非空代理 以绕开零 HTTP 短路。13 类 119 例全绿(1 skip 为既有快照写盘门),test-compile 零错误
This commit is contained in:
+6
-2
@@ -144,7 +144,7 @@ public class CollectDataBrandRecheckScheduler {
|
||||
List<String> brands = distinctBrands(entries);
|
||||
BrandCheckClient.BrandCheckBatchResult check;
|
||||
try {
|
||||
check = brandCheckClient.checkAll(brands, "Terms");
|
||||
check = brandCheckClient.checkAll(brands, "Terms", task.getUserId());
|
||||
} catch (RuntimeException ex) {
|
||||
// 整批检查异常(网络/服务级故障):按实测失败计次退避,下轮重试
|
||||
log.warn("[brand-recheck] 批量品牌检测异常,按失败计次 taskId={} brands={} err={}",
|
||||
@@ -194,6 +194,10 @@ public class CollectDataBrandRecheckScheduler {
|
||||
}
|
||||
|
||||
int exhaustedThisRound = 0;
|
||||
// 补测这一轮到底为什么还是查不了:上游给的原因(供应商拒付/用户未配置代理/出口被拒)
|
||||
// 直接写进条目,收口的条目才读得懂是"查不到"还是"没得查"。上游没给原因时保留旧文案。
|
||||
String checkReason = check == null ? "" : check.reason();
|
||||
String stillFailedReason = checkReason.isBlank() ? "query_faild_data 持续非空" : checkReason;
|
||||
for (CollectDataBrandRecheckEntity entry : cleanEntries) {
|
||||
recheckService.markInjected(entry);
|
||||
}
|
||||
@@ -201,7 +205,7 @@ public class CollectDataBrandRecheckScheduler {
|
||||
recheckService.markRejected(entry);
|
||||
}
|
||||
for (CollectDataBrandRecheckEntity entry : stillFailedEntries) {
|
||||
if (recheckService.markAttempted(entry, "query_faild_data 持续非空")) {
|
||||
if (recheckService.markAttempted(entry, stillFailedReason)) {
|
||||
// updateQuietly 失败时下轮按 PENDING 重放,不算收口
|
||||
if (CollectDataBrandRecheckEntity.STATUS_EXHAUSTED.equals(entry.getStatus())) {
|
||||
exhaustedThisRound++;
|
||||
|
||||
+5
-3
@@ -823,9 +823,11 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
// 比对集认任务归属账号那一家:chunk 回传的过滤结果不该受别家导入过什么影响
|
||||
CollectDataBatchQuery.FilterResult filtered =
|
||||
collectDataBatchQuery.filter(rowsForFiltering, probe.getUserId());
|
||||
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
|
||||
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
||||
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
||||
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome =
|
||||
brandBatchFilter.filter(filtered.kept(), probe.getUserId());
|
||||
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms 品牌失败原因={}",
|
||||
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt,
|
||||
brandOutcome.reason());
|
||||
|
||||
// ASIN 结果缓存旁路写入同样放锁外:这把锁的持有时间是心跳与其他 chunk 的排队预算
|
||||
// (见上方品牌检测移出的注释),缓存写入再快也不该来挤。缓存是"下次少开一次页"的
|
||||
|
||||
+39
-10
@@ -76,13 +76,17 @@ public class CollectDataBrandBatchFilter {
|
||||
|
||||
/**
|
||||
* 按批次执行品牌检查并分类。null/空输入返回空结果;null 行安全跳过不计数。
|
||||
*
|
||||
* @param userId 任务归属用户:品牌检查用的代理是这个用户自己的提取链接
|
||||
* (16890 不再兜底平台 key),谁的套餐被烧就记在谁头上。
|
||||
*/
|
||||
public BrandBatchOutcome filter(List<CollectDataResultRowVo> rows) {
|
||||
public BrandBatchOutcome filter(List<CollectDataResultRowVo> rows, Long userId) {
|
||||
List<CollectDataResultRowVo> rejected = new ArrayList<>();
|
||||
List<CollectDataResultRowVo> queryFailed = new ArrayList<>();
|
||||
List<CollectDataResultRowVo> accepted = new ArrayList<>();
|
||||
List<String> reasons = new ArrayList<>();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted);
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted, "");
|
||||
}
|
||||
long deadlineNanos = chunkBudgetMillis > 0L
|
||||
? System.nanoTime() + chunkBudgetMillis * 1_000_000L : Long.MAX_VALUE;
|
||||
@@ -124,9 +128,13 @@ public class CollectDataBrandBatchFilter {
|
||||
}
|
||||
Map<String, String> batchVerdicts = new LinkedHashMap<>(cachedVerdicts);
|
||||
try {
|
||||
batchVerdicts.putAll(checkAndCache(uncachedBrands, remainingBudgetMillis(deadlineNanos)));
|
||||
BatchVerdicts checked = checkAndCache(uncachedBrands,
|
||||
remainingBudgetMillis(deadlineNanos), userId);
|
||||
batchVerdicts.putAll(checked.verdicts());
|
||||
addReason(reasons, checked.reason());
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[collect-data] brand check batch failed, degrade batch to queryFailed err={}", ex.getMessage());
|
||||
addReason(reasons, "品牌检查批次异常: " + ex.getMessage());
|
||||
for (CollectDataResultRowVo row : batch) {
|
||||
if (row != null) {
|
||||
queryFailed.add(row);
|
||||
@@ -140,7 +148,14 @@ public class CollectDataBrandBatchFilter {
|
||||
log.info("[collect-data] 品牌检查预算用尽,剩余批次转入补测队列 预算={}ms 延后行={} 总行={}",
|
||||
chunkBudgetMillis, budgetCutRows, rows.size());
|
||||
}
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted);
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted, String.join(";", reasons));
|
||||
}
|
||||
|
||||
/** 原因清单去重累积(同一次任务里同一个坏消息不该在日志里重复三遍)。 */
|
||||
private static void addReason(List<String> reasons, String reason) {
|
||||
if (reason != null && !reason.isBlank() && !reasons.contains(reason)) {
|
||||
reasons.add(reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** 本批次还能用的毫秒数;0 表示不限制(交给 checkAll 按配置预算走)。 */
|
||||
@@ -151,9 +166,14 @@ public class CollectDataBrandBatchFilter {
|
||||
return Math.max(1L, (deadlineNanos - System.nanoTime()) / 1_000_000L);
|
||||
}
|
||||
|
||||
/** 远程检查未缓存品牌并写入缓存;返回新查品牌(小写标准化)→ 判定映射。 */
|
||||
private Map<String, String> checkAndCache(List<String> uncachedBrands, long budgetMillis) {
|
||||
BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(uncachedBrands, "Terms", budgetMillis);
|
||||
/** 本批次远程检查的结论与原因(原因=这批为什么查不了,可能为空串)。 */
|
||||
private record BatchVerdicts(Map<String, String> verdicts, String reason) {
|
||||
}
|
||||
|
||||
/** 远程检查未缓存品牌并写入缓存;返回新查品牌(小写标准化)→ 判定映射与失败原因。 */
|
||||
private BatchVerdicts checkAndCache(List<String> uncachedBrands, long budgetMillis, Long userId) {
|
||||
BrandCheckClient.BrandCheckBatchResult check =
|
||||
brandCheckClient.checkAll(uncachedBrands, "Terms", budgetMillis, userId);
|
||||
Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
|
||||
Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
|
||||
Map<String, String> verdicts = new LinkedHashMap<>();
|
||||
@@ -170,7 +190,7 @@ public class CollectDataBrandBatchFilter {
|
||||
putBounded(normalized, verdict);
|
||||
verdicts.put(normalized, verdict);
|
||||
}
|
||||
return verdicts;
|
||||
return new BatchVerdicts(verdicts, check == null ? "" : check.reason());
|
||||
}
|
||||
|
||||
private Map<String, String> snapshotVerdicts(List<String> brands) {
|
||||
@@ -249,10 +269,19 @@ public class CollectDataBrandBatchFilter {
|
||||
}
|
||||
}
|
||||
|
||||
/** 品牌检查分类结果:三类行互斥,顺序与输入一致。 */
|
||||
/**
|
||||
* 品牌检查分类结果:三类行互斥,顺序与输入一致。
|
||||
*
|
||||
* @param reason 这批为什么查不了(供应商拒付 / 用户未配置代理 / 批次异常),多个用;拼接;
|
||||
* 全部有结论时为空串。它存在的意义是让「只看到没采到数据」变成可归因的一句话。
|
||||
*/
|
||||
public record BrandBatchOutcome(List<CollectDataResultRowVo> rejected,
|
||||
List<CollectDataResultRowVo> queryFailed,
|
||||
List<CollectDataResultRowVo> accepted) {
|
||||
List<CollectDataResultRowVo> accepted,
|
||||
String reason) {
|
||||
public String reason() {
|
||||
return reason == null ? "" : reason;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String> distinctNonBlank(List<String> values) {
|
||||
|
||||
Reference in New Issue
Block a user