feat(brand-check): 品牌检测按任务归属用户取代理并计用量——三处调用接入
- BrandCheckClient 契约扩展:构造器收 UserApiSecretService,check/checkAll/ checkTitleText(s) 全系带 userId,请求体新增 user_id + proxy_url(proxy 取自 用户密钥库 module=proxy);响应新增 reason/retryable,retryable=false(供应商 拒付、用户没配代理)当场停重试阶梯并短路整批 - 三处调用方接入:CollectDataService 品牌过滤、CollectDataBrandRecheckScheduler 补测(markAttempted 带真实原因)、AppearancePatentLlmClient 标题查标(取 SecretUsageContext 归属用户) - 用户未配置代理:整批本地定论「用户未配置代理」,零 HTTP,不再有平台钥匙兜底 - 测试迁移:15 支测试文件按新契约迁移(3 参构造、userId 实参、BrandCheckBatchResult 第 4 位 reason、Mockito 匹配器加宽);HTTP 类测试 stub findPlainValue 非空代理 以绕开零 HTTP 短路。13 类 119 例全绿(1 skip 为既有快照写盘门),test-compile 零错误
This commit is contained in:
+7
-1
@@ -1042,7 +1042,13 @@ public class AppearancePatentLlmClient {
|
||||
BrandCheckClient.BrandCheckBatchResult[] byRowIndex =
|
||||
new BrandCheckClient.BrandCheckBatchResult[results.size()];
|
||||
if (!pending.isEmpty()) {
|
||||
List<BrandCheckClient.BrandCheckBatchResult> batch = brandCheckClient.checkTitleTexts(titles, "Terms");
|
||||
// 品牌检查用的代理=这批结果归属那个用户自己的提取链接(16890 不再兜底平台 key)。
|
||||
// 取不到任务上下文就是无人归属,此时不去打上游:上游要么用别人的套餐、要么裸 IP 被拒,
|
||||
// 两种都把一次配置缺失变成一笔算不清的账。
|
||||
SecretUsageContext.Context usage = SecretUsageContext.current();
|
||||
Long ownerUserId = usage == null ? null : usage.userId();
|
||||
List<BrandCheckClient.BrandCheckBatchResult> batch =
|
||||
brandCheckClient.checkTitleTexts(titles, "Terms", ownerUserId);
|
||||
for (int k = 0; k < pending.size(); k++) {
|
||||
byRowIndex[pending.get(k)] = batch.get(k);
|
||||
}
|
||||
|
||||
+132
-29
@@ -2,6 +2,8 @@ package com.nanri.aiimage.modules.brand.client;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -35,14 +37,20 @@ public class BrandCheckClient {
|
||||
/** 品牌检查并发度:避免一次批量检查把第三方服务打满。 */
|
||||
private static final int BRAND_CHECK_CONCURRENCY = 3;
|
||||
|
||||
/** 用户没配代理时交回的原因文案;采集侧与补测队列都按它把行判查询失败。 */
|
||||
public static final String NO_PROXY_REASON = "用户未配置代理";
|
||||
|
||||
private final BrandCheckProperties properties;
|
||||
private final com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics;
|
||||
private final UserApiSecretService userApiSecretService;
|
||||
private volatile RestClient sharedRestClient;
|
||||
|
||||
public BrandCheckClient(BrandCheckProperties properties,
|
||||
com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics) {
|
||||
com.nanri.aiimage.metrics.ExternalCallMetricsRecorder externalCallMetrics,
|
||||
UserApiSecretService userApiSecretService) {
|
||||
this.properties = properties;
|
||||
this.externalCallMetrics = externalCallMetrics;
|
||||
this.userApiSecretService = userApiSecretService;
|
||||
}
|
||||
|
||||
// 有界队列 + CallerRuns:大批量(几千品牌)检查时队列洪峰不无界堆内存,
|
||||
@@ -68,18 +76,32 @@ public class BrandCheckClient {
|
||||
};
|
||||
}
|
||||
|
||||
public BrandCheckResponse check(String brand) {
|
||||
return check(brand, properties.getDefaultStrategy());
|
||||
/**
|
||||
* 单品牌查一次上游。
|
||||
*
|
||||
* <p>{@code userId} 决定这次烧谁的代理:16890 不再自带平台 key(那把 key 的套餐
|
||||
* 2026-10-19 过期,全体用户的品牌检测一起静默失效,而用户在后台只看到「没采到数据」)。
|
||||
* 用户没配代理时**一次上游都不打**,当场按查询失败交回原因——重试救不回缺代理,
|
||||
* 十条阶梯只会把一个人的配置问题摊成上游十分钟的重试流量。
|
||||
*/
|
||||
public BrandCheckResponse check(String brand, String strategy, Long userId) {
|
||||
return postCheck(brand, strategy, userId, proxyUrlFor(userId));
|
||||
}
|
||||
|
||||
public BrandCheckResponse check(String brand, String strategy) {
|
||||
private BrandCheckResponse postCheck(String brand, String strategy, Long userId, String proxyUrl) {
|
||||
String normalizedBrand = normalize(brand);
|
||||
if (normalizedBrand.isBlank()) {
|
||||
return BrandCheckResponse.empty();
|
||||
}
|
||||
if (proxyUrl.isBlank()) {
|
||||
log.warn("[brand-check] {} userId={}(不打上游)", NO_PROXY_REASON, userId);
|
||||
return BrandCheckResponse.noProxy(normalizedBrand);
|
||||
}
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
body.put("brand", normalizedBrand);
|
||||
body.put("strategy", normalize(strategy).isBlank() ? "Terms" : normalize(strategy));
|
||||
body.put("user_id", userId);
|
||||
body.put("proxy_url", proxyUrl);
|
||||
return restClient().post()
|
||||
.uri(joinUrl(properties.getBaseUrl(), properties.getPath()))
|
||||
.headers(headers -> {
|
||||
@@ -94,8 +116,19 @@ public class BrandCheckClient {
|
||||
.body(BrandCheckResponse.class);
|
||||
}
|
||||
|
||||
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
||||
return checkAll(brands, strategy, properties.getTotalTimeoutMillis());
|
||||
/**
|
||||
* 该用户的代理提取链接(module=proxy 明文,支持 {@code url;mode;url;mode} 多 key 写法)。
|
||||
* findPlainValue 自身不抛:未配置/解密失败一律空串。
|
||||
*/
|
||||
private String proxyUrlFor(Long userId) {
|
||||
if (userId == null || userId <= 0L || userApiSecretService == null) {
|
||||
return "";
|
||||
}
|
||||
return normalize(userApiSecretService.findPlainValue(userId, UserSecretModule.PROXY.key()));
|
||||
}
|
||||
|
||||
public BrandCheckBatchResult checkAll(List<String> brands, String strategy, Long userId) {
|
||||
return checkAll(brands, strategy, properties.getTotalTimeoutMillis(), userId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -106,15 +139,26 @@ public class BrandCheckClient {
|
||||
* 是给后台批量任务(外观合批、品牌补测调度)的预算,同步等响应的桌面客户端不该替它扛
|
||||
* ——上游卡死时按批各领一个 90 秒,一个 chunk 十几批能叠到十几分钟。
|
||||
*/
|
||||
public BrandCheckBatchResult checkAll(List<String> brands, String strategy, long budgetMillis) {
|
||||
public BrandCheckBatchResult checkAll(List<String> brands, String strategy,
|
||||
long budgetMillis, Long userId) {
|
||||
List<String> distinctBrands = distinctNonBlank(brands);
|
||||
List<Object> failedData = new ArrayList<>();
|
||||
List<Object> queryFailedData = new ArrayList<>();
|
||||
for (BrandCheckOutcome outcome : checkWords(distinctBrands, strategy, deadlineNanos(budgetMillis)).values()) {
|
||||
Map<String, BrandCheckOutcome> outcomes =
|
||||
checkWords(distinctBrands, strategy, deadlineNanos(budgetMillis), userId);
|
||||
List<String> reasons = new ArrayList<>();
|
||||
for (String brand : distinctBrands) {
|
||||
BrandCheckOutcome outcome = outcomes.get(brand);
|
||||
if (outcome == null) {
|
||||
continue;
|
||||
}
|
||||
failedData.addAll(outcome.failedData());
|
||||
queryFailedData.addAll(outcome.queryFailedData());
|
||||
if (!outcome.reason().isBlank() && !reasons.contains(outcome.reason())) {
|
||||
reasons.add(outcome.reason());
|
||||
}
|
||||
}
|
||||
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
||||
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData, String.join(";", reasons));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -126,7 +170,7 @@ public class BrandCheckClient {
|
||||
* 与货源侧 CollectDataBrandBatchFilter 已上线的口径一致。
|
||||
* 返回值与入参同序同长,空标题行返回空词集合(调用方据此回退成标题原文)。
|
||||
*/
|
||||
public List<BrandCheckBatchResult> checkTitleTexts(List<String> titles, String strategy) {
|
||||
public List<BrandCheckBatchResult> checkTitleTexts(List<String> titles, String strategy, Long userId) {
|
||||
if (titles == null || titles.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
@@ -138,19 +182,24 @@ public class BrandCheckClient {
|
||||
distinctWords.addAll(words);
|
||||
}
|
||||
Map<String, BrandCheckOutcome> outcomes = checkWords(
|
||||
new ArrayList<>(distinctWords), strategy, deadlineNanos(properties.getTotalTimeoutMillis()));
|
||||
new ArrayList<>(distinctWords), strategy,
|
||||
deadlineNanos(properties.getTotalTimeoutMillis()), userId);
|
||||
List<BrandCheckBatchResult> results = new ArrayList<>(titles.size());
|
||||
for (List<String> words : wordsByTitle) {
|
||||
List<Object> failedData = new ArrayList<>();
|
||||
List<Object> queryFailedData = new ArrayList<>();
|
||||
List<String> reasons = new ArrayList<>();
|
||||
for (String word : words) {
|
||||
BrandCheckOutcome outcome = outcomes.get(word);
|
||||
if (outcome != null) {
|
||||
failedData.addAll(outcome.failedData());
|
||||
queryFailedData.addAll(outcome.queryFailedData());
|
||||
if (!outcome.reason().isBlank() && !reasons.contains(outcome.reason())) {
|
||||
reasons.add(outcome.reason());
|
||||
}
|
||||
}
|
||||
}
|
||||
results.add(new BrandCheckBatchResult(words, failedData, queryFailedData));
|
||||
results.add(new BrandCheckBatchResult(words, failedData, queryFailedData, String.join(";", reasons)));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
@@ -159,17 +208,29 @@ public class BrandCheckClient {
|
||||
* 并发查一批**已去重**的词,整批共用调用方给定的截止时刻。
|
||||
* 键顺序即入参顺序,checkAll 的并集顺序因此与改动前一致。
|
||||
*/
|
||||
private Map<String, BrandCheckOutcome> checkWords(List<String> brands, String strategy, long deadlineNanos) {
|
||||
private Map<String, BrandCheckOutcome> checkWords(List<String> brands, String strategy,
|
||||
long deadlineNanos, Long userId) {
|
||||
// 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到
|
||||
// 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。
|
||||
Map<String, BrandCheckOutcome> outcomes = new LinkedHashMap<>();
|
||||
String proxyUrl = proxyUrlFor(userId);
|
||||
if (proxyUrl.isBlank()) {
|
||||
// 缺代理整批一个请求都不发:这是配置问题,十条重试阶梯在这里只会白烧,
|
||||
// 而 16890 拿不到调用方的代理就只能裸打 WIPO——裸 IP 必被拒(2026-10-19 现网)。
|
||||
log.warn("[brand-check] {},整批不打上游 userId={} 品牌数={}", NO_PROXY_REASON, userId, brands.size());
|
||||
for (String brand : brands) {
|
||||
outcomes.put(brand, new BrandCheckOutcome(List.of(), List.of(brand), NO_PROXY_REASON));
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(brands.size());
|
||||
// 一次批量一份判定:本批里只要有一个品牌跑满阶梯仍被判查询失败,其余品牌就不再
|
||||
// 各自烧完整条重试阶梯(见 checkOneBrand 的短路分支)。
|
||||
AtomicBoolean upstreamRejecting = new AtomicBoolean(false);
|
||||
for (String brand : brands) {
|
||||
futures.add(CompletableFuture.supplyAsync(
|
||||
() -> checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting), checkExecutor));
|
||||
() -> checkOneBrand(brand, strategy, deadlineNanos, upstreamRejecting, proxyUrl, userId),
|
||||
checkExecutor));
|
||||
}
|
||||
for (int index = 0; index < brands.size(); index++) {
|
||||
outcomes.put(brands.get(index), awaitOutcome(futures.get(index)));
|
||||
@@ -183,10 +244,11 @@ public class BrandCheckClient {
|
||||
}
|
||||
|
||||
private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos,
|
||||
AtomicBoolean upstreamRejecting) {
|
||||
AtomicBoolean upstreamRejecting, String proxyUrl, Long userId) {
|
||||
int attempts = Math.max(1, properties.getRetryTimes());
|
||||
BrandCheckResponse response = null;
|
||||
Exception lastFailure = null;
|
||||
String reason = "";
|
||||
// 只有「把整条重试阶梯走完」才算拿到"上游在确定性拒绝"的证据。
|
||||
// 被预算或短路掐断的轮次不是证据,拿它去短路同批其余品牌会把偶发抖动放大成整批放弃。
|
||||
boolean ladderCompleted = true;
|
||||
@@ -212,7 +274,7 @@ public class BrandCheckClient {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
response = check(brand, strategy);
|
||||
response = postCheck(brand, strategy, userId, proxyUrl);
|
||||
} catch (Exception ex) {
|
||||
lastFailure = ex;
|
||||
response = null;
|
||||
@@ -226,7 +288,20 @@ public class BrandCheckClient {
|
||||
}
|
||||
if (response != null && nullToEmpty(response.getQueryFaildData()).isEmpty()) {
|
||||
// 本次查询完成且无失败项:命中侵权(faild_data)属有效结论,不重试。
|
||||
return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of());
|
||||
return new BrandCheckOutcome(nullToEmpty(response.getFaildData()), List.of(), "");
|
||||
}
|
||||
// 走到这里 response 必非 null:抛异常的那一轮在 catch 里已经 continue 掉了。
|
||||
if (response.getReason() != null && !response.getReason().isBlank()) {
|
||||
reason = response.getReason().trim();
|
||||
}
|
||||
if (response != null && !response.isRetryable()) {
|
||||
// 上游当场判定"重试也不会变好"(供应商拒付、套餐过期、这类结论是确定的),
|
||||
// 阶梯到此为止并把同批一起短路——旧版这里要跑满 10 轮、约 45~65 秒才知道
|
||||
// 同一个坏消息,而它烧掉的正是后面那些品牌的机会。
|
||||
upstreamRejecting.set(true);
|
||||
ladderCompleted = false;
|
||||
log.warn("[brand-check] 上游判定确定性拒绝,停止重试 brand={} reason={}", brand, reason);
|
||||
break;
|
||||
}
|
||||
if (attempt < attempts) {
|
||||
long retryDelayMillis = retryDelayMillis(attempt);
|
||||
@@ -242,11 +317,11 @@ public class BrandCheckClient {
|
||||
&& !nullToEmpty(response.getQueryFaildData()).isEmpty()) {
|
||||
upstreamRejecting.set(true);
|
||||
}
|
||||
log.warn("[brand-check] 重试耗尽判定查询失败 brand={} attempts={} lastErr={}",
|
||||
log.warn("[brand-check] 重试耗尽判定查询失败 brand={} attempts={} lastErr={} reason={}",
|
||||
brand, attempts,
|
||||
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage());
|
||||
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage(), reason);
|
||||
return new BrandCheckOutcome(List.of(),
|
||||
response == null ? List.of(brand) : nullToEmpty(response.getQueryFaildData()));
|
||||
response == null ? List.of(brand) : nullToEmpty(response.getQueryFaildData()), reason);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -278,19 +353,22 @@ public class BrandCheckClient {
|
||||
} catch (CompletionException ex) {
|
||||
log.warn("[brand-check] unexpected batch failure err={}",
|
||||
ex.getCause() == null ? ex.getMessage() : String.valueOf(ex.getCause().getMessage()));
|
||||
return new BrandCheckOutcome(List.of(), List.of());
|
||||
return new BrandCheckOutcome(List.of(), List.of(), "");
|
||||
}
|
||||
}
|
||||
|
||||
private record BrandCheckOutcome(List<Object> failedData, List<Object> queryFailedData) {
|
||||
/**
|
||||
* 单个品牌的结论。
|
||||
*
|
||||
* @param reason 查询失败时的可展示原因(供应商拒付 / 用户未配置代理 / 上游拒绝代理出口);
|
||||
* 结论正常时为空串。它存在的意义是让采集侧与补测队列能把「为什么查不了」
|
||||
* 写进任务,而不是只留一句 {@code query_faild_data 持续非空}。
|
||||
*/
|
||||
private record BrandCheckOutcome(List<Object> failedData, List<Object> queryFailedData, String reason) {
|
||||
}
|
||||
|
||||
public BrandCheckBatchResult checkTitleText(String titleText) {
|
||||
return checkTitleText(titleText, properties.getDefaultStrategy());
|
||||
}
|
||||
|
||||
public BrandCheckBatchResult checkTitleText(String titleText, String strategy) {
|
||||
return checkAll(splitTitleText(titleText), strategy);
|
||||
public BrandCheckBatchResult checkTitleText(String titleText, String strategy, Long userId) {
|
||||
return checkAll(splitTitleText(titleText), strategy, userId);
|
||||
}
|
||||
|
||||
public List<String> splitTitleText(String titleText) {
|
||||
@@ -392,15 +470,35 @@ public class BrandCheckClient {
|
||||
@JsonAlias({"query_faild_data", "query_failed_data"})
|
||||
private List<Object> queryFaildData = new ArrayList<>();
|
||||
|
||||
/** 查询失败的可展示原因;上游旧版本不回传该字段时为空串,行为与改造前一致。 */
|
||||
private String reason = "";
|
||||
|
||||
/**
|
||||
* false=上游判定这个结论重也不会变好(供应商拒付、用户没配代理),
|
||||
* 重试阶梯应当当场停下。缺省 true 是刻意的:旧版 16890 不带这个字段时,
|
||||
* 不能把"没带"读成"别重试"。
|
||||
*/
|
||||
private boolean retryable = true;
|
||||
|
||||
private static BrandCheckResponse empty() {
|
||||
return new BrandCheckResponse();
|
||||
}
|
||||
|
||||
/** 用户没配代理:Java 侧本地就能定论,一次上游都不该打。 */
|
||||
private static BrandCheckResponse noProxy(String brand) {
|
||||
BrandCheckResponse response = new BrandCheckResponse();
|
||||
response.setQueryFaildData(new ArrayList<>(List.of(brand)));
|
||||
response.setReason(NO_PROXY_REASON);
|
||||
response.setRetryable(false);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
public record BrandCheckBatchResult(
|
||||
List<String> brands,
|
||||
List<Object> faildData,
|
||||
List<Object> queryFaildData
|
||||
List<Object> queryFaildData,
|
||||
String reason
|
||||
) {
|
||||
public boolean hasFailedData() {
|
||||
return faildData != null && !faildData.isEmpty();
|
||||
@@ -409,5 +507,10 @@ public class BrandCheckClient {
|
||||
public boolean hasQueryFailedData() {
|
||||
return queryFaildData != null && !queryFaildData.isEmpty();
|
||||
}
|
||||
|
||||
/** 查询失败原因(多个用;拼接);结论正常时为空串。 */
|
||||
public String reason() {
|
||||
return reason == null ? "" : reason;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+6
-2
@@ -144,7 +144,7 @@ public class CollectDataBrandRecheckScheduler {
|
||||
List<String> brands = distinctBrands(entries);
|
||||
BrandCheckClient.BrandCheckBatchResult check;
|
||||
try {
|
||||
check = brandCheckClient.checkAll(brands, "Terms");
|
||||
check = brandCheckClient.checkAll(brands, "Terms", task.getUserId());
|
||||
} catch (RuntimeException ex) {
|
||||
// 整批检查异常(网络/服务级故障):按实测失败计次退避,下轮重试
|
||||
log.warn("[brand-recheck] 批量品牌检测异常,按失败计次 taskId={} brands={} err={}",
|
||||
@@ -194,6 +194,10 @@ public class CollectDataBrandRecheckScheduler {
|
||||
}
|
||||
|
||||
int exhaustedThisRound = 0;
|
||||
// 补测这一轮到底为什么还是查不了:上游给的原因(供应商拒付/用户未配置代理/出口被拒)
|
||||
// 直接写进条目,收口的条目才读得懂是"查不到"还是"没得查"。上游没给原因时保留旧文案。
|
||||
String checkReason = check == null ? "" : check.reason();
|
||||
String stillFailedReason = checkReason.isBlank() ? "query_faild_data 持续非空" : checkReason;
|
||||
for (CollectDataBrandRecheckEntity entry : cleanEntries) {
|
||||
recheckService.markInjected(entry);
|
||||
}
|
||||
@@ -201,7 +205,7 @@ public class CollectDataBrandRecheckScheduler {
|
||||
recheckService.markRejected(entry);
|
||||
}
|
||||
for (CollectDataBrandRecheckEntity entry : stillFailedEntries) {
|
||||
if (recheckService.markAttempted(entry, "query_faild_data 持续非空")) {
|
||||
if (recheckService.markAttempted(entry, stillFailedReason)) {
|
||||
// updateQuietly 失败时下轮按 PENDING 重放,不算收口
|
||||
if (CollectDataBrandRecheckEntity.STATUS_EXHAUSTED.equals(entry.getStatus())) {
|
||||
exhaustedThisRound++;
|
||||
|
||||
+5
-3
@@ -823,9 +823,11 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
// 比对集认任务归属账号那一家:chunk 回传的过滤结果不该受别家导入过什么影响
|
||||
CollectDataBatchQuery.FilterResult filtered =
|
||||
collectDataBatchQuery.filter(rowsForFiltering, probe.getUserId());
|
||||
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
|
||||
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
||||
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
||||
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome =
|
||||
brandBatchFilter.filter(filtered.kept(), probe.getUserId());
|
||||
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms 品牌失败原因={}",
|
||||
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt,
|
||||
brandOutcome.reason());
|
||||
|
||||
// ASIN 结果缓存旁路写入同样放锁外:这把锁的持有时间是心跳与其他 chunk 的排队预算
|
||||
// (见上方品牌检测移出的注释),缓存写入再快也不该来挤。缓存是"下次少开一次页"的
|
||||
|
||||
+39
-10
@@ -76,13 +76,17 @@ public class CollectDataBrandBatchFilter {
|
||||
|
||||
/**
|
||||
* 按批次执行品牌检查并分类。null/空输入返回空结果;null 行安全跳过不计数。
|
||||
*
|
||||
* @param userId 任务归属用户:品牌检查用的代理是这个用户自己的提取链接
|
||||
* (16890 不再兜底平台 key),谁的套餐被烧就记在谁头上。
|
||||
*/
|
||||
public BrandBatchOutcome filter(List<CollectDataResultRowVo> rows) {
|
||||
public BrandBatchOutcome filter(List<CollectDataResultRowVo> rows, Long userId) {
|
||||
List<CollectDataResultRowVo> rejected = new ArrayList<>();
|
||||
List<CollectDataResultRowVo> queryFailed = new ArrayList<>();
|
||||
List<CollectDataResultRowVo> accepted = new ArrayList<>();
|
||||
List<String> reasons = new ArrayList<>();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted);
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted, "");
|
||||
}
|
||||
long deadlineNanos = chunkBudgetMillis > 0L
|
||||
? System.nanoTime() + chunkBudgetMillis * 1_000_000L : Long.MAX_VALUE;
|
||||
@@ -124,9 +128,13 @@ public class CollectDataBrandBatchFilter {
|
||||
}
|
||||
Map<String, String> batchVerdicts = new LinkedHashMap<>(cachedVerdicts);
|
||||
try {
|
||||
batchVerdicts.putAll(checkAndCache(uncachedBrands, remainingBudgetMillis(deadlineNanos)));
|
||||
BatchVerdicts checked = checkAndCache(uncachedBrands,
|
||||
remainingBudgetMillis(deadlineNanos), userId);
|
||||
batchVerdicts.putAll(checked.verdicts());
|
||||
addReason(reasons, checked.reason());
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[collect-data] brand check batch failed, degrade batch to queryFailed err={}", ex.getMessage());
|
||||
addReason(reasons, "品牌检查批次异常: " + ex.getMessage());
|
||||
for (CollectDataResultRowVo row : batch) {
|
||||
if (row != null) {
|
||||
queryFailed.add(row);
|
||||
@@ -140,7 +148,14 @@ public class CollectDataBrandBatchFilter {
|
||||
log.info("[collect-data] 品牌检查预算用尽,剩余批次转入补测队列 预算={}ms 延后行={} 总行={}",
|
||||
chunkBudgetMillis, budgetCutRows, rows.size());
|
||||
}
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted);
|
||||
return new BrandBatchOutcome(rejected, queryFailed, accepted, String.join(";", reasons));
|
||||
}
|
||||
|
||||
/** 原因清单去重累积(同一次任务里同一个坏消息不该在日志里重复三遍)。 */
|
||||
private static void addReason(List<String> reasons, String reason) {
|
||||
if (reason != null && !reason.isBlank() && !reasons.contains(reason)) {
|
||||
reasons.add(reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** 本批次还能用的毫秒数;0 表示不限制(交给 checkAll 按配置预算走)。 */
|
||||
@@ -151,9 +166,14 @@ public class CollectDataBrandBatchFilter {
|
||||
return Math.max(1L, (deadlineNanos - System.nanoTime()) / 1_000_000L);
|
||||
}
|
||||
|
||||
/** 远程检查未缓存品牌并写入缓存;返回新查品牌(小写标准化)→ 判定映射。 */
|
||||
private Map<String, String> checkAndCache(List<String> uncachedBrands, long budgetMillis) {
|
||||
BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(uncachedBrands, "Terms", budgetMillis);
|
||||
/** 本批次远程检查的结论与原因(原因=这批为什么查不了,可能为空串)。 */
|
||||
private record BatchVerdicts(Map<String, String> verdicts, String reason) {
|
||||
}
|
||||
|
||||
/** 远程检查未缓存品牌并写入缓存;返回新查品牌(小写标准化)→ 判定映射与失败原因。 */
|
||||
private BatchVerdicts checkAndCache(List<String> uncachedBrands, long budgetMillis, Long userId) {
|
||||
BrandCheckClient.BrandCheckBatchResult check =
|
||||
brandCheckClient.checkAll(uncachedBrands, "Terms", budgetMillis, userId);
|
||||
Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
|
||||
Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
|
||||
Map<String, String> verdicts = new LinkedHashMap<>();
|
||||
@@ -170,7 +190,7 @@ public class CollectDataBrandBatchFilter {
|
||||
putBounded(normalized, verdict);
|
||||
verdicts.put(normalized, verdict);
|
||||
}
|
||||
return verdicts;
|
||||
return new BatchVerdicts(verdicts, check == null ? "" : check.reason());
|
||||
}
|
||||
|
||||
private Map<String, String> snapshotVerdicts(List<String> brands) {
|
||||
@@ -249,10 +269,19 @@ public class CollectDataBrandBatchFilter {
|
||||
}
|
||||
}
|
||||
|
||||
/** 品牌检查分类结果:三类行互斥,顺序与输入一致。 */
|
||||
/**
|
||||
* 品牌检查分类结果:三类行互斥,顺序与输入一致。
|
||||
*
|
||||
* @param reason 这批为什么查不了(供应商拒付 / 用户未配置代理 / 批次异常),多个用;拼接;
|
||||
* 全部有结论时为空串。它存在的意义是让「只看到没采到数据」变成可归因的一句话。
|
||||
*/
|
||||
public record BrandBatchOutcome(List<CollectDataResultRowVo> rejected,
|
||||
List<CollectDataResultRowVo> queryFailed,
|
||||
List<CollectDataResultRowVo> accepted) {
|
||||
List<CollectDataResultRowVo> accepted,
|
||||
String reason) {
|
||||
public String reason() {
|
||||
return reason == null ? "" : reason;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String> distinctNonBlank(List<String> values) {
|
||||
|
||||
Reference in New Issue
Block a user