super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-47: 品牌检查结果任务内短期缓存，避免同品牌重复远程调用

Browse Source
This commit is contained in:
黄自达
2026-08-30 13:53:38 +08:00
parent 751f4b89fd
commit b92d3e688d
4 changed files with 305 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java
+99 -21
View File
@@ -8,8 +8,10 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@@ -19,6 +21,10 @@ import java.util.regex.Pattern;
* 非空批次正常检查并按失败/查询失败/通过分类。分类语义与
* CollectDataService 原 filterByBrandCheck 完全等价，仅空品牌批次省掉
* 无效远程请求。远程调用抛错时该批次整组降级 queryFailed，不影响后续批次。
*
* 品牌判定结果在过滤器实例内短期缓存（同一任务多个 chunk 提交复用同一
* 实例）：已判定的品牌再次出现时不再发起远程调用，避免同品牌重复请求。
* 远程抛错不写缓存（可恢复后重查）；缓存有界，超限淘汰最旧条目。
*/
@Slf4j
@Component
@@ -26,13 +32,29 @@ public class CollectDataBrandBatchFilter {
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
private static final String VERDICT_FAILED = "FAILED";
private static final String VERDICT_QUERY_FAILED = "QUERY_FAILED";
private static final String VERDICT_OK = "OK";
private final BrandCheckClient brandCheckClient;
private final int batchSize;
private final int cacheCapacity;
/** 品牌（小写标准化）→ 判定结果；access-order LRU，超限淘汰最旧。 */
private final Map<String, String> verdictCache;
public CollectDataBrandBatchFilter(BrandCheckClient brandCheckClient,
@Value("${aiimage.collect-data.brand-check-batch-size:10}") int batchSize) {
this(brandCheckClient, batchSize, 512);
}
public CollectDataBrandBatchFilter(BrandCheckClient brandCheckClient,
@Value("${aiimage.collect-data.brand-check-batch-size:10}") int batchSize,
@Value("${aiimage.collect-data.brand-check-cache-capacity:512}") int cacheCapacity) {
this.brandCheckClient = brandCheckClient;
this.batchSize = Math.max(1, batchSize);
this.cacheCapacity = Math.max(1, cacheCapacity);
this.verdictCache = new LinkedHashMap<>(Math.max(16, this.cacheCapacity / 2), 0.75f, true);
}
/**
@@ -48,10 +70,21 @@ public class CollectDataBrandBatchFilter {
for (int start = 0; start < rows.size(); start += batchSize) {
int end = Math.min(start + batchSize, rows.size());
List<CollectDataResultRowVo> batch = rows.subList(start, end);
List<String> brands = distinctNonBlank(batch.stream()
List<String> batchBrands = distinctNonBlank(batch.stream()
.filter(row -> row != null)
.map(CollectDataResultRowVo::getBrand).toList());
if (brands.isEmpty()) {
List<String> uncachedBrands = new ArrayList<>();
for (String brand : batchBrands) {
if (!verdictCache.containsKey(normalizeBrand(brand))) {
uncachedBrands.add(brand);
}
}
if (uncachedBrands.isEmpty() && !batchBrands.isEmpty()) {
// 本批次品牌全部命中缓存，无需远程调用。
classify(batch, verdictCache, rejected, queryFailed, accepted);
continue;
}
if (batchBrands.isEmpty()) {
// 空品牌批次：跳过远程检查，行直接归 rejected（与空品牌行语义一致）。
for (CollectDataResultRowVo row : batch) {
if (row != null) {
@@ -60,9 +93,16 @@ public class CollectDataBrandBatchFilter {
}
continue;
}
BrandCheckClient.BrandCheckBatchResult check;
Map<String, String> batchVerdicts = new LinkedHashMap<>();
for (String brand : batchBrands) {
String normalized = normalizeBrand(brand);
String cached = verdictCache.get(normalized);
if (cached != null) {
batchVerdicts.put(normalized, cached);
}
}
try {
check = brandCheckClient.checkAll(brands, "Terms");
batchVerdicts.putAll(checkAndCache(uncachedBrands));
} catch (RuntimeException ex) {
log.warn("[collect-data] brand check batch failed, degrade batch to queryFailed err={}", ex.getMessage());
for (CollectDataResultRowVo row : batch) {
@@ -72,27 +112,65 @@ public class CollectDataBrandBatchFilter {
}
continue;
}
Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
for (CollectDataResultRowVo row : batch) {
if (row == null) {
continue;
}
String brand = normalizeBrand(row.getBrand());
if (brand.isBlank()) {
rejected.add(row);
} else if (failedBrands.contains(brand)) {
rejected.add(row);
} else if (queryFailedBrands.contains(brand)) {
queryFailed.add(row);
} else {
accepted.add(row);
}
}
classify(batch, batchVerdicts, rejected, queryFailed, accepted);
}
return new BrandBatchOutcome(rejected, queryFailed, accepted);
}
/** 远程检查未缓存品牌并写入缓存；返回新查品牌（小写标准化）→ 判定映射。 */
private Map<String, String> checkAndCache(List<String> uncachedBrands) {
BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(uncachedBrands, "Terms");
Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
Map<String, String> verdicts = new LinkedHashMap<>();
for (String brand : uncachedBrands) {
String normalized = normalizeBrand(brand);
String verdict;
if (failedBrands.contains(normalized)) {
verdict = VERDICT_FAILED;
} else if (queryFailedBrands.contains(normalized)) {
verdict = VERDICT_QUERY_FAILED;
} else {
verdict = VERDICT_OK;
}
putBounded(normalized, verdict);
verdicts.put(normalized, verdict);
}
return verdicts;
}
private void putBounded(String brand, String verdict) {
if (verdictCache.containsKey(brand)) {
return;
}
verdictCache.put(brand, verdict);
if (verdictCache.size() > cacheCapacity) {
var it = verdictCache.entrySet().iterator();
it.next();
it.remove();
}
}
private void classify(List<CollectDataResultRowVo> batch, Map<String, String> verdicts,
List<CollectDataResultRowVo> rejected,
List<CollectDataResultRowVo> queryFailed,
List<CollectDataResultRowVo> accepted) {
for (CollectDataResultRowVo row : batch) {
if (row == null) {
continue;
}
String brand = normalizeBrand(row.getBrand());
String verdict = verdicts.get(brand);
if (brand.isBlank() || VERDICT_FAILED.equals(verdict)) {
rejected.add(row);
} else if (VERDICT_QUERY_FAILED.equals(verdict)) {
queryFailed.add(row);
} else {
accepted.add(row);
}
}
}
/** 品牌检查分类结果：三类行互斥，顺序与输入一致。 */
public record BrandBatchOutcome(List<CollectDataResultRowVo> rejected,
List<CollectDataResultRowVo> queryFailed,