feat(采集数据): 品牌检测失败自动补测队列,限流窗口后自动重查并补结果(V140)

WIPO 限流窗口内 queryFailed 的品牌按任务排队(biz_collectdata_brand_recheck,同任务同品牌幂等),Scheduler 按指数退避自动补测:补出无侵权→行补进结果文件,判出侵权→落无效品牌表,连续耗尽→EXHAUSTED 并通知。只存品牌不存行,注入按 payload_hash 幂等。
This commit is contained in:
2026-10-03 00:22:07 +08:00
parent 68b19f22b3
commit 005cda9379
21 changed files with 1642 additions and 11 deletions
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.collectdata.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface CollectDataBrandRecheckMapper extends BaseMapper<CollectDataBrandRecheckEntity> {
}
@@ -0,0 +1,37 @@
package com.nanri.aiimage.modules.collectdata.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 品牌检测失败补测队列表(V140):分片回传时把「查询失败」的品牌按任务入队,
* 调度器在上游限流窗口过去后重查,出结论的行补进结果文件 / 无效品牌表。
* 只存品牌不存行——行数据留在 chunk 全量 payload(RustFS),注入时按品牌回读。
*/
@Data
@TableName("biz_collectdata_brand_recheck")
public class CollectDataBrandRecheckEntity {
public static final String STATUS_PENDING = "PENDING";
public static final String STATUS_INJECTED = "INJECTED";
public static final String STATUS_REJECTED = "REJECTED";
public static final String STATUS_EXHAUSTED = "EXHAUSTED";
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String taskNo;
private Long userId;
/** 归一化小写品牌,与 CollectDataBrandBatchFilter.normalizeBrand 同口径。 */
private String brand;
private String status;
private Integer attemptCount;
private String lastError;
private LocalDateTime nextAttemptAt;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,215 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 补测结论注入器:把一轮补测里出结论的品牌落回该任务的数据链。
*
* <p>行为与实时分片回传(CollectDataService.submitResult)同语义,只是发生在任务收尾之后:
* <ul>
* <li>CLEAN(无侵权命中)→ 行按 chunk 回读,先过 dedupe/无效品牌复查
* (补测窗口内该 ASIN 可能已被其他任务采走,first-writer-wins),
* 再写 chunk 级明细对象(key 带 recheck- 前缀,不覆盖原 chunk 明细)并
* {@code upsertAccepted} 进结果——重复注入靠 payload_hash 跳过语义幂等;</li>
* <li>REJECTED(判出侵权命中)→ 行落无效品牌表(与实时 rejected 路径一致),不进结果;</li>
* <li>注入的行同步写 ASIN 跨任务缓存(resolveCacheCountry 同口径:只收单国家任务)。</li>
* </ul>
*
* <p>结果文件不在这里重建:注入完成后由调度器 {@code enqueueAssembleResult} 走既有
* 异步装配任务(worker 会按 biz_task_result_item 重算并持久化 finalRowCount)。
* 单 chunk 明细对象写失败即整组抛出:已落库的行靠下轮重放的幂等性收敛,不丢不重。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CollectDataBrandRecheckInjector {
public static final String MODULE_TYPE = "COLLECT_DATA";
/** 补测结论:无侵权命中(行进结果) / 侵权命中(行落无效品牌表)。 */
public enum RecheckVerdict { CLEAN, REJECTED }
/** 一轮注入的落地结果:resultId 供调度器触发结果文件重建;两计数为 0 表示无实际写入。 */
public record GroupOutcome(Long resultId, int acceptedRows, int rejectedRows) {
public static final GroupOutcome EMPTY = new GroupOutcome(null, 0, 0);
}
private final TaskChunkMapper taskChunkMapper;
private final FileResultMapper fileResultMapper;
private final CollectDataBatchQuery collectDataBatchQuery;
private final CollectDataResultDetailCodec resultDetailCodec;
private final CollectDataResultItemBatchWriter resultItemBatchWriter;
private final CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
private final TransientPayloadStorageService transientPayloadStorageService;
private final AsinResultCacheService asinResultCacheService;
private final ObjectMapper objectMapper;
/** 只为复用 resolveCacheCountry 的单国家判定口径(缓存键缺国家会把 DE 价当 US 结论)。 */
private final CollectDataService collectDataService;
/**
* 把补测结论落回任务:按品牌回读 chunk 全量 payload,分流 CLEAN/REJECTED。
* 结论映射键必须与队列表 brand 同口径({@link CollectDataBrandBatchFilter#normalizeBrand})。
*/
public GroupOutcome apply(FileTaskEntity task, Map<String, RecheckVerdict> verdicts) {
if (task == null || task.getId() == null || verdicts == null || verdicts.isEmpty()) {
return GroupOutcome.EMPTY;
}
Set<String> cleanBrands = brandsWith(verdicts, RecheckVerdict.CLEAN);
Set<String> rejectedBrands = brandsWith(verdicts, RecheckVerdict.REJECTED);
if (cleanBrands.isEmpty() && rejectedBrands.isEmpty()) {
return GroupOutcome.EMPTY;
}
FileResultEntity result = fileResultMapper.selectOne(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByDesc(FileResultEntity::getId)
.last("limit 1"));
if (result == null) {
log.warn("[brand-recheck] 任务无结果记录,跳过注入 taskId={}", task.getId());
return GroupOutcome.EMPTY;
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, task.getId())
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getChunkIndex)
.orderByAsc(TaskChunkEntity::getId));
int acceptedRows = 0;
int rejectedRows = 0;
List<CollectDataResultRowVo> acceptedKept = new ArrayList<>();
for (TaskChunkEntity chunk : chunks) {
List<CollectDataResultRowVo> rows = readChunkRows(chunk);
if (rows.isEmpty()) {
continue;
}
if (!rejectedBrands.isEmpty()) {
List<CollectDataResultRowVo> chunkRejected = rowsOfBrands(rows, rejectedBrands);
if (!chunkRejected.isEmpty()) {
// INSERT IGNORE 幂等:重复补测同品牌不会在无效品牌表翻倍
rejectedRows += invalidAsinBatchWriter.writeBatch(chunkRejected);
}
}
if (cleanBrands.isEmpty()) {
continue;
}
List<CollectDataResultRowVo> chunkClean = rowsOfBrands(rows, cleanBrands);
if (chunkClean.isEmpty()) {
continue;
}
// 复查去重/无效品牌:补测窗口内该 ASIN 可能已被其他任务采走(first-writer-wins)
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(chunkClean);
if (filtered.kept().isEmpty()) {
log.info("[brand-recheck] 补测行全部被去重/无效过滤,不注入 taskId={} rows={}",
task.getId(), chunkClean.size());
continue;
}
String detailJson = resultDetailCodec.encodeChunk(filtered.kept());
// key 带 recheck- 前缀:不覆盖原 chunk 的明细对象;同 chunk 重复注入覆盖同一对象(幂等)
String storedDetail = transientPayloadStorageService.storeResultPayload(
MODULE_TYPE, task.getId(), chunk.getScopeHash(),
"recheck-chunk-" + chunk.getChunkIndex(), detailJson);
CollectDataResultItemBatchWriter.UpsertCounts counts = resultItemBatchWriter.upsertAccepted(
task.getId(), result.getId(), chunk.getScopeKey(), chunk.getChunkIndex(),
filtered.kept(), storedDetail);
acceptedRows += counts.newlyInserted();
acceptedKept.addAll(filtered.kept());
}
if (!acceptedKept.isEmpty()) {
writeAsinCache(task, acceptedKept);
}
if (acceptedRows > 0 || rejectedRows > 0) {
log.info("[brand-recheck] 补测结论注入完成 taskId={} 新增结果行={} 判定无效行={} 品牌(净/败)={}/{}",
task.getId(), acceptedRows, rejectedRows, cleanBrands.size(), rejectedBrands.size());
}
return new GroupOutcome(result.getId(), acceptedRows, rejectedRows);
}
private Set<String> brandsWith(Map<String, RecheckVerdict> verdicts, RecheckVerdict verdict) {
Set<String> out = new HashSet<>();
for (Map.Entry<String, RecheckVerdict> entry : verdicts.entrySet()) {
if (entry.getValue() == verdict && entry.getKey() != null && !entry.getKey().isBlank()) {
out.add(entry.getKey());
}
}
return out;
}
private List<CollectDataResultRowVo> rowsOfBrands(List<CollectDataResultRowVo> rows, Set<String> brands) {
List<CollectDataResultRowVo> out = new ArrayList<>();
for (CollectDataResultRowVo row : rows) {
if (row == null) {
continue;
}
String brand = CollectDataBrandBatchFilter.normalizeBrand(row.getBrand());
if (!brand.isBlank() && brands.contains(brand)) {
out.add(row);
}
}
return out;
}
/** 单 chunk 全量行回读(与 CollectDataService.streamRawRows 的 chunk 内逻辑一致)。 */
private List<CollectDataResultRowVo> readChunkRows(TaskChunkEntity chunk) {
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
chunk.getPayloadJson(), "read collect data raw chunk failed");
if (payloadJson == null || payloadJson.isBlank()) {
return List.of();
}
List<CollectDataResultRowVo> values = objectMapper.readValue(payloadJson,
new TypeReference<List<CollectDataResultRowVo>>() {
});
if (values == null) {
return List.of();
}
List<CollectDataResultRowVo> out = new ArrayList<>(values.size());
for (CollectDataResultRowVo value : values) {
if (value != null) {
out.add(value);
}
}
return out;
} catch (Exception ex) {
// 单 chunk 丢失只跳不炸:其余 chunk 的行照常注入,该 chunk 的行留待人工
log.warn("[brand-recheck] chunk 全量行回读失败,跳过该 chunk taskId={} chunkIndex={} err={}",
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
return List.of();
}
}
/** 补测出的行同样"顺带贡献结论"进 ASIN 缓存;写失败不影响注入结果。 */
private void writeAsinCache(FileTaskEntity task, List<CollectDataResultRowVo> rows) {
try {
asinResultCacheService.writeCollectDataRows(
collectDataService.resolveCacheCountry(task), String.valueOf(task.getId()), rows);
} catch (RuntimeException ex) {
log.warn("[brand-recheck] ASIN 缓存写入失败,不影响注入 taskId={} err={}",
task.getId(), ex.getMessage());
}
}
}
@@ -0,0 +1,267 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.RecheckVerdict;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 品牌检测失败补测调度器:把补测队列(biz_collectdata_brand_recheck)里到期的
* PENDING 条目按任务聚组,批量重查品牌检测,出结论的行交给
* {@link CollectDataBrandRecheckInjector} 落回结果文件 / 无效品牌表。
*
* <p>轮内熔断:本轮实测品牌中「仍查询失败」占比超阈值(样本 ≥ failRateMinProbe)时
* 中止本轮剩余条目并顺延(不计尝试次数)——上游仍在限流时不必逐批捶它,
* 2026-10-02 WIPO 连续 11 小时 Forbidden 的形状下,逐批重试只会白白消耗预算。
*
* <p>双实例经 Redis 分布式锁互斥;任务级写并发经任务分布式锁互斥(补测发生在
* 任务终态之后,锁主要挡结果文件装配与注入的竞态)。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CollectDataBrandRecheckScheduler {
private static final String LOCK_NAME = "collectdata-brand-recheck";
private static final String MODULE_TYPE = CollectDataBrandRecheckInjector.MODULE_TYPE;
private static final String STATUS_SUCCESS = "SUCCESS";
private static final String STATUS_FAILED = "FAILED";
private static final long TASK_LOCK_WAIT_MILLIS = 2000L;
private final CollectDataBrandRecheckService recheckService;
private final CollectDataBrandRecheckProperties properties;
private final FileTaskMapper fileTaskMapper;
private final BrandCheckClient brandCheckClient;
private final CollectDataBrandRecheckInjector injector;
private final NotificationDispatchService notificationDispatchService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService;
private final DistributedJobLockService distributedJobLockService;
@Scheduled(fixedDelayString = "${aiimage.collect-data.brand-recheck.scan-interval-ms:300000}",
initialDelayString = "${aiimage.collect-data.brand-recheck.scan-initial-delay-ms:120000}")
public void scan() {
if (!properties.isEnabled()) {
log.info("[brand-recheck] 补测已关闭(brand-recheck.enabled=false),跳过本轮");
return;
}
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(5));
if (lock == null) {
log.info("[brand-recheck] 另一实例持有补测锁,跳过本轮");
return;
}
try (lock) {
runRound();
} catch (Exception ex) {
log.warn("[brand-recheck] 补测轮异常终止 err={}", ex.getMessage(), ex);
}
}
/** 一轮补测:取出到期条目、按任务聚组逐组处理;熔断触发后剩余组整组顺延。 */
void runRound() {
List<CollectDataBrandRecheckEntity> due = recheckService.listDue(
LocalDateTime.now(), properties.getRoundBatchSize());
if (due.isEmpty()) {
log.info("[brand-recheck] 无到期的补测条目");
return;
}
Map<Long, List<CollectDataBrandRecheckEntity>> groups = new LinkedHashMap<>();
for (CollectDataBrandRecheckEntity entry : due) {
groups.computeIfAbsent(entry.getTaskId(), key -> new ArrayList<>()).add(entry);
}
log.info("[brand-recheck] 本轮开始 条目={} 任务组={}", due.size(), groups.size());
int probedBrands = 0;
int stillFailedBrands = 0;
boolean tripped = false;
for (Map.Entry<Long, List<CollectDataBrandRecheckEntity>> group : groups.entrySet()) {
if (tripped) {
recheckService.postponeWithoutAttempt(group.getValue(),
properties.getIntervalBaseMinutes());
continue;
}
GroupRound round = processTaskGroup(group.getKey(), group.getValue());
probedBrands += round.probed();
stillFailedBrands += round.stillFailed();
if (probedBrands >= Math.max(1, properties.getFailRateMinProbe())
&& stillFailedBrands >= probedBrands * properties.getFailRateThreshold()) {
tripped = true;
log.warn("[brand-recheck] 轮内熔断:实测 {} 个品牌仍失败 {},上游疑仍在限流,剩余条目顺延",
probedBrands, stillFailedBrands);
}
}
log.info("[brand-recheck] 本轮结束 实测品牌={} 仍失败={} 熔断={}", probedBrands, stillFailedBrands, tripped);
}
/** 处理单个任务组;返回轮计数(实测品牌数 / 仍失败品牌数),异常按全失败计。 */
private GroupRound processTaskGroup(Long taskId, List<CollectDataBrandRecheckEntity> entries) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null) {
// 任务已删除(清理 SPI 回收):条目无主,直接收口避免永久占坑
for (CollectDataBrandRecheckEntity entry : entries) {
recheckService.markExhausted(entry, "任务不存在(可能已被清理)");
}
return new GroupRound(0, 0);
}
if (!STATUS_SUCCESS.equals(task.getStatus()) && !STATUS_FAILED.equals(task.getStatus())) {
// 任务仍在跑:补测只做终态之后的兜底,顺延不计数(避免与分片重提竞态)
recheckService.postponeWithoutAttempt(entries, properties.getFirstDelayMinutes());
return new GroupRound(0, 0);
}
TaskDistributedLockService.LockHandle taskLock =
taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
if (taskLock == null) {
log.info("[brand-recheck] 任务锁竞争,本轮顺延(不计数) taskId={}", taskId);
recheckService.postponeWithoutAttempt(entries, properties.getFirstDelayMinutes());
return new GroupRound(0, 0);
}
try (taskLock) {
return checkAndApply(task, entries);
}
}
private GroupRound checkAndApply(FileTaskEntity task, List<CollectDataBrandRecheckEntity> entries) {
List<String> brands = distinctBrands(entries);
BrandCheckClient.BrandCheckBatchResult check;
try {
check = brandCheckClient.checkAll(brands, "Terms");
} catch (RuntimeException ex) {
// 整批检查异常(网络/服务级故障):按实测失败计次退避,下轮重试
log.warn("[brand-recheck] 批量品牌检测异常,按失败计次 taskId={} brands={} err={}",
task.getId(), brands.size(), ex.getMessage());
for (CollectDataBrandRecheckEntity entry : entries) {
recheckService.markAttempted(entry, ex.getMessage());
}
return new GroupRound(brands.size(), brands.size());
}
Set<String> faildBrands = normalizeVerdicts(check == null ? null : check.faildData());
Set<String> queryFailedBrands = normalizeVerdicts(check == null ? null : check.queryFaildData());
Map<String, RecheckVerdict> verdicts = new LinkedHashMap<>();
List<CollectDataBrandRecheckEntity> cleanEntries = new ArrayList<>();
List<CollectDataBrandRecheckEntity> rejectedEntries = new ArrayList<>();
List<CollectDataBrandRecheckEntity> stillFailedEntries = new ArrayList<>();
for (CollectDataBrandRecheckEntity entry : entries) {
String brand = CollectDataBrandBatchFilter.normalizeBrand(entry.getBrand());
if (faildBrands.contains(brand)) {
verdicts.put(brand, RecheckVerdict.REJECTED);
rejectedEntries.add(entry);
} else if (queryFailedBrands.contains(brand)) {
stillFailedEntries.add(entry);
} else {
verdicts.put(brand, RecheckVerdict.CLEAN);
cleanEntries.add(entry);
}
}
CollectDataBrandRecheckInjector.GroupOutcome outcome = CollectDataBrandRecheckInjector.GroupOutcome.EMPTY;
boolean applyFailed = false;
if (!verdicts.isEmpty()) {
try {
outcome = injector.apply(task, verdicts);
} catch (RuntimeException ex) {
// 注入失败(如结果明细写批次失败):出结论的品牌留队重放(注入幂等),按实测失败计次
applyFailed = true;
log.warn("[brand-recheck] 补测结论注入失败,整组留队重放 taskId={} err={}",
task.getId(), ex.getMessage());
}
}
if (applyFailed) {
for (CollectDataBrandRecheckEntity entry : entries) {
recheckService.markAttempted(entry, "补测结论注入失败,留队重放");
}
return new GroupRound(brands.size(), brands.size());
}
int exhaustedThisRound = 0;
for (CollectDataBrandRecheckEntity entry : cleanEntries) {
recheckService.markInjected(entry);
}
for (CollectDataBrandRecheckEntity entry : rejectedEntries) {
recheckService.markRejected(entry);
}
for (CollectDataBrandRecheckEntity entry : stillFailedEntries) {
if (recheckService.markAttempted(entry, "query_faild_data 持续非空")) {
// updateQuietly 失败时下轮按 PENDING 重放,不算收口
if (CollectDataBrandRecheckEntity.STATUS_EXHAUSTED.equals(entry.getStatus())) {
exhaustedThisRound++;
}
}
}
if (outcome.acceptedRows() > 0) {
// 结果行有新增 → 复用既有异步装配重建结果文件(worker 重算并持久化 finalRowCount)
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, outcome.resultId(),
"task:" + task.getId());
}
notificationDispatchService.pushRecheckOutcome(task.getUserId(), task.getId(), MODULE_TYPE,
task.getTaskNo(), outcome.acceptedRows(), outcome.rejectedRows(),
stillFailedEntries.size(), exhaustedThisRound);
log.info("[brand-recheck] 任务组完成 taskId={} 品牌={} 净={} 败={} 仍失败={} 收口={} 新增结果行={} 判定无效行={}",
task.getId(), brands.size(), cleanEntries.size(), rejectedEntries.size(),
stillFailedEntries.size(), exhaustedThisRound, outcome.acceptedRows(), outcome.rejectedRows());
return new GroupRound(brands.size(), stillFailedEntries.size());
}
private List<String> distinctBrands(List<CollectDataBrandRecheckEntity> entries) {
Set<String> brands = new LinkedHashSet<>();
for (CollectDataBrandRecheckEntity entry : entries) {
String brand = CollectDataBrandBatchFilter.normalizeBrand(entry.getBrand());
if (!brand.isBlank()) {
brands.add(brand);
}
}
return new ArrayList<>(brands);
}
/** 服务端返回的结论对象(字符串或 {brand,...} 映射)→ 归一化品牌集合,与过滤器同口径。 */
private Set<String> normalizeVerdicts(List<Object> values) {
Set<String> out = new LinkedHashSet<>();
if (values == null) {
return out;
}
for (Object value : values) {
String brand = CollectDataBrandBatchFilter.normalizeBrand(String.valueOf(brandValueOf(value)));
if (!brand.isBlank()) {
out.add(brand);
}
}
return out;
}
private Object brandValueOf(Object value) {
if (value instanceof Map<?, ?> map) {
for (String key : List.of("brand", "brandName", "brand_name", "name", "value", "data_value")) {
Object candidate = map.get(key);
if (candidate != null && !String.valueOf(candidate).isBlank()) {
return candidate;
}
}
}
return value;
}
/** 单任务组的轮计数。 */
private record GroupRound(int probed, int stillFailed) {
}
}
@@ -0,0 +1,203 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataBrandRecheckMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
/**
* 品牌检测失败补测队列的入库语义:入队(分片回传侧,锁外)、到期条目取出、
* 状态推进(实测成功/失败/收口)与退避计算。行数据不落表,注入时由
* {@link CollectDataBrandRecheckInjector} 从 chunk 全量 payload 按品牌回读。
*
* <p>状态机:PENDING →(补测无侵权命中)INJECTED /(判出侵权命中)REJECTED /
* (连续 maxAttempts 轮仍失败)EXHAUSTED。「顺延」(任务非终态、任务锁竞争、轮内熔断)
* 只推迟 next_attempt_at,不计尝试次数——上游故障不该消耗品牌的补测预算。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CollectDataBrandRecheckService {
/** last_error 截断长度(与列宽 VARCHAR(500) 对齐,留余量)。 */
private static final int ERROR_MAX_LENGTH = 480;
/** 退避封顶分钟数:intervalBaseMinutes × 2^5 = 16 小时(base=30 时)。 */
private static final int BACKOFF_MAX_EXPONENT = 5;
private final CollectDataBrandRecheckMapper recheckMapper;
private final CollectDataBrandRecheckProperties properties;
/**
* 分片回传侧入队:把本 chunk「查询失败」的品牌写进队列。
*
* <p>同任务同品牌唯一:已有行(任何状态)直接跳过——重复失败不刷新首次时间、
* 不改状态、不重置退避(EXHAUSTED 的品牌不因后续 chunk 再次失败而复活)。
* 整段吞掉异常:补测入队失败只损失一次自动重试机会,绝不能影响分片回传主流程。
*
* @return 实际新建的队列条数(诊断日志用)
*/
public int enqueue(FileTaskEntity task, List<CollectDataResultRowVo> queryFailedRows) {
if (!properties.isEnabled() || task == null
|| queryFailedRows == null || queryFailedRows.isEmpty()) {
return 0;
}
Long taskId = task.getId();
Long userId = task.getUserId();
if (taskId == null || userId == null || userId <= 0) {
return 0;
}
Set<String> brands = new LinkedHashSet<>();
for (CollectDataResultRowVo row : queryFailedRows) {
if (row == null) {
continue;
}
String brand = CollectDataBrandBatchFilter.normalizeBrand(row.getBrand());
if (!brand.isBlank()) {
brands.add(brand);
}
}
if (brands.isEmpty()) {
return 0;
}
LocalDateTime nextAttemptAt = LocalDateTime.now()
.plusMinutes(Math.max(1, properties.getFirstDelayMinutes()));
int created = 0;
for (String brand : brands) {
try {
Long existing = recheckMapper.selectCount(new LambdaQueryWrapper<CollectDataBrandRecheckEntity>()
.eq(CollectDataBrandRecheckEntity::getTaskId, taskId)
.eq(CollectDataBrandRecheckEntity::getBrand, brand));
if (existing != null && existing > 0) {
continue;
}
CollectDataBrandRecheckEntity entity = new CollectDataBrandRecheckEntity();
entity.setTaskId(taskId);
entity.setTaskNo(task.getTaskNo());
entity.setUserId(userId);
entity.setBrand(brand);
entity.setStatus(CollectDataBrandRecheckEntity.STATUS_PENDING);
entity.setAttemptCount(0);
entity.setNextAttemptAt(nextAttemptAt);
recheckMapper.insert(entity);
created++;
} catch (DuplicateKeyException dup) {
// 并发提交的兄弟 chunk 抢先入队:语义一致,静默跳过
} catch (RuntimeException ex) {
log.warn("[brand-recheck] 补测入队失败,不影响本次提交 taskId={} brand={} err={}",
taskId, brand, ex.getMessage());
}
}
if (created > 0) {
log.info("[brand-recheck] 品牌检测查询失败入队 taskId={} 新增={} 品牌={}",
taskId, created, brands);
}
return created;
}
/** 取出到期 PENDING 条目(next_attempt_at 升序,先到期先补)。 */
public List<CollectDataBrandRecheckEntity> listDue(LocalDateTime now, int limit) {
return recheckMapper.selectList(new LambdaQueryWrapper<CollectDataBrandRecheckEntity>()
.eq(CollectDataBrandRecheckEntity::getStatus, CollectDataBrandRecheckEntity.STATUS_PENDING)
.le(CollectDataBrandRecheckEntity::getNextAttemptAt, now)
.orderByAsc(CollectDataBrandRecheckEntity::getNextAttemptAt)
.orderByAsc(CollectDataBrandRecheckEntity::getId)
.last("limit " + Math.max(1, limit)));
}
/**
* 实测失败后的推进:尝试次数 +1,按指数退避排下次;达到 maxAttempts 轮转
* EXHAUSTED(不再自动重试,交还用户手动处理)。
*
* @return 该条目是否就此收口(EXHAUSTED)
*/
public boolean markAttempted(CollectDataBrandRecheckEntity entry, String error) {
int attempts = (entry.getAttemptCount() == null ? 0 : entry.getAttemptCount()) + 1;
entry.setAttemptCount(attempts);
entry.setLastError(truncate(error));
if (attempts >= Math.max(1, properties.getMaxAttempts())) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
} else {
entry.setNextAttemptAt(LocalDateTime.now().plusMinutes(backoffMinutes(attempts)));
}
return updateQuietly(entry, "markAttempted");
}
/** 补测出「无侵权命中」结论且行已补进结果:条目终结。 */
public void markInjected(CollectDataBrandRecheckEntity entry) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_INJECTED);
updateQuietly(entry, "markInjected");
}
/** 补测判出侵权命中且行已落无效品牌表:条目终结(行不进结果,与实时路径语义一致)。 */
public void markRejected(CollectDataBrandRecheckEntity entry) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_REJECTED);
updateQuietly(entry, "markRejected");
}
/** 任务不存在等无主条目:直接收口,避免永久占坑。 */
public void markExhausted(CollectDataBrandRecheckEntity entry, String error) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
entry.setLastError(truncate(error));
updateQuietly(entry, "markExhausted");
}
/**
* 顺延(任务非终态 / 任务锁竞争 / 轮内熔断):只推迟下次时间,不计尝试次数——
* 上游故障不该消耗品牌的补测预算。
*/
public void postponeWithoutAttempt(List<CollectDataBrandRecheckEntity> entries, int delayMinutes) {
LocalDateTime nextAttemptAt = LocalDateTime.now()
.plusMinutes(Math.max(1, delayMinutes));
for (CollectDataBrandRecheckEntity entry : entries) {
entry.setNextAttemptAt(nextAttemptAt);
updateQuietly(entry, "postpone");
}
}
/** 第 attempt 次实测失败后的等待分钟数:30m→1h→2h→4h→8h→16h(封顶)。 */
public long backoffMinutes(int attempt) {
long base = Math.max(1, properties.getIntervalBaseMinutes());
int exponent = Math.min(Math.max(1, attempt) - 1, BACKOFF_MAX_EXPONENT);
return base * (1L << exponent);
}
/** 该任务仍在排队的条目数(判断「全部收口」用)。 */
public long countPending(Long taskId) {
Long count = recheckMapper.selectCount(new LambdaQueryWrapper<CollectDataBrandRecheckEntity>()
.eq(CollectDataBrandRecheckEntity::getTaskId, taskId)
.eq(CollectDataBrandRecheckEntity::getStatus, CollectDataBrandRecheckEntity.STATUS_PENDING));
return count == null ? 0 : count;
}
private boolean updateQuietly(CollectDataBrandRecheckEntity entry, String action) {
try {
entry.setUpdatedAt(LocalDateTime.now());
recheckMapper.updateById(entry);
return true;
} catch (RuntimeException ex) {
log.warn("[brand-recheck] 队列状态更新失败(下轮按旧状态重放,注入侧幂等) action={} id={} err={}",
action, entry.getId(), ex.getMessage());
return false;
}
}
private String truncate(String error) {
if (error == null || error.isBlank()) {
return null;
}
String normalized = error.replaceAll("\\s+", " ").trim();
return normalized.length() <= ERROR_MAX_LENGTH ? normalized : normalized.substring(0, ERROR_MAX_LENGTH) + "…";
}
}
@@ -186,6 +186,9 @@ public class CollectDataService {
/** ASIN 跨任务结果缓存:本链只负责"顺带贡献结论",写入失败绝不影响 chunk 提交。 */
private final AsinResultCacheService asinResultCacheService;
/** 品牌检测失败补测队列:查询失败的品牌在锁外入队,上游恢复后自动重查补结果。 */
private final CollectDataBrandRecheckService collectDataBrandRecheckService;
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
@@ -838,6 +841,9 @@ public class CollectDataService {
// (见上方品牌检测移出的注释),缓存写入再快也不该来挤。缓存是"下次少开一次页"的
// 加速件,同一 chunk 重复提交只是把同样的结论再 upsert 一遍,不需要锁内串行语义。
writeAsinCacheOutsideLock(probe, taskId, brandOutcome.accepted());
// 品牌检测查询失败的品牌入补测队列(服务内部吞异常,绝不影响回传主流程):
// 同步重试只能扛秒级抖动,小时级限流窗口(2026-10-02 WIPO 11h)由收尾后补测兜底。
collectDataBrandRecheckService.enqueue(probe, brandOutcome.queryFailed());
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
@@ -1865,8 +1871,9 @@ public class CollectDataService {
* 缓存键里的国家来自任务入参(Python 回传的结果行本身不带国家,见 CollectDataSubmitRowDto)。
* 只收"这一个任务确定只采一个国家"的情形:多选国家时无法判断某一行属于哪个站点,
* 宁可整批不写 —— 键少了国家就会把 DE 的价格当成 US 的结论发回去。
* 包内可见:品牌补测注入器复用同一口径(CollectDataBrandRecheckInjector)。
*/
private String resolveCacheCountry(FileTaskEntity task) {
String resolveCacheCountry(FileTaskEntity task) {
CollectDataFiltersDto filters = extractFiltersFromRequest(task);
String single = filters.getCountryCode();
if (single != null && !single.isBlank()) {
@@ -239,7 +239,8 @@ public class CollectDataBrandBatchFilter {
return String.valueOf(value);
}
private static String normalizeBrand(String value) {
/** 品牌归一化(去 BOM/全角空格、折叠空白、小写):补测队列等链路复用同一口径。 */
public static String normalizeBrand(String value) {
return normalize(value).toLowerCase(Locale.ROOT);
}