feat(采集数据): 品牌检测失败自动补测队列,限流窗口后自动重查并补结果(V140)

WIPO 限流窗口内 queryFailed 的品牌按任务排队(biz_collectdata_brand_recheck,同任务同品牌幂等),Scheduler 按指数退避自动补测:补出无侵权→行补进结果文件,判出侵权→落无效品牌表,连续耗尽→EXHAUSTED 并通知。只存品牌不存行,注入按 payload_hash 幂等。
This commit is contained in:
2026-10-03 00:22:07 +08:00
parent 68b19f22b3
commit 005cda9379
21 changed files with 1642 additions and 11 deletions
+5 -5
View File
@@ -5,10 +5,10 @@
## 概览
- 版本化迁移文件数:**138**(`src/main/resources/db/V*.sql`,其中整数版本 137 个 + 小版本 V25_1)
- 整数版本范围:**V1..V139**,其中 V132/V133 为已退役空号(见下),其余连续
- 版本化迁移文件数:**139**(`src/main/resources/db/V*.sql`,其中整数版本 138 个 + 小版本 V25_1)
- 整数版本范围:**V1..V140**,其中 V132/V133 为已退役空号(见下),其余连续
- 历史遗留小版本:**V25_1__shop_manage_group_bind_user.sql**(Flyway 语义 25.1,紧跟在 V25 之后、V26 之前执行,属历史命名,保留)
- 最新版本:**V139__asin_result_cache.sql**
- 最新版本:**V140__collectdata_brand_recheck.sql**
- 重复版本:无
- 迁移命名:全部符合 `V<整数>(_<子版本>)?__<描述>.sql`,无空格
@@ -20,9 +20,9 @@
本地曾存在 `V132__web_config_add_is_latest.sql`、`V133__hide_redundant_indexes_invisible.sql`,与另一分支已在生产落库的 V134/V135 冲突。因 Flyway 未开启 `out-of-order`,把任何迁移编到 132/133 会在启动期报 "resolved migration not applied";因此这两条改为 **V136/V137**,132/133 永久退役,不可回填。
后续新增迁移从 **V140** 起取号(只认"当前最大值 + 1",不要为还没写出来的迁移预留号段——预留会在 1..max 里留洞,`integerVersionsSequentialFromOne` 会红)。
后续新增迁移从 **V141** 起取号(只认"当前最大值 + 1",不要为还没写出来的迁移预留号段——预留会在 1..max 里留洞,`integerVersionsSequentialFromOne` 会红)。
原先挂在待办里的"V137 冗余索引 DROP"迁移计划取 V139,该号已被 `V139__asin_result_cache.sql` 占用,顺延到 V140。
原先挂在待办里的"V137 冗余索引 DROP"迁移计划取 V139,该号已被 `V139__asin_result_cache.sql` 占用,顺延到 V140;V140 已被品牌补测队列表 `V140__collectdata_brand_recheck.sql` 落用。
## 约束
@@ -0,0 +1,50 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* 采集数据品牌检测失败补测(requeue)配置:
* 上游 WIPO 限流窗口过去后,把任务里品牌检测「查询失败」的品牌自动重查并补进结果。
*
* <p>节奏设计对标 2026-10-02 的真实限流事件(持续 11 小时):首轮延后 {@link #firstDelayMinutes}
* 分钟避开任务收尾,失败后按 {@link #intervalBaseMinutes} 起指数退避(30m→1h→2h→4h→8h→16h→32h
* 封顶),共 {@link #maxAttempts} 轮实测,合计覆盖约 31 小时的上游抖动窗口。
*/
@Data
@ConfigurationProperties(prefix = "aiimage.collect-data.brand-recheck")
public class CollectDataBrandRecheckProperties {
/** 补测总开关:关闭后不再入队、调度器跳过所有轮次(应急降噪)。 */
private boolean enabled = true;
/** 调度轮询间隔(毫秒),默认 5 分钟;与通知扫描同节拍,双实例由分布式锁互斥。 */
private long scanIntervalMs = 5 * 60 * 1000L;
/** 调度首次执行延迟(毫秒),默认 2 分钟,避开启动阶段。 */
private long scanInitialDelayMs = 2 * 60 * 1000L;
/** 入队后到首次补测的延后分钟数:避开任务收尾与短时限流窗口(客户端同步重试已覆盖)。 */
private int firstDelayMinutes = 30;
/** 退避基准分钟数:第 n 次失败后等待 intervalBaseMinutes × 2^(n-1),封顶 16 小时。 */
private int intervalBaseMinutes = 30;
/**
* 单个品牌最大实测轮次(含首次)。默认 7:失败后等待 30m/1h/2h/4h/8h/16h,
* 首测 + 6 次重测合计覆盖约 31 小时;用尽后条目 EXHAUSTED,不再自动重试。
*/
private int maxAttempts = 7;
/** 单轮最多取出的到期条目数(跨任务合计),防止限流恢复瞬间的补流洪峰。 */
private int roundBatchSize = 200;
/**
* 轮内熔断阈值:本轮已实测品牌中「仍查询失败」占比超过该值(且实测数 ≥
* {@link #failRateMinProbe})时,中止本轮剩余条目——上游仍在限流时不必逐批捶它。
*/
private double failRateThreshold = 0.5;
/** 熔断判定的最小实测品牌数:样本太小不判熔断,避免单品牌偶发失败触发整轮中止。 */
private int failRateMinProbe = 3;
}
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class})
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class})
public class PropertiesConfig {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.collectdata.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface CollectDataBrandRecheckMapper extends BaseMapper<CollectDataBrandRecheckEntity> {
}
@@ -0,0 +1,37 @@
package com.nanri.aiimage.modules.collectdata.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 品牌检测失败补测队列表(V140):分片回传时把「查询失败」的品牌按任务入队,
* 调度器在上游限流窗口过去后重查,出结论的行补进结果文件 / 无效品牌表。
* 只存品牌不存行——行数据留在 chunk 全量 payload(RustFS),注入时按品牌回读。
*/
@Data
@TableName("biz_collectdata_brand_recheck")
public class CollectDataBrandRecheckEntity {
public static final String STATUS_PENDING = "PENDING";
public static final String STATUS_INJECTED = "INJECTED";
public static final String STATUS_REJECTED = "REJECTED";
public static final String STATUS_EXHAUSTED = "EXHAUSTED";
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String taskNo;
private Long userId;
/** 归一化小写品牌,与 CollectDataBrandBatchFilter.normalizeBrand 同口径。 */
private String brand;
private String status;
private Integer attemptCount;
private String lastError;
private LocalDateTime nextAttemptAt;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,215 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 补测结论注入器:把一轮补测里出结论的品牌落回该任务的数据链。
*
* <p>行为与实时分片回传(CollectDataService.submitResult)同语义,只是发生在任务收尾之后:
* <ul>
* <li>CLEAN(无侵权命中)→ 行按 chunk 回读,先过 dedupe/无效品牌复查
* (补测窗口内该 ASIN 可能已被其他任务采走,first-writer-wins),
* 再写 chunk 级明细对象(key 带 recheck- 前缀,不覆盖原 chunk 明细)并
* {@code upsertAccepted} 进结果——重复注入靠 payload_hash 跳过语义幂等;</li>
* <li>REJECTED(判出侵权命中)→ 行落无效品牌表(与实时 rejected 路径一致),不进结果;</li>
* <li>注入的行同步写 ASIN 跨任务缓存(resolveCacheCountry 同口径:只收单国家任务)。</li>
* </ul>
*
* <p>结果文件不在这里重建:注入完成后由调度器 {@code enqueueAssembleResult} 走既有
* 异步装配任务(worker 会按 biz_task_result_item 重算并持久化 finalRowCount)。
* 单 chunk 明细对象写失败即整组抛出:已落库的行靠下轮重放的幂等性收敛,不丢不重。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CollectDataBrandRecheckInjector {
public static final String MODULE_TYPE = "COLLECT_DATA";
/** 补测结论:无侵权命中(行进结果) / 侵权命中(行落无效品牌表)。 */
public enum RecheckVerdict { CLEAN, REJECTED }
/** 一轮注入的落地结果:resultId 供调度器触发结果文件重建;两计数为 0 表示无实际写入。 */
public record GroupOutcome(Long resultId, int acceptedRows, int rejectedRows) {
public static final GroupOutcome EMPTY = new GroupOutcome(null, 0, 0);
}
private final TaskChunkMapper taskChunkMapper;
private final FileResultMapper fileResultMapper;
private final CollectDataBatchQuery collectDataBatchQuery;
private final CollectDataResultDetailCodec resultDetailCodec;
private final CollectDataResultItemBatchWriter resultItemBatchWriter;
private final CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
private final TransientPayloadStorageService transientPayloadStorageService;
private final AsinResultCacheService asinResultCacheService;
private final ObjectMapper objectMapper;
/** 只为复用 resolveCacheCountry 的单国家判定口径(缓存键缺国家会把 DE 价当 US 结论)。 */
private final CollectDataService collectDataService;
/**
* 把补测结论落回任务:按品牌回读 chunk 全量 payload,分流 CLEAN/REJECTED。
* 结论映射键必须与队列表 brand 同口径({@link CollectDataBrandBatchFilter#normalizeBrand})。
*/
public GroupOutcome apply(FileTaskEntity task, Map<String, RecheckVerdict> verdicts) {
if (task == null || task.getId() == null || verdicts == null || verdicts.isEmpty()) {
return GroupOutcome.EMPTY;
}
Set<String> cleanBrands = brandsWith(verdicts, RecheckVerdict.CLEAN);
Set<String> rejectedBrands = brandsWith(verdicts, RecheckVerdict.REJECTED);
if (cleanBrands.isEmpty() && rejectedBrands.isEmpty()) {
return GroupOutcome.EMPTY;
}
FileResultEntity result = fileResultMapper.selectOne(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByDesc(FileResultEntity::getId)
.last("limit 1"));
if (result == null) {
log.warn("[brand-recheck] 任务无结果记录,跳过注入 taskId={}", task.getId());
return GroupOutcome.EMPTY;
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, task.getId())
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getChunkIndex)
.orderByAsc(TaskChunkEntity::getId));
int acceptedRows = 0;
int rejectedRows = 0;
List<CollectDataResultRowVo> acceptedKept = new ArrayList<>();
for (TaskChunkEntity chunk : chunks) {
List<CollectDataResultRowVo> rows = readChunkRows(chunk);
if (rows.isEmpty()) {
continue;
}
if (!rejectedBrands.isEmpty()) {
List<CollectDataResultRowVo> chunkRejected = rowsOfBrands(rows, rejectedBrands);
if (!chunkRejected.isEmpty()) {
// INSERT IGNORE 幂等:重复补测同品牌不会在无效品牌表翻倍
rejectedRows += invalidAsinBatchWriter.writeBatch(chunkRejected);
}
}
if (cleanBrands.isEmpty()) {
continue;
}
List<CollectDataResultRowVo> chunkClean = rowsOfBrands(rows, cleanBrands);
if (chunkClean.isEmpty()) {
continue;
}
// 复查去重/无效品牌:补测窗口内该 ASIN 可能已被其他任务采走(first-writer-wins)
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(chunkClean);
if (filtered.kept().isEmpty()) {
log.info("[brand-recheck] 补测行全部被去重/无效过滤,不注入 taskId={} rows={}",
task.getId(), chunkClean.size());
continue;
}
String detailJson = resultDetailCodec.encodeChunk(filtered.kept());
// key 带 recheck- 前缀:不覆盖原 chunk 的明细对象;同 chunk 重复注入覆盖同一对象(幂等)
String storedDetail = transientPayloadStorageService.storeResultPayload(
MODULE_TYPE, task.getId(), chunk.getScopeHash(),
"recheck-chunk-" + chunk.getChunkIndex(), detailJson);
CollectDataResultItemBatchWriter.UpsertCounts counts = resultItemBatchWriter.upsertAccepted(
task.getId(), result.getId(), chunk.getScopeKey(), chunk.getChunkIndex(),
filtered.kept(), storedDetail);
acceptedRows += counts.newlyInserted();
acceptedKept.addAll(filtered.kept());
}
if (!acceptedKept.isEmpty()) {
writeAsinCache(task, acceptedKept);
}
if (acceptedRows > 0 || rejectedRows > 0) {
log.info("[brand-recheck] 补测结论注入完成 taskId={} 新增结果行={} 判定无效行={} 品牌(净/败)={}/{}",
task.getId(), acceptedRows, rejectedRows, cleanBrands.size(), rejectedBrands.size());
}
return new GroupOutcome(result.getId(), acceptedRows, rejectedRows);
}
private Set<String> brandsWith(Map<String, RecheckVerdict> verdicts, RecheckVerdict verdict) {
Set<String> out = new HashSet<>();
for (Map.Entry<String, RecheckVerdict> entry : verdicts.entrySet()) {
if (entry.getValue() == verdict && entry.getKey() != null && !entry.getKey().isBlank()) {
out.add(entry.getKey());
}
}
return out;
}
private List<CollectDataResultRowVo> rowsOfBrands(List<CollectDataResultRowVo> rows, Set<String> brands) {
List<CollectDataResultRowVo> out = new ArrayList<>();
for (CollectDataResultRowVo row : rows) {
if (row == null) {
continue;
}
String brand = CollectDataBrandBatchFilter.normalizeBrand(row.getBrand());
if (!brand.isBlank() && brands.contains(brand)) {
out.add(row);
}
}
return out;
}
/** 单 chunk 全量行回读(与 CollectDataService.streamRawRows 的 chunk 内逻辑一致)。 */
private List<CollectDataResultRowVo> readChunkRows(TaskChunkEntity chunk) {
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
chunk.getPayloadJson(), "read collect data raw chunk failed");
if (payloadJson == null || payloadJson.isBlank()) {
return List.of();
}
List<CollectDataResultRowVo> values = objectMapper.readValue(payloadJson,
new TypeReference<List<CollectDataResultRowVo>>() {
});
if (values == null) {
return List.of();
}
List<CollectDataResultRowVo> out = new ArrayList<>(values.size());
for (CollectDataResultRowVo value : values) {
if (value != null) {
out.add(value);
}
}
return out;
} catch (Exception ex) {
// 单 chunk 丢失只跳不炸:其余 chunk 的行照常注入,该 chunk 的行留待人工
log.warn("[brand-recheck] chunk 全量行回读失败,跳过该 chunk taskId={} chunkIndex={} err={}",
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
return List.of();
}
}
/** 补测出的行同样"顺带贡献结论"进 ASIN 缓存;写失败不影响注入结果。 */
private void writeAsinCache(FileTaskEntity task, List<CollectDataResultRowVo> rows) {
try {
asinResultCacheService.writeCollectDataRows(
collectDataService.resolveCacheCountry(task), String.valueOf(task.getId()), rows);
} catch (RuntimeException ex) {
log.warn("[brand-recheck] ASIN 缓存写入失败,不影响注入 taskId={} err={}",
task.getId(), ex.getMessage());
}
}
}
@@ -0,0 +1,267 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.RecheckVerdict;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 品牌检测失败补测调度器:把补测队列(biz_collectdata_brand_recheck)里到期的
* PENDING 条目按任务聚组,批量重查品牌检测,出结论的行交给
* {@link CollectDataBrandRecheckInjector} 落回结果文件 / 无效品牌表。
*
* <p>轮内熔断:本轮实测品牌中「仍查询失败」占比超阈值(样本 ≥ failRateMinProbe)时
* 中止本轮剩余条目并顺延(不计尝试次数)——上游仍在限流时不必逐批捶它,
* 2026-10-02 WIPO 连续 11 小时 Forbidden 的形状下,逐批重试只会白白消耗预算。
*
* <p>双实例经 Redis 分布式锁互斥;任务级写并发经任务分布式锁互斥(补测发生在
* 任务终态之后,锁主要挡结果文件装配与注入的竞态)。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CollectDataBrandRecheckScheduler {
private static final String LOCK_NAME = "collectdata-brand-recheck";
private static final String MODULE_TYPE = CollectDataBrandRecheckInjector.MODULE_TYPE;
private static final String STATUS_SUCCESS = "SUCCESS";
private static final String STATUS_FAILED = "FAILED";
private static final long TASK_LOCK_WAIT_MILLIS = 2000L;
private final CollectDataBrandRecheckService recheckService;
private final CollectDataBrandRecheckProperties properties;
private final FileTaskMapper fileTaskMapper;
private final BrandCheckClient brandCheckClient;
private final CollectDataBrandRecheckInjector injector;
private final NotificationDispatchService notificationDispatchService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService;
private final DistributedJobLockService distributedJobLockService;
@Scheduled(fixedDelayString = "${aiimage.collect-data.brand-recheck.scan-interval-ms:300000}",
initialDelayString = "${aiimage.collect-data.brand-recheck.scan-initial-delay-ms:120000}")
public void scan() {
if (!properties.isEnabled()) {
log.info("[brand-recheck] 补测已关闭(brand-recheck.enabled=false),跳过本轮");
return;
}
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(5));
if (lock == null) {
log.info("[brand-recheck] 另一实例持有补测锁,跳过本轮");
return;
}
try (lock) {
runRound();
} catch (Exception ex) {
log.warn("[brand-recheck] 补测轮异常终止 err={}", ex.getMessage(), ex);
}
}
/** 一轮补测:取出到期条目、按任务聚组逐组处理;熔断触发后剩余组整组顺延。 */
void runRound() {
List<CollectDataBrandRecheckEntity> due = recheckService.listDue(
LocalDateTime.now(), properties.getRoundBatchSize());
if (due.isEmpty()) {
log.info("[brand-recheck] 无到期的补测条目");
return;
}
Map<Long, List<CollectDataBrandRecheckEntity>> groups = new LinkedHashMap<>();
for (CollectDataBrandRecheckEntity entry : due) {
groups.computeIfAbsent(entry.getTaskId(), key -> new ArrayList<>()).add(entry);
}
log.info("[brand-recheck] 本轮开始 条目={} 任务组={}", due.size(), groups.size());
int probedBrands = 0;
int stillFailedBrands = 0;
boolean tripped = false;
for (Map.Entry<Long, List<CollectDataBrandRecheckEntity>> group : groups.entrySet()) {
if (tripped) {
recheckService.postponeWithoutAttempt(group.getValue(),
properties.getIntervalBaseMinutes());
continue;
}
GroupRound round = processTaskGroup(group.getKey(), group.getValue());
probedBrands += round.probed();
stillFailedBrands += round.stillFailed();
if (probedBrands >= Math.max(1, properties.getFailRateMinProbe())
&& stillFailedBrands >= probedBrands * properties.getFailRateThreshold()) {
tripped = true;
log.warn("[brand-recheck] 轮内熔断:实测 {} 个品牌仍失败 {},上游疑仍在限流,剩余条目顺延",
probedBrands, stillFailedBrands);
}
}
log.info("[brand-recheck] 本轮结束 实测品牌={} 仍失败={} 熔断={}", probedBrands, stillFailedBrands, tripped);
}
/** 处理单个任务组;返回轮计数(实测品牌数 / 仍失败品牌数),异常按全失败计。 */
private GroupRound processTaskGroup(Long taskId, List<CollectDataBrandRecheckEntity> entries) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null) {
// 任务已删除(清理 SPI 回收):条目无主,直接收口避免永久占坑
for (CollectDataBrandRecheckEntity entry : entries) {
recheckService.markExhausted(entry, "任务不存在(可能已被清理)");
}
return new GroupRound(0, 0);
}
if (!STATUS_SUCCESS.equals(task.getStatus()) && !STATUS_FAILED.equals(task.getStatus())) {
// 任务仍在跑:补测只做终态之后的兜底,顺延不计数(避免与分片重提竞态)
recheckService.postponeWithoutAttempt(entries, properties.getFirstDelayMinutes());
return new GroupRound(0, 0);
}
TaskDistributedLockService.LockHandle taskLock =
taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
if (taskLock == null) {
log.info("[brand-recheck] 任务锁竞争,本轮顺延(不计数) taskId={}", taskId);
recheckService.postponeWithoutAttempt(entries, properties.getFirstDelayMinutes());
return new GroupRound(0, 0);
}
try (taskLock) {
return checkAndApply(task, entries);
}
}
private GroupRound checkAndApply(FileTaskEntity task, List<CollectDataBrandRecheckEntity> entries) {
List<String> brands = distinctBrands(entries);
BrandCheckClient.BrandCheckBatchResult check;
try {
check = brandCheckClient.checkAll(brands, "Terms");
} catch (RuntimeException ex) {
// 整批检查异常(网络/服务级故障):按实测失败计次退避,下轮重试
log.warn("[brand-recheck] 批量品牌检测异常,按失败计次 taskId={} brands={} err={}",
task.getId(), brands.size(), ex.getMessage());
for (CollectDataBrandRecheckEntity entry : entries) {
recheckService.markAttempted(entry, ex.getMessage());
}
return new GroupRound(brands.size(), brands.size());
}
Set<String> faildBrands = normalizeVerdicts(check == null ? null : check.faildData());
Set<String> queryFailedBrands = normalizeVerdicts(check == null ? null : check.queryFaildData());
Map<String, RecheckVerdict> verdicts = new LinkedHashMap<>();
List<CollectDataBrandRecheckEntity> cleanEntries = new ArrayList<>();
List<CollectDataBrandRecheckEntity> rejectedEntries = new ArrayList<>();
List<CollectDataBrandRecheckEntity> stillFailedEntries = new ArrayList<>();
for (CollectDataBrandRecheckEntity entry : entries) {
String brand = CollectDataBrandBatchFilter.normalizeBrand(entry.getBrand());
if (faildBrands.contains(brand)) {
verdicts.put(brand, RecheckVerdict.REJECTED);
rejectedEntries.add(entry);
} else if (queryFailedBrands.contains(brand)) {
stillFailedEntries.add(entry);
} else {
verdicts.put(brand, RecheckVerdict.CLEAN);
cleanEntries.add(entry);
}
}
CollectDataBrandRecheckInjector.GroupOutcome outcome = CollectDataBrandRecheckInjector.GroupOutcome.EMPTY;
boolean applyFailed = false;
if (!verdicts.isEmpty()) {
try {
outcome = injector.apply(task, verdicts);
} catch (RuntimeException ex) {
// 注入失败(如结果明细写批次失败):出结论的品牌留队重放(注入幂等),按实测失败计次
applyFailed = true;
log.warn("[brand-recheck] 补测结论注入失败,整组留队重放 taskId={} err={}",
task.getId(), ex.getMessage());
}
}
if (applyFailed) {
for (CollectDataBrandRecheckEntity entry : entries) {
recheckService.markAttempted(entry, "补测结论注入失败,留队重放");
}
return new GroupRound(brands.size(), brands.size());
}
int exhaustedThisRound = 0;
for (CollectDataBrandRecheckEntity entry : cleanEntries) {
recheckService.markInjected(entry);
}
for (CollectDataBrandRecheckEntity entry : rejectedEntries) {
recheckService.markRejected(entry);
}
for (CollectDataBrandRecheckEntity entry : stillFailedEntries) {
if (recheckService.markAttempted(entry, "query_faild_data 持续非空")) {
// updateQuietly 失败时下轮按 PENDING 重放,不算收口
if (CollectDataBrandRecheckEntity.STATUS_EXHAUSTED.equals(entry.getStatus())) {
exhaustedThisRound++;
}
}
}
if (outcome.acceptedRows() > 0) {
// 结果行有新增 → 复用既有异步装配重建结果文件(worker 重算并持久化 finalRowCount)
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, outcome.resultId(),
"task:" + task.getId());
}
notificationDispatchService.pushRecheckOutcome(task.getUserId(), task.getId(), MODULE_TYPE,
task.getTaskNo(), outcome.acceptedRows(), outcome.rejectedRows(),
stillFailedEntries.size(), exhaustedThisRound);
log.info("[brand-recheck] 任务组完成 taskId={} 品牌={} 净={} 败={} 仍失败={} 收口={} 新增结果行={} 判定无效行={}",
task.getId(), brands.size(), cleanEntries.size(), rejectedEntries.size(),
stillFailedEntries.size(), exhaustedThisRound, outcome.acceptedRows(), outcome.rejectedRows());
return new GroupRound(brands.size(), stillFailedEntries.size());
}
private List<String> distinctBrands(List<CollectDataBrandRecheckEntity> entries) {
Set<String> brands = new LinkedHashSet<>();
for (CollectDataBrandRecheckEntity entry : entries) {
String brand = CollectDataBrandBatchFilter.normalizeBrand(entry.getBrand());
if (!brand.isBlank()) {
brands.add(brand);
}
}
return new ArrayList<>(brands);
}
/** 服务端返回的结论对象(字符串或 {brand,...} 映射)→ 归一化品牌集合,与过滤器同口径。 */
private Set<String> normalizeVerdicts(List<Object> values) {
Set<String> out = new LinkedHashSet<>();
if (values == null) {
return out;
}
for (Object value : values) {
String brand = CollectDataBrandBatchFilter.normalizeBrand(String.valueOf(brandValueOf(value)));
if (!brand.isBlank()) {
out.add(brand);
}
}
return out;
}
private Object brandValueOf(Object value) {
if (value instanceof Map<?, ?> map) {
for (String key : List.of("brand", "brandName", "brand_name", "name", "value", "data_value")) {
Object candidate = map.get(key);
if (candidate != null && !String.valueOf(candidate).isBlank()) {
return candidate;
}
}
}
return value;
}
/** 单任务组的轮计数。 */
private record GroupRound(int probed, int stillFailed) {
}
}
@@ -0,0 +1,203 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataBrandRecheckMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
/**
* 品牌检测失败补测队列的入库语义:入队(分片回传侧,锁外)、到期条目取出、
* 状态推进(实测成功/失败/收口)与退避计算。行数据不落表,注入时由
* {@link CollectDataBrandRecheckInjector} 从 chunk 全量 payload 按品牌回读。
*
* <p>状态机:PENDING →(补测无侵权命中)INJECTED /(判出侵权命中)REJECTED /
* (连续 maxAttempts 轮仍失败)EXHAUSTED。「顺延」(任务非终态、任务锁竞争、轮内熔断)
* 只推迟 next_attempt_at,不计尝试次数——上游故障不该消耗品牌的补测预算。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class CollectDataBrandRecheckService {
/** last_error 截断长度(与列宽 VARCHAR(500) 对齐,留余量)。 */
private static final int ERROR_MAX_LENGTH = 480;
/** 退避封顶分钟数:intervalBaseMinutes × 2^5 = 16 小时(base=30 时)。 */
private static final int BACKOFF_MAX_EXPONENT = 5;
private final CollectDataBrandRecheckMapper recheckMapper;
private final CollectDataBrandRecheckProperties properties;
/**
* 分片回传侧入队:把本 chunk「查询失败」的品牌写进队列。
*
* <p>同任务同品牌唯一:已有行(任何状态)直接跳过——重复失败不刷新首次时间、
* 不改状态、不重置退避(EXHAUSTED 的品牌不因后续 chunk 再次失败而复活)。
* 整段吞掉异常:补测入队失败只损失一次自动重试机会,绝不能影响分片回传主流程。
*
* @return 实际新建的队列条数(诊断日志用)
*/
public int enqueue(FileTaskEntity task, List<CollectDataResultRowVo> queryFailedRows) {
if (!properties.isEnabled() || task == null
|| queryFailedRows == null || queryFailedRows.isEmpty()) {
return 0;
}
Long taskId = task.getId();
Long userId = task.getUserId();
if (taskId == null || userId == null || userId <= 0) {
return 0;
}
Set<String> brands = new LinkedHashSet<>();
for (CollectDataResultRowVo row : queryFailedRows) {
if (row == null) {
continue;
}
String brand = CollectDataBrandBatchFilter.normalizeBrand(row.getBrand());
if (!brand.isBlank()) {
brands.add(brand);
}
}
if (brands.isEmpty()) {
return 0;
}
LocalDateTime nextAttemptAt = LocalDateTime.now()
.plusMinutes(Math.max(1, properties.getFirstDelayMinutes()));
int created = 0;
for (String brand : brands) {
try {
Long existing = recheckMapper.selectCount(new LambdaQueryWrapper<CollectDataBrandRecheckEntity>()
.eq(CollectDataBrandRecheckEntity::getTaskId, taskId)
.eq(CollectDataBrandRecheckEntity::getBrand, brand));
if (existing != null && existing > 0) {
continue;
}
CollectDataBrandRecheckEntity entity = new CollectDataBrandRecheckEntity();
entity.setTaskId(taskId);
entity.setTaskNo(task.getTaskNo());
entity.setUserId(userId);
entity.setBrand(brand);
entity.setStatus(CollectDataBrandRecheckEntity.STATUS_PENDING);
entity.setAttemptCount(0);
entity.setNextAttemptAt(nextAttemptAt);
recheckMapper.insert(entity);
created++;
} catch (DuplicateKeyException dup) {
// 并发提交的兄弟 chunk 抢先入队:语义一致,静默跳过
} catch (RuntimeException ex) {
log.warn("[brand-recheck] 补测入队失败,不影响本次提交 taskId={} brand={} err={}",
taskId, brand, ex.getMessage());
}
}
if (created > 0) {
log.info("[brand-recheck] 品牌检测查询失败入队 taskId={} 新增={} 品牌={}",
taskId, created, brands);
}
return created;
}
/** 取出到期 PENDING 条目(next_attempt_at 升序,先到期先补)。 */
public List<CollectDataBrandRecheckEntity> listDue(LocalDateTime now, int limit) {
return recheckMapper.selectList(new LambdaQueryWrapper<CollectDataBrandRecheckEntity>()
.eq(CollectDataBrandRecheckEntity::getStatus, CollectDataBrandRecheckEntity.STATUS_PENDING)
.le(CollectDataBrandRecheckEntity::getNextAttemptAt, now)
.orderByAsc(CollectDataBrandRecheckEntity::getNextAttemptAt)
.orderByAsc(CollectDataBrandRecheckEntity::getId)
.last("limit " + Math.max(1, limit)));
}
/**
* 实测失败后的推进:尝试次数 +1,按指数退避排下次;达到 maxAttempts 轮转
* EXHAUSTED(不再自动重试,交还用户手动处理)。
*
* @return 该条目是否就此收口(EXHAUSTED)
*/
public boolean markAttempted(CollectDataBrandRecheckEntity entry, String error) {
int attempts = (entry.getAttemptCount() == null ? 0 : entry.getAttemptCount()) + 1;
entry.setAttemptCount(attempts);
entry.setLastError(truncate(error));
if (attempts >= Math.max(1, properties.getMaxAttempts())) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
} else {
entry.setNextAttemptAt(LocalDateTime.now().plusMinutes(backoffMinutes(attempts)));
}
return updateQuietly(entry, "markAttempted");
}
/** 补测出「无侵权命中」结论且行已补进结果:条目终结。 */
public void markInjected(CollectDataBrandRecheckEntity entry) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_INJECTED);
updateQuietly(entry, "markInjected");
}
/** 补测判出侵权命中且行已落无效品牌表:条目终结(行不进结果,与实时路径语义一致)。 */
public void markRejected(CollectDataBrandRecheckEntity entry) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_REJECTED);
updateQuietly(entry, "markRejected");
}
/** 任务不存在等无主条目:直接收口,避免永久占坑。 */
public void markExhausted(CollectDataBrandRecheckEntity entry, String error) {
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
entry.setLastError(truncate(error));
updateQuietly(entry, "markExhausted");
}
/**
* 顺延(任务非终态 / 任务锁竞争 / 轮内熔断):只推迟下次时间,不计尝试次数——
* 上游故障不该消耗品牌的补测预算。
*/
public void postponeWithoutAttempt(List<CollectDataBrandRecheckEntity> entries, int delayMinutes) {
LocalDateTime nextAttemptAt = LocalDateTime.now()
.plusMinutes(Math.max(1, delayMinutes));
for (CollectDataBrandRecheckEntity entry : entries) {
entry.setNextAttemptAt(nextAttemptAt);
updateQuietly(entry, "postpone");
}
}
/** 第 attempt 次实测失败后的等待分钟数:30m→1h→2h→4h→8h→16h(封顶)。 */
public long backoffMinutes(int attempt) {
long base = Math.max(1, properties.getIntervalBaseMinutes());
int exponent = Math.min(Math.max(1, attempt) - 1, BACKOFF_MAX_EXPONENT);
return base * (1L << exponent);
}
/** 该任务仍在排队的条目数(判断「全部收口」用)。 */
public long countPending(Long taskId) {
Long count = recheckMapper.selectCount(new LambdaQueryWrapper<CollectDataBrandRecheckEntity>()
.eq(CollectDataBrandRecheckEntity::getTaskId, taskId)
.eq(CollectDataBrandRecheckEntity::getStatus, CollectDataBrandRecheckEntity.STATUS_PENDING));
return count == null ? 0 : count;
}
private boolean updateQuietly(CollectDataBrandRecheckEntity entry, String action) {
try {
entry.setUpdatedAt(LocalDateTime.now());
recheckMapper.updateById(entry);
return true;
} catch (RuntimeException ex) {
log.warn("[brand-recheck] 队列状态更新失败(下轮按旧状态重放,注入侧幂等) action={} id={} err={}",
action, entry.getId(), ex.getMessage());
return false;
}
}
private String truncate(String error) {
if (error == null || error.isBlank()) {
return null;
}
String normalized = error.replaceAll("\\s+", " ").trim();
return normalized.length() <= ERROR_MAX_LENGTH ? normalized : normalized.substring(0, ERROR_MAX_LENGTH) + "…";
}
}
@@ -186,6 +186,9 @@ public class CollectDataService {
/** ASIN 跨任务结果缓存:本链只负责"顺带贡献结论",写入失败绝不影响 chunk 提交。 */
private final AsinResultCacheService asinResultCacheService;
/** 品牌检测失败补测队列:查询失败的品牌在锁外入队,上游恢复后自动重查补结果。 */
private final CollectDataBrandRecheckService collectDataBrandRecheckService;
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
@@ -838,6 +841,9 @@ public class CollectDataService {
// (见上方品牌检测移出的注释),缓存写入再快也不该来挤。缓存是"下次少开一次页"的
// 加速件,同一 chunk 重复提交只是把同样的结论再 upsert 一遍,不需要锁内串行语义。
writeAsinCacheOutsideLock(probe, taskId, brandOutcome.accepted());
// 品牌检测查询失败的品牌入补测队列(服务内部吞异常,绝不影响回传主流程):
// 同步重试只能扛秒级抖动,小时级限流窗口(2026-10-02 WIPO 11h)由收尾后补测兜底。
collectDataBrandRecheckService.enqueue(probe, brandOutcome.queryFailed());
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
@@ -1865,8 +1871,9 @@ public class CollectDataService {
* 缓存键里的国家来自任务入参(Python 回传的结果行本身不带国家,见 CollectDataSubmitRowDto)。
* 只收"这一个任务确定只采一个国家"的情形:多选国家时无法判断某一行属于哪个站点,
* 宁可整批不写 —— 键少了国家就会把 DE 的价格当成 US 的结论发回去。
* 包内可见:品牌补测注入器复用同一口径(CollectDataBrandRecheckInjector)。
*/
private String resolveCacheCountry(FileTaskEntity task) {
String resolveCacheCountry(FileTaskEntity task) {
CollectDataFiltersDto filters = extractFiltersFromRequest(task);
String single = filters.getCountryCode();
if (single != null && !single.isBlank()) {
@@ -239,7 +239,8 @@ public class CollectDataBrandBatchFilter {
return String.valueOf(value);
}
private static String normalizeBrand(String value) {
/** 品牌归一化(去 BOM/全角空格、折叠空白、小写):补测队列等链路复用同一口径。 */
public static String normalizeBrand(String value) {
return normalize(value).toLowerCase(Locale.ROOT);
}
@@ -129,6 +129,47 @@ public class NotificationDispatchService {
"task_partial_admin:" + taskId, userId);
}
/**
* 「品牌补测出结论」通知:补测队列(biz_collectdata_brand_recheck)在一轮里把
* 查询失败的品牌重查出结论并补进结果文件 / 无效品牌表后,告诉用户结果变了。
*
* <p>只推用户不推管理员:这是部分缺失(task_partial)的自动修复跟进,管理员侧
* 已经在缺失发生时知晓,补测成功属于正向收敛——再扇出一份只放大铃铛噪声。
* 同任务同自然日一条(dedupe_key 带日期):补测按轮推进,同一轮内多品牌合并成一条。
*/
public void pushRecheckOutcome(Long userId, Long taskId, String moduleType, String taskNo,
int injectedRows, int rejectedRows, int pendingRows, int exhaustedRows) {
if (userId == null || userId <= 0 || taskId == null) {
return;
}
boolean fixed = injectedRows > 0 || rejectedRows > 0;
if (!fixed && exhaustedRows <= 0) {
// 本轮没有任何品牌出结论、也没有条目收口:纯排队推进,不打扰用户
return;
}
String label = TaskModuleRegistry.labelOf(moduleType);
String displayTaskNo = taskNo == null || taskNo.isBlank() ? "#" + taskId : taskNo.trim();
String title;
String content;
if (fixed) {
title = label + "任务品牌补测完成";
content = "任务 " + displayTaskNo + ":此前品牌检测未返回结果的行已自动补测出结论"
+ ",新增结果 " + injectedRows + " 行、判定无效品牌 " + rejectedRows + " 行"
+ (pendingRows > 0 ? ",仍有 " + pendingRows + " 行待后续补测" : "")
+ "。结果文件已更新,可在「历史任务」重新下载。";
} else {
title = label + "任务品牌补测未能完成";
content = "任务 " + displayTaskNo + ":品牌检测连续多轮自动补测仍未成功,共 " + exhaustedRows
+ " 行未能补出结论(上游商标库持续不可用)。建议稍后重新提交采集任务重跑这些 ASIN。";
}
String level = fixed && pendingRows <= 0
? NotificationService.LEVEL_INFO : NotificationService.LEVEL_WARNING;
notificationService.pushOrRefresh(userId, NotificationService.AUDIENCE_USER,
NotificationService.SCENE_TASK_FAILED, level, title, content,
"collect_recheck:" + taskId + ":" + java.time.LocalDate.now()
.format(java.time.format.DateTimeFormatter.BASIC_ISO_DATE));
}
/** 解析管理员受众:全部超管 + 管理员(非管理员角色一律排除),并预取各管理员可见用户集。 */
public AdminAudience prepareAdminAudience() {
List<AdminUserEntity> candidates = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
@@ -0,0 +1,39 @@
-- V140: 采集数据品牌检测失败补测队列表
--
-- 背景(2026-10-02 实证):WIPO 全球商标库限流 10:00–21:00 持续 11 小时,两台 java
-- 「brand-check 重试耗尽」2778 次,当天 22 条「部分结果缺失」通知(单任务最高 111 行)。
-- queryFailed 的行按设计不进结果、也不落无效品牌表(task-27265 教训:误杀不可逆),
-- 但现状除一条计数通知外无任何出口——用户既不知道具体哪些 ASIN,也没有自动重试。
-- 本表把「品牌检测失败的品牌」按任务排队,由 CollectDataBrandRecheckScheduler 在
-- 上游限流窗口过去后自动重查,补测出结论的行自动补进结果文件(或判进无效品牌表)。
--
-- 只存品牌不存行:行数据仍以 chunk 全量 payload 形式留在 RustFS(biz_task_chunk),
-- 注入时按品牌回读,避免行级数据双写。
--
-- 状态机:PENDING(排队)→ INJECTED(补测无侵权命中,行已补进结果)/
-- REJECTED(补测判出侵权命中,行已落无效品牌表)/
-- EXHAUSTED(连续 maxAttempts 轮仍失败,不再自动重试)。
--
-- 幂等:同任务同品牌一行(uk_recheck_task_brand),chunk 重复失败不刷新首次时间、
-- 不改状态、不重置退避;结果行注入靠 biz_task_result_item 的 payload_hash 跳过语义幂等。
--
-- 回滚(手工):
-- DROP TABLE IF EXISTS biz_collectdata_brand_recheck;
CREATE TABLE IF NOT EXISTS biz_collectdata_brand_recheck (
id BIGINT NOT NULL AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'biz_file_task.id(COLLECT_DATA 模块)',
task_no VARCHAR(64) NULL COMMENT '任务号(COLLECT_DATA-雪花,仅展示/排查用)',
user_id BIGINT NOT NULL COMMENT '任务归属用户(补测结果通知收件人)',
brand VARCHAR(255) NOT NULL COMMENT '品牌(归一化小写,与 CollectDataBrandBatchFilter.normalizeBrand 同口径)',
status VARCHAR(16) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING / INJECTED / REJECTED / EXHAUSTED',
attempt_count INT NOT NULL DEFAULT 0 COMMENT '已实际执行的补测轮次(顺延不计)',
last_error VARCHAR(500) NULL COMMENT '最近一次补测失败原因(截断)',
next_attempt_at DATETIME NOT NULL COMMENT '下次补测时间(失败后按 30m 起指数退避)',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
PRIMARY KEY (id),
UNIQUE KEY uk_recheck_task_brand (task_id, brand) COMMENT '同任务同品牌一行',
KEY idx_recheck_status_next (status, next_attempt_at) COMMENT '到期 PENDING 条目轮询'
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = DYNAMIC
COMMENT = '采集数据品牌检测失败补测队列(上游限流窗口过去后自动重查并补结果)';
@@ -146,7 +146,8 @@ class CollectDataActivateGuardTest {
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
mock(CollectDataBrandRecheckService.class));
}
@SuppressWarnings("unchecked")
@@ -0,0 +1,209 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.GroupOutcome;
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.RecheckVerdict;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 补测结论注入:CLEAN 行按 chunk 回读并复查去重后补进结果(明细对象 key 带
* recheck- 前缀)、REJECTED 行落无效品牌表、结果记录缺失/单 chunk 损坏安全跳过。
*/
@ExtendWith(MockitoExtension.class)
class CollectDataBrandRecheckInjectorTest {
@Mock
private com.nanri.aiimage.modules.task.mapper.TaskChunkMapper taskChunkMapper;
@Mock
private FileResultMapper fileResultMapper;
@Mock
private CollectDataBatchQuery collectDataBatchQuery;
@Mock
private CollectDataResultItemBatchWriter resultItemBatchWriter;
@Mock
private com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
@Mock
private TransientPayloadStorageService transientPayloadStorageService;
@Mock
private AsinResultCacheService asinResultCacheService;
@Mock
private CollectDataService collectDataService;
private final CollectDataResultDetailCodec codec = new CollectDataResultDetailCodec(new ObjectMapper());
private CollectDataBrandRecheckInjector injector;
@BeforeEach
void setUp() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
FileResultEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
TaskChunkEntity.class);
injector = new CollectDataBrandRecheckInjector(taskChunkMapper, fileResultMapper,
collectDataBatchQuery, codec, resultItemBatchWriter, invalidAsinBatchWriter,
transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService);
}
private FileTaskEntity task(long id) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setTaskNo("COLLECT_DATA-" + id);
task.setUserId(961L);
return task;
}
private FileResultEntity result(long id) {
FileResultEntity result = new FileResultEntity();
result.setId(id);
result.setTaskId(100L);
result.setModuleType("COLLECT_DATA");
return result;
}
private TaskChunkEntity chunk(int index, String scopeKey, String scopeHash, String pointer) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId((long) index);
chunk.setTaskId(100L);
chunk.setModuleType("COLLECT_DATA");
chunk.setScopeKey(scopeKey);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(index);
chunk.setPayloadJson(pointer);
return chunk;
}
private CollectDataResultRowVo row(String asin, String brand) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
row.setBrand(brand);
row.setPrice("$9.99");
return row;
}
@Test
void applyInjectsCleanRowsWithRecheckDetailKey() throws Exception {
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
when(taskChunkMapper.selectList(any())).thenReturn(
List.of(chunk(1, "s1", "hash1", "ptr1")));
String payload = new ObjectMapper().writeValueAsString(List.of(row("B01", "Nike"), row("B02", "Nike")));
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload);
CollectDataResultRowVo kept = row("B01", "Nike");
when(collectDataBatchQuery.filter(anyList())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0));
when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"),
eq("recheck-chunk-1"), anyString())).thenReturn("ptr2");
when(resultItemBatchWriter.upsertAccepted(eq(100L), eq(9L), eq("s1"), eq(1),
eq(List.of(kept)), eq("ptr2"))).thenReturn(new CollectDataResultItemBatchWriter.UpsertCounts(1, 0, 1));
when(collectDataService.resolveCacheCountry(any())).thenReturn("US");
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
assertThat(outcome.resultId()).isEqualTo(9L);
assertThat(outcome.acceptedRows()).isEqualTo(1);
assertThat(outcome.rejectedRows()).isZero();
verify(asinResultCacheService).writeCollectDataRows(eq("US"), eq("100"), eq(List.of(kept)));
}
@Test
void applySendsRejectedRowsToInvalidTableWithoutTouchingResults() {
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
when(taskChunkMapper.selectList(any())).thenReturn(
List.of(chunk(1, "s1", "hash1", "ptr1")));
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(
"[{\"asin\":\"B01\",\"brand\":\"Zara\"},{\"asin\":\"B02\",\"brand\":\"Zara\"}]");
when(invalidAsinBatchWriter.writeBatch(anyList())).thenReturn(2);
GroupOutcome outcome = injector.apply(task(100L), Map.of("zara", RecheckVerdict.REJECTED));
assertThat(outcome.rejectedRows()).isEqualTo(2);
assertThat(outcome.acceptedRows()).isZero();
verify(resultItemBatchWriter, never()).upsertAccepted(
anyLong(), anyLong(), anyString(), anyInt(), anyList(), anyString());
verify(asinResultCacheService, never()).writeCollectDataRows(anyString(), anyString(), anyList());
}
@Test
void applySkipsWhenTaskHasNoResultRecord() {
when(fileResultMapper.selectOne(any())).thenReturn(null);
GroupOutcome outcome = injector.apply(task(100L), Map.of("zara", RecheckVerdict.CLEAN));
assertThat(outcome).isEqualTo(GroupOutcome.EMPTY);
verify(taskChunkMapper, never()).selectList(any());
}
@Test
void applySkipsWhenVerdictsEmptyOrNull() {
assertThat(injector.apply(task(100L), Map.of())).isEqualTo(GroupOutcome.EMPTY);
assertThat(injector.apply(task(100L), null)).isEqualTo(GroupOutcome.EMPTY);
assertThat(injector.apply(null, Map.of("zara", RecheckVerdict.CLEAN))).isEqualTo(GroupOutcome.EMPTY);
verify(fileResultMapper, never()).selectOne(any());
}
@Test
void applyDropsRowsFilteredOutByDedupeRecheck() throws Exception {
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
when(taskChunkMapper.selectList(any())).thenReturn(
List.of(chunk(1, "s1", "hash1", "ptr1")));
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(
"[{\"asin\":\"B01\",\"brand\":\"Nike\"}]");
// 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉
when(collectDataBatchQuery.filter(anyList())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0));
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
assertThat(outcome.acceptedRows()).isZero();
verify(resultItemBatchWriter, never()).upsertAccepted(
anyLong(), anyLong(), anyString(), anyInt(), anyList(), anyString());
verify(asinResultCacheService, never()).writeCollectDataRows(anyString(), anyString(), anyList());
}
@Test
void applySkipsBrokenChunkWithoutFailingTheGroup() {
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
when(taskChunkMapper.selectList(any())).thenReturn(
List.of(chunk(1, "s1", "hash1", "ptr-broken")));
when(transientPayloadStorageService.resolvePayload(eq("ptr-broken"), anyString()))
.thenThrow(new RuntimeException("object not found"));
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
// 单 chunk 损坏只跳过该 chunk:没有行被注入,但没有抛出、也不影响其余 chunk
assertThat(outcome.acceptedRows()).isZero();
assertThat(outcome.rejectedRows()).isZero();
verify(resultItemBatchWriter, never()).upsertAccepted(
anyLong(), anyLong(), anyString(), anyInt(), anyList(), anyString());
}
}
@@ -0,0 +1,251 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.GroupOutcome;
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 补测调度器:终态门禁、任务锁竞争顺延、结论分派(净/败/仍失败)、
* 结果文件重建触发、轮内熔断(样本足够且失败率超阈值时剩余组整组顺延不计数)。
*/
@ExtendWith(MockitoExtension.class)
class CollectDataBrandRecheckSchedulerTest {
@Mock
private CollectDataBrandRecheckService recheckService;
@Mock
private FileTaskMapper fileTaskMapper;
@Mock
private BrandCheckClient brandCheckClient;
@Mock
private CollectDataBrandRecheckInjector injector;
@Mock
private NotificationDispatchService notificationDispatchService;
@Mock
private TaskDistributedLockService taskDistributedLockService;
@Mock
private TaskFileJobService taskFileJobService;
@Mock
private DistributedJobLockService distributedJobLockService;
@Mock
private DistributedJobLockService.LockHandle jobLock;
@Mock
private TaskDistributedLockService.LockHandle taskLock;
private CollectDataBrandRecheckProperties properties;
private CollectDataBrandRecheckScheduler scheduler;
@BeforeEach
void setUp() {
properties = new CollectDataBrandRecheckProperties();
properties.setFirstDelayMinutes(30);
properties.setIntervalBaseMinutes(30);
properties.setMaxAttempts(7);
properties.setRoundBatchSize(200);
properties.setFailRateThreshold(0.5);
properties.setFailRateMinProbe(3);
scheduler = new CollectDataBrandRecheckScheduler(recheckService, properties, fileTaskMapper,
brandCheckClient, injector, notificationDispatchService, taskDistributedLockService,
taskFileJobService, distributedJobLockService);
}
private CollectDataBrandRecheckEntity entry(long id, long taskId, String brand) {
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
entry.setId(id);
entry.setTaskId(taskId);
entry.setTaskNo("COLLECT_DATA-" + taskId);
entry.setUserId(961L);
entry.setBrand(brand);
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_PENDING);
entry.setAttemptCount(0);
entry.setNextAttemptAt(LocalDateTime.now().minusMinutes(1));
return entry;
}
private FileTaskEntity task(long id, String status) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setTaskNo("COLLECT_DATA-" + id);
task.setUserId(961L);
task.setStatus(status);
return task;
}
@Test
void scanSkipsWhenJobLockHeldByOtherInstance() {
when(distributedJobLockService.tryLock(eq("collectdata-brand-recheck"), any(Duration.class)))
.thenReturn(null);
scheduler.scan();
verify(recheckService, never()).listDue(any(), anyInt());
}
@Test
void runRoundPostponesEntriesOfRunningTaskWithoutCountingAttempt() {
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "RUNNING"));
scheduler.runRound();
verify(recheckService).postponeWithoutAttempt(List.of(e1), 30);
verify(brandCheckClient, never()).checkAll(anyList(), anyString());
verify(notificationDispatchService, never()).pushRecheckOutcome(
any(), any(), anyString(), anyString(), anyInt(), anyInt(), anyInt(), anyInt());
}
@Test
void runRoundExhaustsEntriesOfMissingTask() {
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
when(fileTaskMapper.selectById(100L)).thenReturn(null);
scheduler.runRound();
verify(recheckService).markExhausted(e1, "任务不存在(可能已被清理)");
verify(brandCheckClient, never()).checkAll(anyList(), anyString());
}
@Test
void runRoundPostponesWhenTaskLockContended() {
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(null);
scheduler.runRound();
verify(recheckService).postponeWithoutAttempt(List.of(e1), 30);
verify(brandCheckClient, never()).checkAll(anyList(), anyString());
}
@Test
void runRoundAppliesVerdictsTriggersReassemblyAndNotification() {
CollectDataBrandRecheckEntity clean = entry(1, 100L, "Nike ");
CollectDataBrandRecheckEntity rejected = entry(2, 100L, "Zara");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(clean, rejected));
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
// "Nike " 带尾随空格:验证取品牌时按归一化口径去查(入队时已归一化,防御性兜底)
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of("Zara"), List.of()));
when(injector.apply(any(FileTaskEntity.class), any())).thenReturn(
new GroupOutcome(9L, 3, 1));
scheduler.runRound();
verify(recheckService).markInjected(clean);
verify(recheckService).markRejected(rejected);
verify(recheckService, never()).markAttempted(any(), anyString());
verify(taskFileJobService).enqueueAssembleResult(100L, "COLLECT_DATA", 9L, "task:100");
verify(notificationDispatchService).pushRecheckOutcome(
961L, 100L, "COLLECT_DATA", "COLLECT_DATA-100", 3, 1, 0, 0);
}
@Test
void runRoundCountsStillFailedAndNotifiesPendingRows() {
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(), List.of("zara")));
scheduler.runRound();
verify(recheckService).markAttempted(e1, "query_faild_data 持续非空");
verify(injector, never()).apply(any(), any());
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
verify(notificationDispatchService).pushRecheckOutcome(
961L, 100L, "COLLECT_DATA", "COLLECT_DATA-100", 0, 0, 1, 0);
}
@Test
void runRoundTreatsBatchCheckFailureAsFailedAttempt() {
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "FAILED"));
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
when(brandCheckClient.checkAll(anyList(), eq("Terms")))
.thenThrow(new RuntimeException("connection reset"));
scheduler.runRound();
verify(recheckService).markAttempted(eq(e1), contains("connection reset"));
verify(injector, never()).apply(any(), any());
}
@Test
void runRoundReplaysGroupWhenInjectionFails() {
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(), List.of()));
when(injector.apply(any(FileTaskEntity.class), any()))
.thenThrow(new RuntimeException("结果明细写入失败"));
scheduler.runRound();
verify(recheckService).markAttempted(eq(e1), contains("留队重放"));
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
// 注入失败:无修复、无收口,不打扰用户
verify(notificationDispatchService, never()).pushRecheckOutcome(
any(), any(), anyString(), anyString(), anyInt(), anyInt(), anyInt(), anyInt());
}
@Test
void runRoundTripsCircuitBreakerAndPostponesRemainingGroupsWithoutCounting() {
CollectDataBrandRecheckEntity f1 = entry(1, 100L, "bad-one");
CollectDataBrandRecheckEntity f2 = entry(2, 100L, "bad-two");
CollectDataBrandRecheckEntity f3 = entry(3, 100L, "bad-three");
CollectDataBrandRecheckEntity later = entry(4, 200L, "later-brand");
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(f1, f2, f3, later));
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(),
List.of("bad-one", "bad-two", "bad-three")));
scheduler.runRound();
// 第一组 3 个品牌全部仍失败 → 失败率 100% ≥ 阈值,熔断
verify(recheckService).markAttempted(f1, "query_faild_data 持续非空");
verify(recheckService).markAttempted(f2, "query_faild_data 持续非空");
verify(recheckService).markAttempted(f3, "query_faild_data 持续非空");
// 第二组整组顺延:不实测、不计数
verify(brandCheckClient, org.mockito.Mockito.times(1)).checkAll(anyList(), anyString());
verify(recheckService).postponeWithoutAttempt(List.of(later), 30);
verify(recheckService, never()).markInjected(any());
verify(recheckService, never()).markRejected(any());
}
}
@@ -0,0 +1,215 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataBrandRecheckMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.dao.DuplicateKeyException;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.within;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 补测队列入库语义:入队只增不改(重复失败不刷新/不复活)、退避序列、
* maxAttempts 收口、顺延不计次。行数据不落表,入队失败不影响回传主流程。
*/
@ExtendWith(MockitoExtension.class)
class CollectDataBrandRecheckServiceTest {
@Mock
private CollectDataBrandRecheckMapper recheckMapper;
private CollectDataBrandRecheckProperties properties;
private CollectDataBrandRecheckService service;
@BeforeEach
void setUp() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
CollectDataBrandRecheckEntity.class);
properties = new CollectDataBrandRecheckProperties();
properties.setFirstDelayMinutes(30);
properties.setIntervalBaseMinutes(30);
properties.setMaxAttempts(3);
service = new CollectDataBrandRecheckService(recheckMapper, properties);
}
private FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(29842L);
task.setTaskNo("COLLECT_DATA-2105853640493338624");
task.setUserId(961L);
return task;
}
private CollectDataResultRowVo row(String asin, String brand) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
row.setBrand(brand);
return row;
}
@Test
void enqueueSkipsWhenDisabledOrNoRows() {
properties.setEnabled(false);
assertThat(service.enqueue(task(), List.of(row("B01", "Zara")))).isZero();
properties.setEnabled(true);
assertThat(service.enqueue(task(), List.of())).isZero();
assertThat(service.enqueue(task(), null)).isZero();
assertThat(service.enqueue(null, List.of(row("B01", "Zara")))).isZero();
verify(recheckMapper, never()).insert(any(CollectDataBrandRecheckEntity.class));
}
@Test
void enqueueSkipsTaskWithoutUserId() {
FileTaskEntity noUser = task();
noUser.setUserId(null);
assertThat(service.enqueue(noUser, List.of(row("B01", "Zara")))).isZero();
verify(recheckMapper, never()).insert(any(CollectDataBrandRecheckEntity.class));
}
@Test
void enqueueInsertsPendingEntryWithFirstDelay() {
when(recheckMapper.selectCount(any())).thenReturn(0L);
int created = service.enqueue(task(), List.of(
row("B01", "Zara"),
row("B02", " H&M "),
row(null, "Zara"))); // 重复品牌只入一次;空行安全跳过
assertThat(created).isEqualTo(2);
ArgumentCaptor<CollectDataBrandRecheckEntity> captor =
ArgumentCaptor.forClass(CollectDataBrandRecheckEntity.class);
verify(recheckMapper, org.mockito.Mockito.times(2)).insert(captor.capture());
for (CollectDataBrandRecheckEntity entity : captor.getAllValues()) {
assertThat(entity.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_PENDING);
assertThat(entity.getAttemptCount()).isZero();
assertThat(entity.getTaskId()).isEqualTo(29842L);
assertThat(entity.getUserId()).isEqualTo(961L);
assertThat(entity.getNextAttemptAt()).isCloseTo(
LocalDateTime.now().plusMinutes(30), within(5, ChronoUnit.SECONDS));
}
assertThat(captor.getAllValues())
.extracting(CollectDataBrandRecheckEntity::getBrand)
.containsExactlyInAnyOrder("zara", "h&m");
}
@Test
void enqueueSkipsExistingEntryAndSwallowsDuplicateKey() {
when(recheckMapper.selectCount(any())).thenReturn(1L, 0L);
org.mockito.Mockito.doThrow(new DuplicateKeyException("dup"))
.when(recheckMapper).insert(any(CollectDataBrandRecheckEntity.class));
// 第一个品牌已有行跳过;第二个品牌插入撞唯一键(兄弟 chunk 并发)静默吞掉
assertThat(service.enqueue(task(), List.of(row("B01", "Zara"), row("B02", "Nike")))).isZero();
verify(recheckMapper, org.mockito.Mockito.times(1)).insert(any(CollectDataBrandRecheckEntity.class));
}
@Test
void enqueueSwallowsInsertFailureWithoutRethrow() {
when(recheckMapper.selectCount(any())).thenReturn(0L);
org.mockito.Mockito.doThrow(new RuntimeException("db down"))
.when(recheckMapper).insert(any(CollectDataBrandRecheckEntity.class));
assertThat(service.enqueue(task(), List.of(row("B01", "Zara")))).isZero();
}
@Test
void backoffDoublesAndCapsAtSixteenHours() {
assertThat(service.backoffMinutes(1)).isEqualTo(30);
assertThat(service.backoffMinutes(2)).isEqualTo(60);
assertThat(service.backoffMinutes(3)).isEqualTo(120);
assertThat(service.backoffMinutes(6)).isEqualTo(960);
assertThat(service.backoffMinutes(7)).isEqualTo(960);
}
@Test
void markAttemptedSchedulesBackoffAndExhaustsAtMaxAttempts() {
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
entry.setId(1L);
entry.setAttemptCount(0);
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_PENDING);
assertThat(service.markAttempted(entry, "query_faild_data 持续非空")).isTrue();
assertThat(entry.getAttemptCount()).isEqualTo(1);
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_PENDING);
assertThat(entry.getNextAttemptAt()).isCloseTo(
LocalDateTime.now().plusMinutes(30), within(5, ChronoUnit.SECONDS));
service.markAttempted(entry, null);
assertThat(entry.getAttemptCount()).isEqualTo(2);
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_PENDING);
service.markAttempted(entry, "still failing");
assertThat(entry.getAttemptCount()).isEqualTo(3);
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
assertThat(entry.getLastError()).contains("still failing");
}
@Test
void terminalTransitionsSetStatus() {
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
entry.setId(2L);
service.markInjected(entry);
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_INJECTED);
service.markRejected(entry);
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_REJECTED);
service.markExhausted(entry, "任务不存在");
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
assertThat(entry.getLastError()).isEqualTo("任务不存在");
}
@Test
void postponeKeepsAttemptCountUntouched() {
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
entry.setId(3L);
entry.setAttemptCount(2);
entry.setNextAttemptAt(LocalDateTime.now());
service.postponeWithoutAttempt(List.of(entry), 30);
assertThat(entry.getAttemptCount()).isEqualTo(2);
assertThat(entry.getNextAttemptAt()).isCloseTo(
LocalDateTime.now().plusMinutes(30), within(5, ChronoUnit.SECONDS));
}
@Test
void listDueQueriesPendingByNextAttempt() {
CollectDataBrandRecheckEntity due = new CollectDataBrandRecheckEntity();
due.setId(9L);
when(recheckMapper.selectList(any())).thenReturn(List.of(due));
List<CollectDataBrandRecheckEntity> out =
service.listDue(LocalDateTime.now(), 200);
assertThat(out).containsExactly(due);
}
@Test
void countPendingReturnsMapperCount() {
when(recheckMapper.selectCount(any(LambdaQueryWrapper.class))).thenReturn(2L);
assertThat(service.countPending(29842L)).isEqualTo(2);
}
}
@@ -119,7 +119,8 @@ class CollectDataDeleteConsistencyTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
mock(CollectDataBrandRecheckService.class));
}
private FileTaskEntity task(long id, long userId) {
@@ -98,6 +98,7 @@ class CollectDataServiceTxBoundaryTest {
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
@Mock private CollectDataResultDetailCodec resultDetailCodec;
@Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
@Mock private CollectDataBrandRecheckService collectDataBrandRecheckService;
@InjectMocks private CollectDataService service;
@@ -118,7 +118,8 @@ class CollectDataStorageCallCountTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
mock(CollectDataBrandRecheckService.class));
}
private FileResultEntity result(long id, long taskId, long userId) {
@@ -0,0 +1,82 @@
package com.nanri.aiimage.modules.notification.service;
import com.nanri.aiimage.common.mapper.AdminUserMapper;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.support.UserDataScopeSupport;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.startsWith;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
/**
* 品牌补测结果通知:补出结论才推、同任务同日一条(dedupe_key 带日期)、
* 全部收口未修复时改推「未能完成」指引。只推用户不推管理员(正向收敛不放大铃铛)。
*/
@ExtendWith(MockitoExtension.class)
class NotificationDispatchServiceRecheckTest {
@Mock
private NotificationService notificationService;
@Mock
private AdminUserMapper adminUserMapper;
@Mock
private AdminAuthSupport adminAuthSupport;
@Mock
private UserDataScopeSupport userDataScopeSupport;
@InjectMocks
private NotificationDispatchService service;
@Test
void pushRecheckOutcomeNotifiesUserWithInfoLevelWhenAllResolved() {
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
eq("info"), contains("补测完成"), contains("新增结果 3 行"), startsWith("collect_recheck:29842:"));
}
@Test
void pushRecheckOutcomeUsesWarningWhileRowsStillPending() {
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 0, 5, 0);
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
eq("warning"), contains("补测完成"), contains("仍有 5 行"), anyString());
}
@Test
void pushRecheckOutcomeStaysSilentOnPureProgressRounds() {
// 本轮没有品牌出结论、也没有条目收口:纯排队推进,不打扰
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 3, 0);
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 0, 0);
verify(notificationService, never()).pushOrRefresh(
any(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString());
}
@Test
void pushRecheckOutcomeAdvisesManualRerunWhenAllExhausted() {
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 0, 4);
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
eq("warning"), contains("未能完成"), contains("共 4 行"), startsWith("collect_recheck:29842:"));
}
@Test
void pushRecheckOutcomeSkipsInvalidUser() {
service.pushRecheckOutcome(null, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
service.pushRecheckOutcome(0L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
service.pushRecheckOutcome(961L, null, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
verify(notificationService, never()).pushOrRefresh(
any(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString());
}
}
@@ -96,6 +96,7 @@ class DuplicateSubmissionContractTest {
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
@Mock private CollectDataResultDetailCodec resultDetailCodec;
@Mock private com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckService collectDataBrandRecheckService;
@InjectMocks private CollectDataService service;