feat(采集数据): 品牌检测失败自动补测队列,限流窗口后自动重查并补结果(V140)
WIPO 限流窗口内 queryFailed 的品牌按任务排队(biz_collectdata_brand_recheck,同任务同品牌幂等),Scheduler 按指数退避自动补测:补出无侵权→行补进结果文件,判出侵权→落无效品牌表,连续耗尽→EXHAUSTED 并通知。只存品牌不存行,注入按 payload_hash 幂等。
This commit is contained in:
+2
-1
@@ -146,7 +146,8 @@ class CollectDataActivateGuardTest {
|
||||
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
|
||||
mock(CollectDataBrandRecheckService.class));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.GroupOutcome;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.RecheckVerdict;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 补测结论注入:CLEAN 行按 chunk 回读并复查去重后补进结果(明细对象 key 带
|
||||
* recheck- 前缀)、REJECTED 行落无效品牌表、结果记录缺失/单 chunk 损坏安全跳过。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class CollectDataBrandRecheckInjectorTest {
|
||||
|
||||
@Mock
|
||||
private com.nanri.aiimage.modules.task.mapper.TaskChunkMapper taskChunkMapper;
|
||||
@Mock
|
||||
private FileResultMapper fileResultMapper;
|
||||
@Mock
|
||||
private CollectDataBatchQuery collectDataBatchQuery;
|
||||
@Mock
|
||||
private CollectDataResultItemBatchWriter resultItemBatchWriter;
|
||||
@Mock
|
||||
private com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
|
||||
@Mock
|
||||
private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock
|
||||
private AsinResultCacheService asinResultCacheService;
|
||||
@Mock
|
||||
private CollectDataService collectDataService;
|
||||
|
||||
private final CollectDataResultDetailCodec codec = new CollectDataResultDetailCodec(new ObjectMapper());
|
||||
private CollectDataBrandRecheckInjector injector;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
TaskChunkEntity.class);
|
||||
injector = new CollectDataBrandRecheckInjector(taskChunkMapper, fileResultMapper,
|
||||
collectDataBatchQuery, codec, resultItemBatchWriter, invalidAsinBatchWriter,
|
||||
transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService);
|
||||
}
|
||||
|
||||
private FileTaskEntity task(long id) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setTaskNo("COLLECT_DATA-" + id);
|
||||
task.setUserId(961L);
|
||||
return task;
|
||||
}
|
||||
|
||||
private FileResultEntity result(long id) {
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(id);
|
||||
result.setTaskId(100L);
|
||||
result.setModuleType("COLLECT_DATA");
|
||||
return result;
|
||||
}
|
||||
|
||||
private TaskChunkEntity chunk(int index, String scopeKey, String scopeHash, String pointer) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId((long) index);
|
||||
chunk.setTaskId(100L);
|
||||
chunk.setModuleType("COLLECT_DATA");
|
||||
chunk.setScopeKey(scopeKey);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(index);
|
||||
chunk.setPayloadJson(pointer);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private CollectDataResultRowVo row(String asin, String brand) {
|
||||
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||
row.setAsin(asin);
|
||||
row.setBrand(brand);
|
||||
row.setPrice("$9.99");
|
||||
return row;
|
||||
}
|
||||
|
||||
@Test
|
||||
void applyInjectsCleanRowsWithRecheckDetailKey() throws Exception {
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(
|
||||
List.of(chunk(1, "s1", "hash1", "ptr1")));
|
||||
String payload = new ObjectMapper().writeValueAsString(List.of(row("B01", "Nike"), row("B02", "Nike")));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload);
|
||||
CollectDataResultRowVo kept = row("B01", "Nike");
|
||||
when(collectDataBatchQuery.filter(anyList())).thenReturn(
|
||||
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0));
|
||||
when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"),
|
||||
eq("recheck-chunk-1"), anyString())).thenReturn("ptr2");
|
||||
when(resultItemBatchWriter.upsertAccepted(eq(100L), eq(9L), eq("s1"), eq(1),
|
||||
eq(List.of(kept)), eq("ptr2"))).thenReturn(new CollectDataResultItemBatchWriter.UpsertCounts(1, 0, 1));
|
||||
when(collectDataService.resolveCacheCountry(any())).thenReturn("US");
|
||||
|
||||
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
|
||||
|
||||
assertThat(outcome.resultId()).isEqualTo(9L);
|
||||
assertThat(outcome.acceptedRows()).isEqualTo(1);
|
||||
assertThat(outcome.rejectedRows()).isZero();
|
||||
verify(asinResultCacheService).writeCollectDataRows(eq("US"), eq("100"), eq(List.of(kept)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void applySendsRejectedRowsToInvalidTableWithoutTouchingResults() {
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(
|
||||
List.of(chunk(1, "s1", "hash1", "ptr1")));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(
|
||||
"[{\"asin\":\"B01\",\"brand\":\"Zara\"},{\"asin\":\"B02\",\"brand\":\"Zara\"}]");
|
||||
when(invalidAsinBatchWriter.writeBatch(anyList())).thenReturn(2);
|
||||
|
||||
GroupOutcome outcome = injector.apply(task(100L), Map.of("zara", RecheckVerdict.REJECTED));
|
||||
|
||||
assertThat(outcome.rejectedRows()).isEqualTo(2);
|
||||
assertThat(outcome.acceptedRows()).isZero();
|
||||
verify(resultItemBatchWriter, never()).upsertAccepted(
|
||||
anyLong(), anyLong(), anyString(), anyInt(), anyList(), anyString());
|
||||
verify(asinResultCacheService, never()).writeCollectDataRows(anyString(), anyString(), anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void applySkipsWhenTaskHasNoResultRecord() {
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(null);
|
||||
|
||||
GroupOutcome outcome = injector.apply(task(100L), Map.of("zara", RecheckVerdict.CLEAN));
|
||||
|
||||
assertThat(outcome).isEqualTo(GroupOutcome.EMPTY);
|
||||
verify(taskChunkMapper, never()).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void applySkipsWhenVerdictsEmptyOrNull() {
|
||||
assertThat(injector.apply(task(100L), Map.of())).isEqualTo(GroupOutcome.EMPTY);
|
||||
assertThat(injector.apply(task(100L), null)).isEqualTo(GroupOutcome.EMPTY);
|
||||
assertThat(injector.apply(null, Map.of("zara", RecheckVerdict.CLEAN))).isEqualTo(GroupOutcome.EMPTY);
|
||||
verify(fileResultMapper, never()).selectOne(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void applyDropsRowsFilteredOutByDedupeRecheck() throws Exception {
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(
|
||||
List.of(chunk(1, "s1", "hash1", "ptr1")));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(
|
||||
"[{\"asin\":\"B01\",\"brand\":\"Nike\"}]");
|
||||
// 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉
|
||||
when(collectDataBatchQuery.filter(anyList())).thenReturn(
|
||||
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0));
|
||||
|
||||
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
|
||||
|
||||
assertThat(outcome.acceptedRows()).isZero();
|
||||
verify(resultItemBatchWriter, never()).upsertAccepted(
|
||||
anyLong(), anyLong(), anyString(), anyInt(), anyList(), anyString());
|
||||
verify(asinResultCacheService, never()).writeCollectDataRows(anyString(), anyString(), anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void applySkipsBrokenChunkWithoutFailingTheGroup() {
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result(9L));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(
|
||||
List.of(chunk(1, "s1", "hash1", "ptr-broken")));
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr-broken"), anyString()))
|
||||
.thenThrow(new RuntimeException("object not found"));
|
||||
|
||||
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
|
||||
|
||||
// 单 chunk 损坏只跳过该 chunk:没有行被注入,但没有抛出、也不影响其余 chunk
|
||||
assertThat(outcome.acceptedRows()).isZero();
|
||||
assertThat(outcome.rejectedRows()).isZero();
|
||||
verify(resultItemBatchWriter, never()).upsertAccepted(
|
||||
anyLong(), anyLong(), anyString(), anyInt(), anyList(), anyString());
|
||||
}
|
||||
}
|
||||
+251
@@ -0,0 +1,251 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
|
||||
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckInjector.GroupOutcome;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 补测调度器:终态门禁、任务锁竞争顺延、结论分派(净/败/仍失败)、
|
||||
* 结果文件重建触发、轮内熔断(样本足够且失败率超阈值时剩余组整组顺延不计数)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class CollectDataBrandRecheckSchedulerTest {
|
||||
|
||||
@Mock
|
||||
private CollectDataBrandRecheckService recheckService;
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private BrandCheckClient brandCheckClient;
|
||||
@Mock
|
||||
private CollectDataBrandRecheckInjector injector;
|
||||
@Mock
|
||||
private NotificationDispatchService notificationDispatchService;
|
||||
@Mock
|
||||
private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock
|
||||
private TaskFileJobService taskFileJobService;
|
||||
@Mock
|
||||
private DistributedJobLockService distributedJobLockService;
|
||||
@Mock
|
||||
private DistributedJobLockService.LockHandle jobLock;
|
||||
@Mock
|
||||
private TaskDistributedLockService.LockHandle taskLock;
|
||||
|
||||
private CollectDataBrandRecheckProperties properties;
|
||||
private CollectDataBrandRecheckScheduler scheduler;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
properties = new CollectDataBrandRecheckProperties();
|
||||
properties.setFirstDelayMinutes(30);
|
||||
properties.setIntervalBaseMinutes(30);
|
||||
properties.setMaxAttempts(7);
|
||||
properties.setRoundBatchSize(200);
|
||||
properties.setFailRateThreshold(0.5);
|
||||
properties.setFailRateMinProbe(3);
|
||||
scheduler = new CollectDataBrandRecheckScheduler(recheckService, properties, fileTaskMapper,
|
||||
brandCheckClient, injector, notificationDispatchService, taskDistributedLockService,
|
||||
taskFileJobService, distributedJobLockService);
|
||||
}
|
||||
|
||||
private CollectDataBrandRecheckEntity entry(long id, long taskId, String brand) {
|
||||
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
|
||||
entry.setId(id);
|
||||
entry.setTaskId(taskId);
|
||||
entry.setTaskNo("COLLECT_DATA-" + taskId);
|
||||
entry.setUserId(961L);
|
||||
entry.setBrand(brand);
|
||||
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_PENDING);
|
||||
entry.setAttemptCount(0);
|
||||
entry.setNextAttemptAt(LocalDateTime.now().minusMinutes(1));
|
||||
return entry;
|
||||
}
|
||||
|
||||
private FileTaskEntity task(long id, String status) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setTaskNo("COLLECT_DATA-" + id);
|
||||
task.setUserId(961L);
|
||||
task.setStatus(status);
|
||||
return task;
|
||||
}
|
||||
|
||||
@Test
|
||||
void scanSkipsWhenJobLockHeldByOtherInstance() {
|
||||
when(distributedJobLockService.tryLock(eq("collectdata-brand-recheck"), any(Duration.class)))
|
||||
.thenReturn(null);
|
||||
|
||||
scheduler.scan();
|
||||
|
||||
verify(recheckService, never()).listDue(any(), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundPostponesEntriesOfRunningTaskWithoutCountingAttempt() {
|
||||
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "RUNNING"));
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
verify(recheckService).postponeWithoutAttempt(List.of(e1), 30);
|
||||
verify(brandCheckClient, never()).checkAll(anyList(), anyString());
|
||||
verify(notificationDispatchService, never()).pushRecheckOutcome(
|
||||
any(), any(), anyString(), anyString(), anyInt(), anyInt(), anyInt(), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundExhaustsEntriesOfMissingTask() {
|
||||
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(null);
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
verify(recheckService).markExhausted(e1, "任务不存在(可能已被清理)");
|
||||
verify(brandCheckClient, never()).checkAll(anyList(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundPostponesWhenTaskLockContended() {
|
||||
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
|
||||
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(null);
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
verify(recheckService).postponeWithoutAttempt(List.of(e1), 30);
|
||||
verify(brandCheckClient, never()).checkAll(anyList(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundAppliesVerdictsTriggersReassemblyAndNotification() {
|
||||
CollectDataBrandRecheckEntity clean = entry(1, 100L, "Nike ");
|
||||
CollectDataBrandRecheckEntity rejected = entry(2, 100L, "Zara");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(clean, rejected));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
|
||||
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
|
||||
// "Nike " 带尾随空格:验证取品牌时按归一化口径去查(入队时已归一化,防御性兜底)
|
||||
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
|
||||
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of("Zara"), List.of()));
|
||||
when(injector.apply(any(FileTaskEntity.class), any())).thenReturn(
|
||||
new GroupOutcome(9L, 3, 1));
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
verify(recheckService).markInjected(clean);
|
||||
verify(recheckService).markRejected(rejected);
|
||||
verify(recheckService, never()).markAttempted(any(), anyString());
|
||||
verify(taskFileJobService).enqueueAssembleResult(100L, "COLLECT_DATA", 9L, "task:100");
|
||||
verify(notificationDispatchService).pushRecheckOutcome(
|
||||
961L, 100L, "COLLECT_DATA", "COLLECT_DATA-100", 3, 1, 0, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundCountsStillFailedAndNotifiesPendingRows() {
|
||||
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
|
||||
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
|
||||
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
|
||||
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(), List.of("zara")));
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
verify(recheckService).markAttempted(e1, "query_faild_data 持续非空");
|
||||
verify(injector, never()).apply(any(), any());
|
||||
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
|
||||
verify(notificationDispatchService).pushRecheckOutcome(
|
||||
961L, 100L, "COLLECT_DATA", "COLLECT_DATA-100", 0, 0, 1, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundTreatsBatchCheckFailureAsFailedAttempt() {
|
||||
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "FAILED"));
|
||||
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
|
||||
when(brandCheckClient.checkAll(anyList(), eq("Terms")))
|
||||
.thenThrow(new RuntimeException("connection reset"));
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
verify(recheckService).markAttempted(eq(e1), contains("connection reset"));
|
||||
verify(injector, never()).apply(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundReplaysGroupWhenInjectionFails() {
|
||||
CollectDataBrandRecheckEntity e1 = entry(1, 100L, "zara");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(e1));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
|
||||
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
|
||||
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
|
||||
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(), List.of()));
|
||||
when(injector.apply(any(FileTaskEntity.class), any()))
|
||||
.thenThrow(new RuntimeException("结果明细写入失败"));
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
verify(recheckService).markAttempted(eq(e1), contains("留队重放"));
|
||||
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
|
||||
// 注入失败:无修复、无收口,不打扰用户
|
||||
verify(notificationDispatchService, never()).pushRecheckOutcome(
|
||||
any(), any(), anyString(), anyString(), anyInt(), anyInt(), anyInt(), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runRoundTripsCircuitBreakerAndPostponesRemainingGroupsWithoutCounting() {
|
||||
CollectDataBrandRecheckEntity f1 = entry(1, 100L, "bad-one");
|
||||
CollectDataBrandRecheckEntity f2 = entry(2, 100L, "bad-two");
|
||||
CollectDataBrandRecheckEntity f3 = entry(3, 100L, "bad-three");
|
||||
CollectDataBrandRecheckEntity later = entry(4, 200L, "later-brand");
|
||||
when(recheckService.listDue(any(), anyInt())).thenReturn(List.of(f1, f2, f3, later));
|
||||
when(fileTaskMapper.selectById(100L)).thenReturn(task(100L, "SUCCESS"));
|
||||
when(taskDistributedLockService.acquire("COLLECT_DATA", 100L, 2000L)).thenReturn(taskLock);
|
||||
when(brandCheckClient.checkAll(anyList(), eq("Terms"))).thenReturn(
|
||||
new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(),
|
||||
List.of("bad-one", "bad-two", "bad-three")));
|
||||
|
||||
scheduler.runRound();
|
||||
|
||||
// 第一组 3 个品牌全部仍失败 → 失败率 100% ≥ 阈值,熔断
|
||||
verify(recheckService).markAttempted(f1, "query_faild_data 持续非空");
|
||||
verify(recheckService).markAttempted(f2, "query_faild_data 持续非空");
|
||||
verify(recheckService).markAttempted(f3, "query_faild_data 持续非空");
|
||||
// 第二组整组顺延:不实测、不计数
|
||||
verify(brandCheckClient, org.mockito.Mockito.times(1)).checkAll(anyList(), anyString());
|
||||
verify(recheckService).postponeWithoutAttempt(List.of(later), 30);
|
||||
verify(recheckService, never()).markInjected(any());
|
||||
verify(recheckService, never()).markRejected(any());
|
||||
}
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.config.CollectDataBrandRecheckProperties;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataBrandRecheckMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataBrandRecheckEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.within;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 补测队列入库语义:入队只增不改(重复失败不刷新/不复活)、退避序列、
|
||||
* maxAttempts 收口、顺延不计次。行数据不落表,入队失败不影响回传主流程。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class CollectDataBrandRecheckServiceTest {
|
||||
|
||||
@Mock
|
||||
private CollectDataBrandRecheckMapper recheckMapper;
|
||||
|
||||
private CollectDataBrandRecheckProperties properties;
|
||||
private CollectDataBrandRecheckService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
CollectDataBrandRecheckEntity.class);
|
||||
properties = new CollectDataBrandRecheckProperties();
|
||||
properties.setFirstDelayMinutes(30);
|
||||
properties.setIntervalBaseMinutes(30);
|
||||
properties.setMaxAttempts(3);
|
||||
service = new CollectDataBrandRecheckService(recheckMapper, properties);
|
||||
}
|
||||
|
||||
private FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(29842L);
|
||||
task.setTaskNo("COLLECT_DATA-2105853640493338624");
|
||||
task.setUserId(961L);
|
||||
return task;
|
||||
}
|
||||
|
||||
private CollectDataResultRowVo row(String asin, String brand) {
|
||||
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||
row.setAsin(asin);
|
||||
row.setBrand(brand);
|
||||
return row;
|
||||
}
|
||||
|
||||
@Test
|
||||
void enqueueSkipsWhenDisabledOrNoRows() {
|
||||
properties.setEnabled(false);
|
||||
assertThat(service.enqueue(task(), List.of(row("B01", "Zara")))).isZero();
|
||||
|
||||
properties.setEnabled(true);
|
||||
assertThat(service.enqueue(task(), List.of())).isZero();
|
||||
assertThat(service.enqueue(task(), null)).isZero();
|
||||
assertThat(service.enqueue(null, List.of(row("B01", "Zara")))).isZero();
|
||||
verify(recheckMapper, never()).insert(any(CollectDataBrandRecheckEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void enqueueSkipsTaskWithoutUserId() {
|
||||
FileTaskEntity noUser = task();
|
||||
noUser.setUserId(null);
|
||||
assertThat(service.enqueue(noUser, List.of(row("B01", "Zara")))).isZero();
|
||||
verify(recheckMapper, never()).insert(any(CollectDataBrandRecheckEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void enqueueInsertsPendingEntryWithFirstDelay() {
|
||||
when(recheckMapper.selectCount(any())).thenReturn(0L);
|
||||
|
||||
int created = service.enqueue(task(), List.of(
|
||||
row("B01", "Zara"),
|
||||
row("B02", " H&M "),
|
||||
row(null, "Zara"))); // 重复品牌只入一次;空行安全跳过
|
||||
|
||||
assertThat(created).isEqualTo(2);
|
||||
ArgumentCaptor<CollectDataBrandRecheckEntity> captor =
|
||||
ArgumentCaptor.forClass(CollectDataBrandRecheckEntity.class);
|
||||
verify(recheckMapper, org.mockito.Mockito.times(2)).insert(captor.capture());
|
||||
for (CollectDataBrandRecheckEntity entity : captor.getAllValues()) {
|
||||
assertThat(entity.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_PENDING);
|
||||
assertThat(entity.getAttemptCount()).isZero();
|
||||
assertThat(entity.getTaskId()).isEqualTo(29842L);
|
||||
assertThat(entity.getUserId()).isEqualTo(961L);
|
||||
assertThat(entity.getNextAttemptAt()).isCloseTo(
|
||||
LocalDateTime.now().plusMinutes(30), within(5, ChronoUnit.SECONDS));
|
||||
}
|
||||
assertThat(captor.getAllValues())
|
||||
.extracting(CollectDataBrandRecheckEntity::getBrand)
|
||||
.containsExactlyInAnyOrder("zara", "h&m");
|
||||
}
|
||||
|
||||
@Test
|
||||
void enqueueSkipsExistingEntryAndSwallowsDuplicateKey() {
|
||||
when(recheckMapper.selectCount(any())).thenReturn(1L, 0L);
|
||||
org.mockito.Mockito.doThrow(new DuplicateKeyException("dup"))
|
||||
.when(recheckMapper).insert(any(CollectDataBrandRecheckEntity.class));
|
||||
|
||||
// 第一个品牌已有行跳过;第二个品牌插入撞唯一键(兄弟 chunk 并发)静默吞掉
|
||||
assertThat(service.enqueue(task(), List.of(row("B01", "Zara"), row("B02", "Nike")))).isZero();
|
||||
verify(recheckMapper, org.mockito.Mockito.times(1)).insert(any(CollectDataBrandRecheckEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void enqueueSwallowsInsertFailureWithoutRethrow() {
|
||||
when(recheckMapper.selectCount(any())).thenReturn(0L);
|
||||
org.mockito.Mockito.doThrow(new RuntimeException("db down"))
|
||||
.when(recheckMapper).insert(any(CollectDataBrandRecheckEntity.class));
|
||||
|
||||
assertThat(service.enqueue(task(), List.of(row("B01", "Zara")))).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void backoffDoublesAndCapsAtSixteenHours() {
|
||||
assertThat(service.backoffMinutes(1)).isEqualTo(30);
|
||||
assertThat(service.backoffMinutes(2)).isEqualTo(60);
|
||||
assertThat(service.backoffMinutes(3)).isEqualTo(120);
|
||||
assertThat(service.backoffMinutes(6)).isEqualTo(960);
|
||||
assertThat(service.backoffMinutes(7)).isEqualTo(960);
|
||||
}
|
||||
|
||||
@Test
|
||||
void markAttemptedSchedulesBackoffAndExhaustsAtMaxAttempts() {
|
||||
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
|
||||
entry.setId(1L);
|
||||
entry.setAttemptCount(0);
|
||||
entry.setStatus(CollectDataBrandRecheckEntity.STATUS_PENDING);
|
||||
|
||||
assertThat(service.markAttempted(entry, "query_faild_data 持续非空")).isTrue();
|
||||
assertThat(entry.getAttemptCount()).isEqualTo(1);
|
||||
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_PENDING);
|
||||
assertThat(entry.getNextAttemptAt()).isCloseTo(
|
||||
LocalDateTime.now().plusMinutes(30), within(5, ChronoUnit.SECONDS));
|
||||
|
||||
service.markAttempted(entry, null);
|
||||
assertThat(entry.getAttemptCount()).isEqualTo(2);
|
||||
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_PENDING);
|
||||
|
||||
service.markAttempted(entry, "still failing");
|
||||
assertThat(entry.getAttemptCount()).isEqualTo(3);
|
||||
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
|
||||
assertThat(entry.getLastError()).contains("still failing");
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalTransitionsSetStatus() {
|
||||
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
|
||||
entry.setId(2L);
|
||||
|
||||
service.markInjected(entry);
|
||||
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_INJECTED);
|
||||
|
||||
service.markRejected(entry);
|
||||
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_REJECTED);
|
||||
|
||||
service.markExhausted(entry, "任务不存在");
|
||||
assertThat(entry.getStatus()).isEqualTo(CollectDataBrandRecheckEntity.STATUS_EXHAUSTED);
|
||||
assertThat(entry.getLastError()).isEqualTo("任务不存在");
|
||||
}
|
||||
|
||||
@Test
|
||||
void postponeKeepsAttemptCountUntouched() {
|
||||
CollectDataBrandRecheckEntity entry = new CollectDataBrandRecheckEntity();
|
||||
entry.setId(3L);
|
||||
entry.setAttemptCount(2);
|
||||
entry.setNextAttemptAt(LocalDateTime.now());
|
||||
|
||||
service.postponeWithoutAttempt(List.of(entry), 30);
|
||||
|
||||
assertThat(entry.getAttemptCount()).isEqualTo(2);
|
||||
assertThat(entry.getNextAttemptAt()).isCloseTo(
|
||||
LocalDateTime.now().plusMinutes(30), within(5, ChronoUnit.SECONDS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void listDueQueriesPendingByNextAttempt() {
|
||||
CollectDataBrandRecheckEntity due = new CollectDataBrandRecheckEntity();
|
||||
due.setId(9L);
|
||||
when(recheckMapper.selectList(any())).thenReturn(List.of(due));
|
||||
|
||||
List<CollectDataBrandRecheckEntity> out =
|
||||
service.listDue(LocalDateTime.now(), 200);
|
||||
|
||||
assertThat(out).containsExactly(due);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countPendingReturnsMapperCount() {
|
||||
when(recheckMapper.selectCount(any(LambdaQueryWrapper.class))).thenReturn(2L);
|
||||
assertThat(service.countPending(29842L)).isEqualTo(2);
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -119,7 +119,8 @@ class CollectDataDeleteConsistencyTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
|
||||
mock(CollectDataBrandRecheckService.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity task(long id, long userId) {
|
||||
|
||||
+1
@@ -98,6 +98,7 @@ class CollectDataServiceTxBoundaryTest {
|
||||
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
|
||||
@Mock private CollectDataResultDetailCodec resultDetailCodec;
|
||||
@Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
|
||||
@Mock private CollectDataBrandRecheckService collectDataBrandRecheckService;
|
||||
|
||||
@InjectMocks private CollectDataService service;
|
||||
|
||||
|
||||
+2
-1
@@ -118,7 +118,8 @@ class CollectDataStorageCallCountTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class),
|
||||
mock(CollectDataBrandRecheckService.class));
|
||||
}
|
||||
|
||||
private FileResultEntity result(long id, long taskId, long userId) {
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package com.nanri.aiimage.modules.notification.service;
|
||||
|
||||
import com.nanri.aiimage.common.mapper.AdminUserMapper;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.permission.support.UserDataScopeSupport;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.startsWith;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
/**
|
||||
* 品牌补测结果通知:补出结论才推、同任务同日一条(dedupe_key 带日期)、
|
||||
* 全部收口未修复时改推「未能完成」指引。只推用户不推管理员(正向收敛不放大铃铛)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class NotificationDispatchServiceRecheckTest {
|
||||
|
||||
@Mock
|
||||
private NotificationService notificationService;
|
||||
@Mock
|
||||
private AdminUserMapper adminUserMapper;
|
||||
@Mock
|
||||
private AdminAuthSupport adminAuthSupport;
|
||||
@Mock
|
||||
private UserDataScopeSupport userDataScopeSupport;
|
||||
|
||||
@InjectMocks
|
||||
private NotificationDispatchService service;
|
||||
|
||||
@Test
|
||||
void pushRecheckOutcomeNotifiesUserWithInfoLevelWhenAllResolved() {
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
|
||||
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
|
||||
eq("info"), contains("补测完成"), contains("新增结果 3 行"), startsWith("collect_recheck:29842:"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pushRecheckOutcomeUsesWarningWhileRowsStillPending() {
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 0, 5, 0);
|
||||
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
|
||||
eq("warning"), contains("补测完成"), contains("仍有 5 行"), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pushRecheckOutcomeStaysSilentOnPureProgressRounds() {
|
||||
// 本轮没有品牌出结论、也没有条目收口:纯排队推进,不打扰
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 3, 0);
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 0, 0);
|
||||
|
||||
verify(notificationService, never()).pushOrRefresh(
|
||||
any(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pushRecheckOutcomeAdvisesManualRerunWhenAllExhausted() {
|
||||
service.pushRecheckOutcome(961L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 0, 0, 0, 4);
|
||||
|
||||
verify(notificationService).pushOrRefresh(eq(961L), eq("user"), eq("task_failed"),
|
||||
eq("warning"), contains("未能完成"), contains("共 4 行"), startsWith("collect_recheck:29842:"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pushRecheckOutcomeSkipsInvalidUser() {
|
||||
service.pushRecheckOutcome(null, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
|
||||
service.pushRecheckOutcome(0L, 29842L, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
|
||||
service.pushRecheckOutcome(961L, null, "COLLECT_DATA", "COLLECT_DATA-29842", 3, 1, 0, 0);
|
||||
|
||||
verify(notificationService, never()).pushOrRefresh(
|
||||
any(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString());
|
||||
}
|
||||
}
|
||||
+1
@@ -96,6 +96,7 @@ class DuplicateSubmissionContractTest {
|
||||
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
|
||||
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
|
||||
@Mock private CollectDataResultDetailCodec resultDetailCodec;
|
||||
@Mock private com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckService collectDataBrandRecheckService;
|
||||
|
||||
@InjectMocks private CollectDataService service;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user