feat(撞款): 采集明细落库后自动触发重扫,重复检查无需等次日 00:00

- shopdatacrawl 新增 DuplicateCheckRefreshPort 端口,TaskService 在明细落库成功后请求重扫
- shopduplicatecheck 新增 DuplicateCheckRefreshScheduler:10s 合并窗口去抖、单飞、
  扫描锁忙自动重试(最多 6 次),失败不影响采集归档
- 复用现有 SCAN_LOCK,与定时扫描、手动「重新分析」互斥;新增 5 个调度器单测
This commit is contained in:
2026-09-17 23:08:15 +08:00
parent a4bc190cae
commit 0a5cbc5912
17 changed files with 343 additions and 2 deletions
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -137,6 +138,7 @@ class ShopDataCrawlChunkUpsertTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -16,6 +16,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResu
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -174,6 +175,7 @@ class ShopDataCrawlCleanupTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -146,6 +147,7 @@ class ShopDataCrawlDailyFileIncrementalTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileJobSplitTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileLockTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -134,6 +135,7 @@ class ShopDataCrawlLightweightProgressTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -139,6 +140,7 @@ class ShopDataCrawlOwnerColumnTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -146,6 +147,7 @@ class ShopDataCrawlProgressQueryTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -133,6 +134,7 @@ class ShopDataCrawlRowDedupKeyTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -134,6 +135,7 @@ class ShopDataCrawlScopeCounterTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -135,6 +136,7 @@ class ShopDataCrawlScopeMergeTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
@@ -129,6 +130,7 @@ class ShopDataCrawlTaskServiceChunkTest {
instanceMetadata,
dailyFileService,
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class));
@@ -0,0 +1,123 @@
package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
import org.junit.jupiter.api.Test;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BooleanSupplier;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
/**
* 撞款重扫调度器:合并窗口去抖(突发触发聚为一次扫描)、单飞、
* 锁忙有限重试、动作异常吸收(对外零抛出)。
*/
class DuplicateCheckRefreshSchedulerTest {
private static final long AWAIT_TIMEOUT_MILLIS = 5_000L;
@Test
void request_mergesBurstIntoSingleScan() throws Exception {
AtomicInteger scans = new AtomicInteger();
CountDownLatch firstScan = new CountDownLatch(1);
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
scans.incrementAndGet();
firstScan.countDown();
return DuplicateCheckRefreshScheduler.Outcome.DONE;
}, 150L, 30L, 3);
scheduler.request("burst-1");
Thread.sleep(10L);
scheduler.request("burst-2");
Thread.sleep(10L);
scheduler.request("burst-3");
assertTrue(firstScan.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "合并窗口内的触发应执行扫描");
Thread.sleep(300L);
assertEquals(1, scans.get(), "合并窗口内多次触发只执行一次扫描");
}
@Test
void request_retriesWhileLockBusyThenSucceeds() throws Exception {
AtomicInteger attempts = new AtomicInteger();
CountDownLatch succeeded = new CountDownLatch(1);
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
if (attempts.incrementAndGet() <= 2) {
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
}
succeeded.countDown();
return DuplicateCheckRefreshScheduler.Outcome.DONE;
}, 10L, 30L, 5);
scheduler.request("retry");
assertTrue(succeeded.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "锁忙重试后应成功执行");
Thread.sleep(100L);
assertEquals(3, attempts.get(), "2 次锁忙 + 1 次成功");
}
@Test
void request_givesUpAfterMaxLockRetries() throws Exception {
AtomicInteger attempts = new AtomicInteger();
CountDownLatch firstAttempt = new CountDownLatch(1);
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
attempts.incrementAndGet();
firstAttempt.countDown();
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
}, 10L, 20L, 2);
scheduler.request("always-busy");
assertTrue(firstAttempt.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS));
awaitUntil(() -> attempts.get() >= 3, "应完成初试 + 2 次重试");
Thread.sleep(200L);
assertEquals(3, attempts.get(), "超过重试上限后放弃,不再执行");
}
@Test
void request_absorbsActionFailure() throws Exception {
AtomicInteger attempts = new AtomicInteger();
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
attempts.incrementAndGet();
throw new IllegalStateException("模拟扫描动作异常");
}, 10L, 20L, 1);
assertDoesNotThrow(() -> scheduler.request("boom"), "request 不得向调用方抛错");
awaitUntil(() -> attempts.get() >= 1, "动作应被执行");
Thread.sleep(100L);
assertEquals(1, attempts.get(), "动作异常视为失败,不做锁忙重试");
}
@Test
void request_afterPreviousCycleAllowsNewScan() throws Exception {
AtomicInteger scans = new AtomicInteger();
CountDownLatch twoScans = new CountDownLatch(2);
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
scans.incrementAndGet();
twoScans.countDown();
return DuplicateCheckRefreshScheduler.Outcome.DONE;
}, 20L, 20L, 2);
scheduler.request("first");
awaitUntil(() -> scans.get() >= 1, "首轮扫描应执行");
scheduler.request("second");
assertTrue(twoScans.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "新触发应再执行一次扫描");
assertEquals(2, scans.get());
}
private static void awaitUntil(BooleanSupplier condition, String message) throws InterruptedException {
long deadline = System.currentTimeMillis() + AWAIT_TIMEOUT_MILLIS;
while (System.currentTimeMillis() < deadline) {
if (condition.getAsBoolean()) {
return;
}
Thread.sleep(10L);
}
fail(message);
}
}