feat(撞款): 采集明细落库后自动触发重扫,重复检查无需等次日 00:00
- shopdatacrawl 新增 DuplicateCheckRefreshPort 端口,TaskService 在明细落库成功后请求重扫 - shopduplicatecheck 新增 DuplicateCheckRefreshScheduler:10s 合并窗口去抖、单飞、 扫描锁忙自动重试(最多 6 次),失败不影响采集归档 - 复用现有 SCAN_LOCK,与定时扫描、手动「重新分析」互斥;新增 5 个调度器单测
This commit is contained in:
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -137,6 +138,7 @@ class ShopDataCrawlChunkUpsertTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -16,6 +16,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResu
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -174,6 +175,7 @@ class ShopDataCrawlCleanupTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -146,6 +147,7 @@ class ShopDataCrawlDailyFileIncrementalTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileJobSplitTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileLockTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -134,6 +135,7 @@ class ShopDataCrawlLightweightProgressTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -139,6 +140,7 @@ class ShopDataCrawlOwnerColumnTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -146,6 +147,7 @@ class ShopDataCrawlProgressQueryTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -133,6 +134,7 @@ class ShopDataCrawlRowDedupKeyTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -134,6 +135,7 @@ class ShopDataCrawlScopeCounterTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -135,6 +136,7 @@ class ShopDataCrawlScopeMergeTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -129,6 +130,7 @@ class ShopDataCrawlTaskServiceChunkTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.BooleanSupplier;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
/**
|
||||
* 撞款重扫调度器:合并窗口去抖(突发触发聚为一次扫描)、单飞、
|
||||
* 锁忙有限重试、动作异常吸收(对外零抛出)。
|
||||
*/
|
||||
class DuplicateCheckRefreshSchedulerTest {
|
||||
|
||||
private static final long AWAIT_TIMEOUT_MILLIS = 5_000L;
|
||||
|
||||
@Test
|
||||
void request_mergesBurstIntoSingleScan() throws Exception {
|
||||
AtomicInteger scans = new AtomicInteger();
|
||||
CountDownLatch firstScan = new CountDownLatch(1);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
scans.incrementAndGet();
|
||||
firstScan.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||
}, 150L, 30L, 3);
|
||||
|
||||
scheduler.request("burst-1");
|
||||
Thread.sleep(10L);
|
||||
scheduler.request("burst-2");
|
||||
Thread.sleep(10L);
|
||||
scheduler.request("burst-3");
|
||||
|
||||
assertTrue(firstScan.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "合并窗口内的触发应执行扫描");
|
||||
Thread.sleep(300L);
|
||||
assertEquals(1, scans.get(), "合并窗口内多次触发只执行一次扫描");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_retriesWhileLockBusyThenSucceeds() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
CountDownLatch succeeded = new CountDownLatch(1);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
if (attempts.incrementAndGet() <= 2) {
|
||||
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||
}
|
||||
succeeded.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||
}, 10L, 30L, 5);
|
||||
|
||||
scheduler.request("retry");
|
||||
|
||||
assertTrue(succeeded.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "锁忙重试后应成功执行");
|
||||
Thread.sleep(100L);
|
||||
assertEquals(3, attempts.get(), "2 次锁忙 + 1 次成功");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_givesUpAfterMaxLockRetries() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
CountDownLatch firstAttempt = new CountDownLatch(1);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
attempts.incrementAndGet();
|
||||
firstAttempt.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||
}, 10L, 20L, 2);
|
||||
|
||||
scheduler.request("always-busy");
|
||||
|
||||
assertTrue(firstAttempt.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS));
|
||||
awaitUntil(() -> attempts.get() >= 3, "应完成初试 + 2 次重试");
|
||||
Thread.sleep(200L);
|
||||
assertEquals(3, attempts.get(), "超过重试上限后放弃,不再执行");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_absorbsActionFailure() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
attempts.incrementAndGet();
|
||||
throw new IllegalStateException("模拟扫描动作异常");
|
||||
}, 10L, 20L, 1);
|
||||
|
||||
assertDoesNotThrow(() -> scheduler.request("boom"), "request 不得向调用方抛错");
|
||||
awaitUntil(() -> attempts.get() >= 1, "动作应被执行");
|
||||
Thread.sleep(100L);
|
||||
assertEquals(1, attempts.get(), "动作异常视为失败,不做锁忙重试");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_afterPreviousCycleAllowsNewScan() throws Exception {
|
||||
AtomicInteger scans = new AtomicInteger();
|
||||
CountDownLatch twoScans = new CountDownLatch(2);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
scans.incrementAndGet();
|
||||
twoScans.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||
}, 20L, 20L, 2);
|
||||
|
||||
scheduler.request("first");
|
||||
awaitUntil(() -> scans.get() >= 1, "首轮扫描应执行");
|
||||
scheduler.request("second");
|
||||
|
||||
assertTrue(twoScans.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "新触发应再执行一次扫描");
|
||||
assertEquals(2, scans.get());
|
||||
}
|
||||
|
||||
private static void awaitUntil(BooleanSupplier condition, String message) throws InterruptedException {
|
||||
long deadline = System.currentTimeMillis() + AWAIT_TIMEOUT_MILLIS;
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
if (condition.getAsBoolean()) {
|
||||
return;
|
||||
}
|
||||
Thread.sleep(10L);
|
||||
}
|
||||
fail(message);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user