feat(ASIN缓存): 新增 ASIN 采集结果跨任务缓存(asincache 模块 + V139)
同一个 (国家, ASIN) 的详情页被反复重采:单任务内 62.9 倍重复(43,504 行只有 692 个 不同 ASIN)、8 天跨任务 6.7 万行重复(2026-10-01 实测),每条重复都意味着重开一次页、 可能重买一次代理 IP。关键词链与变体链的结果分别落在两处互相看不见,只有 Java 侧能同时 收到两条链的结论,所以缓存放服务端、由服务端判新鲜度。 - 新表 biz_asin_result_cache(V139):一个 (国家, ASIN) 一行,重采覆盖;只收两种可复用 结论 —— OK(页面正常且有价格,复用 60 分钟)、NOT_FOUND(详情页 404,复用 7 天); 空价 / 风控拦截页 / 采集异常一律不入库,避免把瞬时故障锁成永久结论 - 新鲜度只在服务端判(按 observed_at 真正开页时间);采集端拿不到 / 看不懂一律照常开页 (fail-open),任何情况下不许让缓存导致漏行、错行 - 新增内部接口 /api/internal/asin-cache/query|write(X-Internal-Token 自校验,与 InternalBrandTaskController 同口径),供采集端开页前先问、采完回写 - 关键词采集链在任务锁外旁路写入:缓存写失败的代价只是"下次多开一次页",只记 warn, 不影响 chunk 提交 - migration-inventory.md 快照更新至 V139,「V137 冗余索引 DROP」待办顺延 V140 测试:AsinResultCacheServiceTest(13)/ InternalAsinCacheControllerTest(5)/ 采集链 4 个测试类(35)共 53 例全绿。
This commit is contained in:
+2
-1
@@ -145,7 +145,8 @@ class CollectDataActivateGuardTest {
|
||||
mock(CollectDataInvalidAsinBatchWriter.class), mock(CollectDataResultDetailCodec.class),
|
||||
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
+2
-1
@@ -118,7 +118,8 @@ class CollectDataDeleteConsistencyTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity task(long id, long userId) {
|
||||
|
||||
+47
@@ -57,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -96,6 +97,7 @@ class CollectDataServiceTxBoundaryTest {
|
||||
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
|
||||
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
|
||||
@Mock private CollectDataResultDetailCodec resultDetailCodec;
|
||||
@Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
|
||||
|
||||
@InjectMocks private CollectDataService service;
|
||||
|
||||
@@ -288,6 +290,51 @@ class CollectDataServiceTxBoundaryTest {
|
||||
.enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
|
||||
}
|
||||
|
||||
// ==================== ASIN 跨任务缓存旁路写入 ====================
|
||||
|
||||
/** 关键词链的 accepted 行顺带写进跨任务缓存;国家取自任务 request_json,而非结果行本身。 */
|
||||
@Test
|
||||
void acceptedRowsWrittenToAsinCacheWithTaskCountry() {
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithCountry("DE"));
|
||||
|
||||
service.submitResult(TASK_ID, submitRequest());
|
||||
|
||||
verify(asinResultCacheService).writeCollectDataRows(eq("DE"), eq("6161"), any());
|
||||
}
|
||||
|
||||
/** 多选国家的任务不写缓存:结果行本身不带国家,无法判断某一行属于哪个站点。 */
|
||||
@Test
|
||||
void multiCountryTaskWritesNoCache() {
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithRequestJson(
|
||||
"{\"filters\":{\"countryCodes\":[\"DE\",\"FR\"]}}"));
|
||||
|
||||
service.submitResult(TASK_ID, submitRequest());
|
||||
|
||||
verify(asinResultCacheService).writeCollectDataRows(isNull(), anyString(), any());
|
||||
}
|
||||
|
||||
/** 缓存写挂了不能把已经抓到手、正要落库的数据拖成提交失败 —— 旁路只记日志。 */
|
||||
@Test
|
||||
void asinCacheWriteFailureDoesNotAffectSubmit() {
|
||||
when(asinResultCacheService.writeCollectDataRows(any(), any(), any()))
|
||||
.thenThrow(new RuntimeException("缓存库不可用"));
|
||||
|
||||
CollectDataSubmitResultVo vo = service.submitResult(TASK_ID, submitRequest());
|
||||
|
||||
assertEquals(TASK_ID, vo.getTaskId());
|
||||
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity taskWithCountry(String countryCode) {
|
||||
return taskWithRequestJson("{\"filters\":{\"countryCode\":\"" + countryCode + "\"}}");
|
||||
}
|
||||
|
||||
private FileTaskEntity taskWithRequestJson(String requestJson) {
|
||||
FileTaskEntity task = runningTask();
|
||||
task.setRequestJson(requestJson);
|
||||
return task;
|
||||
}
|
||||
|
||||
private CollectDataSubmitResultRequest submitRequest() {
|
||||
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
|
||||
row.setAsin("B0COLLECT1");
|
||||
|
||||
+2
-1
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
}
|
||||
|
||||
private FileResultEntity result(long id, long taskId, long userId) {
|
||||
|
||||
Reference in New Issue
Block a user