feat(ASIN缓存): 新增 ASIN 采集结果跨任务缓存(asincache 模块 + V139)

同一个 (国家, ASIN) 的详情页被反复重采:单任务内 62.9 倍重复(43,504 行只有 692 个
不同 ASIN)、8 天跨任务 6.7 万行重复(2026-10-01 实测),每条重复都意味着重开一次页、
可能重买一次代理 IP。关键词链与变体链的结果分别落在两处互相看不见,只有 Java 侧能同时
收到两条链的结论,所以缓存放服务端、由服务端判新鲜度。

- 新表 biz_asin_result_cache(V139):一个 (国家, ASIN) 一行,重采覆盖;只收两种可复用
  结论 —— OK(页面正常且有价格,复用 60 分钟)、NOT_FOUND(详情页 404,复用 7 天);
  空价 / 风控拦截页 / 采集异常一律不入库,避免把瞬时故障锁成永久结论
- 新鲜度只在服务端判(按 observed_at 真正开页时间);采集端拿不到 / 看不懂一律照常开页
  (fail-open),任何情况下不许让缓存导致漏行、错行
- 新增内部接口 /api/internal/asin-cache/query|write(X-Internal-Token 自校验,与
  InternalBrandTaskController 同口径),供采集端开页前先问、采完回写
- 关键词采集链在任务锁外旁路写入:缓存写失败的代价只是"下次多开一次页",只记 warn,
  不影响 chunk 提交
- migration-inventory.md 快照更新至 V139,「V137 冗余索引 DROP」待办顺延 V140

测试:AsinResultCacheServiceTest(13)/ InternalAsinCacheControllerTest(5)/ 采集链
4 个测试类(35)共 53 例全绿。
This commit is contained in:
2026-10-01 12:19:05 +08:00
parent 051eab6de5
commit 629fdfab79
20 changed files with 1312 additions and 8 deletions
@@ -145,7 +145,8 @@ class CollectDataActivateGuardTest {
mock(CollectDataInvalidAsinBatchWriter.class), mock(CollectDataResultDetailCodec.class),
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
@SuppressWarnings("unchecked")
@@ -118,7 +118,8 @@ class CollectDataDeleteConsistencyTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileTaskEntity task(long id, long userId) {
@@ -57,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -96,6 +97,7 @@ class CollectDataServiceTxBoundaryTest {
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
@Mock private CollectDataResultDetailCodec resultDetailCodec;
@Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
@InjectMocks private CollectDataService service;
@@ -288,6 +290,51 @@ class CollectDataServiceTxBoundaryTest {
.enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
}
// ==================== ASIN 跨任务缓存旁路写入 ====================
/** 关键词链的 accepted 行顺带写进跨任务缓存;国家取自任务 request_json,而非结果行本身。 */
@Test
void acceptedRowsWrittenToAsinCacheWithTaskCountry() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithCountry("DE"));
service.submitResult(TASK_ID, submitRequest());
verify(asinResultCacheService).writeCollectDataRows(eq("DE"), eq("6161"), any());
}
/** 多选国家的任务不写缓存:结果行本身不带国家,无法判断某一行属于哪个站点。 */
@Test
void multiCountryTaskWritesNoCache() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithRequestJson(
"{\"filters\":{\"countryCodes\":[\"DE\",\"FR\"]}}"));
service.submitResult(TASK_ID, submitRequest());
verify(asinResultCacheService).writeCollectDataRows(isNull(), anyString(), any());
}
/** 缓存写挂了不能把已经抓到手、正要落库的数据拖成提交失败 —— 旁路只记日志。 */
@Test
void asinCacheWriteFailureDoesNotAffectSubmit() {
when(asinResultCacheService.writeCollectDataRows(any(), any(), any()))
.thenThrow(new RuntimeException("缓存库不可用"));
CollectDataSubmitResultVo vo = service.submitResult(TASK_ID, submitRequest());
assertEquals(TASK_ID, vo.getTaskId());
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
}
private FileTaskEntity taskWithCountry(String countryCode) {
return taskWithRequestJson("{\"filters\":{\"countryCode\":\"" + countryCode + "\"}}");
}
private FileTaskEntity taskWithRequestJson(String requestJson) {
FileTaskEntity task = runningTask();
task.setRequestJson(requestJson);
return task;
}
private CollectDataSubmitResultRequest submitRequest() {
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
row.setAsin("B0COLLECT1");
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileResultEntity result(long id, long taskId, long userId) {