feat(ASIN缓存): 新增 ASIN 采集结果跨任务缓存(asincache 模块 + V139)

同一个 (国家, ASIN) 的详情页被反复重采:单任务内 62.9 倍重复(43,504 行只有 692 个
不同 ASIN)、8 天跨任务 6.7 万行重复(2026-10-01 实测),每条重复都意味着重开一次页、
可能重买一次代理 IP。关键词链与变体链的结果分别落在两处互相看不见,只有 Java 侧能同时
收到两条链的结论,所以缓存放服务端、由服务端判新鲜度。

- 新表 biz_asin_result_cache(V139):一个 (国家, ASIN) 一行,重采覆盖;只收两种可复用
  结论 —— OK(页面正常且有价格,复用 60 分钟)、NOT_FOUND(详情页 404,复用 7 天);
  空价 / 风控拦截页 / 采集异常一律不入库,避免把瞬时故障锁成永久结论
- 新鲜度只在服务端判(按 observed_at 真正开页时间);采集端拿不到 / 看不懂一律照常开页
  (fail-open),任何情况下不许让缓存导致漏行、错行
- 新增内部接口 /api/internal/asin-cache/query|write(X-Internal-Token 自校验,与
  InternalBrandTaskController 同口径),供采集端开页前先问、采完回写
- 关键词采集链在任务锁外旁路写入:缓存写失败的代价只是"下次多开一次页",只记 warn,
  不影响 chunk 提交
- migration-inventory.md 快照更新至 V139,「V137 冗余索引 DROP」待办顺延 V140

测试:AsinResultCacheServiceTest(13)/ InternalAsinCacheControllerTest(5)/ 采集链
4 个测试类(35)共 53 例全绿。
This commit is contained in:
2026-10-01 12:19:05 +08:00
parent 051eab6de5
commit 629fdfab79
20 changed files with 1312 additions and 8 deletions
@@ -0,0 +1,134 @@
package com.nanri.aiimage.modules.asincache.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 内部缓存端点的鉴权与入参绑定。
*
* <p>/api/internal 前缀虽然已在 AdminApiGuardFilter 兜底名单里,controller 仍自校验一次令牌:
* 配置漂移时这两个端点会直接暴露给匿名请求,而它们背后是全站的采集结论与 ASIN 明细。
*/
class InternalAsinCacheControllerTest {
private final AsinResultCacheService asinResultCacheService = mock(AsinResultCacheService.class);
private final AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
private InternalAsinCacheController newController() {
return new InternalAsinCacheController(asinResultCacheService, adminAuthSupport);
}
@Test
void rejectsQueryAndWriteWithoutTrustedInternalToken() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(false);
InternalAsinCacheController controller = newController();
HttpServletRequest request = mock(HttpServletRequest.class);
AsinCacheQueryDto query = new AsinCacheQueryDto();
query.setCountry("DE");
query.setAsins(List.of("B0ABC12345"));
assertThatThrownBy(() -> controller.query(request, query))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("未授权");
AsinCacheWriteDto write = new AsinCacheWriteDto();
assertThatThrownBy(() -> controller.write(request, write))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("未授权");
verifyNoCacheAccess();
}
@Test
void queryWithoutBodyIsRejected() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
assertThatThrownBy(() -> newController().query(mock(HttpServletRequest.class), null))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("缺少查询参数");
verify(asinResultCacheService, never()).query(any(), any());
}
@Test
void queryReturnsServiceResultAsSuccessEnvelope() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
AsinCacheQueryVo vo = new AsinCacheQueryVo();
vo.setCountry("DE");
when(asinResultCacheService.query("DE", List.of("B0ABC12345"))).thenReturn(vo);
AsinCacheQueryDto body = new AsinCacheQueryDto();
body.setCountry("DE");
body.setAsins(List.of("B0ABC12345"));
ApiResponse<AsinCacheQueryVo> resp = newController().query(mock(HttpServletRequest.class), body);
assertThat(resp.isSuccess()).isTrue();
assertThat(resp.getData().getHits()).isEmpty();
}
/**
* 调用方是 Python 采集端:snake_case 是它的自然写法。这里按真实报文形状反序列化一次,
* 免得字段改名后两端各自"看起来都对",线上才发现写入的行全是空值。
*/
@Test
void writeAcceptsPythonShapedPayload() throws Exception {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
when(asinResultCacheService.write(anyList())).thenReturn(new AsinCacheWriteVo());
String pythonJson = """
{"rows": [
{"country": "UK", "ASIN": "B0ABC12345", "availability": "NOT_FOUND",
"declared_members": ["B0ABC12346"], "source_chain": "VARIANT",
"source_task_id": "28599", "brand": "Nike", "delivery_method": "FBA",
"seller_name": "Nike DE", "seller_id": "A1PXL", "price": ""}
]}""";
AsinCacheWriteDto body = new ObjectMapper().readValue(pythonJson, AsinCacheWriteDto.class);
ApiResponse<AsinCacheWriteVo> resp = newController().write(mock(HttpServletRequest.class), body);
assertThat(resp.isSuccess()).isTrue();
verify(asinResultCacheService).write(org.mockito.ArgumentMatchers.argThat(rows ->
rows.size() == 1
&& "UK".equals(rows.get(0).getCountry())
&& "B0ABC12345".equals(rows.get(0).getAsin())
&& "NOT_FOUND".equals(rows.get(0).getAvailability())
&& List.of("B0ABC12346").equals(rows.get(0).getDeclaredMembers())
&& "28599".equals(rows.get(0).getSourceTaskId())
&& "Nike DE".equals(rows.get(0).getSellerName())
&& "A1PXL".equals(rows.get(0).getSellerId())));
}
@Test
void writeWithoutBodyReportsZeroInsteadOfThrowing() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
when(asinResultCacheService.write(eq(null))).thenReturn(new AsinCacheWriteVo());
ApiResponse<AsinCacheWriteVo> resp = newController().write(mock(HttpServletRequest.class), null);
assertThat(resp.getData().getWritten()).isZero();
}
private void verifyNoCacheAccess() {
verify(asinResultCacheService, never()).query(any(), any());
verify(asinResultCacheService, never()).write(any());
}
}
@@ -0,0 +1,267 @@
package com.nanri.aiimage.modules.asincache.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.asincache.mapper.AsinResultCacheMapper;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteRowDto;
import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
/**
* ASIN 跨任务缓存的可复用口径(新鲜度分档 + 哪些结论根本不入库)。
*
* <p>这些数字是花钱的边界:404 复用 7 天、有价复用一个小时内,判宽了采到过期价格,
* 判窄了白开页。所以连同"未知 availability 一律不复用"这种兜底方向一起固化。
*/
class AsinResultCacheServiceTest {
private static final int PRICED_TTL_MINUTES = 60;
private static final int NOT_FOUND_TTL_HOURS = 168;
private static final int MAX_QUERY_ASINS = 500;
private static final int MAX_WRITE_ROWS = 500;
private static final int WRITE_BATCH_SIZE = 100;
private final AsinResultCacheMapper mapper = mock(AsinResultCacheMapper.class);
private final AsinResultCacheService service = new AsinResultCacheService(
mapper, new ObjectMapper(),
PRICED_TTL_MINUTES, NOT_FOUND_TTL_HOURS, MAX_QUERY_ASINS, MAX_WRITE_ROWS, WRITE_BATCH_SIZE);
// ==================== 新鲜度分档 ====================
@Test
void pricedRowReusableWithinTtlOnly() {
assertTrue(service.isReusable(row("OK", "9.90", minutesAgo(59)), LocalDateTime.now()),
"60 分钟内的价格结论应可复用");
assertFalse(service.isReusable(row("OK", "9.90", minutesAgo(61)), LocalDateTime.now()),
"超过 60 分钟的价格结论不可复用");
}
@Test
void notFoundRowReusableForDays() {
assertTrue(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * 24)), LocalDateTime.now()),
"404 按天级复用,省开页最多");
assertFalse(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * NOT_FOUND_TTL_HOURS + 1)),
LocalDateTime.now()), "超过 not-found-ttl-hours 的 404 必须重新确认");
}
/** 空价格 / 拦截页 / 采集异常这类结论即便被人写进库里,读取端也不能复用。 */
@Test
void unknownAvailabilityNeverReusable() {
for (String availability : List.of("BLOCKED", "NO_PRICE", "ERROR", "", "ok")) {
assertFalse(service.isReusable(row(availability, "9.90", minutesAgo(1)), LocalDateTime.now()),
"未知 availability=" + availability + " 必须按不可复用处理(fail-closed)");
}
assertFalse(service.isReusable(row("OK", "", minutesAgo(1)), LocalDateTime.now()),
"OK 但价格为空的历史脏行不可复用");
assertFalse(service.isReusable(null, LocalDateTime.now()));
}
/** 缺 observed_at 时退回 created_at 判新鲜度,而不是直接判死。 */
@Test
void fallsBackToCreatedAtWhenObservedAtMissing() {
AsinResultCacheEntity fresh = row("OK", "9.90", null);
fresh.setCreatedAt(minutesAgo(1));
assertTrue(service.isReusable(fresh, LocalDateTime.now()));
AsinResultCacheEntity stale = row("OK", "9.90", null);
stale.setCreatedAt(minutesAgo(120));
assertFalse(service.isReusable(stale, LocalDateTime.now()));
}
// ==================== 查询 ====================
@Test
void queryNormalizesKeyAndReturnsOnlyReusableHits() {
when(mapper.selectByCountryAndAsins(eq("DE"), anyList())).thenReturn(List.of(
row("OK", "19.90", minutesAgo(2)),
row("OK", "29.90", minutesAgo(720))));
AsinCacheQueryVo vo = service.query(" de ", Arrays.asList("b0abc12345", "B0ABC12345", "", null));
assertEquals("DE", vo.getCountry());
assertEquals(1, vo.getHits().size(), "过期那行不该作为命中返回");
assertEquals("B0ABC12345", vo.getHits().get(0).getAsin());
assertEquals(List.of("B0ABC12345"), capturedAsins(1), "ASIN 去重且大写归一后才进 SQL");
}
@Test
void queryWithoutCountrySkipsDbEntirely() {
AsinCacheQueryVo blank = service.query(" ", List.of("B0ABC12345"));
assertTrue(blank.getHits().isEmpty());
assertNull(blank.getCountry());
verifyNoInteractions(mapper);
}
/** 旁路查询失败对采集端就等于"没缓存":照常开页,不能把采集打成 500。 */
@Test
void queryFailureDegradesToMiss() {
when(mapper.selectByCountryAndAsins(anyString(), anyList()))
.thenThrow(new RuntimeException("cache table missing"));
AsinCacheQueryVo vo = service.query("DE", List.of("B0ABC12345"));
assertTrue(vo.getHits().isEmpty());
}
@Test
void queryOverCapReportsTruncatedAndSplitsIntoChunks() {
when(mapper.selectByCountryAndAsins(anyString(), anyList())).thenReturn(List.of());
List<String> asins = new ArrayList<>();
for (int i = 0; i < MAX_QUERY_ASINS + 40; i++) {
asins.add(String.format("B0%08d", i));
}
AsinCacheQueryVo vo = service.query("DE", asins);
assertEquals(40, vo.getTruncated(), "超出 max-query-asins 的部分要如实上报");
verify(mapper, times(5)).selectByCountryAndAsins(eq("DE"), anyList());
}
// ==================== 写入 ====================
@Test
void writeDropsRowsThatAreNotReusableConclusions() {
List<AsinCacheWriteRowDto> rows = List.of(
writeRow("OK", "9.90"),
writeRow("NOT_FOUND", ""),
writeRow("OK", ""), // 空价格
writeRow("BLOCKED", "9.90"), // 拦截页
writeRow("OK", "9.90", ""), // 无国家
writeRow("OK", "9.90", null, "")); // 无 ASIN
AsinCacheWriteVo vo = service.write(rows);
assertEquals(4, vo.getDropped(), "只有 404 与有价两种结论可入库");
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).upsertBatch(captor.capture());
assertEquals(List.of("OK", "NOT_FOUND"),
captor.getValue().stream().map(AsinResultCacheEntity::getAvailability).toList());
assertEquals("DE", captor.getValue().get(0).getCountry());
}
/** 一个脏值的长度超列宽会让整批 INSERT 抛 Data too long,因此按列裁剪而不是整批丢。 */
@Test
void writeClipsOverlongValues() {
AsinCacheWriteRowDto dto = writeRow("OK", "1".repeat(200));
dto.setBrand("b".repeat(300));
service.write(List.of(dto));
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).upsertBatch(captor.capture());
assertEquals(32, captor.getValue().get(0).getPrice().length());
assertEquals(255, captor.getValue().get(0).getBrand().length());
}
@Test
void writeSplitsByBatchSizeAndSurvivesFailingBatch() {
when(mapper.upsertBatch(anyList())).thenReturn(100).thenThrow(new RuntimeException("锁等待超时"));
List<AsinCacheWriteRowDto> rows = new ArrayList<>();
for (int i = 0; i < 150; i++) {
rows.add(writeRow("NOT_FOUND", ""));
}
AsinCacheWriteVo vo = service.write(rows);
verify(mapper, times(2)).upsertBatch(anyList());
assertEquals(100, vo.getWritten(), "失败批不计入已写,但也不能让整个提交抛出去"); }
// ==================== 关键词链旁路 ====================
@Test
void collectDataBypassSkipsWholeBatchWhenCountryUnknown() {
int written = service.writeCollectDataRows(null, "6161", List.of(collectRow("B0ABC12345", "9.90")));
assertEquals(0, written);
verifyNoInteractions(mapper);
}
@Test
void collectDataBypassWritesOnlyPricedRows() {
when(mapper.upsertBatch(anyList())).thenReturn(1);
service.writeCollectDataRows("de", "6161", List.of(
collectRow("B0ABC12345", "9.90"),
collectRow("B0ABC12346", " ")));
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).upsertBatch(captor.capture());
AsinResultCacheEntity saved = captor.getValue().get(0);
assertEquals(1, captor.getValue().size(), "空价格行不入库");
assertEquals("DE", saved.getCountry());
assertEquals("OK", saved.getAvailability());
assertEquals("COLLECT_DATA", saved.getSourceChain());
assertEquals("6161", saved.getSourceTaskId());
assertNotNull(saved.getObservedAt(), "缺 observed_at 会让新鲜度判定整体失效");
}
private List<String> capturedAsins(int expectedSize) {
ArgumentCaptor<List<String>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).selectByCountryAndAsins(eq("DE"), captor.capture());
assertEquals(expectedSize, captor.getValue().size());
return captor.getValue();
}
private static LocalDateTime minutesAgo(long minutes) {
return LocalDateTime.now().minusMinutes(minutes);
}
private static AsinResultCacheEntity row(String availability, String price, LocalDateTime observedAt) {
AsinResultCacheEntity entity = new AsinResultCacheEntity();
entity.setCountry("DE");
entity.setAsin("B0ABC12345");
entity.setAvailability(availability);
entity.setPrice(price);
entity.setObservedAt(observedAt);
return entity;
}
private static AsinCacheWriteRowDto writeRow(String availability, String price) {
return writeRow(availability, price, "DE");
}
private static AsinCacheWriteRowDto writeRow(String availability, String price, String country) {
return writeRow(availability, price, country, "B0ABC12345");
}
private static AsinCacheWriteRowDto writeRow(String availability, String price, String country, String asin) {
AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto();
dto.setCountry(country);
dto.setAsin(asin);
dto.setAvailability(availability);
dto.setPrice(price);
return dto;
}
private static CollectDataResultRowVo collectRow(String asin, String price) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
row.setPrice(price);
row.setBrand("Demo Brand");
row.setDeliveryMethod("FBA");
row.setSellerName("Demo Seller");
return row;
}
}
@@ -145,7 +145,8 @@ class CollectDataActivateGuardTest {
mock(CollectDataInvalidAsinBatchWriter.class), mock(CollectDataResultDetailCodec.class),
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
@SuppressWarnings("unchecked")
@@ -118,7 +118,8 @@ class CollectDataDeleteConsistencyTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileTaskEntity task(long id, long userId) {
@@ -57,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -96,6 +97,7 @@ class CollectDataServiceTxBoundaryTest {
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
@Mock private CollectDataResultDetailCodec resultDetailCodec;
@Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
@InjectMocks private CollectDataService service;
@@ -288,6 +290,51 @@ class CollectDataServiceTxBoundaryTest {
.enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
}
// ==================== ASIN 跨任务缓存旁路写入 ====================
/** 关键词链的 accepted 行顺带写进跨任务缓存;国家取自任务 request_json,而非结果行本身。 */
@Test
void acceptedRowsWrittenToAsinCacheWithTaskCountry() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithCountry("DE"));
service.submitResult(TASK_ID, submitRequest());
verify(asinResultCacheService).writeCollectDataRows(eq("DE"), eq("6161"), any());
}
/** 多选国家的任务不写缓存:结果行本身不带国家,无法判断某一行属于哪个站点。 */
@Test
void multiCountryTaskWritesNoCache() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithRequestJson(
"{\"filters\":{\"countryCodes\":[\"DE\",\"FR\"]}}"));
service.submitResult(TASK_ID, submitRequest());
verify(asinResultCacheService).writeCollectDataRows(isNull(), anyString(), any());
}
/** 缓存写挂了不能把已经抓到手、正要落库的数据拖成提交失败 —— 旁路只记日志。 */
@Test
void asinCacheWriteFailureDoesNotAffectSubmit() {
when(asinResultCacheService.writeCollectDataRows(any(), any(), any()))
.thenThrow(new RuntimeException("缓存库不可用"));
CollectDataSubmitResultVo vo = service.submitResult(TASK_ID, submitRequest());
assertEquals(TASK_ID, vo.getTaskId());
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
}
private FileTaskEntity taskWithCountry(String countryCode) {
return taskWithRequestJson("{\"filters\":{\"countryCode\":\"" + countryCode + "\"}}");
}
private FileTaskEntity taskWithRequestJson(String requestJson) {
FileTaskEntity task = runningTask();
task.setRequestJson(requestJson);
return task;
}
private CollectDataSubmitResultRequest submitRequest() {
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
row.setAsin("B0COLLECT1");
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileResultEntity result(long id, long taskId, long userId) {