From 629fdfab791866478e85d5967b091b90bf4441f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Thu, 1 Oct 2026 12:19:05 +0800 Subject: [PATCH] =?UTF-8?q?feat(ASIN=E7=BC=93=E5=AD=98):=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=20ASIN=20=E9=87=87=E9=9B=86=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E8=B7=A8=E4=BB=BB=E5=8A=A1=E7=BC=93=E5=AD=98=EF=BC=88asincache?= =?UTF-8?q?=20=E6=A8=A1=E5=9D=97=20+=20V139=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 同一个 (国家, ASIN) 的详情页被反复重采:单任务内 62.9 倍重复(43,504 行只有 692 个 不同 ASIN)、8 天跨任务 6.7 万行重复(2026-10-01 实测),每条重复都意味着重开一次页、 可能重买一次代理 IP。关键词链与变体链的结果分别落在两处互相看不见,只有 Java 侧能同时 收到两条链的结论,所以缓存放服务端、由服务端判新鲜度。 - 新表 biz_asin_result_cache(V139):一个 (国家, ASIN) 一行,重采覆盖;只收两种可复用 结论 —— OK(页面正常且有价格,复用 60 分钟)、NOT_FOUND(详情页 404,复用 7 天); 空价 / 风控拦截页 / 采集异常一律不入库,避免把瞬时故障锁成永久结论 - 新鲜度只在服务端判(按 observed_at 真正开页时间);采集端拿不到 / 看不懂一律照常开页 (fail-open),任何情况下不许让缓存导致漏行、错行 - 新增内部接口 /api/internal/asin-cache/query|write(X-Internal-Token 自校验,与 InternalBrandTaskController 同口径),供采集端开页前先问、采完回写 - 关键词采集链在任务锁外旁路写入:缓存写失败的代价只是"下次多开一次页",只记 warn, 不影响 chunk 提交 - migration-inventory.md 快照更新至 V139,「V137 冗余索引 DROP」待办顺延 V140 测试:AsinResultCacheServiceTest(13)/ InternalAsinCacheControllerTest(5)/ 采集链 4 个测试类(35)共 53 例全绿。 --- backend-java/docs/migration-inventory.md | 12 +- .../InternalAsinCacheController.java | 80 ++++ .../mapper/AsinResultCacheMapper.java | 73 ++++ .../model/dto/AsinCacheQueryDto.java | 20 + .../model/dto/AsinCacheWriteDto.java | 17 + .../model/dto/AsinCacheWriteRowDto.java | 69 ++++ .../model/entity/AsinResultCacheEntity.java | 41 ++ .../asincache/model/vo/AsinCacheHitVo.java | 39 ++ .../asincache/model/vo/AsinCacheQueryVo.java | 21 + .../asincache/model/vo/AsinCacheWriteVo.java | 18 + .../service/AsinResultCacheService.java | 374 ++++++++++++++++++ .../service/CollectDataService.java | 44 +++ .../src/main/resources/application.yml | 10 + .../resources/db/V139__asin_result_cache.sql | 45 +++ .../InternalAsinCacheControllerTest.java | 134 +++++++ .../service/AsinResultCacheServiceTest.java | 267 +++++++++++++ .../service/CollectDataActivateGuardTest.java | 3 +- .../CollectDataDeleteConsistencyTest.java | 3 +- .../CollectDataServiceTxBoundaryTest.java | 47 +++ .../CollectDataStorageCallCountTest.java | 3 +- 20 files changed, 1312 insertions(+), 8 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/mapper/AsinResultCacheMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheQueryDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteRowDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/entity/AsinResultCacheEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheQueryVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheWriteVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheService.java create mode 100644 backend-java/src/main/resources/db/V139__asin_result_cache.sql create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheControllerTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheServiceTest.java diff --git a/backend-java/docs/migration-inventory.md b/backend-java/docs/migration-inventory.md index 0eb889227..8890b64ff 100644 --- a/backend-java/docs/migration-inventory.md +++ b/backend-java/docs/migration-inventory.md @@ -1,14 +1,14 @@ # Flyway 迁移盘点(task-193) > 生成方式:`src/test/java/com/nanri/aiimage/config/MigrationInventoryTest.java`(只读审计,可重复)。 -> 快照日期:2026-09-28。 +> 快照日期:2026-10-01。 ## 概览 -- 版本化迁移文件数:**137**(`src/main/resources/db/V*.sql`) -- 整数版本范围:**V1..V138**,其中 V132/V133 为已退役空号(见下),其余连续 +- 版本化迁移文件数:**138**(`src/main/resources/db/V*.sql`,其中整数版本 137 个 + 小版本 V25_1) +- 整数版本范围:**V1..V139**,其中 V132/V133 为已退役空号(见下),其余连续 - 历史遗留小版本:**V25_1__shop_manage_group_bind_user.sql**(Flyway 语义 25.1,紧跟在 V25 之后、V26 之前执行,属历史命名,保留) -- 最新版本:**V138__client_task_console.sql** +- 最新版本:**V139__asin_result_cache.sql** - 重复版本:无 - 迁移命名:全部符合 `V<整数>(_<子版本>)?__<描述>.sql`,无空格 @@ -20,7 +20,9 @@ 本地曾存在 `V132__web_config_add_is_latest.sql`、`V133__hide_redundant_indexes_invisible.sql`,与另一分支已在生产落库的 V134/V135 冲突。因 Flyway 未开启 `out-of-order`,把任何迁移编到 132/133 会在启动期报 "resolved migration not applied";因此这两条改为 **V136/V137**,132/133 永久退役,不可回填。 -后续新增迁移从 **V139** 起取号。 +后续新增迁移从 **V140** 起取号(只认"当前最大值 + 1",不要为还没写出来的迁移预留号段——预留会在 1..max 里留洞,`integerVersionsSequentialFromOne` 会红)。 + +原先挂在待办里的"V137 冗余索引 DROP"迁移计划取 V139,该号已被 `V139__asin_result_cache.sql` 占用,顺延到 V140。 ## 约束 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java new file mode 100644 index 000000000..03fff9518 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java @@ -0,0 +1,80 @@ +package com.nanri.aiimage.modules.asincache.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.security.AdminAuthSupport; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo; +import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 内部接口:ASIN 采集结果缓存的读写,供采集端(麦象采集机 / 桌面端)在开页前先问一次 + * 「这个 (国家, ASIN) 是否已经有人开过」。 + * + *
鉴权与 {@code InternalBrandTaskController} 同口径:/api/internal 前缀已在 + * AdminApiGuardFilter 兜底名单内,controller 仍自校验 X-Internal-Token —— 防止配置漂移时匿名可达。 + * + *
这两个端点是旁路:采集端拿不到响应时必须照常开页(fail-open),所以
+ * 服务端任何异常都只可能让采集变慢,绝不可能让数据变少或变错。
+ */
+@RestController
+@RequiredArgsConstructor
+@Slf4j
+@RequestMapping("/api/internal/asin-cache")
+@Tag(name = "内部接口", description = "服务间调用(X-Internal-Token 鉴权)。")
+public class InternalAsinCacheController {
+
+ private final AsinResultCacheService asinResultCacheService;
+ private final AdminAuthSupport adminAuthSupport;
+
+ @PostMapping("/query")
+ @Operation(summary = "批量查可复用的 ASIN 采集结论(采集端调用)",
+ description = "只返回新鲜度达标(OK 60 分钟 / NOT_FOUND 7 天,均可配置)的行;"
+ + "未出现在 hits 里的 ASIN 按未命中处理,调用方照常开页。")
+ public ApiResponse 走唯一键 uk_asin_cache_country_asin (country, asin) 的前缀,所以必须是
+ * 「country = ? AND asin IN (...)」这种等值 + IN 的形状;把 country 放进 IN 里
+ * 成对匹配((country, asin) IN ((..),(..)))在 MySQL 上用不上索引。
+ */
+ @Select("""
+
+ """)
+ List 两条采集链往同一行里写的字段并不一样多 —— 关键词链没有 seller_id 也没有变体
+ * 家族,变体链没有 seller 信息。这些"这次没采到"的列在实体里是 NULL(空白值入库前
+ * 已被 clip 成 NULL),所以它们用 COALESCE 继承旧值:新观测带了就覆盖,没带就保留,
+ * 否则一条变体记录会把关键词记录里的卖家名抹成空,下次谁都读不到。
+ *
+ * price / availability / source_* / observed_at 不参与继承:它们是本行结论的主体,
+ * 404 就该把价格清空,"谁最后开的页"也必须如实反映最后一次开页。
+ */
+ @Insert("""
+
+ """)
+ int upsertBatch(@Param("rows") List 字段一律同时接受 camelCase 与 snake_case:调用方是 Python 采集端,snake_case 是它的
+ * 自然写法;而 Java 侧序列化出来的是 camelCase,两头都认就不用任何一侧迁就。
+ */
+@Data
+@Schema(description = "ASIN 缓存写入行(内部接口)")
+public class AsinCacheWriteRowDto {
+
+ @JsonAlias({"country_code", "countryCode"})
+ @Schema(description = "国家代码", example = "UK")
+ private String country;
+
+ @JsonAlias({"ASIN", "asin"})
+ @Schema(description = "商品 ASIN", example = "B0XXXXXXX")
+ private String asin;
+
+ @JsonAlias({"availability", "status"})
+ @Schema(description = "结论类型:OK(页面正常且有价格)/ NOT_FOUND(404 不存在)。"
+ + "其余取值服务端一律不入库 —— 空价/拦截页/采集异常都可能是采集端自己的解析或风控问题",
+ example = "OK")
+ private String availability;
+
+ @Schema(description = "价格原文(保留当地币种符号,不数值化)", example = "£12.34")
+ private String price;
+
+ @JsonAlias({"品牌", "brand_name"})
+ @Schema(description = "品牌")
+ private String brand;
+
+ @JsonAlias({"delivery_method", "fulfillment"})
+ @Schema(description = "配送方式", example = "FBA")
+ private String deliveryMethod;
+
+ @JsonAlias({"seller_name", "卖家名称"})
+ @Schema(description = "卖家名称")
+ private String sellerName;
+
+ @JsonAlias({"seller_id", "sellerID", "merchant_id"})
+ @Schema(description = "merchantID")
+ private String sellerId;
+
+ @JsonAlias({"declared_members", "declaredMembers"})
+ @Schema(description = "该 ASIN 页面自己声明的变体成员列表;跨任务跳过开页时靠它补齐产出行")
+ private List 一个 (country, asin) 一行,重采即覆盖;新鲜度按 {@code observedAt}(采集机上真正
+ * 打开页面的时间)算,不按入库时间 —— 否则客户端攒批延迟回传会把旧结论算成新结论。
+ */
+@Data
+@TableName("biz_asin_result_cache")
+public class AsinResultCacheEntity {
+
+ /** 可复用结论:页面正常且取到价格。 */
+ public static final String AVAILABILITY_OK = "OK";
+ /** 可复用结论:详情页 404,该 ASIN 不存在。 */
+ public static final String AVAILABILITY_NOT_FOUND = "NOT_FOUND";
+
+ @TableId(type = IdType.AUTO)
+ private Long id;
+ private String country;
+ private String asin;
+ private String availability;
+ private String price;
+ private String brand;
+ private String deliveryMethod;
+ private String sellerName;
+ private String sellerId;
+ private String declaredMembersJson;
+ private String sourceChain;
+ private String sourceTaskId;
+ private LocalDateTime observedAt;
+ private LocalDateTime createdAt;
+ private LocalDateTime updatedAt;
+}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java
new file mode 100644
index 000000000..28fddefb6
--- /dev/null
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java
@@ -0,0 +1,39 @@
+package com.nanri.aiimage.modules.asincache.model.vo;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.Data;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+@Data
+@Schema(description = "可复用的 ASIN 缓存结论")
+public class AsinCacheHitVo {
+
+ @Schema(description = "商品 ASIN")
+ private String asin;
+
+ @Schema(description = "结论类型:OK / NOT_FOUND")
+ private String availability;
+
+ @Schema(description = "价格原文;NOT_FOUND 时为空")
+ private String price;
+
+ @Schema(description = "品牌")
+ private String brand;
+
+ @Schema(description = "配送方式")
+ private String deliveryMethod;
+
+ @Schema(description = "卖家名称")
+ private String sellerName;
+
+ @Schema(description = "merchantID")
+ private String sellerId;
+
+ @Schema(description = "该 ASIN 页面声明过的变体成员列表(跳过开页时靠它补齐产出行)")
+ private List 新鲜度只在这里判,采集端不参与。两条链的采集端版本不可能同时升级,
+ * 把口径放在服务端意味着老采集端拿到 404/未知字段只会当没缓存用(fail-open),
+ * 而不是各按各的理解复用到过期数据。
+ *
+ * 可复用只有两档(其余一律不入库,见 {@link #ttlMinutesOf}):
+ * 单批失败只记日志、不抛:调用方是采集热路径与关键词结果提交链路,
+ * 缓存写不进去的代价是"下次多开一次页",把任务整体拖失败是更大的代价。
+ */
+ public AsinCacheWriteVo write(List country 缺失时整批不写 —— 缓存键少了国家,就可能把 DE 的价格当成 US 的结论发回去。
+ * 关键词链按 ASIN 逐个采详情页,一批 accepted 行同属一个任务、一个国家,所以这里
+ * 只按行计数,不逐行判国家。
+ */
+ public int writeCollectDataRows(String country, String sourceTaskId,
+ List 整段吞掉 RuntimeException:缓存写不进去的代价是"下次多开一次页",把已经抓到手、
+ * 正要落库的 chunk 提交拖失败是更大的代价。
+ */
+ private void writeAsinCacheOutsideLock(FileTaskEntity task, Long taskId,
+ List /api/internal 前缀虽然已在 AdminApiGuardFilter 兜底名单里,controller 仍自校验一次令牌:
+ * 配置漂移时这两个端点会直接暴露给匿名请求,而它们背后是全站的采集结论与 ASIN 明细。
+ */
+class InternalAsinCacheControllerTest {
+
+ private final AsinResultCacheService asinResultCacheService = mock(AsinResultCacheService.class);
+ private final AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
+
+ private InternalAsinCacheController newController() {
+ return new InternalAsinCacheController(asinResultCacheService, adminAuthSupport);
+ }
+
+ @Test
+ void rejectsQueryAndWriteWithoutTrustedInternalToken() {
+ when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(false);
+ InternalAsinCacheController controller = newController();
+ HttpServletRequest request = mock(HttpServletRequest.class);
+
+ AsinCacheQueryDto query = new AsinCacheQueryDto();
+ query.setCountry("DE");
+ query.setAsins(List.of("B0ABC12345"));
+ assertThatThrownBy(() -> controller.query(request, query))
+ .isInstanceOf(BusinessException.class)
+ .hasMessageContaining("未授权");
+
+ AsinCacheWriteDto write = new AsinCacheWriteDto();
+ assertThatThrownBy(() -> controller.write(request, write))
+ .isInstanceOf(BusinessException.class)
+ .hasMessageContaining("未授权");
+
+ verifyNoCacheAccess();
+ }
+
+ @Test
+ void queryWithoutBodyIsRejected() {
+ when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
+
+ assertThatThrownBy(() -> newController().query(mock(HttpServletRequest.class), null))
+ .isInstanceOf(BusinessException.class)
+ .hasMessageContaining("缺少查询参数");
+ verify(asinResultCacheService, never()).query(any(), any());
+ }
+
+ @Test
+ void queryReturnsServiceResultAsSuccessEnvelope() {
+ when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
+ AsinCacheQueryVo vo = new AsinCacheQueryVo();
+ vo.setCountry("DE");
+ when(asinResultCacheService.query("DE", List.of("B0ABC12345"))).thenReturn(vo);
+ AsinCacheQueryDto body = new AsinCacheQueryDto();
+ body.setCountry("DE");
+ body.setAsins(List.of("B0ABC12345"));
+
+ ApiResponse 这些数字是花钱的边界:404 复用 7 天、有价复用一个小时内,判宽了采到过期价格,
+ * 判窄了白开页。所以连同"未知 availability 一律不复用"这种兜底方向一起固化。
+ */
+class AsinResultCacheServiceTest {
+
+ private static final int PRICED_TTL_MINUTES = 60;
+ private static final int NOT_FOUND_TTL_HOURS = 168;
+ private static final int MAX_QUERY_ASINS = 500;
+ private static final int MAX_WRITE_ROWS = 500;
+ private static final int WRITE_BATCH_SIZE = 100;
+
+ private final AsinResultCacheMapper mapper = mock(AsinResultCacheMapper.class);
+ private final AsinResultCacheService service = new AsinResultCacheService(
+ mapper, new ObjectMapper(),
+ PRICED_TTL_MINUTES, NOT_FOUND_TTL_HOURS, MAX_QUERY_ASINS, MAX_WRITE_ROWS, WRITE_BATCH_SIZE);
+
+ // ==================== 新鲜度分档 ====================
+
+ @Test
+ void pricedRowReusableWithinTtlOnly() {
+ assertTrue(service.isReusable(row("OK", "9.90", minutesAgo(59)), LocalDateTime.now()),
+ "60 分钟内的价格结论应可复用");
+ assertFalse(service.isReusable(row("OK", "9.90", minutesAgo(61)), LocalDateTime.now()),
+ "超过 60 分钟的价格结论不可复用");
+ }
+
+ @Test
+ void notFoundRowReusableForDays() {
+ assertTrue(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * 24)), LocalDateTime.now()),
+ "404 按天级复用,省开页最多");
+ assertFalse(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * NOT_FOUND_TTL_HOURS + 1)),
+ LocalDateTime.now()), "超过 not-found-ttl-hours 的 404 必须重新确认");
+ }
+
+ /** 空价格 / 拦截页 / 采集异常这类结论即便被人写进库里,读取端也不能复用。 */
+ @Test
+ void unknownAvailabilityNeverReusable() {
+ for (String availability : List.of("BLOCKED", "NO_PRICE", "ERROR", "", "ok")) {
+ assertFalse(service.isReusable(row(availability, "9.90", minutesAgo(1)), LocalDateTime.now()),
+ "未知 availability=" + availability + " 必须按不可复用处理(fail-closed)");
+ }
+ assertFalse(service.isReusable(row("OK", "", minutesAgo(1)), LocalDateTime.now()),
+ "OK 但价格为空的历史脏行不可复用");
+ assertFalse(service.isReusable(null, LocalDateTime.now()));
+ }
+
+ /** 缺 observed_at 时退回 created_at 判新鲜度,而不是直接判死。 */
+ @Test
+ void fallsBackToCreatedAtWhenObservedAtMissing() {
+ AsinResultCacheEntity fresh = row("OK", "9.90", null);
+ fresh.setCreatedAt(minutesAgo(1));
+ assertTrue(service.isReusable(fresh, LocalDateTime.now()));
+
+ AsinResultCacheEntity stale = row("OK", "9.90", null);
+ stale.setCreatedAt(minutesAgo(120));
+ assertFalse(service.isReusable(stale, LocalDateTime.now()));
+ }
+
+ // ==================== 查询 ====================
+
+ @Test
+ void queryNormalizesKeyAndReturnsOnlyReusableHits() {
+ when(mapper.selectByCountryAndAsins(eq("DE"), anyList())).thenReturn(List.of(
+ row("OK", "19.90", minutesAgo(2)),
+ row("OK", "29.90", minutesAgo(720))));
+
+ AsinCacheQueryVo vo = service.query(" de ", Arrays.asList("b0abc12345", "B0ABC12345", "", null));
+
+ assertEquals("DE", vo.getCountry());
+ assertEquals(1, vo.getHits().size(), "过期那行不该作为命中返回");
+ assertEquals("B0ABC12345", vo.getHits().get(0).getAsin());
+ assertEquals(List.of("B0ABC12345"), capturedAsins(1), "ASIN 去重且大写归一后才进 SQL");
+ }
+
+ @Test
+ void queryWithoutCountrySkipsDbEntirely() {
+ AsinCacheQueryVo blank = service.query(" ", List.of("B0ABC12345"));
+ assertTrue(blank.getHits().isEmpty());
+ assertNull(blank.getCountry());
+ verifyNoInteractions(mapper);
+ }
+
+ /** 旁路查询失败对采集端就等于"没缓存":照常开页,不能把采集打成 500。 */
+ @Test
+ void queryFailureDegradesToMiss() {
+ when(mapper.selectByCountryAndAsins(anyString(), anyList()))
+ .thenThrow(new RuntimeException("cache table missing"));
+
+ AsinCacheQueryVo vo = service.query("DE", List.of("B0ABC12345"));
+
+ assertTrue(vo.getHits().isEmpty());
+ }
+
+ @Test
+ void queryOverCapReportsTruncatedAndSplitsIntoChunks() {
+ when(mapper.selectByCountryAndAsins(anyString(), anyList())).thenReturn(List.of());
+ List
+ *
+ * 空价格、验证码/风控拦截页、采集异常都不写:这些结论要么是瞬时故障,要么是我们自己的
+ * 解析问题,缓存它等于把这个 ASIN 后面每次出现都锁成空行。
+ */
+@Slf4j
+@Service
+public class AsinResultCacheService {
+
+ public static final String CHAIN_COLLECT_DATA = "COLLECT_DATA";
+
+ /** 列宽截断上限:超长值会让整批 INSERT 抛 Data too long,一行的脏数据拖垮整个 chunk。 */
+ private static final int MAX_ASIN_LENGTH = 32;
+ private static final int MAX_COUNTRY_LENGTH = 16;
+ private static final int MAX_PRICE_LENGTH = 32;
+ private static final int MAX_BRAND_LENGTH = 255;
+ private static final int MAX_DELIVERY_LENGTH = 32;
+ private static final int MAX_SELLER_NAME_LENGTH = 255;
+ private static final int MAX_SELLER_ID_LENGTH = 64;
+ private static final int MAX_SOURCE_TASK_ID_LENGTH = 64;
+ private static final int MAX_DECLARED_MEMBERS = 500;
+ /** IN (...) 单条 SQL 的 ASIN 个数上限:查询与写入共用,避免超长 SQL 打爆解析器。 */
+ private static final int SQL_CHUNK_SIZE = 100;
+
+ private final AsinResultCacheMapper asinResultCacheMapper;
+ private final ObjectMapper objectMapper;
+ private final int pricedTtlMinutes;
+ private final int notFoundTtlMinutes;
+ private final int maxQueryAsins;
+ private final int maxWriteRows;
+ private final int writeBatchSize;
+
+ public AsinResultCacheService(AsinResultCacheMapper asinResultCacheMapper,
+ ObjectMapper objectMapper,
+ @Value("${aiimage.asin-cache.priced-ttl-minutes:60}") int pricedTtlMinutes,
+ @Value("${aiimage.asin-cache.not-found-ttl-hours:168}") int notFoundTtlHours,
+ @Value("${aiimage.asin-cache.max-query-asins:500}") int maxQueryAsins,
+ @Value("${aiimage.asin-cache.max-write-rows:500}") int maxWriteRows,
+ @Value("${aiimage.asin-cache.write-batch-size:100}") int writeBatchSize) {
+ this.asinResultCacheMapper = asinResultCacheMapper;
+ this.objectMapper = objectMapper;
+ this.pricedTtlMinutes = Math.max(0, pricedTtlMinutes);
+ this.notFoundTtlMinutes = Math.max(0, notFoundTtlHours) * 60;
+ this.maxQueryAsins = Math.max(1, maxQueryAsins);
+ this.maxWriteRows = Math.max(1, maxWriteRows);
+ this.writeBatchSize = Math.max(1, writeBatchSize);
+ }
+
+ /**
+ * 批量查可复用结论。未出现在返回值里的 ASIN 就是"没有可复用结论",调用方照常开页 ——
+ * 复用是省时间的旁路,任何情况下都不许让采集因为缓存而漏行。
+ */
+ public AsinCacheQueryVo query(String country, Collection> captor = ArgumentCaptor.forClass(List.class);
+ verify(mapper).upsertBatch(captor.capture());
+ assertEquals(List.of("OK", "NOT_FOUND"),
+ captor.getValue().stream().map(AsinResultCacheEntity::getAvailability).toList());
+ assertEquals("DE", captor.getValue().get(0).getCountry());
+ }
+
+ /** 一个脏值的长度超列宽会让整批 INSERT 抛 Data too long,因此按列裁剪而不是整批丢。 */
+ @Test
+ void writeClipsOverlongValues() {
+ AsinCacheWriteRowDto dto = writeRow("OK", "1".repeat(200));
+ dto.setBrand("b".repeat(300));
+
+ service.write(List.of(dto));
+
+ ArgumentCaptor
> captor = ArgumentCaptor.forClass(List.class);
+ verify(mapper).upsertBatch(captor.capture());
+ assertEquals(32, captor.getValue().get(0).getPrice().length());
+ assertEquals(255, captor.getValue().get(0).getBrand().length());
+ }
+
+ @Test
+ void writeSplitsByBatchSizeAndSurvivesFailingBatch() {
+ when(mapper.upsertBatch(anyList())).thenReturn(100).thenThrow(new RuntimeException("锁等待超时"));
+ List
> captor = ArgumentCaptor.forClass(List.class);
+ verify(mapper).upsertBatch(captor.capture());
+ AsinResultCacheEntity saved = captor.getValue().get(0);
+ assertEquals(1, captor.getValue().size(), "空价格行不入库");
+ assertEquals("DE", saved.getCountry());
+ assertEquals("OK", saved.getAvailability());
+ assertEquals("COLLECT_DATA", saved.getSourceChain());
+ assertEquals("6161", saved.getSourceTaskId());
+ assertNotNull(saved.getObservedAt(), "缺 observed_at 会让新鲜度判定整体失效");
+ }
+
+ private List
> captor = ArgumentCaptor.forClass(List.class);
+ verify(mapper).selectByCountryAndAsins(eq("DE"), captor.capture());
+ assertEquals(expectedSize, captor.getValue().size());
+ return captor.getValue();
+ }
+
+ private static LocalDateTime minutesAgo(long minutes) {
+ return LocalDateTime.now().minusMinutes(minutes);
+ }
+
+ private static AsinResultCacheEntity row(String availability, String price, LocalDateTime observedAt) {
+ AsinResultCacheEntity entity = new AsinResultCacheEntity();
+ entity.setCountry("DE");
+ entity.setAsin("B0ABC12345");
+ entity.setAvailability(availability);
+ entity.setPrice(price);
+ entity.setObservedAt(observedAt);
+ return entity;
+ }
+
+ private static AsinCacheWriteRowDto writeRow(String availability, String price) {
+ return writeRow(availability, price, "DE");
+ }
+
+ private static AsinCacheWriteRowDto writeRow(String availability, String price, String country) {
+ return writeRow(availability, price, country, "B0ABC12345");
+ }
+
+ private static AsinCacheWriteRowDto writeRow(String availability, String price, String country, String asin) {
+ AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto();
+ dto.setCountry(country);
+ dto.setAsin(asin);
+ dto.setAvailability(availability);
+ dto.setPrice(price);
+ return dto;
+ }
+
+ private static CollectDataResultRowVo collectRow(String asin, String price) {
+ CollectDataResultRowVo row = new CollectDataResultRowVo();
+ row.setAsin(asin);
+ row.setPrice(price);
+ row.setBrand("Demo Brand");
+ row.setDeliveryMethod("FBA");
+ row.setSellerName("Demo Seller");
+ return row;
+ }
+}
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java
index 5172bcb2d..5c8a2e7f8 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java
@@ -145,7 +145,8 @@ class CollectDataActivateGuardTest {
mock(CollectDataInvalidAsinBatchWriter.class), mock(CollectDataResultDetailCodec.class),
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
- mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
+ mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
+ mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
@SuppressWarnings("unchecked")
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java
index f0b4fbed4..f122002fc 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java
@@ -118,7 +118,8 @@ class CollectDataDeleteConsistencyTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
- mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
+ mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
+ mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileTaskEntity task(long id, long userId) {
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java
index e89c4486e..698735ce6 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java
@@ -57,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -96,6 +97,7 @@ class CollectDataServiceTxBoundaryTest {
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
@Mock private CollectDataResultDetailCodec resultDetailCodec;
+ @Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
@InjectMocks private CollectDataService service;
@@ -288,6 +290,51 @@ class CollectDataServiceTxBoundaryTest {
.enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
}
+ // ==================== ASIN 跨任务缓存旁路写入 ====================
+
+ /** 关键词链的 accepted 行顺带写进跨任务缓存;国家取自任务 request_json,而非结果行本身。 */
+ @Test
+ void acceptedRowsWrittenToAsinCacheWithTaskCountry() {
+ when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithCountry("DE"));
+
+ service.submitResult(TASK_ID, submitRequest());
+
+ verify(asinResultCacheService).writeCollectDataRows(eq("DE"), eq("6161"), any());
+ }
+
+ /** 多选国家的任务不写缓存:结果行本身不带国家,无法判断某一行属于哪个站点。 */
+ @Test
+ void multiCountryTaskWritesNoCache() {
+ when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithRequestJson(
+ "{\"filters\":{\"countryCodes\":[\"DE\",\"FR\"]}}"));
+
+ service.submitResult(TASK_ID, submitRequest());
+
+ verify(asinResultCacheService).writeCollectDataRows(isNull(), anyString(), any());
+ }
+
+ /** 缓存写挂了不能把已经抓到手、正要落库的数据拖成提交失败 —— 旁路只记日志。 */
+ @Test
+ void asinCacheWriteFailureDoesNotAffectSubmit() {
+ when(asinResultCacheService.writeCollectDataRows(any(), any(), any()))
+ .thenThrow(new RuntimeException("缓存库不可用"));
+
+ CollectDataSubmitResultVo vo = service.submitResult(TASK_ID, submitRequest());
+
+ assertEquals(TASK_ID, vo.getTaskId());
+ verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
+ }
+
+ private FileTaskEntity taskWithCountry(String countryCode) {
+ return taskWithRequestJson("{\"filters\":{\"countryCode\":\"" + countryCode + "\"}}");
+ }
+
+ private FileTaskEntity taskWithRequestJson(String requestJson) {
+ FileTaskEntity task = runningTask();
+ task.setRequestJson(requestJson);
+ return task;
+ }
+
private CollectDataSubmitResultRequest submitRequest() {
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
row.setAsin("B0COLLECT1");
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java
index ca7823c1e..71cbcd59d 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
- mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
+ mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
+ mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileResultEntity result(long id, long taskId, long userId) {