From 629fdfab791866478e85d5967b091b90bf4441f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Thu, 1 Oct 2026 12:19:05 +0800 Subject: [PATCH] =?UTF-8?q?feat(ASIN=E7=BC=93=E5=AD=98):=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=20ASIN=20=E9=87=87=E9=9B=86=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E8=B7=A8=E4=BB=BB=E5=8A=A1=E7=BC=93=E5=AD=98=EF=BC=88asincache?= =?UTF-8?q?=20=E6=A8=A1=E5=9D=97=20+=20V139=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 同一个 (国家, ASIN) 的详情页被反复重采:单任务内 62.9 倍重复(43,504 行只有 692 个 不同 ASIN)、8 天跨任务 6.7 万行重复(2026-10-01 实测),每条重复都意味着重开一次页、 可能重买一次代理 IP。关键词链与变体链的结果分别落在两处互相看不见,只有 Java 侧能同时 收到两条链的结论,所以缓存放服务端、由服务端判新鲜度。 - 新表 biz_asin_result_cache(V139):一个 (国家, ASIN) 一行,重采覆盖;只收两种可复用 结论 —— OK(页面正常且有价格,复用 60 分钟)、NOT_FOUND(详情页 404,复用 7 天); 空价 / 风控拦截页 / 采集异常一律不入库,避免把瞬时故障锁成永久结论 - 新鲜度只在服务端判(按 observed_at 真正开页时间);采集端拿不到 / 看不懂一律照常开页 (fail-open),任何情况下不许让缓存导致漏行、错行 - 新增内部接口 /api/internal/asin-cache/query|write(X-Internal-Token 自校验,与 InternalBrandTaskController 同口径),供采集端开页前先问、采完回写 - 关键词采集链在任务锁外旁路写入:缓存写失败的代价只是"下次多开一次页",只记 warn, 不影响 chunk 提交 - migration-inventory.md 快照更新至 V139,「V137 冗余索引 DROP」待办顺延 V140 测试:AsinResultCacheServiceTest(13)/ InternalAsinCacheControllerTest(5)/ 采集链 4 个测试类(35)共 53 例全绿。 --- backend-java/docs/migration-inventory.md | 12 +- .../InternalAsinCacheController.java | 80 ++++ .../mapper/AsinResultCacheMapper.java | 73 ++++ .../model/dto/AsinCacheQueryDto.java | 20 + .../model/dto/AsinCacheWriteDto.java | 17 + .../model/dto/AsinCacheWriteRowDto.java | 69 ++++ .../model/entity/AsinResultCacheEntity.java | 41 ++ .../asincache/model/vo/AsinCacheHitVo.java | 39 ++ .../asincache/model/vo/AsinCacheQueryVo.java | 21 + .../asincache/model/vo/AsinCacheWriteVo.java | 18 + .../service/AsinResultCacheService.java | 374 ++++++++++++++++++ .../service/CollectDataService.java | 44 +++ .../src/main/resources/application.yml | 10 + .../resources/db/V139__asin_result_cache.sql | 45 +++ .../InternalAsinCacheControllerTest.java | 134 +++++++ .../service/AsinResultCacheServiceTest.java | 267 +++++++++++++ .../service/CollectDataActivateGuardTest.java | 3 +- .../CollectDataDeleteConsistencyTest.java | 3 +- .../CollectDataServiceTxBoundaryTest.java | 47 +++ .../CollectDataStorageCallCountTest.java | 3 +- 20 files changed, 1312 insertions(+), 8 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/mapper/AsinResultCacheMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheQueryDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteRowDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/entity/AsinResultCacheEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheQueryVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheWriteVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheService.java create mode 100644 backend-java/src/main/resources/db/V139__asin_result_cache.sql create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheControllerTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheServiceTest.java diff --git a/backend-java/docs/migration-inventory.md b/backend-java/docs/migration-inventory.md index 0eb889227..8890b64ff 100644 --- a/backend-java/docs/migration-inventory.md +++ b/backend-java/docs/migration-inventory.md @@ -1,14 +1,14 @@ # Flyway 迁移盘点(task-193) > 生成方式:`src/test/java/com/nanri/aiimage/config/MigrationInventoryTest.java`(只读审计,可重复)。 -> 快照日期:2026-09-28。 +> 快照日期:2026-10-01。 ## 概览 -- 版本化迁移文件数:**137**(`src/main/resources/db/V*.sql`) -- 整数版本范围:**V1..V138**,其中 V132/V133 为已退役空号(见下),其余连续 +- 版本化迁移文件数:**138**(`src/main/resources/db/V*.sql`,其中整数版本 137 个 + 小版本 V25_1) +- 整数版本范围:**V1..V139**,其中 V132/V133 为已退役空号(见下),其余连续 - 历史遗留小版本:**V25_1__shop_manage_group_bind_user.sql**(Flyway 语义 25.1,紧跟在 V25 之后、V26 之前执行,属历史命名,保留) -- 最新版本:**V138__client_task_console.sql** +- 最新版本:**V139__asin_result_cache.sql** - 重复版本:无 - 迁移命名:全部符合 `V<整数>(_<子版本>)?__<描述>.sql`,无空格 @@ -20,7 +20,9 @@ 本地曾存在 `V132__web_config_add_is_latest.sql`、`V133__hide_redundant_indexes_invisible.sql`,与另一分支已在生产落库的 V134/V135 冲突。因 Flyway 未开启 `out-of-order`,把任何迁移编到 132/133 会在启动期报 "resolved migration not applied";因此这两条改为 **V136/V137**,132/133 永久退役,不可回填。 -后续新增迁移从 **V139** 起取号。 +后续新增迁移从 **V140** 起取号(只认"当前最大值 + 1",不要为还没写出来的迁移预留号段——预留会在 1..max 里留洞,`integerVersionsSequentialFromOne` 会红)。 + +原先挂在待办里的"V137 冗余索引 DROP"迁移计划取 V139,该号已被 `V139__asin_result_cache.sql` 占用,顺延到 V140。 ## 约束 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java new file mode 100644 index 000000000..03fff9518 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheController.java @@ -0,0 +1,80 @@ +package com.nanri.aiimage.modules.asincache.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.security.AdminAuthSupport; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo; +import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 内部接口:ASIN 采集结果缓存的读写,供采集端(麦象采集机 / 桌面端)在开页前先问一次 + * 「这个 (国家, ASIN) 是否已经有人开过」。 + * + *

鉴权与 {@code InternalBrandTaskController} 同口径:/api/internal 前缀已在 + * AdminApiGuardFilter 兜底名单内,controller 仍自校验 X-Internal-Token —— 防止配置漂移时匿名可达。 + * + *

这两个端点是旁路:采集端拿不到响应时必须照常开页(fail-open),所以 + * 服务端任何异常都只可能让采集变慢,绝不可能让数据变少或变错。 + */ +@RestController +@RequiredArgsConstructor +@Slf4j +@RequestMapping("/api/internal/asin-cache") +@Tag(name = "内部接口", description = "服务间调用(X-Internal-Token 鉴权)。") +public class InternalAsinCacheController { + + private final AsinResultCacheService asinResultCacheService; + private final AdminAuthSupport adminAuthSupport; + + @PostMapping("/query") + @Operation(summary = "批量查可复用的 ASIN 采集结论(采集端调用)", + description = "只返回新鲜度达标(OK 60 分钟 / NOT_FOUND 7 天,均可配置)的行;" + + "未出现在 hits 里的 ASIN 按未命中处理,调用方照常开页。") + public ApiResponse query(HttpServletRequest request, + @RequestBody(required = false) AsinCacheQueryDto body) { + requireTrustedToken(request, "query"); + if (body == null) { + throw new BusinessException(400, "缺少查询参数"); + } + AsinCacheQueryVo vo = asinResultCacheService.query(body.getCountry(), body.getAsins()); + log.info("[internal-asin-cache] 查询 country={} 请求={} 命中={} 截断={}", + vo.getCountry(), body.getAsins() == null ? 0 : body.getAsins().size(), + vo.getHits().size(), vo.getTruncated()); + return ApiResponse.success(vo); + } + + @PostMapping("/write") + @Operation(summary = "批量写入 ASIN 采集结论(采集端调用)", + description = "不可复用的结论(空价/拦截页/采集异常)服务端逐行丢弃;单批写失败只记日志,不返回失败。") + public ApiResponse write(HttpServletRequest request, + @RequestBody(required = false) AsinCacheWriteDto body) { + requireTrustedToken(request, "write"); + AsinCacheWriteVo vo = asinResultCacheService.write( + body == null ? null : body.getRows()); + log.info("[internal-asin-cache] 写入 请求={} 入库={} 丢弃={} 截断={}", + body == null || body.getRows() == null ? 0 : body.getRows().size(), + vo.getWritten(), vo.getDropped(), vo.getTruncated()); + return ApiResponse.success(vo); + } + + private void requireTrustedToken(HttpServletRequest request, String action) { + if (adminAuthSupport.isTrustedInternalToken(request)) { + return; + } + log.warn("[internal-asin-cache] 拒绝未携带可信内部令牌的请求 action={} remoteAddr={}", + action, request.getRemoteAddr()); + throw new BusinessException(401, "未授权"); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/mapper/AsinResultCacheMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/mapper/AsinResultCacheMapper.java new file mode 100644 index 000000000..b8f26ed94 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/mapper/AsinResultCacheMapper.java @@ -0,0 +1,73 @@ +package com.nanri.aiimage.modules.asincache.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +@Mapper +public interface AsinResultCacheMapper extends BaseMapper { + + /** + * 一次取回一个国家下这批 ASIN 的缓存行。 + * + *

走唯一键 uk_asin_cache_country_asin (country, asin) 的前缀,所以必须是 + * 「country = ? AND asin IN (...)」这种等值 + IN 的形状;把 country 放进 IN 里 + * 成对匹配((country, asin) IN ((..),(..)))在 MySQL 上用不上索引。 + */ + @Select(""" + + """) + List selectByCountryAndAsins(@Param("country") String country, + @Param("asins") List asins); + + /** + * 按唯一键 (country, asin) 批量 upsert:重采覆盖旧结论,不新增行。 + * + *

两条采集链往同一行里写的字段并不一样多 —— 关键词链没有 seller_id 也没有变体 + * 家族,变体链没有 seller 信息。这些"这次没采到"的列在实体里是 NULL(空白值入库前 + * 已被 clip 成 NULL),所以它们用 COALESCE 继承旧值:新观测带了就覆盖,没带就保留, + * 否则一条变体记录会把关键词记录里的卖家名抹成空,下次谁都读不到。 + * + *

price / availability / source_* / observed_at 不参与继承:它们是本行结论的主体, + * 404 就该把价格清空,"谁最后开的页"也必须如实反映最后一次开页。 + */ + @Insert(""" + + """) + int upsertBatch(@Param("rows") List rows); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheQueryDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheQueryDto.java new file mode 100644 index 000000000..4a71387d6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheQueryDto.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.asincache.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "ASIN 缓存批量查询请求(内部接口)") +public class AsinCacheQueryDto { + + @JsonAlias({"country_code", "countryCode"}) + @Schema(description = "国家代码(大小写不敏感)", example = "UK") + private String country; + + @Schema(description = "待查 ASIN 列表,超出上限的部分由服务端丢弃(按请求顺序保留前面那些)") + private List asins = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteDto.java new file mode 100644 index 000000000..67f1da82c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteDto.java @@ -0,0 +1,17 @@ +package com.nanri.aiimage.modules.asincache.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "ASIN 缓存批量写入请求(内部接口)") +public class AsinCacheWriteDto { + + @JsonAlias({"rows", "items"}) + @Schema(description = "待写入行;不可复用的结论(空价/拦截/异常)服务端会逐行丢弃") + private List rows = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteRowDto.java new file mode 100644 index 000000000..925319b0f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/dto/AsinCacheWriteRowDto.java @@ -0,0 +1,69 @@ +package com.nanri.aiimage.modules.asincache.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * ASIN 缓存写入行。 + * + *

字段一律同时接受 camelCase 与 snake_case:调用方是 Python 采集端,snake_case 是它的 + * 自然写法;而 Java 侧序列化出来的是 camelCase,两头都认就不用任何一侧迁就。 + */ +@Data +@Schema(description = "ASIN 缓存写入行(内部接口)") +public class AsinCacheWriteRowDto { + + @JsonAlias({"country_code", "countryCode"}) + @Schema(description = "国家代码", example = "UK") + private String country; + + @JsonAlias({"ASIN", "asin"}) + @Schema(description = "商品 ASIN", example = "B0XXXXXXX") + private String asin; + + @JsonAlias({"availability", "status"}) + @Schema(description = "结论类型:OK(页面正常且有价格)/ NOT_FOUND(404 不存在)。" + + "其余取值服务端一律不入库 —— 空价/拦截页/采集异常都可能是采集端自己的解析或风控问题", + example = "OK") + private String availability; + + @Schema(description = "价格原文(保留当地币种符号,不数值化)", example = "£12.34") + private String price; + + @JsonAlias({"品牌", "brand_name"}) + @Schema(description = "品牌") + private String brand; + + @JsonAlias({"delivery_method", "fulfillment"}) + @Schema(description = "配送方式", example = "FBA") + private String deliveryMethod; + + @JsonAlias({"seller_name", "卖家名称"}) + @Schema(description = "卖家名称") + private String sellerName; + + @JsonAlias({"seller_id", "sellerID", "merchant_id"}) + @Schema(description = "merchantID") + private String sellerId; + + @JsonAlias({"declared_members", "declaredMembers"}) + @Schema(description = "该 ASIN 页面自己声明的变体成员列表;跨任务跳过开页时靠它补齐产出行") + private List declaredMembers; + + @JsonAlias({"source_chain", "sourceChain"}) + @Schema(description = "来源采集链:COLLECT_DATA / VARIANT / DETAIL", example = "VARIANT") + private String sourceChain; + + @JsonAlias({"source_task_id", "sourceTaskId", "task_id"}) + @Schema(description = "来源任务标识,仅用于排查") + private String sourceTaskId; + + @JsonAlias({"observed_at", "observedAt"}) + @Schema(description = "采集机上真正打开该页面的时间;缺省时服务端按当前时间记账", + example = "2026-10-01T10:12:30") + private LocalDateTime observedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/entity/AsinResultCacheEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/entity/AsinResultCacheEntity.java new file mode 100644 index 000000000..f8bf076c9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/entity/AsinResultCacheEntity.java @@ -0,0 +1,41 @@ +package com.nanri.aiimage.modules.asincache.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * ASIN 采集结果缓存行(biz_asin_result_cache)。 + * + *

一个 (country, asin) 一行,重采即覆盖;新鲜度按 {@code observedAt}(采集机上真正 + * 打开页面的时间)算,不按入库时间 —— 否则客户端攒批延迟回传会把旧结论算成新结论。 + */ +@Data +@TableName("biz_asin_result_cache") +public class AsinResultCacheEntity { + + /** 可复用结论:页面正常且取到价格。 */ + public static final String AVAILABILITY_OK = "OK"; + /** 可复用结论:详情页 404,该 ASIN 不存在。 */ + public static final String AVAILABILITY_NOT_FOUND = "NOT_FOUND"; + + @TableId(type = IdType.AUTO) + private Long id; + private String country; + private String asin; + private String availability; + private String price; + private String brand; + private String deliveryMethod; + private String sellerName; + private String sellerId; + private String declaredMembersJson; + private String sourceChain; + private String sourceTaskId; + private LocalDateTime observedAt; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java new file mode 100644 index 000000000..28fddefb6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheHitVo.java @@ -0,0 +1,39 @@ +package com.nanri.aiimage.modules.asincache.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.time.LocalDateTime; +import java.util.List; + +@Data +@Schema(description = "可复用的 ASIN 缓存结论") +public class AsinCacheHitVo { + + @Schema(description = "商品 ASIN") + private String asin; + + @Schema(description = "结论类型:OK / NOT_FOUND") + private String availability; + + @Schema(description = "价格原文;NOT_FOUND 时为空") + private String price; + + @Schema(description = "品牌") + private String brand; + + @Schema(description = "配送方式") + private String deliveryMethod; + + @Schema(description = "卖家名称") + private String sellerName; + + @Schema(description = "merchantID") + private String sellerId; + + @Schema(description = "该 ASIN 页面声明过的变体成员列表(跳过开页时靠它补齐产出行)") + private List declaredMembers; + + @Schema(description = "结论产生时间,采集端可据此在日志里报出复用了多久以前的数据") + private LocalDateTime observedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheQueryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheQueryVo.java new file mode 100644 index 000000000..12f80e578 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheQueryVo.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.modules.asincache.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "ASIN 缓存批量查询结果") +public class AsinCacheQueryVo { + + @Schema(description = "回显国家代码(大写)") + private String country; + + @Schema(description = "命中的结论;未列出的 ASIN 就是没有可复用结论,调用方照常开页") + private List hits = new ArrayList<>(); + + @Schema(description = "因超出单次上限而被忽略的 ASIN 个数(这些一律按未命中处理)") + private int truncated; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheWriteVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheWriteVo.java new file mode 100644 index 000000000..a57a77993 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/model/vo/AsinCacheWriteVo.java @@ -0,0 +1,18 @@ +package com.nanri.aiimage.modules.asincache.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "ASIN 缓存批量写入结果") +public class AsinCacheWriteVo { + + @Schema(description = "实际入库的行数(含覆盖旧行)") + private int written; + + @Schema(description = "被服务端丢弃的行数:结论不可复用、ASIN 非法、或超出单次上限") + private int dropped; + + @Schema(description = "本次请求被丢弃的行数上限之外的部分,调用方无需重试(重试也一样被丢)") + private int truncated; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheService.java new file mode 100644 index 000000000..4b499ec5c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheService.java @@ -0,0 +1,374 @@ +package com.nanri.aiimage.modules.asincache.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.asincache.mapper.AsinResultCacheMapper; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteRowDto; +import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheHitVo; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo; +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; + +/** + * ASIN 采集结果缓存:跨任务、跨采集链复用「同一个 (国家, ASIN) 详情页已经开过」这个事实。 + * + *

新鲜度只在这里判,采集端不参与。两条链的采集端版本不可能同时升级, + * 把口径放在服务端意味着老采集端拿到 404/未知字段只会当没缓存用(fail-open), + * 而不是各按各的理解复用到过期数据。 + * + *

可复用只有两档(其余一律不入库,见 {@link #ttlMinutesOf}): + *

+ * 空价格、验证码/风控拦截页、采集异常都不写:这些结论要么是瞬时故障,要么是我们自己的 + * 解析问题,缓存它等于把这个 ASIN 后面每次出现都锁成空行。 + */ +@Slf4j +@Service +public class AsinResultCacheService { + + public static final String CHAIN_COLLECT_DATA = "COLLECT_DATA"; + + /** 列宽截断上限:超长值会让整批 INSERT 抛 Data too long,一行的脏数据拖垮整个 chunk。 */ + private static final int MAX_ASIN_LENGTH = 32; + private static final int MAX_COUNTRY_LENGTH = 16; + private static final int MAX_PRICE_LENGTH = 32; + private static final int MAX_BRAND_LENGTH = 255; + private static final int MAX_DELIVERY_LENGTH = 32; + private static final int MAX_SELLER_NAME_LENGTH = 255; + private static final int MAX_SELLER_ID_LENGTH = 64; + private static final int MAX_SOURCE_TASK_ID_LENGTH = 64; + private static final int MAX_DECLARED_MEMBERS = 500; + /** IN (...) 单条 SQL 的 ASIN 个数上限:查询与写入共用,避免超长 SQL 打爆解析器。 */ + private static final int SQL_CHUNK_SIZE = 100; + + private final AsinResultCacheMapper asinResultCacheMapper; + private final ObjectMapper objectMapper; + private final int pricedTtlMinutes; + private final int notFoundTtlMinutes; + private final int maxQueryAsins; + private final int maxWriteRows; + private final int writeBatchSize; + + public AsinResultCacheService(AsinResultCacheMapper asinResultCacheMapper, + ObjectMapper objectMapper, + @Value("${aiimage.asin-cache.priced-ttl-minutes:60}") int pricedTtlMinutes, + @Value("${aiimage.asin-cache.not-found-ttl-hours:168}") int notFoundTtlHours, + @Value("${aiimage.asin-cache.max-query-asins:500}") int maxQueryAsins, + @Value("${aiimage.asin-cache.max-write-rows:500}") int maxWriteRows, + @Value("${aiimage.asin-cache.write-batch-size:100}") int writeBatchSize) { + this.asinResultCacheMapper = asinResultCacheMapper; + this.objectMapper = objectMapper; + this.pricedTtlMinutes = Math.max(0, pricedTtlMinutes); + this.notFoundTtlMinutes = Math.max(0, notFoundTtlHours) * 60; + this.maxQueryAsins = Math.max(1, maxQueryAsins); + this.maxWriteRows = Math.max(1, maxWriteRows); + this.writeBatchSize = Math.max(1, writeBatchSize); + } + + /** + * 批量查可复用结论。未出现在返回值里的 ASIN 就是"没有可复用结论",调用方照常开页 —— + * 复用是省时间的旁路,任何情况下都不许让采集因为缓存而漏行。 + */ + public AsinCacheQueryVo query(String country, Collection asins) { + AsinCacheQueryVo vo = new AsinCacheQueryVo(); + String normalizedCountry = normalizeCountry(country); + vo.setCountry(normalizedCountry); + if (normalizedCountry == null || asins == null || asins.isEmpty()) { + return vo; + } + List distinct = new ArrayList<>(new LinkedHashSet<>(cleanAsins(asins))); + if (distinct.isEmpty()) { + return vo; + } + if (distinct.size() > maxQueryAsins) { + vo.setTruncated(distinct.size() - maxQueryAsins); + distinct = distinct.subList(0, maxQueryAsins); + } + LocalDateTime now = LocalDateTime.now(); + List hits = new ArrayList<>(); + for (int from = 0; from < distinct.size(); from += SQL_CHUNK_SIZE) { + List batch = distinct.subList(from, Math.min(from + SQL_CHUNK_SIZE, distinct.size())); + List rows; + try { + rows = asinResultCacheMapper.selectByCountryAndAsins(normalizedCountry, batch); + } catch (RuntimeException ex) { + // 查询失败与"没有缓存"对采集端是同一件事:照常开页。不能因为旁路挂了就 500。 + log.warn("[asin-cache] 缓存查询失败,按未命中处理 country={} size={} err={}", + normalizedCountry, batch.size(), ex.getMessage()); + continue; + } + for (AsinResultCacheEntity row : rows) { + if (isReusable(row, now)) { + hits.add(toHit(row)); + } + } + } + vo.setHits(hits); + return vo; + } + + /** + * 批量写入结论。不可复用的行在这里丢弃(而不是在采集端):采集端有版本差, + * 老客户端可能还在上报空价格 —— 服务端不收,表里就不会出现被误复用的脏结论。 + * + *

单批失败只记日志、不抛:调用方是采集热路径与关键词结果提交链路, + * 缓存写不进去的代价是"下次多开一次页",把任务整体拖失败是更大的代价。 + */ + public AsinCacheWriteVo write(List rows) { + AsinCacheWriteVo vo = new AsinCacheWriteVo(); + if (rows == null || rows.isEmpty()) { + return vo; + } + if (rows.size() > maxWriteRows) { + vo.setTruncated(rows.size() - maxWriteRows); + rows = rows.subList(0, maxWriteRows); + } + List entities = new ArrayList<>(rows.size()); + for (AsinCacheWriteRowDto row : rows) { + AsinResultCacheEntity entity = toEntity(row); + if (entity == null) { + vo.setDropped(vo.getDropped() + 1); + continue; + } + entities.add(entity); + } + if (entities.isEmpty()) { + return vo; + } + for (int from = 0; from < entities.size(); from += writeBatchSize) { + List batch = entities.subList(from, Math.min(from + writeBatchSize, entities.size())); + try { + vo.setWritten(vo.getWritten() + asinResultCacheMapper.upsertBatch(batch)); + } catch (RuntimeException ex) { + log.warn("[asin-cache] 缓存批量写入失败,跳过该批 size={} err={}", batch.size(), ex.getMessage()); + } + } + return vo; + } + + /** + * 关键词采集链的旁路写入:结果行落 biz_task_result_item 之后,把"这个 (国家, ASIN) + * 的价格/品牌/配送已经开页看过"这件事同步记进缓存,让变体采集下次不必重开。 + * + *

country 缺失时整批不写 —— 缓存键少了国家,就可能把 DE 的价格当成 US 的结论发回去。 + * 关键词链按 ASIN 逐个采详情页,一批 accepted 行同属一个任务、一个国家,所以这里 + * 只按行计数,不逐行判国家。 + */ + public int writeCollectDataRows(String country, String sourceTaskId, + List rows) { + if (rows == null || rows.isEmpty()) { + return 0; + } + String normalizedCountry = normalizeCountry(country); + if (normalizedCountry == null) { + log.info("[asin-cache] 任务未带国家代码,跳过缓存写入(避免跨国家错用)rows={}", rows.size()); + return 0; + } + List payload = new ArrayList<>(rows.size()); + for (CollectDataResultRowVo row : rows) { + if (row == null || isBlank(row.getPrice())) { + continue; // 空价格不可复用,见类注释 + } + AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto(); + dto.setCountry(normalizedCountry); + dto.setAsin(row.getAsin()); + dto.setAvailability(AsinResultCacheEntity.AVAILABILITY_OK); + dto.setPrice(row.getPrice()); + dto.setBrand(row.getBrand()); + dto.setDeliveryMethod(row.getDeliveryMethod()); + dto.setSellerName(row.getSellerName()); + dto.setSourceChain(CHAIN_COLLECT_DATA); + dto.setSourceTaskId(sourceTaskId); + payload.add(dto); + } + if (payload.isEmpty()) { + return 0; + } + AsinCacheWriteVo written = write(payload); + // 关键词链是"顺带"贡献缓存的:写满/写坏都不该在业务日志里响,降到 debug 级别由排查时打开。 + log.debug("[asin-cache] 关键词链旁路写入 taskId={} rows={} accepted={} dropped={} truncated={}", + sourceTaskId, payload.size(), written.getWritten(), written.getDropped(), written.getTruncated()); + return written.getWritten(); + } + + // ==================== 口径 ==================== + + /** 该 availability 的可复用时长(分钟);返回 0 表示永不复用。未知取值一律按 0 处理。 */ + int ttlMinutesOf(String availability) { + if (AsinResultCacheEntity.AVAILABILITY_OK.equals(availability)) { + return pricedTtlMinutes; + } + if (AsinResultCacheEntity.AVAILABILITY_NOT_FOUND.equals(availability)) { + return notFoundTtlMinutes; + } + return 0; + } + + /** 新鲜度判定:按 observed_at(真正开页时间),没有才退到 created_at(入库时间)。 */ + boolean isReusable(AsinResultCacheEntity row, LocalDateTime now) { + if (row == null || row.getAsin() == null) { + return false; + } + int ttlMinutes = ttlMinutesOf(row.getAvailability()); + if (ttlMinutes <= 0) { + return false; + } + if (AsinResultCacheEntity.AVAILABILITY_OK.equals(row.getAvailability()) && isBlank(row.getPrice())) { + // 理论上写不进来(write 已挡),但库里万一有历史脏行,不能让它被当成有效价格复用 + return false; + } + LocalDateTime observedAt = row.getObservedAt() != null ? row.getObservedAt() : row.getCreatedAt(); + if (observedAt == null) { + return false; + } + return observedAt.plusMinutes(ttlMinutes).isAfter(now); + } + + // ==================== 归一化 ==================== + + private AsinResultCacheEntity toEntity(AsinCacheWriteRowDto row) { + if (row == null) { + return null; + } + String country = normalizeCountry(row.getCountry()); + String asin = normalizeAsin(row.getAsin()); + if (country == null || asin == null) { + return null; + } + String availability = row.getAvailability() == null ? "" + : row.getAvailability().trim().toUpperCase(Locale.ROOT); + if (ttlMinutesOf(availability) <= 0) { + return null; // 不可复用的结论(含未知取值)不进表 + } + if (AsinResultCacheEntity.AVAILABILITY_OK.equals(availability) && isBlank(row.getPrice())) { + return null; + } + AsinResultCacheEntity entity = new AsinResultCacheEntity(); + entity.setCountry(country); + entity.setAsin(asin); + entity.setAvailability(availability); + entity.setPrice(clip(row.getPrice(), MAX_PRICE_LENGTH)); + entity.setBrand(clip(row.getBrand(), MAX_BRAND_LENGTH)); + entity.setDeliveryMethod(clip(row.getDeliveryMethod(), MAX_DELIVERY_LENGTH)); + entity.setSellerName(clip(row.getSellerName(), MAX_SELLER_NAME_LENGTH)); + entity.setSellerId(clip(row.getSellerId(), MAX_SELLER_ID_LENGTH)); + entity.setDeclaredMembersJson(toDeclaredMembersJson(row.getDeclaredMembers())); + entity.setSourceChain(clip(row.getSourceChain(), 32)); + entity.setSourceTaskId(clip(row.getSourceTaskId(), MAX_SOURCE_TASK_ID_LENGTH)); + // 缺 observed_at 时按当前时间记账:等价于"刚采到",新鲜度判定不会因为客户端没传就整体失效 + entity.setObservedAt(row.getObservedAt() != null ? row.getObservedAt() : LocalDateTime.now()); + LocalDateTime now = LocalDateTime.now(); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + return entity; + } + + private AsinCacheHitVo toHit(AsinResultCacheEntity row) { + AsinCacheHitVo hit = new AsinCacheHitVo(); + hit.setAsin(row.getAsin()); + hit.setAvailability(row.getAvailability()); + hit.setPrice(row.getPrice()); + hit.setBrand(row.getBrand()); + hit.setDeliveryMethod(row.getDeliveryMethod()); + hit.setSellerName(row.getSellerName()); + hit.setSellerId(row.getSellerId()); + hit.setDeclaredMembers(fromDeclaredMembersJson(row.getDeclaredMembersJson())); + hit.setObservedAt(row.getObservedAt() != null ? row.getObservedAt() : row.getCreatedAt()); + return hit; + } + + /** 国家代码是缓存键的一半:大写归一、空值即拒,绝不用空国家写行(会与其他国家撞键)。 */ + private static String normalizeCountry(String country) { + String value = country == null ? "" : country.trim().toUpperCase(Locale.ROOT); + if (value.isEmpty()) { + return null; + } + return value.length() > MAX_COUNTRY_LENGTH ? null : value; + } + + private static String normalizeAsin(String asin) { + String value = asin == null ? "" : asin.trim().toUpperCase(Locale.ROOT); + if (value.isEmpty() || value.length() > MAX_ASIN_LENGTH) { + return null; + } + return value; + } + + private static List cleanAsins(Collection asins) { + List out = new ArrayList<>(asins.size()); + for (String asin : asins) { + String normalized = normalizeAsin(asin); + if (normalized != null) { + out.add(normalized); + } + } + return out; + } + + private String toDeclaredMembersJson(List members) { + if (members == null || members.isEmpty()) { + return null; + } + Collection clean = new LinkedHashSet<>(); + for (String member : members) { + String normalized = normalizeAsin(member); + if (normalized != null) { + clean.add(normalized); + } + if (clean.size() >= MAX_DECLARED_MEMBERS) { + break; + } + } + if (clean.isEmpty()) { + return null; + } + try { + return objectMapper.writeValueAsString(clean); + } catch (Exception ex) { + log.warn("[asin-cache] 变体成员列表序列化失败,按无成员入库 err={}", ex.getMessage()); + return null; + } + } + + private List fromDeclaredMembersJson(String json) { + if (isBlank(json)) { + return List.of(); + } + try { + List members = objectMapper.readValue(json, + objectMapper.getTypeFactory().constructCollectionType(List.class, String.class)); + return members == null ? List.of() : members; + } catch (Exception ex) { + log.warn("[asin-cache] 变体成员列表解析失败,按无成员返回 json={} err={}", + json.length() > 120 ? json.substring(0, 120) + "..." : json, ex.getMessage()); + return List.of(); + } + } + + private static String clip(String value, int maxLength) { + if (value == null) { + return null; + } + String trimmed = value.trim(); + if (trimmed.isEmpty()) { + return null; + } + return trimmed.length() > maxLength ? trimmed.substring(0, maxLength) : trimmed; + } + + private static boolean isBlank(String value) { + return value == null || value.isBlank(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index 9ae19f36d..edc7d8694 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -12,6 +12,7 @@ import com.nanri.aiimage.common.exception.BusinessCodes; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.security.AdminAuthSupport; import com.nanri.aiimage.common.util.ExcelStreamReader; +import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService; import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper; import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest; @@ -182,6 +183,9 @@ public class CollectDataService { private final TaskProgressLightAssembler taskProgressLightAssembler; private final NotificationDispatchService notificationDispatchService; + /** ASIN 跨任务结果缓存:本链只负责"顺带贡献结论",写入失败绝不影响 chunk 提交。 */ + private final AsinResultCacheService asinResultCacheService; + @Value("${aiimage.collect-data.stale-timeout-minutes:30}") private long staleTimeoutMinutes; @@ -830,6 +834,11 @@ public class CollectDataService { log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms", taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt); + // ASIN 结果缓存旁路写入同样放锁外:这把锁的持有时间是心跳与其他 chunk 的排队预算 + // (见上方品牌检测移出的注释),缓存写入再快也不该来挤。缓存是"下次少开一次页"的 + // 加速件,同一 chunk 重复提交只是把同样的结论再 upsert 一遍,不需要锁内串行语义。 + writeAsinCacheOutsideLock(probe, taskId, brandOutcome.accepted()); + try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { FileTaskEntity task = fileTaskMapper.selectById(taskId); if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { @@ -1830,6 +1839,41 @@ public class CollectDataService { return vo; } + /** + * 关键词采集链的缓存旁路写入:把"这个 (国家, ASIN) 的价格/品牌/配送已经开页看过"这件事 + * 记进跨任务缓存,供下一条采集链复用。 + * + *

整段吞掉 RuntimeException:缓存写不进去的代价是"下次多开一次页",把已经抓到手、 + * 正要落库的 chunk 提交拖失败是更大的代价。 + */ + private void writeAsinCacheOutsideLock(FileTaskEntity task, Long taskId, + List accepted) { + if (accepted == null || accepted.isEmpty()) { + return; + } + try { + asinResultCacheService.writeCollectDataRows(resolveCacheCountry(task), String.valueOf(taskId), accepted); + } catch (RuntimeException ex) { + log.warn("[collect-data] ASIN 缓存写入失败,不影响本次提交 taskId={} err={}", + taskId, ex.getMessage()); + } + } + + /** + * 缓存键里的国家来自任务入参(Python 回传的结果行本身不带国家,见 CollectDataSubmitRowDto)。 + * 只收"这一个任务确定只采一个国家"的情形:多选国家时无法判断某一行属于哪个站点, + * 宁可整批不写 —— 键少了国家就会把 DE 的价格当成 US 的结论发回去。 + */ + private String resolveCacheCountry(FileTaskEntity task) { + CollectDataFiltersDto filters = extractFiltersFromRequest(task); + String single = filters.getCountryCode(); + if (single != null && !single.isBlank()) { + return single; + } + List codes = filters.getCountryCodes(); + return codes != null && codes.size() == 1 ? codes.get(0) : null; + } + private CollectDataFiltersDto extractFiltersFromRequest(FileTaskEntity task) { if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) { return new CollectDataFiltersDto(); diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 115cc75b2..922317a2e 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -396,6 +396,16 @@ aiimage: brand-check-cache-capacity: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CACHE_CAPACITY:512} invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100} result-item-batch-size: ${AIIMAGE_COLLECT_DATA_RESULT_ITEM_BATCH_SIZE:100} + # ASIN 采集结果跨任务缓存(详见 AsinResultCacheService)。新鲜度口径放在服务端, + # 采集端版本不齐时老客户端只会"问不到就照常开页",不会各按各的理解复用到过期数据。 + asin-cache: + # 页面正常且有价格:价格会变,只复用这么短时间内(分钟) + priced-ttl-minutes: ${AIIMAGE_ASIN_CACHE_PRICED_TTL_MINUTES:60} + # 详情页 404「此 ASIN 不存在」:基本永久,缓存最长(小时) + not-found-ttl-hours: ${AIIMAGE_ASIN_CACHE_NOT_FOUND_TTL_HOURS:168} + max-query-asins: ${AIIMAGE_ASIN_CACHE_MAX_QUERY_ASINS:500} + max-write-rows: ${AIIMAGE_ASIN_CACHE_MAX_WRITE_ROWS:500} + write-batch-size: ${AIIMAGE_ASIN_CACHE_WRITE_BATCH_SIZE:100} image-video: coze-base-url: ${AIIMAGE_IMAGE_VIDEO_COZE_BASE_URL:https://api.coze.cn} coze-token: ${AIIMAGE_IMAGE_VIDEO_COZE_TOKEN:} diff --git a/backend-java/src/main/resources/db/V139__asin_result_cache.sql b/backend-java/src/main/resources/db/V139__asin_result_cache.sql new file mode 100644 index 000000000..089bb0d17 --- /dev/null +++ b/backend-java/src/main/resources/db/V139__asin_result_cache.sql @@ -0,0 +1,45 @@ +-- V139: ASIN 采集结果缓存表(跨任务/跨链复用的唯一事实源) +-- +-- 为什么要这张表(2026-10-01 实测):同一个 (国家, ASIN) 的详情页被反复重采。 +-- · 任务内:变体采集单任务 d56024ff 写出 43,504 行,去重后只有 692 个不同 ASIN(重复 62.9 倍); +-- · 跨任务:8 天自连接重复 67,169 行。 +-- 每一行都要开一次页、可能买一次代理 IP,而关键词采集与变体采集的结果分别落在 +-- biz_task_result_item 与麦象 MySQL 的 variant_collection 两处,互相看不见 —— +-- 只有 Java 这侧能同时收到两条链的结论,所以缓存放这里、由这里判新鲜度。 +-- +-- 复用口径(服务端唯一判定,客户端不参与,见 AsinResultCacheService): +-- · OK 页面正常且取到价格 —— 只复用很短时间内的(默认 60 分钟,价格会变) +-- · NOT_FOUND 详情页 404「此 ASIN 不存在」—— 基本永久,复用最长(默认 7 天,省开页最多) +-- · 其余(空价/拦截页/采集异常/不可售)一律不写本表:那些结论要么可能是我们 +-- 自己的解析/风控问题,要么换一次出口 IP 就能救回,缓存它等于把这个 ASIN +-- 永久锁成空行。 +-- +-- collation 显式写死 utf8mb4_general_ci:库里不同表的默认排序规则不一致 +-- (biz_dedupe_total_data / biz_collect_data_item 是 general_ci,部分新表走库默认), +-- 注解 SQL 里 (country, asin) 与入参比较一旦跨排序规则会抛 Illegal mix of collations, +-- 这类错误只在运行期出现、编译期查不出来。 +-- +-- 回滚(手工): +-- DROP TABLE IF EXISTS biz_asin_result_cache; + +CREATE TABLE IF NOT EXISTS biz_asin_result_cache ( + id BIGINT NOT NULL AUTO_INCREMENT COMMENT 'primary key', + country VARCHAR(16) NOT NULL COMMENT '国家代码(大写),与 ASIN 一起构成缓存键', + asin VARCHAR(32) NOT NULL COMMENT '商品 ASIN', + availability VARCHAR(16) NOT NULL COMMENT '可复用结论:OK / NOT_FOUND(其余取值不入库)', + price VARCHAR(32) NULL COMMENT '价格原文(带当地币种符号,不做数值化以免丢信息)', + brand VARCHAR(255) NULL COMMENT '品牌', + delivery_method VARCHAR(32) NULL COMMENT '配送方式:FBA / FBM / AMZ / NA', + seller_name VARCHAR(255) NULL COMMENT '卖家名称', + seller_id VARCHAR(64) NULL COMMENT 'merchantID', + declared_members_json TEXT NULL COMMENT '该 ASIN 页面自己声明的变体成员列表(JSON 数组);跳过开页时靠它补齐产出行,见 VariantCollectionHandler._remember_family', + source_chain VARCHAR(32) NOT NULL COMMENT '结论来源链:COLLECT_DATA / VARIANT / DETAIL', + source_task_id VARCHAR(64) NULL COMMENT '来源任务标识(Java 任务 id 或麦象 task_id,仅用于排查)', + observed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '采集机上真正打开该页面的时间(新鲜度按它算,不是按入库时间)', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + PRIMARY KEY (id), + UNIQUE KEY uk_asin_cache_country_asin (country, asin) COMMENT '一个 (国家, ASIN) 一行,重采即覆盖', + KEY idx_asin_cache_observed (availability, observed_at) COMMENT '新鲜度判定与后续保留期清理' +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = DYNAMIC + COMMENT = 'ASIN 采集结果跨任务缓存(关键词链与变体链共用)'; diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheControllerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheControllerTest.java new file mode 100644 index 000000000..1580458c3 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/asincache/controller/InternalAsinCacheControllerTest.java @@ -0,0 +1,134 @@ +package com.nanri.aiimage.modules.asincache.controller; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.security.AdminAuthSupport; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo; +import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService; +import jakarta.servlet.http.HttpServletRequest; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 内部缓存端点的鉴权与入参绑定。 + * + *

/api/internal 前缀虽然已在 AdminApiGuardFilter 兜底名单里,controller 仍自校验一次令牌: + * 配置漂移时这两个端点会直接暴露给匿名请求,而它们背后是全站的采集结论与 ASIN 明细。 + */ +class InternalAsinCacheControllerTest { + + private final AsinResultCacheService asinResultCacheService = mock(AsinResultCacheService.class); + private final AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class); + + private InternalAsinCacheController newController() { + return new InternalAsinCacheController(asinResultCacheService, adminAuthSupport); + } + + @Test + void rejectsQueryAndWriteWithoutTrustedInternalToken() { + when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(false); + InternalAsinCacheController controller = newController(); + HttpServletRequest request = mock(HttpServletRequest.class); + + AsinCacheQueryDto query = new AsinCacheQueryDto(); + query.setCountry("DE"); + query.setAsins(List.of("B0ABC12345")); + assertThatThrownBy(() -> controller.query(request, query)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("未授权"); + + AsinCacheWriteDto write = new AsinCacheWriteDto(); + assertThatThrownBy(() -> controller.write(request, write)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("未授权"); + + verifyNoCacheAccess(); + } + + @Test + void queryWithoutBodyIsRejected() { + when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true); + + assertThatThrownBy(() -> newController().query(mock(HttpServletRequest.class), null)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("缺少查询参数"); + verify(asinResultCacheService, never()).query(any(), any()); + } + + @Test + void queryReturnsServiceResultAsSuccessEnvelope() { + when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true); + AsinCacheQueryVo vo = new AsinCacheQueryVo(); + vo.setCountry("DE"); + when(asinResultCacheService.query("DE", List.of("B0ABC12345"))).thenReturn(vo); + AsinCacheQueryDto body = new AsinCacheQueryDto(); + body.setCountry("DE"); + body.setAsins(List.of("B0ABC12345")); + + ApiResponse resp = newController().query(mock(HttpServletRequest.class), body); + + assertThat(resp.isSuccess()).isTrue(); + assertThat(resp.getData().getHits()).isEmpty(); + } + + /** + * 调用方是 Python 采集端:snake_case 是它的自然写法。这里按真实报文形状反序列化一次, + * 免得字段改名后两端各自"看起来都对",线上才发现写入的行全是空值。 + */ + @Test + void writeAcceptsPythonShapedPayload() throws Exception { + when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true); + when(asinResultCacheService.write(anyList())).thenReturn(new AsinCacheWriteVo()); + String pythonJson = """ + {"rows": [ + {"country": "UK", "ASIN": "B0ABC12345", "availability": "NOT_FOUND", + "declared_members": ["B0ABC12346"], "source_chain": "VARIANT", + "source_task_id": "28599", "brand": "Nike", "delivery_method": "FBA", + "seller_name": "Nike DE", "seller_id": "A1PXL", "price": ""} + ]}"""; + AsinCacheWriteDto body = new ObjectMapper().readValue(pythonJson, AsinCacheWriteDto.class); + + ApiResponse resp = newController().write(mock(HttpServletRequest.class), body); + + assertThat(resp.isSuccess()).isTrue(); + verify(asinResultCacheService).write(org.mockito.ArgumentMatchers.argThat(rows -> + rows.size() == 1 + && "UK".equals(rows.get(0).getCountry()) + && "B0ABC12345".equals(rows.get(0).getAsin()) + && "NOT_FOUND".equals(rows.get(0).getAvailability()) + && List.of("B0ABC12346").equals(rows.get(0).getDeclaredMembers()) + && "28599".equals(rows.get(0).getSourceTaskId()) + && "Nike DE".equals(rows.get(0).getSellerName()) + && "A1PXL".equals(rows.get(0).getSellerId()))); + } + + @Test + void writeWithoutBodyReportsZeroInsteadOfThrowing() { + when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true); + when(asinResultCacheService.write(eq(null))).thenReturn(new AsinCacheWriteVo()); + + ApiResponse resp = newController().write(mock(HttpServletRequest.class), null); + + assertThat(resp.getData().getWritten()).isZero(); + } + + private void verifyNoCacheAccess() { + verify(asinResultCacheService, never()).query(any(), any()); + verify(asinResultCacheService, never()).write(any()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheServiceTest.java new file mode 100644 index 000000000..543ad9c41 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/asincache/service/AsinResultCacheServiceTest.java @@ -0,0 +1,267 @@ +package com.nanri.aiimage.modules.asincache.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.asincache.mapper.AsinResultCacheMapper; +import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteRowDto; +import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo; +import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo; +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * ASIN 跨任务缓存的可复用口径(新鲜度分档 + 哪些结论根本不入库)。 + * + *

这些数字是花钱的边界:404 复用 7 天、有价复用一个小时内,判宽了采到过期价格, + * 判窄了白开页。所以连同"未知 availability 一律不复用"这种兜底方向一起固化。 + */ +class AsinResultCacheServiceTest { + + private static final int PRICED_TTL_MINUTES = 60; + private static final int NOT_FOUND_TTL_HOURS = 168; + private static final int MAX_QUERY_ASINS = 500; + private static final int MAX_WRITE_ROWS = 500; + private static final int WRITE_BATCH_SIZE = 100; + + private final AsinResultCacheMapper mapper = mock(AsinResultCacheMapper.class); + private final AsinResultCacheService service = new AsinResultCacheService( + mapper, new ObjectMapper(), + PRICED_TTL_MINUTES, NOT_FOUND_TTL_HOURS, MAX_QUERY_ASINS, MAX_WRITE_ROWS, WRITE_BATCH_SIZE); + + // ==================== 新鲜度分档 ==================== + + @Test + void pricedRowReusableWithinTtlOnly() { + assertTrue(service.isReusable(row("OK", "9.90", minutesAgo(59)), LocalDateTime.now()), + "60 分钟内的价格结论应可复用"); + assertFalse(service.isReusable(row("OK", "9.90", minutesAgo(61)), LocalDateTime.now()), + "超过 60 分钟的价格结论不可复用"); + } + + @Test + void notFoundRowReusableForDays() { + assertTrue(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * 24)), LocalDateTime.now()), + "404 按天级复用,省开页最多"); + assertFalse(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * NOT_FOUND_TTL_HOURS + 1)), + LocalDateTime.now()), "超过 not-found-ttl-hours 的 404 必须重新确认"); + } + + /** 空价格 / 拦截页 / 采集异常这类结论即便被人写进库里,读取端也不能复用。 */ + @Test + void unknownAvailabilityNeverReusable() { + for (String availability : List.of("BLOCKED", "NO_PRICE", "ERROR", "", "ok")) { + assertFalse(service.isReusable(row(availability, "9.90", minutesAgo(1)), LocalDateTime.now()), + "未知 availability=" + availability + " 必须按不可复用处理(fail-closed)"); + } + assertFalse(service.isReusable(row("OK", "", minutesAgo(1)), LocalDateTime.now()), + "OK 但价格为空的历史脏行不可复用"); + assertFalse(service.isReusable(null, LocalDateTime.now())); + } + + /** 缺 observed_at 时退回 created_at 判新鲜度,而不是直接判死。 */ + @Test + void fallsBackToCreatedAtWhenObservedAtMissing() { + AsinResultCacheEntity fresh = row("OK", "9.90", null); + fresh.setCreatedAt(minutesAgo(1)); + assertTrue(service.isReusable(fresh, LocalDateTime.now())); + + AsinResultCacheEntity stale = row("OK", "9.90", null); + stale.setCreatedAt(minutesAgo(120)); + assertFalse(service.isReusable(stale, LocalDateTime.now())); + } + + // ==================== 查询 ==================== + + @Test + void queryNormalizesKeyAndReturnsOnlyReusableHits() { + when(mapper.selectByCountryAndAsins(eq("DE"), anyList())).thenReturn(List.of( + row("OK", "19.90", minutesAgo(2)), + row("OK", "29.90", minutesAgo(720)))); + + AsinCacheQueryVo vo = service.query(" de ", Arrays.asList("b0abc12345", "B0ABC12345", "", null)); + + assertEquals("DE", vo.getCountry()); + assertEquals(1, vo.getHits().size(), "过期那行不该作为命中返回"); + assertEquals("B0ABC12345", vo.getHits().get(0).getAsin()); + assertEquals(List.of("B0ABC12345"), capturedAsins(1), "ASIN 去重且大写归一后才进 SQL"); + } + + @Test + void queryWithoutCountrySkipsDbEntirely() { + AsinCacheQueryVo blank = service.query(" ", List.of("B0ABC12345")); + assertTrue(blank.getHits().isEmpty()); + assertNull(blank.getCountry()); + verifyNoInteractions(mapper); + } + + /** 旁路查询失败对采集端就等于"没缓存":照常开页,不能把采集打成 500。 */ + @Test + void queryFailureDegradesToMiss() { + when(mapper.selectByCountryAndAsins(anyString(), anyList())) + .thenThrow(new RuntimeException("cache table missing")); + + AsinCacheQueryVo vo = service.query("DE", List.of("B0ABC12345")); + + assertTrue(vo.getHits().isEmpty()); + } + + @Test + void queryOverCapReportsTruncatedAndSplitsIntoChunks() { + when(mapper.selectByCountryAndAsins(anyString(), anyList())).thenReturn(List.of()); + List asins = new ArrayList<>(); + for (int i = 0; i < MAX_QUERY_ASINS + 40; i++) { + asins.add(String.format("B0%08d", i)); + } + + AsinCacheQueryVo vo = service.query("DE", asins); + + assertEquals(40, vo.getTruncated(), "超出 max-query-asins 的部分要如实上报"); + verify(mapper, times(5)).selectByCountryAndAsins(eq("DE"), anyList()); + } + + // ==================== 写入 ==================== + + @Test + void writeDropsRowsThatAreNotReusableConclusions() { + List rows = List.of( + writeRow("OK", "9.90"), + writeRow("NOT_FOUND", ""), + writeRow("OK", ""), // 空价格 + writeRow("BLOCKED", "9.90"), // 拦截页 + writeRow("OK", "9.90", ""), // 无国家 + writeRow("OK", "9.90", null, "")); // 无 ASIN + + AsinCacheWriteVo vo = service.write(rows); + + assertEquals(4, vo.getDropped(), "只有 404 与有价两种结论可入库"); + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mapper).upsertBatch(captor.capture()); + assertEquals(List.of("OK", "NOT_FOUND"), + captor.getValue().stream().map(AsinResultCacheEntity::getAvailability).toList()); + assertEquals("DE", captor.getValue().get(0).getCountry()); + } + + /** 一个脏值的长度超列宽会让整批 INSERT 抛 Data too long,因此按列裁剪而不是整批丢。 */ + @Test + void writeClipsOverlongValues() { + AsinCacheWriteRowDto dto = writeRow("OK", "1".repeat(200)); + dto.setBrand("b".repeat(300)); + + service.write(List.of(dto)); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mapper).upsertBatch(captor.capture()); + assertEquals(32, captor.getValue().get(0).getPrice().length()); + assertEquals(255, captor.getValue().get(0).getBrand().length()); + } + + @Test + void writeSplitsByBatchSizeAndSurvivesFailingBatch() { + when(mapper.upsertBatch(anyList())).thenReturn(100).thenThrow(new RuntimeException("锁等待超时")); + List rows = new ArrayList<>(); + for (int i = 0; i < 150; i++) { + rows.add(writeRow("NOT_FOUND", "")); + } + + AsinCacheWriteVo vo = service.write(rows); + + verify(mapper, times(2)).upsertBatch(anyList()); + assertEquals(100, vo.getWritten(), "失败批不计入已写,但也不能让整个提交抛出去"); } + + // ==================== 关键词链旁路 ==================== + + @Test + void collectDataBypassSkipsWholeBatchWhenCountryUnknown() { + int written = service.writeCollectDataRows(null, "6161", List.of(collectRow("B0ABC12345", "9.90"))); + + assertEquals(0, written); + verifyNoInteractions(mapper); + } + + @Test + void collectDataBypassWritesOnlyPricedRows() { + when(mapper.upsertBatch(anyList())).thenReturn(1); + + service.writeCollectDataRows("de", "6161", List.of( + collectRow("B0ABC12345", "9.90"), + collectRow("B0ABC12346", " "))); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mapper).upsertBatch(captor.capture()); + AsinResultCacheEntity saved = captor.getValue().get(0); + assertEquals(1, captor.getValue().size(), "空价格行不入库"); + assertEquals("DE", saved.getCountry()); + assertEquals("OK", saved.getAvailability()); + assertEquals("COLLECT_DATA", saved.getSourceChain()); + assertEquals("6161", saved.getSourceTaskId()); + assertNotNull(saved.getObservedAt(), "缺 observed_at 会让新鲜度判定整体失效"); + } + + private List capturedAsins(int expectedSize) { + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mapper).selectByCountryAndAsins(eq("DE"), captor.capture()); + assertEquals(expectedSize, captor.getValue().size()); + return captor.getValue(); + } + + private static LocalDateTime minutesAgo(long minutes) { + return LocalDateTime.now().minusMinutes(minutes); + } + + private static AsinResultCacheEntity row(String availability, String price, LocalDateTime observedAt) { + AsinResultCacheEntity entity = new AsinResultCacheEntity(); + entity.setCountry("DE"); + entity.setAsin("B0ABC12345"); + entity.setAvailability(availability); + entity.setPrice(price); + entity.setObservedAt(observedAt); + return entity; + } + + private static AsinCacheWriteRowDto writeRow(String availability, String price) { + return writeRow(availability, price, "DE"); + } + + private static AsinCacheWriteRowDto writeRow(String availability, String price, String country) { + return writeRow(availability, price, country, "B0ABC12345"); + } + + private static AsinCacheWriteRowDto writeRow(String availability, String price, String country, String asin) { + AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto(); + dto.setCountry(country); + dto.setAsin(asin); + dto.setAvailability(availability); + dto.setPrice(price); + return dto; + } + + private static CollectDataResultRowVo collectRow(String asin, String price) { + CollectDataResultRowVo row = new CollectDataResultRowVo(); + row.setAsin(asin); + row.setPrice(price); + row.setBrand("Demo Brand"); + row.setDeliveryMethod("FBA"); + row.setSellerName("Demo Seller"); + return row; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java index 5172bcb2d..5c8a2e7f8 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java @@ -145,7 +145,8 @@ class CollectDataActivateGuardTest { mock(CollectDataInvalidAsinBatchWriter.class), mock(CollectDataResultDetailCodec.class), mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class), mock(TaskProgressLightAssembler.class), - mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class)); + mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class), + mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class)); } @SuppressWarnings("unchecked") diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java index f0b4fbed4..f122002fc 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java @@ -118,7 +118,8 @@ class CollectDataDeleteConsistencyTest { mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class), mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class), mock(TaskProgressLightAssembler.class), - mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class)); + mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class), + mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class)); } private FileTaskEntity task(long id, long userId) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java index e89c4486e..698735ce6 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java @@ -57,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -96,6 +97,7 @@ class CollectDataServiceTxBoundaryTest { @Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter; @Mock private CollectDataResultItemBatchWriter resultItemBatchWriter; @Mock private CollectDataResultDetailCodec resultDetailCodec; + @Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService; @InjectMocks private CollectDataService service; @@ -288,6 +290,51 @@ class CollectDataServiceTxBoundaryTest { .enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString()); } + // ==================== ASIN 跨任务缓存旁路写入 ==================== + + /** 关键词链的 accepted 行顺带写进跨任务缓存;国家取自任务 request_json,而非结果行本身。 */ + @Test + void acceptedRowsWrittenToAsinCacheWithTaskCountry() { + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithCountry("DE")); + + service.submitResult(TASK_ID, submitRequest()); + + verify(asinResultCacheService).writeCollectDataRows(eq("DE"), eq("6161"), any()); + } + + /** 多选国家的任务不写缓存:结果行本身不带国家,无法判断某一行属于哪个站点。 */ + @Test + void multiCountryTaskWritesNoCache() { + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithRequestJson( + "{\"filters\":{\"countryCodes\":[\"DE\",\"FR\"]}}")); + + service.submitResult(TASK_ID, submitRequest()); + + verify(asinResultCacheService).writeCollectDataRows(isNull(), anyString(), any()); + } + + /** 缓存写挂了不能把已经抓到手、正要落库的数据拖成提交失败 —— 旁路只记日志。 */ + @Test + void asinCacheWriteFailureDoesNotAffectSubmit() { + when(asinResultCacheService.writeCollectDataRows(any(), any(), any())) + .thenThrow(new RuntimeException("缓存库不可用")); + + CollectDataSubmitResultVo vo = service.submitResult(TASK_ID, submitRequest()); + + assertEquals(TASK_ID, vo.getTaskId()); + verify(taskChunkMapper).insert(any(TaskChunkEntity.class)); + } + + private FileTaskEntity taskWithCountry(String countryCode) { + return taskWithRequestJson("{\"filters\":{\"countryCode\":\"" + countryCode + "\"}}"); + } + + private FileTaskEntity taskWithRequestJson(String requestJson) { + FileTaskEntity task = runningTask(); + task.setRequestJson(requestJson); + return task; + } + private CollectDataSubmitResultRequest submitRequest() { CollectDataSubmitRowDto row = new CollectDataSubmitRowDto(); row.setAsin("B0COLLECT1"); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java index ca7823c1e..71cbcd59d 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java @@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest { mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class), mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class), mock(TaskProgressLightAssembler.class), - mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class)); + mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class), + mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class)); } private FileResultEntity result(long id, long taskId, long userId) {