feat(ASIN缓存): 新增 ASIN 采集结果跨任务缓存(asincache 模块 + V139)
同一个 (国家, ASIN) 的详情页被反复重采:单任务内 62.9 倍重复(43,504 行只有 692 个 不同 ASIN)、8 天跨任务 6.7 万行重复(2026-10-01 实测),每条重复都意味着重开一次页、 可能重买一次代理 IP。关键词链与变体链的结果分别落在两处互相看不见,只有 Java 侧能同时 收到两条链的结论,所以缓存放服务端、由服务端判新鲜度。 - 新表 biz_asin_result_cache(V139):一个 (国家, ASIN) 一行,重采覆盖;只收两种可复用 结论 —— OK(页面正常且有价格,复用 60 分钟)、NOT_FOUND(详情页 404,复用 7 天); 空价 / 风控拦截页 / 采集异常一律不入库,避免把瞬时故障锁成永久结论 - 新鲜度只在服务端判(按 observed_at 真正开页时间);采集端拿不到 / 看不懂一律照常开页 (fail-open),任何情况下不许让缓存导致漏行、错行 - 新增内部接口 /api/internal/asin-cache/query|write(X-Internal-Token 自校验,与 InternalBrandTaskController 同口径),供采集端开页前先问、采完回写 - 关键词采集链在任务锁外旁路写入:缓存写失败的代价只是"下次多开一次页",只记 warn, 不影响 chunk 提交 - migration-inventory.md 快照更新至 V139,「V137 冗余索引 DROP」待办顺延 V140 测试:AsinResultCacheServiceTest(13)/ InternalAsinCacheControllerTest(5)/ 采集链 4 个测试类(35)共 53 例全绿。
This commit is contained in:
@@ -1,14 +1,14 @@
|
||||
# Flyway 迁移盘点(task-193)
|
||||
|
||||
> 生成方式:`src/test/java/com/nanri/aiimage/config/MigrationInventoryTest.java`(只读审计,可重复)。
|
||||
> 快照日期:2026-09-28。
|
||||
> 快照日期:2026-10-01。
|
||||
|
||||
## 概览
|
||||
|
||||
- 版本化迁移文件数:**137**(`src/main/resources/db/V*.sql`)
|
||||
- 整数版本范围:**V1..V138**,其中 V132/V133 为已退役空号(见下),其余连续
|
||||
- 版本化迁移文件数:**138**(`src/main/resources/db/V*.sql`,其中整数版本 137 个 + 小版本 V25_1)
|
||||
- 整数版本范围:**V1..V139**,其中 V132/V133 为已退役空号(见下),其余连续
|
||||
- 历史遗留小版本:**V25_1__shop_manage_group_bind_user.sql**(Flyway 语义 25.1,紧跟在 V25 之后、V26 之前执行,属历史命名,保留)
|
||||
- 最新版本:**V138__client_task_console.sql**
|
||||
- 最新版本:**V139__asin_result_cache.sql**
|
||||
- 重复版本:无
|
||||
- 迁移命名:全部符合 `V<整数>(_<子版本>)?__<描述>.sql`,无空格
|
||||
|
||||
@@ -20,7 +20,9 @@
|
||||
|
||||
本地曾存在 `V132__web_config_add_is_latest.sql`、`V133__hide_redundant_indexes_invisible.sql`,与另一分支已在生产落库的 V134/V135 冲突。因 Flyway 未开启 `out-of-order`,把任何迁移编到 132/133 会在启动期报 "resolved migration not applied";因此这两条改为 **V136/V137**,132/133 永久退役,不可回填。
|
||||
|
||||
后续新增迁移从 **V139** 起取号。
|
||||
后续新增迁移从 **V140** 起取号(只认"当前最大值 + 1",不要为还没写出来的迁移预留号段——预留会在 1..max 里留洞,`integerVersionsSequentialFromOne` 会红)。
|
||||
|
||||
原先挂在待办里的"V137 冗余索引 DROP"迁移计划取 V139,该号已被 `V139__asin_result_cache.sql` 占用,顺延到 V140。
|
||||
|
||||
## 约束
|
||||
|
||||
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package com.nanri.aiimage.modules.asincache.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto;
|
||||
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
|
||||
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* 内部接口:ASIN 采集结果缓存的读写,供采集端(麦象采集机 / 桌面端)在开页前先问一次
|
||||
* 「这个 (国家, ASIN) 是否已经有人开过」。
|
||||
*
|
||||
* <p>鉴权与 {@code InternalBrandTaskController} 同口径:/api/internal 前缀已在
|
||||
* AdminApiGuardFilter 兜底名单内,controller 仍自校验 X-Internal-Token —— 防止配置漂移时匿名可达。
|
||||
*
|
||||
* <p>这两个端点是<b>旁路</b>:采集端拿不到响应时必须照常开页(fail-open),所以
|
||||
* 服务端任何异常都只可能让采集变慢,绝不可能让数据变少或变错。
|
||||
*/
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
@RequestMapping("/api/internal/asin-cache")
|
||||
@Tag(name = "内部接口", description = "服务间调用(X-Internal-Token 鉴权)。")
|
||||
public class InternalAsinCacheController {
|
||||
|
||||
private final AsinResultCacheService asinResultCacheService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
@PostMapping("/query")
|
||||
@Operation(summary = "批量查可复用的 ASIN 采集结论(采集端调用)",
|
||||
description = "只返回新鲜度达标(OK 60 分钟 / NOT_FOUND 7 天,均可配置)的行;"
|
||||
+ "未出现在 hits 里的 ASIN 按未命中处理,调用方照常开页。")
|
||||
public ApiResponse<AsinCacheQueryVo> query(HttpServletRequest request,
|
||||
@RequestBody(required = false) AsinCacheQueryDto body) {
|
||||
requireTrustedToken(request, "query");
|
||||
if (body == null) {
|
||||
throw new BusinessException(400, "缺少查询参数");
|
||||
}
|
||||
AsinCacheQueryVo vo = asinResultCacheService.query(body.getCountry(), body.getAsins());
|
||||
log.info("[internal-asin-cache] 查询 country={} 请求={} 命中={} 截断={}",
|
||||
vo.getCountry(), body.getAsins() == null ? 0 : body.getAsins().size(),
|
||||
vo.getHits().size(), vo.getTruncated());
|
||||
return ApiResponse.success(vo);
|
||||
}
|
||||
|
||||
@PostMapping("/write")
|
||||
@Operation(summary = "批量写入 ASIN 采集结论(采集端调用)",
|
||||
description = "不可复用的结论(空价/拦截页/采集异常)服务端逐行丢弃;单批写失败只记日志,不返回失败。")
|
||||
public ApiResponse<AsinCacheWriteVo> write(HttpServletRequest request,
|
||||
@RequestBody(required = false) AsinCacheWriteDto body) {
|
||||
requireTrustedToken(request, "write");
|
||||
AsinCacheWriteVo vo = asinResultCacheService.write(
|
||||
body == null ? null : body.getRows());
|
||||
log.info("[internal-asin-cache] 写入 请求={} 入库={} 丢弃={} 截断={}",
|
||||
body == null || body.getRows() == null ? 0 : body.getRows().size(),
|
||||
vo.getWritten(), vo.getDropped(), vo.getTruncated());
|
||||
return ApiResponse.success(vo);
|
||||
}
|
||||
|
||||
private void requireTrustedToken(HttpServletRequest request, String action) {
|
||||
if (adminAuthSupport.isTrustedInternalToken(request)) {
|
||||
return;
|
||||
}
|
||||
log.warn("[internal-asin-cache] 拒绝未携带可信内部令牌的请求 action={} remoteAddr={}",
|
||||
action, request.getRemoteAddr());
|
||||
throw new BusinessException(401, "未授权");
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package com.nanri.aiimage.modules.asincache.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface AsinResultCacheMapper extends BaseMapper<AsinResultCacheEntity> {
|
||||
|
||||
/**
|
||||
* 一次取回一个国家下这批 ASIN 的缓存行。
|
||||
*
|
||||
* <p>走唯一键 uk_asin_cache_country_asin (country, asin) 的前缀,所以必须是
|
||||
* 「country = ? AND asin IN (...)」这种等值 + IN 的形状;把 country 放进 IN 里
|
||||
* 成对匹配((country, asin) IN ((..),(..)))在 MySQL 上用不上索引。
|
||||
*/
|
||||
@Select("""
|
||||
<script>
|
||||
SELECT id, country, asin, availability, price, brand, delivery_method, seller_name,
|
||||
seller_id, declared_members_json, source_chain, source_task_id,
|
||||
observed_at, created_at, updated_at
|
||||
FROM biz_asin_result_cache
|
||||
WHERE country = #{country}
|
||||
AND asin IN
|
||||
<foreach collection="asins" item="asin" open="(" separator="," close=")">#{asin}</foreach>
|
||||
</script>
|
||||
""")
|
||||
List<AsinResultCacheEntity> selectByCountryAndAsins(@Param("country") String country,
|
||||
@Param("asins") List<String> asins);
|
||||
|
||||
/**
|
||||
* 按唯一键 (country, asin) 批量 upsert:重采覆盖旧结论,不新增行。
|
||||
*
|
||||
* <p>两条采集链往同一行里写的字段并不一样多 —— 关键词链没有 seller_id 也没有变体
|
||||
* 家族,变体链没有 seller 信息。这些"这次没采到"的列在实体里是 NULL(空白值入库前
|
||||
* 已被 clip 成 NULL),所以它们用 COALESCE 继承旧值:新观测带了就覆盖,没带就保留,
|
||||
* 否则一条变体记录会把关键词记录里的卖家名抹成空,下次谁都读不到。
|
||||
*
|
||||
* <p>price / availability / source_* / observed_at 不参与继承:它们是本行结论的主体,
|
||||
* 404 就该把价格清空,"谁最后开的页"也必须如实反映最后一次开页。
|
||||
*/
|
||||
@Insert("""
|
||||
<script>
|
||||
INSERT INTO biz_asin_result_cache
|
||||
(country, asin, availability, price, brand, delivery_method, seller_name, seller_id,
|
||||
declared_members_json, source_chain, source_task_id, observed_at, created_at, updated_at)
|
||||
VALUES
|
||||
<foreach collection="rows" item="row" separator=",">
|
||||
(#{row.country}, #{row.asin}, #{row.availability}, #{row.price}, #{row.brand},
|
||||
#{row.deliveryMethod}, #{row.sellerName}, #{row.sellerId}, #{row.declaredMembersJson},
|
||||
#{row.sourceChain}, #{row.sourceTaskId}, #{row.observedAt}, #{row.createdAt}, #{row.updatedAt})
|
||||
</foreach>
|
||||
ON DUPLICATE KEY UPDATE
|
||||
availability = VALUES(availability),
|
||||
price = VALUES(price),
|
||||
brand = COALESCE(VALUES(brand), brand),
|
||||
delivery_method = COALESCE(VALUES(delivery_method), delivery_method),
|
||||
seller_name = COALESCE(VALUES(seller_name), seller_name),
|
||||
seller_id = COALESCE(VALUES(seller_id), seller_id),
|
||||
declared_members_json = COALESCE(VALUES(declared_members_json), declared_members_json),
|
||||
source_chain = VALUES(source_chain),
|
||||
source_task_id = VALUES(source_task_id),
|
||||
observed_at = VALUES(observed_at),
|
||||
updated_at = VALUES(updated_at)
|
||||
</script>
|
||||
""")
|
||||
int upsertBatch(@Param("rows") List<AsinResultCacheEntity> rows);
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.asincache.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "ASIN 缓存批量查询请求(内部接口)")
|
||||
public class AsinCacheQueryDto {
|
||||
|
||||
@JsonAlias({"country_code", "countryCode"})
|
||||
@Schema(description = "国家代码(大小写不敏感)", example = "UK")
|
||||
private String country;
|
||||
|
||||
@Schema(description = "待查 ASIN 列表,超出上限的部分由服务端丢弃(按请求顺序保留前面那些)")
|
||||
private List<String> asins = new ArrayList<>();
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.asincache.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "ASIN 缓存批量写入请求(内部接口)")
|
||||
public class AsinCacheWriteDto {
|
||||
|
||||
@JsonAlias({"rows", "items"})
|
||||
@Schema(description = "待写入行;不可复用的结论(空价/拦截/异常)服务端会逐行丢弃")
|
||||
private List<AsinCacheWriteRowDto> rows = new ArrayList<>();
|
||||
}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package com.nanri.aiimage.modules.asincache.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* ASIN 缓存写入行。
|
||||
*
|
||||
* <p>字段一律同时接受 camelCase 与 snake_case:调用方是 Python 采集端,snake_case 是它的
|
||||
* 自然写法;而 Java 侧序列化出来的是 camelCase,两头都认就不用任何一侧迁就。
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "ASIN 缓存写入行(内部接口)")
|
||||
public class AsinCacheWriteRowDto {
|
||||
|
||||
@JsonAlias({"country_code", "countryCode"})
|
||||
@Schema(description = "国家代码", example = "UK")
|
||||
private String country;
|
||||
|
||||
@JsonAlias({"ASIN", "asin"})
|
||||
@Schema(description = "商品 ASIN", example = "B0XXXXXXX")
|
||||
private String asin;
|
||||
|
||||
@JsonAlias({"availability", "status"})
|
||||
@Schema(description = "结论类型:OK(页面正常且有价格)/ NOT_FOUND(404 不存在)。"
|
||||
+ "其余取值服务端一律不入库 —— 空价/拦截页/采集异常都可能是采集端自己的解析或风控问题",
|
||||
example = "OK")
|
||||
private String availability;
|
||||
|
||||
@Schema(description = "价格原文(保留当地币种符号,不数值化)", example = "£12.34")
|
||||
private String price;
|
||||
|
||||
@JsonAlias({"品牌", "brand_name"})
|
||||
@Schema(description = "品牌")
|
||||
private String brand;
|
||||
|
||||
@JsonAlias({"delivery_method", "fulfillment"})
|
||||
@Schema(description = "配送方式", example = "FBA")
|
||||
private String deliveryMethod;
|
||||
|
||||
@JsonAlias({"seller_name", "卖家名称"})
|
||||
@Schema(description = "卖家名称")
|
||||
private String sellerName;
|
||||
|
||||
@JsonAlias({"seller_id", "sellerID", "merchant_id"})
|
||||
@Schema(description = "merchantID")
|
||||
private String sellerId;
|
||||
|
||||
@JsonAlias({"declared_members", "declaredMembers"})
|
||||
@Schema(description = "该 ASIN 页面自己声明的变体成员列表;跨任务跳过开页时靠它补齐产出行")
|
||||
private List<String> declaredMembers;
|
||||
|
||||
@JsonAlias({"source_chain", "sourceChain"})
|
||||
@Schema(description = "来源采集链:COLLECT_DATA / VARIANT / DETAIL", example = "VARIANT")
|
||||
private String sourceChain;
|
||||
|
||||
@JsonAlias({"source_task_id", "sourceTaskId", "task_id"})
|
||||
@Schema(description = "来源任务标识,仅用于排查")
|
||||
private String sourceTaskId;
|
||||
|
||||
@JsonAlias({"observed_at", "observedAt"})
|
||||
@Schema(description = "采集机上真正打开该页面的时间;缺省时服务端按当前时间记账",
|
||||
example = "2026-10-01T10:12:30")
|
||||
private LocalDateTime observedAt;
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package com.nanri.aiimage.modules.asincache.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* ASIN 采集结果缓存行(biz_asin_result_cache)。
|
||||
*
|
||||
* <p>一个 (country, asin) 一行,重采即覆盖;新鲜度按 {@code observedAt}(采集机上真正
|
||||
* 打开页面的时间)算,不按入库时间 —— 否则客户端攒批延迟回传会把旧结论算成新结论。
|
||||
*/
|
||||
@Data
|
||||
@TableName("biz_asin_result_cache")
|
||||
public class AsinResultCacheEntity {
|
||||
|
||||
/** 可复用结论:页面正常且取到价格。 */
|
||||
public static final String AVAILABILITY_OK = "OK";
|
||||
/** 可复用结论:详情页 404,该 ASIN 不存在。 */
|
||||
public static final String AVAILABILITY_NOT_FOUND = "NOT_FOUND";
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String country;
|
||||
private String asin;
|
||||
private String availability;
|
||||
private String price;
|
||||
private String brand;
|
||||
private String deliveryMethod;
|
||||
private String sellerName;
|
||||
private String sellerId;
|
||||
private String declaredMembersJson;
|
||||
private String sourceChain;
|
||||
private String sourceTaskId;
|
||||
private LocalDateTime observedAt;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
package com.nanri.aiimage.modules.asincache.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "可复用的 ASIN 缓存结论")
|
||||
public class AsinCacheHitVo {
|
||||
|
||||
@Schema(description = "商品 ASIN")
|
||||
private String asin;
|
||||
|
||||
@Schema(description = "结论类型:OK / NOT_FOUND")
|
||||
private String availability;
|
||||
|
||||
@Schema(description = "价格原文;NOT_FOUND 时为空")
|
||||
private String price;
|
||||
|
||||
@Schema(description = "品牌")
|
||||
private String brand;
|
||||
|
||||
@Schema(description = "配送方式")
|
||||
private String deliveryMethod;
|
||||
|
||||
@Schema(description = "卖家名称")
|
||||
private String sellerName;
|
||||
|
||||
@Schema(description = "merchantID")
|
||||
private String sellerId;
|
||||
|
||||
@Schema(description = "该 ASIN 页面声明过的变体成员列表(跳过开页时靠它补齐产出行)")
|
||||
private List<String> declaredMembers;
|
||||
|
||||
@Schema(description = "结论产生时间,采集端可据此在日志里报出复用了多久以前的数据")
|
||||
private LocalDateTime observedAt;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.nanri.aiimage.modules.asincache.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "ASIN 缓存批量查询结果")
|
||||
public class AsinCacheQueryVo {
|
||||
|
||||
@Schema(description = "回显国家代码(大写)")
|
||||
private String country;
|
||||
|
||||
@Schema(description = "命中的结论;未列出的 ASIN 就是没有可复用结论,调用方照常开页")
|
||||
private List<AsinCacheHitVo> hits = new ArrayList<>();
|
||||
|
||||
@Schema(description = "因超出单次上限而被忽略的 ASIN 个数(这些一律按未命中处理)")
|
||||
private int truncated;
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.nanri.aiimage.modules.asincache.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "ASIN 缓存批量写入结果")
|
||||
public class AsinCacheWriteVo {
|
||||
|
||||
@Schema(description = "实际入库的行数(含覆盖旧行)")
|
||||
private int written;
|
||||
|
||||
@Schema(description = "被服务端丢弃的行数:结论不可复用、ASIN 非法、或超出单次上限")
|
||||
private int dropped;
|
||||
|
||||
@Schema(description = "本次请求被丢弃的行数上限之外的部分,调用方无需重试(重试也一样被丢)")
|
||||
private int truncated;
|
||||
}
|
||||
+374
@@ -0,0 +1,374 @@
|
||||
package com.nanri.aiimage.modules.asincache.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.asincache.mapper.AsinResultCacheMapper;
|
||||
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteRowDto;
|
||||
import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheHitVo;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* ASIN 采集结果缓存:跨任务、跨采集链复用「同一个 (国家, ASIN) 详情页已经开过」这个事实。
|
||||
*
|
||||
* <p><b>新鲜度只在这里判</b>,采集端不参与。两条链的采集端版本不可能同时升级,
|
||||
* 把口径放在服务端意味着老采集端拿到 404/未知字段只会当没缓存用(fail-open),
|
||||
* 而不是各按各的理解复用到过期数据。
|
||||
*
|
||||
* <p>可复用只有两档(其余一律不入库,见 {@link #ttlMinutesOf}):
|
||||
* <ul>
|
||||
* <li>{@code NOT_FOUND} —— 详情页 404。ASIN 不会"复活",缓存最久(默认 7 天),省开页最多;</li>
|
||||
* <li>{@code OK} —— 页面正常且取到价格。价格/配送会变,只复用很短一段时间(默认 60 分钟)。</li>
|
||||
* </ul>
|
||||
* 空价格、验证码/风控拦截页、采集异常都不写:这些结论要么是瞬时故障,要么是我们自己的
|
||||
* 解析问题,缓存它等于把这个 ASIN 后面每次出现都锁成空行。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class AsinResultCacheService {
|
||||
|
||||
public static final String CHAIN_COLLECT_DATA = "COLLECT_DATA";
|
||||
|
||||
/** 列宽截断上限:超长值会让整批 INSERT 抛 Data too long,一行的脏数据拖垮整个 chunk。 */
|
||||
private static final int MAX_ASIN_LENGTH = 32;
|
||||
private static final int MAX_COUNTRY_LENGTH = 16;
|
||||
private static final int MAX_PRICE_LENGTH = 32;
|
||||
private static final int MAX_BRAND_LENGTH = 255;
|
||||
private static final int MAX_DELIVERY_LENGTH = 32;
|
||||
private static final int MAX_SELLER_NAME_LENGTH = 255;
|
||||
private static final int MAX_SELLER_ID_LENGTH = 64;
|
||||
private static final int MAX_SOURCE_TASK_ID_LENGTH = 64;
|
||||
private static final int MAX_DECLARED_MEMBERS = 500;
|
||||
/** IN (...) 单条 SQL 的 ASIN 个数上限:查询与写入共用,避免超长 SQL 打爆解析器。 */
|
||||
private static final int SQL_CHUNK_SIZE = 100;
|
||||
|
||||
private final AsinResultCacheMapper asinResultCacheMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final int pricedTtlMinutes;
|
||||
private final int notFoundTtlMinutes;
|
||||
private final int maxQueryAsins;
|
||||
private final int maxWriteRows;
|
||||
private final int writeBatchSize;
|
||||
|
||||
public AsinResultCacheService(AsinResultCacheMapper asinResultCacheMapper,
|
||||
ObjectMapper objectMapper,
|
||||
@Value("${aiimage.asin-cache.priced-ttl-minutes:60}") int pricedTtlMinutes,
|
||||
@Value("${aiimage.asin-cache.not-found-ttl-hours:168}") int notFoundTtlHours,
|
||||
@Value("${aiimage.asin-cache.max-query-asins:500}") int maxQueryAsins,
|
||||
@Value("${aiimage.asin-cache.max-write-rows:500}") int maxWriteRows,
|
||||
@Value("${aiimage.asin-cache.write-batch-size:100}") int writeBatchSize) {
|
||||
this.asinResultCacheMapper = asinResultCacheMapper;
|
||||
this.objectMapper = objectMapper;
|
||||
this.pricedTtlMinutes = Math.max(0, pricedTtlMinutes);
|
||||
this.notFoundTtlMinutes = Math.max(0, notFoundTtlHours) * 60;
|
||||
this.maxQueryAsins = Math.max(1, maxQueryAsins);
|
||||
this.maxWriteRows = Math.max(1, maxWriteRows);
|
||||
this.writeBatchSize = Math.max(1, writeBatchSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量查可复用结论。未出现在返回值里的 ASIN 就是"没有可复用结论",调用方照常开页 ——
|
||||
* 复用是省时间的旁路,任何情况下都不许让采集因为缓存而漏行。
|
||||
*/
|
||||
public AsinCacheQueryVo query(String country, Collection<String> asins) {
|
||||
AsinCacheQueryVo vo = new AsinCacheQueryVo();
|
||||
String normalizedCountry = normalizeCountry(country);
|
||||
vo.setCountry(normalizedCountry);
|
||||
if (normalizedCountry == null || asins == null || asins.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
List<String> distinct = new ArrayList<>(new LinkedHashSet<>(cleanAsins(asins)));
|
||||
if (distinct.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
if (distinct.size() > maxQueryAsins) {
|
||||
vo.setTruncated(distinct.size() - maxQueryAsins);
|
||||
distinct = distinct.subList(0, maxQueryAsins);
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
List<AsinCacheHitVo> hits = new ArrayList<>();
|
||||
for (int from = 0; from < distinct.size(); from += SQL_CHUNK_SIZE) {
|
||||
List<String> batch = distinct.subList(from, Math.min(from + SQL_CHUNK_SIZE, distinct.size()));
|
||||
List<AsinResultCacheEntity> rows;
|
||||
try {
|
||||
rows = asinResultCacheMapper.selectByCountryAndAsins(normalizedCountry, batch);
|
||||
} catch (RuntimeException ex) {
|
||||
// 查询失败与"没有缓存"对采集端是同一件事:照常开页。不能因为旁路挂了就 500。
|
||||
log.warn("[asin-cache] 缓存查询失败,按未命中处理 country={} size={} err={}",
|
||||
normalizedCountry, batch.size(), ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
for (AsinResultCacheEntity row : rows) {
|
||||
if (isReusable(row, now)) {
|
||||
hits.add(toHit(row));
|
||||
}
|
||||
}
|
||||
}
|
||||
vo.setHits(hits);
|
||||
return vo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量写入结论。不可复用的行在这里丢弃(而不是在采集端):采集端有版本差,
|
||||
* 老客户端可能还在上报空价格 —— 服务端不收,表里就不会出现被误复用的脏结论。
|
||||
*
|
||||
* <p>单批失败只记日志、不抛:调用方是采集热路径与关键词结果提交链路,
|
||||
* 缓存写不进去的代价是"下次多开一次页",把任务整体拖失败是更大的代价。
|
||||
*/
|
||||
public AsinCacheWriteVo write(List<AsinCacheWriteRowDto> rows) {
|
||||
AsinCacheWriteVo vo = new AsinCacheWriteVo();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
if (rows.size() > maxWriteRows) {
|
||||
vo.setTruncated(rows.size() - maxWriteRows);
|
||||
rows = rows.subList(0, maxWriteRows);
|
||||
}
|
||||
List<AsinResultCacheEntity> entities = new ArrayList<>(rows.size());
|
||||
for (AsinCacheWriteRowDto row : rows) {
|
||||
AsinResultCacheEntity entity = toEntity(row);
|
||||
if (entity == null) {
|
||||
vo.setDropped(vo.getDropped() + 1);
|
||||
continue;
|
||||
}
|
||||
entities.add(entity);
|
||||
}
|
||||
if (entities.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
for (int from = 0; from < entities.size(); from += writeBatchSize) {
|
||||
List<AsinResultCacheEntity> batch = entities.subList(from, Math.min(from + writeBatchSize, entities.size()));
|
||||
try {
|
||||
vo.setWritten(vo.getWritten() + asinResultCacheMapper.upsertBatch(batch));
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[asin-cache] 缓存批量写入失败,跳过该批 size={} err={}", batch.size(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 关键词采集链的旁路写入:结果行落 biz_task_result_item 之后,把"这个 (国家, ASIN)
|
||||
* 的价格/品牌/配送已经开页看过"这件事同步记进缓存,让变体采集下次不必重开。
|
||||
*
|
||||
* <p>country 缺失时整批不写 —— 缓存键少了国家,就可能把 DE 的价格当成 US 的结论发回去。
|
||||
* 关键词链按 ASIN 逐个采详情页,一批 accepted 行同属一个任务、一个国家,所以这里
|
||||
* 只按行计数,不逐行判国家。
|
||||
*/
|
||||
public int writeCollectDataRows(String country, String sourceTaskId,
|
||||
List<CollectDataResultRowVo> rows) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
String normalizedCountry = normalizeCountry(country);
|
||||
if (normalizedCountry == null) {
|
||||
log.info("[asin-cache] 任务未带国家代码,跳过缓存写入(避免跨国家错用)rows={}", rows.size());
|
||||
return 0;
|
||||
}
|
||||
List<AsinCacheWriteRowDto> payload = new ArrayList<>(rows.size());
|
||||
for (CollectDataResultRowVo row : rows) {
|
||||
if (row == null || isBlank(row.getPrice())) {
|
||||
continue; // 空价格不可复用,见类注释
|
||||
}
|
||||
AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto();
|
||||
dto.setCountry(normalizedCountry);
|
||||
dto.setAsin(row.getAsin());
|
||||
dto.setAvailability(AsinResultCacheEntity.AVAILABILITY_OK);
|
||||
dto.setPrice(row.getPrice());
|
||||
dto.setBrand(row.getBrand());
|
||||
dto.setDeliveryMethod(row.getDeliveryMethod());
|
||||
dto.setSellerName(row.getSellerName());
|
||||
dto.setSourceChain(CHAIN_COLLECT_DATA);
|
||||
dto.setSourceTaskId(sourceTaskId);
|
||||
payload.add(dto);
|
||||
}
|
||||
if (payload.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
AsinCacheWriteVo written = write(payload);
|
||||
// 关键词链是"顺带"贡献缓存的:写满/写坏都不该在业务日志里响,降到 debug 级别由排查时打开。
|
||||
log.debug("[asin-cache] 关键词链旁路写入 taskId={} rows={} accepted={} dropped={} truncated={}",
|
||||
sourceTaskId, payload.size(), written.getWritten(), written.getDropped(), written.getTruncated());
|
||||
return written.getWritten();
|
||||
}
|
||||
|
||||
// ==================== 口径 ====================
|
||||
|
||||
/** 该 availability 的可复用时长(分钟);返回 0 表示永不复用。未知取值一律按 0 处理。 */
|
||||
int ttlMinutesOf(String availability) {
|
||||
if (AsinResultCacheEntity.AVAILABILITY_OK.equals(availability)) {
|
||||
return pricedTtlMinutes;
|
||||
}
|
||||
if (AsinResultCacheEntity.AVAILABILITY_NOT_FOUND.equals(availability)) {
|
||||
return notFoundTtlMinutes;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** 新鲜度判定:按 observed_at(真正开页时间),没有才退到 created_at(入库时间)。 */
|
||||
boolean isReusable(AsinResultCacheEntity row, LocalDateTime now) {
|
||||
if (row == null || row.getAsin() == null) {
|
||||
return false;
|
||||
}
|
||||
int ttlMinutes = ttlMinutesOf(row.getAvailability());
|
||||
if (ttlMinutes <= 0) {
|
||||
return false;
|
||||
}
|
||||
if (AsinResultCacheEntity.AVAILABILITY_OK.equals(row.getAvailability()) && isBlank(row.getPrice())) {
|
||||
// 理论上写不进来(write 已挡),但库里万一有历史脏行,不能让它被当成有效价格复用
|
||||
return false;
|
||||
}
|
||||
LocalDateTime observedAt = row.getObservedAt() != null ? row.getObservedAt() : row.getCreatedAt();
|
||||
if (observedAt == null) {
|
||||
return false;
|
||||
}
|
||||
return observedAt.plusMinutes(ttlMinutes).isAfter(now);
|
||||
}
|
||||
|
||||
// ==================== 归一化 ====================
|
||||
|
||||
private AsinResultCacheEntity toEntity(AsinCacheWriteRowDto row) {
|
||||
if (row == null) {
|
||||
return null;
|
||||
}
|
||||
String country = normalizeCountry(row.getCountry());
|
||||
String asin = normalizeAsin(row.getAsin());
|
||||
if (country == null || asin == null) {
|
||||
return null;
|
||||
}
|
||||
String availability = row.getAvailability() == null ? ""
|
||||
: row.getAvailability().trim().toUpperCase(Locale.ROOT);
|
||||
if (ttlMinutesOf(availability) <= 0) {
|
||||
return null; // 不可复用的结论(含未知取值)不进表
|
||||
}
|
||||
if (AsinResultCacheEntity.AVAILABILITY_OK.equals(availability) && isBlank(row.getPrice())) {
|
||||
return null;
|
||||
}
|
||||
AsinResultCacheEntity entity = new AsinResultCacheEntity();
|
||||
entity.setCountry(country);
|
||||
entity.setAsin(asin);
|
||||
entity.setAvailability(availability);
|
||||
entity.setPrice(clip(row.getPrice(), MAX_PRICE_LENGTH));
|
||||
entity.setBrand(clip(row.getBrand(), MAX_BRAND_LENGTH));
|
||||
entity.setDeliveryMethod(clip(row.getDeliveryMethod(), MAX_DELIVERY_LENGTH));
|
||||
entity.setSellerName(clip(row.getSellerName(), MAX_SELLER_NAME_LENGTH));
|
||||
entity.setSellerId(clip(row.getSellerId(), MAX_SELLER_ID_LENGTH));
|
||||
entity.setDeclaredMembersJson(toDeclaredMembersJson(row.getDeclaredMembers()));
|
||||
entity.setSourceChain(clip(row.getSourceChain(), 32));
|
||||
entity.setSourceTaskId(clip(row.getSourceTaskId(), MAX_SOURCE_TASK_ID_LENGTH));
|
||||
// 缺 observed_at 时按当前时间记账:等价于"刚采到",新鲜度判定不会因为客户端没传就整体失效
|
||||
entity.setObservedAt(row.getObservedAt() != null ? row.getObservedAt() : LocalDateTime.now());
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
entity.setCreatedAt(now);
|
||||
entity.setUpdatedAt(now);
|
||||
return entity;
|
||||
}
|
||||
|
||||
private AsinCacheHitVo toHit(AsinResultCacheEntity row) {
|
||||
AsinCacheHitVo hit = new AsinCacheHitVo();
|
||||
hit.setAsin(row.getAsin());
|
||||
hit.setAvailability(row.getAvailability());
|
||||
hit.setPrice(row.getPrice());
|
||||
hit.setBrand(row.getBrand());
|
||||
hit.setDeliveryMethod(row.getDeliveryMethod());
|
||||
hit.setSellerName(row.getSellerName());
|
||||
hit.setSellerId(row.getSellerId());
|
||||
hit.setDeclaredMembers(fromDeclaredMembersJson(row.getDeclaredMembersJson()));
|
||||
hit.setObservedAt(row.getObservedAt() != null ? row.getObservedAt() : row.getCreatedAt());
|
||||
return hit;
|
||||
}
|
||||
|
||||
/** 国家代码是缓存键的一半:大写归一、空值即拒,绝不用空国家写行(会与其他国家撞键)。 */
|
||||
private static String normalizeCountry(String country) {
|
||||
String value = country == null ? "" : country.trim().toUpperCase(Locale.ROOT);
|
||||
if (value.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
return value.length() > MAX_COUNTRY_LENGTH ? null : value;
|
||||
}
|
||||
|
||||
private static String normalizeAsin(String asin) {
|
||||
String value = asin == null ? "" : asin.trim().toUpperCase(Locale.ROOT);
|
||||
if (value.isEmpty() || value.length() > MAX_ASIN_LENGTH) {
|
||||
return null;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static List<String> cleanAsins(Collection<String> asins) {
|
||||
List<String> out = new ArrayList<>(asins.size());
|
||||
for (String asin : asins) {
|
||||
String normalized = normalizeAsin(asin);
|
||||
if (normalized != null) {
|
||||
out.add(normalized);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private String toDeclaredMembersJson(List<String> members) {
|
||||
if (members == null || members.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
Collection<String> clean = new LinkedHashSet<>();
|
||||
for (String member : members) {
|
||||
String normalized = normalizeAsin(member);
|
||||
if (normalized != null) {
|
||||
clean.add(normalized);
|
||||
}
|
||||
if (clean.size() >= MAX_DECLARED_MEMBERS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (clean.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.writeValueAsString(clean);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[asin-cache] 变体成员列表序列化失败,按无成员入库 err={}", ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> fromDeclaredMembersJson(String json) {
|
||||
if (isBlank(json)) {
|
||||
return List.of();
|
||||
}
|
||||
try {
|
||||
List<String> members = objectMapper.readValue(json,
|
||||
objectMapper.getTypeFactory().constructCollectionType(List.class, String.class));
|
||||
return members == null ? List.of() : members;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[asin-cache] 变体成员列表解析失败,按无成员返回 json={} err={}",
|
||||
json.length() > 120 ? json.substring(0, 120) + "..." : json, ex.getMessage());
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
private static String clip(String value, int maxLength) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String trimmed = value.trim();
|
||||
if (trimmed.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
return trimmed.length() > maxLength ? trimmed.substring(0, maxLength) : trimmed;
|
||||
}
|
||||
|
||||
private static boolean isBlank(String value) {
|
||||
return value == null || value.isBlank();
|
||||
}
|
||||
}
|
||||
+44
@@ -12,6 +12,7 @@ import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
@@ -182,6 +183,9 @@ public class CollectDataService {
|
||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
private final NotificationDispatchService notificationDispatchService;
|
||||
|
||||
/** ASIN 跨任务结果缓存:本链只负责"顺带贡献结论",写入失败绝不影响 chunk 提交。 */
|
||||
private final AsinResultCacheService asinResultCacheService;
|
||||
|
||||
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
||||
private long staleTimeoutMinutes;
|
||||
|
||||
@@ -830,6 +834,11 @@ public class CollectDataService {
|
||||
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
||||
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
||||
|
||||
// ASIN 结果缓存旁路写入同样放锁外:这把锁的持有时间是心跳与其他 chunk 的排队预算
|
||||
// (见上方品牌检测移出的注释),缓存写入再快也不该来挤。缓存是"下次少开一次页"的
|
||||
// 加速件,同一 chunk 重复提交只是把同样的结论再 upsert 一遍,不需要锁内串行语义。
|
||||
writeAsinCacheOutsideLock(probe, taskId, brandOutcome.accepted());
|
||||
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
@@ -1830,6 +1839,41 @@ public class CollectDataService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 关键词采集链的缓存旁路写入:把"这个 (国家, ASIN) 的价格/品牌/配送已经开页看过"这件事
|
||||
* 记进跨任务缓存,供下一条采集链复用。
|
||||
*
|
||||
* <p>整段吞掉 RuntimeException:缓存写不进去的代价是"下次多开一次页",把已经抓到手、
|
||||
* 正要落库的 chunk 提交拖失败是更大的代价。
|
||||
*/
|
||||
private void writeAsinCacheOutsideLock(FileTaskEntity task, Long taskId,
|
||||
List<CollectDataResultRowVo> accepted) {
|
||||
if (accepted == null || accepted.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
asinResultCacheService.writeCollectDataRows(resolveCacheCountry(task), String.valueOf(taskId), accepted);
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[collect-data] ASIN 缓存写入失败,不影响本次提交 taskId={} err={}",
|
||||
taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 缓存键里的国家来自任务入参(Python 回传的结果行本身不带国家,见 CollectDataSubmitRowDto)。
|
||||
* 只收"这一个任务确定只采一个国家"的情形:多选国家时无法判断某一行属于哪个站点,
|
||||
* 宁可整批不写 —— 键少了国家就会把 DE 的价格当成 US 的结论发回去。
|
||||
*/
|
||||
private String resolveCacheCountry(FileTaskEntity task) {
|
||||
CollectDataFiltersDto filters = extractFiltersFromRequest(task);
|
||||
String single = filters.getCountryCode();
|
||||
if (single != null && !single.isBlank()) {
|
||||
return single;
|
||||
}
|
||||
List<String> codes = filters.getCountryCodes();
|
||||
return codes != null && codes.size() == 1 ? codes.get(0) : null;
|
||||
}
|
||||
|
||||
private CollectDataFiltersDto extractFiltersFromRequest(FileTaskEntity task) {
|
||||
if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) {
|
||||
return new CollectDataFiltersDto();
|
||||
|
||||
@@ -396,6 +396,16 @@ aiimage:
|
||||
brand-check-cache-capacity: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CACHE_CAPACITY:512}
|
||||
invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100}
|
||||
result-item-batch-size: ${AIIMAGE_COLLECT_DATA_RESULT_ITEM_BATCH_SIZE:100}
|
||||
# ASIN 采集结果跨任务缓存(详见 AsinResultCacheService)。新鲜度口径放在服务端,
|
||||
# 采集端版本不齐时老客户端只会"问不到就照常开页",不会各按各的理解复用到过期数据。
|
||||
asin-cache:
|
||||
# 页面正常且有价格:价格会变,只复用这么短时间内(分钟)
|
||||
priced-ttl-minutes: ${AIIMAGE_ASIN_CACHE_PRICED_TTL_MINUTES:60}
|
||||
# 详情页 404「此 ASIN 不存在」:基本永久,缓存最长(小时)
|
||||
not-found-ttl-hours: ${AIIMAGE_ASIN_CACHE_NOT_FOUND_TTL_HOURS:168}
|
||||
max-query-asins: ${AIIMAGE_ASIN_CACHE_MAX_QUERY_ASINS:500}
|
||||
max-write-rows: ${AIIMAGE_ASIN_CACHE_MAX_WRITE_ROWS:500}
|
||||
write-batch-size: ${AIIMAGE_ASIN_CACHE_WRITE_BATCH_SIZE:100}
|
||||
image-video:
|
||||
coze-base-url: ${AIIMAGE_IMAGE_VIDEO_COZE_BASE_URL:https://api.coze.cn}
|
||||
coze-token: ${AIIMAGE_IMAGE_VIDEO_COZE_TOKEN:}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
-- V139: ASIN 采集结果缓存表(跨任务/跨链复用的唯一事实源)
|
||||
--
|
||||
-- 为什么要这张表(2026-10-01 实测):同一个 (国家, ASIN) 的详情页被反复重采。
|
||||
-- · 任务内:变体采集单任务 d56024ff 写出 43,504 行,去重后只有 692 个不同 ASIN(重复 62.9 倍);
|
||||
-- · 跨任务:8 天自连接重复 67,169 行。
|
||||
-- 每一行都要开一次页、可能买一次代理 IP,而关键词采集与变体采集的结果分别落在
|
||||
-- biz_task_result_item 与麦象 MySQL 的 variant_collection 两处,互相看不见 ——
|
||||
-- 只有 Java 这侧能同时收到两条链的结论,所以缓存放这里、由这里判新鲜度。
|
||||
--
|
||||
-- 复用口径(服务端唯一判定,客户端不参与,见 AsinResultCacheService):
|
||||
-- · OK 页面正常且取到价格 —— 只复用很短时间内的(默认 60 分钟,价格会变)
|
||||
-- · NOT_FOUND 详情页 404「此 ASIN 不存在」—— 基本永久,复用最长(默认 7 天,省开页最多)
|
||||
-- · 其余(空价/拦截页/采集异常/不可售)一律不写本表:那些结论要么可能是我们
|
||||
-- 自己的解析/风控问题,要么换一次出口 IP 就能救回,缓存它等于把这个 ASIN
|
||||
-- 永久锁成空行。
|
||||
--
|
||||
-- collation 显式写死 utf8mb4_general_ci:库里不同表的默认排序规则不一致
|
||||
-- (biz_dedupe_total_data / biz_collect_data_item 是 general_ci,部分新表走库默认),
|
||||
-- 注解 SQL 里 (country, asin) 与入参比较一旦跨排序规则会抛 Illegal mix of collations,
|
||||
-- 这类错误只在运行期出现、编译期查不出来。
|
||||
--
|
||||
-- 回滚(手工):
|
||||
-- DROP TABLE IF EXISTS biz_asin_result_cache;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS biz_asin_result_cache (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT COMMENT 'primary key',
|
||||
country VARCHAR(16) NOT NULL COMMENT '国家代码(大写),与 ASIN 一起构成缓存键',
|
||||
asin VARCHAR(32) NOT NULL COMMENT '商品 ASIN',
|
||||
availability VARCHAR(16) NOT NULL COMMENT '可复用结论:OK / NOT_FOUND(其余取值不入库)',
|
||||
price VARCHAR(32) NULL COMMENT '价格原文(带当地币种符号,不做数值化以免丢信息)',
|
||||
brand VARCHAR(255) NULL COMMENT '品牌',
|
||||
delivery_method VARCHAR(32) NULL COMMENT '配送方式:FBA / FBM / AMZ / NA',
|
||||
seller_name VARCHAR(255) NULL COMMENT '卖家名称',
|
||||
seller_id VARCHAR(64) NULL COMMENT 'merchantID',
|
||||
declared_members_json TEXT NULL COMMENT '该 ASIN 页面自己声明的变体成员列表(JSON 数组);跳过开页时靠它补齐产出行,见 VariantCollectionHandler._remember_family',
|
||||
source_chain VARCHAR(32) NOT NULL COMMENT '结论来源链:COLLECT_DATA / VARIANT / DETAIL',
|
||||
source_task_id VARCHAR(64) NULL COMMENT '来源任务标识(Java 任务 id 或麦象 task_id,仅用于排查)',
|
||||
observed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '采集机上真正打开该页面的时间(新鲜度按它算,不是按入库时间)',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uk_asin_cache_country_asin (country, asin) COMMENT '一个 (国家, ASIN) 一行,重采即覆盖',
|
||||
KEY idx_asin_cache_observed (availability, observed_at) COMMENT '新鲜度判定与后续保留期清理'
|
||||
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = DYNAMIC
|
||||
COMMENT = 'ASIN 采集结果跨任务缓存(关键词链与变体链共用)';
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
package com.nanri.aiimage.modules.asincache.controller;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto;
|
||||
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
|
||||
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 内部缓存端点的鉴权与入参绑定。
|
||||
*
|
||||
* <p>/api/internal 前缀虽然已在 AdminApiGuardFilter 兜底名单里,controller 仍自校验一次令牌:
|
||||
* 配置漂移时这两个端点会直接暴露给匿名请求,而它们背后是全站的采集结论与 ASIN 明细。
|
||||
*/
|
||||
class InternalAsinCacheControllerTest {
|
||||
|
||||
private final AsinResultCacheService asinResultCacheService = mock(AsinResultCacheService.class);
|
||||
private final AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
|
||||
|
||||
private InternalAsinCacheController newController() {
|
||||
return new InternalAsinCacheController(asinResultCacheService, adminAuthSupport);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsQueryAndWriteWithoutTrustedInternalToken() {
|
||||
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(false);
|
||||
InternalAsinCacheController controller = newController();
|
||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
||||
|
||||
AsinCacheQueryDto query = new AsinCacheQueryDto();
|
||||
query.setCountry("DE");
|
||||
query.setAsins(List.of("B0ABC12345"));
|
||||
assertThatThrownBy(() -> controller.query(request, query))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("未授权");
|
||||
|
||||
AsinCacheWriteDto write = new AsinCacheWriteDto();
|
||||
assertThatThrownBy(() -> controller.write(request, write))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("未授权");
|
||||
|
||||
verifyNoCacheAccess();
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryWithoutBodyIsRejected() {
|
||||
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
|
||||
|
||||
assertThatThrownBy(() -> newController().query(mock(HttpServletRequest.class), null))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("缺少查询参数");
|
||||
verify(asinResultCacheService, never()).query(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryReturnsServiceResultAsSuccessEnvelope() {
|
||||
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
|
||||
AsinCacheQueryVo vo = new AsinCacheQueryVo();
|
||||
vo.setCountry("DE");
|
||||
when(asinResultCacheService.query("DE", List.of("B0ABC12345"))).thenReturn(vo);
|
||||
AsinCacheQueryDto body = new AsinCacheQueryDto();
|
||||
body.setCountry("DE");
|
||||
body.setAsins(List.of("B0ABC12345"));
|
||||
|
||||
ApiResponse<AsinCacheQueryVo> resp = newController().query(mock(HttpServletRequest.class), body);
|
||||
|
||||
assertThat(resp.isSuccess()).isTrue();
|
||||
assertThat(resp.getData().getHits()).isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用方是 Python 采集端:snake_case 是它的自然写法。这里按真实报文形状反序列化一次,
|
||||
* 免得字段改名后两端各自"看起来都对",线上才发现写入的行全是空值。
|
||||
*/
|
||||
@Test
|
||||
void writeAcceptsPythonShapedPayload() throws Exception {
|
||||
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
|
||||
when(asinResultCacheService.write(anyList())).thenReturn(new AsinCacheWriteVo());
|
||||
String pythonJson = """
|
||||
{"rows": [
|
||||
{"country": "UK", "ASIN": "B0ABC12345", "availability": "NOT_FOUND",
|
||||
"declared_members": ["B0ABC12346"], "source_chain": "VARIANT",
|
||||
"source_task_id": "28599", "brand": "Nike", "delivery_method": "FBA",
|
||||
"seller_name": "Nike DE", "seller_id": "A1PXL", "price": ""}
|
||||
]}""";
|
||||
AsinCacheWriteDto body = new ObjectMapper().readValue(pythonJson, AsinCacheWriteDto.class);
|
||||
|
||||
ApiResponse<AsinCacheWriteVo> resp = newController().write(mock(HttpServletRequest.class), body);
|
||||
|
||||
assertThat(resp.isSuccess()).isTrue();
|
||||
verify(asinResultCacheService).write(org.mockito.ArgumentMatchers.argThat(rows ->
|
||||
rows.size() == 1
|
||||
&& "UK".equals(rows.get(0).getCountry())
|
||||
&& "B0ABC12345".equals(rows.get(0).getAsin())
|
||||
&& "NOT_FOUND".equals(rows.get(0).getAvailability())
|
||||
&& List.of("B0ABC12346").equals(rows.get(0).getDeclaredMembers())
|
||||
&& "28599".equals(rows.get(0).getSourceTaskId())
|
||||
&& "Nike DE".equals(rows.get(0).getSellerName())
|
||||
&& "A1PXL".equals(rows.get(0).getSellerId())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void writeWithoutBodyReportsZeroInsteadOfThrowing() {
|
||||
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
|
||||
when(asinResultCacheService.write(eq(null))).thenReturn(new AsinCacheWriteVo());
|
||||
|
||||
ApiResponse<AsinCacheWriteVo> resp = newController().write(mock(HttpServletRequest.class), null);
|
||||
|
||||
assertThat(resp.getData().getWritten()).isZero();
|
||||
}
|
||||
|
||||
private void verifyNoCacheAccess() {
|
||||
verify(asinResultCacheService, never()).query(any(), any());
|
||||
verify(asinResultCacheService, never()).write(any());
|
||||
}
|
||||
}
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
package com.nanri.aiimage.modules.asincache.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.asincache.mapper.AsinResultCacheMapper;
|
||||
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteRowDto;
|
||||
import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
|
||||
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* ASIN 跨任务缓存的可复用口径(新鲜度分档 + 哪些结论根本不入库)。
|
||||
*
|
||||
* <p>这些数字是花钱的边界:404 复用 7 天、有价复用一个小时内,判宽了采到过期价格,
|
||||
* 判窄了白开页。所以连同"未知 availability 一律不复用"这种兜底方向一起固化。
|
||||
*/
|
||||
class AsinResultCacheServiceTest {
|
||||
|
||||
private static final int PRICED_TTL_MINUTES = 60;
|
||||
private static final int NOT_FOUND_TTL_HOURS = 168;
|
||||
private static final int MAX_QUERY_ASINS = 500;
|
||||
private static final int MAX_WRITE_ROWS = 500;
|
||||
private static final int WRITE_BATCH_SIZE = 100;
|
||||
|
||||
private final AsinResultCacheMapper mapper = mock(AsinResultCacheMapper.class);
|
||||
private final AsinResultCacheService service = new AsinResultCacheService(
|
||||
mapper, new ObjectMapper(),
|
||||
PRICED_TTL_MINUTES, NOT_FOUND_TTL_HOURS, MAX_QUERY_ASINS, MAX_WRITE_ROWS, WRITE_BATCH_SIZE);
|
||||
|
||||
// ==================== 新鲜度分档 ====================
|
||||
|
||||
@Test
|
||||
void pricedRowReusableWithinTtlOnly() {
|
||||
assertTrue(service.isReusable(row("OK", "9.90", minutesAgo(59)), LocalDateTime.now()),
|
||||
"60 分钟内的价格结论应可复用");
|
||||
assertFalse(service.isReusable(row("OK", "9.90", minutesAgo(61)), LocalDateTime.now()),
|
||||
"超过 60 分钟的价格结论不可复用");
|
||||
}
|
||||
|
||||
@Test
|
||||
void notFoundRowReusableForDays() {
|
||||
assertTrue(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * 24)), LocalDateTime.now()),
|
||||
"404 按天级复用,省开页最多");
|
||||
assertFalse(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * NOT_FOUND_TTL_HOURS + 1)),
|
||||
LocalDateTime.now()), "超过 not-found-ttl-hours 的 404 必须重新确认");
|
||||
}
|
||||
|
||||
/** 空价格 / 拦截页 / 采集异常这类结论即便被人写进库里,读取端也不能复用。 */
|
||||
@Test
|
||||
void unknownAvailabilityNeverReusable() {
|
||||
for (String availability : List.of("BLOCKED", "NO_PRICE", "ERROR", "", "ok")) {
|
||||
assertFalse(service.isReusable(row(availability, "9.90", minutesAgo(1)), LocalDateTime.now()),
|
||||
"未知 availability=" + availability + " 必须按不可复用处理(fail-closed)");
|
||||
}
|
||||
assertFalse(service.isReusable(row("OK", "", minutesAgo(1)), LocalDateTime.now()),
|
||||
"OK 但价格为空的历史脏行不可复用");
|
||||
assertFalse(service.isReusable(null, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
/** 缺 observed_at 时退回 created_at 判新鲜度,而不是直接判死。 */
|
||||
@Test
|
||||
void fallsBackToCreatedAtWhenObservedAtMissing() {
|
||||
AsinResultCacheEntity fresh = row("OK", "9.90", null);
|
||||
fresh.setCreatedAt(minutesAgo(1));
|
||||
assertTrue(service.isReusable(fresh, LocalDateTime.now()));
|
||||
|
||||
AsinResultCacheEntity stale = row("OK", "9.90", null);
|
||||
stale.setCreatedAt(minutesAgo(120));
|
||||
assertFalse(service.isReusable(stale, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
// ==================== 查询 ====================
|
||||
|
||||
@Test
|
||||
void queryNormalizesKeyAndReturnsOnlyReusableHits() {
|
||||
when(mapper.selectByCountryAndAsins(eq("DE"), anyList())).thenReturn(List.of(
|
||||
row("OK", "19.90", minutesAgo(2)),
|
||||
row("OK", "29.90", minutesAgo(720))));
|
||||
|
||||
AsinCacheQueryVo vo = service.query(" de ", Arrays.asList("b0abc12345", "B0ABC12345", "", null));
|
||||
|
||||
assertEquals("DE", vo.getCountry());
|
||||
assertEquals(1, vo.getHits().size(), "过期那行不该作为命中返回");
|
||||
assertEquals("B0ABC12345", vo.getHits().get(0).getAsin());
|
||||
assertEquals(List.of("B0ABC12345"), capturedAsins(1), "ASIN 去重且大写归一后才进 SQL");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryWithoutCountrySkipsDbEntirely() {
|
||||
AsinCacheQueryVo blank = service.query(" ", List.of("B0ABC12345"));
|
||||
assertTrue(blank.getHits().isEmpty());
|
||||
assertNull(blank.getCountry());
|
||||
verifyNoInteractions(mapper);
|
||||
}
|
||||
|
||||
/** 旁路查询失败对采集端就等于"没缓存":照常开页,不能把采集打成 500。 */
|
||||
@Test
|
||||
void queryFailureDegradesToMiss() {
|
||||
when(mapper.selectByCountryAndAsins(anyString(), anyList()))
|
||||
.thenThrow(new RuntimeException("cache table missing"));
|
||||
|
||||
AsinCacheQueryVo vo = service.query("DE", List.of("B0ABC12345"));
|
||||
|
||||
assertTrue(vo.getHits().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryOverCapReportsTruncatedAndSplitsIntoChunks() {
|
||||
when(mapper.selectByCountryAndAsins(anyString(), anyList())).thenReturn(List.of());
|
||||
List<String> asins = new ArrayList<>();
|
||||
for (int i = 0; i < MAX_QUERY_ASINS + 40; i++) {
|
||||
asins.add(String.format("B0%08d", i));
|
||||
}
|
||||
|
||||
AsinCacheQueryVo vo = service.query("DE", asins);
|
||||
|
||||
assertEquals(40, vo.getTruncated(), "超出 max-query-asins 的部分要如实上报");
|
||||
verify(mapper, times(5)).selectByCountryAndAsins(eq("DE"), anyList());
|
||||
}
|
||||
|
||||
// ==================== 写入 ====================
|
||||
|
||||
@Test
|
||||
void writeDropsRowsThatAreNotReusableConclusions() {
|
||||
List<AsinCacheWriteRowDto> rows = List.of(
|
||||
writeRow("OK", "9.90"),
|
||||
writeRow("NOT_FOUND", ""),
|
||||
writeRow("OK", ""), // 空价格
|
||||
writeRow("BLOCKED", "9.90"), // 拦截页
|
||||
writeRow("OK", "9.90", ""), // 无国家
|
||||
writeRow("OK", "9.90", null, "")); // 无 ASIN
|
||||
|
||||
AsinCacheWriteVo vo = service.write(rows);
|
||||
|
||||
assertEquals(4, vo.getDropped(), "只有 404 与有价两种结论可入库");
|
||||
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mapper).upsertBatch(captor.capture());
|
||||
assertEquals(List.of("OK", "NOT_FOUND"),
|
||||
captor.getValue().stream().map(AsinResultCacheEntity::getAvailability).toList());
|
||||
assertEquals("DE", captor.getValue().get(0).getCountry());
|
||||
}
|
||||
|
||||
/** 一个脏值的长度超列宽会让整批 INSERT 抛 Data too long,因此按列裁剪而不是整批丢。 */
|
||||
@Test
|
||||
void writeClipsOverlongValues() {
|
||||
AsinCacheWriteRowDto dto = writeRow("OK", "1".repeat(200));
|
||||
dto.setBrand("b".repeat(300));
|
||||
|
||||
service.write(List.of(dto));
|
||||
|
||||
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mapper).upsertBatch(captor.capture());
|
||||
assertEquals(32, captor.getValue().get(0).getPrice().length());
|
||||
assertEquals(255, captor.getValue().get(0).getBrand().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
void writeSplitsByBatchSizeAndSurvivesFailingBatch() {
|
||||
when(mapper.upsertBatch(anyList())).thenReturn(100).thenThrow(new RuntimeException("锁等待超时"));
|
||||
List<AsinCacheWriteRowDto> rows = new ArrayList<>();
|
||||
for (int i = 0; i < 150; i++) {
|
||||
rows.add(writeRow("NOT_FOUND", ""));
|
||||
}
|
||||
|
||||
AsinCacheWriteVo vo = service.write(rows);
|
||||
|
||||
verify(mapper, times(2)).upsertBatch(anyList());
|
||||
assertEquals(100, vo.getWritten(), "失败批不计入已写,但也不能让整个提交抛出去"); }
|
||||
|
||||
// ==================== 关键词链旁路 ====================
|
||||
|
||||
@Test
|
||||
void collectDataBypassSkipsWholeBatchWhenCountryUnknown() {
|
||||
int written = service.writeCollectDataRows(null, "6161", List.of(collectRow("B0ABC12345", "9.90")));
|
||||
|
||||
assertEquals(0, written);
|
||||
verifyNoInteractions(mapper);
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectDataBypassWritesOnlyPricedRows() {
|
||||
when(mapper.upsertBatch(anyList())).thenReturn(1);
|
||||
|
||||
service.writeCollectDataRows("de", "6161", List.of(
|
||||
collectRow("B0ABC12345", "9.90"),
|
||||
collectRow("B0ABC12346", " ")));
|
||||
|
||||
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mapper).upsertBatch(captor.capture());
|
||||
AsinResultCacheEntity saved = captor.getValue().get(0);
|
||||
assertEquals(1, captor.getValue().size(), "空价格行不入库");
|
||||
assertEquals("DE", saved.getCountry());
|
||||
assertEquals("OK", saved.getAvailability());
|
||||
assertEquals("COLLECT_DATA", saved.getSourceChain());
|
||||
assertEquals("6161", saved.getSourceTaskId());
|
||||
assertNotNull(saved.getObservedAt(), "缺 observed_at 会让新鲜度判定整体失效");
|
||||
}
|
||||
|
||||
private List<String> capturedAsins(int expectedSize) {
|
||||
ArgumentCaptor<List<String>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mapper).selectByCountryAndAsins(eq("DE"), captor.capture());
|
||||
assertEquals(expectedSize, captor.getValue().size());
|
||||
return captor.getValue();
|
||||
}
|
||||
|
||||
private static LocalDateTime minutesAgo(long minutes) {
|
||||
return LocalDateTime.now().minusMinutes(minutes);
|
||||
}
|
||||
|
||||
private static AsinResultCacheEntity row(String availability, String price, LocalDateTime observedAt) {
|
||||
AsinResultCacheEntity entity = new AsinResultCacheEntity();
|
||||
entity.setCountry("DE");
|
||||
entity.setAsin("B0ABC12345");
|
||||
entity.setAvailability(availability);
|
||||
entity.setPrice(price);
|
||||
entity.setObservedAt(observedAt);
|
||||
return entity;
|
||||
}
|
||||
|
||||
private static AsinCacheWriteRowDto writeRow(String availability, String price) {
|
||||
return writeRow(availability, price, "DE");
|
||||
}
|
||||
|
||||
private static AsinCacheWriteRowDto writeRow(String availability, String price, String country) {
|
||||
return writeRow(availability, price, country, "B0ABC12345");
|
||||
}
|
||||
|
||||
private static AsinCacheWriteRowDto writeRow(String availability, String price, String country, String asin) {
|
||||
AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto();
|
||||
dto.setCountry(country);
|
||||
dto.setAsin(asin);
|
||||
dto.setAvailability(availability);
|
||||
dto.setPrice(price);
|
||||
return dto;
|
||||
}
|
||||
|
||||
private static CollectDataResultRowVo collectRow(String asin, String price) {
|
||||
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||
row.setAsin(asin);
|
||||
row.setPrice(price);
|
||||
row.setBrand("Demo Brand");
|
||||
row.setDeliveryMethod("FBA");
|
||||
row.setSellerName("Demo Seller");
|
||||
return row;
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -145,7 +145,8 @@ class CollectDataActivateGuardTest {
|
||||
mock(CollectDataInvalidAsinBatchWriter.class), mock(CollectDataResultDetailCodec.class),
|
||||
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
+2
-1
@@ -118,7 +118,8 @@ class CollectDataDeleteConsistencyTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity task(long id, long userId) {
|
||||
|
||||
+47
@@ -57,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -96,6 +97,7 @@ class CollectDataServiceTxBoundaryTest {
|
||||
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
|
||||
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
|
||||
@Mock private CollectDataResultDetailCodec resultDetailCodec;
|
||||
@Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
|
||||
|
||||
@InjectMocks private CollectDataService service;
|
||||
|
||||
@@ -288,6 +290,51 @@ class CollectDataServiceTxBoundaryTest {
|
||||
.enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
|
||||
}
|
||||
|
||||
// ==================== ASIN 跨任务缓存旁路写入 ====================
|
||||
|
||||
/** 关键词链的 accepted 行顺带写进跨任务缓存;国家取自任务 request_json,而非结果行本身。 */
|
||||
@Test
|
||||
void acceptedRowsWrittenToAsinCacheWithTaskCountry() {
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithCountry("DE"));
|
||||
|
||||
service.submitResult(TASK_ID, submitRequest());
|
||||
|
||||
verify(asinResultCacheService).writeCollectDataRows(eq("DE"), eq("6161"), any());
|
||||
}
|
||||
|
||||
/** 多选国家的任务不写缓存:结果行本身不带国家,无法判断某一行属于哪个站点。 */
|
||||
@Test
|
||||
void multiCountryTaskWritesNoCache() {
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithRequestJson(
|
||||
"{\"filters\":{\"countryCodes\":[\"DE\",\"FR\"]}}"));
|
||||
|
||||
service.submitResult(TASK_ID, submitRequest());
|
||||
|
||||
verify(asinResultCacheService).writeCollectDataRows(isNull(), anyString(), any());
|
||||
}
|
||||
|
||||
/** 缓存写挂了不能把已经抓到手、正要落库的数据拖成提交失败 —— 旁路只记日志。 */
|
||||
@Test
|
||||
void asinCacheWriteFailureDoesNotAffectSubmit() {
|
||||
when(asinResultCacheService.writeCollectDataRows(any(), any(), any()))
|
||||
.thenThrow(new RuntimeException("缓存库不可用"));
|
||||
|
||||
CollectDataSubmitResultVo vo = service.submitResult(TASK_ID, submitRequest());
|
||||
|
||||
assertEquals(TASK_ID, vo.getTaskId());
|
||||
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity taskWithCountry(String countryCode) {
|
||||
return taskWithRequestJson("{\"filters\":{\"countryCode\":\"" + countryCode + "\"}}");
|
||||
}
|
||||
|
||||
private FileTaskEntity taskWithRequestJson(String requestJson) {
|
||||
FileTaskEntity task = runningTask();
|
||||
task.setRequestJson(requestJson);
|
||||
return task;
|
||||
}
|
||||
|
||||
private CollectDataSubmitResultRequest submitRequest() {
|
||||
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
|
||||
row.setAsin("B0COLLECT1");
|
||||
|
||||
+2
-1
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
|
||||
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
|
||||
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
|
||||
}
|
||||
|
||||
private FileResultEntity result(long id, long taskId, long userId) {
|
||||
|
||||
Reference in New Issue
Block a user