feat(ASIN缓存): 新增 ASIN 采集结果跨任务缓存(asincache 模块 + V139)

同一个 (国家, ASIN) 的详情页被反复重采:单任务内 62.9 倍重复(43,504 行只有 692 个
不同 ASIN)、8 天跨任务 6.7 万行重复(2026-10-01 实测),每条重复都意味着重开一次页、
可能重买一次代理 IP。关键词链与变体链的结果分别落在两处互相看不见,只有 Java 侧能同时
收到两条链的结论,所以缓存放服务端、由服务端判新鲜度。

- 新表 biz_asin_result_cache(V139):一个 (国家, ASIN) 一行,重采覆盖;只收两种可复用
  结论 —— OK(页面正常且有价格,复用 60 分钟)、NOT_FOUND(详情页 404,复用 7 天);
  空价 / 风控拦截页 / 采集异常一律不入库,避免把瞬时故障锁成永久结论
- 新鲜度只在服务端判(按 observed_at 真正开页时间);采集端拿不到 / 看不懂一律照常开页
  (fail-open),任何情况下不许让缓存导致漏行、错行
- 新增内部接口 /api/internal/asin-cache/query|write(X-Internal-Token 自校验,与
  InternalBrandTaskController 同口径),供采集端开页前先问、采完回写
- 关键词采集链在任务锁外旁路写入:缓存写失败的代价只是"下次多开一次页",只记 warn,
  不影响 chunk 提交
- migration-inventory.md 快照更新至 V139,「V137 冗余索引 DROP」待办顺延 V140

测试:AsinResultCacheServiceTest(13)/ InternalAsinCacheControllerTest(5)/ 采集链
4 个测试类(35)共 53 例全绿。
This commit is contained in:
2026-10-01 12:19:05 +08:00
parent 051eab6de5
commit 629fdfab79
20 changed files with 1312 additions and 8 deletions
@@ -0,0 +1,80 @@
package com.nanri.aiimage.modules.asincache.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* 内部接口:ASIN 采集结果缓存的读写,供采集端(麦象采集机 / 桌面端)在开页前先问一次
* 「这个 (国家, ASIN) 是否已经有人开过」。
*
* <p>鉴权与 {@code InternalBrandTaskController} 同口径:/api/internal 前缀已在
* AdminApiGuardFilter 兜底名单内,controller 仍自校验 X-Internal-Token —— 防止配置漂移时匿名可达。
*
* <p>这两个端点是<b>旁路</b>:采集端拿不到响应时必须照常开页(fail-open),所以
* 服务端任何异常都只可能让采集变慢,绝不可能让数据变少或变错。
*/
@RestController
@RequiredArgsConstructor
@Slf4j
@RequestMapping("/api/internal/asin-cache")
@Tag(name = "内部接口", description = "服务间调用(X-Internal-Token 鉴权)。")
public class InternalAsinCacheController {
private final AsinResultCacheService asinResultCacheService;
private final AdminAuthSupport adminAuthSupport;
@PostMapping("/query")
@Operation(summary = "批量查可复用的 ASIN 采集结论(采集端调用)",
description = "只返回新鲜度达标(OK 60 分钟 / NOT_FOUND 7 天,均可配置)的行;"
+ "未出现在 hits 里的 ASIN 按未命中处理,调用方照常开页。")
public ApiResponse<AsinCacheQueryVo> query(HttpServletRequest request,
@RequestBody(required = false) AsinCacheQueryDto body) {
requireTrustedToken(request, "query");
if (body == null) {
throw new BusinessException(400, "缺少查询参数");
}
AsinCacheQueryVo vo = asinResultCacheService.query(body.getCountry(), body.getAsins());
log.info("[internal-asin-cache] 查询 country={} 请求={} 命中={} 截断={}",
vo.getCountry(), body.getAsins() == null ? 0 : body.getAsins().size(),
vo.getHits().size(), vo.getTruncated());
return ApiResponse.success(vo);
}
@PostMapping("/write")
@Operation(summary = "批量写入 ASIN 采集结论(采集端调用)",
description = "不可复用的结论(空价/拦截页/采集异常)服务端逐行丢弃;单批写失败只记日志,不返回失败。")
public ApiResponse<AsinCacheWriteVo> write(HttpServletRequest request,
@RequestBody(required = false) AsinCacheWriteDto body) {
requireTrustedToken(request, "write");
AsinCacheWriteVo vo = asinResultCacheService.write(
body == null ? null : body.getRows());
log.info("[internal-asin-cache] 写入 请求={} 入库={} 丢弃={} 截断={}",
body == null || body.getRows() == null ? 0 : body.getRows().size(),
vo.getWritten(), vo.getDropped(), vo.getTruncated());
return ApiResponse.success(vo);
}
private void requireTrustedToken(HttpServletRequest request, String action) {
if (adminAuthSupport.isTrustedInternalToken(request)) {
return;
}
log.warn("[internal-asin-cache] 拒绝未携带可信内部令牌的请求 action={} remoteAddr={}",
action, request.getRemoteAddr());
throw new BusinessException(401, "未授权");
}
}
@@ -0,0 +1,73 @@
package com.nanri.aiimage.modules.asincache.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.util.List;
@Mapper
public interface AsinResultCacheMapper extends BaseMapper<AsinResultCacheEntity> {
/**
* 一次取回一个国家下这批 ASIN 的缓存行。
*
* <p>走唯一键 uk_asin_cache_country_asin (country, asin) 的前缀,所以必须是
* 「country = ? AND asin IN (...)」这种等值 + IN 的形状;把 country 放进 IN 里
* 成对匹配((country, asin) IN ((..),(..)))在 MySQL 上用不上索引。
*/
@Select("""
<script>
SELECT id, country, asin, availability, price, brand, delivery_method, seller_name,
seller_id, declared_members_json, source_chain, source_task_id,
observed_at, created_at, updated_at
FROM biz_asin_result_cache
WHERE country = #{country}
AND asin IN
<foreach collection="asins" item="asin" open="(" separator="," close=")">#{asin}</foreach>
</script>
""")
List<AsinResultCacheEntity> selectByCountryAndAsins(@Param("country") String country,
@Param("asins") List<String> asins);
/**
* 按唯一键 (country, asin) 批量 upsert:重采覆盖旧结论,不新增行。
*
* <p>两条采集链往同一行里写的字段并不一样多 —— 关键词链没有 seller_id 也没有变体
* 家族,变体链没有 seller 信息。这些"这次没采到"的列在实体里是 NULL(空白值入库前
* 已被 clip 成 NULL),所以它们用 COALESCE 继承旧值:新观测带了就覆盖,没带就保留,
* 否则一条变体记录会把关键词记录里的卖家名抹成空,下次谁都读不到。
*
* <p>price / availability / source_* / observed_at 不参与继承:它们是本行结论的主体,
* 404 就该把价格清空,"谁最后开的页"也必须如实反映最后一次开页。
*/
@Insert("""
<script>
INSERT INTO biz_asin_result_cache
(country, asin, availability, price, brand, delivery_method, seller_name, seller_id,
declared_members_json, source_chain, source_task_id, observed_at, created_at, updated_at)
VALUES
<foreach collection="rows" item="row" separator=",">
(#{row.country}, #{row.asin}, #{row.availability}, #{row.price}, #{row.brand},
#{row.deliveryMethod}, #{row.sellerName}, #{row.sellerId}, #{row.declaredMembersJson},
#{row.sourceChain}, #{row.sourceTaskId}, #{row.observedAt}, #{row.createdAt}, #{row.updatedAt})
</foreach>
ON DUPLICATE KEY UPDATE
availability = VALUES(availability),
price = VALUES(price),
brand = COALESCE(VALUES(brand), brand),
delivery_method = COALESCE(VALUES(delivery_method), delivery_method),
seller_name = COALESCE(VALUES(seller_name), seller_name),
seller_id = COALESCE(VALUES(seller_id), seller_id),
declared_members_json = COALESCE(VALUES(declared_members_json), declared_members_json),
source_chain = VALUES(source_chain),
source_task_id = VALUES(source_task_id),
observed_at = VALUES(observed_at),
updated_at = VALUES(updated_at)
</script>
""")
int upsertBatch(@Param("rows") List<AsinResultCacheEntity> rows);
}
@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.asincache.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "ASIN 缓存批量查询请求(内部接口)")
public class AsinCacheQueryDto {
@JsonAlias({"country_code", "countryCode"})
@Schema(description = "国家代码(大小写不敏感)", example = "UK")
private String country;
@Schema(description = "待查 ASIN 列表,超出上限的部分由服务端丢弃(按请求顺序保留前面那些)")
private List<String> asins = new ArrayList<>();
}
@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.asincache.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "ASIN 缓存批量写入请求(内部接口)")
public class AsinCacheWriteDto {
@JsonAlias({"rows", "items"})
@Schema(description = "待写入行;不可复用的结论(空价/拦截/异常)服务端会逐行丢弃")
private List<AsinCacheWriteRowDto> rows = new ArrayList<>();
}
@@ -0,0 +1,69 @@
package com.nanri.aiimage.modules.asincache.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.time.LocalDateTime;
import java.util.List;
/**
* ASIN 缓存写入行。
*
* <p>字段一律同时接受 camelCase 与 snake_case:调用方是 Python 采集端,snake_case 是它的
* 自然写法;而 Java 侧序列化出来的是 camelCase,两头都认就不用任何一侧迁就。
*/
@Data
@Schema(description = "ASIN 缓存写入行(内部接口)")
public class AsinCacheWriteRowDto {
@JsonAlias({"country_code", "countryCode"})
@Schema(description = "国家代码", example = "UK")
private String country;
@JsonAlias({"ASIN", "asin"})
@Schema(description = "商品 ASIN", example = "B0XXXXXXX")
private String asin;
@JsonAlias({"availability", "status"})
@Schema(description = "结论类型:OK(页面正常且有价格)/ NOT_FOUND(404 不存在)。"
+ "其余取值服务端一律不入库 —— 空价/拦截页/采集异常都可能是采集端自己的解析或风控问题",
example = "OK")
private String availability;
@Schema(description = "价格原文(保留当地币种符号,不数值化)", example = "£12.34")
private String price;
@JsonAlias({"品牌", "brand_name"})
@Schema(description = "品牌")
private String brand;
@JsonAlias({"delivery_method", "fulfillment"})
@Schema(description = "配送方式", example = "FBA")
private String deliveryMethod;
@JsonAlias({"seller_name", "卖家名称"})
@Schema(description = "卖家名称")
private String sellerName;
@JsonAlias({"seller_id", "sellerID", "merchant_id"})
@Schema(description = "merchantID")
private String sellerId;
@JsonAlias({"declared_members", "declaredMembers"})
@Schema(description = "该 ASIN 页面自己声明的变体成员列表;跨任务跳过开页时靠它补齐产出行")
private List<String> declaredMembers;
@JsonAlias({"source_chain", "sourceChain"})
@Schema(description = "来源采集链:COLLECT_DATA / VARIANT / DETAIL", example = "VARIANT")
private String sourceChain;
@JsonAlias({"source_task_id", "sourceTaskId", "task_id"})
@Schema(description = "来源任务标识,仅用于排查")
private String sourceTaskId;
@JsonAlias({"observed_at", "observedAt"})
@Schema(description = "采集机上真正打开该页面的时间;缺省时服务端按当前时间记账",
example = "2026-10-01T10:12:30")
private LocalDateTime observedAt;
}
@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.asincache.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
/**
* ASIN 采集结果缓存行(biz_asin_result_cache)。
*
* <p>一个 (country, asin) 一行,重采即覆盖;新鲜度按 {@code observedAt}(采集机上真正
* 打开页面的时间)算,不按入库时间 —— 否则客户端攒批延迟回传会把旧结论算成新结论。
*/
@Data
@TableName("biz_asin_result_cache")
public class AsinResultCacheEntity {
/** 可复用结论:页面正常且取到价格。 */
public static final String AVAILABILITY_OK = "OK";
/** 可复用结论:详情页 404,该 ASIN 不存在。 */
public static final String AVAILABILITY_NOT_FOUND = "NOT_FOUND";
@TableId(type = IdType.AUTO)
private Long id;
private String country;
private String asin;
private String availability;
private String price;
private String brand;
private String deliveryMethod;
private String sellerName;
private String sellerId;
private String declaredMembersJson;
private String sourceChain;
private String sourceTaskId;
private LocalDateTime observedAt;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.asincache.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.time.LocalDateTime;
import java.util.List;
@Data
@Schema(description = "可复用的 ASIN 缓存结论")
public class AsinCacheHitVo {
@Schema(description = "商品 ASIN")
private String asin;
@Schema(description = "结论类型:OK / NOT_FOUND")
private String availability;
@Schema(description = "价格原文;NOT_FOUND 时为空")
private String price;
@Schema(description = "品牌")
private String brand;
@Schema(description = "配送方式")
private String deliveryMethod;
@Schema(description = "卖家名称")
private String sellerName;
@Schema(description = "merchantID")
private String sellerId;
@Schema(description = "该 ASIN 页面声明过的变体成员列表(跳过开页时靠它补齐产出行)")
private List<String> declaredMembers;
@Schema(description = "结论产生时间,采集端可据此在日志里报出复用了多久以前的数据")
private LocalDateTime observedAt;
}
@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.asincache.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "ASIN 缓存批量查询结果")
public class AsinCacheQueryVo {
@Schema(description = "回显国家代码(大写)")
private String country;
@Schema(description = "命中的结论;未列出的 ASIN 就是没有可复用结论,调用方照常开页")
private List<AsinCacheHitVo> hits = new ArrayList<>();
@Schema(description = "因超出单次上限而被忽略的 ASIN 个数(这些一律按未命中处理)")
private int truncated;
}
@@ -0,0 +1,18 @@
package com.nanri.aiimage.modules.asincache.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "ASIN 缓存批量写入结果")
public class AsinCacheWriteVo {
@Schema(description = "实际入库的行数(含覆盖旧行)")
private int written;
@Schema(description = "被服务端丢弃的行数:结论不可复用、ASIN 非法、或超出单次上限")
private int dropped;
@Schema(description = "本次请求被丢弃的行数上限之外的部分,调用方无需重试(重试也一样被丢)")
private int truncated;
}
@@ -0,0 +1,374 @@
package com.nanri.aiimage.modules.asincache.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.asincache.mapper.AsinResultCacheMapper;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteRowDto;
import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheHitVo;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
/**
* ASIN 采集结果缓存:跨任务、跨采集链复用「同一个 (国家, ASIN) 详情页已经开过」这个事实。
*
* <p><b>新鲜度只在这里判</b>,采集端不参与。两条链的采集端版本不可能同时升级,
* 把口径放在服务端意味着老采集端拿到 404/未知字段只会当没缓存用(fail-open),
* 而不是各按各的理解复用到过期数据。
*
* <p>可复用只有两档(其余一律不入库,见 {@link #ttlMinutesOf}):
* <ul>
* <li>{@code NOT_FOUND} —— 详情页 404。ASIN 不会"复活",缓存最久(默认 7 天),省开页最多;</li>
* <li>{@code OK} —— 页面正常且取到价格。价格/配送会变,只复用很短一段时间(默认 60 分钟)。</li>
* </ul>
* 空价格、验证码/风控拦截页、采集异常都不写:这些结论要么是瞬时故障,要么是我们自己的
* 解析问题,缓存它等于把这个 ASIN 后面每次出现都锁成空行。
*/
@Slf4j
@Service
public class AsinResultCacheService {
public static final String CHAIN_COLLECT_DATA = "COLLECT_DATA";
/** 列宽截断上限:超长值会让整批 INSERT 抛 Data too long,一行的脏数据拖垮整个 chunk。 */
private static final int MAX_ASIN_LENGTH = 32;
private static final int MAX_COUNTRY_LENGTH = 16;
private static final int MAX_PRICE_LENGTH = 32;
private static final int MAX_BRAND_LENGTH = 255;
private static final int MAX_DELIVERY_LENGTH = 32;
private static final int MAX_SELLER_NAME_LENGTH = 255;
private static final int MAX_SELLER_ID_LENGTH = 64;
private static final int MAX_SOURCE_TASK_ID_LENGTH = 64;
private static final int MAX_DECLARED_MEMBERS = 500;
/** IN (...) 单条 SQL 的 ASIN 个数上限:查询与写入共用,避免超长 SQL 打爆解析器。 */
private static final int SQL_CHUNK_SIZE = 100;
private final AsinResultCacheMapper asinResultCacheMapper;
private final ObjectMapper objectMapper;
private final int pricedTtlMinutes;
private final int notFoundTtlMinutes;
private final int maxQueryAsins;
private final int maxWriteRows;
private final int writeBatchSize;
public AsinResultCacheService(AsinResultCacheMapper asinResultCacheMapper,
ObjectMapper objectMapper,
@Value("${aiimage.asin-cache.priced-ttl-minutes:60}") int pricedTtlMinutes,
@Value("${aiimage.asin-cache.not-found-ttl-hours:168}") int notFoundTtlHours,
@Value("${aiimage.asin-cache.max-query-asins:500}") int maxQueryAsins,
@Value("${aiimage.asin-cache.max-write-rows:500}") int maxWriteRows,
@Value("${aiimage.asin-cache.write-batch-size:100}") int writeBatchSize) {
this.asinResultCacheMapper = asinResultCacheMapper;
this.objectMapper = objectMapper;
this.pricedTtlMinutes = Math.max(0, pricedTtlMinutes);
this.notFoundTtlMinutes = Math.max(0, notFoundTtlHours) * 60;
this.maxQueryAsins = Math.max(1, maxQueryAsins);
this.maxWriteRows = Math.max(1, maxWriteRows);
this.writeBatchSize = Math.max(1, writeBatchSize);
}
/**
* 批量查可复用结论。未出现在返回值里的 ASIN 就是"没有可复用结论",调用方照常开页 ——
* 复用是省时间的旁路,任何情况下都不许让采集因为缓存而漏行。
*/
public AsinCacheQueryVo query(String country, Collection<String> asins) {
AsinCacheQueryVo vo = new AsinCacheQueryVo();
String normalizedCountry = normalizeCountry(country);
vo.setCountry(normalizedCountry);
if (normalizedCountry == null || asins == null || asins.isEmpty()) {
return vo;
}
List<String> distinct = new ArrayList<>(new LinkedHashSet<>(cleanAsins(asins)));
if (distinct.isEmpty()) {
return vo;
}
if (distinct.size() > maxQueryAsins) {
vo.setTruncated(distinct.size() - maxQueryAsins);
distinct = distinct.subList(0, maxQueryAsins);
}
LocalDateTime now = LocalDateTime.now();
List<AsinCacheHitVo> hits = new ArrayList<>();
for (int from = 0; from < distinct.size(); from += SQL_CHUNK_SIZE) {
List<String> batch = distinct.subList(from, Math.min(from + SQL_CHUNK_SIZE, distinct.size()));
List<AsinResultCacheEntity> rows;
try {
rows = asinResultCacheMapper.selectByCountryAndAsins(normalizedCountry, batch);
} catch (RuntimeException ex) {
// 查询失败与"没有缓存"对采集端是同一件事:照常开页。不能因为旁路挂了就 500。
log.warn("[asin-cache] 缓存查询失败,按未命中处理 country={} size={} err={}",
normalizedCountry, batch.size(), ex.getMessage());
continue;
}
for (AsinResultCacheEntity row : rows) {
if (isReusable(row, now)) {
hits.add(toHit(row));
}
}
}
vo.setHits(hits);
return vo;
}
/**
* 批量写入结论。不可复用的行在这里丢弃(而不是在采集端):采集端有版本差,
* 老客户端可能还在上报空价格 —— 服务端不收,表里就不会出现被误复用的脏结论。
*
* <p>单批失败只记日志、不抛:调用方是采集热路径与关键词结果提交链路,
* 缓存写不进去的代价是"下次多开一次页",把任务整体拖失败是更大的代价。
*/
public AsinCacheWriteVo write(List<AsinCacheWriteRowDto> rows) {
AsinCacheWriteVo vo = new AsinCacheWriteVo();
if (rows == null || rows.isEmpty()) {
return vo;
}
if (rows.size() > maxWriteRows) {
vo.setTruncated(rows.size() - maxWriteRows);
rows = rows.subList(0, maxWriteRows);
}
List<AsinResultCacheEntity> entities = new ArrayList<>(rows.size());
for (AsinCacheWriteRowDto row : rows) {
AsinResultCacheEntity entity = toEntity(row);
if (entity == null) {
vo.setDropped(vo.getDropped() + 1);
continue;
}
entities.add(entity);
}
if (entities.isEmpty()) {
return vo;
}
for (int from = 0; from < entities.size(); from += writeBatchSize) {
List<AsinResultCacheEntity> batch = entities.subList(from, Math.min(from + writeBatchSize, entities.size()));
try {
vo.setWritten(vo.getWritten() + asinResultCacheMapper.upsertBatch(batch));
} catch (RuntimeException ex) {
log.warn("[asin-cache] 缓存批量写入失败,跳过该批 size={} err={}", batch.size(), ex.getMessage());
}
}
return vo;
}
/**
* 关键词采集链的旁路写入:结果行落 biz_task_result_item 之后,把"这个 (国家, ASIN)
* 的价格/品牌/配送已经开页看过"这件事同步记进缓存,让变体采集下次不必重开。
*
* <p>country 缺失时整批不写 —— 缓存键少了国家,就可能把 DE 的价格当成 US 的结论发回去。
* 关键词链按 ASIN 逐个采详情页,一批 accepted 行同属一个任务、一个国家,所以这里
* 只按行计数,不逐行判国家。
*/
public int writeCollectDataRows(String country, String sourceTaskId,
List<CollectDataResultRowVo> rows) {
if (rows == null || rows.isEmpty()) {
return 0;
}
String normalizedCountry = normalizeCountry(country);
if (normalizedCountry == null) {
log.info("[asin-cache] 任务未带国家代码,跳过缓存写入(避免跨国家错用)rows={}", rows.size());
return 0;
}
List<AsinCacheWriteRowDto> payload = new ArrayList<>(rows.size());
for (CollectDataResultRowVo row : rows) {
if (row == null || isBlank(row.getPrice())) {
continue; // 空价格不可复用,见类注释
}
AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto();
dto.setCountry(normalizedCountry);
dto.setAsin(row.getAsin());
dto.setAvailability(AsinResultCacheEntity.AVAILABILITY_OK);
dto.setPrice(row.getPrice());
dto.setBrand(row.getBrand());
dto.setDeliveryMethod(row.getDeliveryMethod());
dto.setSellerName(row.getSellerName());
dto.setSourceChain(CHAIN_COLLECT_DATA);
dto.setSourceTaskId(sourceTaskId);
payload.add(dto);
}
if (payload.isEmpty()) {
return 0;
}
AsinCacheWriteVo written = write(payload);
// 关键词链是"顺带"贡献缓存的:写满/写坏都不该在业务日志里响,降到 debug 级别由排查时打开。
log.debug("[asin-cache] 关键词链旁路写入 taskId={} rows={} accepted={} dropped={} truncated={}",
sourceTaskId, payload.size(), written.getWritten(), written.getDropped(), written.getTruncated());
return written.getWritten();
}
// ==================== 口径 ====================
/** 该 availability 的可复用时长(分钟);返回 0 表示永不复用。未知取值一律按 0 处理。 */
int ttlMinutesOf(String availability) {
if (AsinResultCacheEntity.AVAILABILITY_OK.equals(availability)) {
return pricedTtlMinutes;
}
if (AsinResultCacheEntity.AVAILABILITY_NOT_FOUND.equals(availability)) {
return notFoundTtlMinutes;
}
return 0;
}
/** 新鲜度判定:按 observed_at(真正开页时间),没有才退到 created_at(入库时间)。 */
boolean isReusable(AsinResultCacheEntity row, LocalDateTime now) {
if (row == null || row.getAsin() == null) {
return false;
}
int ttlMinutes = ttlMinutesOf(row.getAvailability());
if (ttlMinutes <= 0) {
return false;
}
if (AsinResultCacheEntity.AVAILABILITY_OK.equals(row.getAvailability()) && isBlank(row.getPrice())) {
// 理论上写不进来(write 已挡),但库里万一有历史脏行,不能让它被当成有效价格复用
return false;
}
LocalDateTime observedAt = row.getObservedAt() != null ? row.getObservedAt() : row.getCreatedAt();
if (observedAt == null) {
return false;
}
return observedAt.plusMinutes(ttlMinutes).isAfter(now);
}
// ==================== 归一化 ====================
private AsinResultCacheEntity toEntity(AsinCacheWriteRowDto row) {
if (row == null) {
return null;
}
String country = normalizeCountry(row.getCountry());
String asin = normalizeAsin(row.getAsin());
if (country == null || asin == null) {
return null;
}
String availability = row.getAvailability() == null ? ""
: row.getAvailability().trim().toUpperCase(Locale.ROOT);
if (ttlMinutesOf(availability) <= 0) {
return null; // 不可复用的结论(含未知取值)不进表
}
if (AsinResultCacheEntity.AVAILABILITY_OK.equals(availability) && isBlank(row.getPrice())) {
return null;
}
AsinResultCacheEntity entity = new AsinResultCacheEntity();
entity.setCountry(country);
entity.setAsin(asin);
entity.setAvailability(availability);
entity.setPrice(clip(row.getPrice(), MAX_PRICE_LENGTH));
entity.setBrand(clip(row.getBrand(), MAX_BRAND_LENGTH));
entity.setDeliveryMethod(clip(row.getDeliveryMethod(), MAX_DELIVERY_LENGTH));
entity.setSellerName(clip(row.getSellerName(), MAX_SELLER_NAME_LENGTH));
entity.setSellerId(clip(row.getSellerId(), MAX_SELLER_ID_LENGTH));
entity.setDeclaredMembersJson(toDeclaredMembersJson(row.getDeclaredMembers()));
entity.setSourceChain(clip(row.getSourceChain(), 32));
entity.setSourceTaskId(clip(row.getSourceTaskId(), MAX_SOURCE_TASK_ID_LENGTH));
// 缺 observed_at 时按当前时间记账:等价于"刚采到",新鲜度判定不会因为客户端没传就整体失效
entity.setObservedAt(row.getObservedAt() != null ? row.getObservedAt() : LocalDateTime.now());
LocalDateTime now = LocalDateTime.now();
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
return entity;
}
private AsinCacheHitVo toHit(AsinResultCacheEntity row) {
AsinCacheHitVo hit = new AsinCacheHitVo();
hit.setAsin(row.getAsin());
hit.setAvailability(row.getAvailability());
hit.setPrice(row.getPrice());
hit.setBrand(row.getBrand());
hit.setDeliveryMethod(row.getDeliveryMethod());
hit.setSellerName(row.getSellerName());
hit.setSellerId(row.getSellerId());
hit.setDeclaredMembers(fromDeclaredMembersJson(row.getDeclaredMembersJson()));
hit.setObservedAt(row.getObservedAt() != null ? row.getObservedAt() : row.getCreatedAt());
return hit;
}
/** 国家代码是缓存键的一半:大写归一、空值即拒,绝不用空国家写行(会与其他国家撞键)。 */
private static String normalizeCountry(String country) {
String value = country == null ? "" : country.trim().toUpperCase(Locale.ROOT);
if (value.isEmpty()) {
return null;
}
return value.length() > MAX_COUNTRY_LENGTH ? null : value;
}
private static String normalizeAsin(String asin) {
String value = asin == null ? "" : asin.trim().toUpperCase(Locale.ROOT);
if (value.isEmpty() || value.length() > MAX_ASIN_LENGTH) {
return null;
}
return value;
}
private static List<String> cleanAsins(Collection<String> asins) {
List<String> out = new ArrayList<>(asins.size());
for (String asin : asins) {
String normalized = normalizeAsin(asin);
if (normalized != null) {
out.add(normalized);
}
}
return out;
}
private String toDeclaredMembersJson(List<String> members) {
if (members == null || members.isEmpty()) {
return null;
}
Collection<String> clean = new LinkedHashSet<>();
for (String member : members) {
String normalized = normalizeAsin(member);
if (normalized != null) {
clean.add(normalized);
}
if (clean.size() >= MAX_DECLARED_MEMBERS) {
break;
}
}
if (clean.isEmpty()) {
return null;
}
try {
return objectMapper.writeValueAsString(clean);
} catch (Exception ex) {
log.warn("[asin-cache] 变体成员列表序列化失败,按无成员入库 err={}", ex.getMessage());
return null;
}
}
private List<String> fromDeclaredMembersJson(String json) {
if (isBlank(json)) {
return List.of();
}
try {
List<String> members = objectMapper.readValue(json,
objectMapper.getTypeFactory().constructCollectionType(List.class, String.class));
return members == null ? List.of() : members;
} catch (Exception ex) {
log.warn("[asin-cache] 变体成员列表解析失败,按无成员返回 json={} err={}",
json.length() > 120 ? json.substring(0, 120) + "..." : json, ex.getMessage());
return List.of();
}
}
private static String clip(String value, int maxLength) {
if (value == null) {
return null;
}
String trimmed = value.trim();
if (trimmed.isEmpty()) {
return null;
}
return trimmed.length() > maxLength ? trimmed.substring(0, maxLength) : trimmed;
}
private static boolean isBlank(String value) {
return value == null || value.isBlank();
}
}
@@ -12,6 +12,7 @@ import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
@@ -182,6 +183,9 @@ public class CollectDataService {
private final TaskProgressLightAssembler taskProgressLightAssembler;
private final NotificationDispatchService notificationDispatchService;
/** ASIN 跨任务结果缓存:本链只负责"顺带贡献结论",写入失败绝不影响 chunk 提交。 */
private final AsinResultCacheService asinResultCacheService;
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
@@ -830,6 +834,11 @@ public class CollectDataService {
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
// ASIN 结果缓存旁路写入同样放锁外:这把锁的持有时间是心跳与其他 chunk 的排队预算
// (见上方品牌检测移出的注释),缓存写入再快也不该来挤。缓存是"下次少开一次页"的
// 加速件,同一 chunk 重复提交只是把同样的结论再 upsert 一遍,不需要锁内串行语义。
writeAsinCacheOutsideLock(probe, taskId, brandOutcome.accepted());
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
@@ -1830,6 +1839,41 @@ public class CollectDataService {
return vo;
}
/**
* 关键词采集链的缓存旁路写入:把"这个 (国家, ASIN) 的价格/品牌/配送已经开页看过"这件事
* 记进跨任务缓存,供下一条采集链复用。
*
* <p>整段吞掉 RuntimeException:缓存写不进去的代价是"下次多开一次页",把已经抓到手、
* 正要落库的 chunk 提交拖失败是更大的代价。
*/
private void writeAsinCacheOutsideLock(FileTaskEntity task, Long taskId,
List<CollectDataResultRowVo> accepted) {
if (accepted == null || accepted.isEmpty()) {
return;
}
try {
asinResultCacheService.writeCollectDataRows(resolveCacheCountry(task), String.valueOf(taskId), accepted);
} catch (RuntimeException ex) {
log.warn("[collect-data] ASIN 缓存写入失败,不影响本次提交 taskId={} err={}",
taskId, ex.getMessage());
}
}
/**
* 缓存键里的国家来自任务入参(Python 回传的结果行本身不带国家,见 CollectDataSubmitRowDto)。
* 只收"这一个任务确定只采一个国家"的情形:多选国家时无法判断某一行属于哪个站点,
* 宁可整批不写 —— 键少了国家就会把 DE 的价格当成 US 的结论发回去。
*/
private String resolveCacheCountry(FileTaskEntity task) {
CollectDataFiltersDto filters = extractFiltersFromRequest(task);
String single = filters.getCountryCode();
if (single != null && !single.isBlank()) {
return single;
}
List<String> codes = filters.getCountryCodes();
return codes != null && codes.size() == 1 ? codes.get(0) : null;
}
private CollectDataFiltersDto extractFiltersFromRequest(FileTaskEntity task) {
if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) {
return new CollectDataFiltersDto();
@@ -396,6 +396,16 @@ aiimage:
brand-check-cache-capacity: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CACHE_CAPACITY:512}
invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100}
result-item-batch-size: ${AIIMAGE_COLLECT_DATA_RESULT_ITEM_BATCH_SIZE:100}
# ASIN 采集结果跨任务缓存(详见 AsinResultCacheService)。新鲜度口径放在服务端,
# 采集端版本不齐时老客户端只会"问不到就照常开页",不会各按各的理解复用到过期数据。
asin-cache:
# 页面正常且有价格:价格会变,只复用这么短时间内(分钟)
priced-ttl-minutes: ${AIIMAGE_ASIN_CACHE_PRICED_TTL_MINUTES:60}
# 详情页 404「此 ASIN 不存在」:基本永久,缓存最长(小时)
not-found-ttl-hours: ${AIIMAGE_ASIN_CACHE_NOT_FOUND_TTL_HOURS:168}
max-query-asins: ${AIIMAGE_ASIN_CACHE_MAX_QUERY_ASINS:500}
max-write-rows: ${AIIMAGE_ASIN_CACHE_MAX_WRITE_ROWS:500}
write-batch-size: ${AIIMAGE_ASIN_CACHE_WRITE_BATCH_SIZE:100}
image-video:
coze-base-url: ${AIIMAGE_IMAGE_VIDEO_COZE_BASE_URL:https://api.coze.cn}
coze-token: ${AIIMAGE_IMAGE_VIDEO_COZE_TOKEN:}
@@ -0,0 +1,45 @@
-- V139: ASIN 采集结果缓存表(跨任务/跨链复用的唯一事实源)
--
-- 为什么要这张表(2026-10-01 实测):同一个 (国家, ASIN) 的详情页被反复重采。
-- · 任务内:变体采集单任务 d56024ff 写出 43,504 行,去重后只有 692 个不同 ASIN(重复 62.9 倍);
-- · 跨任务:8 天自连接重复 67,169 行。
-- 每一行都要开一次页、可能买一次代理 IP,而关键词采集与变体采集的结果分别落在
-- biz_task_result_item 与麦象 MySQL 的 variant_collection 两处,互相看不见 ——
-- 只有 Java 这侧能同时收到两条链的结论,所以缓存放这里、由这里判新鲜度。
--
-- 复用口径(服务端唯一判定,客户端不参与,见 AsinResultCacheService):
-- · OK 页面正常且取到价格 —— 只复用很短时间内的(默认 60 分钟,价格会变)
-- · NOT_FOUND 详情页 404「此 ASIN 不存在」—— 基本永久,复用最长(默认 7 天,省开页最多)
-- · 其余(空价/拦截页/采集异常/不可售)一律不写本表:那些结论要么可能是我们
-- 自己的解析/风控问题,要么换一次出口 IP 就能救回,缓存它等于把这个 ASIN
-- 永久锁成空行。
--
-- collation 显式写死 utf8mb4_general_ci:库里不同表的默认排序规则不一致
-- (biz_dedupe_total_data / biz_collect_data_item 是 general_ci,部分新表走库默认),
-- 注解 SQL 里 (country, asin) 与入参比较一旦跨排序规则会抛 Illegal mix of collations,
-- 这类错误只在运行期出现、编译期查不出来。
--
-- 回滚(手工):
-- DROP TABLE IF EXISTS biz_asin_result_cache;
CREATE TABLE IF NOT EXISTS biz_asin_result_cache (
id BIGINT NOT NULL AUTO_INCREMENT COMMENT 'primary key',
country VARCHAR(16) NOT NULL COMMENT '国家代码(大写),与 ASIN 一起构成缓存键',
asin VARCHAR(32) NOT NULL COMMENT '商品 ASIN',
availability VARCHAR(16) NOT NULL COMMENT '可复用结论:OK / NOT_FOUND(其余取值不入库)',
price VARCHAR(32) NULL COMMENT '价格原文(带当地币种符号,不做数值化以免丢信息)',
brand VARCHAR(255) NULL COMMENT '品牌',
delivery_method VARCHAR(32) NULL COMMENT '配送方式:FBA / FBM / AMZ / NA',
seller_name VARCHAR(255) NULL COMMENT '卖家名称',
seller_id VARCHAR(64) NULL COMMENT 'merchantID',
declared_members_json TEXT NULL COMMENT '该 ASIN 页面自己声明的变体成员列表(JSON 数组);跳过开页时靠它补齐产出行,见 VariantCollectionHandler._remember_family',
source_chain VARCHAR(32) NOT NULL COMMENT '结论来源链:COLLECT_DATA / VARIANT / DETAIL',
source_task_id VARCHAR(64) NULL COMMENT '来源任务标识(Java 任务 id 或麦象 task_id,仅用于排查)',
observed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '采集机上真正打开该页面的时间(新鲜度按它算,不是按入库时间)',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
PRIMARY KEY (id),
UNIQUE KEY uk_asin_cache_country_asin (country, asin) COMMENT '一个 (国家, ASIN) 一行,重采即覆盖',
KEY idx_asin_cache_observed (availability, observed_at) COMMENT '新鲜度判定与后续保留期清理'
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = DYNAMIC
COMMENT = 'ASIN 采集结果跨任务缓存(关键词链与变体链共用)';
@@ -0,0 +1,134 @@
package com.nanri.aiimage.modules.asincache.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheQueryDto;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteDto;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 内部缓存端点的鉴权与入参绑定。
*
* <p>/api/internal 前缀虽然已在 AdminApiGuardFilter 兜底名单里,controller 仍自校验一次令牌:
* 配置漂移时这两个端点会直接暴露给匿名请求,而它们背后是全站的采集结论与 ASIN 明细。
*/
class InternalAsinCacheControllerTest {
private final AsinResultCacheService asinResultCacheService = mock(AsinResultCacheService.class);
private final AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
private InternalAsinCacheController newController() {
return new InternalAsinCacheController(asinResultCacheService, adminAuthSupport);
}
@Test
void rejectsQueryAndWriteWithoutTrustedInternalToken() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(false);
InternalAsinCacheController controller = newController();
HttpServletRequest request = mock(HttpServletRequest.class);
AsinCacheQueryDto query = new AsinCacheQueryDto();
query.setCountry("DE");
query.setAsins(List.of("B0ABC12345"));
assertThatThrownBy(() -> controller.query(request, query))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("未授权");
AsinCacheWriteDto write = new AsinCacheWriteDto();
assertThatThrownBy(() -> controller.write(request, write))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("未授权");
verifyNoCacheAccess();
}
@Test
void queryWithoutBodyIsRejected() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
assertThatThrownBy(() -> newController().query(mock(HttpServletRequest.class), null))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("缺少查询参数");
verify(asinResultCacheService, never()).query(any(), any());
}
@Test
void queryReturnsServiceResultAsSuccessEnvelope() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
AsinCacheQueryVo vo = new AsinCacheQueryVo();
vo.setCountry("DE");
when(asinResultCacheService.query("DE", List.of("B0ABC12345"))).thenReturn(vo);
AsinCacheQueryDto body = new AsinCacheQueryDto();
body.setCountry("DE");
body.setAsins(List.of("B0ABC12345"));
ApiResponse<AsinCacheQueryVo> resp = newController().query(mock(HttpServletRequest.class), body);
assertThat(resp.isSuccess()).isTrue();
assertThat(resp.getData().getHits()).isEmpty();
}
/**
* 调用方是 Python 采集端:snake_case 是它的自然写法。这里按真实报文形状反序列化一次,
* 免得字段改名后两端各自"看起来都对",线上才发现写入的行全是空值。
*/
@Test
void writeAcceptsPythonShapedPayload() throws Exception {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
when(asinResultCacheService.write(anyList())).thenReturn(new AsinCacheWriteVo());
String pythonJson = """
{"rows": [
{"country": "UK", "ASIN": "B0ABC12345", "availability": "NOT_FOUND",
"declared_members": ["B0ABC12346"], "source_chain": "VARIANT",
"source_task_id": "28599", "brand": "Nike", "delivery_method": "FBA",
"seller_name": "Nike DE", "seller_id": "A1PXL", "price": ""}
]}""";
AsinCacheWriteDto body = new ObjectMapper().readValue(pythonJson, AsinCacheWriteDto.class);
ApiResponse<AsinCacheWriteVo> resp = newController().write(mock(HttpServletRequest.class), body);
assertThat(resp.isSuccess()).isTrue();
verify(asinResultCacheService).write(org.mockito.ArgumentMatchers.argThat(rows ->
rows.size() == 1
&& "UK".equals(rows.get(0).getCountry())
&& "B0ABC12345".equals(rows.get(0).getAsin())
&& "NOT_FOUND".equals(rows.get(0).getAvailability())
&& List.of("B0ABC12346").equals(rows.get(0).getDeclaredMembers())
&& "28599".equals(rows.get(0).getSourceTaskId())
&& "Nike DE".equals(rows.get(0).getSellerName())
&& "A1PXL".equals(rows.get(0).getSellerId())));
}
@Test
void writeWithoutBodyReportsZeroInsteadOfThrowing() {
when(adminAuthSupport.isTrustedInternalToken(any())).thenReturn(true);
when(asinResultCacheService.write(eq(null))).thenReturn(new AsinCacheWriteVo());
ApiResponse<AsinCacheWriteVo> resp = newController().write(mock(HttpServletRequest.class), null);
assertThat(resp.getData().getWritten()).isZero();
}
private void verifyNoCacheAccess() {
verify(asinResultCacheService, never()).query(any(), any());
verify(asinResultCacheService, never()).write(any());
}
}
@@ -0,0 +1,267 @@
package com.nanri.aiimage.modules.asincache.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.asincache.mapper.AsinResultCacheMapper;
import com.nanri.aiimage.modules.asincache.model.dto.AsinCacheWriteRowDto;
import com.nanri.aiimage.modules.asincache.model.entity.AsinResultCacheEntity;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheQueryVo;
import com.nanri.aiimage.modules.asincache.model.vo.AsinCacheWriteVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
/**
* ASIN 跨任务缓存的可复用口径(新鲜度分档 + 哪些结论根本不入库)。
*
* <p>这些数字是花钱的边界:404 复用 7 天、有价复用一个小时内,判宽了采到过期价格,
* 判窄了白开页。所以连同"未知 availability 一律不复用"这种兜底方向一起固化。
*/
class AsinResultCacheServiceTest {
private static final int PRICED_TTL_MINUTES = 60;
private static final int NOT_FOUND_TTL_HOURS = 168;
private static final int MAX_QUERY_ASINS = 500;
private static final int MAX_WRITE_ROWS = 500;
private static final int WRITE_BATCH_SIZE = 100;
private final AsinResultCacheMapper mapper = mock(AsinResultCacheMapper.class);
private final AsinResultCacheService service = new AsinResultCacheService(
mapper, new ObjectMapper(),
PRICED_TTL_MINUTES, NOT_FOUND_TTL_HOURS, MAX_QUERY_ASINS, MAX_WRITE_ROWS, WRITE_BATCH_SIZE);
// ==================== 新鲜度分档 ====================
@Test
void pricedRowReusableWithinTtlOnly() {
assertTrue(service.isReusable(row("OK", "9.90", minutesAgo(59)), LocalDateTime.now()),
"60 分钟内的价格结论应可复用");
assertFalse(service.isReusable(row("OK", "9.90", minutesAgo(61)), LocalDateTime.now()),
"超过 60 分钟的价格结论不可复用");
}
@Test
void notFoundRowReusableForDays() {
assertTrue(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * 24)), LocalDateTime.now()),
"404 按天级复用,省开页最多");
assertFalse(service.isReusable(row("NOT_FOUND", null, minutesAgo(60 * NOT_FOUND_TTL_HOURS + 1)),
LocalDateTime.now()), "超过 not-found-ttl-hours 的 404 必须重新确认");
}
/** 空价格 / 拦截页 / 采集异常这类结论即便被人写进库里,读取端也不能复用。 */
@Test
void unknownAvailabilityNeverReusable() {
for (String availability : List.of("BLOCKED", "NO_PRICE", "ERROR", "", "ok")) {
assertFalse(service.isReusable(row(availability, "9.90", minutesAgo(1)), LocalDateTime.now()),
"未知 availability=" + availability + " 必须按不可复用处理(fail-closed)");
}
assertFalse(service.isReusable(row("OK", "", minutesAgo(1)), LocalDateTime.now()),
"OK 但价格为空的历史脏行不可复用");
assertFalse(service.isReusable(null, LocalDateTime.now()));
}
/** 缺 observed_at 时退回 created_at 判新鲜度,而不是直接判死。 */
@Test
void fallsBackToCreatedAtWhenObservedAtMissing() {
AsinResultCacheEntity fresh = row("OK", "9.90", null);
fresh.setCreatedAt(minutesAgo(1));
assertTrue(service.isReusable(fresh, LocalDateTime.now()));
AsinResultCacheEntity stale = row("OK", "9.90", null);
stale.setCreatedAt(minutesAgo(120));
assertFalse(service.isReusable(stale, LocalDateTime.now()));
}
// ==================== 查询 ====================
@Test
void queryNormalizesKeyAndReturnsOnlyReusableHits() {
when(mapper.selectByCountryAndAsins(eq("DE"), anyList())).thenReturn(List.of(
row("OK", "19.90", minutesAgo(2)),
row("OK", "29.90", minutesAgo(720))));
AsinCacheQueryVo vo = service.query(" de ", Arrays.asList("b0abc12345", "B0ABC12345", "", null));
assertEquals("DE", vo.getCountry());
assertEquals(1, vo.getHits().size(), "过期那行不该作为命中返回");
assertEquals("B0ABC12345", vo.getHits().get(0).getAsin());
assertEquals(List.of("B0ABC12345"), capturedAsins(1), "ASIN 去重且大写归一后才进 SQL");
}
@Test
void queryWithoutCountrySkipsDbEntirely() {
AsinCacheQueryVo blank = service.query(" ", List.of("B0ABC12345"));
assertTrue(blank.getHits().isEmpty());
assertNull(blank.getCountry());
verifyNoInteractions(mapper);
}
/** 旁路查询失败对采集端就等于"没缓存":照常开页,不能把采集打成 500。 */
@Test
void queryFailureDegradesToMiss() {
when(mapper.selectByCountryAndAsins(anyString(), anyList()))
.thenThrow(new RuntimeException("cache table missing"));
AsinCacheQueryVo vo = service.query("DE", List.of("B0ABC12345"));
assertTrue(vo.getHits().isEmpty());
}
@Test
void queryOverCapReportsTruncatedAndSplitsIntoChunks() {
when(mapper.selectByCountryAndAsins(anyString(), anyList())).thenReturn(List.of());
List<String> asins = new ArrayList<>();
for (int i = 0; i < MAX_QUERY_ASINS + 40; i++) {
asins.add(String.format("B0%08d", i));
}
AsinCacheQueryVo vo = service.query("DE", asins);
assertEquals(40, vo.getTruncated(), "超出 max-query-asins 的部分要如实上报");
verify(mapper, times(5)).selectByCountryAndAsins(eq("DE"), anyList());
}
// ==================== 写入 ====================
@Test
void writeDropsRowsThatAreNotReusableConclusions() {
List<AsinCacheWriteRowDto> rows = List.of(
writeRow("OK", "9.90"),
writeRow("NOT_FOUND", ""),
writeRow("OK", ""), // 空价格
writeRow("BLOCKED", "9.90"), // 拦截页
writeRow("OK", "9.90", ""), // 无国家
writeRow("OK", "9.90", null, "")); // 无 ASIN
AsinCacheWriteVo vo = service.write(rows);
assertEquals(4, vo.getDropped(), "只有 404 与有价两种结论可入库");
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).upsertBatch(captor.capture());
assertEquals(List.of("OK", "NOT_FOUND"),
captor.getValue().stream().map(AsinResultCacheEntity::getAvailability).toList());
assertEquals("DE", captor.getValue().get(0).getCountry());
}
/** 一个脏值的长度超列宽会让整批 INSERT 抛 Data too long,因此按列裁剪而不是整批丢。 */
@Test
void writeClipsOverlongValues() {
AsinCacheWriteRowDto dto = writeRow("OK", "1".repeat(200));
dto.setBrand("b".repeat(300));
service.write(List.of(dto));
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).upsertBatch(captor.capture());
assertEquals(32, captor.getValue().get(0).getPrice().length());
assertEquals(255, captor.getValue().get(0).getBrand().length());
}
@Test
void writeSplitsByBatchSizeAndSurvivesFailingBatch() {
when(mapper.upsertBatch(anyList())).thenReturn(100).thenThrow(new RuntimeException("锁等待超时"));
List<AsinCacheWriteRowDto> rows = new ArrayList<>();
for (int i = 0; i < 150; i++) {
rows.add(writeRow("NOT_FOUND", ""));
}
AsinCacheWriteVo vo = service.write(rows);
verify(mapper, times(2)).upsertBatch(anyList());
assertEquals(100, vo.getWritten(), "失败批不计入已写,但也不能让整个提交抛出去"); }
// ==================== 关键词链旁路 ====================
@Test
void collectDataBypassSkipsWholeBatchWhenCountryUnknown() {
int written = service.writeCollectDataRows(null, "6161", List.of(collectRow("B0ABC12345", "9.90")));
assertEquals(0, written);
verifyNoInteractions(mapper);
}
@Test
void collectDataBypassWritesOnlyPricedRows() {
when(mapper.upsertBatch(anyList())).thenReturn(1);
service.writeCollectDataRows("de", "6161", List.of(
collectRow("B0ABC12345", "9.90"),
collectRow("B0ABC12346", " ")));
ArgumentCaptor<List<AsinResultCacheEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).upsertBatch(captor.capture());
AsinResultCacheEntity saved = captor.getValue().get(0);
assertEquals(1, captor.getValue().size(), "空价格行不入库");
assertEquals("DE", saved.getCountry());
assertEquals("OK", saved.getAvailability());
assertEquals("COLLECT_DATA", saved.getSourceChain());
assertEquals("6161", saved.getSourceTaskId());
assertNotNull(saved.getObservedAt(), "缺 observed_at 会让新鲜度判定整体失效");
}
private List<String> capturedAsins(int expectedSize) {
ArgumentCaptor<List<String>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper).selectByCountryAndAsins(eq("DE"), captor.capture());
assertEquals(expectedSize, captor.getValue().size());
return captor.getValue();
}
private static LocalDateTime minutesAgo(long minutes) {
return LocalDateTime.now().minusMinutes(minutes);
}
private static AsinResultCacheEntity row(String availability, String price, LocalDateTime observedAt) {
AsinResultCacheEntity entity = new AsinResultCacheEntity();
entity.setCountry("DE");
entity.setAsin("B0ABC12345");
entity.setAvailability(availability);
entity.setPrice(price);
entity.setObservedAt(observedAt);
return entity;
}
private static AsinCacheWriteRowDto writeRow(String availability, String price) {
return writeRow(availability, price, "DE");
}
private static AsinCacheWriteRowDto writeRow(String availability, String price, String country) {
return writeRow(availability, price, country, "B0ABC12345");
}
private static AsinCacheWriteRowDto writeRow(String availability, String price, String country, String asin) {
AsinCacheWriteRowDto dto = new AsinCacheWriteRowDto();
dto.setCountry(country);
dto.setAsin(asin);
dto.setAvailability(availability);
dto.setPrice(price);
return dto;
}
private static CollectDataResultRowVo collectRow(String asin, String price) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
row.setPrice(price);
row.setBrand("Demo Brand");
row.setDeliveryMethod("FBA");
row.setSellerName("Demo Seller");
return row;
}
}
@@ -145,7 +145,8 @@ class CollectDataActivateGuardTest {
mock(CollectDataInvalidAsinBatchWriter.class), mock(CollectDataResultDetailCodec.class),
mock(CollectDataResultItemBatchWriter.class), mock(CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
@SuppressWarnings("unchecked")
@@ -118,7 +118,8 @@ class CollectDataDeleteConsistencyTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileTaskEntity task(long id, long userId) {
@@ -57,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -96,6 +97,7 @@ class CollectDataServiceTxBoundaryTest {
@Mock private CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
@Mock private CollectDataResultItemBatchWriter resultItemBatchWriter;
@Mock private CollectDataResultDetailCodec resultDetailCodec;
@Mock private com.nanri.aiimage.modules.asincache.service.AsinResultCacheService asinResultCacheService;
@InjectMocks private CollectDataService service;
@@ -288,6 +290,51 @@ class CollectDataServiceTxBoundaryTest {
.enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
}
// ==================== ASIN 跨任务缓存旁路写入 ====================
/** 关键词链的 accepted 行顺带写进跨任务缓存;国家取自任务 request_json,而非结果行本身。 */
@Test
void acceptedRowsWrittenToAsinCacheWithTaskCountry() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithCountry("DE"));
service.submitResult(TASK_ID, submitRequest());
verify(asinResultCacheService).writeCollectDataRows(eq("DE"), eq("6161"), any());
}
/** 多选国家的任务不写缓存:结果行本身不带国家,无法判断某一行属于哪个站点。 */
@Test
void multiCountryTaskWritesNoCache() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithRequestJson(
"{\"filters\":{\"countryCodes\":[\"DE\",\"FR\"]}}"));
service.submitResult(TASK_ID, submitRequest());
verify(asinResultCacheService).writeCollectDataRows(isNull(), anyString(), any());
}
/** 缓存写挂了不能把已经抓到手、正要落库的数据拖成提交失败 —— 旁路只记日志。 */
@Test
void asinCacheWriteFailureDoesNotAffectSubmit() {
when(asinResultCacheService.writeCollectDataRows(any(), any(), any()))
.thenThrow(new RuntimeException("缓存库不可用"));
CollectDataSubmitResultVo vo = service.submitResult(TASK_ID, submitRequest());
assertEquals(TASK_ID, vo.getTaskId());
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
}
private FileTaskEntity taskWithCountry(String countryCode) {
return taskWithRequestJson("{\"filters\":{\"countryCode\":\"" + countryCode + "\"}}");
}
private FileTaskEntity taskWithRequestJson(String requestJson) {
FileTaskEntity task = runningTask();
task.setRequestJson(requestJson);
return task;
}
private CollectDataSubmitResultRequest submitRequest() {
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
row.setAsin("B0COLLECT1");
@@ -117,7 +117,8 @@ class CollectDataStorageCallCountTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class),
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class));
mock(com.nanri.aiimage.modules.notification.service.NotificationDispatchService.class),
mock(com.nanri.aiimage.modules.asincache.service.AsinResultCacheService.class));
}
private FileResultEntity result(long id, long taskId, long userId) {