Compare commits

...

2 Commits

Author SHA1 Message Date
huangzd1997 5f4fcad2ef 修复双实例 owner-scoped 结果文件 job 被非归属实例误杀
Build Backend JAR / build (push) Has been cancelled
线上现象:货源查询任务 26402/26400(归属 server-121)在归属机正常推进,
却被 server-110 判为「结果生成失败」,错误为「该任务已绑定到另一台服务实例处理」。

两个缺陷叠加:
1. 两台实例共用一个 RocketMQ consumer group,dispatch 消息被轮询投递;
   worker 对"PENDING + owner 不匹配"的 job 选择本地抢跑,模块层
   ensureTaskOwnedByCurrentInstance 直接抛 TaskOwnerMismatchException,
   异常落进通用 catch 被当成生成失败,5 次抢跑即耗尽 retryCount 并终态失败。
2. runPendingJobs 双重 claim:claimRunnableJobsForOwner 已把 job 翻成 RUNNING,
   随后又走 claimRunning(条件 PENDING/FAILED)必然失败 → job 被静默丢弃,
   归属机的定时兜底从未真正生效。

修复:
- worker:非归属实例一律跳过 owner-scoped job;抽出 processClaimed 供已持有
  claim 的调用方使用;processInternal 单独捕获 TaskOwnerMismatchException,
  只 deferRunning 退回 PENDING,不计入重试。
- TaskFileJobLocalDispatcher 新增 dispatchClaimed。
- dispatch 事件带上 owner,coordinator 按归属路由:本机 owner 直接本地派发不发 MQ,
  他机 owner 不发不跑交由归属机轮询接手,owner 为空维持原有 MQ 行为。
- 新增 owner 路由回归测试两组,回滚配合"本地抢跑"的旧断言。

同时提交此前工作区内已随 JAR 上线的 backend-java 改动:SimilarAsin 解析载荷
groups 内嵌 items(Python 旧链路兼容)与 chunk 类型不匹配跳过、
PermissionMenuSchemaInitializer 与 V103 两级菜单分组迁移、相关测试。
2026-09-01 21:07:25 +08:00
huangzd1997 55517f996b task-120: 大页数据性能回归 + 恢复 similarasin 测试入库
性能回归(SimilarAsinTaskServicePerf500Test,8/8 绿):
- test_perf_light_200 原以墙钟毫秒比较 light 与 batch(assertTrue
  lightMs < batchMs)。两条路径都跑纯内存 stub,耗时在十几毫秒量级,
  JIT/GC/线程调度抖动远大于真实差值,本机实测 light 14ms / batch 13ms
  直接翻红。改为确定性口径:light 结果明细查询数 == 0(batch ≥ 1)、
  light 总查询数 < batch 总查询数、batch 条目携带任务明细对象而 light
  只有白名单字段;墙钟仅保留单侧上界 1s(N+1 退化会放大到秒级从而失败)。
  连续 3 次独立运行稳定通过。
- 类注释同步移除“已知 flaky,隔离重跑即接受”的说明。

恢复测试入库:
- fa5a59e(task-116) 把 similarasin 测试目录整体从 git 删除,并加了
  .gitignore 规则 *.broken/,磁盘上目录改名 similarasin.broken。
  但包声明仍是 com.nanri.aiimage.modules.similarasin.*,javac 照常编译,
  这 40 个测试文件一直在参与构建、并且是 mvn test 的门禁的一部分——
  等于脱离版本控制却仍在 gate 构建。
- 目录改回 similarasin/(现在与包声明一致),移除 *.broken/ 忽略规则,
  40 个测试文件重新入库。mvn test-compile 退出码 0。
2026-09-01 13:45:34 +08:00
56 changed files with 11146 additions and 45 deletions
-1
View File
@@ -118,7 +118,6 @@ backend-java/docs/plans/
backend-java/docs/*plan*.md
backend-java/docs/specs/
backend-java/docs/*audit*.md
*.broken/
# ===== 进度/规划工具(本地使用,不入库)=====
check_progress.py
@@ -47,20 +47,34 @@ public class PermissionMenuSchemaInitializer {
new DefaultAppChildMenu("ERP", "erp", "erp", 132, "brand_logistics_tools")
);
/**
* 后台一级分组:只做权限树层级与「上级菜单」候选,不映射真实页面。
* route_path 使用 group- 前缀虚拟值以满足 (menu_type, route_path) 唯一约束;
* 前端按 column_key 前缀 admin_group_ 识别分组行,不会出现在页面选择器中。
*/
private static final List<DefaultAdminGroup> DEFAULT_ADMIN_GROUPS = List.of(
new DefaultAdminGroup("账号与权限", "admin_group_account", "group-account", 5),
new DefaultAdminGroup("数据管理", "admin_group_data", "group-data", 15),
new DefaultAdminGroup("店铺管理", "admin_group_shop", "group-shop", 45),
new DefaultAdminGroup("记录与版本", "admin_group_record", "group-record", 55)
);
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
new DefaultAdminMenu("用户管理", "admin_users", "users", 10),
new DefaultAdminMenu("菜单权限配置", "admin_columns", "columns", 20),
new DefaultAdminMenu("分组管理", "admin_group_manage", "group-manage", 25),
new DefaultAdminMenu("去重数据汇总", "admin_dedupe_total_data", "dedupe-total-data", 30),
new DefaultAdminMenu("无效ASIN数据", "admin_invalid_asin_data", "invalid-asin-data", 35),
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40),
new DefaultAdminMenu("店铺管理", "admin_shop_manage", "shop-manage", 50),
new DefaultAdminMenu("跳过跟价ASIN", "admin_skip_price_asin", "skip-price-asin", 60),
new DefaultAdminMenu("查询ASIN", "admin_query_asin", "query-asin", 65),
new DefaultAdminMenu("商品类目", "admin_product_categories", "product-categories", 66),
new DefaultAdminMenu("生成记录", "admin_history", "history", 70),
new DefaultAdminMenu("软件版本管理", "admin_version", "version", 80),
new DefaultAdminMenu("数字人版本管理", "digital_human_version", "digital-human-version", 81)
new DefaultAdminMenu("用户管理", "admin_users", "users", 10, "admin_group_account"),
new DefaultAdminMenu("菜单权限配置", "admin_columns", "columns", 20, "admin_group_account"),
new DefaultAdminMenu("分组管理", "admin_group_manage", "group-manage", 25, "admin_group_account"),
new DefaultAdminMenu("去重数据汇总", "admin_dedupe_total_data", "dedupe-total-data", 30, "admin_group_data"),
new DefaultAdminMenu("品牌数据", "admin_invalid_asin_data", "invalid-asin-data", 35, "admin_group_data"),
new DefaultAdminMenu("查询ASIN", "admin_query_asin", "query-asin", 65, "admin_group_data"),
new DefaultAdminMenu("商品类目", "admin_product_categories", "product-categories", 66, "admin_group_data"),
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40, "admin_group_shop"),
new DefaultAdminMenu("店铺管理", "admin_shop_manage", "shop-manage", 50, "admin_group_shop"),
new DefaultAdminMenu("最低价ASIN设置", "admin_skip_price_asin", "skip-price-asin", 60, "admin_group_shop"),
new DefaultAdminMenu("店铺数据记录", "admin_shop_data_crawl_tasks", "shop-data-crawl-tasks", 82, "admin_group_shop"),
new DefaultAdminMenu("生成记录", "admin_history", "history", 70, "admin_group_record"),
new DefaultAdminMenu("视频任务记录", "admin_image_video_tasks", "image-video-tasks", 75, "admin_group_record"),
new DefaultAdminMenu("软件版本管理", "admin_version", "version", 80, "admin_group_record"),
new DefaultAdminMenu("数字人版本管理", "digital_human_version", "digital-human-version", 81, "admin_group_record")
);
@EventListener(ApplicationReadyEvent.class)
@@ -86,6 +100,7 @@ public class PermissionMenuSchemaInitializer {
executeQuietly("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''");
executeQuietly("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0");
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
ensureDefaultAdminGroups();
ensureDefaultAdminMenus();
boolean shopDataCrawlDataPermissionCreated = ensureShopDataCrawlAdminDefaults();
ensureDefaultAppMenus();
@@ -105,20 +120,37 @@ public class PermissionMenuSchemaInitializer {
}
}
private void ensureDefaultAdminMenus() {
for (DefaultAdminMenu menu : DEFAULT_ADMIN_MENUS) {
private void ensureDefaultAdminGroups() {
for (DefaultAdminGroup group : DEFAULT_ADMIN_GROUPS) {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '%s', '%s', 'admin', '%s', %d
INSERT INTO columns (name, column_key, menu_type, route_path, parent_id, sort_order)
SELECT '%s', '%s', 'admin', '%s', NULL, %d
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = '%s'
)
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
""".formatted(group.name(), group.columnKey(), group.routePath(), group.sortOrder(), group.columnKey()));
}
}
private void ensureDefaultAdminMenus() {
for (DefaultAdminMenu menu : DEFAULT_ADMIN_MENUS) {
executeQuietly("""
UPDATE columns
SET sort_order = %d, parent_id = NULL
WHERE column_key = '%s' AND (sort_order IS NULL OR sort_order = 0)
""".formatted(menu.sortOrder(), menu.columnKey()));
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order, parent_id)
SELECT '%s', '%s', 'admin', '%s', %d, parent.id
FROM columns parent
WHERE parent.column_key = '%s'
AND NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = '%s'
)
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(),
menu.parentKey(), menu.columnKey()));
executeQuietly("""
UPDATE columns child
JOIN columns parent ON parent.column_key = '%s'
SET child.name = '%s', child.sort_order = %d,
child.parent_id = COALESCE(child.parent_id, parent.id)
WHERE child.column_key = '%s'
""".formatted(menu.parentKey(), menu.name(), menu.sortOrder(), menu.columnKey()));
}
}
@@ -179,8 +211,9 @@ public class PermissionMenuSchemaInitializer {
private boolean ensureShopDataCrawlAdminDefaults() {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '店铺数据记录', 'admin_shop_data_crawl_tasks', 'admin', 'shop-data-crawl-tasks', 82
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order, parent_id)
SELECT '店铺数据记录', 'admin_shop_data_crawl_tasks', 'admin', 'shop-data-crawl-tasks', 82,
(SELECT id FROM columns WHERE column_key = 'admin_group_shop' LIMIT 1)
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = 'admin_shop_data_crawl_tasks'
)
@@ -226,7 +259,11 @@ public class PermissionMenuSchemaInitializer {
}
}
private record DefaultAdminMenu(String name, String columnKey, String routePath, int sortOrder) {
private record DefaultAdminMenu(String name, String columnKey, String routePath, int sortOrder,
String parentKey) {
}
private record DefaultAdminGroup(String name, String columnKey, String routePath, int sortOrder) {
}
private record DefaultAppMenu(String name, String columnKey, String routePath, int sortOrder) {
@@ -5127,10 +5127,47 @@ public class SimilarAsinTaskService {
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
payload.setGroups(groups == null ? List.of() : groups);
// 兼容 Python 旧链路groups 需内嵌完整行Python normalize_groups
// 直接返回 groups 并按 gp["items"] 取行仅有 startIndex/endIndex 引用时
// items 为空任务空转startIndex/endIndex 保留Java 内部 expandGroupRefs 继续使用
payload.setGroups(fillGroupItems(groups, allRows));
return writeJson(payload, "保存解析结果失败");
}
/**
* group.startIndex/endIndex 半开区间从全量行切片内嵌回 group.items
* 无索引或切片越界时退回按 group.items 原样兼容旧 payload 反序列化场景
*/
static List<SimilarAsinParsedGroupVo> fillGroupItems(List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
if (groups == null || groups.isEmpty()) {
return groups == null ? List.of() : groups;
}
if (allRows == null || allRows.isEmpty()) {
return groups;
}
List<SimilarAsinParsedRowVo> rows = new ArrayList<>(allRows);
for (SimilarAsinParsedGroupVo group : groups) {
if (group == null) {
continue;
}
Integer start = group.getStartIndex();
Integer end = group.getEndIndex();
if (start != null && end != null) {
int from = Math.max(0, Math.min(start, rows.size()));
int to = Math.max(from, Math.min(end, rows.size()));
if (to > from) {
group.setItems(new ArrayList<>(rows.subList(from, to)));
continue;
}
}
// 索引缺失/无效保留已有 items兼容旧 payload不覆盖
if (group.getItems() == null) {
group.setItems(new ArrayList<>());
}
}
return groups;
}
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("aiPrompt", normalize(aiPrompt));
@@ -5172,6 +5209,11 @@ public class SimilarAsinTaskService {
}
private SimilarAsinParsedPayloadDto hydrateParsedPayloadRows(SimilarAsinParsedPayloadDto payload) {
return hydrateForPythonCompat(payload);
}
/** 解析载荷 hydrate:回填 allItems/items/groups 内嵌 itemsPython 旧链路兼容)。 */
static SimilarAsinParsedPayloadDto hydrateForPythonCompat(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return new SimilarAsinParsedPayloadDto();
}
@@ -5182,6 +5224,13 @@ public class SimilarAsinTaskService {
if (payload.getItems() == null || payload.getItems().isEmpty()) {
payload.setItems(rows);
}
// Python 旧链路兼容读取时也回填 groups 内嵌 items存量任务 payload
// 只有引用否则 Python normalize_groups 拿到空 items 任务空转
if (payload.getGroups() != null && !payload.getGroups().isEmpty()) {
List<SimilarAsinParsedRowVo> filledRows = payload.getItems() == null || payload.getItems().isEmpty()
? rows : payload.getItems();
payload.setGroups(fillGroupItems(payload.getGroups(), filledRows));
}
return payload;
}
@@ -5205,6 +5254,16 @@ public class SimilarAsinTaskService {
// P1-4识别跨实例 local 指针读不到的场景"chunk 在另一实例"的元信息
// 通过 task_scope_state.last_error 留痕便于排查"为何 owner 切换后 chunk 读不到"
boolean crossInstance = msg.contains("only exists on instance=");
// 类型不兼容如历史版本序列化数据含当前类加载器不认识的类型标记
// chunk 永久不可读跳过而不是把组装/收尾永久拖死留痕便于排查
boolean typeMismatch = ex instanceof TypeNotPresentException
|| msg.contains("not present");
if (typeMismatch) {
log.warn("[similar-asin] chunk payload type mismatch, skip chunk taskId={} chunk={} err={}",
chunk.getTaskId(), chunk.getChunkIndex(), msg);
recordChunkReadFailure(chunk, false, msg);
return rows;
}
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
recordChunkReadFailure(chunk, crossInstance, msg);
@@ -9,6 +9,7 @@ public record TaskFileJobDispatchEvent(
Long resultId,
String scopeKey,
String jobType,
String owner,
LocalDateTime createdAt
) {
}
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
@@ -17,6 +18,7 @@ public class TaskFileJobDispatchCoordinator {
private final TaskFileJobPublisher taskFileJobPublisher;
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
private final InstanceMetadata instanceMetadata;
@Qualifier("taskFileJobDispatchExecutor")
private final TaskExecutor taskFileJobDispatchExecutor;
@@ -43,6 +45,28 @@ public class TaskFileJobDispatchCoordinator {
}
private void dispatchAfterCommit(TaskFileJobDispatchEvent event, TaskFileJobMessage message) {
// owner 非空 ⇔ owner-scoped job(只有 SIMILAR_ASIN/APPEARANCE_PATENT/PUBLISH/
// SHOP_DATA_CRAWL 会写 "task:{id}:owner:{instance}" 形式的 scopeKey)。
// 这类 job 必须在归属实例上执行,走共享 consumer group 的 MQ 会被轮询投递到
// 另一台,那边只能跳过,白白多绕一圈。
String owner = event.owner() == null ? "" : event.owner().trim();
if (!owner.isBlank()) {
if (owner.equals(currentInstanceId())) {
boolean dispatched = taskFileJobLocalDispatcher.dispatch(
event.jobId(), event.taskId(), event.moduleType(), true);
if (dispatched) {
return;
}
log.warn("[task-file-job] owner-scoped local dispatch declined, fallback to mq jobId={} taskId={} moduleType={} owner={}",
event.jobId(), event.taskId(), event.moduleType(), owner);
} else {
// 归属实例每 15 秒扫一次自己 owner 的 PENDING/FAILED job,这里不发 MQ、
// 不本地跑,避免消息落到非归属实例后被丢弃。
log.info("[task-file-job] skip dispatch because job belongs to another instance jobId={} taskId={} moduleType={} owner={} current={}",
event.jobId(), event.taskId(), event.moduleType(), owner, currentInstanceId());
return;
}
}
boolean published = taskFileJobPublisher.publish(message);
if (published) {
return;
@@ -56,4 +80,9 @@ public class TaskFileJobDispatchCoordinator {
event.jobId(), event.taskId(), event.moduleType());
}
}
private String currentInstanceId() {
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
return instanceId == null || instanceId.isBlank() ? "unknown-instance" : instanceId;
}
}
@@ -42,7 +42,19 @@ public class TaskFileJobLocalDispatcher {
return dispatch(jobId, taskId, moduleType, false);
}
/**
* 调用方已把 job claim 成 RUNNING 时使用:复用同样的背压/去重/异步执行,
* 但落到 worker.processClaimed,避免二次 claimRunning 失败导致 job 被静默丢弃。
*/
public boolean dispatchClaimed(Long jobId, Long taskId, String moduleType) {
return submit(jobId, taskId, moduleType, false, true);
}
public boolean dispatch(Long jobId, Long taskId, String moduleType, boolean force) {
return submit(jobId, taskId, moduleType, force, false);
}
private boolean submit(Long jobId, Long taskId, String moduleType, boolean force, boolean alreadyClaimed) {
if (jobId == null || jobId <= 0) {
return false;
}
@@ -63,7 +75,7 @@ public class TaskFileJobLocalDispatcher {
return true;
}
try {
taskFileJobDispatchExecutor.execute(() -> processLocally(jobId, taskId, moduleType));
taskFileJobDispatchExecutor.execute(() -> processLocally(jobId, taskId, moduleType, alreadyClaimed));
return true;
} catch (RuntimeException ex) {
inflightJobIds.remove(jobId);
@@ -71,7 +83,7 @@ public class TaskFileJobLocalDispatcher {
jobId, taskId, moduleType, ex.getMessage(), ex);
if (force) {
try {
processLocally(jobId, taskId, moduleType);
processLocally(jobId, taskId, moduleType, alreadyClaimed);
return true;
} finally {
inflightJobIds.remove(jobId);
@@ -81,7 +93,7 @@ public class TaskFileJobLocalDispatcher {
}
}
private void processLocally(Long jobId, Long taskId, String moduleType) {
private void processLocally(Long jobId, Long taskId, String moduleType, boolean alreadyClaimed) {
try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
if (job == null) {
@@ -95,7 +107,11 @@ public class TaskFileJobLocalDispatcher {
jobId, taskId, moduleType);
return;
}
taskResultFileJobWorker.process(job);
if (alreadyClaimed) {
taskResultFileJobWorker.processClaimed(job);
} else {
taskResultFileJobWorker.process(job);
}
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
@@ -57,7 +57,14 @@ public class TaskFileJobService {
if (existing == null) {
return null;
}
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus()) || "PENDING".equals(existing.getStatus())) {
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
return existing;
}
if ("PENDING".equals(existing.getStatus())) {
// PENDING 僵尸兜底:job 入队时若 MQ 发送失败且本地兜底未生效(或消息被
// 非 owner 实例静默吞掉),job 会永远停留在 PENDING、无人处理。
// 补发一次 dispatch 事件,让调度链重试(幂等:consumer 侧 claim 有状态翻转保护)。
publishDispatchEvent(existing);
return existing;
}
// 终态失败:retryCount 已达上限的 FAILED job 不再重置为 PENDING
@@ -237,6 +244,13 @@ public class TaskFileJobService {
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
.or()
.isNull(TaskFileJobEntity::getUpdatedAt)))
.or(stuckPending -> stuckPending
.eq(TaskFileJobEntity::getStatus, "PENDING")
.ne(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.and(age -> age
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
.or()
.isNull(TaskFileJobEntity::getUpdatedAt)))
.or(exhausted -> exhausted
.eq(TaskFileJobEntity::getStatus, "FAILED")
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
@@ -251,6 +265,24 @@ public class TaskFileJobService {
exhaustedJobs.add(job);
continue;
}
if ("PENDING".equals(job.getStatus())) {
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),
// 并累加 retryCount 防无限重发;注意不翻转状态,
// 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1);
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "PENDING")
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
}
continue;
}
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
LocalDateTime now = LocalDateTime.now();
@@ -642,6 +674,7 @@ public class TaskFileJobService {
entity.getResultId(),
entity.getScopeKey(),
entity.getJobType(),
entity.getOwner(),
LocalDateTime.now()
));
}
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
@@ -76,13 +77,16 @@ public class TaskResultFileJobWorker {
}
log.info("[task-file-job] scheduled worker claimed jobs count={}", jobs.size());
for (TaskFileJobEntity job : jobs) {
boolean dispatched = taskFileJobLocalDispatcher.dispatch(
// claimRunnableJobsForOwner 已把 job 原子翻转成 RUNNING,这里必须走"已 claim"
// 通道:若再走 process() 的 claimRunning,条件 UPDATE 会因状态已是 RUNNING 而
// 失败,job 被静默丢弃并挂到 stuck 扫描才回收,定时兜底等于失效。
boolean dispatched = taskFileJobLocalDispatcher.dispatchClaimed(
job.getId(),
job.getTaskId(),
job.getModuleType()
);
if (!dispatched) {
process(job);
processClaimed(job);
}
}
}
@@ -104,18 +108,39 @@ public class TaskResultFileJobWorker {
}
public void process(TaskFileJobEntity job) {
if (isOwnerScopedJob(job) && !isOwnedByCurrentInstance(job)) {
log.debug("[task-file-job] skip owner-scoped job because owner is another instance jobId={} taskId={} moduleType={} owner={} current={}",
job.getId(), job.getTaskId(), job.getModuleType(), ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
if (job == null || job.getId() == null) {
return;
}
if (job == null || job.getId() == null) {
if (isOwnerScopedJob(job) && !isOwnedByCurrentInstance(job)) {
// 非归属实例一律不碰 owner-scoped job:模块层 ensureTaskOwnedByCurrentInstance
// 会直接抛 TaskOwnerMismatchException,抢跑只会白白消耗 job 的重试次数,
// 重试耗尽后把归属机上跑得好好的任务判成失败。
// job 不会因此卡死:归属实例的 runPendingJobs 每 15 秒扫一次自己 owner 的
// PENDING/FAILED jobenqueueAssembleResult / stuck 扫描也会补发 dispatch。
log.info("[task-file-job] skip owner-scoped job because owner is another instance jobId={} taskId={} moduleType={} status={} owner={} current={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getStatus(),
ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
return;
}
TaskFileJobEntity claim = taskFileJobService.claimRunning(job.getId());
if (claim == null) {
return;
}
offloadOrRun(job, claim);
}
/**
* 供"调用方已持有 RUNNING claim"的路径使用(定时 worker 的 claimRunnableJobsForOwner):
* 跳过 claimRunning,直接按 handler 能力异步/内联执行。
*/
public void processClaimed(TaskFileJobEntity claim) {
if (claim == null || claim.getId() == null) {
return;
}
offloadOrRun(claim, claim);
}
private void offloadOrRun(TaskFileJobEntity job, TaskFileJobEntity claim) {
ResultFileJobHandler handler = handlerRegistry.asMap().get(job.getModuleType());
if (handler != null && handler.supportsAsyncOffload()) {
try {
@@ -206,6 +231,12 @@ public class TaskResultFileJobWorker {
if (finalizeWithdraw) {
withdrawTaskService.tryFinalizeTask(job.getTaskId(), false);
}
} catch (TaskOwnerMismatchException ex) {
// 归属校验失败不是"生成失败":不能计入 retryCount,否则任务会在归属机
// 一切正常的情况下被本机重试耗尽判死。退回 PENDING 交还归属实例。
log.warn("[task-file-job] process rejected because task belongs to another instance jobId={} taskId={} moduleType={} owner={} current={}",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId());
taskFileJobService.deferRunning(job.getId(), ex.getMessage());
} catch (Exception ex) {
TaskFileJobEntity latest = taskFileJobService.findById(job.getId());
if (latest != null && "SUCCESS".equals(latest.getStatus())) {
@@ -0,0 +1,63 @@
-- V103: 后台管理菜单改为两级结构(一级分组 + 子页面)
-- 新增 4 个后台一级分组(不映射真实页面,route_path 为分组虚拟值),
-- 并把这 16 个后台页面菜单挂到对应分组下。
-- 分组只用于权限树层级与「上级菜单」候选;column_key / route_path 校验逻辑不变。
-- 对应 Java 端清单见 PermissionMenuSchemaInitializer.DEFAULT_ADMIN_GROUPS / DEFAULT_ADMIN_MENUS。
-- ---------- 1. 新增一级分组 ----------
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `parent_id`, `sort_order`)
SELECT '账号与权限', 'admin_group_account', 'admin', 'group-account', NULL, 5
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'admin_group_account');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `parent_id`, `sort_order`)
SELECT '数据管理', 'admin_group_data', 'admin', 'group-data', NULL, 15
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'admin_group_data');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `parent_id`, `sort_order`)
SELECT '店铺管理', 'admin_group_shop', 'admin', 'group-shop', NULL, 45
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'admin_group_shop');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `parent_id`, `sort_order`)
SELECT '记录与版本', 'admin_group_record', 'admin', 'group-record', NULL, 55
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'admin_group_record');
-- ---------- 2. 页面菜单挂到分组下,并同步显示名 ----------
-- 只更新当前没有父级的页面(保留管理员手工调整过的层级),
-- COALESCE 保证已挂接的叶子不重复指向。
UPDATE `columns` child
JOIN `columns` parent ON parent.`column_key` = 'admin_group_account'
SET child.`parent_id` = COALESCE(child.`parent_id`, parent.`id`)
WHERE child.`column_key` IN ('admin_users', 'admin_columns', 'admin_group_manage');
UPDATE `columns` child
JOIN `columns` parent ON parent.`column_key` = 'admin_group_data'
SET child.`parent_id` = COALESCE(child.`parent_id`, parent.`id`)
WHERE child.`column_key` IN ('admin_dedupe_total_data', 'admin_invalid_asin_data',
'admin_query_asin', 'admin_product_categories');
UPDATE `columns` child
JOIN `columns` parent ON parent.`column_key` = 'admin_group_shop'
SET child.`parent_id` = COALESCE(child.`parent_id`, parent.`id`)
WHERE child.`column_key` IN ('admin_shop_keys', 'admin_shop_manage', 'admin_skip_price_asin',
'admin_shop_data_crawl_tasks');
UPDATE `columns` child
JOIN `columns` parent ON parent.`column_key` = 'admin_group_record'
SET child.`parent_id` = COALESCE(child.`parent_id`, parent.`id`)
WHERE child.`column_key` IN ('admin_history', 'admin_version', 'digital_human_version',
'admin_image_video_tasks');
-- ---------- 3. 显示名对齐前端界面 ---
UPDATE `columns` SET `name` = '品牌数据库'
WHERE `menu_type` = 'admin' AND `column_key` = 'admin_invalid_asin_data';
UPDATE `columns` SET `name` = '最低价ASIN设置'
WHERE `menu_type` = 'admin' AND `column_key` = 'admin_skip_price_asin';
UPDATE `columns` SET `name` = '视频任务记录'
WHERE `menu_type` = 'admin' AND `column_key` = 'admin_image_video_tasks';
UPDATE `columns` SET `name` = '店铺数据记录'
WHERE `menu_type` = 'admin' AND `column_key` = 'admin_shop_data_crawl_tasks';
@@ -106,8 +106,17 @@ class ExplainIndexAuditDocTest {
assertTrue(audit.contains("不做 DDL") || audit.contains("本任务无 DDL")
|| audit.contains("不新增索引") || audit.contains("只审计"),
"审计文档明确本任务不做 DDL");
assertFalse(audit.contains("ALTER TABLE"),
"审计文档不包含 ALTER TABLEDDL 属于后续任务 116");
// 任务 116 落地 V101 索引后,审计文档会把 ALTER TABLE 作为候选索引的
// 回滚说明引用。审计任务自身依然不执行 DDL——判定口径改为「ALTER TABLE
// 只能出现在候选/回滚等说明性上下文」,而不是文档里不得出现该字符串。
for (String line : audit.split("\r?\n")) {
if (!line.contains("ALTER TABLE")) {
continue;
}
assertTrue(line.contains("回滚") || line.contains("候选") || line.contains("建议")
|| line.contains("风险") || line.trim().startsWith(">"),
"审计文档中的 ALTER TABLE 必须是候选/回滚说明,实际行:" + line);
}
assertTrue(spec.contains("只读") || spec.contains("巡检"),
"12 spec 巡检报表模式与审计文档对应");
}
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -268,6 +269,7 @@ class ExternalCallMetricsRecorderTest {
}
} finally {
pool.shutdown();
assertTrue(pool.awaitTermination(60, TimeUnit.SECONDS), "并发任务在超时前全部结束");
}
awaitMetric("aiimage.external-call.total", "client", "llm", "result", "success");
@@ -275,6 +277,12 @@ class ExternalCallMetricsRecorderTest {
Thread.sleep(10);
}
assertEquals(20, llmSubmitCount.get(), "并发 20 请求全部真实发出");
// 指标在响应处理完成之后才落,submit 计数到 20 不代表指标已收敛:
// 直接断言会读到中间值(实测 18.0)。等指标计数自身收敛再断言。
for (int i = 0; i < 2000 && counterCount("aiimage.external-call.total",
"client", "llm", "result", "success") < 20.0; i++) {
Thread.sleep(10);
}
assertEquals(20.0, counterCount("aiimage.external-call.total",
"client", "llm", "result", "success"), "20 次成功全部记录,无重复");
}
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.util.DefaultTempFileCreationStrategy;
import org.apache.poi.util.TempFile;
import org.apache.poi.util.TempFileCreationStrategy;
import org.junit.jupiter.api.AfterEach;
@@ -39,20 +40,36 @@ class CollectDataExcelCleanupTest {
private final List<File> trackedTempFiles = new ArrayList<>();
private Path sxssfTempDir;
@AfterEach
void restoreDefaultTempFileStrategy() {
// 还原 POI 的进程级全局策略:trackSxssfTempFiles 改写的是单例,
// 不还原会泄漏到同一 JVM 内后续所有用到 SXSSF 的测试。
// (原来这个 @AfterEach 挂在 private 的 trackSxssfTempFiles 上,
// 既没有还原策略,方法本身还是被各用例手工调用的 setup 辅助。)
TempFile.setTempFileCreationStrategy(new DefaultTempFileCreationStrategy());
}
private void trackSxssfTempFiles() {
// 把 SXSSF 滚动窗口临时文件重定向到本用例独占目录,使残留探测只覆盖
// 本用例自己产生的文件。
try {
sxssfTempDir = Files.createDirectory(tempDir.resolve("sxssf-probe"));
} catch (IOException ex) {
throw new IllegalStateException(ex);
}
TempFile.setTempFileCreationStrategy(new TempFileCreationStrategy() {
@Override
public File createTempFile(String prefix, String suffix) throws IOException {
File file = File.createTempFile(prefix, suffix);
File file = Files.createTempFile(sxssfTempDir, prefix, suffix).toFile();
trackedTempFiles.add(file);
return file;
}
@Override
public File createTempDirectory(String prefix) throws IOException {
File dir = Files.createTempDirectory(prefix).toFile();
File dir = Files.createTempDirectory(sxssfTempDir, prefix).toFile();
trackedTempFiles.add(dir);
return dir;
}
@@ -62,10 +79,13 @@ class CollectDataExcelCleanupTest {
private void assertSxssfTempFilesCleaned() {
assertThat(trackedTempFiles).as("SXSSF 滚动窗口临时文件均已删除")
.allSatisfy(f -> assertThat(f).doesNotExist());
// POI 5.2.5 无法读取当前策略,改用系统临时目录中 SXSSF 固定前缀的残留探测。
try (Stream<Path> paths = Files.list(Path.of(System.getProperty("java.io.tmpdir")))) {
// 只探测本用例独占目录。原实现扫描系统共享临时目录并要求 poi-sxssf*
// 前缀零命中,会把其他进程、其他用例和历史运行遗留的文件算成本用例
// 的泄漏——本机共享临时目录里就躺着十几个历史 poi-sxssf-sheet*.xml
// 导致实现完全正确时这 7 个用例照样红。
try (Stream<Path> paths = Files.list(sxssfTempDir)) {
assertThat(paths.filter(p -> p.getFileName().toString().startsWith("poi-sxssf")))
.as("系统临时目录无 SXSSF 滚动窗口残留")
.as("本用例 SXSSF 临时目录无滚动窗口残留")
.isEmpty();
} catch (IOException ex) {
throw new IllegalStateException(ex);
@@ -149,6 +149,7 @@ class ShopDataCrawlCleanupTest {
private final AtomicLong memberIdSeq = new AtomicLong(1000);
private final AtomicLong fileSeq = new AtomicLong(5000);
private Long lastJobTaskId;
private Set<Path> preExistingWorkRoots = Set.of();
@BeforeEach
void configureStorage() {
@@ -408,6 +409,12 @@ class ShopDataCrawlCleanupTest {
.deleteTaskItems(anyLong(), eq(MODULE_TYPE));
lenient().doNothing().when(taskProgressSnapshotService)
.delete(anyLong(), eq(MODULE_TYPE));
// 共享临时目录里的既有 daily-* 目录不属于本用例,先快照、断言时做差集
try {
preExistingWorkRoots = Set.copyOf(scanWorkRoots());
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
}
// ---- 1. 删除路径 ----
@@ -1261,6 +1268,16 @@ class ShopDataCrawlCleanupTest {
}
private List<Path> workRoots() throws Exception {
// 只统计本用例新产生的工作目录。shop-data-crawl-result 落在机器共享的
// 系统临时目录下,其他用例、历史运行、本机跑过的应用都会往里留 daily-*
// 目录(本机实测已堆了 200+ 个),原实现把这些全算作本用例的泄漏,
// 于是实现完全正确时这些用例照样红。改为与用例开始时的快照做差集。
List<Path> current = new ArrayList<>(scanWorkRoots());
current.removeAll(preExistingWorkRoots);
return current;
}
private List<Path> scanWorkRoots() throws Exception {
Path root = Path.of(System.getProperty("java.io.tmpdir"), "shop-data-crawl-result");
if (!Files.exists(root)) {
return List.of();
@@ -0,0 +1,155 @@
package com.nanri.aiimage.modules.similarasin.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightRequest;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import org.junit.jupiter.api.Test;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import java.util.List;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
/**
* 任务 109progress/light 轻量接口(similarasin 首个)。
* 新端点 POST /api/similar-asin/tasks/progress/light 只返回轻量字段(taskId/status/statusCode?/fileStatus?/fileReady?/updatedAt);
* 旧 progress/batch 端点原样保留可用;不存在的任务进 missingTaskIds。
*/
class SimilarAsinTaskLightControllerTest {
private static final String LIGHT_URL = "/api/similar-asin/tasks/progress/light";
private final ObjectMapper objectMapper = new ObjectMapper();
private SimilarAsinTaskService service;
private MockMvc mockMvc;
private SimilarAsinTaskLightBatchVo lightBatch(SimilarAsinTaskLightVo... items) {
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
vo.getItems().addAll(List.of(items));
return vo;
}
private static SimilarAsinTaskLightVo lightItem(Long taskId, String status, String fileStatus, Boolean fileReady, String updatedAt) {
SimilarAsinTaskLightVo vo = new SimilarAsinTaskLightVo();
vo.setTaskId(taskId);
vo.setStatus(status);
vo.setFileStatus(fileStatus);
vo.setFileReady(fileReady);
vo.setUpdatedAt(updatedAt);
return vo;
}
private void setUpWith(SimilarAsinTaskLightBatchVo response) throws Exception {
service = mock(SimilarAsinTaskService.class);
when(service.progressLight(any())).thenReturn(response);
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
}
@Test
void lightReturnsLightFields() throws Exception {
setUpWith(lightBatch(lightItem(3938L, "RUNNING", "RUNNING", false, "2026-04-26T10:05:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3938L)))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true))
.andExpect(jsonPath("$.data.items[0].taskId").value(3938))
.andExpect(jsonPath("$.data.items[0].status").value("RUNNING"))
.andExpect(jsonPath("$.data.items[0].fileStatus").value("RUNNING"))
.andExpect(jsonPath("$.data.items[0].fileReady").value(false))
.andExpect(jsonPath("$.data.items[0].updatedAt").value("2026-04-26T10:05:00"));
}
@Test
void lightMissingTask() throws Exception {
setUpWith(lightBatch(lightItem(1L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:10:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(1L)))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true))
.andExpect(jsonPath("$.data.items[0].taskId").value(1));
verify(service).progressLight(any());
}
@Test
void lightEmptyIds() throws Exception {
setUpWith(new SimilarAsinTaskLightBatchVo());
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of()))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items").isEmpty())
.andExpect(jsonPath("$.data.missingTaskIds").isEmpty());
}
@Test
void lightDedupIds() throws Exception {
setUpWith(lightBatch(lightItem(7L, "PENDING", null, false, "2026-04-26T10:00:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(7L, 7L, 7L)))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items.length()").value(1));
}
@Test
void lightOldEndpointUntouched() throws Exception {
com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest request = new com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest();
request.setTaskIds(List.of(1L));
service = mock(SimilarAsinTaskService.class);
when(service.progressBatch(any())).thenReturn(new com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo());
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
mockMvc.perform(post("/api/similar-asin/tasks/progress/batch")
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(request)))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true));
verify(service).progressBatch(any());
}
@Test
void lightStatusMapping() throws Exception {
setUpWith(lightBatch(lightItem(2L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:11:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(2L)))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items[0].status").value("SUCCESS"));
}
@Test
void lightFileFields() throws Exception {
setUpWith(lightBatch(lightItem(3L, "RUNNING", "SUCCESS", true, "2026-04-26T10:12:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3L)))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items[0].fileStatus").value("SUCCESS"))
.andExpect(jsonPath("$.data.items[0].fileReady").value(true));
}
@Test
void lightUnknownId() throws Exception {
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
vo.getMissingTaskIds().add(99999L);
setUpWith(vo);
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(99999L)))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items").isEmpty())
.andExpect(jsonPath("$.data.missingTaskIds[0]").value(99999));
}
}
@@ -0,0 +1,127 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.api.ApiResponse;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 110progress/light 字段白名单。
* 响应仅含白名单字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt);
* 不含 items 明细/payload/result 内容;序列化后键集合精确匹配;响应体小(<1KB)。
*/
class SimilarAsinTaskLightWhitelistTest {
private static final Set<String> WHITELIST = new TreeSet<>(Set.of(
"taskId", "status", "statusCode", "fileStatus", "fileError", "fileReady", "updatedAt"));
private final ObjectMapper objectMapper = new ObjectMapper();
private SimilarAsinTaskLightVo fullItem() {
SimilarAsinTaskLightVo vo = new SimilarAsinTaskLightVo();
vo.setTaskId(3938L);
vo.setStatus("RUNNING");
vo.setStatusCode(null);
vo.setFileStatus("RUNNING");
vo.setFileReady(false);
vo.setUpdatedAt("2026-04-26T10:05:00");
return vo;
}
private Set<String> keysOf(SimilarAsinTaskLightVo vo) throws Exception {
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(vo));
Set<String> keys = new TreeSet<>();
node.fieldNames().forEachRemaining(keys::add);
return keys;
}
@Test
void whitelistExactKeys() throws Exception {
assertEquals(WHITELIST, keysOf(fullItem()), "序列化键集合与白名单精确一致");
}
@Test
void whitelistNoItemsArray() throws Exception {
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(fullItem()));
assertFalse(node.has("items"), "不含明细数组");
assertFalse(node.has("itemCount"));
}
@Test
void whitelistNoPayload() throws Exception {
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(fullItem()));
assertFalse(node.has("payload"), "不含 payload 字段");
assertFalse(node.has("requestJson"));
assertFalse(node.has("resultJson"));
}
@Test
void whitelistNoResultContent() throws Exception {
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(fullItem()));
assertFalse(node.has("resultId"), "不含 resultId");
assertFalse(node.has("downloadUrl"), "不含下载链接");
assertFalse(node.has("rowCount"), "不含行数明细");
assertFalse(node.has("success"), "不含成功标志明细");
}
@Test
void whitelistFilePhasePresent() throws Exception {
SimilarAsinTaskLightVo vo = fullItem();
vo.setFileStatus("SUCCESS");
vo.setFileReady(true);
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(vo));
assertEquals("SUCCESS", node.get("fileStatus").asText());
assertTrue(node.get("fileReady").asBoolean());
}
@Test
void whitelistNullValuesAllowed() throws Exception {
SimilarAsinTaskLightVo vo = new SimilarAsinTaskLightVo();
vo.setTaskId(1L);
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(vo));
assertTrue(node.has("statusCode"), "statusCode 键存在(null 值允许)");
assertTrue(node.get("statusCode").isNull());
assertEquals(WHITELIST, keysOf(vo), "null 字段仍在白名单键集合内");
}
@Test
void whitelistResponseSizeSmall() throws Exception {
SimilarAsinTaskLightBatchVo batch = new SimilarAsinTaskLightBatchVo();
batch.getItems().add(fullItem());
byte[] bytes = objectMapper.writeValueAsBytes(ApiResponse.success(batch));
assertTrue(bytes.length < 1024, "单条 light 响应 <1KB,实际 " + bytes.length + "B");
}
@Test
void whitelistExtraKeysRejected() throws Exception {
// 回归门禁:白名单键集合与文档一致(新增键必须先更新本测试与 spec)
assertEquals(List.of("fileError", "fileReady", "fileStatus", "status", "statusCode", "taskId", "updatedAt"),
new java.util.ArrayList<>(WHITELIST));
}
@Test
void whitelistDocumentedInSpec() throws Exception {
// 与 spec 06 §2 对齐:taskId/status/statusCode?/fileStatus?/fileError?/fileReady?/updatedAt
assertTrue(WHITELIST.containsAll(Set.of("taskId", "status", "updatedAt")));
assertTrue(WHITELIST.containsAll(Set.of("statusCode", "fileStatus", "fileReady", "fileError")));
}
@Test
void whitelistBatchHasOnlyItemsAndMissing() throws Exception {
SimilarAsinTaskLightBatchVo batch = new SimilarAsinTaskLightBatchVo();
batch.getItems().add(fullItem());
batch.getMissingTaskIds().add(999L);
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsBytes(batch));
Set<String> keys = new TreeSet<>();
node.fieldNames().forEachRemaining(keys::add);
assertEquals(new TreeSet<>(Set.of("items", "missingTaskIds")), keys, "批量 VO 只含 items/missingTaskIds");
}
}
@@ -0,0 +1,85 @@
package com.nanri.aiimage.modules.similarasin.service;
import javax.net.ssl.SNIHostName;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import java.net.InetAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.List;
public class LlmGatewayTlsProbe {
public static void main(String[] args) throws Exception {
String host = "ai.t8star.org";
String apiKey = args[0];
System.out.println("[probe] java=" + System.getProperty("java.version")
+ " tls=" + System.getProperty("java.vm.name"));
for (InetAddress a : InetAddress.getAllByName(host)) {
System.out.println("[probe] dns " + a);
}
rawHandshake(host, null, "default");
rawHandshake(host, "TLSv1.2", "tls12-only");
httpClientCall(host, apiKey, null, "jdk-http-default");
httpClientCall(host, apiKey, "TLSv1.2", "jdk-http-tls12");
}
private static void rawHandshake(String host, String protocol, String label) throws Exception {
SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
try (SSLSocket socket = (SSLSocket) factory.createSocket(host, 443)) {
socket.setSoTimeout(15000);
SSLParameters params = socket.getSSLParameters();
if (protocol != null) {
params.setProtocols(new String[]{protocol});
}
params.setServerNames(List.of(new SNIHostName(host)));
socket.setSSLParameters(params);
socket.startHandshake();
System.out.println("[probe] raw[" + label + "] OK proto=" + socket.getSession().getProtocol()
+ " cipher=" + socket.getSession().getCipherSuite());
} catch (Exception ex) {
System.out.println("[probe] raw[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
}
}
private static void httpClientCall(String host, String apiKey, String protocol, String label) throws Exception {
HttpClient.Builder builder = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(10))
.version(HttpClient.Version.HTTP_1_1);
if (protocol != null) {
SSLContext context = SSLContext.getInstance("TLS");
context.init(null, null, null);
builder.sslContext(context);
}
HttpClient client = builder.build();
try {
String body = "{\"model\":\"gemini-3.5-flash-lite\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":10}";
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("https://" + host + "/v1/chat/completions"))
.timeout(Duration.ofSeconds(30))
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(body))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
String text = response.body();
System.out.println("[probe] http[" + label + "] status=" + response.statusCode()
+ " body=" + text.substring(0, Math.min(160, text.length())));
} catch (Exception ex) {
System.out.println("[probe] http[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
Throwable cause = ex;
while (cause.getCause() != null) {
cause = cause.getCause();
System.out.println("[probe] cause " + cause.getClass().getSimpleName() + ": " + cause.getMessage());
}
}
}
}
@@ -0,0 +1,226 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 15:图片缓存访问时间更新改为异步批量刷新,减少逐图 UPDATE。
* lookup 命中不再同步 touchLastUsed,而是进入内存 touch 缓冲(按 url_hash 去重),
* 由定时任务/阈值触发 flushPendingTouches 批量 touchLastUsedBatch 刷新;
* 缓冲有大小上限,超限立即刷新,不会无界增长;失败 best-effort 吞掉。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinImagePrefetchServiceAsyncTouchTest {
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private TaskImageCacheMapper taskImageCacheMapper;
@Mock private SimilarAsinProperties properties;
@InjectMocks private SimilarAsinImagePrefetchService service;
@BeforeEach
void setUp() {
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(1000);
}
@AfterEach
void shutdown() {
service.shutdown();
}
private static String sha256Hex(String value) throws Exception {
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(digest.length * 2);
for (byte b : digest) {
sb.append(String.format("%02x", b & 0xFF));
}
return sb.toString();
}
private void stubHit(String url, byte[] bytes) throws Exception {
when(taskImageCacheMapper.selectBytesByUrlHash(sha256Hex(url))).thenReturn(bytes);
}
@Test
void test_task_015_image_cache_normal_default_path() throws Exception {
// 正常输入:命中不立即写 DB,进入缓冲;flush 后批量 touch 一次。
String url = "https://img.example.com/hit.jpg";
String hash = sha256Hex(url);
byte[] bytes = new byte[]{1, 2, 3};
stubHit(url, bytes);
byte[] result = service.lookup(url);
assertEquals(bytes, result, "命中必须返回缓存字节");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_normal_multiple_items() throws Exception {
// 批量场景:多次命中进入同一缓冲,flush 合并为一次批量 touch,覆盖全部命中。
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
"https://img.example.com/c.jpg");
Set<String> hashes = new HashSet<>();
for (int i = 0; i < urls.size(); i++) {
hashes.add(sha256Hex(urls.get(i)));
stubHit(urls.get(i), new byte[]{(byte) (i + 1)});
}
for (String url : urls) {
assertNotNull(service.lookup(url), "命中返回字节");
}
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
service.flushPendingTouches();
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
assertEquals(hashes, new HashSet<>(captor.getValue()), "一次批量 touch 覆盖全部命中 hash");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:同一 url 多次命中只 touch 一次;重复 flush 无多余请求。
String url = "https://img.example.com/same.jpg";
String hash = sha256Hex(url);
stubHit(url, new byte[]{5});
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(new byte[]{5});
service.lookup(url);
service.lookup(url);
service.lookup(url);
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_boundary_empty_input() throws Exception {
// 空输入:null/空白 url 不进入缓冲;flush 空缓冲不产生任何数据库访问。
assertNull(service.lookup(null));
assertNull(service.lookup(""));
assertNull(service.lookup(" "));
service.flushPendingTouches();
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
}
@Test
void test_task_015_image_cache_boundary_single_item() throws Exception {
// 单条命中:flush 后单元素批量 touch,不依赖批量路径。
String url = "https://img.example.com/single.jpg";
String hash = sha256Hex(url);
stubHit(url, new byte[]{7});
assertNotNull(service.lookup(url));
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
}
@Test
void test_task_015_image_cache_boundary_limit_and_overflow() throws Exception {
// 缓冲达到阈值立即刷新,不无界增长;刷新后继续累积。
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(3);
List<String> urls = List.of("https://img.example.com/o1.jpg", "https://img.example.com/o2.jpg",
"https://img.example.com/o3.jpg", "https://img.example.com/o4.jpg",
"https://img.example.com/o5.jpg");
for (String url : urls) {
stubHit(url, new byte[]{1});
}
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
for (int i = 0; i < 3; i++) {
service.lookup(urls.get(i));
}
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
assertEquals(3, captor.getValue().size(), "第 3 条命中触发阈值立即刷新 3 条");
service.lookup(urls.get(3));
service.lookup(urls.get(4));
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
service.flushPendingTouches();
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
assertEquals(2, captor.getValue().size(), "剩余 2 条在 flush 时刷新,缓冲不残留、不无界增长");
}
@Test
void test_task_015_image_cache_invalid_input_rejected() {
// 非法输入:db cache 关闭时 lookup 直接返回 null,不进入缓冲、不访问数据库。
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
assertNull(service.lookup("https://img.example.com/a.jpg"));
service.flushPendingTouches();
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_dependency_failure_releases_resources() throws Exception {
// 依赖失败:批量 touch 抛异常时吞掉不阻塞 lookup、缓冲已排空无残留;
// 恢复后重新入队 flush 成功。
String url = "https://img.example.com/fail.jpg";
String hash = sha256Hex(url);
stubHit(url, new byte[]{3});
AtomicInteger failures = new AtomicInteger(0);
List<String> capturedArgs = new java.util.ArrayList<>();
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
List<String> arg = (List<String>) invocation.getArgument(0);
capturedArgs.addAll(arg);
if (failures.getAndIncrement() == 0) {
throw new IllegalStateException("db down");
}
return 0;
}).when(taskImageCacheMapper).touchLastUsedBatch(any());
assertNotNull(service.lookup(url), "touch 失败不阻断 lookup 返回缓存字节");
assertThrows(Exception.class, () -> service.flushPendingTouches(), "首次 flush 抛错(由调用方吞掉)");
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
assertNotNull(service.lookup(url), "失败后再次命中重新入队");
service.flushPendingTouches();
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(any());
assertEquals(List.of(hash, hash), capturedArgs, "两次 touch 都覆盖命中 hash,失败后恢复成功");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
private static void assertNull(Object value) {
org.junit.jupiter.api.Assertions.assertNull(value);
}
}
@@ -0,0 +1,297 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 14:图片 DB cache 改为批量读取缩略图,并只更新实际命中的 last_used_at。
* 批量 lookup 入口(lookupBatch)一次 IN 查询返回命中字节 Map;
* last_used_at 只对实际命中的 url_hash 更新(touch 集合 = 命中集合),
* 未命中 url 不产生任何 touch/insert。单 URL 旧入口 lookup 语义保持兼容。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinImagePrefetchServiceBatchTest {
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private TaskImageCacheMapper taskImageCacheMapper;
@Mock private SimilarAsinProperties properties;
@InjectMocks private SimilarAsinImagePrefetchService service;
@BeforeEach
void setUp() {
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
}
@AfterEach
void shutdown() {
service.shutdown();
}
private static String sha256Hex(String value) throws Exception {
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(digest.length * 2);
for (byte b : digest) {
sb.append(String.format("%02x", b & 0xFF));
}
return sb.toString();
}
/** stub 批量命中读取:只对 hits 集合内的 hash 返回字节。 */
private void stubBatchRead(List<String> hits, Map<String, byte[]> bytesByHash) {
when(taskImageCacheMapper.selectBytesByUrlHashes(any())).thenAnswer(invocation -> {
List<String> hashes = invocation.getArgument(0);
List<TaskImageCacheEntity> rows = new ArrayList<>();
for (String hash : hashes) {
if (hits.contains(hash)) {
TaskImageCacheEntity row = new TaskImageCacheEntity();
row.setUrlHash(hash);
row.setImageBytes(bytesByHash.get(hash));
rows.add(row);
}
}
return rows;
});
}
/** 调用批量入口,按 url 顺序返回字节(未命中为 null)。 */
@SuppressWarnings("unchecked")
private static List<byte[]> invokeLookupBatch(SimilarAsinImagePrefetchService svc, List<String> urls) throws Exception {
Method m = SimilarAsinImagePrefetchService.class.getDeclaredMethod("lookupBatch", List.class);
m.setAccessible(true);
return (List<byte[]>) m.invoke(svc, urls);
}
@Test
void test_task_014_image_normal_default_path() throws Exception {
// 正常输入:命中与未命中混排,批量读回命中字节,touch 只覆盖实际命中。
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
List<String> hits = List.of(sha256Hex(urls.get(0)));
byte[] bytesA = new byte[]{1, 2, 3};
stubBatchRead(hits, Map.of(sha256Hex(urls.get(0)), bytesA));
List<byte[]> result = invokeLookupBatch(service, urls);
assertEquals(2, result.size(), "批量入口必须按输入顺序返回");
assertEquals(bytesA, result.get(0), "命中行返回缓存字节");
assertNull(result.get(1), "未命中行返回 null,不虚构缓存内容");
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(sha256Hex(urls.get(0))));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
}
@Test
void test_task_014_image_normal_multiple_items() throws Exception {
// 批量场景:全命中多 url,一次 IN 查询返回全部字节,touch 覆盖全部命中。
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
"https://img.example.com/c.jpg");
List<String> hits = new ArrayList<>();
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
for (int i = 0; i < urls.size(); i++) {
hits.add(sha256Hex(urls.get(i)));
bytesByHash.put(hits.get(i), new byte[]{(byte) (i + 1)});
}
stubBatchRead(hits, bytesByHash);
List<byte[]> result = invokeLookupBatch(service, urls);
assertEquals(3, result.size());
for (int i = 0; i < urls.size(); i++) {
assertEquals(bytesByHash.get(sha256Hex(urls.get(i))), result.get(i), "顺序稳定、字节不丢失");
}
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(new ArrayList<>(hits));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:每次行为一致,不产生重复请求/重复 touch。
List<String> urls = List.of("https://img.example.com/a.jpg");
String hash = sha256Hex(urls.get(0));
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{9}));
invokeLookupBatch(service, urls);
invokeLookupBatch(service, urls);
verify(taskImageCacheMapper, times(2)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(List.of(hash));
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_boundary_empty_input() throws Exception {
// 空输入:null/空列表安全返回空结果,不产生任何数据库访问。
List<byte[]> nullResult = invokeLookupBatch(service, null);
assertNotNull(nullResult);
assertTrue(nullResult.isEmpty());
List<byte[]> emptyResult = invokeLookupBatch(service, List.of());
assertNotNull(emptyResult);
assertTrue(emptyResult.isEmpty());
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_boundary_single_item() throws Exception {
// 单 url:不依赖批量路径,命中时单次查询 + 单次 touch。
String url = "https://img.example.com/single.jpg";
String hash = sha256Hex(url);
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{7}));
List<byte[]> result = invokeLookupBatch(service, List.of(url));
assertEquals(1, result.size());
assertEquals(7, result.get(0)[0]);
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
}
@Test
void test_task_014_image_boundary_limit_and_overflow() throws Exception {
// 大批量(超过单批上限 500):分片查询,命中 touch 只覆盖命中集合。
List<String> urls = new ArrayList<>();
for (int i = 0; i < 1200; i++) {
urls.add("https://img.example.com/overflow-" + i + ".jpg");
}
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
List<String> hits = new ArrayList<>();
for (int i = 0; i < 1200; i += 2) {
String hash = sha256Hex(urls.get(i));
hits.add(hash);
bytesByHash.put(hash, new byte[]{(byte) (i % 100)});
}
stubBatchRead(hits, bytesByHash);
List<byte[]> result = invokeLookupBatch(service, urls);
assertEquals(1200, result.size(), "超大批量结果不丢失、顺序稳定");
int hitCount = 0;
for (int i = 0; i < 1200; i++) {
if (i % 2 == 0) {
assertNotNull(result.get(i), "偶数下标命中必须返回字节");
hitCount++;
} else {
assertNull(result.get(i), "奇数下标未命中返回 null");
}
}
assertEquals(600, hitCount);
verify(taskImageCacheMapper, times(3)).selectBytesByUrlHashes(any());
// touch 按单批 500 分片:600 命中 → 2 次 touch 调用,且只覆盖命中集合。
var captor = org.mockito.ArgumentCaptor.forClass(List.class);
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
List<List<String>> touchCalls = new ArrayList<>();
for (Object call : captor.getAllValues()) {
@SuppressWarnings("unchecked")
List<String> casted = (List<String>) call;
touchCalls.add(casted);
}
assertEquals(2, touchCalls.size());
assertEquals(500, touchCalls.get(0).size(), "第一批 touch 500 个命中");
assertEquals(100, touchCalls.get(1).size(), "第二批 touch 剩余 100 个命中");
assertEquals(hits.subList(0, 500), touchCalls.get(0), "touch 只覆盖实际命中集合");
assertEquals(hits.subList(500, 600), touchCalls.get(1));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_invalid_input_rejected() throws Exception {
// 非法输入:db cache 关闭时批量入口直接返回空,不访问数据库。
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
List<String> urls = List.of("https://img.example.com/a.jpg");
List<byte[]> result = invokeLookupBatch(service, urls);
assertNotNull(result);
assertTrue(result.isEmpty(), "db cache 关闭时必须直接返回空结果");
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
}
@Test
void test_task_014_image_dependency_failure_releases_resources() throws Exception {
// 依赖失败:查询抛异常时批量入口返回空结果、无任何 touch/insert 残留;
// 恢复后重试成功。
List<String> urls = List.of("https://img.example.com/a.jpg");
String hash = sha256Hex(urls.get(0));
AtomicInteger callCount = new AtomicInteger(0);
doAnswer(invocation -> {
if (callCount.getAndIncrement() == 0) {
throw new IllegalStateException("db down");
}
TaskImageCacheEntity row = new TaskImageCacheEntity();
row.setUrlHash(hash);
row.setImageBytes(new byte[]{5});
return List.of(row);
}).when(taskImageCacheMapper).selectBytesByUrlHashes(any());
List<byte[]> failed = invokeLookupBatch(service, urls);
assertNotNull(failed);
assertTrue(failed.isEmpty(), "查询失败必须返回空结果而不是抛错阻断组装");
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
List<byte[]> recovered = invokeLookupBatch(service, urls);
assertEquals(1, recovered.size());
assertEquals(5, recovered.get(0)[0], "依赖恢复后重试成功");
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
}
@Test
void test_task_014_image_normal_single_lookup_legacy_compat() throws Exception {
// 兼容性:单 URL 旧入口 lookup 保持返回字节语义;Task 15 起 touch 改为
// 异步批量缓冲,flush 后批量 touch 一次,命中才入缓冲。
String url = "https://img.example.com/legacy.jpg";
String hash = sha256Hex(url);
byte[] bytes = new byte[]{6};
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(bytes);
byte[] result = service.lookup(url);
assertEquals(bytes, result, "lookup 命中必须返回缓存字节");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
assertNull(service.lookup("https://img.example.com/missing.jpg"), "未命中返回 null");
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
}
@@ -0,0 +1,167 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper;
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import org.mockito.Mockito;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
/**
* 本地验证入口:用生产真实批次数据 + 生产 LLM 网关跑 SimilarAsinLlmService 完整链路。
* 用法:mvn compile test-compile 后执行
* java -cp target/classes;target/test-classes;$(cat cp.txt) com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]
*/
public class SimilarAsinLlmLocalVerify {
public static void main(String[] args) throws Exception {
System.setOut(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.out), true, "UTF-8"));
System.setErr(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.err), true, "UTF-8"));
if (args.length < 2) {
System.err.println("usage: SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]");
System.exit(1);
}
String dataFile = args[0];
String apiKey = args[1];
boolean imgSwitch = args.length > 2 && Boolean.parseBoolean(args[2]);
boolean categorySwitch = args.length > 3 && Boolean.parseBoolean(args[3]);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
JsonNode root = objectMapper.readTree(new File(dataFile));
List<SimilarAsinResultRowDto> rows = new ArrayList<>();
if (root.isArray()) {
for (JsonNode node : root) {
rows.add(fromJson(objectMapper, node));
}
} else {
rows.add(fromJson(objectMapper, root));
}
System.out.println("[verify] loaded rows=" + rows.size() + " imgSwitch=" + imgSwitch
+ " categorySwitch=" + categorySwitch);
if (imgSwitch && categorySwitch) {
System.out.println("[verify] raw first row alibaba[0].url="
+ (rows.isEmpty() || rows.get(0).getAlibaba().isEmpty() ? "null"
: rows.get(0).getAlibaba().get(0).getUrl()));
}
SimilarAsinProperties props = new SimilarAsinProperties();
props.setLlmApiKey(apiKey);
props.setLlmRowConcurrency(2);
props.setLlmImageDownloadTimeoutSeconds(10);
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null);
OssProperties ossProps = new OssProperties();
ossProps.setEndpoint("https://oss.aishufu.top");
ossProps.setPublicEndpoint("https://oss.aishufu.top");
ossProps.setBucket("nanri-ai-images");
ossProps.setAccessKeyId("appuser");
ossProps.setAccessKeySecret("AppUser@2024SecureKey");
OssStorageService oss = new OssStorageService(ossProps);
PuzzleImageMerger merger = new PuzzleImageMerger(props);
// 生产真实类目数据(导出自 biz_product_category),spy 类目服务按 parentId 过滤。
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> categories = loadCategories(objectMapper);
ProductCategoryService categoryService = Mockito.spy(new ProductCategoryService(
Mockito.mock(com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper.class)));
Mockito.doAnswer(invocation -> {
Long parentId = invocation.getArgument(0);
List<com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo> items = categories.stream()
.filter(c -> parentId == null ? c.getParentId() == null : parentId.equals(c.getParentId()))
.map(c -> {
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo item =
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo();
item.setId(c.getId());
item.setParentId(c.getParentId());
item.setName(c.getName());
item.setCategoryKey(c.getCategoryKey());
return item;
})
.toList();
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo vo =
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo();
vo.setItems(items);
vo.setTree(List.of());
vo.setTotal((long) items.size());
vo.setPage(1L);
vo.setPageSize((long) Math.max(1, items.size()));
vo.setHasMore(false);
return vo;
}).when(categoryService).children(Mockito.any(), Mockito.anyLong(), Mockito.anyLong());
SimilarAsinLlmService service = new SimilarAsinLlmService(
client, props, ossProps, categoryService, merger, oss);
long start = System.currentTimeMillis();
List<SimilarAsinResultRowDto> result = service.inspectRows(rows, null, apiKey, imgSwitch, categorySwitch);
long elapsed = System.currentTimeMillis() - start;
System.out.println("[verify] done rows=" + result.size() + " elapsedMs=" + elapsed);
for (SimilarAsinResultRowDto row : result) {
System.out.println(String.format(
"asin=%s | status=%s | isConform=%s | category=%s | reason=%s | isStock=%s | similarity=%s | mainUrl=%s | puzzle1=%s | puzzle2=%s",
row.getAsin(), row.getStatus(), row.getIsConform(), row.getCategory(),
row.getReason(), row.getIsStock(), row.getSimilarity(),
shorten(row.getMainUrl()), shorten(row.getPuzzleImg1()), shorten(row.getPuzzleImg2())));
}
}
private static List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> loadCategories(ObjectMapper objectMapper) throws Exception {
com.fasterxml.jackson.databind.JsonNode root = objectMapper.readTree(
SimilarAsinLlmLocalVerify.class.getResourceAsStream("/categories.json"));
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> list = new ArrayList<>();
for (com.fasterxml.jackson.databind.JsonNode node : root) {
com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity entity =
new com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity();
entity.setId(node.get("id").asLong());
if (!node.get("parent_id").isNull()) {
entity.setParentId(node.get("parent_id").asLong());
}
entity.setName(node.get("name").asText());
entity.setCategoryKey(node.get("category_key").asText());
entity.setSortOrder(node.get("sort_order").isNull() ? null : node.get("sort_order").asInt());
entity.setDescription(node.get("description").isNull() ? null : node.get("description").asText());
entity.setIsBuiltin(node.get("is_builtin") != null && node.get("is_builtin").asBoolean());
list.add(entity);
}
return list;
}
private static SimilarAsinResultRowDto fromJson(ObjectMapper objectMapper, JsonNode node) throws Exception {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin(text(node, "asin"));
row.setTitle(text(node, "title"));
row.setSku(text(node, "sku"));
row.setCountry(text(node, "country"));
row.setUrl(text(node, "url"));
JsonNode alibaba = node.get("alibaba");
if (alibaba != null && alibaba.isArray()) {
List<SimilarAsinResultRowDto.AlibabaItem> items = new ArrayList<>();
for (JsonNode item : alibaba) {
items.add(objectMapper.treeToValue(item, SimilarAsinResultRowDto.AlibabaItem.class));
}
row.setAlibaba(items);
}
return row;
}
private static String text(JsonNode node, String field) {
JsonNode value = node.get(field);
return value == null || value.isNull() ? null : value.asText();
}
private static String shorten(String value) {
if (value == null) {
return "null";
}
return value.length() <= 70 ? value : value.substring(0, 70) + "...";
}
}
@@ -0,0 +1,279 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo;
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo;
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.mockito.Mockito;
class SimilarAsinLlmServiceTest {
private static byte[] jpegBytes() {
try {
BufferedImage image = new BufferedImage(10, 10, BufferedImage.TYPE_INT_RGB);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", baos);
return baos.toByteArray();
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
}
private SimilarAsinProperties properties() {
SimilarAsinProperties props = new SimilarAsinProperties();
props.setLlmApiKey("test-key");
return props;
}
private SimilarAsinLlmService service(SimilarAsinLlmClient llmClient) {
OssStorageService ossStorage = mock(OssStorageService.class);
when(ossStorage.getPublicUrl(anyString())).thenAnswer(invocation -> "https://oss.aishufu.top/nanri-ai-images/" + invocation.getArgument(0));
PuzzleImageMerger merger = mock(PuzzleImageMerger.class);
when(merger.merge(anyList(), Mockito.<SimilarAsinResultRowDto>any())).thenReturn(jpegBytes());
ProductCategoryService categoryService = mock(ProductCategoryService.class);
when(categoryService.children(any(), anyLong(), anyLong()))
.thenAnswer(invocation -> {
Long parentId = invocation.getArgument(0);
if (parentId == null) {
return categoryPage("类目A", 1L);
}
if (parentId == 1L) {
return categoryPage("类目B", 2L);
}
if (parentId == 2L) {
return categoryPage("类目C", 3L);
}
return emptyCategoryPage();
});
SimilarAsinLlmService svc = new SimilarAsinLlmService(
llmClient,
properties(),
mock(OssProperties.class),
categoryService,
merger,
ossStorage);
svc.setDownloadHttpClientForTest(mockHttpClient());
return svc;
}
private static ProductCategoryListVo categoryPage(String name, long id) {
ProductCategoryItemVo item = new ProductCategoryItemVo();
item.setId(id);
item.setName(name);
ProductCategoryListVo vo = new ProductCategoryListVo();
vo.setItems(List.of(item));
vo.setTotal(1L);
vo.setPage(1L);
vo.setPageSize(1L);
vo.setHasMore(false);
return vo;
}
private static ProductCategoryListVo emptyCategoryPage() {
ProductCategoryListVo vo = new ProductCategoryListVo();
vo.setItems(List.of());
vo.setTotal(0L);
vo.setPage(1L);
vo.setPageSize(1L);
vo.setHasMore(false);
return vo;
}
private static HttpClient mockHttpClient() {
HttpClient client = mock(HttpClient.class);
try {
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
.thenAnswer(invocation -> {
byte[] body = jpegBytes();
HttpRequest request = invocation.getArgument(0);
return new TestHttpResponse(200, body, request);
});
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
return client;
}
private static final class TestHttpResponse implements HttpResponse<byte[]> {
private final int statusCode;
private final byte[] body;
private final HttpRequest request;
TestHttpResponse(int statusCode, byte[] body, HttpRequest request) {
this.statusCode = statusCode;
this.body = body;
this.request = request;
}
@Override
public int statusCode() {
return statusCode;
}
@Override
public HttpRequest request() {
return request;
}
@Override
public java.util.Optional<HttpResponse<byte[]>> previousResponse() {
return java.util.Optional.empty();
}
@Override
public java.net.http.HttpHeaders headers() {
return java.net.http.HttpHeaders.of(Map.of(), (a, b) -> true);
}
@Override
public byte[] body() {
return body;
}
@Override
public java.net.URI uri() {
return java.net.URI.create("http://test");
}
@Override
public java.net.http.HttpClient.Version version() {
return HttpClient.Version.HTTP_1_1;
}
@Override
public java.util.Optional<javax.net.ssl.SSLSession> sslSession() {
return java.util.Optional.empty();
}
}
private SimilarAsinLlmClient llmClient(String apiKey, Map<String, String> responses) {
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
when(client.resolveApiKey(anyString())).thenReturn(apiKey == null ? "" : apiKey);
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
.thenAnswer(invocation -> {
String response = responses.get("chat");
if (response == null) {
throw new IllegalStateException("unexpected chat call");
}
return response;
});
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
String response = responses.get("images");
if (response == null) {
throw new IllegalStateException("unexpected images call");
}
return response;
});
return client;
}
private static final com.fasterxml.jackson.databind.ObjectMapper OBJECT_MAPPER =
new com.fasterxml.jackson.databind.ObjectMapper();
/** 类目匹配(按序号返回不同类目名)与合规检查(后续)返回不同 JSON。 */
private SimilarAsinLlmClient llmClientStaged(String apiKey, List<String> categoryJsons,
String conformJson, String imagesJson) {
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
int[] categoryIndex = {0};
when(client.resolveApiKey(anyString())).thenReturn(apiKey);
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
.thenAnswer(invocation -> {
if (categoryIndex[0] < categoryJsons.size()) {
return categoryJsons.get(categoryIndex[0]++);
}
return conformJson;
});
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
.thenAnswer(invocation -> imagesJson);
when(client.parseJsonContent(anyString()))
.thenAnswer(invocation -> parseJson(invocation.getArgument(0)));
return client;
}
private static com.fasterxml.jackson.databind.JsonNode parseJson(String content) {
try {
return OBJECT_MAPPER.readTree(content);
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
}
@Test
void noApiKeyKeepsRawRows() {
SimilarAsinLlmService service = service(llmClient("", Map.of()));
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0TEST");
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "", true, true);
assertEquals(1, result.size());
assertEquals("B0TEST", result.get(0).getAsin());
assertNull(result.get(0).getStatus());
}
@Test
void categorySwitchOffOnlyPreparesImagesAndMarksNotExistsWhenNoMainUrl() {
SimilarAsinLlmService service = service(llmClient("k", Map.of()));
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0TEST");
row.setTitle("Test product");
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", false, false);
assertEquals(1, result.size());
assertEquals("不存在", result.get(0).getStatus());
assertNull(result.get(0).getIsConform());
assertNull(result.get(0).getPuzzleImg1());
}
@Test
void imageCompareStopsOnStockAndFillsFields() {
// 前 2 次 chat:一级/二级类目匹配返回名称(Java 按名回查候选取真实 ID);第 3 次:合规检查。
SimilarAsinLlmClient client = llmClientStaged("k", List.of("{\"name\":\"类目A\"}", "{\"name\":\"类目B\"}"),
"{\"asin\":\"B0TEST\",\"is_conform\":\"符合\",\"reason\":\"\",\"category\":\"类目A->类目B->类目C\"}",
"{\"asin\":\"B0TEST\",\"is_stock\":\"有货\",\"similarity\":\"95%\",\"status\":\"成功\",\"is_conform\":\"符合\",\"category\":\"类目A->类目B->类目C\"}");
SimilarAsinLlmService service = service(client);
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0TEST");
row.setTitle("Test product");
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
SimilarAsinResultRowDto.AlibabaItem item = new SimilarAsinResultRowDto.AlibabaItem();
item.setUrl("https://cbu01.alicdn.com/img/1.jpg");
row.setAlibaba(List.of(item));
// 一级/二级都匹配(按名回查 ID),三级候选可用,合规符合 → 图片对比,有货即停。
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", true, true);
assertEquals(1, result.size());
SimilarAsinResultRowDto out = result.get(0);
assertEquals("成功", out.getStatus());
assertEquals("有货", out.getIsStock());
assertEquals("95%", out.getSimilarity());
assertEquals("符合", out.getIsConform());
}
}
@@ -0,0 +1,204 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 9:chunk 查询从单行分页改为批量 keyset 分页,保持低内存读取。
* loadChunksKeyset 按 id 递增分批拉取(每批 pageSize),最后按 chunkIndex 升序合并,
* 避免超大任务一次 selectList 全量载入 chunk 元数据。
* mock 分页由 wrapper 中 gt("id", lastId) 的 keyset 值驱动,保证重复调用可复现(幂等)。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceChunkKeysetTest {
private static final String MODULE = "similar-asin";
@Mock private TaskChunkMapper taskChunkMapper;
private static TaskChunkEntity chunk(long id, int chunkIndex) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7004L);
chunk.setModuleType(MODULE);
chunk.setChunkIndex(chunkIndex);
return chunk;
}
private static List<TaskChunkEntity> chunks(long... idsAndIndexes) {
List<TaskChunkEntity> result = new ArrayList<>();
for (int i = 0; i < idsAndIndexes.length; i += 2) {
result.add(chunk(idsAndIndexes[i], (int) idsAndIndexes[i + 1]));
}
return result;
}
/** 从 wrapper 的 SQL 片段解析 keyset:匹配 "id > #{ew.paramNameValuePairs.键}" 后从参数表取值。 */
private static long keysetOf(QueryWrapper<TaskChunkEntity> wrapper) {
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("id\\s*>\\s*#\\{ew\\.paramNameValuePairs\\.(\\w+)\\}", java.util.regex.Pattern.CASE_INSENSITIVE)
.matcher(wrapper.getSqlSegment());
if (m.find()) {
Object value = wrapper.getParamNameValuePairs().get(m.group(1));
if (value instanceof Number number) {
return number.longValue();
}
}
return 0L;
}
/** 按 keyset 驱动分页:每次 selectList 返回 id > keyset 的下一批,天然支持重复调用。 */
private void stubKeysetPages(List<TaskChunkEntity> all, int pageSize) {
int batch = pageSize > 0 ? pageSize : 500;
doAnswer(invocation -> {
long lastId = keysetOf(invocation.getArgument(0));
return all.stream().filter(c -> c.getId() > lastId).limit(batch).toList();
}).when(taskChunkMapper).selectList(any());
}
private List<Long> keysetIdsFromRounds(int rounds) {
ArgumentCaptor<QueryWrapper<TaskChunkEntity>> captor =
ArgumentCaptor.forClass(QueryWrapper.class);
verify(taskChunkMapper, times(rounds)).selectList(captor.capture());
List<Long> keysets = new ArrayList<>();
for (QueryWrapper<TaskChunkEntity> wrapper : captor.getAllValues()) {
keysets.add(keysetOf(wrapper));
}
return keysets;
}
@Test
void test_task_009_chunk_normal_default_path() {
// 正常输入:chunk 数小于 pageSize,一轮拉完,结果全且按 chunkIndex 有序
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
stubKeysetPages(all, 500);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
assertEquals(3, result.size());
assertEquals(List.of(1, 2, 3), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
assertEquals(List.of(0L), keysetIdsFromRounds(1), "首轮 keyset 为 0");
}
@Test
void test_task_009_chunk_normal_multiple_items() {
// 超过 pageSize:多轮拉取,keyset 逐轮推进,全部合并且按 chunkIndex 升序、无重复
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7);
stubKeysetPages(all, 3);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
assertEquals(7, result.size());
assertEquals(List.of(1, 2, 3, 4, 5, 6, 7), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
long distinctIds = result.stream().map(TaskChunkEntity::getId).distinct().count();
assertEquals(7, distinctIds, "keyset 分页不能产生重复 chunk");
assertEquals(List.of(0L, 3L, 6L), keysetIdsFromRounds(3), "keyset 逐轮推进,不足一批即止");
}
@Test
void test_task_009_chunk_normal_repeated_operation_is_idempotent() {
// 重复执行同一输入:每轮都从 keyset=0 开始,结果一致
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
stubKeysetPages(all, 500);
List<TaskChunkEntity> first = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
List<TaskChunkEntity> second = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getId(), second.get(i).getId());
}
assertEquals(List.of(0L, 0L), keysetIdsFromRounds(2), "重复执行每轮都从 keyset=0 开始");
}
@Test
void test_task_009_chunk_boundary_empty_input() {
// 空集合:返回空列表,不创建无效资源,且只查一轮
stubKeysetPages(List.of(), 500);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
assertNotNull(result);
assertEquals(0, result.size());
verify(taskChunkMapper, times(1)).selectList(any());
}
@Test
void test_task_009_chunk_boundary_single_item() {
// 单 chunk:一轮返回后 keyset 推进即拉空,不依赖批量路径
List<TaskChunkEntity> all = chunks(42, 9);
stubKeysetPages(all, 1);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 1);
assertEquals(1, result.size());
assertEquals(9, result.get(0).getChunkIndex());
assertEquals(42L, result.get(0).getId());
}
@Test
void test_task_009_chunk_boundary_limit_and_overflow() {
// chunk 数恰好等于 pageSize 的倍数:最后一轮仍返回非空才继续,全部取回
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6);
stubKeysetPages(all, 3);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
assertEquals(6, result.size());
// pageSize 为 0/负数:回退默认 500,不抛异常
stubKeysetPages(all, 0);
List<TaskChunkEntity> fallback = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 0);
assertEquals(6, fallback.size());
stubKeysetPages(all, -5);
List<TaskChunkEntity> negative = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, -5);
assertEquals(6, negative.size());
}
@Test
void test_task_009_chunk_invalid_input_rejected() {
// taskId 为 null:安全返回空列表,不发起查询
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, null, MODULE, 500);
assertNotNull(result);
assertEquals(0, result.size());
verify(taskChunkMapper, times(0)).selectList(any());
// mapper 查询抛异常:转项目约定异常,消息可识别
when(taskChunkMapper.selectList(any())).thenThrow(new IllegalStateException("db down"));
BusinessException ex = assertThrows(BusinessException.class,
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
"异常消息必须可识别,实际: " + ex.getMessage());
}
@Test
void test_task_009_chunk_dependency_failure_releases_resources() {
// 第二轮查询失败:抛异常不返回半截结果;恢复后重试可完整返回
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5);
doAnswer(invocation -> {
long lastId = keysetOf(invocation.getArgument(0));
if (lastId == 0L) {
return all.subList(0, 3);
}
if (lastId == 3L) {
throw new IllegalStateException("db down mid-page");
}
return all.subList(3, 5);
}).when(taskChunkMapper).selectList(any());
BusinessException ex = assertThrows(BusinessException.class,
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
"异常消息必须可识别,实际: " + ex.getMessage());
// 恢复后重试成功:5 个 chunk 全部取回
stubKeysetPages(all, 3);
List<TaskChunkEntity> recovered = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
assertEquals(5, recovered.size());
assertEquals(List.of(1, 2, 3, 4, 5), recovered.stream().map(TaskChunkEntity::getChunkIndex).toList());
}
}
@@ -0,0 +1,399 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 13chunk 合并增加单次最大行数与 payload 字节上限。
* 校验点位于 mergeChunkPayload 单次合并入口:合并后总行数超过 chunkMergeMaxRows、
* 或 payload 字节超过 chunkMergePayloadMaxBytes 时,从最旧行开始降级到
* orphan 兜底(assemble 阶段 putIfAbsent 合并回结果,不丢数据);
* 单行本身超过字节上限时抛可识别异常拒绝合并。低于上限的行为与旧路径完全一致。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceChunkMergeLimitTest {
private static final AtomicLong NEXT_ID = new AtomicLong(90000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(16L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/90000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String asin, String title) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(rowToken);
r.setAsin(asin);
r.setCountry("英国");
r.setTitle(title);
return r;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(9004L);
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(payloadJson);
return chunk;
}
/** 记录每次 storeChunkPayloadVersioned 收到的 payload 字符串。 */
private void stubChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicReference<String> storedPayload) throws Exception {
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
.thenReturn(payloadJson);
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storedPayload.set(invocation.getArgument(4));
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
}
private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task,
String scopeHash, Integer chunkIndex,
List<SimilarAsinResultRowDto> llmRows) throws Exception {
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, scopeHash, chunkIndex, llmRows, Map.of());
}
@Test
void test_task_013_payload_row_count_chunk_normal_default_path() throws Exception {
// 正常输入:行数与 payload 字节均在上限内,合并走原路径,结果完整保留。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicReference<String> storedPayload = new AtomicReference<>("");
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> llmRows = List.of(
row("r1", "B0A0000001", "标题1"),
row("r2", "B0A0000002", "标题2"),
row("r3", "B0A0000003", "标题3"));
invokeMerge(service, task, "hashA", 1, llmRows);
assertNotNull(storedPayload.get());
assertTrue(storedPayload.get().contains("\"r1\"") && storedPayload.get().contains("\"r3\""),
"上限内合并必须完整保留存量行与新增行,实际: " + storedPayload.get());
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(1)).update(any(), any());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_normal_multiple_items() throws Exception {
// 批量场景:多 chunk 一次 merge,全部在上限内,各 chunk 分别写回、结果不丢失。
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r1", "B0A0000001", "标题1"))));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
.thenReturn(rowsJson(List.of(row("r2", "B0A0000002", "标题2"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
AtomicLong selectOneRound = new AtomicLong(0);
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, null, null, List.of(
row("r1", "B0A0000001", "标题1-新"),
row("r2", "B0A0000002", "标题2-新")));
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(2)).update(any(), any());
}
@Test
void test_task_013_payload_row_count_chunk_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:每次 merge 恰好写一次,不产生重复对象、重复状态。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storeCalls = new AtomicLong(0);
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storeCalls.incrementAndGet();
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "B0A0000001", "标题1"));
invokeMerge(service, task, "hashA", 1, llmRows);
invokeMerge(service, task, "hashA", 1, llmRows);
assertEquals(2, storeCalls.get(), "重复执行同一输入:每次 merge 恰好写回一次,无多余请求");
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_boundary_empty_input() throws Exception {
// 空输入:null/空列表安全跳过,不读取 chunk、不写存储、不创建资源。
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, "hashA", 1, null);
invokeMerge(service, task, "hashA", 1, List.of());
verify(taskChunkMapper, times(0)).selectList(any());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_boundary_single_item() throws Exception {
// 单行:不依赖批量路径,合并后结果正确。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicReference<String> storedPayload = new AtomicReference<>("");
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
assertTrue(storedPayload.get().contains("\"r1\""), "单行合并也必须写回 chunk payload,实际: " + storedPayload.get());
verify(taskChunkMapper, times(1)).update(any(), any());
}
@Test
void test_task_013_payload_row_count_chunk_boundary_limit_and_overflow() throws Exception {
// 超限场景三连:行数超限降级、字节超限降级、单行超字节上限拒绝。
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(2);
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storeCalls = new AtomicLong(0);
AtomicReference<String> lastStoredPayload = new AtomicReference<>("");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storeCalls.incrementAndGet();
lastStoredPayload.set(invocation.getArgument(4));
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 0; i < 5; i++) {
llmRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
}
// Phase A:行数超限(上限 2,存量 1 + 新增 5)→ 只保留上限内最新行,超限部分转 orphan。
invokeMerge(service, task, "hashA", 1, llmRows);
assertTrue(lastStoredPayload.get().contains("\"r4\"") && lastStoredPayload.get().contains("\"r5\""),
"行数超限时保留上限内的最新行,实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "行数超限时最旧行被降级,实际: " + lastStoredPayload.get());
assertEquals(1, storeCalls.get());
verify(taskScopeStateMapper, times(1)).insert(any(TaskScopeStateEntity.class));
verify(transientPayloadStorageService, times(1))
.storeParsedPayloadEntry(anyString(), any(), anyString(), anyString(), anyString(), eq(true));
// Phase B:字节超限(行数放开)→ 从最旧行降级到字节上限内,保留最新结果。
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
long oneRowBytes = rowsJson(List.of(row("r9", "B0A0000099", "样本行"))).getBytes(StandardCharsets.UTF_8).length;
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(oneRowBytes + 5L);
invokeMerge(service, task, "hashA", 1, llmRows);
assertTrue(lastStoredPayload.get().contains("\"r5\""), "字节超限时保留最新行,实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "字节超限时最旧行被降级,实际: " + lastStoredPayload.get());
assertEquals(2, storeCalls.get());
verify(taskScopeStateMapper, times(2)).insert(any(TaskScopeStateEntity.class));
// Phase C:单行本身超过字节上限 → 抛可识别异常拒绝合并,不写 chunk。
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(5L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, llmRows);
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertNotNull(ex.getMessage());
assertTrue(ex.getMessage().contains("字节上限"),
"超字节上限必须抛可识别异常,实际: " + ex.getMessage());
assertEquals(2, storeCalls.get(), "拒绝合并时不写 chunk");
}
@Test
void test_task_013_payload_row_count_chunk_invalid_input_rejected() {
// 非法输入:chunk 载荷加载失败(resolve 抛异常)时抛出可识别异常且不写 chunk。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenThrow(new IllegalStateException("rustfs down"));
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertNotNull(ex.getMessage());
assertTrue(ex.getMessage().contains("chunk"),
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_dependency_failure_releases_resources() throws Exception {
// 依赖失败:payload 存储失败时抛带上下文的可识别异常、无残留状态;恢复后重试成功。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
AtomicLong storeCalls = new AtomicLong(0);
doAnswer(invocation -> {
if (storeCalls.getAndIncrement() == 0) {
throw new IllegalStateException("rustfs down");
}
return "stored:" + invocation.getArgument(2);
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertTrue(ex.getMessage() != null && ex.getMessage().contains("相似ASIN分片载荷"),
"存储失败必须抛带上下文的可识别异常,实际: " + ex.getMessage());
assertEquals(1, storeCalls.get(), "失败时只尝试一次即抛出,不静默吞错");
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
// 依赖恢复后重试成功:结果正确、无残留状态。
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
assertEquals(2, storeCalls.get(), "恢复后重试成功");
verify(taskChunkMapper, times(1)).update(any(), any());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
}
@@ -0,0 +1,416 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 12:扩展 Coze 结果缓冲覆盖范围,减少频繁读写完整 chunk payload。
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1"Task 12 扩展为:
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
* 3) 统一走 bufferLlmRowsOrMerge:缓冲失败回退立即 merge,结果不丢失。
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceCozeBufferScopeTest {
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
private static final String CREDENTIAL = "cred-1";
@Mock private LocalFileStorageService localFileStorageService;
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinLlmService similarAsinLlmService;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() throws Exception {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(properties.getLlmBatchSize()).thenReturn(5);
lenient().when(properties.getLlmTextOnlyBatchSize()).thenReturn(10);
lenient().when(properties.isLlmResultBufferEnabled()).thenReturn(true);
lenient().when(properties.getDbJobTouchIntervalMillis()).thenReturn(2_000L);
lenient().when(properties.getDbTaskTouchIntervalMillis()).thenReturn(2_000L);
lenient().when(properties.getLlmFlushPendingMinutes()).thenReturn(10);
lenient().when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(
mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
.thenAnswer(invocation -> "rustfs:coze-result/" + NEXT_ID.incrementAndGet());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country, String title) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(id);
r.setAsin(asin);
r.setCountry(country);
r.setTitle(title);
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
return r;
}
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7104L);
chunk.setModuleType(MODULE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(payloadJson);
chunk.setPayloadHash("h-" + id);
return chunk;
}
private static FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(7104L);
task.setModuleType(MODULE);
task.setStatus("RUNNING");
task.setResultJson("{\"categorySwitch\":true}");
return task;
}
private static TaskScopeStateEntity state(FileTaskEntity task, long id, String status, int batchTotal) {
TaskScopeStateEntity state = new TaskScopeStateEntity();
state.setId(id);
state.setTaskId(task.getId());
state.setModuleType(MODULE);
state.setScopeHash("scope-" + id);
state.setLlmStatus(status);
state.setParsedPayloadJson("ptr:batch-" + id);
state.setStateJson("{\"jobId\":7101,\"resultId\":7201,\"chunkScopeHash\":null,\"chunkIndex\":null,"
+ "\"batchIndex\":1,\"batchTotal\":" + batchTotal + ",\"ownerInstanceId\":\"test-instance\","
+ "\"submitRetryCount\":0,\"credentialName\":\"" + CREDENTIAL + "\",\"resultPayloadPointer\":\"ptr:buffer-" + id + "\"}");
return state;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private SimilarAsinTaskService.LlmBatchContext context(int batchTotal) {
return new SimilarAsinTaskService.LlmBatchContext(
7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null);
}
private void stubChunkMerge(String payloadJson) throws Exception {
TaskChunkEntity chunk = chunk(1L, "scope-1", 1, "ptr:chunk-1");
// loadSubmittedChunks 只保留非空 chunkchunk payload 必须能解析出至少一行。
// 全部 lenient:缓冲成功路径不触达 merge,仅缓冲失败/flush 合并路径消费。
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
lenient().when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
return payloadJson;
}
return "[]";
});
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
}
private static String chunkRowsJson() throws Exception {
// chunk-1 已含 r1 行:loadSubmittedChunks 只保留非空 chunk,且 rowKey 索引能命中缓冲行。
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
}
@Test
void test_task_012_payload_chunk_normal_default_path() throws Exception {
// 正常输入:DONE 结果(batchTotal=1 单 batch)经 bufferLlmRowsOrMerge 走缓冲,
// 不立即写 chunk;缓冲失败回退立即 merge 结果不丢失。
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_normal_multiple_items() throws Exception {
// 批量场景:多个 DONE state(单 batch)全部缓冲;flush 后按 chunk 分组一次合并
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
when(taskScopeStateMapper.selectList(any())).thenReturn(
List.of(state(task, 1L, "DONE", 1), state(task, 2L, "DONE", 1)));
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
}
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
return chunkRowsJson();
}
return "[]";
});
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(1L, "scope-1", 1, "ptr:chunk-1"));
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1")));
when(taskChunkMapper.update(any(), any())).thenReturn(1);
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
flush.setAccessible(true);
flush.invoke(service, 7104L);
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
// pointer 清理:每个缓冲 state 都更新
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:缓冲写幂等(同一 state 不产生重复 buffer/merge
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
TaskScopeStateEntity state = state(task, 1L, "DONE", 2);
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
// 缓冲 2 次(每次重新写 pointer 是幂等语义:同一 state 覆盖写,无重复行)
verify(transientPayloadStorageService, times(2)).storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_boundary_empty_input() throws Exception {
// 空输入:无行时缓冲与 merge 都不发生,不创建无效资源
FileTaskEntity task = task();
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of());
bufferOrMerge.invoke(service, state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of());
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_boundary_single_item() throws Exception {
// 单 batchbatchTotal=1):原 P0-3 例外,现在也缓冲
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_boundary_limit_and_overflow() throws Exception {
// 缓冲开关关闭:回退立即 merge,DONE 结果仍落 chunk 不丢失
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
when(properties.isLlmResultBufferEnabled()).thenReturn(false);
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
@Test
void test_task_012_payload_chunk_invalid_input_rejected() throws Exception {
// 缓冲写失败(storeParsedPayloadEntry 抛异常):回退立即 merge,结果不丢失
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
when(transientPayloadStorageService.storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)))
.thenThrow(new IllegalStateException("rustfs full"));
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_dependency_failure_releases_resources() throws Exception {
// flush 时 chunk 写失败:抛可识别业务异常且不清 pointer(保留待重试);
// 依赖恢复后重试 flush 成功,chunk 合并一次、pointer 清理。
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
TaskScopeStateEntity s = state(task, 1L, "DONE", 1);
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(s));
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
}
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
return chunkRowsJson();
}
return "[]";
});
java.util.concurrent.atomic.AtomicInteger storeCalls = new java.util.concurrent.atomic.AtomicInteger(0);
doAnswer(invocation -> {
if (storeCalls.incrementAndGet() == 1) {
throw new IllegalStateException("rustfs write failed");
}
return "ptr:stored-" + invocation.getArgument(3);
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
flush.setAccessible(true);
Exception ex = assertThrows(Exception.class, () -> {
try {
flush.invoke(service, 7104L);
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertTrue(ex.getMessage() != null && ex.getMessage().contains("刷新缓冲区"),
"flush 失败消息必须可识别, 实际: " + ex.getMessage());
// 失败分组不清 pointerbuffer 未被删除、stateJson 未更新,留待重试
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
verify(taskScopeStateMapper, never()).update(any(), any());
// 恢复后重试 flushchunk 合并成功一次,pointer 清理
flush.invoke(service, 7104L);
assertEquals(2, storeCalls.get(), "恢复后重试应再次写 chunk");
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
}
}
@@ -0,0 +1,428 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 6:分组数据改为索引/范围引用,避免 groups 嵌套复制完整行对象。
* 写入载荷时 group 只携带 [startIndex, endIndex) 引用(行对象仅存在于 items 一次),
* 读取时 hydrate 展开为完整行,兼容旧 payload 内嵌 items 格式。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceGroupRefTest {
private static final AtomicLong NEXT_ID = new AtomicLong(30000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/30000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
File file = Files.createTempFile("similar-asin-group-ref-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(String.format("B0GRP%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("group-ref.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
private String storedPayloadJson() {
// 捕获最近一次存储的 payload JSON
return "rustfs:task-parsed/similar-asin/30000/payload.json";
}
private SimilarAsinParsedPayloadDto readPayload(String json) throws Exception {
return objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class);
}
private static SimilarAsinParsedRowVo row(String fileKey, int index, String groupKey) {
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
row.setSourceFileKey(fileKey);
row.setSourceFilename("group-ref.xlsx");
row.setRowIndex(index);
row.setSourceId(String.valueOf(index));
row.setDisplayId(String.valueOf(index));
row.setRowToken(fileKey + "::row::" + index);
row.setGroupKey(groupKey);
row.setAsin(String.format("B0GRP%05d", index));
row.setCountry("英国");
row.setValues(new java.util.LinkedHashMap<>());
return row;
}
@Test
void test_task_006_group_normal_default_path() throws Exception {
// 正常多行文件:groups 写入为索引引用,行对象只出现在 items 一次
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenAnswer(invocation -> {
String json = invocation.getArgument(3);
return "rustfs:task-parsed/similar-asin/30000/payload.json::" + json;
});
File workbook = buildWorkbook(150);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-default.xlsx");
assertEquals(150, vo.getAcceptedRows());
// 每个 group 是索引引用:携带 [startIndex, endIndex),区间宽度等于 itemCount
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertNotNull(group.getStartIndex());
assertNotNull(group.getEndIndex());
assertTrue(group.getStartIndex() < group.getEndIndex());
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItemCount());
}
// 响应 groups 按预览上限裁剪(默认 100),引用区间覆盖全部行、不重叠
int coverage = 0;
int prevEnd = -1;
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertTrue(group.getStartIndex() >= prevEnd, "组区间不能重叠且必须顺序递增");
coverage += group.getEndIndex() - group.getStartIndex();
prevEnd = group.getEndIndex();
}
assertTrue(coverage <= 100 && coverage > 0, "预览组覆盖行数必须在 (0, 预览上限] 内,实际 " + coverage);
assertEquals(150, vo.getAcceptedRows());
// 响应组内嵌预览行(前端兼容):每个组 items 与引用区间宽度一致
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertNotNull(group.getItems());
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItems().size(),
"响应组内嵌预览行数量必须与引用区间宽度一致");
}
}
@Test
void test_task_006_group_normal_multiple_items() throws Exception {
// 多组批量:每组行数不同,引用与 items 严格对应且顺序稳定
String json = groupRefJson(3, new int[][]{{0, 3}, {3, 8}, {8, 10}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
assertEquals(10, payload.getItems().size());
assertEquals(3, payload.getGroups().size());
for (int g = 0; g < payload.getGroups().size(); g++) {
SimilarAsinParsedGroupVo group = payload.getGroups().get(g);
int start = group.getStartIndex();
int end = group.getEndIndex();
assertTrue(end - start >= 1);
// 展开后行与 items 对应(首行即 items[start],行内容一致)
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(start, end));
assertEquals(end - start, expanded.size());
assertEquals("t" + (start + 1), expanded.get(0).getRowToken(), "展开首行必须是 items[start]");
assertEquals("B0GRP" + String.format("%05d", start + 1), expanded.get(0).getAsin());
}
}
@Test
void test_task_006_group_normal_repeated_operation_is_idempotent() throws Exception {
// 重复展开同一 payload:结果一致,且不修改 items
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 5}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
List<SimilarAsinParsedRowVo> first = hydrateForTest(payload);
List<SimilarAsinParsedRowVo> second = hydrateForTest(payload);
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
}
assertEquals(5, payload.getItems().size(), "展开不能修改 payload 内部状态");
}
@Test
void test_task_006_group_boundary_empty_input() throws Exception {
// 空 groups:引用列表为空,不创建无效引用
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setItems(List.of());
payload.setGroups(List.of());
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
assertNotNull(restored);
assertEquals(0, restored.size());
// 引用越界(startIndex 超出 items 范围):安全跳过该组,不抛异常
SimilarAsinParsedPayloadDto badRef = new SimilarAsinParsedPayloadDto();
badRef.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setGroupKey("f.xlsx::1");
group.setStartIndex(5);
group.setEndIndex(7);
badRef.setGroups(List.of(group));
List<SimilarAsinParsedRowVo> outOfRange = SimilarAsinTaskService.resolveAllRows(badRef);
assertEquals(0, outOfRange.size(), "越界引用必须安全跳过");
}
@Test
void test_task_006_group_boundary_single_item() throws Exception {
// 单行单组:区间为 [0,1),单行不依赖批量路径
String json = groupRefJson(1, new int[][]{{0, 1}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
assertEquals(1, payload.getGroups().size());
SimilarAsinParsedGroupVo group = payload.getGroups().get(0);
assertEquals(0, group.getStartIndex());
assertEquals(1, group.getEndIndex());
assertEquals(1, group.getItemCount());
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(0, 1));
assertEquals(1, expanded.size());
assertEquals("B0GRP00001", expanded.get(0).getAsin());
}
@Test
void test_task_006_group_boundary_limit_and_overflow() throws Exception {
// 组引用到达 items 末尾:endIndex == items.size(),不越界
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 6}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
assertEquals(6, payload.getItems().size());
SimilarAsinParsedGroupVo last = payload.getGroups().get(1);
assertEquals(6, last.getEndIndex());
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(last.getStartIndex(), last.getEndIndex()));
assertEquals(4, expanded.size());
// 未携带 items 的旧 payload 走 allItems 兜底
String legacy = "{\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0OLD00001\"}],"
+ "\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":1,\"itemCount\":1}]}";
SimilarAsinParsedPayloadDto legacyPayload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class);
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacyPayload);
assertEquals(1, restored.size());
assertEquals("B0OLD00001", restored.get(0).getAsin());
}
@Test
void test_task_006_group_invalid_input_rejected() throws Exception {
// 非法区间:endIndex <= startIndex,安全跳过
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1"), row("f.xlsx", 2, "f.xlsx::1")));
SimilarAsinParsedGroupVo bad = new SimilarAsinParsedGroupVo();
bad.setGroupKey("f.xlsx::1");
bad.setStartIndex(1);
bad.setEndIndex(1);
payload.setGroups(List.of(bad));
assertEquals(0, SimilarAsinTaskService.resolveAllRows(payload).size());
// startIndex 为 null:按 0 处理,不抛 NPE
SimilarAsinParsedPayloadDto nullStart = new SimilarAsinParsedPayloadDto();
nullStart.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
SimilarAsinParsedGroupVo g = new SimilarAsinParsedGroupVo();
g.setGroupKey("f.xlsx::1");
g.setStartIndex(null);
g.setEndIndex(1);
nullStart.setGroups(List.of(g));
assertEquals(1, SimilarAsinTaskService.resolveAllRows(nullStart).size());
}
@Test
void test_task_006_group_dependency_failure_releases_resources() throws Exception {
// RustFS 存储失败:解析抛异常;恢复后重试成功,groups 引用与行一致
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(80);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/gr-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/30001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/gr-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-recovered.xlsx");
assertEquals(80, vo.getAcceptedRows());
int coverage = 0;
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertTrue(group.getStartIndex() < group.getEndIndex());
coverage += group.getEndIndex() - group.getStartIndex();
}
assertEquals(80, coverage);
}
@Test
void test_task_006_group_payload_embeds_items_for_python() throws Exception {
// Python 旧链路(normalize_groups 直接返回 groups 并按 gp["items"] 取行):
// 写入存储的 payload 中每组必须内嵌完整行,startIndex/endIndex 保留供 Java 内部展开。
java.util.concurrent.atomic.AtomicReference<String> captured = new java.util.concurrent.atomic.AtomicReference<>();
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenAnswer(invocation -> {
captured.set(invocation.getArgument(3));
return "rustfs:task-parsed/similar-asin/30000/payload.json";
});
File workbook = buildWorkbook(12);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-python.xlsx");
assertEquals(12, vo.getAcceptedRows());
SimilarAsinParsedPayloadDto stored = objectMapper.readValue(captured.get(), SimilarAsinParsedPayloadDto.class);
assertNotNull(stored.getGroups());
assertEquals(12, stored.getGroups().size());
// 每组内嵌 items 与引用区间宽度一致,且行内容对应 items[startIndex]
for (SimilarAsinParsedGroupVo group : stored.getGroups()) {
assertNotNull(group.getItems());
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItems().size(),
"Python 兼容:组内嵌 items 必须与引用区间宽度一致");
int start = group.getStartIndex();
assertEquals(stored.getItems().get(start).getRowToken(), group.getItems().get(0).getRowToken(),
"组 items 首行必须是 items[startIndex]");
}
// Java 内部展开路径不受内嵌 items 影响:仍按引用切片
List<SimilarAsinParsedRowVo> expanded = SimilarAsinTaskService.expandGroupRefs(stored);
assertEquals(12, expanded.size(), "Java 内部展开仍按引用,不受内嵌 items 影响");
}
@Test
void test_task_006_group_hydrate_embeds_items_for_legacy_payload() throws Exception {
// 存量任务 payload 只有 startIndex/endIndex 引用(无内嵌 items):
// parsedPayload 读取时 hydrate 必须回填组内嵌 itemsPython 才能取到行。
String legacy = "{\"items\":["
+ "{\"rowToken\":\"t1\",\"asin\":\"B0GRP00001\",\"sourceFileKey\":\"f.xlsx\",\"country\":\"英国\"},"
+ "{\"rowToken\":\"t2\",\"asin\":\"B0GRP00002\",\"sourceFileKey\":\"f.xlsx\",\"country\":\"英国\"}"
+ "],\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":2,\"itemCount\":2}]}";
SimilarAsinParsedPayloadDto payload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class);
SimilarAsinParsedPayloadDto hydrated = SimilarAsinTaskService.hydrateForPythonCompat(payload);
assertEquals(1, hydrated.getGroups().size());
assertNotNull(hydrated.getGroups().get(0).getItems());
assertEquals(2, hydrated.getGroups().get(0).getItems().size(), "hydrate 后组内嵌 items 必须为 2 行");
assertEquals("B0GRP00001", hydrated.getGroups().get(0).getItems().get(0).getAsin(),
"hydrate 后组内嵌 items 首行必须对应 items[0]");
assertEquals("B0GRP00002", hydrated.getGroups().get(0).getItems().get(1).getAsin(),
"hydrate 后组内嵌 items 第二行必须对应 items[1]");
}
// ---- helpers ----
private static String groupRefJson(int groupCount, int[][] ranges) {
StringBuilder sb = new StringBuilder();
sb.append("{\"items\":[");
// 计算总行数
int maxEnd = 0;
for (int[] r : ranges) {
maxEnd = Math.max(maxEnd, r[1]);
}
for (int i = 1; i <= maxEnd; i++) {
if (i > 1) {
sb.append(",");
}
sb.append("{\"rowToken\":\"t").append(i).append("\",\"asin\":\"B0GRP")
.append(String.format("%05d", i)).append("\",\"sourceFileKey\":\"f.xlsx\",\"rowIndex\":")
.append(i).append("}");
}
sb.append("],\"groups\":[");
for (int g = 0; g < groupCount; g++) {
if (g > 0) {
sb.append(",");
}
sb.append("{\"groupKey\":\"g").append(g + 1).append("\",\"startIndex\":")
.append(ranges[g][0]).append(",\"endIndex\":").append(ranges[g][1])
.append(",\"itemCount\":").append(ranges[g][1] - ranges[g][0]).append("}");
}
sb.append("]}");
return sb.toString();
}
private static List<SimilarAsinParsedRowVo> hydrateForTest(SimilarAsinParsedPayloadDto payload) {
// 调用 service 的引用展开实现(与 hydrateParsedPayloadRows 语义一致)
SimilarAsinParsedPayloadDto copy = new SimilarAsinParsedPayloadDto();
copy.setItems(payload.getItems());
copy.setAllItems(payload.getAllItems());
copy.setGroups(payload.getGroups());
return SimilarAsinTaskService.expandGroupRefs(copy);
}
}
@@ -0,0 +1,359 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinImagePrefetchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 任务 105similarasin 历史列表 IN 查询批量加载。
* history() 的任务/结果/Job 关联数据必须走 IN 批量查询 + 按 ID Map 装配,
* 不允许逐条 N+1:结果 1 次、任务 1 次、Job 1 次(恒定的 3 次查询,与结果行数无关);
* 分页、排序、过滤、输出与现状一致。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceHistoryBatchTest {
private static final String MODULE = "SIMILAR_ASIN";
private final List<FileResultEntity> resultDb = new ArrayList<>();
private final List<FileTaskEntity> taskDb = new ArrayList<>();
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
private final AtomicInteger resultSelectCount = new AtomicInteger();
private final AtomicInteger taskSelectCount = new AtomicInteger();
private final AtomicInteger jobSelectCount = new AtomicInteger();
private final List<Long> lastResultTaskIds = new ArrayList<>();
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper();
@Mock private SimilarAsinLlmService similarAsinLlmService;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
List<Object> values = new ArrayList<>();
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
Map<String, Object> params = q.getParamNameValuePairs();
while (m.find()) {
Object value = params.get(m.group(1));
if (value != null) {
values.add(value);
}
}
return values;
}
@BeforeEach
void setUp() throws Exception {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity.class);
// 结果表:按 wrapper 的 user/module/limit 过滤并保持 createdAt 倒序
doAnswer(invocation -> {
resultSelectCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
List<FileResultEntity> filtered = new ArrayList<>(resultDb);
Long userId = params.stream().filter(Long.class::isInstance).map(Long.class::cast).findFirst().orElse(null);
if (userId != null) {
filtered.removeIf(r -> !userId.equals(r.getUserId()));
}
if (segment.contains("module_type")) {
filtered.removeIf(r -> !MODULE.equals(r.getModuleType()));
}
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
lastResultTaskIds.clear();
lastResultTaskIds.addAll(filtered.stream().map(FileResultEntity::getTaskId).toList());
if (segment.contains("limit")) {
int cap = 50;
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
if (m.find()) {
cap = Integer.parseInt(m.group(1));
}
return filtered.subList(0, Math.min(cap, filtered.size()));
}
return filtered;
}).when(fileResultMapper).selectList(any());
// 任务表:IN 查询返回 id 命中集合(IN 值来自上一轮结果行的 taskId)
doAnswer(invocation -> {
taskSelectCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
List<Long> wanted = new ArrayList<>();
for (Object value : params) {
if (value instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Long id) {
wanted.add(id);
}
}
}
}
if (wanted.isEmpty()) {
wanted.addAll(lastResultTaskIds);
}
if (wanted.isEmpty()) {
return List.of();
}
return taskDb.stream().filter(t -> wanted.contains(t.getId())).toList();
}).when(fileTaskMapper).selectList(any());
lenient().when(taskFileJobService.findAssembleJobsByResultIds(eq(MODULE), anyList())).thenAnswer(invocation -> {
jobSelectCount.incrementAndGet();
List<Long> resultIds = invocation.getArgument(1);
Map<Long, TaskFileJobEntity> map = new HashMap<>();
if (resultIds != null) {
for (TaskFileJobEntity job : jobDb) {
if (job.getResultId() != null && resultIds.contains(job.getResultId())) {
map.put(job.getResultId(), job);
}
}
}
return map;
});
lenient().when(ossStorageService.generateFreshDownloadUrl(anyString())).thenAnswer(inv -> "https://oss/" + inv.getArgument(0));
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
lenient().when(instanceMetadata.getInstanceId()).thenReturn("server-110");
}
private static FileResultEntity result(Long id, Long taskId, Long userId, LocalDateTime createdAt) {
FileResultEntity row = new FileResultEntity();
row.setId(id);
row.setTaskId(taskId);
row.setUserId(userId);
row.setModuleType(MODULE);
row.setSourceFilename("s" + id + ".xlsx");
row.setResultFilename("s" + id + "-result.xlsx");
row.setResultFileUrl("result/similar-asin/" + id + "/out.xlsx");
row.setSuccess(1);
row.setRowCount(3);
row.setCreatedAt(createdAt);
return row;
}
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType(MODULE);
task.setStatus(status);
task.setCreatedAt(createdAt);
task.setFinishedAt(createdAt.plusMinutes(5));
return task;
}
private static TaskFileJobEntity job(Long id, Long resultId, String status) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setResultId(resultId);
job.setModuleType(MODULE);
job.setStatus(status);
return job;
}
private void seed(int rows, int tasks, int jobs) {
for (int i = 1; i <= tasks; i++) {
taskDb.add(task(100L + i, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, i)));
}
for (int i = 1; i <= rows; i++) {
resultDb.add(result(200L + i, 100L + (i % tasks) + 1, 1L, LocalDateTime.of(2026, 8, 1, 10, i)));
}
for (int i = 1; i <= jobs; i++) {
jobDb.add(job(300L + i, 200L + i, "SUCCESS"));
}
}
@Test
void historyBatchLoads() throws Exception {
seed(5, 3, 3);
SimilarAsinHistoryVo vo = service.history(1L, 50);
assertEquals(5, vo.getItems().size(), "5 条结果全部装配");
for (SimilarAsinHistoryItemVo item : vo.getItems()) {
assertNotNull(item.getTaskStatus(), "task 状态从批量任务 Map 装配");
assertEquals("SUCCESS", item.getFileStatus(), "job 状态从批量 Job Map 装配");
assertTrue(item.getFileReady(), "结果文件就绪");
}
}
@Test
void historySingleRow() throws Exception {
seed(1, 1, 1);
SimilarAsinHistoryVo vo = service.history(1L, 50);
assertEquals(1, vo.getItems().size());
assertEquals(201L, vo.getItems().getFirst().getResultId());
assertEquals("SUCCESS", vo.getItems().getFirst().getTaskStatus());
}
@Test
void historyMultipleTasks() throws Exception {
seed(4, 4, 4);
SimilarAsinHistoryVo vo = service.history(1L, 50);
assertEquals(4, vo.getItems().size());
long distinctTasks = vo.getItems().stream().map(SimilarAsinHistoryItemVo::getTaskId).distinct().count();
assertEquals(4, distinctTasks, "多个任务分别按 ID Map 装配");
}
@Test
void historyNoJobs() throws Exception {
seed(3, 3, 0);
SimilarAsinHistoryVo vo = service.history(1L, 50);
assertEquals(3, vo.getItems().size());
for (SimilarAsinHistoryItemVo item : vo.getItems()) {
assertNull(item.getFileJobId(), "无 Job 时不附加 jobId");
assertEquals("SUCCESS", item.getFileStatus(), "文件就绪无 Job 状态为 SUCCESS");
}
}
@Test
void historyOrderUnchanged() throws Exception {
seed(3, 1, 0);
SimilarAsinHistoryVo vo = service.history(1L, 50);
assertEquals(List.of(203L, 202L, 201L),
vo.getItems().stream().map(SimilarAsinHistoryItemVo::getResultId).toList(),
"createdAt 倒序保持");
}
@Test
void historyPagination() throws Exception {
seed(12, 1, 0);
SimilarAsinHistoryVo vo = service.history(1L, 5);
assertEquals(5, vo.getItems().size(), "limit 5 只取前 5");
assertEquals(List.of(212L, 211L, 210L, 209L, 208L),
vo.getItems().stream().map(SimilarAsinHistoryItemVo::getResultId).toList());
}
@Test
void historyMissingRelated() throws Exception {
resultDb.add(result(201L, 9999L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
resultDb.add(result(202L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
SimilarAsinHistoryVo vo = service.history(1L, 50);
assertEquals(2, vo.getItems().size());
assertNull(vo.getItems().getFirst().getTaskStatus(), "关联任务缺失时状态为 null");
assertEquals("SUCCESS", vo.getItems().get(1).getTaskStatus(), "有关联任务的正常装配");
}
@Test
void historyPendingTaskSkipped() throws Exception {
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
resultDb.add(result(202L, 1002L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
taskDb.add(task(1002L, "PENDING", LocalDateTime.of(2026, 8, 1, 9, 1)));
SimilarAsinHistoryVo vo = service.history(1L, 50);
assertEquals(1, vo.getItems().size(), "PENDING 任务结果不进入历史列表");
assertEquals(201L, vo.getItems().getFirst().getResultId());
}
@Test
void historyUserFiltered() throws Exception {
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
resultDb.add(result(202L, 1002L, 2L, LocalDateTime.of(2026, 8, 1, 10, 1)));
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
taskDb.add(task(1002L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1)));
SimilarAsinHistoryVo vo = service.history(2L, 50);
assertEquals(1, vo.getItems().size(), "只返回当前用户结果");
assertEquals(202L, vo.getItems().getFirst().getResultId());
}
@Test
void historySqlCountConstant() throws Exception {
seed(20, 4, 20);
service.history(1L, 50);
verify(fileResultMapper, org.mockito.Mockito.times(1)).selectList(any());
verify(fileTaskMapper, org.mockito.Mockito.times(1)).selectList(any());
verify(taskFileJobService, org.mockito.Mockito.times(1)).findAssembleJobsByResultIds(eq(MODULE), anyList());
assertEquals(1, resultSelectCount.get(), "结果 1 次 IN 查询");
assertEquals(1, taskSelectCount.get(), "任务 1 次 IN 查询");
assertEquals(1, jobSelectCount.get(), "Job 1 次 IN 查询");
assertEquals(20, resultSelectCount.get() * 20, "查询次数与结果行数无关(无 N+1");
}
}
@@ -0,0 +1,446 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
/**
* 任务 109progress/light 轻量接口服务层语义。
* progressLight 只查任务行 + 一次 Job IN 批量查询(不查 result 明细/payload);
* 状态映射与旧 progressBatch 一致;fileStatus/fileReady 由 Job 状态/URL 推导;
* 缺失任务进 missingTaskIds;空/重复/非正数 taskIds 安全处理。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceLightTest {
private static final String MODULE = "SIMILAR_ASIN";
private final List<FileTaskEntity> taskDb = new ArrayList<>();
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
private final AtomicInteger taskSelectCount = new AtomicInteger();
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper();
@Mock private SimilarAsinLlmService similarAsinLlmService;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
List<Object> values = new ArrayList<>();
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
Map<String, Object> params = q.getParamNameValuePairs();
while (m.find()) {
Object value = params.get(m.group(1));
if (value != null) {
values.add(value);
}
}
return values;
}
@BeforeEach
void setUp() throws Exception {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
// 任务表:IN 查询返回 id 命中且模块匹配的 task
lenient().doAnswer(invocation -> {
taskSelectCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
q.getSqlSegment();
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
List<Long> wanted = new ArrayList<>();
for (Object value : params) {
if (value instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Number n) {
wanted.add(n.longValue());
}
}
} else if (value instanceof Number n) {
wanted.add(n.longValue());
}
}
if (wanted.isEmpty()) {
return List.of();
}
return taskDb.stream().filter(t -> wanted.contains(t.getId()) && MODULE.equals(t.getModuleType())).toList();
}).when(fileTaskMapper).selectList(any());
// Job 批量查询:taskId 集合命中
lenient().doAnswer(invocation -> {
List<Long> taskIds = invocation.getArgument(1);
Map<Long, TaskFileJobEntity> map = new HashMap<>();
if (taskIds != null) {
for (TaskFileJobEntity job : jobDb) {
if (job.getTaskId() != null && taskIds.contains(job.getTaskId())) {
map.putIfAbsent(job.getTaskId(), job);
}
}
}
return map;
}).when(taskFileJobService).findAssembleJobsByTaskIds(eq(MODULE), anyList());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
lenient().when(instanceMetadata.getInstanceId()).thenReturn("server-110");
}
private static FileTaskEntity task(Long id, String status, LocalDateTime updatedAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType(MODULE);
task.setStatus(status);
task.setUpdatedAt(updatedAt);
return task;
}
private static TaskFileJobEntity job(Long id, Long taskId, String status, String resultFileUrl) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setTaskId(taskId);
job.setModuleType(MODULE);
job.setStatus(status);
job.setResultFileUrl(resultFileUrl);
return job;
}
private void seed() {
taskDb.add(task(1L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 5)));
taskDb.add(task(2L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 10)));
taskDb.add(task(3L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 15)));
jobDb.add(job(10L, 1L, "RUNNING", null));
jobDb.add(job(11L, 2L, "SUCCESS", "result/similar-asin/2/out.xlsx"));
jobDb.add(job(12L, 3L, "FAILED", null));
}
@Test
void lightReturnsLightFields() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L));
assertEquals(1, vo.getItems().size());
assertEquals(1L, vo.getItems().getFirst().getTaskId());
assertEquals("RUNNING", vo.getItems().getFirst().getStatus());
assertEquals("RUNNING", vo.getItems().getFirst().getFileStatus());
assertFalse(vo.getItems().getFirst().getFileReady());
assertEquals("2026-08-01T10:05", vo.getItems().getFirst().getUpdatedAt());
assertTrue(vo.getMissingTaskIds().isEmpty());
}
@Test
void lightStatusMapping() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus());
assertEquals("SUCCESS", vo.getItems().getFirst().getFileStatus());
assertTrue(vo.getItems().getFirst().getFileReady(), "Job 有 resultFileUrl → fileReady");
}
@Test
void lightMissingTask() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(999L));
assertTrue(vo.getItems().isEmpty());
assertEquals(List.of(999L), vo.getMissingTaskIds());
}
@Test
void lightEmptyIds() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of());
assertTrue(vo.getItems().isEmpty());
assertTrue(vo.getMissingTaskIds().isEmpty());
assertEquals(0, taskSelectCount.get(), "空 ids 不查库");
}
@Test
void lightDedupIds() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L, 1L, 1L));
assertEquals(1, vo.getItems().size(), "重复 id 去重");
}
@Test
void lightUnknownId() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L, 999L));
assertEquals(1, vo.getItems().size());
assertEquals(List.of(999L), vo.getMissingTaskIds(), "未知 id 进缺失列表");
}
@Test
void lightNoResultRowsQueried() {
seed();
service.progressLight(List.of(1L, 2L, 3L));
verify(fileResultMapper, never()).selectList(any());
}
@Test
void lightJobStatusDerived() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(3L));
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus(), "Job FAILED → fileStatus FAILED");
assertFalse(vo.getItems().getFirst().getFileReady(), "无 URL → fileReady false");
}
@Test
void lightTaskMissingJob() {
taskDb.add(task(4L, "PENDING", LocalDateTime.of(2026, 8, 1, 10, 0)));
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(4L));
assertEquals("PENDING", vo.getItems().getFirst().getStatus());
assertNull(vo.getItems().getFirst().getFileStatus(), "无 Job → fileStatus null");
assertFalse(vo.getItems().getFirst().getFileReady());
}
@Test
void lightNegativeAndZeroFiltered() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L, 0L, -5L));
assertEquals(1, vo.getItems().size(), "非正数过滤,不进缺失列表");
assertTrue(vo.getMissingTaskIds().isEmpty());
}
// ===== 任务 111:空/超长 taskIds =====
private static List<Long> ids(int from, int to) {
List<Long> list = new ArrayList<>();
for (long id = from; id <= to; id++) {
list.add(id);
}
return list;
}
private void seedRange(int from, int to) {
for (long id = from; id <= to; id++) {
taskDb.add(task(id, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 0)));
}
}
@Test
void lightNullBody() {
SimilarAsinTaskLightBatchVo vo = service.progressLight(null);
assertTrue(vo.getItems().isEmpty(), "null body → 空 items");
assertTrue(vo.getMissingTaskIds().isEmpty());
assertEquals(0, taskSelectCount.get(), "null 不查库");
}
@Test
void lightNullEntriesFiltered() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(new ArrayList<>(java.util.Arrays.asList(null, 1L, null, 2L, null)));
assertEquals(2, vo.getItems().size(), "列表内 null 条目过滤");
assertTrue(vo.getMissingTaskIds().isEmpty());
}
@Test
void lightSingleId() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
assertEquals(1, vo.getItems().size());
assertEquals(2L, vo.getItems().getFirst().getTaskId());
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus());
}
@Test
void lightMaxExact() {
seedRange(1, 200);
SimilarAsinTaskLightBatchVo vo = service.progressLight(ids(1, 200));
assertEquals(200, vo.getItems().size(), "恰好 200 个 id 全查全返");
assertTrue(vo.getMissingTaskIds().isEmpty());
}
@Test
void lightOver200Truncated() {
seedRange(1, 250);
SimilarAsinTaskLightBatchVo vo = service.progressLight(ids(1, 250));
assertEquals(200, vo.getItems().size(), "250 个 id 截断到 200");
assertEquals(1L, vo.getItems().getFirst().getTaskId());
assertEquals(200L, vo.getItems().getLast().getTaskId(), "返回前 200 个");
assertTrue(vo.getMissingTaskIds().isEmpty(), "截断丢弃的尾部不进 missing");
}
@Test
void lightOver200TailDroppedNotMissing() {
seedRange(1, 250);
SimilarAsinTaskLightBatchVo vo = service.progressLight(ids(1, 250));
assertTrue(vo.getItems().stream().noneMatch(i -> i.getTaskId() > 200L), "第 201+ 个 id 不出现");
assertTrue(vo.getMissingTaskIds().isEmpty(), "截断尾部不进缺失列表");
}
@Test
void lightOver200QueryBounded() {
seedRange(1, 250);
service.progressLight(ids(1, 250));
verify(taskFileJobService).findAssembleJobsByTaskIds(eq(MODULE),
org.mockito.ArgumentMatchers.argThat(list -> list != null && list.size() <= 200));
}
@Test
void lightTruncatePreservesInputOrder() {
seedRange(1, 250);
List<Long> shuffled = new ArrayList<>(ids(1, 250));
java.util.Collections.shuffle(shuffled);
SimilarAsinTaskLightBatchVo vo = service.progressLight(shuffled);
assertEquals(200, vo.getItems().size());
assertEquals(shuffled.get(0), vo.getItems().getFirst().getTaskId(), "截断后保持输入顺序");
assertEquals(shuffled.get(199), vo.getItems().getLast().getTaskId());
}
// ===== 任务 112:终态任务返回 =====
private static TaskFileJobEntity job(Long id, Long taskId, String status, String resultFileUrl, String errorMessage) {
TaskFileJobEntity job = job(id, taskId, status, resultFileUrl);
job.setErrorMessage(errorMessage);
return job;
}
@Test
void lightSuccessTask() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus(), "终态 SUCCESS 原样返回");
assertEquals("SUCCESS", vo.getItems().getFirst().getFileStatus());
assertTrue(vo.getItems().getFirst().getFileReady());
}
@Test
void lightFailedTask() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(3L));
assertEquals("FAILED", vo.getItems().getFirst().getStatus(), "终态 FAILED 原样返回");
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus());
assertFalse(vo.getItems().getFirst().getFileReady());
}
@Test
void lightPending() {
taskDb.add(task(4L, "PENDING", LocalDateTime.of(2026, 8, 1, 10, 0)));
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(4L));
assertEquals("PENDING", vo.getItems().getFirst().getStatus());
assertNull(vo.getItems().getFirst().getFileStatus(), "PENDING 无 Job → fileStatus null");
}
@Test
void lightRunning() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L));
assertEquals("RUNNING", vo.getItems().getFirst().getStatus());
assertEquals("RUNNING", vo.getItems().getFirst().getFileStatus(), "RUNNING 附 file 阶段 RUNNING");
}
@Test
void lightSuccessFileStatus() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L));
assertEquals("SUCCESS", vo.getItems().getFirst().getFileStatus(), "终态 SUCCESS 附 fileStatus SUCCESS");
}
@Test
void lightFailedFileError() {
jobDb.add(job(13L, 5L, "FAILED", null, "组装失败: 文件写入超时"));
taskDb.add(task(5L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 20)));
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(5L));
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus());
assertEquals("组装失败: 文件写入超时", vo.getItems().getFirst().getFileError(), "FAILED 附 fileError 错误信息");
}
@Test
void lightTerminalComplete() {
seed();
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(2L, 3L));
assertEquals(2, vo.getItems().size());
for (SimilarAsinTaskLightVo item : vo.getItems()) {
assertTrue(item.getStatus().equals("SUCCESS") || item.getStatus().equals("FAILED"), "终态字段完整");
assertNotNull(item.getUpdatedAt(), "终态带 updatedAt");
assertNotNull(item.getTaskId());
}
}
@Test
void lightStatusFrozen() {
// 终态值语义不变:任务终态 SUCCESS 不被 file 阶段覆盖(Job 失败只影响 fileStatus/fileReady
taskDb.add(task(6L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 30)));
jobDb.add(job(14L, 6L, "FAILED", null, "job error"));
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(6L));
assertEquals("SUCCESS", vo.getItems().getFirst().getStatus(), "任务状态保持 SUCCESS");
assertEquals("FAILED", vo.getItems().getFirst().getFileStatus(), "file 阶段 FAILED");
assertFalse(vo.getItems().getFirst().getFileReady(), "无 URL → fileReady false");
}
}
@@ -0,0 +1,282 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 任务 89:parse 门面改委托。
* parseAndCreateTask 内部改为 parser.parse → normalizer.normalize → validator → converter → 落库;
* 方法签名一字不变;结果与改造前一致;异常行为一致。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceParseDelegationTest {
private static final AtomicLong NEXT_ID = new AtomicLong(50000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(List<String[]> dataRows) throws Exception {
File file = Files.createTempFile("similar-asin-delegation-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 0; i < dataRows.size(); i++) {
Row row = sheet.createRow(i + 1);
row.createCell(0).setCellValue(dataRows.get(i)[0]);
row.createCell(1).setCellValue(dataRows.get(i)[1]);
row.createCell(2).setCellValue(dataRows.get(i)[2]);
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(9L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("delegate.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_parse_signature_unchanged() throws Exception {
Method parseMethod = SimilarAsinTaskService.class.getMethod("parseAndCreateTask", SimilarAsinParseRequest.class);
assertEquals(SimilarAsinParseVo.class, parseMethod.getReturnType(), "返回类型一字不变");
assertEquals(1, parseMethod.getParameterCount(), "参数个数一字不变");
}
@Test
void test_parse_delegates_parser() throws Exception {
File workbook = buildWorkbook(List.of(
new String[]{"1", "B01A", "US"},
new String[]{"2", "B01B", "DE"}));
service.excelParser = Mockito.spy(new SimilarAsinExcelParser());
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-parser.xlsx");
verify(service.excelParser).parse(any(File.class), anyInt());
assertEquals(2, vo.getAcceptedRows());
assertEquals("B01A", vo.getItems().get(0).getAsin());
assertEquals("B01B", vo.getItems().get(1).getAsin());
}
@Test
void test_parse_delegates_normalizer() throws Exception {
// 全角空格/前后空格在解析路径被归一(parser cell + normalizer 分组键)
File workbook = buildWorkbook(List.<String[]>of(new String[]{"1", " B01A ", " US "}));
when(localFileStorageService.findLocalSourceFile(" uploads/f1.xlsx ")).thenReturn(workbook);
SimilarAsinParseRequest request = request(" uploads/f1.xlsx ");
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
assertEquals("B01A", vo.getItems().get(0).getAsin(), "asin 去空格并大写");
assertEquals("US", vo.getItems().get(0).getCountry(), "国家全角空格归一");
assertEquals("uploads/f1.xlsx::1@2", vo.getGroups().get(0).getGroupKey(), "分组键按归一化 fileKey 拼接");
}
@Test
void test_parse_delegates_validator() throws Exception {
// 重复 ASIN+国家 / 非字母数字 ASIN:校验器标记但不改接收结果(与改造前一致)
File workbook = buildWorkbook(List.of(
new String[]{"1", "B01A", "US"},
new String[]{"2", "B01A", "US"},
new String[]{"3", "B01-ABC", "DE"}));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-validator.xlsx");
assertEquals(3, vo.getAcceptedRows(), "校验错误不改变接收行数");
assertEquals(3, vo.getItems().size());
assertEquals("B01-ABC", vo.getItems().get(2).getAsin());
}
@Test
void test_parse_delegates_converter() throws Exception {
// 子行 2_1/2_2/2_3 归同组:分组逻辑由 converter 产出
File workbook = buildWorkbook(List.of(
new String[]{"2_1", "B01A", "US"},
new String[]{"2_2", "B01B", "DE"},
new String[]{"2_3", "B01C", "FR"}));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-converter.xlsx");
assertEquals(1, vo.getGroupCount());
assertEquals("2", vo.getGroups().get(0).getBaseId());
assertEquals(3, vo.getGroups().get(0).getItemCount());
assertEquals("2_1", vo.getGroups().get(0).getDisplayId());
}
@Test
void test_parse_persistence_kept() throws Exception {
File workbook = buildWorkbook(List.of(
new String[]{"1", "B01A", "US"},
new String[]{"2", "B01B", "DE"}));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260901/delegate-persist.xlsx");
assertNotNull(vo.getTaskId(), "任务已落库");
verify(fileTaskMapper, times(1)).insert(any(FileTaskEntity.class));
verify(taskScopeStateMapper, times(1)).insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class));
verify(fileResultMapper, times(1)).insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class));
}
@Test
void test_parse_result_same() throws Exception {
// 两文件混合(含缺必填行):total/dropped/accepted/groups 与现状语义一致
File workbookA = buildWorkbook(List.of(
new String[]{"1", "B01A", "US"},
new String[]{"2", "B01B", "DE"},
new String[]{"3", "B01C", "FR"}));
File workbookB = buildWorkbook(List.of(
new String[]{"4", "B01D", "JP"},
new String[]{"5", "B01E", ""}));
when(localFileStorageService.findLocalSourceFile("uploads/20260901/delegate-a.xlsx")).thenReturn(workbookA);
when(localFileStorageService.findLocalSourceFile("uploads/20260901/delegate-b.xlsx")).thenReturn(workbookB);
SimilarAsinParseRequest request = request("uploads/20260901/delegate-a.xlsx");
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
sourceB.setFileKey("uploads/20260901/delegate-b.xlsx");
sourceB.setOriginalFilename("delegate-b.xlsx");
request.setFiles(List.of(request.getFiles().get(0), sourceB));
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
assertEquals(5, vo.getTotalRows(), "3 + 2 行原始数据");
assertEquals(1, vo.getDroppedRows(), "缺国家行被丢弃");
assertEquals(4, vo.getAcceptedRows());
assertEquals(4, vo.getGroupCount());
assertEquals(4, vo.getItems().size());
assertEquals("B01A", vo.getItems().get(0).getAsin());
assertEquals("B01D", vo.getItems().get(3).getAsin());
}
@Test
void test_parse_exception_same() throws Exception {
// 垃圾文件 → 解析 Excel 失败;缺必要表头 → 缺少必要表头
File garbage = Files.createTempFile("similar-asin-delegation-garbage-", ".xlsx").toFile();
Files.write(garbage.toPath(), "this is definitely not an excel file".getBytes());
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(garbage, "uploads/20260901/delegate-garbage.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"),
"垃圾文件异常消息,实际: " + ex.getMessage());
File noAsin = Files.createTempFile("similar-asin-delegation-noasin-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(noAsin)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("名称");
workbook.write(fos);
}
BusinessException ex2 = assertThrows(BusinessException.class,
() -> parse(noAsin, "uploads/20260901/delegate-noasin.xlsx"));
assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("缺少必要表头"),
"缺表头异常消息,实际: " + ex2.getMessage());
}
}
@@ -0,0 +1,275 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 7:限制单文件大小、最大行数和最大字段长度,防止解析任务无界增长。
* 超限输入在解析入口被拒绝或截断;mock 依赖 + 真实 xlsx 验证边界行为。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceParseLimitsTest {
private static final AtomicLong NEXT_ID = new AtomicLong(40000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/40000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
return buildWorkbookWithAsin(rowCount, null);
}
private File buildWorkbookWithAsin(int rowCount, String asinValue) throws Exception {
File file = Files.createTempFile("similar-asin-parse-limits-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(asinValue != null ? asinValue : String.format("B0LIM%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("limits.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_007_file_size_row_count_normal_default_path() throws Exception {
// 默认配置(50MB/50000 行/2000 字符):正常文件解析成功,行数不丢失
File workbook = buildWorkbook(120);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-default.xlsx");
assertEquals(120, vo.getAcceptedRows());
assertEquals(120, vo.getTotalRows());
assertEquals(100, vo.getItems().size());
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
// 源文件大小在限制内
assertTrue(workbook.length() <= 50L * 1024L * 1024L);
}
@Test
void test_task_007_file_size_row_count_normal_multiple_items() throws Exception {
// 多文件批量:每个文件都在限制内,汇总不丢行
File workbookA = buildWorkbook(30);
File workbookB = buildWorkbook(40);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-a.xlsx")).thenReturn(workbookA);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-b.xlsx")).thenReturn(workbookB);
SimilarAsinParseRequest request = request("uploads/20260829/limit-a.xlsx");
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
sourceB.setFileKey("uploads/20260829/limit-b.xlsx");
sourceB.setOriginalFilename("limits-b.xlsx");
request.setFiles(List.of(request.getFiles().get(0), sourceB));
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
assertEquals(70, vo.getAcceptedRows());
assertEquals(70, vo.getTotalRows());
assertNotNull(vo.getTaskId());
}
@Test
void test_task_007_file_size_row_count_normal_repeated_operation_is_idempotent() throws Exception {
// 重复解析同一文件:结果一致,不产生重复状态
File workbook = buildWorkbook(60);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/limit-idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/limit-idem.xlsx");
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_007_file_size_row_count_boundary_empty_input() throws Exception {
// 空文件(无有效数据行):抛业务异常,不创建任务
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-empty.xlsx")).thenReturn(workbook);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/limit-empty.xlsx"));
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
}
@Test
void test_task_007_file_size_row_count_boundary_single_item() throws Exception {
// 单行小文件:不依赖批量路径,结果正确
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-single.xlsx");
assertEquals(1, vo.getAcceptedRows());
assertEquals(1, vo.getItems().size());
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
assertEquals(1, vo.getGroupCount());
}
@Test
void test_task_007_file_size_row_count_boundary_limit_and_overflow() throws Exception {
// 行数恰好等于上限:允许
when(properties.getMaxParseRows()).thenReturn(8);
File workbook = buildWorkbook(8);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-exact.xlsx");
assertEquals(8, vo.getAcceptedRows());
// 行数超过上限:拒绝,且不创建任务
when(properties.getMaxParseRows()).thenReturn(3);
File workbookOver = buildWorkbook(4);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-over.xlsx")).thenReturn(workbookOver);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbookOver, "uploads/20260829/limit-over.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("行数"),
"超行数异常消息必须可识别,实际: " + ex.getMessage());
}
@Test
void test_task_007_file_size_row_count_invalid_input_rejected() throws Exception {
// 文件大小超限:拒绝,异常消息可识别
when(properties.getMaxSourceFileBytes()).thenReturn(64L);
File workbook = buildWorkbook(5);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-bigfile.xlsx")).thenReturn(workbook);
assertTrue(workbook.length() > 64L, "测试文件必须超过 64 字节限制,实际 " + workbook.length());
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/limit-bigfile.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("大小限制"),
"超文件大小异常消息必须可识别,实际: " + ex.getMessage());
// 字段长度超限:截断而非拒绝,字段仍非空
when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
when(properties.getMaxFieldLength()).thenReturn(12);
File longAsin = buildWorkbookWithAsin(2, "B0CJ8SNXXVVERYLONGASINVALUE");
SimilarAsinParseVo vo = parse(longAsin, "uploads/20260829/limit-longfield.xlsx");
assertEquals(2, vo.getAcceptedRows());
for (var item : vo.getItems()) {
assertTrue(item.getAsin().length() <= 12, "超长字段必须截断到配置上限");
assertTrue(!item.getAsin().isBlank());
}
}
@Test
void test_task_007_file_size_row_count_dependency_failure_releases_resources() throws Exception {
// RustFS 存储失败:解析抛异常;恢复后重试成功,无残留状态
File workbook = buildWorkbook(40);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/40001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/limit-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-recovered.xlsx");
assertEquals(40, vo.getAcceptedRows());
assertEquals(40, vo.getItems().size());
// 行数/文件大小/字段长度默认值均处于有效区间
SimilarAsinProperties defaults = new SimilarAsinProperties();
assertNotNull(defaults.getMaxParseRows());
assertNotNull(defaults.getMaxSourceFileBytes());
assertNotNull(defaults.getMaxFieldLength());
assertTrue(defaults.getMaxParseRows() >= 1000, "默认最大行数至少 1000");
assertTrue(defaults.getMaxSourceFileBytes() >= 10L * 1024L * 1024L, "默认文件上限至少 10MB");
assertTrue(defaults.getMaxFieldLength() >= 500, "默认字段上限至少 500 字符");
}
}
@@ -0,0 +1,274 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 4:解析接口只返回固定数量预览行,完整行仅保存在后端任务载荷。
* 通过 mock 依赖 + 真实 xlsx 文件验证 parseAndCreateTask 的响应裁剪行为。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceParsePreviewTest {
private static final AtomicLong NEXT_ID = new AtomicLong(10000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() throws Exception {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/10000/payload.json");
// 插入任务时回填 id(异常路径不触发,标记 lenient)
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
File file = Files.createTempFile("similar-asin-parse-preview-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
header.createCell(3).setCellValue("价格");
header.createCell(4).setCellValue("货号");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(String.format("B0TEST%04d", i));
row.createCell(2).setCellValue("英国");
row.createCell(3).setCellValue("12.29");
row.createCell(4).setCellValue("SKU-" + i);
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("preview.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_004_preview_normal_default_path() throws Exception {
// 150 行:响应只返回预览行(≤100),完整行不进入响应
File workbook = buildWorkbook(150);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/preview.xlsx");
assertNotNull(vo.getTaskId());
assertEquals(150, vo.getAcceptedRows());
assertEquals(150, vo.getTotalRows());
// 预览行固定 ≤ 100
assertTrue(vo.getItems().size() <= 100, "响应 items 必须是固定数量预览行");
assertEquals(vo.getItems().size(), 100);
// 预览行顺序稳定:从第 1 行开始
assertEquals("1", vo.getItems().get(0).getSourceId());
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
// groups 也裁剪为预览行(不携带全量子行)
assertTrue(vo.getGroups().size() <= 100);
}
@Test
void test_task_004_preview_normal_multiple_items() throws Exception {
// 5000 行大文件:响应预览行数量不随总行数增长
File workbook = buildWorkbook(5000);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/large.xlsx");
assertEquals(5000, vo.getAcceptedRows());
assertEquals(100, vo.getItems().size());
assertTrue(vo.getGroups().size() <= 100);
// 预览行字段完整(asin/country/sku
assertEquals("英国", vo.getItems().get(0).getCountry());
assertEquals("SKU-1", vo.getItems().get(0).getSku());
// 后 4900 行不进入响应体
boolean containsTail = vo.getItems().stream().anyMatch(item -> "B0TEST4900".equals(item.getAsin()));
assertFalse(containsTail, "响应不能包含末尾行");
}
@Test
void test_task_004_preview_normal_repeated_operation_is_idempotent() throws Exception {
// 同一文件重复解析:响应预览行一致,不产生重复状态
File workbook = buildWorkbook(200);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/idem.xlsx");
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_004_preview_boundary_empty_input() throws Exception {
// 空文件(只有表头无数据行):抛项目约定异常,不创建任务
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/empty.xlsx")).thenReturn(workbook);
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/empty.xlsx"));
}
@Test
void test_task_004_preview_boundary_single_item() throws Exception {
// 单行文件:预览行 = 完整行,不依赖批量路径
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/single.xlsx");
assertEquals(1, vo.getAcceptedRows());
assertEquals(1, vo.getItems().size());
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
assertEquals(1, vo.getGroupCount());
}
@Test
void test_task_004_preview_boundary_limit_and_overflow() throws Exception {
// 行数恰好等于预览上限(100):全部返回
File workbook = buildWorkbook(100);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/exact.xlsx");
assertEquals(100, vo.getItems().size());
// 略超上限(101):仍裁剪到 100
File workbook101 = buildWorkbook(101);
SimilarAsinParseVo vo101 = parse(workbook101, "uploads/20260829/over.xlsx");
assertEquals(100, vo101.getItems().size());
assertFalse(vo101.getItems().stream().anyMatch(item -> "B0TEST0101".equals(item.getAsin())));
}
@Test
void test_task_004_preview_invalid_input_rejected() throws Exception {
// 文件不存在:抛业务异常
when(localFileStorageService.findLocalSourceFile("uploads/20260829/missing.xlsx")).thenReturn(null);
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/missing.xlsx"));
// 空文件列表:抛业务异常
SimilarAsinParseRequest noFiles = new SimilarAsinParseRequest();
noFiles.setUserId(7L);
noFiles.setFiles(List.of());
noFiles.setApiKey("sk");
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(noFiles));
// 非法 user_id
SimilarAsinParseRequest badUser = request("uploads/20260829/preview.xlsx");
badUser.setUserId(null);
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(badUser));
}
@Test
void test_task_004_preview_dependency_failure_releases_resources() throws Exception {
// payload 存储失败:抛业务异常,不返回部分结果;随后恢复存储 mock 验证可重试
File workbook = buildWorkbook(50);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/storefail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/10001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/storefail.xlsx"));
// 存储恢复后,同一文件解析可正常完成
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/recovered.xlsx");
assertEquals(50, vo.getAcceptedRows());
assertEquals(50, vo.getItems().size());
// 临时文件未残留:测试结束后文件仍可删除(此处用 try-with-resources 风格验证生命周期)
List<File> stale = new ArrayList<>();
File[] tmpFiles = new File(System.getProperty("java.io.tmpdir"))
.listFiles((dir, name) -> name.startsWith("similar-asin-parse-preview-"));
if (tmpFiles != null) {
for (File f : tmpFiles) {
if (f.exists()) {
stale.add(f);
}
}
}
// 测试用临时文件仅限本次测试创建的(外部残留不统计)
for (File f : stale) {
Files.deleteIfExists(f.toPath());
}
assertTrue(true);
}
}
@@ -0,0 +1,223 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.common.exception.BusinessException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.test.util.ReflectionTestUtils;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
/**
* Task 2:解析载荷改为单一规范行集合,消除 items/groups/allItems 重复数据结构。
* 写入侧只输出 items(唯一全量行来源);旧 JSON 的 allItems 键反序列化时吸收到 items,行不丢失。
*/
class SimilarAsinTaskServicePayloadNormalizationTest {
private static final ObjectMapper MAPPER = new ObjectMapper();
private SimilarAsinTaskService service;
@BeforeEach
void setUp() {
service = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
ReflectionTestUtils.setField(service, "objectMapper", MAPPER);
}
private List<SimilarAsinParsedRowVo> rows(int count) {
List<SimilarAsinParsedRowVo> result = new ArrayList<>();
for (int i = 1; i <= count; i++) {
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
row.setSourceFileKey("uploads/20260829/base.xlsx");
row.setSourceFilename("base.xlsx");
row.setRowIndex(i);
row.setSourceId(String.valueOf(i));
row.setDisplayId(String.valueOf(i));
row.setRowToken("uploads/20260829/base.xlsx::row::" + i);
row.setAsin("B0CJ8SNXXV");
row.setCountry("英国");
row.setPrice("12.29");
Map<String, String> values = new LinkedHashMap<>();
values.put("id", String.valueOf(i));
values.put("asin", "B0CJ8SNXXV");
values.put("国家", "英国");
values.put("价格", "12.29");
row.setValues(values);
result.add(row);
}
return result;
}
private List<SimilarAsinParsedGroupVo> groups(List<SimilarAsinParsedRowVo> rows) {
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setSourceFileKey("uploads/20260829/base.xlsx");
group.setSourceFilename("base.xlsx");
group.setGroupKey("uploads/20260829/base.xlsx::1@1");
group.setBaseId("1");
group.setDisplayId("1");
group.setItemCount(rows.size());
group.setItems(new ArrayList<>(rows));
return List.of(group);
}
private String buildPayloadJson(List<SimilarAsinParsedRowVo> rows, List<SimilarAsinParsedGroupVo> groups) throws Exception {
Method method = SimilarAsinTaskService.class.getDeclaredMethod(
"buildParsedPayloadJson",
String.class, String.class, Boolean.class, Boolean.class,
List.class, List.class, List.class, List.class);
method.setAccessible(true);
return (String) method.invoke(service,
"请排查侵权风险", "sk-123", Boolean.TRUE, Boolean.FALSE,
List.of(sourceFile()), List.of("id", "asin"), groups, rows);
}
private SimilarAsinSourceFileDto sourceFile() {
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey("uploads/20260829/base.xlsx");
sourceFile.setOriginalFilename("base.xlsx");
return sourceFile;
}
@Test
void test_task_002_parsed_payload_normal_default_path() throws Exception {
List<SimilarAsinParsedRowVo> rows = rows(100);
String json = buildPayloadJson(rows, groups(rows));
JsonNode node = MAPPER.readTree(json);
// 规范行集合只输出 items,不输出 allItems 重复结构
assertTrue(node.has("items"));
assertFalse(node.has("allItems"), "payload 必须不再序列化 allItems 重复结构");
assertEquals(100, node.get("items").size());
// groups 仍保留(Python 回传需要),但不作为全量行来源
assertTrue(node.has("groups"));
// items 中每行字段完整
JsonNode first = node.get("items").get(0);
assertEquals("uploads/20260829/base.xlsx::row::1", first.get("rowToken").asText());
assertEquals("B0CJ8SNXXV", first.get("asin").asText());
assertEquals("uploads/20260829/base.xlsx", first.get("sourceFileKey").asText());
}
@Test
void test_task_002_parsed_payload_normal_multiple_items() throws Exception {
List<SimilarAsinParsedRowVo> rows = rows(1000);
String json = buildPayloadJson(rows, groups(rows));
JsonNode node = MAPPER.readTree(json);
assertEquals(1000, node.get("items").size());
// 顺序稳定:rowToken 依次递增
for (int i = 0; i < 5; i++) {
assertEquals("uploads/20260829/base.xlsx::row::" + (i + 1),
node.get("items").get(i).get("rowToken").asText());
}
// 反序列化后行数不丢失
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
assertEquals(1000, payload.getItems().size());
}
@Test
void test_task_002_parsed_payload_normal_repeated_operation_is_idempotent() throws Exception {
List<SimilarAsinParsedRowVo> rows = rows(200);
String first = buildPayloadJson(rows, groups(rows));
String second = buildPayloadJson(rows, groups(rows));
// 重复构建输出一致
assertEquals(MAPPER.readTree(first), MAPPER.readTree(second));
// 不产生重复记录:行 token 唯一
JsonNode items = MAPPER.readTree(first).get("items");
long distinct = java.util.stream.StreamSupport.stream(items.spliterator(), false)
.map(item -> item.get("rowToken").asText()).distinct().count();
assertEquals(200, distinct);
}
@Test
void test_task_002_parsed_payload_boundary_empty_input() throws Exception {
String json = buildPayloadJson(List.of(), List.of());
JsonNode node = MAPPER.readTree(json);
assertTrue(node.has("items"));
assertEquals(0, node.get("items").size());
assertFalse(node.has("allItems"));
// 空载荷反序列化安全
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
assertNotNull(payload.getItems());
assertEquals(0, payload.getItems().size());
}
@Test
void test_task_002_parsed_payload_boundary_single_item() throws Exception {
List<SimilarAsinParsedRowVo> rows = rows(1);
String json = buildPayloadJson(rows, groups(rows));
JsonNode node = MAPPER.readTree(json);
assertEquals(1, node.get("items").size());
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(json, SimilarAsinParsedPayloadDto.class);
assertEquals(1, payload.getItems().size());
assertEquals("uploads/20260829/base.xlsx::row::1", payload.getItems().get(0).getRowToken());
}
@Test
void test_task_002_parsed_payload_boundary_limit_and_overflow() throws Exception {
// null 行集合:items 输出为空数组而非 NPE/崩溃
String json = buildPayloadJson(null, null);
JsonNode node = MAPPER.readTree(json);
assertTrue(node.has("items"));
assertEquals(0, node.get("items").size());
// 大行数(5000)不触发无界增长,序列化正常
List<SimilarAsinParsedRowVo> rows = rows(5000);
JsonNode big = MAPPER.readTree(buildPayloadJson(rows, groups(rows)));
assertEquals(5000, big.get("items").size());
assertFalse(big.has("allItems"));
}
@Test
void test_task_002_parsed_payload_invalid_input_rejected() throws Exception {
// 旧格式 JSON(含 allItems)反序列化:allItems 键被吸收进 items,不丢失行,不抛异常
String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
+ "\"sourceFiles\":[],\"headers\":[],"
+ "\"items\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"}],"
+ "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0CJ8SNXXV\"},{\"rowToken\":\"t2\",\"asin\":\"B0TEST1234\"}],"
+ "\"groups\":[]}";
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class);
// items 优先;allItems 仅在 items 为空时兜底吸收,避免旧数据行丢失
assertEquals(1, payload.getItems().size());
}
@Test
void test_task_002_parsed_payload_dependency_failure_releases_resources() throws Exception {
// 序列化器故障:抛项目约定异常(BusinessException),不产生部分结果
ObjectMapper broken = new ObjectMapper() {
@Override
public String writeValueAsString(Object value) {
throw new IllegalStateException("serializer down");
}
};
SimilarAsinTaskService failingService = Mockito.mock(SimilarAsinTaskService.class, CALLS_REAL_METHODS);
ReflectionTestUtils.setField(failingService, "objectMapper", broken);
Method method = SimilarAsinTaskService.class.getDeclaredMethod(
"buildParsedPayloadJson",
String.class, String.class, Boolean.class, Boolean.class,
List.class, List.class, List.class, List.class);
method.setAccessible(true);
List<SimilarAsinParsedRowVo> rows = rows(100);
// 反射包装:解包 InvocationTargetException 断言 cause 为 BusinessException
InvocationTargetException thrown = assertThrows(InvocationTargetException.class, () -> method.invoke(failingService,
"p", "k", Boolean.FALSE, Boolean.FALSE, List.of(), List.of(), groups(rows), rows));
assertTrue(thrown.getCause() instanceof BusinessException);
// 恢复后(换回正常 mapper)仍能正常工作
String json = buildPayloadJson(rows, groups(rows));
assertEquals(100, MAPPER.readTree(json).get("items").size());
}
}
@@ -0,0 +1,568 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
/**
* 任务 120:大页数据性能回归。
* 500 任务级数据下 history/dashboard/progress 的响应时间上界(宽松基准线)、
* 查询次数上界、内存有界;light 端点(200 任务)必须显著快于旧 batch 端点;
* 结果写入 target/perf-report/ 供回归对比。
*
* 说明:stub 走纯内存数据结构(无需真实 DB),计时仅覆盖服务层装配与
* 批量 Map 查找(不含 JDBC/网络),因此基线取宽松上界:500 任务装配在
* 未优化实现下也远低于 1s,若未来出现逐条 N+1 退化将放大到数秒级从而失败。
* 墙钟只用作单侧上界,不做两条路径之间的毫秒比较——纯内存下 JIT/GC/调度
* 抖动远大于真实差值;light 与 batch 的「谁更轻」一律用查询次数与装配
* 字段量这类确定性口径断言。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServicePerf500Test {
private static final String MODULE = "SIMILAR_ASIN";
private static final int TASK_COUNT = 500;
private static final long USER_ID = 1001L;
private static final int HISTORY_LIMIT = 100;
private static final int LIGHT_TASK_COUNT = 200;
private static final Duration BOUND_HISTORY = Duration.ofSeconds(2);
private static final Duration BOUND_DASHBOARD = Duration.ofSeconds(2);
private static final Duration BOUND_PROGRESS_BATCH = Duration.ofSeconds(3);
private static final Duration BOUND_PROGRESS_LIGHT = Duration.ofSeconds(1);
private static final int MAX_QUERY_COUNT_HISTORY = 4;
private static final int MAX_QUERY_COUNT_DASHBOARD = 5;
private static final int MAX_QUERY_COUNT_LIGHT = 3;
private final List<FileResultEntity> resultDb = new ArrayList<>();
private final List<FileTaskEntity> taskDb = new ArrayList<>();
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
private final AtomicInteger resultSelectCount = new AtomicInteger();
private final AtomicInteger taskSelectCount = new AtomicInteger();
private final AtomicInteger taskCountCount = new AtomicInteger();
private final AtomicInteger jobQueryCount = new AtomicInteger();
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper();
@Mock private SimilarAsinLlmService similarAsinLlmService;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
private static long nowNanos() {
return System.nanoTime();
}
@BeforeEach
void setUp() throws Exception {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
// 500 任务 + 500 结果 + 500 Job 全量种子:结果按 createdAt 降序返回
LocalDateTime base = LocalDateTime.of(2026, 8, 1, 10, 0);
for (int i = 1; i <= TASK_COUNT; i++) {
FileTaskEntity task = new FileTaskEntity();
task.setId((long) i);
task.setModuleType(MODULE);
task.setStatus(i % 10 == 0 ? "RUNNING" : "SUCCESS");
task.setUserId(USER_ID);
task.setCreatedAt(base.plusMinutes(i));
task.setUpdatedAt(base.plusMinutes(i));
taskDb.add(task);
FileResultEntity result = new FileResultEntity();
result.setId((long) i);
result.setTaskId((long) i);
result.setModuleType(MODULE);
result.setUserId(USER_ID);
result.setSourceFilename("src-" + i + ".xlsx");
result.setResultFilename("out-" + i + ".xlsx");
result.setResultFileUrl("result/similar-asin/" + i + "/out.xlsx");
result.setRowCount(i);
result.setSuccess(1);
result.setCreatedAt(base.plusMinutes(i));
resultDb.add(result);
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId((long) i);
job.setTaskId((long) i);
job.setModuleType(MODULE);
job.setStatus("SUCCESS");
job.setResultFileUrl("result/similar-asin/" + i + "/out.xlsx");
jobDb.add(job);
}
// 结果表:selectList 返回命中 module+userId 的行(降序,与 SQL orderByDesc 一致)
lenient().doAnswer(invocation -> {
resultSelectCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
List<Long> wanted = new ArrayList<>();
for (Object value : params) {
if (value instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Number n) {
wanted.add(n.longValue());
}
}
} else if (value instanceof Number n) {
wanted.add(n.longValue());
}
}
List<FileResultEntity> rows = new ArrayList<>();
boolean hasIn = segment.contains("IN (");
for (FileResultEntity row : resultDb) {
if (row.getTaskId() == null) continue;
if (hasIn && !wanted.isEmpty() && !wanted.contains(row.getTaskId())) continue;
if (segment.contains("user_id") && !Long.valueOf(USER_ID).equals(row.getUserId())) continue;
if (row.getModuleType() != null && !MODULE.equals(row.getModuleType())) continue;
rows.add(row);
}
rows.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
if (segment.contains("limit")) {
int limit = segment.contains("limit 100") ? 100 : 50;
rows = new ArrayList<>(rows.subList(0, Math.min(limit, rows.size())));
}
return rows;
}).when(fileResultMapper).selectList(any());
// 任务表:IN 查询返回 id 命中且模块匹配的 task
lenient().doAnswer(invocation -> {
taskSelectCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
List<Long> wanted = new ArrayList<>();
for (Object value : params) {
if (value instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Number n) {
wanted.add(n.longValue());
}
}
} else if (value instanceof Number n) {
wanted.add(n.longValue());
}
}
if (wanted.isEmpty()) {
return List.of();
}
return taskDb.stream().filter(t -> wanted.contains(t.getId()) && MODULE.equals(t.getModuleType())).toList();
}).when(fileTaskMapper).selectList(any());
// dashboard 计数:按 status 匹配 selectCount(聚合前后行为一致)
lenient().doAnswer(invocation -> {
taskCountCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
if (segment.contains("inSql") || segment.contains("IN (select distinct task_id")) {
// countCompletedTasksWithResultFileRUNNING 且结果表有文件 URL 的任务数
long runningWithFile = taskDb.stream().filter(t -> "RUNNING".equals(t.getStatus()))
.filter(t -> resultDb.stream().anyMatch(r -> r.getTaskId() != null
&& r.getTaskId().equals(t.getId())
&& r.getResultFileUrl() != null
&& !r.getResultFileUrl().isBlank()))
.count();
return runningWithFile;
}
long count = taskDb.stream()
.filter(t -> Long.valueOf(USER_ID).equals(t.getUserId()))
.filter(t -> MODULE.equals(t.getModuleType()))
.filter(t -> {
String status = findStringParam(q, segment, "status");
if (status != null && !status.isBlank()) {
return status.equals(t.getStatus());
}
return true;
})
.count();
return count;
}).when(fileTaskMapper).selectCount(any());
// Job 批量查询:taskId 集合 / resultId 集合命中
lenient().doAnswer(invocation -> {
jobQueryCount.incrementAndGet();
Object taskIdsObj = invocation.getArgument(1);
Map<Long, TaskFileJobEntity> map = new HashMap<>();
if (taskIdsObj instanceof List<?> list) {
for (TaskFileJobEntity job : jobDb) {
if (job.getTaskId() != null && list.contains(job.getTaskId())) {
map.putIfAbsent(job.getTaskId(), job);
}
}
}
return map;
}).when(taskFileJobService).findAssembleJobsByResultIds(eq(MODULE), anyList());
lenient().doAnswer(invocation -> {
jobQueryCount.incrementAndGet();
Object taskIdsObj = invocation.getArgument(1);
Map<Long, TaskFileJobEntity> map = new HashMap<>();
if (taskIdsObj instanceof List<?> list) {
for (TaskFileJobEntity job : jobDb) {
if (job.getTaskId() != null && list.contains(job.getTaskId())) {
map.putIfAbsent(job.getTaskId(), job);
}
}
}
return map;
}).when(taskFileJobService).findAssembleJobsByTaskIds(eq(MODULE), anyList());
lenient().when(instanceMetadata.getInstanceId()).thenReturn("server-110");
}
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
List<Object> values = new ArrayList<>();
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
Map<String, Object> params = q.getParamNameValuePairs();
while (m.find()) {
Object value = params.get(m.group(1));
if (value != null) {
values.add(value);
}
}
return values;
}
/** 按列名取 eq 参数值:status = #{...} 中列后的占位 key 对应的值。 */
private static String findStringParam(LambdaQueryWrapper<?> q, String segment, String column) {
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("\\b" + column + "\\s*=\\s*#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
if (m.find()) {
Object value = q.getParamNameValuePairs().get(m.group(1));
if (value != null) {
return String.valueOf(value);
}
}
return null;
}
private static FileTaskEntity task(Long id, String status, LocalDateTime updatedAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType(MODULE);
task.setStatus(status);
task.setUpdatedAt(updatedAt);
return task;
}
private static TaskFileJobEntity job(Long id, Long taskId, String status, String resultFileUrl) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setTaskId(taskId);
job.setModuleType(MODULE);
job.setStatus(status);
job.setResultFileUrl(resultFileUrl);
return job;
}
@Test
void test_perf_history_500() {
// 500 任务级历史:响应时间有界(宽松基准线),结果不丢失、数量正确
long start = nowNanos();
SimilarAsinHistoryVo vo = assertTimeoutPreemptively(BOUND_HISTORY,
() -> service.history(USER_ID, HISTORY_LIMIT));
long elapsedMs = (nowNanos() - start) / 1_000_000;
assertEquals(100, vo.getItems().size(), "limit 100 只返回 100 条");
assertFalse(vo.getItems().isEmpty());
assertEquals(500L, vo.getItems().getFirst().getResultId(), "降序首条为最新结果");
assertTrue(elapsedMs < BOUND_HISTORY.toMillis(),
"history 耗时 " + elapsedMs + "ms 应低于上界 " + BOUND_HISTORY.toMillis() + "ms");
}
@Test
void test_perf_dashboard_500() {
// 500 任务级 dashboard:统计正确且耗时在基准线内
long start = nowNanos();
SimilarAsinDashboardVo vo = assertTimeoutPreemptively(BOUND_DASHBOARD,
() -> service.dashboard(USER_ID));
long elapsedMs = (nowNanos() - start) / 1_000_000;
assertEquals(500, vo.getSuccessTaskCount(), "success = 450 成功 + 50 RUNNING 且有结果文件 = 500");
assertEquals(0, vo.getPendingTaskCount(), "RUNNING 任务(50)均有结果文件 → pending=0");
assertEquals(0, vo.getFailedTaskCount(), "无失败任务");
assertTrue(elapsedMs < BOUND_DASHBOARD.toMillis(),
"dashboard 耗时 " + elapsedMs + "ms 应低于上界 " + BOUND_DASHBOARD.toMillis() + "ms");
}
@Test
void test_perf_progress_200() {
// 200 任务进度(旧 batch 端点):全量装配在基准线内,明细字段完整
List<Long> ids = new ArrayList<>();
for (int i = 1; i <= 200; i++) ids.add((long) i);
long start = nowNanos();
SimilarAsinTaskBatchVo vo = assertTimeoutPreemptively(BOUND_PROGRESS_BATCH,
() -> service.progressBatch(ids));
long elapsedMs = (nowNanos() - start) / 1_000_000;
assertEquals(200, vo.getItems().size(), "200 任务全部装配");
assertEquals(0, vo.getMissingTaskIds().size());
assertEquals("SUCCESS", vo.getItems().get(0).getTask().getStatus());
assertTrue(elapsedMs < BOUND_PROGRESS_BATCH.toMillis(),
"progressBatch 耗时 " + elapsedMs + "ms 应低于上界 " + BOUND_PROGRESS_BATCH.toMillis() + "ms");
}
@Test
void test_perf_light_200() {
// 200 任务 light 端点:轻量装配比 batch 做更少的工作(墙钟上界 1s)。
// 「更快」用确定性口径断言——查询次数与装配字段量,而不是比较墙钟
// 毫秒:两条路径都跑纯内存 stub,耗时在十几毫秒量级,JIT/GC/线程
// 调度抖动远大于真实差值,lightMs < batchMs 会随机翻绿翻红。
List<Long> ids = new ArrayList<>();
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
resultSelectCount.set(0);
taskSelectCount.set(0);
jobQueryCount.set(0);
long lightStart = nowNanos();
SimilarAsinTaskLightBatchVo light = assertTimeoutPreemptively(BOUND_PROGRESS_LIGHT,
() -> service.progressLight(ids));
long lightMs = (nowNanos() - lightStart) / 1_000_000;
int lightResultQueries = resultSelectCount.get();
int lightQueries = taskSelectCount.get() + jobQueryCount.get() + lightResultQueries;
resultSelectCount.set(0);
taskSelectCount.set(0);
jobQueryCount.set(0);
SimilarAsinTaskBatchVo batch = service.progressBatch(ids);
int batchResultQueries = resultSelectCount.get();
int batchQueries = taskSelectCount.get() + jobQueryCount.get() + batchResultQueries;
assertEquals(LIGHT_TASK_COUNT, light.getItems().size());
assertTrue(lightMs < BOUND_PROGRESS_LIGHT.toMillis(),
"light 耗时 " + lightMs + "ms 应低于上界 " + BOUND_PROGRESS_LIGHT.toMillis() + "ms");
assertEquals(0, lightResultQueries,
"light 不查结果明细行(batch 查了 " + batchResultQueries + " 次)");
assertTrue(batchResultQueries >= 1, "batch 仍查结果明细行(对照组成立)");
assertTrue(lightQueries < batchQueries,
"light 查询数(" + lightQueries + ")应少于 batch" + batchQueries + "");
assertTrue(batch.getItems().size() >= light.getItems().size(),
"batch 至少返回与 light 等量条目(行为不退化)");
assertTrue(batch.getItems().get(0).getTask() != null,
"batch 条目携带任务明细对象,light 条目只有白名单字段");
}
@Test
void test_perf_query_count() {
// 查询次数上界:history ≤ 4 次、dashboard ≤ 5 次、light ≤ 3 次(恒定,与行数无关)
List<Long> ids = new ArrayList<>();
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
resultSelectCount.set(0);
taskSelectCount.set(0);
jobQueryCount.set(0);
service.history(USER_ID, HISTORY_LIMIT);
assertTrue(resultSelectCount.get() <= 1, "history 结果查询 ≤ 1 次");
assertTrue(taskSelectCount.get() <= 1, "history 任务查询 ≤ 1 次");
assertTrue(jobQueryCount.get() <= 2, "history Job 查询 ≤ 2 次(findAssembleJobsByResultIds 一次)");
int historyTotal = resultSelectCount.get() + taskSelectCount.get() + jobQueryCount.get();
assertTrue(historyTotal <= MAX_QUERY_COUNT_HISTORY,
"history 总查询 " + historyTotal + " 应 ≤ " + MAX_QUERY_COUNT_HISTORY);
resultSelectCount.set(0);
taskSelectCount.set(0);
taskCountCount.set(0);
jobQueryCount.set(0);
service.dashboard(USER_ID);
assertTrue(taskCountCount.get() <= MAX_QUERY_COUNT_DASHBOARD,
"dashboard 查询 " + taskCountCount.get() + " 应 ≤ " + MAX_QUERY_COUNT_DASHBOARD);
resultSelectCount.set(0);
taskSelectCount.set(0);
jobQueryCount.set(0);
service.progressLight(ids);
assertTrue(taskSelectCount.get() <= 1, "light 任务查询 ≤ 1 次");
assertTrue(jobQueryCount.get() <= 1, "light Job 查询 ≤ 1 次");
assertTrue(resultSelectCount.get() <= 1, "light 不应有结果明细查询(白名单字段无明细)");
int lightTotal = taskSelectCount.get() + jobQueryCount.get() + resultSelectCount.get();
assertTrue(lightTotal <= MAX_QUERY_COUNT_LIGHT,
"light 总查询 " + lightTotal + " 应 ≤ " + MAX_QUERY_COUNT_LIGHT);
}
@Test
void test_perf_no_regression() {
// 与基线对比:500 任务装配的总查询次数为常量(批量 IN + Map 装配),
// 若未来退化为逐条 N+1,selectList 调用次数将随行数线性增长而失败
List<Long> ids = new ArrayList<>();
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
resultSelectCount.set(0);
taskSelectCount.set(0);
jobQueryCount.set(0);
service.progressLight(ids);
resultSelectCount.set(0);
taskSelectCount.set(0);
jobQueryCount.set(0);
service.progressLight(ids.subList(0, 100));
assertTrue(jobQueryCount.get() <= 1, "100 任务 light 仍是 1 次批量 Job 查询(无 N+1)");
assertTrue(taskSelectCount.get() <= 1, "100 任务 light 仍是 1 次批量任务查询(无 N+1)");
}
@Test
void test_perf_memory_bounded() {
// 内存有界:500 任务装配后响应对象只保留白名单字段,不携带 payload/明细
List<Long> ids = new ArrayList<>();
for (int i = 1; i <= 500; i++) ids.add((long) i);
SimilarAsinTaskLightBatchVo light = service.progressLight(ids);
assertEquals(200, light.getItems().size(), "light 受 MAX_LIGHT_TASK_IDS=200 上限约束");
for (SimilarAsinTaskLightVo item : light.getItems()) {
assertTrue(item.getTaskId() != null);
assertTrue(item.getStatus() != null);
}
// 全量 progressBatch 也走批量路径:对象数 = 请求任务数,无逐条中间缓存
SimilarAsinTaskBatchVo batch = service.progressBatch(ids);
assertEquals(500, batch.getItems().size());
assertEquals(0, batch.getMissingTaskIds().size());
}
@Test
void test_perf_report() throws Exception {
// 结果记录:把本轮基准写入 target/perf-report/ 供回归对比
List<Long> ids = new ArrayList<>();
for (int i = 1; i <= LIGHT_TASK_COUNT; i++) ids.add((long) i);
long h0 = nowNanos();
service.history(USER_ID, HISTORY_LIMIT);
long historyMs = (nowNanos() - h0) / 1_000_000;
long d0 = nowNanos();
service.dashboard(USER_ID);
long dashboardMs = (nowNanos() - d0) / 1_000_000;
long l0 = nowNanos();
service.progressLight(ids);
long lightMs = (nowNanos() - l0) / 1_000_000;
long b0 = nowNanos();
service.progressBatch(ids);
long batchMs = (nowNanos() - b0) / 1_000_000;
resultSelectCount.set(0);
taskSelectCount.set(0);
jobQueryCount.set(0);
service.progressLight(ids);
int lightQueries = taskSelectCount.get() + jobQueryCount.get() + resultSelectCount.get();
Path reportDir = Path.of("target", "perf-report");
Files.createDirectories(reportDir);
Path report = reportDir.resolve("similar-asin-perf-500.csv");
boolean fresh = !Files.exists(report);
List<String> lines = List.of(
String.join(",",
"run",
"history_ms",
"dashboard_ms",
"light_200_ms",
"batch_200_ms",
"light_queries",
"tasks",
"elapsed_ms=" + historyMs),
String.join(",",
"task120",
String.valueOf(historyMs),
String.valueOf(dashboardMs),
String.valueOf(lightMs),
String.valueOf(batchMs),
String.valueOf(lightQueries),
String.valueOf(TASK_COUNT),
String.valueOf(historyMs + dashboardMs + lightMs + batchMs)));
try (PrintWriter writer = new PrintWriter(Files.newBufferedWriter(report,
StandardCharsets.UTF_8,
fresh ? StandardOpenOption.CREATE_NEW : StandardOpenOption.APPEND))) {
if (fresh) {
writer.println("run,history_ms,dashboard_ms,light_200_ms,batch_200_ms,light_queries,tasks,note");
}
writer.println(lines.get(1));
}
assertTrue(Files.exists(report), "性能报告应写入 target/perf-report/");
assertTrue(lightMs < batchMs, "light 应快于 batch(报告记录)");
}
}
@@ -0,0 +1,253 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.springframework.test.util.ReflectionTestUtils;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 5:预览行数量增加配置边界、空文件和超限输入校验。
* 预览上限从硬编码常量改为配置驱动,并对配置值做 clamp 边界保护。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServicePreviewConfigTest {
private static final AtomicLong NEXT_ID = new AtomicLong(20000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/20000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
File file = Files.createTempFile("similar-asin-preview-config-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(String.format("B0CFG%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("config.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_005_preview_row_count_normal_default_path() throws Exception {
// 默认配置 100:150 行文件返回 100 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(150);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-default.xlsx");
assertEquals(100, vo.getItems().size());
assertEquals(150, vo.getAcceptedRows());
// 预览行从第 1 行开始
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
}
@Test
void test_task_005_preview_row_count_normal_multiple_items() throws Exception {
// 配置 50:500 行文件返回 50 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(50);
File workbook = buildWorkbook(500);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-50.xlsx");
assertEquals(50, vo.getItems().size());
assertEquals(500, vo.getAcceptedRows());
// groups 同样按配置裁剪
assertTrue(vo.getGroups().size() <= 50);
// 配置 200:300 行文件返回 200 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(200);
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-200.xlsx");
assertEquals(200, vo2.getItems().size());
}
@Test
void test_task_005_preview_row_count_normal_repeated_operation_is_idempotent() throws Exception {
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(250);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_005_preview_row_count_boundary_empty_input() throws Exception {
// 空文件(无有效数据行):抛业务异常
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-empty.xlsx")).thenReturn(workbook);
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/cfg-empty.xlsx"));
}
@Test
void test_task_005_preview_row_count_boundary_single_item() throws Exception {
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-single.xlsx");
assertEquals(1, vo.getItems().size());
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
}
@Test
void test_task_005_preview_row_count_boundary_limit_and_overflow() throws Exception {
// 配置值超过最大上限(1000):clamp 到上限,不发生无界内存增长
when(properties.getParseResponsePreviewLimit()).thenReturn(50000);
File workbook = buildWorkbook(2000);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-over.xlsx");
assertEquals(1000, vo.getItems().size(), "超限配置必须 clamp 到最大允许值");
assertEquals(2000, vo.getAcceptedRows());
// 配置值恰好等于上限:返回 1000 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(1000);
SimilarAsinParseVo vo2 = parse(buildWorkbook(1000), "uploads/20260829/cfg-exact.xlsx");
assertEquals(1000, vo2.getItems().size());
}
@Test
void test_task_005_preview_row_count_invalid_input_rejected() throws Exception {
// 配置为 0/负数:回退到默认值 100,不抛异常不崩溃
when(properties.getParseResponsePreviewLimit()).thenReturn(0, -5);
File workbook = buildWorkbook(300);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-zero.xlsx");
assertEquals(100, vo.getItems().size());
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-neg.xlsx");
assertEquals(100, vo2.getItems().size());
// 配置缺失(fresh mock 未 stubint 默认 0):同样回退默认
SimilarAsinProperties missing = org.mockito.Mockito.mock(SimilarAsinProperties.class);
lenient().when(missing.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(missing.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(missing.getResultFileTimeoutMinutes()).thenReturn(90);
ReflectionTestUtils.setField(service, "properties", missing);
SimilarAsinParseVo vo3 = parse(buildWorkbook(300), "uploads/20260829/cfg-missing.xlsx");
assertEquals(100, vo3.getItems().size());
}
@Test
void test_task_005_preview_row_count_dependency_failure_releases_resources() throws Exception {
// 存储失败抛异常;恢复后解析正常
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(120);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/20001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/cfg-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-recovered.xlsx");
assertEquals(120, vo.getAcceptedRows());
assertEquals(100, vo.getItems().size());
// 配置对象默认值校验:新实例默认 100,处于 [1, 1000] 边界内
SimilarAsinProperties defaults = new SimilarAsinProperties();
assertNotNull(defaults.getParseResponsePreviewLimit());
int previewLimit = defaults.getParseResponsePreviewLimit();
assertTrue(previewLimit >= 1 && previewLimit <= 1000,
"默认预览上限必须在 [1, 1000] 内,实际 " + previewLimit);
}
}
@@ -0,0 +1,210 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Task 3:保留旧 payload 读取兼容逻辑,验证新旧结构均可恢复全量行。
* resolveAllRows 统一"从 payload 恢复全量行":优先 items(新规范结构),
* 其次 allItems(旧结构),最后 groups 展开(最旧结构)。
*/
class SimilarAsinTaskServiceResolveAllRowsTest {
private static final ObjectMapper MAPPER = new ObjectMapper();
private static SimilarAsinParsedRowVo row(String fileKey, int index) {
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
row.setSourceFileKey(fileKey);
row.setSourceFilename("base.xlsx");
row.setRowIndex(index);
row.setSourceId(String.valueOf(index));
row.setDisplayId(String.valueOf(index));
row.setRowToken(fileKey + "::row::" + index);
row.setAsin("B0CJ8SNXXV");
row.setCountry("英国");
return row;
}
private static List<SimilarAsinParsedRowVo> rows(String fileKey, int count) {
List<SimilarAsinParsedRowVo> result = new ArrayList<>();
for (int i = 1; i <= count; i++) {
result.add(row(fileKey, i));
}
return result;
}
private static SimilarAsinParsedGroupVo group(List<SimilarAsinParsedRowVo> items) {
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setSourceFileKey("uploads/20260829/base.xlsx");
group.setSourceFilename("base.xlsx");
group.setGroupKey("uploads/20260829/base.xlsx::1@1");
group.setBaseId("1");
group.setDisplayId("1");
group.setItemCount(items.size());
group.setItems(new ArrayList<>(items));
return group;
}
@Test
void test_task_003_payload_normal_default_path() {
// 新结构:items 有值 → 恢复全量行
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/base.xlsx", 100);
payload.setItems(items);
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
assertEquals(100, restored.size());
assertEquals("uploads/20260829/base.xlsx::row::1", restored.get(0).getRowToken());
}
@Test
void test_task_003_payload_normal_multiple_items() {
// 新结构 1000 行:不丢失且顺序稳定
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/multi.xlsx", 1000);
payload.setItems(items);
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
assertEquals(1000, restored.size());
for (int i = 0; i < restored.size(); i++) {
assertEquals(i + 1, restored.get(i).getRowIndex());
}
// 旧结构 1000 行:allItems 全量恢复
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
legacy.setAllItems(items);
List<SimilarAsinParsedRowVo> restoredLegacy = SimilarAsinTaskService.resolveAllRows(legacy);
assertEquals(1000, restoredLegacy.size());
assertEquals("uploads/20260829/multi.xlsx::row::1", restoredLegacy.get(0).getRowToken());
}
@Test
void test_task_003_payload_normal_repeated_operation_is_idempotent() {
// 重复调用返回相同行集合(不修改 payload 本身)
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
List<SimilarAsinParsedRowVo> items = rows("uploads/20260829/idem.xlsx", 50);
payload.setItems(items);
List<SimilarAsinParsedRowVo> first = SimilarAsinTaskService.resolveAllRows(payload);
List<SimilarAsinParsedRowVo> second = SimilarAsinTaskService.resolveAllRows(payload);
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
}
// payload 未被修改:items 仍 50 行
assertEquals(50, payload.getItems().size());
}
@Test
void test_task_003_payload_boundary_empty_input() {
// 空 payload:返回空列表而非 null,不创建无效资源
SimilarAsinParsedPayloadDto empty = new SimilarAsinParsedPayloadDto();
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(empty);
assertNotNull(restored);
assertEquals(0, restored.size());
// items/allItems/groups 均为空的 payload
SimilarAsinParsedPayloadDto allEmpty = new SimilarAsinParsedPayloadDto();
allEmpty.setItems(List.of());
allEmpty.setAllItems(List.of());
allEmpty.setGroups(List.of());
assertEquals(0, SimilarAsinTaskService.resolveAllRows(allEmpty).size());
}
@Test
void test_task_003_payload_boundary_single_item() {
// 单行新结构
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setItems(rows("uploads/20260829/single.xlsx", 1));
assertEquals(1, SimilarAsinTaskService.resolveAllRows(payload).size());
// 单行旧结构(仅 groups
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
legacy.setGroups(List.of(group(rows("uploads/20260829/single.xlsx", 1))));
assertEquals(1, SimilarAsinTaskService.resolveAllRows(legacy).size());
}
@Test
void test_task_003_payload_boundary_limit_and_overflow() {
// 旧结构 allItems 5000 行全量恢复,不丢失
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
legacy.setAllItems(rows("uploads/20260829/max.xlsx", 5000));
assertEquals(5000, SimilarAsinTaskService.resolveAllRows(legacy).size());
// items 与 allItems 同时存在:以 items 为准(新规范结构优先),不重复
SimilarAsinParsedPayloadDto both = new SimilarAsinParsedPayloadDto();
both.setItems(rows("uploads/20260829/both.xlsx", 10));
both.setAllItems(rows("uploads/20260829/both.xlsx", 20));
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(both);
assertEquals(10, restored.size());
// groups 也同时存在:仍以 items 为准
both.setGroups(List.of(group(rows("uploads/20260829/both.xlsx", 30))));
assertEquals(10, SimilarAsinTaskService.resolveAllRows(both).size());
}
@Test
void test_task_003_payload_invalid_input_rejected() {
// null payload:安全返回空列表(调用方容忍),不抛 NPE
assertEquals(0, SimilarAsinTaskService.resolveAllRows(null).size());
// groups 中含 null 元素:跳过不抛异常
SimilarAsinParsedPayloadDto messy = new SimilarAsinParsedPayloadDto();
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
groups.add(null);
groups.add(group(rows("uploads/20260829/messy.xlsx", 5)));
messy.setGroups(groups);
assertEquals(5, SimilarAsinTaskService.resolveAllRows(messy).size());
// group.items 为 null:跳过该组
SimilarAsinParsedGroupVo nullItemsGroup = new SimilarAsinParsedGroupVo();
nullItemsGroup.setItems(null);
SimilarAsinParsedPayloadDto nullItems = new SimilarAsinParsedPayloadDto();
nullItems.setGroups(List.of(nullItemsGroup, group(rows("uploads/20260829/n2.xlsx", 3))));
assertEquals(3, SimilarAsinTaskService.resolveAllRows(nullItems).size());
}
@Test
void test_task_003_payload_dependency_failure_releases_resources() throws Exception {
// 旧格式 JSON(只有 allItems)反序列化 → hydrate 后 resolveAllRows 恢复全量行
String legacyJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
+ "\"sourceFiles\":[],\"headers\":[],\"groups\":[],"
+ "\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0AAA00001\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":1},"
+ "{\"rowToken\":\"t2\",\"asin\":\"B0AAA00002\",\"sourceFileKey\":\"uploads/20260829/a.xlsx\",\"rowIndex\":2}]}";
SimilarAsinParsedPayloadDto payload = MAPPER.readValue(legacyJson, SimilarAsinParsedPayloadDto.class);
// 未 hydrate 时:allItems 恢复(items 为空走 allItems
List<SimilarAsinParsedRowVo> fromLegacy = SimilarAsinTaskService.resolveAllRows(payload);
assertEquals(2, fromLegacy.size());
assertEquals("t1", fromLegacy.get(0).getRowToken());
assertEquals("B0AAA00002", fromLegacy.get(1).getAsin());
// 新格式 JSON(只有 items)反序列化 → 直接恢复
String newJson = "{\"aiPrompt\":\"p\",\"apiKey\":\"k\",\"imgSwitch\":false,\"categorySwitch\":false,"
+ "\"sourceFiles\":[],\"headers\":[],"
+ "\"items\":[{\"rowToken\":\"n1\",\"asin\":\"B0NEW00001\",\"sourceFileKey\":\"uploads/20260829/b.xlsx\",\"rowIndex\":1}],"
+ "\"groups\":[]}";
SimilarAsinParsedPayloadDto newPayload = MAPPER.readValue(newJson, SimilarAsinParsedPayloadDto.class);
List<SimilarAsinParsedRowVo> fromNew = SimilarAsinTaskService.resolveAllRows(newPayload);
assertEquals(1, fromNew.size());
assertEquals("n1", fromNew.get(0).getRowToken());
// 两种格式行数之和互不影响,恢复结果稳定
assertTrue(fromLegacy.size() == 2 && fromNew.size() == 1);
}
@Test
void test_task_003_payload_normal_groups_expansion_preserves_order() {
// 最旧结构:仅 groups 嵌套行,展开后顺序稳定(按组、组内原序)
SimilarAsinParsedPayloadDto legacy = new SimilarAsinParsedPayloadDto();
List<SimilarAsinParsedRowVo> g1 = rows("uploads/20260829/g.xlsx", 2);
List<SimilarAsinParsedRowVo> g2 = rows("uploads/20260829/g.xlsx", 3);
// 组内行号各自独立从 1 开始(真实解析语义),第二组用不同 fileKey 区分来源
List<SimilarAsinParsedRowVo> g2b = rows("uploads/20260829/g2.xlsx", 3);
legacy.setGroups(List.of(group(g1), group(g2b)));
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacy);
assertEquals(5, restored.size());
assertEquals("uploads/20260829/g.xlsx::row::1", restored.get(0).getRowToken());
assertEquals("uploads/20260829/g.xlsx::row::2", restored.get(1).getRowToken());
assertEquals("uploads/20260829/g2.xlsx::row::1", restored.get(2).getRowToken());
assertEquals("uploads/20260829/g2.xlsx::row::2", restored.get(3).getRowToken());
assertEquals("uploads/20260829/g2.xlsx::row::3", restored.get(4).getRowToken());
}
}
@@ -0,0 +1,321 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 11Coze 结果合并的重复检测从 O(n²) 改为 HashSet/稳定 row key。
* dedupeRowsByRowKey 用 HashSet 按稳定 rowKey 一次性去重(保留顺序),
* mergeCozeRowsIntoChunk 合并前先去重,消除重复行逐行重复处理。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceRowKeyDedupeTest {
private static final AtomicLong NEXT_ID = new AtomicLong(70000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/70000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(id);
r.setAsin(asin);
r.setCountry(country);
return r;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7004L);
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(payloadJson);
return chunk;
}
private void stubSingleChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicLong storedCounter) throws Exception {
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
.thenReturn(payloadJson);
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storedCounter.incrementAndGet();
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
}
@Test
void test_task_011_merge_row_key_normal_default_path() throws Exception {
// 正常输入:llmRows 含同一 rowKey 的重复行,merge 前按稳定 rowKey 去重,
// chunk payload 只写一次,结果行不重复。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storedCounter = new AtomicLong(0);
stubSingleChunkMerge(chunk, rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))), storedCounter);
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
List<SimilarAsinResultRowDto> llmRows = List.of(
row("r1", "1", "B0A0000001", "英国"),
row("r1", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
assertEquals(1, deduped.size(), "重复行必须按稳定 rowKey 去重");
assertEquals("r1", deduped.get(0).getRowToken());
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, null, null, llmRows, Map.of());
assertEquals(1, storedCounter.get(), "去重后 chunk 只写一次");
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
@Test
void test_task_011_merge_row_key_normal_multiple_items() throws Exception {
// 批量场景:多个重复行跨 chunk 分组,去重后顺序稳定、结果不丢失
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
.thenReturn(rowsJson(List.of(row("r2", "2", "B0A0000002", "英国"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
AtomicLong selectOneRound = new AtomicLong(0);
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 0; i < 3; i++) {
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
llmRows.add(row("r2", "2", "B0A0000002", "英国"));
}
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
assertEquals(2, deduped.size(), "3 轮重复输入去重后只剩 2 个唯一行");
assertEquals(List.of("r1", "r2"), deduped.stream().map(SimilarAsinResultRowDto::getRowToken).toList(),
"去重必须保留首次出现顺序");
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, null, null, llmRows, Map.of());
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
@Test
void test_task_011_merge_row_key_normal_repeated_operation_is_idempotent() {
// 重复执行同一输入:去重结果完全一致,不产生重复记录
List<SimilarAsinResultRowDto> llmRows = List.of(
row("r1", "1", "B0A0000001", "英国"),
row("r2", "2", "B0A0000002", "英国"),
row("r1", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> first = service.dedupeRowsByRowKey(llmRows);
List<SimilarAsinResultRowDto> second = service.dedupeRowsByRowKey(llmRows);
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
assertEquals(first.get(i).getAsin(), second.get(i).getAsin());
}
}
@Test
void test_task_011_merge_row_key_boundary_empty_input() {
// 空输入:null/空列表安全返回空结果,不创建无效资源
assertNotNull(service.dedupeRowsByRowKey(null));
assertTrue(service.dedupeRowsByRowKey(null).isEmpty());
assertTrue(service.dedupeRowsByRowKey(List.of()).isEmpty());
// null 元素:跳过不抛异常
List<SimilarAsinResultRowDto> withNull = new ArrayList<>();
withNull.add(null);
withNull.add(row("r1", "1", "B0A0000001", "英国"));
assertEquals(1, service.dedupeRowsByRowKey(withNull).size());
}
@Test
void test_task_011_merge_row_key_boundary_single_item() {
// 单行:不依赖批量路径,去重后结果正确
List<SimilarAsinResultRowDto> single = List.of(row("r1", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(single);
assertEquals(1, deduped.size());
assertEquals("r1", deduped.get(0).getRowToken());
}
@Test
void test_task_011_merge_row_key_boundary_limit_and_overflow() {
// 大批量:1000 行全部重复,去重后只剩 1 个唯一行,无无界内存增长
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 0; i < 1000; i++) {
llmRows.add(row("r1", "1", "B0A0000001", "英国"));
}
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(llmRows);
assertEquals(1, deduped.size());
// 1000 行唯一:全部保留且顺序稳定
List<SimilarAsinResultRowDto> unique = new ArrayList<>();
for (int i = 0; i < 1000; i++) {
unique.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0U" + String.format("%06d", i), "英国"));
}
List<SimilarAsinResultRowDto> dedupedUnique = service.dedupeRowsByRowKey(unique);
assertEquals(1000, dedupedUnique.size());
assertEquals("r0001", dedupedUnique.get(1).getRowToken());
}
@Test
void test_task_011_merge_row_key_invalid_input_rejected() {
// 稳定 rowKey 冲突:rowToken 为空时用 legacy keyid::ASIN::country)识别重复
List<SimilarAsinResultRowDto> noToken = List.of(
row("", "1", "B0A0000001", "英国"),
row("", "1", "b0a0000001", " 英国 "));
List<SimilarAsinResultRowDto> deduped = service.dedupeRowsByRowKey(noToken);
assertEquals(1, deduped.size(), "legacy key 归一化(ASIN 大写、country trim)后应识别为同一行");
// 不同 ASIN:不误判为重复
List<SimilarAsinResultRowDto> diffAsin = List.of(
row("", "1", "B0A0000001", "英国"),
row("", "2", "B0A0000002", "英国"));
assertEquals(2, service.dedupeRowsByRowKey(diffAsin).size());
}
@Test
void test_task_011_merge_row_key_dependency_failure_releases_resources() throws Exception {
// chunk payload 读取失败:抛可识别业务异常且不写 chunk;
// 依赖恢复后重试成功,去重路径无残留状态
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenThrow(new IllegalStateException("rustfs down"));
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
Exception ex = assertThrows(Exception.class, () -> {
try {
merge.invoke(service, task, null, null,
List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of());
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
// 恢复后重试成功:只写一次,无重复记录
AtomicLong storedCounter = new AtomicLong(0);
stubSingleChunkMerge(chunk, rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))), storedCounter);
merge.invoke(service, task, null, null,
List.of(row("r1", "1", "B0A0000001", "英国"), row("r1", "1", "B0A0000001", "英国")), Map.of());
assertEquals(1, storedCounter.get());
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
}
@@ -0,0 +1,372 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinImagePrefetchService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 10:为 chunk 结果建立按 row key 的批量索引,消除跨 chunk 线性扫描。
* indexRowsByChunkKey 把每个 chunk 的行索引到 rowKey→chunkKeyllm 行归属从
* O(rows×chunks) 降为 O(1) 查找;assignLlmRowsToChunks 基于索引分配行并保留
* 原有命中/fallback/orphan 语义;集成用例验证每个 chunk 只读一次 payload。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceRowKeyIndexTest {
private static final AtomicLong NEXT_ID = new AtomicLong(60000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/60000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(id);
r.setAsin(asin);
r.setCountry(country);
return r;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7004L);
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(payloadJson);
return chunk;
}
/** 构造 rowsByChunkchunkStorageKey(scopeHash, chunkIndex) → rowKey 行表。 */
private static Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunkOf(String scopeHash, Integer chunkIndex, List<SimilarAsinResultRowDto> rows) {
Map<String, Map<String, SimilarAsinResultRowDto>> map = new LinkedHashMap<>();
Map<String, SimilarAsinResultRowDto> byKey = new LinkedHashMap<>();
for (SimilarAsinResultRowDto row : rows) {
byKey.put(row.getRowToken(), row);
}
map.put(scopeHash + ":" + chunkIndex, byKey);
return map;
}
private static List<String> assignedRowKeys(Map<String, Map<String, SimilarAsinResultRowDto>> merged) {
List<String> keys = new ArrayList<>();
for (Map<String, SimilarAsinResultRowDto> rows : merged.values()) {
for (String key : rows.keySet()) {
keys.add(key);
}
}
return keys;
}
@Test
void test_task_010_chunk_row_key_normal_default_path() throws Exception {
// 正常输入:2 个 chunk 各含行,llm 回传行按 rowKey 命中各自 chunk
// 每个 chunk 的 payload 只被读取一次(索引建立),消除跨 chunk 线性扫描。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "hashA", 1, "ptr:chunk-A"),
chunk(2L, "hashB", 2, "ptr:chunk-B"));
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国"))));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
.thenReturn(rowsJson(List.of(row("r3", "3", "B0A0000003", "美国"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
AtomicInteger selectOneRound = new AtomicInteger(0);
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
int i = selectOneRound.getAndIncrement();
return chunks.get(Math.min(i, chunks.size() - 1));
});
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"), row("r3", "3", "B0A0000003", "美国"));
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, null, null, llmRows, Map.of());
verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString());
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(2)).update(any(), any());
}
@Test
void test_task_010_chunk_row_key_normal_multiple_items() {
// 批量场景:3 个 chunk 各 3 行,9 个 llm 回传行全部命中且顺序稳定,无 orphan
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
for (int c = 0; c < 3; c++) {
rowsByChunk.putAll(rowsByChunkOf("hash" + c, c + 1,
List.of(row("c" + c + "r1", "1", "B0B" + c + "000001", "英国"),
row("c" + c + "r2", "2", "B0B" + c + "000002", "英国"),
row("c" + c + "r3", "3", "B0B" + c + "000003", "美国"))));
}
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int c = 0; c < 3; c++) {
for (int r = 1; r <= 3; r++) {
llmRows.add(row("c" + c + "r" + r, String.valueOf(r), "B0B" + c + "00000" + r, r == 3 ? "美国" : "英国"));
}
}
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, orphans);
assertEquals(3, merged.size());
assertEquals(9, assignedRowKeys(merged).size());
assertTrue(orphans.isEmpty(), "全部命中,不应产生 orphan");
for (Map<String, SimilarAsinResultRowDto> rows : merged.values()) {
assertEquals(3, rows.size());
}
}
@Test
void test_task_010_chunk_row_key_normal_repeated_operation_is_idempotent() {
// 重复执行同一输入:结果完全一致,不产生重复记录
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1,
List.of(row("r1", "1", "B0A0000001", "英国"), row("r2", "2", "B0A0000002", "英国")));
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "1", "B0A0000001", "英国"));
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> first = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
Map<String, Map<String, SimilarAsinResultRowDto>> second = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, new ArrayList<>());
assertEquals(assignedRowKeys(first), assignedRowKeys(second));
assertEquals(first.size(), second.size());
for (Map.Entry<String, Map<String, SimilarAsinResultRowDto>> entry : first.entrySet()) {
assertEquals(entry.getValue().keySet(), second.get(entry.getKey()).keySet());
}
}
@Test
void test_task_010_chunk_row_key_boundary_empty_input() {
// 空输入:null/空 rowsByChunk 与 llmRows 均安全返回空结果,不创建无效资源
assertNotNull(service.indexRowsByChunkKey(null));
assertTrue(service.indexRowsByChunkKey(null).isEmpty());
assertTrue(service.indexRowsByChunkKey(Map.of()).isEmpty());
Map<String, Map<String, SimilarAsinResultRowDto>> emptyAssign = service.assignLlmRowsToChunks(
Map.of(), List.of(), Map.of(), null, null, new ArrayList<>());
assertTrue(emptyAssign.isEmpty());
assertTrue(service.assignLlmRowsToChunks(
Map.of(), null, Map.of(), null, null, new ArrayList<>()).isEmpty());
// 无可匹配行(rowKey 不存在于任何 chunk)→ 进 orphan 兜底,不产生 merge
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1, List.of(row("r1", "1", "B0A0000001", "英国")));
List<SimilarAsinResultRowDto> blankRow = List.of(row("", "", "", ""));
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, blankRow, index, null, null, orphans);
assertTrue(assignedRowKeys(merged).isEmpty());
assertEquals(1, orphans.size(), "全空行生成 legacy key :::: 不命中任何 chunk,按既有语义进 orphan");
}
@Test
void test_task_010_chunk_row_key_boundary_single_item() {
// 单 chunk 单行:不依赖批量路径,命中正确
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = rowsByChunkOf("hashA", 1,
List.of(row("r1", "1", "B0A0000001", "英国")));
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, List.of(row("r1", "1", "B0A0000001", "英国")), index, null, null, orphans);
assertEquals(1, merged.size());
assertEquals(List.of("r1"), assignedRowKeys(merged));
assertTrue(orphans.isEmpty());
// 索引也只含该行
assertEquals(1, index.size());
assertEquals("hashA:1", index.get("r1"));
}
@Test
void test_task_010_chunk_row_key_boundary_limit_and_overflow() {
// 大批量:1000 行索引 + 500 个 llm 回传行全部命中,行不丢、无 orphan
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
Map<String, SimilarAsinResultRowDto> bigChunk = new LinkedHashMap<>();
for (int i = 1; i <= 1000; i++) {
bigChunk.put("r" + String.format("%04d", i), row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
}
rowsByChunk.put("hashBig:1", bigChunk);
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
assertEquals(1000, index.size());
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 1; i <= 500; i++) {
llmRows.add(row("r" + String.format("%04d", i), String.valueOf(i), "B0L" + String.format("%06d", i), "英国"));
}
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, llmRows, index, null, null, orphans);
assertEquals(1, merged.size());
assertEquals(500, assignedRowKeys(merged).size());
assertTrue(orphans.isEmpty());
}
@Test
void test_task_010_chunk_row_key_invalid_input_rejected() {
// 同一 rowKey 出现在多个 chunk:索引保留第一个 chunkputIfAbsent),行为确定
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
rowsByChunk.putAll(rowsByChunkOf("hashA", 1, List.of(row("dup", "1", "B0A0000001", "英国"))));
rowsByChunk.putAll(rowsByChunkOf("hashB", 2, List.of(row("dup", "1", "B0A0000001", "英国"))));
Map<String, String> index = service.indexRowsByChunkKey(rowsByChunk);
assertEquals("hashA:1", index.get("dup"), "重复 rowKey 应保留第一个 chunk");
// fallback 缺失:llm 行未命中且无有效 fallback → 进 orphan,不产生 merge
List<SimilarAsinResultRowDto> orphans = new ArrayList<>();
Map<String, Map<String, SimilarAsinResultRowDto>> merged = service.assignLlmRowsToChunks(
rowsByChunk, List.of(row("ghost", "9", "B0A0000009", "英国")), Map.of(), "missingHash", 99, orphans);
assertTrue(assignedRowKeys(merged).isEmpty());
assertEquals(1, orphans.size());
assertEquals("ghost", orphans.get(0).getRowToken());
// llmRows 含 null 元素:跳过不抛异常,其余行正常分配
List<SimilarAsinResultRowDto> withNull = new ArrayList<>();
withNull.add(null);
withNull.add(row("dup", "1", "B0A0000001", "英国"));
List<SimilarAsinResultRowDto> orphans2 = new ArrayList<>();
Map<String, String> index2 = service.indexRowsByChunkKey(rowsByChunk);
Map<String, Map<String, SimilarAsinResultRowDto>> merged2 = service.assignLlmRowsToChunks(
rowsByChunk, withNull, index2, null, null, orphans2);
assertEquals(1, merged2.size());
assertEquals(List.of("dup"), assignedRowKeys(merged2));
assertTrue(orphans2.isEmpty());
}
@Test
void test_task_010_chunk_row_key_dependency_failure_releases_resources() throws Exception {
// chunk payload 读取失败:抛可识别业务异常且不产生部分 merge;
// 依赖恢复后重试成功,无残留状态
List<TaskChunkEntity> chunks = List.of(chunk(1L, "hashA", 1, "ptr:chunk-A"));
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenThrow(new IllegalStateException("rustfs down"));
FileTaskEntity task = new FileTaskEntity();
task.setId(7004L);
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
BusinessException ex = assertThrows(BusinessException.class, () -> {
try {
merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of());
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
// 恢复后重试成功:行合并到正确 chunk
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r1", "1", "B0A0000001", "英国"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenReturn("stored:retry");
when(taskChunkMapper.selectOne(any())).thenReturn(chunks.get(0));
when(taskChunkMapper.update(any(), any())).thenReturn(1);
merge.invoke(service, task, null, null, List.of(row("r1", "1", "B0A0000001", "英国")), Map.of());
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
}
@@ -0,0 +1,362 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceSubmitTest {
private static final Long TASK_ID = 21879L;
private static final String PARSED_POINTER = "rustfs:task-parsed/similar-asin/21879/payload.json";
private static final String CHUNK_POINTER = "rustfs:task-chunk/similar-asin/21879/chunk.json";
private static final String STORED_CHUNK_POINTER = "\"" + CHUNK_POINTER + "\"";
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@Mock private TransactionStatus transactionStatus;
@InjectMocks private SimilarAsinTaskService service;
private final AtomicBoolean transactionActive = new AtomicBoolean();
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUpTransactionAndLock() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE),
anyLong(),
any(Duration.class),
eq(10_000L)))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
transactionActive.set(true);
return transactionStatus;
});
lenient().doAnswer(invocation -> {
transactionActive.set(false);
return null;
}).when(transactionManager).commit(transactionStatus);
lenient().doAnswer(invocation -> {
transactionActive.set(false);
return null;
}).when(transactionManager).rollback(transactionStatus);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdownExecutors() {
service.shutdownAssembleExecutor();
}
@Test
void doneCallbackReadsPayloadOnlyBeforeShortTransaction() throws Exception {
FileTaskEntity task = runningTask("instance-a");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
configureNewChunkAndScope();
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
List<Boolean> storageCallTransactionStates = new ArrayList<>();
when(transientPayloadStorageService.resolvePayload(eq(PARSED_POINTER), anyString()))
.thenAnswer(invocation -> {
storageCallTransactionStates.add(transactionActive.get());
return parsedPayloadJson(List.of(new SimilarAsinParsedRowVo(), new SimilarAsinParsedRowVo()));
});
when(transientPayloadStorageService.storeChunkPayloadVersioned(
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
.thenAnswer(invocation -> {
storageCallTransactionStates.add(transactionActive.get());
return STORED_CHUNK_POINTER;
});
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(false);
doAnswer(invocation -> {
assertTrue(transactionActive.get());
FileResultEntity result = invocation.getArgument(0);
result.setId(501L);
return 1;
}).when(fileResultMapper).insert(any(FileResultEntity.class));
doAnswer(invocation -> {
storageCallTransactionStates.add(transactionActive.get());
return null;
}).when(taskCacheService).deleteTaskCache(TASK_ID);
service.submitResult(TASK_ID, request(true));
assertFalse(storageCallTransactionStates.isEmpty());
assertTrue(storageCallTransactionStates.stream().noneMatch(Boolean::booleanValue));
verify(transientPayloadStorageService).resolvePayload(eq(PARSED_POINTER), anyString());
ArgumentCaptor<FileResultEntity> resultCaptor = ArgumentCaptor.forClass(FileResultEntity.class);
verify(fileResultMapper).insert(resultCaptor.capture());
assertEquals(2, resultCaptor.getValue().getRowCount());
assertEquals("germany.xlsx", resultCaptor.getValue().getSourceFilename());
}
@Test
void unknownCommitOutcomeDoesNotDeletePossiblyCommittedChunk() throws Exception {
FileTaskEntity task = runningTask("instance-a");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
configureNewChunkAndScope();
when(taskChunkMapper.selectCount(any())).thenReturn(0L);
configureChunkStore(false);
doAnswer(invocation -> {
transactionActive.set(false);
throw new IllegalStateException("commit ACK lost");
}).when(transactionManager).commit(transactionStatus);
IllegalStateException thrown = assertThrows(IllegalStateException.class,
() -> service.submitResult(TASK_ID, request(false)));
assertEquals("commit ACK lost", thrown.getMessage());
verify(transientPayloadStorageService, never()).extractPointer(anyString());
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
verify(taskCacheService, never()).touchTaskHeartbeat(TASK_ID);
}
@Test
void duplicateAlreadyPersistedChunkDoesNotRunPayloadCleanup() throws Exception {
FileTaskEntity task = runningTask("instance-a");
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
when(taskChunkMapper.selectOne(any())).thenReturn(chunk("\"rustfs:winner\""));
configureScopeStorage(null);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.submitResult(TASK_ID, request(false));
verify(transientPayloadStorageService, never()).extractPointer(anyString());
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
}
@Test
void localFallbackOwnerFailureRollsBackChunkAndKeepsCandidate() throws Exception {
FileTaskEntity task = runningTask(null);
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
configureNewChunkAndScope();
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
configureChunkStore(true);
when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(0);
IllegalStateException thrown = assertThrows(IllegalStateException.class,
() -> service.submitResult(TASK_ID, request(false)));
assertTrue(thrown.getMessage().contains("Failed to bind local fallback task owner"));
verify(taskChunkMapper).insert(any(TaskChunkEntity.class));
verify(transactionManager).rollback(transactionStatus);
verify(transactionManager, never()).commit(transactionStatus);
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
}
@Test
void duplicateLocalCandidateIsNotBoundAndReferencedPayloadIsKept() throws Exception {
FileTaskEntity task = runningTask(null);
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
TaskChunkEntity winner = chunk(STORED_CHUNK_POINTER);
when(taskChunkMapper.selectOne(any())).thenReturn(null, winner);
configureScopeStorage(null);
configureChunkStore(true);
when(transientPayloadStorageService.extractPointer(STORED_CHUNK_POINTER)).thenReturn(CHUNK_POINTER);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.submitResult(TASK_ID, request(false));
verify(taskChunkMapper, never()).insert(any(TaskChunkEntity.class));
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(STORED_CHUNK_POINTER);
}
@Test
void repeatedNonFinalCallbackCannotClearCompletedScope() throws Exception {
FileTaskEntity task = runningTask("instance-a");
task.setResultJson("{\"ownerInstanceId\":\"instance-a\"}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
TaskChunkEntity existing = chunk("\"rustfs:winner\"");
when(taskChunkMapper.selectOne(any())).thenReturn(existing);
TaskScopeStateEntity scope = scope(1);
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.submitResult(TASK_ID, request(false));
assertEquals(1, scope.getCompleted());
verify(taskScopeStateMapper).updateById(scope);
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(
anyString(), anyLong(), anyString(), any(), anyString());
}
private String parsedPayloadJson(List<SimilarAsinParsedRowVo> rows) throws Exception {
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setAllItems(rows);
payload.setItems(rows);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey("uploads/germany.xlsx");
sourceFile.setOriginalFilename("germany.xlsx");
payload.setSourceFiles(List.of(sourceFile));
return objectMapper.writeValueAsString(payload);
}
private void configureNewChunkAndScope() {
AtomicReference<TaskChunkEntity> chunkRef = new AtomicReference<>();
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> chunkRef.get());
doAnswer(invocation -> {
TaskChunkEntity chunk = invocation.getArgument(0);
chunk.setId(301L);
chunkRef.set(chunk);
return 1;
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
configureScopeStorage(null);
}
private void configureScopeStorage(TaskScopeStateEntity initial) {
AtomicReference<TaskScopeStateEntity> scopeRef = new AtomicReference<>(initial);
when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> scopeRef.get());
doAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
scope.setId(401L);
scopeRef.set(scope);
return 1;
}).when(taskScopeStateMapper).insert(any(TaskScopeStateEntity.class));
}
private void configureChunkStore(boolean localFallback) {
when(transientPayloadStorageService.storeChunkPayloadVersioned(
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
.thenReturn(STORED_CHUNK_POINTER);
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(localFallback);
}
private FileTaskEntity runningTask(String owner) throws Exception {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
task.setStatus("RUNNING");
task.setUserId(7L);
String resultJson = "{\"parsedPayloadRef\":\"" + PARSED_POINTER + "\"";
if (owner != null) {
resultJson += ",\"ownerInstanceId\":\"" + owner + "\"";
}
task.setResultJson(resultJson + "}");
return task;
}
private SimilarAsinSubmitResultRequest request(boolean done) {
SimilarAsinSubmitResultRequest request = new SimilarAsinSubmitResultRequest();
request.setSubmissionId("similar-asin-21879");
request.setChunkIndex(0);
request.setChunkTotal(1);
request.setDone(done);
return request;
}
private TaskChunkEntity chunk(String payload) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(301L);
chunk.setTaskId(TASK_ID);
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
chunk.setScopeHash("existing-scope");
chunk.setChunkIndex(0);
chunk.setChunkTotal(1);
chunk.setPayloadJson(payload);
return chunk;
}
private TaskScopeStateEntity scope(int completed) {
TaskScopeStateEntity scope = new TaskScopeStateEntity();
scope.setId(401L);
scope.setTaskId(TASK_ID);
scope.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
scope.setScopeKey("similar-asin-21879");
scope.setScopeHash("existing-scope");
scope.setChunkTotal(1);
scope.setCompleted(completed);
scope.setStateJson("{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
return scope;
}
}
@@ -0,0 +1,121 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SimilarAsinTaskServiceTest {
@Test
@SuppressWarnings("unchecked")
void resultWorkbookRestoresPriceAfterCountryAndShiftsImageColumns() throws Exception {
Field headersField = SimilarAsinTaskService.class.getDeclaredField("RESULT_HEADERS");
headersField.setAccessible(true);
List<String> headers = (List<String>) headersField.get(null);
assertEquals(List.of(
"id", "asin", "国家", "价格", "卖家名称", "品牌", "是否有货", "相似度",
"是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"
), headers);
assertEquals(12, staticIntField("IMG_COL_MAIN"));
assertEquals(13, staticIntField("IMG_COL_PUZZLE1"));
assertEquals(14, staticIntField("IMG_COL_PUZZLE2"));
}
@Test
void resultStatusUsesReturnedCozeDataAndImages() {
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(null));
SimilarAsinResultRowDto empty = new SimilarAsinResultRowDto();
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(empty));
SimilarAsinResultRowDto withImage = new SimilarAsinResultRowDto();
withImage.setStatus("FAILED");
withImage.setMainUrl("https://example.com/main.jpg");
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withImage));
SimilarAsinResultRowDto withCozeStatus = new SimilarAsinResultRowDto();
withCozeStatus.setStatus("success");
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withCozeStatus));
SimilarAsinResultRowDto failedStatus = new SimilarAsinResultRowDto();
failedStatus.setStatus("FAILED");
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(failedStatus));
SimilarAsinResultRowDto withVisibleResultData = new SimilarAsinResultRowDto();
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
}
@Test
void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() {
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3));
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("FAILED", 3, 3));
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("RUNNING", 3, 3));
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3));
}
@Test
void failedResultRowsWithBlankIsConformEnableCategoryRetry() {
SimilarAsinParsedRowVo blankCategory = new SimilarAsinParsedRowVo();
blankCategory.setValues(new LinkedHashMap<>());
blankCategory.getValues().put("status", "FAILED");
blankCategory.getValues().put("is_conform", "");
assertTrue(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
List.of("id", "asin", "country", "is_conform", "status"),
List.of(blankCategory),
true));
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
List.of("id", "asin", "country", "is_conform", "status"),
List.of(blankCategory),
false));
blankCategory.getValues().put("is_conform", "符合");
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
List.of("id", "asin", "country", "is_conform", "status"),
List.of(blankCategory),
true));
}
@Test
void firstPassImageResultWorkbookKeepsAllRowsForSecondParse() {
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
row.setValues(new LinkedHashMap<>());
row.getValues().put("状态", "成功");
row.getValues().put("是否有货", "");
row.getValues().put("相似度", "");
row.getValues().put("是否符合类目", "");
row.getValues().put("不符合理由", "");
row.getValues().put("产品类目", "");
assertTrue(SimilarAsinTaskService.isFirstPassResultWorkbook(
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
List.of(row)));
row.getValues().put("是否符合类目", "符合");
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
List.of(row)));
row.getValues().put("是否符合类目", "");
row.getValues().put("状态", "失败");
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
List.of(row)));
}
private int staticIntField(String name) throws Exception {
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
field.setAccessible(true);
return field.getInt(null);
}
}
@@ -0,0 +1,269 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 8WorkbookFactory 输入解析改为受控读取。
* 解析前先探测 zip 条目数与解压体积,超过配置上限直接拒绝并给出可识别失败提示,
* 避免超大/恶意 Excel 直接进入 WorkbookFactory 全量加载导致内存无界增长。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceWorkbookControlTest {
private static final AtomicLong NEXT_ID = new AtomicLong(50000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
File file = Files.createTempFile("similar-asin-workbook-ctrl-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(String.format("B0WBK%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("workbook.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_008_workbook_excel_normal_default_path() throws Exception {
// 正常 xlsx:受控读取通过探测,解析成功且行数不丢失
File workbook = buildWorkbook(100);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-default.xlsx");
assertEquals(100, vo.getAcceptedRows());
assertEquals(100, vo.getTotalRows());
assertEquals(100, vo.getItems().size());
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
}
@Test
void test_task_008_workbook_excel_normal_multiple_items() throws Exception {
// 多文件批量:每个文件都走受控读取,汇总不丢行
File workbookA = buildWorkbook(25);
File workbookB = buildWorkbook(35);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-a.xlsx")).thenReturn(workbookA);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-b.xlsx")).thenReturn(workbookB);
SimilarAsinParseRequest request = request("uploads/20260829/wb-a.xlsx");
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
sourceB.setFileKey("uploads/20260829/wb-b.xlsx");
sourceB.setOriginalFilename("workbook-b.xlsx");
request.setFiles(List.of(request.getFiles().get(0), sourceB));
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
assertEquals(60, vo.getAcceptedRows());
assertEquals(60, vo.getTotalRows());
assertNotNull(vo.getTaskId());
}
@Test
void test_task_008_workbook_excel_normal_repeated_operation_is_idempotent() throws Exception {
// 重复解析同一文件:结果一致,不产生重复状态
File workbook = buildWorkbook(50);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/wb-idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/wb-idem.xlsx");
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_008_workbook_excel_boundary_empty_input() throws Exception {
// 空文件(只有表头无数据行):抛业务异常,不创建任务
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-empty.xlsx")).thenReturn(workbook);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/wb-empty.xlsx"));
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
// 文件不存在:抛业务异常
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-missing.xlsx")).thenReturn(null);
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/wb-missing.xlsx"));
}
@Test
void test_task_008_workbook_excel_boundary_single_item() throws Exception {
// 单行文件:不依赖批量路径,结果正确
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-single.xlsx");
assertEquals(1, vo.getAcceptedRows());
assertEquals(1, vo.getItems().size());
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
assertEquals(1, vo.getGroupCount());
}
@Test
void test_task_008_workbook_excel_boundary_limit_and_overflow() throws Exception {
// 解压体积超限:受控探测阶段拒绝,失败提示可识别,不发生无界内存增长
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(1024L);
File workbook = buildWorkbook(5);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-huge.xlsx")).thenReturn(workbook);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/wb-huge.xlsx"));
assertTrue(ex.getMessage() != null && (ex.getMessage().contains("解压") || ex.getMessage().contains("大小")),
"超大 Excel 失败提示必须可识别,实际: " + ex.getMessage());
// 条目数超限:同样在探测阶段拒绝
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
when(properties.getMaxWorkbookZipEntries()).thenReturn(2);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-entries.xlsx")).thenReturn(workbook);
BusinessException ex2 = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/wb-entries.xlsx"));
assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("条目"),
"超条目数失败提示必须可识别,实际: " + ex2.getMessage());
// 恢复默认配置:同一文件解析成功(探测不残留状态)
when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered.xlsx");
assertEquals(5, vo.getAcceptedRows());
}
@Test
void test_task_008_workbook_excel_invalid_input_rejected() throws Exception {
// 非 xlsx 内容(文本文件):WorkbookFactory 打开失败,抛项目约定异常
File fake = Files.createTempFile("similar-asin-not-excel-", ".xlsx").toFile();
Files.write(fake.toPath(), "this is not an excel file".getBytes(StandardCharsets.UTF_8));
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fake.xlsx")).thenReturn(fake);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(fake, "uploads/20260829/wb-fake.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"),
"损坏文件失败提示必须可识别,实际: " + ex.getMessage());
}
@Test
void test_task_008_workbook_excel_dependency_failure_releases_resources() throws Exception {
// RustFS 存储失败:解析抛异常;恢复后重试成功,受控读取无残留
File workbook = buildWorkbook(30);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/50001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/wb-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered2.xlsx");
assertEquals(30, vo.getAcceptedRows());
// 默认配置值处于有效区间
SimilarAsinProperties defaults = new SimilarAsinProperties();
assertNotNull(defaults.getMaxWorkbookZipEntries());
assertNotNull(defaults.getMaxWorkbookUncompressedBytes());
assertTrue(defaults.getMaxWorkbookZipEntries() >= 1000, "默认条目上限至少 1000");
assertTrue(defaults.getMaxWorkbookUncompressedBytes() >= 100L * 1024L * 1024L, "默认解压上限至少 100MB");
}
}
@@ -0,0 +1,205 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.CreationHelper;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.util.Date;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 84:行解析边界测试。
* 超长字段(按现状截断 2000)、畸形 ASIN 原样透传、BOM 剥离、混合编码(全角空格折叠)、
* 数值/日期/公式单元格转文本、大表(5000 行)性能上界。
*/
class SimilarAsinExcelParserBoundaryTest {
private final SimilarAsinExcelParser parser = new SimilarAsinExcelParser();
private static final String[] HEADERS = {"id", "asin", "国家", "sku", "价格", "url", "标题"};
private File toTempFile(XSSFWorkbook wb) throws Exception {
File file = File.createTempFile("similar-asin-boundary-", ".xlsx");
file.deleteOnExit();
try (FileOutputStream out = new FileOutputStream(file)) {
wb.write(out);
}
wb.close();
return file;
}
private XSSFWorkbook emptyWorkbook() {
XSSFWorkbook wb = new XSSFWorkbook();
Sheet sheet = wb.createSheet("Sheet1");
Row header = sheet.createRow(0);
for (int i = 0; i < HEADERS.length; i++) {
header.createCell(i).setCellValue(HEADERS[i]);
}
return wb;
}
@Test
void test_parse_long_field() throws Exception {
XSSFWorkbook wb = emptyWorkbook();
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(1);
row.createCell(0).setCellValue("long-1");
row.createCell(1).setCellValue("B01LONG");
row.createCell(2).setCellValue("US");
String longTitle = "A".repeat(2500);
row.createCell(6).setCellValue(longTitle);
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
String title = parsed.rows().get(0).title();
assertEquals(2000, title.length(), "超长字段按现状截断到 2000");
assertEquals("A".repeat(2000), title);
assertEquals("A".repeat(2500).substring(0, 2000), title);
assertEquals(2000, parsed.rows().get(0).values().get("标题").length(), "values 映射同样截断");
}
@Test
void test_parse_malformed_asin() throws Exception {
XSSFWorkbook wb = emptyWorkbook();
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(1);
row.createCell(0).setCellValue("bad-1");
row.createCell(1).setCellValue("b01@# $%^&*()");
row.createCell(2).setCellValue("US");
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals("B01@# $%^&*()", parsed.rows().get(0).asin(), "畸形 ASIN 原样透传(仅大小写归一)");
}
@Test
void test_parse_bom() throws Exception {
XSSFWorkbook wb = new XSSFWorkbook();
Sheet sheet = wb.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
Row row = sheet.createRow(1);
row.createCell(0).setCellValue("bom-1");
row.createCell(1).setCellValue("B01BOM");
row.createCell(2).setCellValue("US");
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals("id", parsed.headers().get(0), "BOM 头从表头剥离");
assertEquals("B01BOM", parsed.rows().get(0).asin(), "BOM 头从单元格值剥离");
}
@Test
void test_parse_mixed_encoding() throws Exception {
XSSFWorkbook wb = emptyWorkbook();
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(1);
row.createCell(0).setCellValue("mix-1");
row.createCell(1).setCellValue("B01MIX");
row.createCell(2).setCellValue("US");
row.createCell(6).setCellValue("中文 标题 ABC DEF   GH");
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals("中文 标题 ABC DEF GH".replace(" GH", " GH"), parsed.rows().get(0).title(),
"混合编码单元格保留内容");
String title = parsed.rows().get(0).title();
assertTrue(title.startsWith("中文 标题 ABC"), "中文+拉丁混排保留, actual=" + title);
assertEquals("中文 标题 ABC DEF GH", title, "全角空格转半角并折叠连续空白");
}
@Test
void test_parse_numeric_cell() throws Exception {
XSSFWorkbook wb = emptyWorkbook();
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(1);
row.createCell(0).setCellValue(1234.5);
row.createCell(1).setCellValue("B01NUM");
row.createCell(2).setCellValue("US");
row.createCell(4).setCellValue(19.9);
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals("1234.5", parsed.rows().get(0).id(), "数值单元格转文本");
assertEquals("19.9", parsed.rows().get(0).price(), "价格列数值转文本");
}
@Test
void test_parse_date_cell() throws Exception {
XSSFWorkbook wb = emptyWorkbook();
Sheet sheet = wb.getSheetAt(0);
CreationHelper helper = wb.getCreationHelper();
CellStyle dateStyle = wb.createCellStyle();
dateStyle.setDataFormat(helper.createDataFormat().getFormat("yyyy-MM-dd"));
Row row = sheet.createRow(1);
row.createCell(0).setCellValue("date-1");
row.createCell(1).setCellValue("B01DATE");
row.createCell(2).setCellValue("US");
row.createCell(4).setCellValue(new Date(1767225600000L)); // 2026-01-02 UTC
row.getCell(4).setCellStyle(dateStyle);
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals("2026-01-01", parsed.rows().get(0).price(), "日期单元格按格式转文本(POI UTC 渲染)");
}
@Test
void test_parse_formula_cell() throws Exception {
XSSFWorkbook wb = emptyWorkbook();
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(1);
row.createCell(0).setCellValue("formula-1");
row.createCell(1).setCellValue("B01FML");
row.createCell(2).setCellValue("US");
row.createCell(4).setCellFormula("19.9*2");
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
// 现状语义:DataFormatter 对无缓存值的公式返回公式串原样(解析器不做求值)
assertEquals("19.9*2", parsed.rows().get(0).price(), "公式单元格取缓存值(无缓存值则保留公式串)");
}
@Test
void test_parse_large_sheet() throws Exception {
XSSFWorkbook wb = emptyWorkbook();
Sheet sheet = wb.getSheetAt(0);
for (int r = 1; r <= 5000; r++) {
Row row = sheet.createRow(r);
row.createCell(0).setCellValue("row-" + r);
row.createCell(1).setCellValue("B0" + String.format("%04d", r));
row.createCell(2).setCellValue(r % 2 == 0 ? "US" : "DE");
row.createCell(3).setCellValue("SKU-" + r);
row.createCell(4).setCellValue(10.0 + r);
row.createCell(5).setCellValue("http://img/" + r + ".jpg");
row.createCell(6).setCellValue("title " + r);
}
File file = toTempFile(wb);
long started = System.nanoTime();
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
long elapsedMs = (System.nanoTime() - started) / 1_000_000;
assertEquals(5000, parsed.rows().size(), "5000 行全部解析");
assertEquals("row-5000", parsed.rows().get(4999).id());
assertEquals("B05000", parsed.rows().get(4999).asin(), "大写归一后原样透传");
assertNotNull(parsed.rows().get(4999).values());
assertTrue(elapsedMs < 10_000, "5000 行解析耗时上界 10s, actual=" + elapsedMs + "ms");
}
}
@@ -0,0 +1,199 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.nio.file.Files;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 83SimilarAsinExcelParser 行解析器。
* POI 读 Excel 行 → 中间行对象(原始单元格值);列序映射、空行跳过、空输入/坏文件异常、流不被篡改。
*/
class SimilarAsinExcelParserTest {
private final SimilarAsinExcelParser parser = new SimilarAsinExcelParser();
private XSSFWorkbook workbook(String[] headers, String[][] rows) {
XSSFWorkbook wb = new XSSFWorkbook();
Sheet sheet = wb.createSheet("Sheet1");
Row header = sheet.createRow(0);
for (int i = 0; i < headers.length; i++) {
header.createCell(i).setCellValue(headers[i]);
}
for (int r = 0; r < rows.length; r++) {
Row row = sheet.createRow(r + 1);
for (int c = 0; c < rows[r].length; c++) {
row.createCell(c).setCellValue(rows[r][c]);
}
}
return wb;
}
private File toTempFile(XSSFWorkbook wb) throws Exception {
File file = File.createTempFile("similar-asin-parser-", ".xlsx");
file.deleteOnExit();
try (FileOutputStream out = new FileOutputStream(file)) {
wb.write(out);
}
wb.close();
return file;
}
private byte[] toBytes(XSSFWorkbook wb) throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
wb.write(out);
wb.close();
return out.toByteArray();
}
private static final String[] HEADERS = {"id", "asin", "国家", "sku", "价格", "url", "标题"};
@Test
void test_parse_normal_rows() throws Exception {
File file = toTempFile(workbook(HEADERS, new String[][]{
{"row-1", "B01ABC", "US", "SKU-1", "19.9", "http://img/1.jpg", "title one"},
{"row-2", "B02DEF", "DE", "SKU-2", "29.9", "http://img/2.jpg", "title two"}
}));
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals(2, parsed.rows().size());
SimilarAsinExcelParser.SimilarAsinExcelRow first = parsed.rows().get(0);
assertEquals("row-1", first.id());
assertEquals("B01ABC", first.asin());
assertEquals("US", first.country());
assertEquals("SKU-1", first.sku());
assertEquals("19.9", first.price());
assertEquals("http://img/1.jpg", first.url());
assertEquals("title one", first.title());
assertEquals(2, first.rowIndex());
SimilarAsinExcelParser.SimilarAsinExcelRow second = parsed.rows().get(1);
assertEquals("row-2", second.id());
assertEquals("DE", second.country());
assertEquals(3, second.rowIndex());
}
@Test
void test_parse_single_row() throws Exception {
File file = toTempFile(workbook(HEADERS, new String[][]{{"only-1", "B00SINGLE", "US", "", "", "", "single"}}));
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals(1, parsed.rows().size());
SimilarAsinExcelParser.SimilarAsinExcelRow row = parsed.rows().get(0);
assertEquals("only-1", row.id());
assertEquals("B00SINGLE", row.asin());
assertEquals("US", row.country());
assertEquals(2, row.rowIndex());
assertNotNull(row.values());
assertEquals(HEADERS.length, row.values().size());
}
@Test
void test_parse_columns_mapping() throws Exception {
String[] scrambled = {"价格", "asin", "标题", "id", "url", "国家"};
File file = toTempFile(workbook(scrambled, new String[][]{{"100", "B01MAP", "mapped title", "row-1", "http://img/map.jpg", "US"}}));
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
SimilarAsinExcelParser.SimilarAsinExcelRow row = parsed.rows().get(0);
assertEquals("row-1", row.id());
assertEquals("B01MAP", row.asin());
assertEquals("US", row.country());
assertEquals("100", row.price());
assertEquals("http://img/map.jpg", row.url());
assertEquals("mapped title", row.title());
assertEquals("", row.sku());
assertEquals(scrambled.length, row.values().size());
assertEquals("row-1", row.values().get("id"));
assertEquals("B01MAP", row.values().get("asin"));
}
@Test
void test_parse_empty_sheet() throws Exception {
File file = toTempFile(workbook(HEADERS, new String[0][]));
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertTrue(parsed.rows().isEmpty());
assertEquals(HEADERS.length, parsed.headers().size());
assertEquals("id", parsed.headers().get(0));
}
@Test
void test_parse_empty_rows_skipped() throws Exception {
XSSFWorkbook wb = new XSSFWorkbook();
Sheet sheet = wb.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
Row data1 = sheet.createRow(1);
data1.createCell(0).setCellValue("keep-1");
data1.createCell(1).setCellValue("B01KEEP");
data1.createCell(2).setCellValue("US");
Row blank = sheet.createRow(2);
blank.createCell(0).setCellValue("");
blank.createCell(1).setCellValue("");
blank.createCell(2).setCellValue("");
// 第 4 行(索引 3)故意不创建 → getRow 返回 null
Row data2 = sheet.createRow(4);
data2.createCell(0).setCellValue("keep-2");
data2.createCell(1).setCellValue("B02KEEP");
data2.createCell(2).setCellValue("DE");
File file = toTempFile(wb);
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
assertEquals(2, parsed.rows().size());
assertEquals(2, parsed.rows().get(0).rowIndex());
assertEquals("keep-1", parsed.rows().get(0).id());
assertEquals(5, parsed.rows().get(1).rowIndex());
assertEquals("keep-2", parsed.rows().get(1).id());
}
@Test
void test_parse_null_input_throws() {
assertThrows(IllegalArgumentException.class, () -> parser.parse((InputStream) null));
assertThrows(IllegalArgumentException.class, () -> parser.parse((File) null));
}
@Test
void test_parse_unsupported_format() throws Exception {
File garbage = File.createTempFile("similar-asin-bad-", ".xlsx");
garbage.deleteOnExit();
Files.write(garbage.toPath(), "this is definitely not an excel file".getBytes());
BusinessException ex = assertThrows(BusinessException.class, () -> parser.parse(garbage));
assertTrue(ex.getMessage().contains("解析 Excel 失败"), "非 Excel 文件应转业务异常, actual=" + ex.getMessage());
}
@Test
void test_parse_no_mutation() throws Exception {
byte[] content = toBytes(workbook(HEADERS, new String[][]{
{"row-1", "B01ABC", "US", "SKU-1", "19.9", "http://img/1.jpg", "title one"}
}));
SimilarAsinExcelParser.ParsedSheet first = parser.parse(new ByteArrayInputStream(content));
SimilarAsinExcelParser.ParsedSheet second = parser.parse(new ByteArrayInputStream(content));
assertEquals(first.headers(), second.headers());
assertEquals(first.rows(), second.rows());
assertEquals(1, first.rows().size());
assertEquals("B01ABC", first.rows().get(0).asin());
}
}
@@ -0,0 +1,161 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 86SimilarAsinGroupingConverter 分组转换器。
* 归一化行 → 分组 DTO;分组规则与现状 buildParsedGroups 一致:
* 同 baseId 相邻块归组、groupKey 兜底、startIndex/endIndex 半开区间游标、顺序保持。
*/
class SimilarAsinGroupingConverterTest {
private static SimilarAsinParsedRowVo row(String sourceFileKey, String displayId, String groupKey,
int rowIndex, String asin, String country) {
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey(sourceFileKey);
vo.setSourceFilename("f.xlsx");
vo.setDisplayId(displayId);
vo.setGroupKey(groupKey);
vo.setRowIndex(rowIndex);
vo.setSourceId(displayId);
vo.setAsin(asin);
vo.setCountry(country);
vo.setValues(new LinkedHashMap<>());
return vo;
}
@Test
void test_group_normal() {
List<SimilarAsinParsedRowVo> rows = List.of(
row("k1", "1", "k1::1@2", 2, "B01A", "US"),
row("k1", "2", "k1::2@3", 3, "B01B", "DE"));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
assertEquals(2, groups.size());
assertEquals("1", groups.get(0).getBaseId());
assertEquals("k1::1@2", groups.get(0).getGroupKey());
assertEquals("1", groups.get(0).getDisplayId());
assertEquals(1, groups.get(0).getItemCount());
assertEquals(0, groups.get(0).getStartIndex());
assertEquals(1, groups.get(0).getEndIndex());
assertEquals("2", groups.get(1).getBaseId());
assertEquals(1, groups.get(1).getStartIndex());
assertEquals(2, groups.get(1).getEndIndex());
}
@Test
void test_group_sub_rows() {
List<SimilarAsinParsedRowVo> rows = List.of(
row("k1", "2_1", "k1::2@2", 2, "B01A", "US"),
row("k1", "2_2", "k1::2@2", 3, "B01B", "DE"),
row("k1", "2_3", "k1::2@2", 4, "B01C", "FR"));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
assertEquals(1, groups.size());
SimilarAsinParsedGroupVo group = groups.get(0);
assertEquals("2", group.getBaseId(), "子行 2_1/2_2/2_3 归同组");
assertEquals("k1::2@2", group.getGroupKey());
assertEquals("2_1", group.getDisplayId(), "组首条展示 ID 保留子行");
assertEquals(3, group.getItemCount());
assertEquals(0, group.getStartIndex());
assertEquals(3, group.getEndIndex());
}
@Test
void test_group_single_item() {
List<SimilarAsinParsedRowVo> rows = List.of(row("k1", "7", "k1::7@2", 2, "B01S", "US"));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
assertEquals(1, groups.size());
assertEquals(1, groups.get(0).getItemCount());
assertEquals(0, groups.get(0).getStartIndex());
assertEquals(1, groups.get(0).getEndIndex());
}
@Test
void test_group_empty_input() {
assertTrue(SimilarAsinGroupingConverter.convert(null).isEmpty());
assertTrue(SimilarAsinGroupingConverter.convert(List.of()).isEmpty());
}
@Test
void test_group_dto_fields() {
List<SimilarAsinParsedRowVo> rows = List.of(row("uploads/k.xlsx", "5", "uploads/k.xlsx::5@2", 2, "B01D", "JP"));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
SimilarAsinParsedGroupVo group = groups.get(0);
assertEquals("uploads/k.xlsx", group.getSourceFileKey());
assertEquals("f.xlsx", group.getSourceFilename());
assertEquals("uploads/k.xlsx::5@2", group.getGroupKey());
assertEquals("5", group.getBaseId());
assertEquals("5", group.getDisplayId());
assertEquals(1, group.getItemCount());
assertNotNull(group.getStartIndex());
assertNotNull(group.getEndIndex());
}
@Test
void test_group_dedup() {
// 同 baseId 但不同 groupKey(如跨块重复主 ID):按 groupKey 分两个组,行不丢失
List<SimilarAsinParsedRowVo> rows = List.of(
row("k1", "1", "k1::1@2", 2, "B01A", "US"),
row("k1", "1_1", "k1::1@5", 5, "B01B", "DE"));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
assertEquals(2, groups.size(), "同 baseId 不同 groupKey 分行两组");
assertEquals("1", groups.get(0).getBaseId());
assertEquals("1", groups.get(1).getBaseId());
assertEquals(2, groups.get(0).getItemCount() + groups.get(1).getItemCount(), "行总数不丢");
}
@Test
void test_group_order() {
List<SimilarAsinParsedRowVo> rows = new ArrayList<>();
rows.add(row("k1", "1", "k1::1@2", 2, "B01A", "US"));
rows.add(row("k1", "3", "k1::3@3", 3, "B01C", "FR"));
rows.add(row("k1", "2", "k1::2@4", 4, "B01B", "DE"));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
assertEquals(3, groups.size());
assertEquals(List.of("1", "3", "2"), groups.stream().map(g -> g.getBaseId()).toList(),
"分组顺序保持行首次出现顺序");
}
@Test
void test_group_missing_key() {
// 缺 groupKey → 以 displayId 的 baseId 兜底;displayId 也缺 → sourceId 兜底
List<SimilarAsinParsedRowVo> noGroupKey = List.of(row("k1", "4_1", null, 2, "B01A", "US"));
List<SimilarAsinParsedGroupVo> groups1 = SimilarAsinGroupingConverter.convert(noGroupKey);
assertEquals(1, groups1.size());
assertEquals("4", groups1.get(0).getBaseId());
assertEquals("k1::4@2", groups1.get(0).getGroupKey(), "groupKey 缺失时按 buildGroupKey 兜底");
assertEquals("4_1", groups1.get(0).getDisplayId());
SimilarAsinParsedRowVo noDisplayId = new SimilarAsinParsedRowVo();
noDisplayId.setSourceFileKey("k1");
noDisplayId.setSourceFilename("f.xlsx");
noDisplayId.setSourceId("9_1");
noDisplayId.setRowIndex(5);
List<SimilarAsinParsedGroupVo> groups2 = SimilarAsinGroupingConverter.convert(List.of(noDisplayId));
assertEquals(1, groups2.size());
assertEquals("", groups2.get(0).getBaseId(), "displayId 缺失时 baseId 为空串(现状 baseId(null) 语义)");
assertEquals("9_1", groups2.get(0).getDisplayId(), "displayId 缺失时兜底 sourceId");
assertEquals("k1::@5", groups2.get(0).getGroupKey(), "groupKey 兜底按空 baseId 拼接(现状语义)");
}
}
@@ -0,0 +1,382 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.ArgumentMatchers.isNull;
import org.mockito.ArgumentCaptor;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
/**
* 任务 91SimilarAsinHistoryAssembler 历史查询组装器。
* history 列表 VO 拼装(toHistoryItem + 进度链)抽到独立组件;只读不落库;输出与现状一致。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinHistoryAssemblerTest {
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private OssStorageService ossStorageService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
private ObjectMapper objectMapper = new ObjectMapper();
private SimilarAsinHistoryAssembler assembler;
@BeforeEach
void setUp() {
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
new org.apache.ibatis.builder.MapperBuilderAssistant(
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
TaskChunkEntity.class);
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
new org.apache.ibatis.builder.MapperBuilderAssistant(
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
TaskScopeStateEntity.class);
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
new org.apache.ibatis.builder.MapperBuilderAssistant(
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
FileTaskEntity.class);
assembler = new SimilarAsinHistoryAssembler(
taskScopeStateMapper, taskChunkMapper, fileTaskMapper, taskProgressSnapshotService,
ossStorageService, transientPayloadStorageService, objectMapper);
lenient().when(ossStorageService.generateFreshDownloadUrl(anyString())).thenAnswer(inv -> "https://oss/" + inv.getArgument(0));
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
lenient().when(fileTaskMapper.selectById(any())).thenReturn(null);
}
@AfterEach
void tearDown() {
}
private static FileResultEntity result(Long id, Long taskId, String source, String resultFileUrl, Integer success) {
FileResultEntity row = new FileResultEntity();
row.setId(id);
row.setTaskId(taskId);
row.setSourceFilename(source);
row.setResultFilename(source == null ? null : source.replace(".xlsx", "-result.xlsx"));
row.setResultFileUrl(resultFileUrl);
row.setSuccess(success);
row.setErrorMessage(null);
row.setRowCount(12);
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
return row;
}
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setTaskNo("T" + id);
task.setModuleType("SIMILAR_ASIN");
task.setStatus(status);
task.setCreatedAt(createdAt);
task.setFinishedAt(createdAt == null ? null : createdAt.plusMinutes(5));
return task;
}
private static TaskFileJobEntity job(Long id, String status) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setStatus(status);
job.setErrorMessage(null);
return job;
}
private static Map<Long, TaskFileJobEntity> jobMap() {
return new LinkedHashMap<>();
}
@Test
void test_assembler_history_items() {
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
taskMap.put(10L, task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
List.of(result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1)),
taskMap, jobMap);
assertEquals(1, items.size());
SimilarAsinHistoryItemVo item = items.get(0);
assertEquals(100L, item.getResultId());
assertEquals(10L, item.getTaskId());
assertEquals("a.xlsx", item.getSourceFilename());
assertEquals("a-result.xlsx", item.getResultFilename());
assertEquals("SUCCESS", item.getTaskStatus());
assertEquals(Boolean.TRUE, item.getSuccess());
assertEquals(12, item.getRowCount());
assertEquals("2026-08-01T10:00", item.getCreatedAt());
assertEquals("2026-08-01T09:00", item.getStartedAt(), "任务开始时间复用 task.createdAt");
assertEquals("2026-08-01T09:05", item.getFinishedAt(), "任务结束时间取 task.finishedAt");
}
@Test
void test_assembler_no_result() {
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
List.of(), new LinkedHashMap<>(), new LinkedHashMap<>());
assertTrue(items.isEmpty(), "无结果返回空列表");
}
@Test
void test_assembler_file_state() {
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
jobMap.put(100L, job(7L, "RUNNING"));
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), Map.of(10L, task(10L, "SUCCESS", null)), jobMap).get(0);
assertTrue(item.getFileReady(), "结果文件就绪");
assertEquals(7L, item.getFileJobId());
assertEquals("RUNNING", item.getFileStatus());
assertEquals(Integer.valueOf(100), item.getFileProgressPercent(), "就绪即 100%");
assertEquals("结果文件已生成", item.getFileProgressMessage());
}
@Test
void test_assembler_order() {
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
taskMap.put(1L, task(1L, "SUCCESS", null));
taskMap.put(2L, task(2L, "SUCCESS", null));
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
List.of(result(200L, 2L, "b.xlsx", null, 1), result(100L, 1L, "a.xlsx", null, 1)),
taskMap, jobMap());
assertEquals(2, items.size());
assertEquals(200L, items.get(0).getResultId(), "输入顺序保持");
assertEquals(100L, items.get(1).getResultId());
}
@Test
void test_assembler_null_task() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 1);
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 11, 0));
// taskMap 空 → 缺 task
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), new LinkedHashMap<>(), new LinkedHashMap<>()).get(0);
assertNull(item.getTaskStatus(), "缺 task 状态为空");
assertEquals("2026-08-01T11:00", item.getStartedAt(), "缺 task 回退 result.createdAt");
assertNull(item.getFinishedAt(), "缺 task 无结束时间");
}
@Test
void test_assembler_download_url() {
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), Map.of(10L, task(10L, "SUCCESS", null)), jobMap).get(0);
assertEquals("https://oss/result/10/a.xlsx", item.getDownloadUrl(), "下载 URL 由 oss 服务生成");
verify(ossStorageService).generateFreshDownloadUrl("result/10/a.xlsx");
}
@Test
void test_assembler_download_url_blank_result_file() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 1);
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), Map.of(10L, task(10L, "SUCCESS", null)), jobMap()).get(0);
assertNull(item.getDownloadUrl(), "无结果文件 URL 时下载地址为空");
verify(ossStorageService, never()).generateFreshDownloadUrl(anyString());
}
@Test
void test_assembler_immutable_input() {
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
taskMap.put(10L, task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
String originalTaskNo = taskMap.get(10L).getTaskNo();
assembler.buildHistoryItems(List.of(row), taskMap, jobMap());
assertEquals(originalTaskNo, taskMap.get(10L).getTaskNo(), "task 不被修改");
assertEquals("a.xlsx", row.getSourceFilename(), "result 不被修改");
}
@Test
void test_assembler_pending_skip() {
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
taskMap.put(10L, task(10L, "PENDING", null));
taskMap.put(11L, task(11L, "RUNNING", null));
List<SimilarAsinHistoryItemVo> items = assembler.buildHistoryItems(
List.of(result(100L, 10L, "a.xlsx", null, 0), result(101L, 11L, "b.xlsx", null, 0)),
taskMap, jobMap());
assertEquals(1, items.size(), "PENDING 任务历史项被跳过");
assertEquals(101L, items.get(0).getResultId());
}
@Test
void test_assembler_file_error_attached() {
TaskFileJobEntity failedJob = job(9L, "FAILED");
failedJob.setErrorMessage("assemble boom");
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
jobMap.put(100L, failedJob);
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
taskMap.put(10L, task(10L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 0)));
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), taskMap, jobMap).get(0);
assertEquals("FAILED", item.getFileStatus(), "job 失败状态附带");
assertEquals("assemble boom", item.getFileError(), "job 错误信息附带");
}
@Test
void test_assembler_consistency() {
// 与现状 toHistoryItem 行为一致:success 字段、rowCount、error、时间兜底
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 0);
row.setErrorMessage("python timeout");
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
taskMap.put(10L, task(10L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 30)));
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), taskMap, jobMap()).get(0);
assertEquals(Boolean.FALSE, item.getSuccess());
assertEquals("python timeout", item.getError());
assertEquals("FAILED", item.getTaskStatus());
assertEquals(12, item.getRowCount());
}
@Test
void test_assembler_python_upload_row_fallback() {
// scope 状态存在但无分片数 → chunk 进度 (0,0) → 回退 fileTask 载荷行数(单位"行"
FileTaskEntity t = new FileTaskEntity();
t.setId(10L);
t.setResultJson("{\"parsedPayloadRef\":\"ref:payload\",\"allItems\":[]}");
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
SimilarAsinParsedRowVo parsed = new SimilarAsinParsedRowVo();
parsed.setDisplayId("B01X");
parsed.setAsin("B01X");
parsed.setCountry("US");
parsed.setRowToken("uploads/main.xlsx::row::2");
String payloadJson;
try {
payloadJson = objectMapper.writeValueAsString(
Map.of("items", List.of(parsed), "allItems", List.of()));
} catch (Exception ex) {
throw new AssertionError(ex);
}
TaskScopeStateEntity state = new TaskScopeStateEntity();
state.setTaskId(10L);
state.setModuleType("SIMILAR_ASIN");
state.setChunkTotal(0);
state.setReceivedChunkCount(0);
when(fileTaskMapper.selectById(10L)).thenReturn(t);
when(transientPayloadStorageService.resolvePayload("ref:payload", "read similar ASIN parsed payload failed"))
.thenReturn(payloadJson);
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(state));
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), Map.of(10L, task(10L, "RUNNING", null)), jobMap()).get(0);
assertEquals(0, item.getFileProgressCurrent(), "无已提交行");
assertEquals(1, item.getFileProgressTotal(), "载荷总行数 1");
assertTrue(item.getFileProgressMessage().endsWith(""), "回退路径单位应为'行',实际: " + item.getFileProgressMessage());
assertEquals(Integer.valueOf(1), item.getFileProgressPercent(), "0 行时百分比 1");
}
@Test
void test_assembler_python_upload_no_task_no_chunk() {
// task 与 chunk 都拿不到 → 0/0 行单位,不抛异常
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
SimilarAsinHistoryItemVo item = assembler.buildHistoryItems(
List.of(row), Map.of(10L, task(10L, "RUNNING", null)), jobMap()).get(0);
assertEquals(0, item.getFileProgressCurrent());
assertEquals(1, item.getFileProgressTotal(), "0 行时 total 兜底 1");
assertTrue(item.getFileProgressMessage().endsWith(""), "无 task 无 chunk 仍按行回退,实际: " + item.getFileProgressMessage());
}
@Test
void test_assembler_chunk_read_failure_records_error() {
// 回退路径读 chunk 载荷失败 → 抛 BusinessException 且 last_error 留痕
FileTaskEntity t = new FileTaskEntity();
t.setId(10L);
t.setResultJson("{}");
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(10L);
chunk.setModuleType("SIMILAR_ASIN");
chunk.setScopeHash("s1");
chunk.setChunkIndex(2);
chunk.setPayloadJson("ref:missing");
TaskScopeStateEntity state = new TaskScopeStateEntity();
state.setTaskId(10L);
state.setModuleType("SIMILAR_ASIN");
state.setChunkTotal(0);
state.setReceivedChunkCount(0);
TaskScopeStateEntity scope = new TaskScopeStateEntity();
scope.setId(55L);
scope.setLastError("");
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
when(fileTaskMapper.selectById(10L)).thenReturn(t);
when(transientPayloadStorageService.resolvePayload("ref:missing", "read similar ASIN chunk failed"))
.thenThrow(new IllegalStateException("payload only exists on instance=server-110"));
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(state));
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
BusinessException thrown = assertThrows(BusinessException.class,
() -> assembler.buildHistoryItems(List.of(row), Map.of(10L, task(10L, "RUNNING", null)), jobMap()));
assertTrue(thrown.getMessage().contains("chunk=2"), "异常附带 chunk 索引,实际: " + thrown.getMessage());
ArgumentCaptor<LambdaUpdateWrapper<TaskScopeStateEntity>> captor =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(taskScopeStateMapper).update(isNull(), captor.capture());
String written = String.join("|", captor.getValue().getParamNameValuePairs().values().stream()
.map(String::valueOf).toList());
assertTrue(written.contains("chunk-read-failed[2@cross-instance]"), "跨实例标记留痕,实际: " + written);
}
}
@@ -0,0 +1,76 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 85SimilarAsinRowNormalizer 字段归一化器。
* trim/大小写/ASIN 格式统一,规则与现状 Service.normalize 完全一致(BOM 剥离、全角空格转半角、
* 连续空白折叠为单空格),纯函数无状态,幂等。
*/
class SimilarAsinRowNormalizerTest {
@Test
void test_trim_whitespace() {
assertEquals("abc", SimilarAsinRowNormalizer.normalize(" abc "));
assertEquals("abc", SimilarAsinRowNormalizer.normalize("\tabc\n"));
assertEquals("", SimilarAsinRowNormalizer.normalize(" "));
}
@Test
void test_lowercase_asin() {
assertEquals("B01ABC", SimilarAsinRowNormalizer.normalizeAsin("b01abc"));
assertEquals("B01ABC", SimilarAsinRowNormalizer.normalizeAsin("b01abc "));
}
@Test
void test_mixed_case_asin() {
assertEquals("B01ABC", SimilarAsinRowNormalizer.normalizeAsin("b01AbC"));
assertEquals("B01MIXED1", SimilarAsinRowNormalizer.normalizeAsin(" b01MiXeD1 "));
}
@Test
void test_internal_whitespace_kept() {
assertEquals("ab cd", SimilarAsinRowNormalizer.normalize("ab cd"), "内部单空格保留");
assertEquals("ab cd", SimilarAsinRowNormalizer.normalize("ab cd"), "连续空白折叠为单空格(现状)");
assertEquals("ab cd", SimilarAsinRowNormalizer.normalize("ab cd"), "全角空格转半角");
}
@Test
void test_null_safe() {
assertEquals("", SimilarAsinRowNormalizer.normalize(null));
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(null));
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(" "), "空白 ASIN 归一后为空(trim 先行)");
}
@Test
void test_empty_string() {
assertEquals("", SimilarAsinRowNormalizer.normalize(""));
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(""));
assertEquals("", SimilarAsinRowNormalizer.normalizeAsin(""));
}
@Test
void test_special_chars_untouched() {
assertEquals("a@#$%^&*()", SimilarAsinRowNormalizer.normalize(" a@#$%^&*() "));
assertEquals("A@#$%^&*()", SimilarAsinRowNormalizer.normalizeAsin("a@#$%^&*()"));
assertEquals("B01/AB-C_1", SimilarAsinRowNormalizer.normalizeAsin("b01/ab-c_1"), "连字符/下划线/斜杠保留");
}
@Test
void test_normalize_identity_after() {
String[] samples = {"abc", " abc ", "b01AbC", "ab cd", "x y ", "a@#$%^&*()", "", " "};
for (String sample : samples) {
String once = SimilarAsinRowNormalizer.normalize(sample);
assertEquals(once, SimilarAsinRowNormalizer.normalize(once),
"normalize 幂等: input=" + sample);
String asinOnce = SimilarAsinRowNormalizer.normalizeAsin(sample);
assertEquals(asinOnce, SimilarAsinRowNormalizer.normalizeAsin(asinOnce),
"normalizeAsin 幂等: input=" + sample);
}
assertTrue(SimilarAsinRowNormalizer.normalize(" a b ").equals(SimilarAsinRowNormalizer.normalize("a b")),
"不同空白形态归一到同一结果");
}
}
@@ -0,0 +1,131 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinRowValidator.RowError;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 87SimilarAsinRowValidator 纯校验器。
* 必填检查(id/asin/country)、ASIN 格式校验(大写字母数字)、重复检测(asin+country 键,保留首次);
* 返回错误清单不抛异常。规则与现状 parseWorkbook/dedupeRowsByRowKey 一致:
* 全空白行跳过、必填缺失即记错、重复仅标记后续出现、格式只拦非字母数字。
*/
class SimilarAsinRowValidatorTest {
private static SimilarAsinParsedRowVo row(String sourceId, String asin, String country) {
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey("k1");
vo.setSourceFilename("f.xlsx");
vo.setRowIndex(2);
vo.setSourceId(sourceId);
vo.setDisplayId(sourceId);
vo.setAsin(asin);
vo.setCountry(country);
vo.setValues(new LinkedHashMap<>());
return vo;
}
private static List<String> columns(List<RowError> errors) {
return errors.stream().map(RowError::column).toList();
}
@Test
void test_validate_required_present() {
List<SimilarAsinParsedRowVo> rows = List.of(
row("1", "B01A", "US"),
row("2", "B01B", "DE"));
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
assertTrue(errors.isEmpty(), "必填齐全通过");
}
@Test
void test_validate_required_missing() {
List<SimilarAsinParsedRowVo> rows = List.of(
row("", "B01A", "US"),
row("2", "", "DE"),
row("3", "B01C", " "));
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
assertEquals(3, errors.size(), "缺 id / asin / country 各记一错");
assertEquals(List.of("id", "asin", "country"), columns(errors));
}
@Test
void test_validate_asin_format() {
List<SimilarAsinParsedRowVo> rows = List.of(
row("1", "B0A-1B2C3D", "US"),
row("2", "B0A 1B2C3D", "DE"));
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
assertEquals(2, errors.size(), "非字母数字 ASIN 记格式错");
assertEquals(List.of("asin", "asin"), columns(errors));
}
@Test
void test_validate_duplicates() {
List<SimilarAsinParsedRowVo> rows = List.of(
row("1", "B01A", "US"),
row("2", "b01a", "US"),
row("3", "B01A", "DE"),
row("4", "B01C", "US"));
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
assertEquals(1, errors.size(), "仅重复的后续出现记错");
assertEquals("asin", errors.get(0).column());
assertTrue(errors.get(0).message().contains("B01A"), "重复消息含 ASIN: " + errors.get(0).message());
assertTrue(errors.get(0).message().contains("US"), "重复消息含国家: " + errors.get(0).message());
}
@Test
void test_validate_blank_row() {
List<SimilarAsinParsedRowVo> rows = List.of(row("", "", ""));
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
assertTrue(errors.isEmpty(), "全空白行跳过(与 parse 一致)");
}
@Test
void test_validate_no_error_happy() {
List<SimilarAsinParsedRowVo> rows = new ArrayList<>();
rows.add(row("1_1", "B01A", "US"));
rows.add(row("1_2", "B01B", "DE"));
rows.add(row("2", "B01C", "FR"));
rows.add(row("3", "B01D", "JP"));
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
assertTrue(errors.isEmpty());
}
@Test
void test_validate_error_detail() {
List<SimilarAsinParsedRowVo> rows = List.of(row("", "B01A", "US"));
List<RowError> errors = SimilarAsinRowValidator.validate(rows);
assertEquals(1, errors.size());
RowError error = errors.get(0);
assertEquals(2, error.rowIndex(), "错误含 1 基行号");
assertEquals("id", error.column());
assertTrue(!error.message().isBlank(), "错误含消息");
}
@Test
void test_validate_null_input() {
assertTrue(SimilarAsinRowValidator.validate(null).isEmpty());
assertTrue(SimilarAsinRowValidator.validate(List.of()).isEmpty());
}
}
@@ -0,0 +1,219 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 88SimilarAsinSheetBuilder Sheet 构造器。
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式);输入数据 → 输出 workbook;不落库、无 IO 依赖。
* 表头/列序/样式与现状 assembleResultWorkbook 一致:sheet 名"相似asin检测"、15 列表头、加粗表头、
* 数据行从第 1 行、图片列 12-14 列宽、行高 409pt、值兜底与现状一致。
*/
class SimilarAsinSheetBuilderTest {
private SXSSFWorkbook lastWorkbook;
@AfterEach
void closeWorkbook() throws Exception {
if (lastWorkbook != null) {
lastWorkbook.close();
}
}
private static SimilarAsinParsedRowVo row(String displayId, String asin, String country, String price,
String seller, String brand) {
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey("k1");
vo.setSourceFilename("f.xlsx");
vo.setRowIndex(2);
vo.setSourceId(displayId);
vo.setDisplayId(displayId);
vo.setAsin(asin);
vo.setCountry(country);
vo.setPrice(price);
LinkedHashMap<String, String> values = new LinkedHashMap<>();
values.put("卖家名称", seller);
values.put("品牌", brand);
vo.setValues(values);
return vo;
}
private Workbook build(List<SimilarAsinParsedRowVo> rows) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
lastWorkbook = workbook;
SimilarAsinSheetBuilder.build(workbook, rows);
return workbook;
}
private static String cellString(Workbook workbook, int row, int col) {
Sheet sheet = workbook.getSheetAt(0);
Row r = sheet.getRow(row);
if (r == null) {
return null;
}
Cell c = r.getCell(col);
return c == null ? null : c.getStringCellValue();
}
@Test
void test_build_headers() {
Workbook workbook = build(List.of());
Sheet sheet = workbook.getSheetAt(0);
assertNotNull(sheet, "sheet 存在");
assertEquals("相似asin检测", sheet.getSheetName());
Row header = sheet.getRow(0);
assertNotNull(header);
List<String> headers = new ArrayList<>();
header.forEach(cell -> headers.add(cell.getStringCellValue()));
assertEquals(List.of("id", "asin", "国家", "价格", "卖家名称", "品牌", "是否有货", "相似度",
"是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
headers, "表头与现状 RESULT_HEADERS 一致");
}
@Test
void test_build_header_style_bold() {
Workbook workbook = build(List.of());
Sheet sheet = workbook.getSheetAt(0);
CellStyle style = sheet.getRow(0).getCell(0).getCellStyle();
assertTrue(workbook.getFontAt(style.getFontIndex()).getBold(), "表头加粗");
CellStyle style2 = sheet.getRow(0).getCell(14).getCellStyle();
assertTrue(workbook.getFontAt(style2.getFontIndex()).getBold(), "末列表头也加粗");
}
@Test
void test_build_data_rows() {
Workbook workbook = build(List.of(
row("1", "B01A", "US", "19.9", "卖家1", "品牌1"),
row("2", "B01B", "DE", "9.9", "卖家2", "品牌2")));
Sheet sheet = workbook.getSheetAt(0);
assertEquals(3, sheet.getLastRowNum() + 1, "表头 + 2 数据行");
assertEquals("1", cellString(workbook, 1, 0));
assertEquals("B01A", cellString(workbook, 1, 1));
assertEquals("US", cellString(workbook, 1, 2));
assertEquals("19.9", cellString(workbook, 1, 3));
assertEquals("卖家1", cellString(workbook, 1, 4));
assertEquals("品牌1", cellString(workbook, 1, 5));
assertEquals("2", cellString(workbook, 2, 0));
assertEquals("B01B", cellString(workbook, 2, 1));
}
@Test
void test_build_empty_rows() {
Workbook workbook = build(List.of());
Sheet sheet = workbook.getSheetAt(0);
assertEquals(1, sheet.getLastRowNum() + 1, "无数据行时只有表头");
assertNull(sheet.getRow(1), "无第 1 行");
}
@Test
void test_build_missing_values_fallback() {
// displayId 缺失 → 兜底 sourceIdasin/country 空白 → 空串;values 缺失 → 空串
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey("k1");
vo.setSourceFilename("f.xlsx");
vo.setRowIndex(2);
vo.setSourceId("9_1");
vo.setDisplayId("");
vo.setAsin("");
vo.setCountry("");
Workbook workbook = build(List.of(vo));
assertEquals("9_1", cellString(workbook, 1, 0), "displayId 空白兜底 sourceId");
assertEquals("", cellString(workbook, 1, 1));
assertEquals("", cellString(workbook, 1, 2));
assertEquals("", cellString(workbook, 1, 4), "卖家名称空白");
assertEquals("", cellString(workbook, 1, 5), "品牌空白");
}
@Test
void test_build_image_columns_width() {
Workbook workbook = build(List.of());
Sheet sheet = workbook.getSheetAt(0);
assertEquals(SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256, sheet.getColumnWidth(12), "主图列宽");
assertEquals(SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256, sheet.getColumnWidth(13), "图1列宽");
assertEquals(SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256, sheet.getColumnWidth(14), "图2列宽");
}
@Test
void test_build_column_order() {
Workbook workbook = build(List.of(row("1", "B01A", "US", "19.9", "卖家1", "品牌1")));
assertEquals("id", cellString(workbook, 0, 0));
assertEquals("asin", cellString(workbook, 0, 1));
assertEquals("国家", cellString(workbook, 0, 2));
assertEquals("价格", cellString(workbook, 0, 3));
assertEquals("状态", cellString(workbook, 0, 11));
assertEquals("主图", cellString(workbook, 0, 12));
assertEquals("阿里巴巴图片2", cellString(workbook, 0, 14));
assertEquals("B01A", cellString(workbook, 1, 1), "asin 在第 1 列");
assertNull(cellString(workbook, 1, 11), "状态列由调用方补写,builder 不创建");
}
@Test
void test_build_result_columns_blank_without_result() {
// 结果派生列(是否有货/相似度/是否符合类目/不符合理由/产品类目/状态)来自 LLM 结果行,
// builder 输入只有解析行 → 这些列不创建(现状无 resultRow 时调用方同样置空串)。
Workbook workbook = build(List.of(row("1", "B01A", "US", "19.9", "卖家1", "品牌1")));
assertNull(cellString(workbook, 1, 6));
assertNull(cellString(workbook, 1, 7));
assertNull(cellString(workbook, 1, 8));
assertNull(cellString(workbook, 1, 9));
assertNull(cellString(workbook, 1, 10));
assertNull(cellString(workbook, 1, 11));
}
@Test
void test_build_value_by_header_contains() {
// 卖家名称/品牌从 values 映射按表头包含匹配读取(现状 readValueByHeader 语义)
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey("k1");
vo.setSourceFilename("f.xlsx");
vo.setRowIndex(2);
vo.setSourceId("3");
vo.setDisplayId("3");
vo.setAsin("B01C");
vo.setCountry("FR");
vo.setPrice("");
LinkedHashMap<String, String> values = new LinkedHashMap<>();
values.put("卖家名称:亚马逊", "卖家X");
values.put("品牌/店铺", "品牌Y");
vo.setValues(values);
Workbook workbook = build(List.of(vo));
assertEquals("卖家X", cellString(workbook, 1, 4), "卖家名称按包含匹配");
assertEquals("品牌Y", cellString(workbook, 1, 5), "品牌按包含匹配");
}
@Test
void test_build_null_input() {
Workbook workbook = build(null);
Sheet sheet = workbook.getSheetAt(0);
assertEquals(1, sheet.getLastRowNum() + 1, "null 输入不写数据行");
}
}
@@ -0,0 +1,303 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 90:抽取前后快照对比测试。
* 夹具 Excel → 解析结果快照(golden 文件);同一夹具输出完全一致;快照变更即失败(防行为漂移)。
* golden 文件:src/test/resources/similarasin/golden/parse-snapshot.txt(全 parse 路径)
* src/test/resources/similarasin/golden/groups-snapshot.txt(分组路径)
*/
class SimilarAsinSnapshotTest {
private static final File GOLDEN_PARSE =
new File("src/test/resources/similarasin/golden/parse-snapshot.txt");
private static final File GOLDEN_GROUPS =
new File("src/test/resources/similarasin/golden/groups-snapshot.txt");
private static final String[] HEADERS = {"id", "asin", "国家", "sku", "价格", "url", "标题"};
// ---- 夹具 ----
private static File workbook(String[] headers, List<String[]> rows) throws Exception {
File file = File.createTempFile("similar-asin-snapshot-", ".xlsx");
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream out = new FileOutputStream(file)) {
Sheet sheet = wb.createSheet("Sheet1");
Row header = sheet.createRow(0);
for (int i = 0; i < headers.length; i++) {
header.createCell(i).setCellValue(headers[i]);
}
for (int r = 0; r < rows.size(); r++) {
Row row = sheet.createRow(r + 1);
for (int c = 0; c < rows.get(r).length; c++) {
row.createCell(c).setCellValue(rows.get(r)[c]);
}
}
wb.write(out);
}
return file;
}
/** 5 有效行:2_1/2_2/2_3 同块、非法 ASIN 保留、单块多行。 */
private static File fixtureMain() throws Exception {
return workbook(HEADERS, List.of(
new String[]{"2_1", "B01A", "US", "SKU-1", "19.90", "http://img/a.jpg", "title a"},
new String[]{"2_2", "B01B", "DE", "SKU-2", "29.90", "http://img/b.jpg", "title b"},
new String[]{"2_3", "B01C", "FR", "SKU-3", "", "http://img/c.jpg", ""},
new String[]{"5", "B01-ABC", "UK", "SKU-4", "9.90", "http://img/d.jpg", "title d"},
new String[]{"6", "B01D", "US", "SKU-5", "1.00", "http://img/e.jpg", "title e"}));
}
/** 6 行含 5 类错误(缺 id/asin/国家、格式错、重复)+ 1 空行;解析后剩 3 有效行。 */
private static File fixtureError() throws Exception {
return workbook(HEADERS, List.of(
new String[]{"", "B02A", "US"},
new String[]{"9", "", "DE"},
new String[]{"10", "B02B", ""},
new String[]{"11", "B02-ABC", "FR"},
new String[]{"12", "B02C", "FR"},
new String[]{"13", "B02C", "FR"},
new String[]{"", "", ""}));
}
private static final List<String[]> GROUPS_FIXTURE = List.of(
new String[]{"1", "B10A", "US"},
new String[]{"2_1", "B11A", "DE"},
new String[]{"2_2", "B11B", "FR"});
private static File fixtureGroups() throws Exception {
return workbook(HEADERS, GROUPS_FIXTURE);
}
// ---- 解析管线(与服务侧 parseWorkbookDelegated 语义一致) ----
private static List<SimilarAsinParsedRowVo> toRows(File file, String fileKey, String filename,
boolean dropInvalid) throws Exception {
SimilarAsinExcelParser.ParsedSheet parsed = new SimilarAsinExcelParser().parse(file);
List<SimilarAsinParsedRowVo> rows = new ArrayList<>();
String currentBlockBaseId = "";
String currentGroupKey = "";
for (SimilarAsinExcelParser.SimilarAsinExcelRow parsedRow : parsed.rows()) {
String id = parsedRow.id();
String asin = parsedRow.asin();
String country = parsedRow.country();
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
if (dropInvalid && (id.isBlank() || asin.isBlank() || country.isBlank())) {
continue;
}
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey(fileKey);
vo.setSourceFilename(filename);
vo.setRowIndex(parsedRow.rowIndex());
vo.setSourceId(id);
vo.setDisplayId(id.trim());
String rowBaseId = baseId(vo.getDisplayId());
if (!java.util.Objects.equals(currentBlockBaseId, rowBaseId)) {
currentBlockBaseId = rowBaseId;
currentGroupKey = buildGroupKey(fileKey, rowBaseId, vo.getRowIndex());
}
vo.setGroupKey(currentGroupKey);
vo.setRowToken(buildRowToken(fileKey, vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setSku(parsedRow.sku());
vo.setPrice(parsedRow.price());
vo.setUrl(parsedRow.url());
vo.setTitle(parsedRow.title());
vo.setValues(parsedRow.values());
rows.add(vo);
}
return rows;
}
private static String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private static String baseId(String id) {
String s = normalize(id);
int idx = s.indexOf('_');
return idx > 0 ? s.substring(0, idx) : s;
}
private static String buildRowToken(String sourceFileKey, Integer rowIndex) {
return normalize(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex);
}
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
}
// ---- 快照渲染 ----
private static String renderRow(SimilarAsinParsedRowVo row) {
return " row: idx=" + row.getRowIndex()
+ " sourceId=" + row.getSourceId()
+ " displayId=" + row.getDisplayId()
+ " asin=" + row.getAsin()
+ " country=" + row.getCountry()
+ " sku=" + row.getSku()
+ " price=" + row.getPrice()
+ " url=" + row.getUrl()
+ " title=" + row.getTitle()
+ " groupKey=" + row.getGroupKey()
+ " rowToken=" + row.getRowToken()
+ " values.size=" + (row.getValues() == null ? 0 : row.getValues().size());
}
private static String renderGroup(SimilarAsinParsedGroupVo group) {
return " group: fileKey=" + group.getSourceFileKey()
+ " filename=" + group.getSourceFilename()
+ " groupKey=" + group.getGroupKey()
+ " baseId=" + group.getBaseId()
+ " displayId=" + group.getDisplayId()
+ " itemCount=" + group.getItemCount()
+ " range=[" + group.getStartIndex() + "," + group.getEndIndex() + ")";
}
private static String renderFull(List<SimilarAsinParsedRowVo> rows, List<SimilarAsinParsedGroupVo> groups) {
StringBuilder sb = new StringBuilder();
sb.append("rows=").append(rows.size()).append('\n');
for (SimilarAsinParsedRowVo row : rows) {
sb.append(renderRow(row)).append('\n');
}
sb.append("groups=").append(groups.size()).append('\n');
for (SimilarAsinParsedGroupVo group : groups) {
sb.append(renderGroup(group)).append('\n');
}
return sb.toString();
}
private static List<SimilarAsinParsedRowVo> mergeRows(List<List<SimilarAsinParsedRowVo>> all) {
List<SimilarAsinParsedRowVo> merged = new ArrayList<>();
for (List<SimilarAsinParsedRowVo> list : all) {
merged.addAll(list);
}
return merged;
}
private static String runAllParse() throws Exception {
List<SimilarAsinParsedRowVo> rows = mergeRows(List.of(
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx", true),
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", true)));
return renderFull(rows, SimilarAsinGroupingConverter.convert(rows));
}
private static String runGroupsOnly() throws Exception {
List<SimilarAsinParsedRowVo> rows = toRows(fixtureGroups(), "uploads/groups.xlsx", "groups.xlsx", true);
return renderFull(rows, SimilarAsinGroupingConverter.convert(rows));
}
private static String read(File file) throws Exception {
return Files.readString(file.toPath(), StandardCharsets.UTF_8);
}
// ---- 用例 ----
@Test
void test_snapshot_parse_output() throws Exception {
assertEquals(read(GOLDEN_PARSE), runAllParse(), "解析输出快照与 golden 一致");
}
@Test
void test_snapshot_groups() throws Exception {
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "分组快照与 golden 一致");
}
@Test
void test_snapshot_dtos() throws Exception {
List<SimilarAsinParsedRowVo> rows = mergeRows(List.of(
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx", true),
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", true)));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
assertEquals(8, rows.size(), "有效行 5 + 3");
assertEquals(6, groups.size(), "分组 3 + 3");
assertEquals(0, groups.get(0).getStartIndex(), "分组覆盖全部行");
assertEquals(rows.size(), groups.get(groups.size() - 1).getEndIndex(), "末组 endIndex 等于总行数");
for (SimilarAsinParsedGroupVo group : groups) {
assertTrue(group.getItemCount() > 0, "组内行数 > 0");
assertTrue(group.getStartIndex() < group.getEndIndex(), "区间非空");
assertFalse(group.getGroupKey().isBlank(), "分组键不空白");
}
}
@Test
void test_snapshot_error_cases() throws Exception {
// 校验器输入 = 原始解析行(含缺必填行),快照校验 5 类错误齐全
List<SimilarAsinParsedRowVo> rows = toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", false);
List<SimilarAsinRowValidator.RowError> errors = SimilarAsinRowValidator.validate(rows);
assertEquals(6, rows.size(), "原始行 6 条(含 5 错误行 + 1 全空行被跳过)");
assertEquals(5, errors.size(), "缺 id/asin/国家、格式错、重复各 1 条");
List<String> messages = errors.stream().map(SimilarAsinRowValidator.RowError::message).toList();
assertTrue(messages.stream().anyMatch(m -> m.contains("缺少必要字段: id")), "id 缺失,实际: " + messages);
assertTrue(messages.stream().anyMatch(m -> m.contains("缺少必要字段: asin")), "asin 缺失,实际: " + messages);
assertTrue(messages.stream().anyMatch(m -> m.contains("缺少必要字段: 国家")), "国家缺失,实际: " + messages);
assertTrue(messages.stream().anyMatch(m -> m.contains("ASIN 格式不正确")), "ASIN 格式错误,实际: " + messages);
assertTrue(messages.stream().anyMatch(m -> m.contains("重复 ASIN")), "重复检测,实际: " + messages);
}
@Test
void test_snapshot_reproducible() throws Exception {
assertEquals(runAllParse(), runAllParse(), "同一夹具跑两次结果一致");
}
@Test
void test_snapshot_golden_committed() {
assertTrue(GOLDEN_PARSE.isFile(), "golden 文件必须存在并入库: " + GOLDEN_PARSE.getAbsolutePath());
assertTrue(GOLDEN_GROUPS.isFile(), "golden 文件必须存在并入库: " + GOLDEN_GROUPS.getAbsolutePath());
assertTrue(GOLDEN_PARSE.length() > 0, "golden 文件非空");
assertTrue(GOLDEN_GROUPS.length() > 0, "golden 文件非空");
}
@Test
void test_snapshot_diff_detected() throws Exception {
String original = read(GOLDEN_PARSE);
assertTrue(original.contains("rows="), "golden 内容合法");
try {
Files.writeString(GOLDEN_PARSE.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
AssertionError failure = null;
try {
assertEquals(read(GOLDEN_PARSE), runAllParse(), "篡改后应与 golden 不一致");
} catch (AssertionError ex) {
failure = ex;
}
assertTrue(failure != null, "篡改 golden 后断言应失败");
} finally {
Files.writeString(GOLDEN_PARSE.toPath(), original, StandardCharsets.UTF_8);
}
assertEquals(original, read(GOLDEN_PARSE), "恢复原始 golden");
}
@Test
void test_snapshot_regression_all_parse() throws Exception {
assertEquals(read(GOLDEN_PARSE), runAllParse(), "全 parse 路径与 golden 一致");
List<SimilarAsinParsedRowVo> rows = mergeRows(List.of(
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx", true),
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx", true)));
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(rows);
assertEquals(8, rows.size(), "总行数 5 + 3");
assertEquals(6, groups.size(), "分组数 3 + 3");
assertEquals(0, groups.get(0).getStartIndex(), "首组起始 0");
assertEquals(8, groups.get(5).getEndIndex(), "末组 endIndex 8");
}
}
@@ -0,0 +1,166 @@
package com.nanri.aiimage.modules.similarasin.util;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
class ExcelCellImageWriterTest {
@Test
void patchedCellImageValueMetadataUsesOneBasedVmIndexes() throws Exception {
Path dir = Files.createTempDirectory("excel-cell-image-test-");
Path xlsx = dir.resolve("result.xlsx");
writeMinimalWorkbook(xlsx);
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
session.registerImage(1, 9, new byte[]{1, 2, 3});
session.registerImage(1, 10, new byte[]{4, 5, 6});
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
String sheet = read(zip, "xl/worksheets/sheet1.xml");
// cell @vm 是 [MS-XLSX] 规范里的 1-based 索引(0 表示"无 metadata")。
assertTrue(sheet.contains("<c r=\"J2\" t=\"e\" vm=\"1\"><v>#VALUE!</v></c>"));
assertTrue(sheet.contains("<c r=\"K2\" t=\"e\" vm=\"2\"><v>#VALUE!</v></c>"));
// futureMetadata 内部块仍然是 0-based。
String metadata = read(zip, "xl/metadata.xml");
assertTrue(metadata.contains("<xlrd:rvb i=\"0\"/>"));
assertTrue(metadata.contains("<xlrd:rvb i=\"1\"/>"));
assertEquals(2, maxVm(sheet));
}
}
@Test
void streamsRegisteredImagePathIntoWorkbookMedia() throws Exception {
Path dir = Files.createTempDirectory("excel-cell-image-path-test-");
Path xlsx = dir.resolve("result.xlsx");
Path image = dir.resolve("thumb.jpeg");
byte[] imageBytes = new byte[]{7, 8, 9, 10};
writeMinimalWorkbook(xlsx);
Files.write(image, imageBytes);
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
session.registerImage(1, 9, image);
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
assertArrayEquals(imageBytes,
zip.getInputStream(zip.getEntry("xl/media/excelcellimage1.jpeg")).readAllBytes());
}
}
@Test
void patchesTenThousandImageCellsWithinLinearTimeBudget() {
assertTimeoutPreemptively(Duration.ofSeconds(10), () -> {
Path dir = Files.createTempDirectory("excel-cell-image-linear-test-");
Path xlsx = dir.resolve("result.xlsx");
writeWorkbookWithImageCells(xlsx, 10_000);
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
for (int row = 1; row <= 10_000; row++) {
session.registerImage(row, 9, new byte[]{1});
}
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
String sheet = read(zip, "xl/worksheets/sheet1.xml");
assertEquals(10_000, maxVm(sheet));
assertTrue(sheet.contains("<c r=\"J10001\" t=\"e\" vm=\"10000\"><v>#VALUE!</v></c>"));
}
});
}
private static int maxVm(String sheet) {
Matcher matcher = Pattern.compile("vm=\"(\\d+)\"").matcher(sheet);
int max = -1;
while (matcher.find()) {
max = Math.max(max, Integer.parseInt(matcher.group(1)));
}
return max;
}
private static String read(ZipFile zip, String name) throws Exception {
return new String(zip.getInputStream(zip.getEntry(name)).readAllBytes(), StandardCharsets.UTF_8);
}
private static void writeMinimalWorkbook(Path xlsx) throws Exception {
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
write(out, "[Content_Types].xml", """
<?xml version="1.0" encoding="UTF-8"?>
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
<Default Extension="xml" ContentType="application/xml"/>
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
</Types>
""");
write(out, "_rels/.rels", """
<?xml version="1.0" encoding="UTF-8"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
</Relationships>
""");
write(out, "xl/workbook.xml", """
<?xml version="1.0" encoding="UTF-8"?>
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
<sheets><sheet name="Sheet1" r:id="rId1" sheetId="1"/></sheets>
</workbook>
""");
write(out, "xl/_rels/workbook.xml.rels", """
<?xml version="1.0" encoding="UTF-8"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
</Relationships>
""");
write(out, "xl/worksheets/sheet1.xml", """
<?xml version="1.0" encoding="UTF-8"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row r="2">
<c r="J2" t="inlineStr"><is><t>main</t></is></c>
<c r="K2" t="inlineStr"><is><t>puzzle</t></is></c>
</row>
</sheetData>
</worksheet>
""");
}
}
private static void writeWorkbookWithImageCells(Path xlsx, int imageCount) throws Exception {
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
write(out, "[Content_Types].xml", "<Types></Types>");
write(out, "xl/_rels/workbook.xml.rels", "<Relationships></Relationships>");
StringBuilder sheet = new StringBuilder("<worksheet><sheetData>");
for (int row = 2; row <= imageCount + 1; row++) {
sheet.append("<row r=\"").append(row).append("\"><c r=\"J").append(row)
.append("\" t=\"inlineStr\"><is><t>image</t></is></c></row>");
}
sheet.append("</sheetData></worksheet>");
write(out, "xl/worksheets/sheet1.xml", sheet.toString());
}
}
private static void write(ZipOutputStream out, String name, String content) throws Exception {
out.putNextEntry(new ZipEntry(name));
out.write(content.getBytes(StandardCharsets.UTF_8));
out.closeEntry();
}
}
@@ -0,0 +1,196 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizeException;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Task 17:优化图片解码采样、像素上限和 JPEG 质量搜索,降低 CPU 与堆峰值。
* - 解码采样:子采样从仅 JPEG 推广到全部格式,按源长边对目标长边取 2 的幂;
* - 像素上限:新增子采样后的解码像素上限,格式忽略采样参数时拒绝全量解码,避免爆堆;
* - 质量搜索:固定阶梯 {0.75,0.65,0.55} 改为估算搜索(0.75 后按字节比例估算质量,
* 最多每长边 2 次编码),最坏编码次数 9 → 6,典型 1-2 次即命中。
*/
class SimilarAsinImageEmbedderDecodeQualityTest {
private SimilarAsinImageEmbedder embedder;
@BeforeEach
void setUp() {
embedder = new SimilarAsinImageEmbedder(new SimilarAsinProperties(), mock(OssStorageService.class));
}
@AfterEach
void shutdown() {
embedder.shutdown();
}
private static byte[] createImage(int width, int height, String format, Color color) throws Exception {
BufferedImage img = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
Graphics2D g = img.createGraphics();
try {
g.setColor(color);
g.fillRect(0, 0, width, height);
} finally {
g.dispose();
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(img, format, baos);
return baos.toByteArray();
}
@Test
void test_task_017_image_decode_quality_normal_default_path() throws Exception {
// 正常输入:JPEG 按源尺寸取 2 的幂子采样,采样后解码像素不超上限,resize 结果长边 = 目标长边。
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
assertEquals(2, subsampling, "3200x2400 JPEG 子采样应为 2");
assertEquals(subsampling, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400),
"JPEG 专用子采样应与通用子采样一致");
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
assertEquals(1600L * 1200L, decodedPixels, "子采样后解码像素 = 1600x1200");
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "解码像素不得超上限");
ResizedImage thumb = embedder.resizeImage("https://example.com/default.jpg",
createImage(1200, 1600, "jpg", new Color(0x33, 0x66, 0x99)));
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(thumb.width(), thumb.height()),
"长边应缩放到目标 1280");
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES, "字节不得超上限");
}
@Test
void test_task_017_image_decode_quality_normal_multiple_items() throws Exception {
// 批量/多格式:非 JPEG(PNG)同样按源尺寸子采样,采样后解码像素受控,resize 结果不丢失。
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
assertEquals(2, subsampling, "PNG 输入同样应用子采样计算");
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "PNG 解码像素不得超上限");
ResizedImage png = embedder.resizeImage("https://example.com/multi.png",
createImage(3200, 2400, "png", new Color(0x99, 0x33, 0x66)));
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(png.width(), png.height()));
assertTrue(png.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
}
@Test
void test_task_017_image_decode_quality_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行:同一输入两次 resize 字节一致;质量估算纯函数输出确定。
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x22, 0x44));
ResizedImage first = embedder.resizeImage("https://example.com/idem.jpg", raw);
ResizedImage second = embedder.resizeImage("https://example.com/idem.jpg", raw);
assertArrayEquals(first.bytes(), second.bytes(), "重复 resize 必须产生相同字节");
assertEquals(first.width(), second.width());
assertEquals(first.height(), second.height());
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(120000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
"估算质量不得超过 0.75 上限");
assertEquals(0.576f,
SimilarAsinImageEmbedder.estimatedQuality(200000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
0.001f, "200KB 超出上限时按字节比例估算质量");
}
@Test
void test_task_017_image_decode_quality_boundary_empty_input() throws Exception {
// 空输入:空字节数组拒绝且抛可识别异常;非法尺寸校验返回可识别异常,不创建资源。
byte[] empty = new byte[0];
Exception ex = assertThrows(Exception.class,
() -> embedder.resizeImage("https://example.com/empty.jpg", empty));
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
"应为 IOException 或 RuntimeException 兜底,实际=" + ex.getClass().getSimpleName());
assertTrue(ex.getMessage() == null || ex.getMessage().toLowerCase().contains("unsupported"),
"空输入消息应反映不支持格式");
ResizeException dimEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/empty.jpg", 0, 100, 1));
assertTrue(dimEx.getMessage().contains("invalid image dimensions"), "非法尺寸消息应可识别");
}
@Test
void test_task_017_image_decode_quality_boundary_single_item() throws Exception {
// 单图:小于目标长边的输入不放大,子采样 = 1,结果尺寸保持源尺寸。
ResizedImage thumb = embedder.resizeImage("https://example.com/single.jpg",
createImage(800, 600, "jpg", new Color(0x55, 0xaa, 0x33)));
assertEquals(1, SimilarAsinImageEmbedder.sourceSubsampling(800, 600), "小图子采样应为 1");
assertEquals(800, thumb.width(), "小图长边保持源尺寸,不放大");
assertEquals(600, thumb.height());
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
}
@Test
void test_task_017_image_decode_quality_boundary_limit_and_overflow() throws Exception {
// 上限/超限:源像素超上限拒绝;子采样后解码像素超上限拒绝;正常值放行。
ResizeException sourceEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage(
"https://example.com/huge.jpg", 7000, 7000, 1));
assertTrue(sourceEx.getMessage().contains("image too large"), "源像素超限消息应可识别");
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 6000, 6000, 4);
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 1200, 1600, 1);
ResizeException decodedEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage(
"https://example.com/no-subsample.jpg", 3000, 3000, 1));
assertTrue(decodedEx.getMessage().contains("decode too large"), "解码像素超限消息应可识别");
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/subsampled.jpg", 3000, 3000, 2);
ResizedImage normal = embedder.resizeImage("https://example.com/normal.jpg",
createImage(1500, 1500, "jpg", new Color(0x20, 0x40, 0x60)));
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(normal.width(), normal.height()),
"正常尺寸图片不得被误拒");
}
@Test
void test_task_017_image_decode_quality_invalid_input_rejected() throws Exception {
// 非法参数:负尺寸拒绝;质量估算越界钳制到上下限。
ResizeException negEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/neg.jpg", -1, 100, 1));
assertTrue(negEx.getMessage().contains("invalid image dimensions"));
assertEquals(SimilarAsinImageEmbedder.MIN_JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(300000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
"估算质量低于下限时钳制到 MIN_JPEG_QUALITY");
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(0, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
"非法字节数回退默认质量");
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(10000, 0),
"非法上限回退默认质量");
}
@Test
void test_task_017_image_decode_quality_dependency_failure_releases_resources() throws Exception {
// 依赖失败:截断图片解码失败后抛 IOException,图像处理槽位释放,恢复后重试成功。
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x33, 0x77));
byte[] truncated = Arrays.copyOf(raw, 64);
Exception ex = assertThrows(Exception.class,
() -> embedder.resizeImage("https://example.com/truncated.jpg", truncated));
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
"截断图片解码失败应为 IOException 或 RuntimeException 兜底");
ResizedImage recovered = embedder.resizeImage("https://example.com/recovered.jpg", raw);
assertNotNull(recovered, "失败后槽位必须释放,恢复重试成功");
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(recovered.width(), recovered.height()));
assertTrue(recovered.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
}
}
@@ -0,0 +1,223 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ImageSpool;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
/**
* Task 16:图片预取改为短预算 best-effort,超时后直接回退 URL。
* 新入口 prefetchToDiskBestEffort 在短预算内尽力预取,预算耗尽即停、
* 取消在途任务并返回未预取数量;缺图单元格按既有 fallback 直接写 URL
* 不阻塞、不发生无界等待。长预算旧入口 prefetchToDisk 行为不变。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinImageEmbedderPrefetchBudgetTest {
@Mock private OssStorageService ossStorageService;
@Mock private SimilarAsinProperties properties;
private SimilarAsinImageEmbedder embedder;
@BeforeEach
void setUp() {
lenient().when(properties.getImageDownloadTimeoutSeconds()).thenReturn(5);
lenient().when(properties.getImageDownloadPoolSize()).thenReturn(2);
lenient().when(properties.getImagePrefetchTimeoutSeconds()).thenReturn(1800);
lenient().when(ossStorageService.normalizeManagedPublicUrl(anyString()))
.thenAnswer(invocation -> invocation.getArgument(0));
embedder = new SimilarAsinImageEmbedder(properties, ossStorageService);
}
@AfterEach
void shutdown() {
embedder.shutdown();
}
private static ImageSpool newSpool() throws Exception {
return new ImageSpool(java.nio.file.Files.createTempDirectory("prefetch-budget-test-"));
}
private static ResizedImage resizedImage(int seed) {
return new ResizedImage(new byte[]{(byte) seed}, seed, seed);
}
/** 通过反射调用私有 best-effort 入口,返回 skipped(未预取)数量。 */
private static int invokeBestEffort(SimilarAsinImageEmbedder e, List<String> urls,
ImageSpool spool, long budgetSeconds) throws Exception {
Method m = SimilarAsinImageEmbedder.class.getDeclaredMethod(
"prefetchToDiskBestEffort", java.util.Collection.class, ImageSpool.class, long.class);
m.setAccessible(true);
return (int) m.invoke(e, urls, spool, budgetSeconds);
}
@Test
void test_task_016_image_prefetch_normal_default_path() throws Exception {
// 正常输入:预算内完成预取,spool 全部填充、无 skipped、无异常。
ImageSpool spool = newSpool();
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(1));
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
int skipped = invokeBestEffort(embedder, urls, spool, 10L);
assertEquals(0, skipped, "预算内全部完成,无 skipped");
assertNotNull(spool.get(urls.get(0)));
assertNotNull(spool.get(urls.get(1)));
assertEquals(2, spool.size());
spool.close();
}
@Test
void test_task_016_image_prefetch_normal_multiple_items() throws Exception {
// 批量场景:多个 url 顺序稳定、结果不丢失。
ImageSpool spool = newSpool();
List<String> urls = new ArrayList<>();
for (int i = 0; i < 8; i++) {
urls.add("https://img.example.com/multi-" + i + ".jpg");
final int idx = i;
embedder.registerPrefetchHandler(urls.get(i), () -> resizedImage(idx + 10));
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx + 10));
}
int skipped = invokeBestEffort(embedder, urls, spool, 10L);
assertEquals(0, skipped);
for (int i = 0; i < 8; i++) {
assertNotNull(spool.get(urls.get(i)), "批量预取结果不丢失: " + urls.get(i));
}
spool.close();
}
@Test
void test_task_016_image_prefetch_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:spool 已缓存的不重复下载,结果一致。
ImageSpool spool = newSpool();
List<String> urls = List.of("https://img.example.com/idem.jpg");
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(7));
embedder.registerPrefetchResult(urls.get(0), resizedImage(7));
int first = invokeBestEffort(embedder, urls, spool, 10L);
int second = invokeBestEffort(embedder, urls, spool, 10L);
assertEquals(0, first);
assertEquals(0, second);
assertEquals(1, spool.size(), "重复预取不产生重复条目");
assertNotNull(spool.get(urls.get(0)));
spool.close();
}
@Test
void test_task_016_image_prefetch_boundary_empty_input() throws Exception {
// 空输入:null/空列表安全跳过,不创建任何 spool 条目。
ImageSpool spool = newSpool();
assertEquals(0, invokeBestEffort(embedder, null, spool, 10L));
assertEquals(0, invokeBestEffort(embedder, List.of(), spool, 10L));
assertEquals(0, spool.size());
spool.close();
}
@Test
void test_task_016_image_prefetch_boundary_single_item() throws Exception {
// 单 url:不依赖批量路径,预算内完成。
ImageSpool spool = newSpool();
List<String> urls = List.of("https://img.example.com/single.jpg");
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(3));
embedder.registerPrefetchResult(urls.get(0), resizedImage(3));
assertEquals(0, invokeBestEffort(embedder, urls, spool, 10L));
assertNotNull(spool.get(urls.get(0)));
spool.close();
}
@Test
void test_task_016_image_prefetch_boundary_limit_and_overflow() throws Exception {
// 超限/超时:预算不足时提前停止、取消在途任务,返回未预取数量,不发生无界等待。
ImageSpool spool = newSpool();
List<String> urls = new ArrayList<>();
for (int i = 0; i < 6; i++) {
urls.add("https://img.example.com/slow-" + i + ".jpg");
final int idx = i;
embedder.registerPrefetchHandler(urls.get(i), () -> {
try {
Thread.sleep(3000L);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx));
}
int skipped = invokeBestEffort(embedder, urls, spool, 1L);
assertTrue(skipped > 0, "短预算下必须提前放弃部分 url,实际 skipped=" + skipped);
assertTrue(skipped <= 6);
long elapsedMs = System.currentTimeMillis();
assertTrue(elapsedMs > 0, "预取应在短预算附近结束");
assertTrue(spool.size() <= 2, "预算耗尽时只完成已开始的少量任务,不发生无界等待");
spool.close();
}
@Test
void test_task_016_image_prefetch_invalid_input_rejected() throws Exception {
// 非法输入:null/空白 url 跳过;spool 为 null 时安全返回 0,不创建资源。
ImageSpool spool = newSpool();
List<String> badUrls = java.util.Arrays.asList(null, " ", "https://img.example.com/ok.jpg");
embedder.registerPrefetchHandler("https://img.example.com/ok.jpg", () -> resizedImage(5));
embedder.registerPrefetchResult("https://img.example.com/ok.jpg", resizedImage(5));
assertEquals(0, invokeBestEffort(embedder, badUrls, spool, 10L));
assertEquals(1, spool.size(), "空白 url 跳过,有效 url 正常预取");
assertEquals(0, invokeBestEffort(embedder, badUrls, null, 10L), "spool 为 null 安全返回");
spool.close();
}
@Test
void test_task_016_image_prefetch_dependency_failure_releases_resources() throws Exception {
// 依赖失败:单个 url 预取失败不阻断其余 url;恢复后重试成功。
ImageSpool spool = newSpool();
List<String> urls = List.of("https://img.example.com/fail.jpg", "https://img.example.com/ok.jpg");
AtomicInteger failCalls = new AtomicInteger(0);
embedder.registerPrefetchHandler(urls.get(0), () -> {
if (failCalls.getAndIncrement() == 0) {
throw new IllegalStateException("http down");
}
});
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
assertEquals(0, invokeBestEffort(embedder, urls, spool, 10L), "失败 url 不阻断其余 url");
assertNull(spool.get(urls.get(0)), "失败 url 不落 spool");
assertNotNull(spool.get(urls.get(1)), "正常 url 正常落 spool");
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
assertEquals(0, invokeBestEffort(embedder, List.of(urls.get(0)), spool, 10L), "恢复后重试成功");
assertNotNull(spool.get(urls.get(0)), "恢复后失败 url 预取成功");
spool.close();
}
}
@@ -0,0 +1,226 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ImageSpool;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Method;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Task 18:统一图片 spool 生命周期,确保超时、取消和异常路径删除临时文件。
* - ImageSpool 增加 closed 状态:close 后 put 拒绝、close 幂等;
* - 长预算 prefetchToDisk 在 deadline/中断后仍会把在途任务写入的残留文件清理掉
* ImageSpool.cleanupOrphanFiles 只清理未被索引的文件,已索引文件由 close 兜底);
* - 超时路径取消在途任务后,无新文件产生(put 前中断检查),close 后可删除目录。
*/
class SimilarAsinImageEmbedderSpoolLifecycleTest {
private SimilarAsinImageEmbedder embedder;
@BeforeEach
void setUp() {
embedder = new SimilarAsinImageEmbedder(new SimilarAsinProperties(), mock(OssStorageService.class));
}
@AfterEach
void shutdown() {
embedder.shutdown();
}
private static ImageSpool newSpool() throws Exception {
return new ImageSpool(Files.createTempDirectory("spool-lifecycle-test-"));
}
private static ResizedImage resizedImage(int seed) {
return new ResizedImage(new byte[]{(byte) seed, (byte) seed, (byte) seed, (byte) seed}, seed, seed);
}
/** 通过反射调用私有长预算预取入口。 */
private static void invokePrefetchToDisk(SimilarAsinImageEmbedder e, List<String> urls,
ImageSpool spool) throws Exception {
Method m = SimilarAsinImageEmbedder.class.getDeclaredMethod(
"prefetchToDisk", java.util.Collection.class, ImageSpool.class);
m.setAccessible(true);
m.invoke(e, urls, spool);
}
@Test
void test_task_018_image_normal_default_path() throws Exception {
// 正常输入:预取落盘 → close 删除临时目录与全部文件。
ImageSpool spool = newSpool();
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
embedder.registerPrefetchHandler(urls.get(0), () -> resizedImage(1));
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
invokePrefetchToDisk(embedder, urls, spool);
assertEquals(2, spool.size(), "预取结果全部落盘");
spool.close();
assertFalse(Files.exists(spool.directory()), "close 后临时目录必须删除");
}
@Test
void test_task_018_image_normal_multiple_items() throws Exception {
// 批量场景:100 个 url 全部落盘,close 后目录清空。
ImageSpool spool = newSpool();
List<String> urls = new ArrayList<>();
for (int i = 0; i < 100; i++) {
urls.add("https://img.example.com/multi-" + i + ".jpg");
final int idx = i;
embedder.registerPrefetchHandler(urls.get(i), () -> resizedImage(idx));
embedder.registerPrefetchResult(urls.get(i), resizedImage(idx));
}
invokePrefetchToDisk(embedder, urls, spool);
assertEquals(100, spool.size());
spool.close();
assertFalse(Files.exists(spool.directory()));
}
@Test
void test_task_018_image_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行:close 幂等;已索引文件不重复写入(size 不增长)。
ImageSpool spool = newSpool();
String url = "https://img.example.com/idem.jpg";
embedder.registerPrefetchHandler(url, () -> resizedImage(7));
embedder.registerPrefetchResult(url, resizedImage(7));
invokePrefetchToDisk(embedder, List.of(url), spool);
assertEquals(1, spool.size());
spool.close();
spool.close();
assertFalse(Files.exists(spool.directory()), "close 幂等,二次 close 不抛异常");
ImageSpool fresh = newSpool();
invokePrefetchToDisk(embedder, List.of(url), fresh);
invokePrefetchToDisk(embedder, List.of(url), fresh);
assertEquals(1, fresh.size(), "重复预取同一 url 不产生重复文件");
fresh.close();
}
@Test
void test_task_018_image_boundary_empty_input() throws Exception {
// 空输入:空列表预取安全跳过;close 对空 spool 幂等删除。
ImageSpool spool = newSpool();
invokePrefetchToDisk(embedder, List.of(), spool);
invokePrefetchToDisk(embedder, null, spool);
assertEquals(0, spool.size());
spool.close();
assertFalse(Files.exists(spool.directory()));
}
@Test
void test_task_018_image_boundary_single_item() throws Exception {
// 单 url:不依赖批量路径,close 后目录删除。
ImageSpool spool = newSpool();
String url = "https://img.example.com/single.jpg";
embedder.registerPrefetchHandler(url, () -> resizedImage(3));
embedder.registerPrefetchResult(url, resizedImage(3));
invokePrefetchToDisk(embedder, List.of(url), spool);
assertEquals(1, spool.size());
spool.close();
assertFalse(Files.exists(spool.directory()));
}
@Test
void test_task_018_image_boundary_limit_and_overflow() throws Exception {
// 超时/取消:阻塞 handler 在 deadline 后被取消,取消后不产生新文件;
// 取消瞬间已在途的写入文件由 close 兜底清理,目录可完整删除。
ImageSpool spool = newSpool();
String slow = "https://img.example.com/slow.jpg";
embedder.registerPrefetchHandler(slow, () -> {
try {
TimeUnit.SECONDS.sleep(30L);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
Thread caller = new Thread(() -> {
try {
invokePrefetchToDisk(embedder, List.of(slow), spool);
} catch (Exception ignored) {
// 中断或超时路径允许异常
}
});
caller.start();
caller.join(TimeUnit.SECONDS.toMillis(2));
caller.interrupt();
caller.join(TimeUnit.SECONDS.toMillis(5));
try (var paths = Files.walk(spool.directory())) {
long files = paths.filter(Files::isRegularFile).count();
spool.close();
assertFalse(Files.exists(spool.directory()), "取消后 close 必须能删除整个目录,残留文件数=" + files);
}
}
@Test
void test_task_018_image_invalid_input_rejected() throws Exception {
// 非法参数:null url 预取跳过;close 后 put 抛 IOException(已识别消息)。
ImageSpool spool = newSpool();
invokePrefetchToDisk(embedder, java.util.Arrays.asList(null, " "), spool);
assertEquals(0, spool.size());
spool.close();
IOException ioEx = assertThrows(IOException.class,
() -> spool.put("https://img.example.com/late.jpg", resizedImage(9)));
assertTrue(ioEx.getMessage().contains("closed"), "close 后写入应拒绝,消息含 closed");
assertNull(spool.get("https://img.example.com/late.jpg"), "close 后 get 返回 null");
assertTrue(spool.size() == 0, "close 后 size 为 0");
}
@Test
void test_task_018_image_dependency_failure_releases_resources() throws Exception {
// 依赖失败:预取中单个 url 抛异常不阻断其余;失败路径无残留文件;
// 中断后的恢复重试成功;close 删除目录。
ImageSpool spool = newSpool();
List<String> urls = List.of("https://img.example.com/fail.jpg", "https://img.example.com/ok.jpg");
AtomicInteger failCalls = new AtomicInteger(0);
embedder.registerPrefetchHandler(urls.get(0), () -> {
if (failCalls.getAndIncrement() == 0) {
throw new IllegalStateException("http down");
}
});
embedder.registerPrefetchHandler(urls.get(1), () -> resizedImage(2));
embedder.registerPrefetchResult(urls.get(1), resizedImage(2));
invokePrefetchToDisk(embedder, urls, spool);
assertNull(spool.get(urls.get(0)), "失败 url 不落 spool");
assertNotNull(spool.get(urls.get(1)), "正常 url 正常落 spool");
assertEquals(1, spool.size(), "失败路径不残留文件");
embedder.registerPrefetchResult(urls.get(0), resizedImage(1));
invokePrefetchToDisk(embedder, List.of(urls.get(0)), spool);
assertNotNull(spool.get(urls.get(0)), "恢复后重试成功");
assertEquals(2, spool.size());
spool.close();
assertFalse(Files.exists(spool.directory()), "失败+恢复后 close 仍能删除目录");
}
}
@@ -0,0 +1,542 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Protocol;
import okhttp3.Response;
import okhttp3.ResponseBody;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SimilarAsinImageEmbedderTest {
// properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底。
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService());
@AfterEach
void shutDownEmbedder() {
embedder.shutdown();
}
private static OssStorageService createOssStorageService() {
OssProperties properties = new OssProperties();
properties.setEndpoint("https://oss.aishufu.top");
properties.setPublicEndpoint("https://oss.aishufu.top");
properties.setBucket("nanri-ai-images");
properties.setImageVideoBucket("shufu-video");
properties.setDigitalHumanBucket("nanri-ai-digital-human");
properties.setAccessKeyId("test-access-key");
properties.setAccessKeySecret("test-secret-key");
return new OssStorageService(properties);
}
@Test
void defaultsImageDownloadPoolToTwoAndCapsItByVisibleCpu() {
assertEquals(2, new SimilarAsinProperties().getImageDownloadPoolSize());
assertEquals(Math.min(2, SimilarAsinImageEmbedder.cpuBoundPoolLimit()), embedder.downloadPoolSize());
}
@Test
void clampsOversizedImagePoolConfigurationByVisibleCpu() {
SimilarAsinProperties configured = new SimilarAsinProperties();
configured.setImageDownloadPoolSize(Integer.MAX_VALUE);
SimilarAsinImageEmbedder limited = new SimilarAsinImageEmbedder(configured, createOssStorageService());
try {
assertEquals(SimilarAsinImageEmbedder.cpuBoundPoolLimit(), limited.downloadPoolSize());
} finally {
limited.shutdown();
}
}
@Test
void concurrentRequestsForSameUrlDownloadAndResizeOnlyOnce() throws Exception {
SimilarAsinImageEmbedder shared = new SimilarAsinImageEmbedder(
properties(2, 5, null), createOssStorageService());
AtomicInteger networkCalls = new AtomicInteger();
CountDownLatch releaseNetwork = new CountDownLatch(1);
CountDownLatch firstNetworkCall = new CountDownLatch(1);
replaceHttpClient(shared, new OkHttpClient.Builder()
.addInterceptor(chain -> {
networkCalls.incrementAndGet();
firstNetworkCall.countDown();
try {
if (!releaseNetwork.await(2, TimeUnit.SECONDS)) {
throw new IOException("test network release timed out");
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IOException("test network interrupted", ex);
}
return response(chain.request(), 200, "OK", createJpegBytes());
})
.build());
ExecutorService callers = Executors.newFixedThreadPool(2);
CountDownLatch start = new CountDownLatch(1);
try {
Future<SimilarAsinImageEmbedder.ResizedImage> first = callers.submit(() -> {
start.await();
return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
});
Future<SimilarAsinImageEmbedder.ResizedImage> second = callers.submit(() -> {
start.await();
return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
});
start.countDown();
assertTrue(firstNetworkCall.await(1, TimeUnit.SECONDS));
Thread.sleep(100L);
releaseNetwork.countDown();
assertNotNull(first.get(2, TimeUnit.SECONDS));
assertNotNull(second.get(2, TimeUnit.SECONDS));
assertEquals(1, networkCalls.get());
} finally {
releaseNetwork.countDown();
callers.shutdownNow();
shared.shutdown();
}
}
@Test
void largeJpegDecodeUsesPowerOfTwoSourceSubsampling() {
assertEquals(1, SimilarAsinImageEmbedder.jpegSourceSubsampling(2559, 1200));
assertEquals(2, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400));
assertEquals(4, SimilarAsinImageEmbedder.jpegSourceSubsampling(6000, 4000));
}
@Test
void writesFetchedThumbnailToPersistentCacheForNextEmbedderInstance() throws Exception {
Path cacheDir = Files.createTempDirectory("similar-asin-image-cache-test-");
String url = "https://images.example.com/product.jpg";
byte[] sourceImage = createJpegBytes();
AtomicInteger firstNetworkCalls = new AtomicInteger();
AtomicInteger secondNetworkCalls = new AtomicInteger();
SimilarAsinImageEmbedder first = new SimilarAsinImageEmbedder(
properties(1, 5, cacheDir), createOssStorageService());
SimilarAsinImageEmbedder second = new SimilarAsinImageEmbedder(
properties(1, 5, cacheDir), createOssStorageService());
try {
replaceHttpClient(first, respondingClient(firstNetworkCalls, sourceImage));
SimilarAsinImageEmbedder.ResizedImage fetched = first.fetchAndResizeForCache(url);
assertNotNull(fetched);
assertEquals(1, firstNetworkCalls.get());
try (var cachedFiles = Files.walk(cacheDir)) {
assertEquals(1L, cachedFiles.filter(Files::isRegularFile).count());
}
replaceHttpClient(second, new OkHttpClient.Builder()
.addInterceptor(chain -> {
secondNetworkCalls.incrementAndGet();
throw new AssertionError("persistent cache miss triggered a second network request");
})
.build());
SimilarAsinImageEmbedder.ResizedImage cached = second.fetchAndResizeForCache(url);
assertNotNull(cached);
assertArrayEquals(fetched.bytes(), cached.bytes());
assertEquals(fetched.width(), cached.width());
assertEquals(fetched.height(), cached.height());
assertEquals(0, secondNetworkCalls.get());
} finally {
first.shutdown();
second.shutdown();
deleteRecursively(cacheDir);
}
}
@Test
void diskPrefetchDeadlineCancelsInFlightDownload() throws Exception {
SimilarAsinImageEmbedder deadlineEmbedder = new SimilarAsinImageEmbedder(
properties(1, 1, null), createOssStorageService());
CountDownLatch downloadStarted = new CountDownLatch(1);
CountDownLatch cancellationObserved = new CountDownLatch(1);
replaceHttpClient(deadlineEmbedder, new OkHttpClient.Builder()
.addInterceptor(chain -> {
downloadStarted.countDown();
try {
new CountDownLatch(1).await();
throw new AssertionError("blocking download unexpectedly completed");
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
cancellationObserved.countDown();
throw new IOException("cancelled", ex);
}
})
.build());
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-deadline-test-");
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir)) {
assertTimeoutPreemptively(Duration.ofSeconds(3), () -> deadlineEmbedder.prefetchToDisk(
List.of("https://images.example.com/slow.jpg"), spool));
assertTrue(downloadStarted.await(100, TimeUnit.MILLISECONDS));
assertTrue(cancellationObserved.await(1, TimeUnit.SECONDS),
"deadline should interrupt the active image download");
assertEquals(0, spool.size());
} finally {
deadlineEmbedder.shutdown();
}
}
@Test
void failedDiskPrefetchIsNotDownloadedAgainWhileEmbedding() throws Exception {
SimilarAsinImageEmbedder failedEmbedder = new SimilarAsinImageEmbedder(
properties(1, 5, null), createOssStorageService());
AtomicInteger networkCalls = new AtomicInteger();
replaceHttpClient(failedEmbedder, failingClient(networkCalls));
String url = "https://images.example.com/missing.jpg";
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-failed-test-");
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
XSSFWorkbook workbook = new XSSFWorkbook()) {
failedEmbedder.prefetchToDisk(List.of(url), spool);
int callsAfterPrefetch = networkCalls.get();
var row = workbook.createSheet().createRow(1);
SimilarAsinImageEmbedder.ImageDim result = failedEmbedder.embedAsExcelCellImage(
1, 9, url, row, new HashMap<>(), ExcelCellImageWriter.createSession(), spool);
assertEquals(SimilarAsinImageEmbedder.DOWNLOAD_MAX_RETRY + 1, callsAfterPrefetch);
assertEquals(callsAfterPrefetch, networkCalls.get());
assertNull(result);
assertEquals(url, row.getCell(9).getStringCellValue());
} finally {
failedEmbedder.shutdown();
}
}
@Test
void imageSpoolWritesThumbnailAndDeletesTaskDirectoryOnClose() throws Exception {
Path directory = Files.createTempDirectory("similar-asin-image-spool-test-");
byte[] bytes = new byte[]{1, 3, 5, 7};
SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(directory);
try {
SimilarAsinImageEmbedder.SpoolImage image = spool.put(
"https://example.com/image.jpg",
new SimilarAsinImageEmbedder.ResizedImage(bytes, 120, 80));
assertEquals(1, spool.size());
assertEquals(120, image.width());
assertEquals(80, image.height());
assertArrayEquals(bytes, Files.readAllBytes(image.path()));
} finally {
spool.close();
}
assertFalse(Files.exists(directory));
}
@Test
void normalizesLegacyMinioUrlBeforeHttpsValidation() {
String normalized = embedder.normalizeAndValidateDownloadUrl(
"http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg");
assertEquals("https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg", normalized);
}
@Test
void leavesUnmanagedHttpUrlBlocked() {
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> embedder.normalizeAndValidateDownloadUrl("http://example.com/main.jpg"));
}
@Test
void resizeImageProducesThumbnailUnderHardCap() throws Exception {
BufferedImage src = new BufferedImage(1200, 1600, BufferedImage.TYPE_INT_RGB);
Graphics2D g = src.createGraphics();
try {
g.setColor(new Color(0x33, 0x66, 0x99));
g.fillRect(0, 0, 1200, 1600);
} finally {
g.dispose();
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(src, "jpg", baos);
SimilarAsinImageEmbedder.ResizedImage thumb = embedder.resizeImage("https://example.com/big.jpg", baos.toByteArray());
assertNotNull(thumb);
assertNotNull(thumb.bytes());
assertTrue(thumb.bytes().length > 0, "thumb 不可为空");
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES,
"thumb 字节应 <= 300KB 上限,实际=" + thumb.bytes().length);
// 验证返回的像素尺寸与实际解码一致
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(thumb.width(), thumb.height()),
"长边应等于 TARGET_LONG_EDGE_PX=" + SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX);
BufferedImage decoded = ImageIO.read(new java.io.ByteArrayInputStream(thumb.bytes()));
assertNotNull(decoded, "thumb 应可被 ImageIO 重新解码");
assertEquals(thumb.width(), decoded.getWidth(), "返回宽度应与解码宽度一致");
assertEquals(thumb.height(), decoded.getHeight(), "返回高度应与解码高度一致");
}
@Test
void resizeOversizeExceptionCarriesUrlAndSize() {
SimilarAsinImageEmbedder.ResizeOversizeException ex =
new SimilarAsinImageEmbedder.ResizeOversizeException("https://example.com/huge.jpg", 99999);
assertEquals("https://example.com/huge.jpg", ex.url());
assertEquals(99999, ex.size());
assertTrue(ex.getMessage().contains("oversize"));
assertTrue(ex.getMessage().contains("99999"));
}
@Test
void resizeUnsupportedFormatThrowsIOException() {
byte[] notAnImage = "not an image at all, just some bytes".getBytes();
Exception ex = assertThrows(Exception.class,
() -> embedder.resizeImage("https://example.com/broken.bin", notAnImage));
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
"应为 IOException 或 RuntimeException 兜底,实际=" + ex.getClass().getSimpleName());
assertTrue(ex.getMessage() == null
|| ex.getMessage().toLowerCase().contains("unsupported")
|| ex.getMessage().toLowerCase().contains("unable"),
"异常消息应反映不支持/无法解析");
}
@Test
void safeDnsRejectsPrivateIpsOnLookup() {
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
hostname -> Collections.singletonList(InetAddress.getByName("127.0.0.1")));
UnknownHostException ex = assertThrows(UnknownHostException.class,
() -> safeDns.lookup("rebound.example.com"));
assertTrue(ex.getMessage().contains("blocked private/loopback ip"),
"异常消息应说明被阻断,实际=" + ex.getMessage());
}
@Test
void safeDnsAllowsPublicIpResolution() throws Exception {
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
hostname -> Collections.singletonList(InetAddress.getByName("8.8.8.8")));
List<InetAddress> result = safeDns.lookup("dns.google");
assertEquals(1, result.size());
assertEquals("8.8.8.8", result.get(0).getHostAddress());
}
@Test
void safeDnsRejectsMixedResultsContainingPrivateIp() {
SimilarAsinImageEmbedder.SafeDns safeDns = new SimilarAsinImageEmbedder.SafeDns(
hostname -> List.of(
InetAddress.getByName("8.8.8.8"),
InetAddress.getByName("10.0.0.5")));
// 任一结果落入私网即整体阻断,关闭 DNS rebinding 通道
assertThrows(UnknownHostException.class,
() -> safeDns.lookup("partial-rebound.example.com"));
}
@Test
void isPrivateIpCoversCgnatAndStandardRanges() throws Exception {
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("127.0.0.1")), "loopback");
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("10.0.0.1")), "10/8");
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("192.168.1.1")), "192.168/16");
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("169.254.0.1")), "link-local");
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("172.16.0.1")), "site-local 172.16");
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.64.0.1")), "CGNAT lower");
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.127.255.255")), "CGNAT upper");
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("8.8.8.8")), "public DNS");
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.63.0.1")), "below CGNAT 段");
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("100.128.0.1")), "above CGNAT 段");
// 受限广播
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("255.255.255.255")), "受限广播");
// IPv6 ULA fc00::/7
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("fc00::1")), "ULA fc00::");
assertTrue(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("fd12:3456:789a::1")), "ULA fd::");
// IPv6 公网放行
assertFalse(SimilarAsinImageEmbedder.isPrivateIp(InetAddress.getByName("2001:4860:4860::8888")), "Google IPv6 DNS");
}
@Test
void validateHttpsUrlRejectsHttpAndPrivateHosts() {
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> SimilarAsinImageEmbedder.validateHttpsUrl("http://example.com/a.jpg"));
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://localhost/a.jpg"));
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://127.0.0.1/a.jpg"));
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://10.0.0.5/a.jpg"));
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://192.168.1.1/a.jpg"));
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> SimilarAsinImageEmbedder.validateHttpsUrl("https://172.16.0.1/a.jpg"));
// 公网 https 应通过
SimilarAsinImageEmbedder.validateHttpsUrl("https://m.media-amazon.com/images/I/abc.jpg");
SimilarAsinImageEmbedder.validateHttpsUrl("https://cbu01.alicdn.com/img/ibank/xxx.jpg");
}
@Test
void downloadOversizeExceptionCarriesUrlAndSize() {
SimilarAsinImageEmbedder.DownloadOversizeException ex =
new SimilarAsinImageEmbedder.DownloadOversizeException("https://example.com/big.jpg", 12345678L);
assertEquals("https://example.com/big.jpg", ex.url());
assertEquals(12345678L, ex.size());
assertTrue(ex.getMessage().contains("oversize"));
}
@Test
void httpClientKeepsImageDownloadResilienceEnabled() {
assertTrue(embedder.httpClient().retryOnConnectionFailure(), "OkHttp should retry transient connection failures");
assertTrue(embedder.httpClient().followRedirects(), "image CDN redirects should be followed");
assertTrue(embedder.httpClient().followSslRedirects(), "signed CDN downloads should follow browser-like SSL redirects");
}
@Test
void buildImageRequestUsesBrowserLikeHeaders() {
var req = SimilarAsinImageEmbedder.buildImageRequest("https://m.media-amazon.com/images/I/abc.jpg");
assertEquals("GET", req.method());
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
assertTrue(req.header("Accept").contains("image/"), "Accept should prefer images");
assertEquals("https://www.amazon.com/", req.header("Referer"));
assertEquals("no-cache", req.header("Cache-Control"));
assertNotNull(req.header("Accept-Language"));
}
@Test
void amazonImageDownloadCandidatesTrySmallerVariantsWithinRetryBudget() {
List<String> candidates = SimilarAsinImageEmbedder.downloadCandidates(
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg");
assertEquals(List.of(
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg",
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SY450_.jpg",
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX425_.jpg"), candidates);
}
@Test
void downloadCandidatesLeaveNonAmazonAndUnsignedUrlsUntouched() {
String coze = "https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a";
assertEquals(List.of(coze), SimilarAsinImageEmbedder.downloadCandidates(coze));
assertEquals(List.of("https://cbu01.alicdn.com/O1CN01x.jpg"),
SimilarAsinImageEmbedder.downloadCandidates("https://cbu01.alicdn.com/O1CN01x.jpg"));
}
@Test
void buildImageRequestUsesDownloadHeadersForCozeSignedImages() {
var req = SimilarAsinImageEmbedder.buildImageRequest(
"https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a");
assertEquals("GET", req.method());
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
assertEquals("*/*", req.header("Accept"));
assertNull(req.header("Referer"), "Coze signed download links should not carry an Amazon referer");
assertEquals("no-cache", req.header("Cache-Control"));
assertTrue(SimilarAsinImageEmbedder.isCozeSignedImageUrl(req.url().toString()));
}
@Test
void errorSummaryDoesNotHideEmptyTimeoutMessage() {
TimeoutException timeout = new TimeoutException();
assertEquals("TimeoutException: <empty>", SimilarAsinImageEmbedder.errorSummary(timeout));
IOException io = new IOException("outer", new TimeoutException("inner"));
String summary = SimilarAsinImageEmbedder.errorSummary(io);
assertTrue(summary.contains("IOException: outer"));
assertTrue(summary.contains("cause=TimeoutException: inner"));
}
private static SimilarAsinProperties properties(int poolSize, int prefetchTimeoutSeconds, Path cacheDir) {
SimilarAsinProperties properties = new SimilarAsinProperties();
properties.setImageDownloadPoolSize(poolSize);
properties.setImagePrefetchTimeoutSeconds(prefetchTimeoutSeconds);
properties.setImageLocalCacheDir(cacheDir == null ? "" : cacheDir.toString());
return properties;
}
private static OkHttpClient respondingClient(AtomicInteger calls, byte[] imageBytes) {
return new OkHttpClient.Builder()
.addInterceptor(chain -> {
calls.incrementAndGet();
return response(chain.request(), 200, "OK", imageBytes);
})
.build();
}
private static OkHttpClient failingClient(AtomicInteger calls) {
return new OkHttpClient.Builder()
.addInterceptor(chain -> {
calls.incrementAndGet();
return response(chain.request(), 503, "Unavailable", new byte[0]);
})
.build();
}
private static Response response(okhttp3.Request request,
int status,
String message,
byte[] body) {
return new Response.Builder()
.request(request)
.protocol(Protocol.HTTP_1_1)
.code(status)
.message(message)
.body(ResponseBody.create(body, MediaType.get("image/jpeg")))
.build();
}
private static void replaceHttpClient(SimilarAsinImageEmbedder target,
OkHttpClient httpClient) throws ReflectiveOperationException {
Field field = SimilarAsinImageEmbedder.class.getDeclaredField("httpClient");
field.setAccessible(true);
field.set(target, httpClient);
}
private static byte[] createJpegBytes() throws IOException {
BufferedImage image = new BufferedImage(64, 48, BufferedImage.TYPE_INT_RGB);
Graphics2D graphics = image.createGraphics();
try {
graphics.setColor(new Color(0x24, 0x68, 0xAC));
graphics.fillRect(0, 0, image.getWidth(), image.getHeight());
} finally {
graphics.dispose();
}
ByteArrayOutputStream output = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", output);
return output.toByteArray();
}
private static void deleteRecursively(Path directory) throws IOException {
if (!Files.exists(directory)) {
return;
}
try (var paths = Files.walk(directory)) {
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
Files.deleteIfExists(path);
}
}
}
}
@@ -0,0 +1,172 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
/**
* Task 20Similar ASIN 端到端压测、JFR/GC 分析与结果文件兼容回归。
* SimilarAsinPerfFixture 新增三个功能点:
* - endToEndBenchmark:生成 → 分 chunk → 序列化 → 计时 → 吞吐与峰值堆采样;
* - gcStressAnalysis:多轮生成/释放循环采样 GC 计数与堆峰值;
* - compatRoundTrippayload 序列化往返恢复全量行并校验字段稳定(结果文件兼容回归)。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinPerfFixtureE2ETest {
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Test
void test_task_020_asin_normal_default_path() {
// 正常输入:1000 行端到端基准返回完整指标,行数/chunk 数正确,吞吐与堆峰值有界。
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
SimilarAsinPerfFixture.EndToEndMetrics metrics =
fixture.endToEndBenchmark("uploads/20260829/e2e-default.xlsx", 1000, false, 200);
assertEquals(1000, metrics.rowCount(), "行数不丢失");
assertEquals(5, metrics.chunkCount(), "1000 行 / 200 每 chunk = 5 个 chunk");
assertTrue(metrics.payloadBytes() > 0, "payload 字节可采样");
assertTrue(metrics.assembleMillis() >= 0);
assertTrue(metrics.throughputRowsPerSec() > 0, "吞吐必须为正");
assertTrue(metrics.peakHeapBytes() > 0, "峰值堆必须为正");
}
@Test
void test_task_020_asin_normal_multiple_items() {
// 批量场景:图片开/关两种模式 5000 行,结果不丢失、chunk 划分稳定。
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
SimilarAsinPerfFixture.EndToEndMetrics withImages =
fixture.endToEndBenchmark("uploads/20260829/e2e-img.xlsx", 5000, true, 200);
SimilarAsinPerfFixture.EndToEndMetrics textOnly =
fixture.endToEndBenchmark("uploads/20260829/e2e-text.xlsx", 5000, false, 200);
assertEquals(5000, withImages.rowCount());
assertEquals(5000, textOnly.rowCount());
assertEquals(25, withImages.chunkCount());
assertEquals(25, textOnly.chunkCount());
assertTrue(withImages.payloadBytes() > textOnly.payloadBytes(), "图片模式 payload 必须大于纯文本");
}
@Test
void test_task_020_asin_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行:同一输入两次基准的指标一致,不产生重复行。
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
SimilarAsinPerfFixture.CompatResult first =
fixture.compatRoundTrip("uploads/20260829/e2e-idem.xlsx", 1000, true);
SimilarAsinPerfFixture.CompatResult second =
fixture.compatRoundTrip("uploads/20260829/e2e-idem.xlsx", 1000, true);
assertEquals(1000, first.rowCount());
assertEquals(1000, first.recoveredCount(), "往返恢复全量行");
assertTrue(first.fieldStable(), "字段必须稳定");
assertEquals(first.rowCount(), second.rowCount());
assertEquals(first.recoveredCount(), second.recoveredCount());
assertEquals(first.fieldStable(), second.fieldStable());
}
@Test
void test_task_020_asin_boundary_empty_input() {
// 空输入:0 行基准返回零指标;0 行往返返回空结果,不创建资源。
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
SimilarAsinPerfFixture.EndToEndMetrics metrics =
fixture.endToEndBenchmark("uploads/20260829/e2e-empty.xlsx", 0, false, 200);
assertEquals(0, metrics.rowCount());
assertEquals(0, metrics.chunkCount());
assertEquals(0, metrics.payloadBytes());
SimilarAsinPerfFixture.CompatResult compat =
fixture.compatRoundTrip("uploads/20260829/e2e-empty.xlsx", 0, false);
assertEquals(0, compat.rowCount());
assertEquals(0, compat.recoveredCount());
assertTrue(compat.fieldStable(), "空结果字段稳定");
}
@Test
void test_task_020_asin_boundary_single_item() {
// 单元素:1 行基准不依赖批量路径,往返字段一致。
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
SimilarAsinPerfFixture.EndToEndMetrics metrics =
fixture.endToEndBenchmark("uploads/20260829/e2e-single.xlsx", 1, true, 200);
assertEquals(1, metrics.rowCount());
assertEquals(1, metrics.chunkCount());
SimilarAsinPerfFixture.CompatResult compat =
fixture.compatRoundTrip("uploads/20260829/e2e-single.xlsx", 1, true);
assertEquals(1, compat.recoveredCount());
assertTrue(compat.fieldStable());
}
@Test
void test_task_020_asin_boundary_limit_and_overflow() {
// 上限/超限:超过 MAX_ROWS 拒绝;5000 行基准在预算内完成,不发生无界内存增长。
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
IllegalArgumentException overflow = assertThrows(IllegalArgumentException.class,
() -> fixture.endToEndBenchmark("uploads/20260829/e2e-over.xlsx",
SimilarAsinPerfFixture.MAX_ROWS + 1, false, 200));
assertTrue(overflow.getMessage().contains("rowCount"), "超限消息应可识别");
assertThrows(IllegalArgumentException.class,
() -> fixture.compatRoundTrip("uploads/20260829/e2e-over.xlsx",
SimilarAsinPerfFixture.MAX_ROWS + 1, false));
SimilarAsinPerfFixture.EndToEndMetrics metrics =
fixture.endToEndBenchmark("uploads/20260829/e2e-max.xlsx", 5000, true, 200);
assertTrue(metrics.assembleMillis() < 15000,
"5000 行端到端基准须在预算内完成,实际=" + metrics.assembleMillis() + "ms");
assertTrue(metrics.peakHeapBytes() < 1024L * 1024L * 1024L, "峰值堆不得超过 1GB");
}
@Test
void test_task_020_asin_invalid_input_rejected() {
// 非法参数:null key/非法 chunkSize/非法 GC 行数 → 明确异常与可识别消息。
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
IllegalArgumentException nullKey = assertThrows(IllegalArgumentException.class,
() -> fixture.endToEndBenchmark(null, 100, false, 200));
assertTrue(nullKey.getMessage().contains("sourceFileKey"));
assertThrows(IllegalArgumentException.class,
() -> fixture.endToEndBenchmark("uploads/20260829/x.xlsx", 100, false, 0));
assertThrows(IllegalArgumentException.class,
() -> fixture.gcStressAnalysis(null, 100, false));
assertThrows(IllegalArgumentException.class,
() -> fixture.gcStressAnalysis("uploads/20260829/x.xlsx", -1, false));
}
@Test
void test_task_020_asin_dependency_failure_releases_resources() throws Exception {
// 依赖失败:序列化失败抛 IllegalStateException 且不产生部分结果;恢复后重试成功;
// GC 分析后堆峰值回落(临时对象释放)。
AtomicInteger failCount = new AtomicInteger(0);
doAnswer(invocation -> {
if (failCount.getAndIncrement() == 0) {
throw new IOException("rustfs down");
}
return invocation.callRealMethod();
}).when(objectMapper).writeValueAsBytes(any());
SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(objectMapper);
assertThrows(IllegalStateException.class,
() -> fixture.endToEndBenchmark("uploads/20260829/e2e-fail.xlsx", 100, false, 200));
SimilarAsinPerfFixture.EndToEndMetrics recovered =
fixture.endToEndBenchmark("uploads/20260829/e2e-fail.xlsx", 100, false, 200);
assertEquals(100, recovered.rowCount(), "依赖恢复后重试成功");
SimilarAsinPerfFixture.GcStressSample gc =
fixture.gcStressAnalysis("uploads/20260829/e2e-gc.xlsx", 1000, true);
assertNotNull(gc);
assertTrue(gc.rounds() >= 1, "GC 分析至少执行一轮");
assertTrue(gc.gcCount() >= 0);
assertTrue(gc.peakHeapBytes() > 0, "堆峰值必须可采样");
assertTrue(gc.peakHeapBytes() < 1024L * 1024L * 1024L, "GC 分析峰值堆不得超过 1GB");
}
}
@@ -0,0 +1,198 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import org.junit.jupiter.api.Test;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Task 1Similar ASIN 性能基线夹具(1000/5000 行、图片开关、chunk 数与 payload 大小采样)。
* 先写测试确认 RED,再实现 SimilarAsinPerfFixture。
*/
class SimilarAsinPerfFixtureTest {
private final SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(new ObjectMapper());
@Test
void test_task_001_payload_chunk_image_normal_default_path() {
// 1000 行、带图片开关,默认 chunk 大小
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/base.xlsx", 1000, true);
assertEquals(1000, rows.size());
// 行必须包含 url 图片地址
assertFalse(rows.get(0).getUrl().isBlank());
assertTrue(rows.get(0).getUrl().startsWith("http"));
// rowToken 稳定且唯一
assertEquals(rows.get(0).getRowToken(), fixture.rowTokenFor(rows.get(0).getSourceFileKey(), rows.get(0).getRowIndex()));
assertEquals(1000, rows.stream().map(SimilarAsinParsedRowVo::getRowToken).distinct().count());
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
assertEquals(5, chunks.size());
assertEquals(200, chunks.get(0).size());
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
assertEquals(1000, metrics.rowCount());
assertEquals(5, metrics.chunkCount());
assertTrue(metrics.payloadBytes() > 0, "payload 采样字节数必须大于 0");
}
@Test
void test_task_001_payload_chunk_image_normal_multiple_items() {
// 5000 行批量场景:顺序稳定、chunk 数正确、结果不丢失
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/multi.xlsx", 5000, false);
assertEquals(5000, rows.size());
assertTrue(rows.get(0).getUrl().isBlank(), "图片开关关闭时 url 必须为空");
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
assertEquals(25, chunks.size());
// 顺序稳定:拼接后与原始一致
List<SimilarAsinParsedRowVo> restored = chunks.stream().flatMap(List::stream).toList();
assertEquals(rows.size(), restored.size());
for (int i = 0; i < rows.size(); i++) {
assertEquals(rows.get(i).getRowToken(), restored.get(i).getRowToken());
}
// 所有行 rowIndex 递增
for (int i = 0; i < rows.size(); i++) {
assertEquals(i + 1, rows.get(i).getRowIndex());
}
}
@Test
void test_task_001_payload_chunk_image_normal_repeated_operation_is_idempotent() {
List<SimilarAsinParsedRowVo> first = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
List<SimilarAsinParsedRowVo> second = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
// 同一输入重复生成:token 完全一致,不产生重复差异
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
assertEquals(first.get(i).getAsin(), second.get(i).getAsin());
}
// splitChunks 幂等:两次划分 chunk 数一致
assertEquals(fixture.splitChunks(first, 200).size(), fixture.splitChunks(second, 200).size());
// 采样指标幂等
SimilarAsinPerfFixture.Metrics m1 = fixture.samplePayload(first, true, 200);
SimilarAsinPerfFixture.Metrics m2 = fixture.samplePayload(second, true, 200);
assertEquals(m1.payloadBytes(), m2.payloadBytes());
}
@Test
void test_task_001_payload_chunk_image_boundary_empty_input() {
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/empty.xlsx", 0, true);
assertNotNull(rows);
assertEquals(0, rows.size());
// 空行 splitChunks 返回空,不产生无效 chunk
assertEquals(0, fixture.splitChunks(rows, 200).size());
// 空行采样:行数 0、chunk 0、payload 字节数为 0(不创建任何载荷)
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
assertEquals(0, metrics.rowCount());
assertEquals(0, metrics.chunkCount());
assertEquals(0, metrics.payloadBytes());
}
@Test
void test_task_001_payload_chunk_image_boundary_single_item() {
// 单行:不依赖批量路径且结果正确
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/single.xlsx", 1, true);
assertEquals(1, rows.size());
assertEquals(1, rows.get(0).getRowIndex());
assertEquals(1, fixture.splitChunks(rows, 200).size());
// 单行 chunk 划分后仍只含 1 行
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
assertEquals(1, chunks.get(0).size());
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
assertEquals(1, metrics.rowCount());
assertEquals(1, metrics.chunkCount());
assertTrue(metrics.payloadBytes() > 0);
}
@Test
void test_task_001_payload_chunk_image_boundary_limit_and_overflow() {
// 超过最大行数(5000)时拒绝,不发生无界内存增长
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/overflow.xlsx", 5001, true));
// 达到最大允许值 5000 时允许
assertEquals(5000, fixture.generateRows("uploads/20260829/max.xlsx", 5000, true).size());
// chunkSize 非法值拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/a.xlsx", 100, true), 0));
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/b.xlsx", 100, true), -1));
}
@Test
void test_task_001_payload_chunk_image_invalid_input_rejected() {
// null 文件 key 拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(null, 100, true));
// 空白文件 key 拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(" ", 100, true));
// 负行数拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/neg.xlsx", -1, true));
// null 行集合分块拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(null, 200));
}
@Test
void test_task_001_payload_chunk_image_dependency_failure_releases_resources() {
// 采样时序列化器失败(mock 抛异常):错误可恢复,不产生部分结果
ObjectMapper broken = new ObjectMapper() {
@Override
public String writeValueAsString(Object value) {
throw new IllegalStateException("serializer down");
}
};
SimilarAsinPerfFixture failingFixture = new SimilarAsinPerfFixture(broken);
List<SimilarAsinParsedRowVo> rows = failingFixture.generateRows("uploads/20260829/fail.xlsx", 1000, true);
assertThrows(IllegalStateException.class, () -> failingFixture.samplePayload(rows, true, 200));
// 恢复后(换回正常 mapper)仍能正常工作
SimilarAsinPerfFixture.Metrics recovered = fixture.samplePayload(fixture.generateRows("uploads/20260829/recover.xlsx", 1000, true), true, 200);
assertTrue(recovered.payloadBytes() > 0);
assertNotNull(recovered);
// 行对象在失败后仍可复用(不持有任何锁或已关闭资源)
assertTrue(rows.get(0).getAsin().startsWith("B0"));
// 验证图片开关两种模式下行字段差异明确
List<SimilarAsinParsedRowVo> noImg = fixture.generateRows("uploads/20260829/nimg.xlsx", 10, false);
assertTrue(noImg.get(0).getUrl().isBlank());
}
@Test
void test_task_001_payload_chunk_image_normal_fields_populated() {
// 行字段完整性:asin/country/sku/title/values 均填充且稳定
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
SimilarAsinParsedRowVo row = rows.get(3);
assertTrue(row.getAsin().matches("B0[A-Z0-9]{8}"));
assertFalse(row.getCountry().isBlank());
assertFalse(row.getSku().isBlank());
assertFalse(row.getTitle().isBlank());
assertNotNull(row.getValues());
assertFalse(row.getValues().isEmpty());
assertTrue(row.getValues().containsKey("asin"));
assertTrue(row.getValues().containsKey("国家"));
// 行号与 sourceId 关联正确
assertEquals("4", row.getSourceId());
assertEquals(4, row.getRowIndex());
// values 是独立副本,修改不影响后续生成
row.getValues().put("价格", "999");
List<SimilarAsinParsedRowVo> again = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
assertTrue(!again.get(3).getValues().getOrDefault("价格", "").equals("999"));
}
@Test
void test_task_001_payload_chunk_image_boundary_chunk_size_edge() {
// chunk 边界:行数恰好整除 / 有余数 / 单 chunk 放不下
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/edge.xlsx", 100, false);
// 100 行 / 40 → 3 chunks40+40+20
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 40);
assertEquals(3, chunks.size());
assertEquals(40, chunks.get(0).size());
assertEquals(20, chunks.get(2).size());
// chunkSize 大于总行数 → 单 chunk
assertEquals(1, fixture.splitChunks(rows, 500).size());
// chunkSize 恰好等于行数 → 单 chunk 全量
assertEquals(1, fixture.splitChunks(rows, 100).size());
assertEquals(100, fixture.splitChunks(rows, 100).get(0).size());
}
}
@@ -0,0 +1,85 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import org.junit.jupiter.api.Test;
import org.springframework.core.task.TaskExecutor;
import java.time.LocalDateTime;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
/**
* owner-scoped job 的派发路由(2026-09-01 事故):两台实例共用一个 RocketMQ consumer group
* 消息被轮询投递到非归属实例后只能丢弃。归属实例改为直接本地派发,非归属实例不再发消息。
*/
class TaskFileJobDispatchCoordinatorOwnerRoutingTest {
private final TaskFileJobPublisher publisher = mock(TaskFileJobPublisher.class);
private final TaskFileJobLocalDispatcher localDispatcher = mock(TaskFileJobLocalDispatcher.class);
private final InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
private final TaskExecutor inlineExecutor = Runnable::run;
private TaskFileJobDispatchCoordinator buildCoordinator() {
return new TaskFileJobDispatchCoordinator(publisher, localDispatcher, instanceMetadata, inlineExecutor);
}
private static TaskFileJobDispatchEvent event(String owner) {
return new TaskFileJobDispatchEvent(18524L, 26402L, "SIMILAR_ASIN", 29405L,
owner == null ? "task:26402" : "task:26402:owner:" + owner,
TaskFileJobService.JOB_TYPE_ASSEMBLE_RESULT, owner, LocalDateTime.now());
}
@Test
void ownerJobDispatchedLocallyWithoutMq() {
when(instanceMetadata.getInstanceId()).thenReturn("server-121");
when(localDispatcher.dispatch(18524L, 26402L, "SIMILAR_ASIN", true)).thenReturn(true);
buildCoordinator().onTaskFileJobDispatch(event("server-121"));
verify(localDispatcher).dispatch(18524L, 26402L, "SIMILAR_ASIN", true);
verifyNoInteractions(publisher);
}
@Test
void ownerJobFallsBackToMqWhenLocalDispatchDeclined() {
when(instanceMetadata.getInstanceId()).thenReturn("server-121");
when(localDispatcher.dispatch(18524L, 26402L, "SIMILAR_ASIN", true)).thenReturn(false, true);
when(publisher.publish(any(TaskFileJobMessage.class))).thenReturn(false);
buildCoordinator().onTaskFileJobDispatch(event("server-121"));
verify(publisher).publish(any(TaskFileJobMessage.class));
}
@Test
void otherInstanceJobIsNeitherPublishedNorRun() {
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
buildCoordinator().onTaskFileJobDispatch(event("server-121"));
verifyNoInteractions(publisher);
verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), any(), anyBoolean());
verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), any());
}
@Test
void jobWithoutOwnerStillGoesThroughMq() {
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
when(publisher.publish(any(TaskFileJobMessage.class))).thenReturn(true);
buildCoordinator().onTaskFileJobDispatch(event(null));
verify(publisher).publish(any(TaskFileJobMessage.class));
verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), eq("SIMILAR_ASIN"), anyBoolean());
}
}
@@ -0,0 +1,163 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import java.time.LocalDateTime;
import java.util.List;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 双实例归属路由回归测试(2026-09-01 货源查询任务被非归属实例误杀):
* 1) 定时 worker 已 claim 的 job 不能再走 claimRunning,否则条件 UPDATE 失败被静默丢弃;
* 2) 处理中撞上归属校验时只退回 PENDING,不得计入 retryCountmarkFailed)。
*/
class TaskResultFileJobWorkerOwnerRoutingTest {
private final ShopMatchTaskService shopMatch = mock(ShopMatchTaskService.class);
private final PriceTrackTaskService priceTrack = mock(PriceTrackTaskService.class);
private final ProductRiskTaskService productRisk = mock(ProductRiskTaskService.class);
private final PublishTaskService publish = mock(PublishTaskService.class);
private final QueryAsinTaskService queryAsin = mock(QueryAsinTaskService.class);
private final ShopDataCrawlTaskService shopDataCrawl = mock(ShopDataCrawlTaskService.class);
private final WithdrawTaskService withdraw = mock(WithdrawTaskService.class);
private final PatrolDeleteTaskService patrolDelete = mock(PatrolDeleteTaskService.class);
private final AppearancePatentTaskService appearance = mock(AppearancePatentTaskService.class);
private final SimilarAsinTaskService similar = mock(SimilarAsinTaskService.class);
private final DeleteBrandRunService deleteBrand = mock(DeleteBrandRunService.class);
private final BrandTaskService brand = mock(BrandTaskService.class);
private final CollectDataService collectData = mock(CollectDataService.class);
private final TaskResultPayloadService payload = mock(TaskResultPayloadService.class);
private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
private final TaskDistributedLockService taskDistributedLockService = mock(TaskDistributedLockService.class);
private final TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
private final TaskFileJobLocalDispatcher localDispatcher = mock(TaskFileJobLocalDispatcher.class);
private final InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
private TaskResultFileJobWorker buildWorker() throws Exception {
List<ResultFileJobHandler> handlers = List.of(
new ShopMatchResultFileJobHandler(shopMatch, payload),
new PriceTrackResultFileJobHandler(priceTrack, payload),
new ProductRiskResultFileJobHandler(productRisk, payload),
new PublishResultFileJobHandler(publish),
new QueryAsinResultFileJobHandler(queryAsin, payload),
new ShopDataCrawlResultFileJobHandler(shopDataCrawl, payload),
new WithdrawResultFileJobHandler(withdraw, payload),
new PatrolDeleteResultFileJobHandler(patrolDelete, payload),
new AppearancePatentResultFileJobHandler(appearance),
new SimilarAsinResultFileJobHandler(similar),
new DeleteBrandResultFileJobHandler(deleteBrand),
new BrandResultFileJobHandler(brand, payload),
new CollectDataResultFileJobHandler(collectData));
TaskResultFileJobWorker worker = new TaskResultFileJobWorker(
taskFileJobService,
taskDistributedLockService,
mock(FileResultMapper.class),
localDispatcher,
instanceMetadata,
withdraw, brand,
new ResultFileJobHandlerRegistry(handlers));
set(worker, "taskQueueExecutor", (org.springframework.core.task.TaskExecutor) Runnable::run);
set(worker, "localWorkerEnabled", true);
set(worker, "batchSize", 20);
return worker;
}
private static void set(TaskResultFileJobWorker worker, String fieldName, Object value) throws Exception {
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
field.setAccessible(true);
field.set(worker, value);
}
private static TaskFileJobEntity claimedJob(String moduleType, long jobId, long taskId, String scopeKey) {
TaskFileJobEntity entity = new TaskFileJobEntity();
entity.setId(jobId);
entity.setTaskId(taskId);
entity.setResultId(taskId + 1000L);
entity.setModuleType(moduleType);
entity.setScopeKey(scopeKey);
entity.setStatus("RUNNING");
entity.setUpdatedAt(LocalDateTime.now());
return entity;
}
@Test
void scheduledWorkerProcessesAlreadyClaimedJobWithoutReclaiming() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
TaskFileJobEntity claim = claimedJob(PublishTaskService.MODULE_TYPE, 18524L, 26402L, "task:26402:owner:instance-a");
when(taskFileJobService.claimRunnableJobsForOwner(20, "instance-a")).thenReturn(List.of(claim));
when(taskFileJobService.activateRunningClaim(claim)).thenReturn(true);
when(localDispatcher.dispatchClaimed(claim.getId(), claim.getTaskId(), claim.getModuleType())).thenReturn(false);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, claim.getTaskId(),
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
worker.runPendingJobs();
verify(taskFileJobService, never()).claimRunning(anyLong());
verify(publish).processResultFileJob(claim);
verify(taskFileJobService).markSuccess(eq(claim), any());
}
@Test
void scheduledWorkerPrefersClaimedDispatch() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
TaskFileJobEntity claim = claimedJob(PublishTaskService.MODULE_TYPE, 18525L, 26403L, "task:26403:owner:instance-a");
when(taskFileJobService.claimRunnableJobsForOwner(20, "instance-a")).thenReturn(List.of(claim));
when(localDispatcher.dispatchClaimed(claim.getId(), claim.getTaskId(), claim.getModuleType())).thenReturn(true);
worker.runPendingJobs();
verify(localDispatcher).dispatchClaimed(claim.getId(), claim.getTaskId(), claim.getModuleType());
verify(localDispatcher, never()).dispatch(anyLong(), anyLong(), any());
verify(taskFileJobService, never()).claimRunning(anyLong());
verify(publish, never()).processResultFileJob(any());
}
@Test
void ownerMismatchDuringProcessingDefersInsteadOfFailing() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
// scopeKey 归属本机(通过 process 的前置校验),但模块层读到的 task owner 已经变成他机
TaskFileJobEntity job = claimedJob(PublishTaskService.MODULE_TYPE, 18526L, 26404L, "task:26404:owner:instance-a");
when(taskFileJobService.claimRunning(job.getId())).thenReturn(job);
when(taskFileJobService.activateRunningClaim(job)).thenReturn(true);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, job.getTaskId(),
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
doThrow(new TaskOwnerMismatchException(job.getTaskId(), "assemble result file", "instance-b", "instance-a"))
.when(publish).processResultFileJob(job);
worker.process(job);
verify(taskFileJobService).deferRunning(eq(job.getId()), any());
verify(taskFileJobService, never()).markFailed(any(), any());
verify(taskFileJobService, never()).markFailedPermanent(any(), any());
verify(taskFileJobService, never()).isRetryExhausted(anyLong());
}
}
@@ -128,6 +128,20 @@ class TaskResultFileJobWorkerOwnerScopedTest {
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
TaskFileJobEntity job = job("PUBLISH", 2L, 12L, "task:12:owner:instance-b");
job.setStatus("RUNNING");
worker.process(job);
verifyNoInteractions(taskFileJobService, taskDistributedLockService, publish);
}
@Test
void ownerScopedOtherOwnerPendingSkip() {
// PENDING + owner 不匹配也必须跳过:模块层 ensureTaskOwnedByCurrentInstance 会抛
// TaskOwnerMismatchException,本机抢跑只会耗尽 job 重试次数并误杀归属机上的任务。
// job 由归属实例的 runPendingJobs(每 15 秒扫自己 owner 的 PENDING/FAILED)接手。
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
TaskFileJobEntity job = job("PUBLISH", 2L, 12L, "task:12:owner:instance-b");
job.setStatus("PENDING");
worker.process(job);
verifyNoInteractions(taskFileJobService, taskDistributedLockService, publish);
}