task-116: 后台管理新增/导入表单弹窗化 + 分组管理独立菜单与UI优化 + 轻量进度端点

- 后台管理页(admin)所有面板的新增/导入表单改为弹窗操作,原有字段 ID 全部保留、提交逻辑不变;导入删除入口不再触发二次确认拦截
- 分组管理升级为独立菜单(V102 + schema initializer),移除 5 个面板内的管理分组按钮;分组列表改为蓝白主题、增加权限分组横幅
- Python 侧 group-manage 权限守卫(_ensure_backend_menu_access 补充 group-manage)
- 引入 V101(biz_task_file_job 复合索引)+ 新增 TaskProgressLight/TaskFileJob 轻量端点与进度聚合作
- 前端 progress-light / page-separated-loads / dispatch-guard 共享模块及单元测试
This commit is contained in:
2026-09-01 12:54:13 +08:00
parent 652ec1bbc9
commit b148dc57eb
139 changed files with 10540 additions and 8834 deletions
+5
View File
@@ -114,6 +114,11 @@ OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
# ===== 本地规划文档(不入库)===== # ===== 本地规划文档(不入库)=====
docs/plans/ docs/plans/
docs/*plan*.md docs/*plan*.md
backend-java/docs/plans/
backend-java/docs/*plan*.md
backend-java/docs/specs/
backend-java/docs/*audit*.md
*.broken/
# ===== 进度/规划工具(本地使用,不入库)===== # ===== 进度/规划工具(本地使用,不入库)=====
check_progress.py check_progress.py
@@ -31,6 +31,8 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -98,6 +100,14 @@ public class AppearancePatentController {
return ApiResponse.success(service.progressBatch(request.getTaskIds())); return ApiResponse.success(service.progressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(service.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/activate") @PostMapping("/tasks/{taskId}/activate")
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING,并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新,不需要单独心跳接口。") @Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING,并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新,不需要单独心跳接口。")
public ApiResponse<Void> activate( public ApiResponse<Void> activate(
@@ -24,6 +24,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSour
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentHistoryAssembler;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
@@ -31,6 +32,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParse
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskDetailVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskDetailVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskItemVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskItemVo;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentSheetBuilder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService; import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
@@ -88,6 +90,8 @@ import java.util.UUID;
import java.util.function.Supplier; import java.util.function.Supplier;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -95,6 +99,10 @@ import java.util.zip.ZipOutputStream;
public class AppearancePatentTaskService { public class AppearancePatentTaskService {
public static final String MODULE_TYPE = "APPEARANCE_PATENT"; public static final String MODULE_TYPE = "APPEARANCE_PATENT";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String STATUS_PENDING = "PENDING"; private static final String STATUS_PENDING = "PENDING";
private static final String STATUS_RUNNING = "RUNNING"; private static final String STATUS_RUNNING = "RUNNING";
private static final String STATUS_SUCCESS = "SUCCESS"; private static final String STATUS_SUCCESS = "SUCCESS";
@@ -107,18 +115,6 @@ public class AppearancePatentTaskService {
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5); private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
private static final long TASK_LOCK_WAIT_MILLIS = 10000L; private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
private static final long TASK_LOCK_RETRY_DELAY_MILLIS = 200L; private static final long TASK_LOCK_RETRY_DELAY_MILLIS = 200L;
private static final List<String> RESULT_HEADERS = List.of(
"id",
"asin",
"国家",
"卖家名称",
"品牌",
"价格",
"标题维度(商标)",
"外观维度(外观设计专利)",
"结论",
"状态"
);
private final LocalFileStorageService localFileStorageService; private final LocalFileStorageService localFileStorageService;
private final OssStorageService ossStorageService; private final OssStorageService ossStorageService;
@@ -138,6 +134,7 @@ public class AppearancePatentTaskService {
private final DistributedJobLockService distributedJobLockService; private final DistributedJobLockService distributedJobLockService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final InstanceMetadata instanceMetadata; private final InstanceMetadata instanceMetadata;
private final TaskProgressLightAssembler taskProgressLightAssembler;
public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) { public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) {
long startedAt = System.nanoTime(); long startedAt = System.nanoTime();
@@ -2199,44 +2196,8 @@ public class AppearancePatentTaskService {
List<AppearancePatentParsedRowVo> receivedRows, List<AppearancePatentParsedRowVo> receivedRows,
Map<String, AppearancePatentResultRowDto> resultMap) { Map<String, AppearancePatentResultRowDto> resultMap) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
Sheet sheet = workbook.createSheet("外观专利检测结果"); AppearancePatentSheetBuilder.buildResultSheet(workbook, receivedRows, resultMap,
CellStyle headerStyle = workbook.createCellStyle(); parsedRow -> findResultRow(parsedRow, resultMap));
Font font = workbook.createFont();
font.setBold(true);
headerStyle.setFont(font);
List<String> resultHeaders = new ArrayList<>(RESULT_HEADERS);
resultHeaders.add(6, "标题");
resultHeaders.add(7, "图片链接");
resultHeaders.add(8, "sku");
Row header = sheet.createRow(0);
for (int i = 0; i < resultHeaders.size(); i++) {
Cell cell = header.createCell(i);
cell.setCellValue(resultHeaders.get(i));
cell.setCellStyle(headerStyle);
}
int rowIndex = 1;
List<AppearancePatentParsedRowVo> rowsToWrite = receivedRows == null ? List.of() : receivedRows;
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite) {
AppearancePatentResultRowDto resultRow = findResultRow(parsedRow, resultMap);
Row row = sheet.createRow(rowIndex++);
int col = 0;
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
row.createCell(col++).setCellValue(resolveBrand(resultRow, parsedRow));
row.createCell(col++).setCellValue(resolvePrice(resultRow));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
}
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap);
workbook.write(fos); workbook.write(fos);
workbook.dispose(); workbook.dispose();
} catch (Exception ex) { } catch (Exception ex) {
@@ -2244,36 +2205,6 @@ public class AppearancePatentTaskService {
} }
} }
private void writeReasonSheet(SXSSFWorkbook workbook,
CellStyle headerStyle,
List<AppearancePatentParsedRowVo> rowsToWrite,
Map<String, AppearancePatentResultRowDto> resultMap) {
Sheet sheet = workbook.createSheet("原因");
Row header = sheet.createRow(0);
List<String> headers = List.of("ASIN", "外观原因", "专利原因", "标题原因");
for (int i = 0; i < headers.size(); i++) {
Cell cell = header.createCell(i);
cell.setCellValue(headers.get(i));
cell.setCellStyle(headerStyle);
}
Set<String> writtenAsins = new LinkedHashSet<>();
int rowIndex = 1;
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite == null ? List.<AppearancePatentParsedRowVo>of() : rowsToWrite) {
String asin = normalize(parsedRow.getAsin()).toUpperCase(Locale.ROOT);
if (asin.isBlank() || !writtenAsins.add(asin)) {
continue;
}
AppearancePatentResultRowDto resultRow = findResultRow(parsedRow, resultMap);
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(asin);
row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), ""));
row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), ""));
row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), ""));
rowIndex++;
}
}
private AppearancePatentResultRowDto findResultRowByAsin(String asin, Map<String, AppearancePatentResultRowDto> resultMap) { private AppearancePatentResultRowDto findResultRowByAsin(String asin, Map<String, AppearancePatentResultRowDto> resultMap) {
String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT); String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT);
if (normalizedAsin.isBlank() || resultMap == null || resultMap.isEmpty()) { if (normalizedAsin.isBlank() || resultMap == null || resultMap.isEmpty()) {
@@ -2496,7 +2427,7 @@ public class AppearancePatentTaskService {
private boolean hasUsableLlmField(String value) { private boolean hasUsableLlmField(String value) {
String normalized = normalize(value); String normalized = normalize(value);
return !normalized.isBlank() && !isTechnicalLlmFailure(normalized); return !normalized.isBlank() && !AppearancePatentSheetBuilder.isTechnicalLlmFailure(normalized);
} }
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) { private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
@@ -2641,119 +2572,33 @@ public class AppearancePatentTaskService {
return vo; return vo;
} }
private AppearancePatentHistoryAssembler historyAssembler() {
return new AppearancePatentHistoryAssembler(
taskProgressSnapshotService,
this::calculateDisplayProgressPercent,
this::extractSnapshotDisplayPercent);
}
private AppearancePatentHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) { private AppearancePatentHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo(); return historyAssembler().toHistoryItem(row, task, job);
vo.setResultId(row.getId());
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(null);
attachFileJobState(vo, row, job);
vo.setTaskStatus(task == null ? null : task.getStatus());
vo.setSuccess(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()
|| row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
vo.setCreatedAt(fmt(row.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
} }
private int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) { private int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
String taskStatus = task == null ? null : task.getStatus(); return historyAssembler().historyPriority(row, task, job);
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
return 0;
}
return 1;
} }
private LocalDateTime historyActivityTime(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) { private LocalDateTime historyActivityTime(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
LocalDateTime latest = latestTime( return historyAssembler().historyActivityTime(row, task, job);
task == null ? null : task.getUpdatedAt(),
job == null ? null : job.getUpdatedAt(),
task == null ? null : task.getFinishedAt(),
row == null ? null : row.getCreatedAt(),
task == null ? null : task.getCreatedAt());
return latest == null && row != null ? row.getCreatedAt() : latest;
} }
private boolean isHistoryFileBuilding(FileResultEntity row, String taskStatus, TaskFileJobEntity job) { private boolean isHistoryFileBuilding(FileResultEntity row, String taskStatus, TaskFileJobEntity job) {
if (!STATUS_SUCCESS.equals(taskStatus)) { return historyAssembler().isHistoryFileBuilding(row, taskStatus, job);
return false;
}
boolean fileReady = row != null && row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank();
if (fileReady) {
return false;
}
String fileStatus = job == null ? null : job.getStatus();
return !STATUS_SUCCESS.equals(fileStatus) && !STATUS_FAILED.equals(fileStatus);
}
private void attachFileJobState(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
attachFileProgress(vo, row, job);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(STATUS_FAILED.equals(job.getStatus()) ? firstNonBlank(job.getErrorMessage(), null) : null);
attachFileProgress(vo, row, job);
}
private void attachFileProgress(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
if (row == null || row.getTaskId() == null) {
return;
}
if (Boolean.TRUE.equals(vo.getFileReady())) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(row.getTaskId(), MODULE_TYPE);
if (snapshot == null) {
return;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
if (total <= 0) {
return;
}
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(snapshot.getMessage());
} }
private String fmt(LocalDateTime t) { private String fmt(LocalDateTime t) {
return t == null ? null : t.toString(); return t == null ? null : t.toString();
} }
private LocalDateTime latestTime(LocalDateTime... values) {
LocalDateTime latest = null;
if (values == null) {
return null;
}
for (LocalDateTime value : values) {
if (value == null) {
continue;
}
if (latest == null || value.isAfter(latest)) {
latest = value;
}
}
return latest;
}
private String firstNonBlank(String preferred, String fallback) { private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
} }
@@ -3004,43 +2849,6 @@ public class AppearancePatentTaskService {
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country); return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
} }
private String readValueByHeader(AppearancePatentParsedRowVo row, String... candidates) {
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
return "";
}
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT);
for (String candidate : candidates) {
String expected = normalize(candidate).toLowerCase(Locale.ROOT);
if (!expected.isBlank() && header.contains(expected)) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
}
return "";
}
static String resolveBrand(AppearancePatentResultRowDto resultRow, AppearancePatentParsedRowVo parsedRow) {
String pythonBrand = resultRow == null ? "" : resultRow.getBrand();
if (pythonBrand != null && !pythonBrand.isBlank()) {
return pythonBrand.trim();
}
if (parsedRow == null || parsedRow.getValues() == null) {
return "";
}
for (Map.Entry<String, String> entry : parsedRow.getValues().entrySet()) {
String header = entry.getKey() == null ? "" : entry.getKey().trim().toLowerCase(Locale.ROOT);
if (header.contains("品牌") || header.contains("brand")) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
return "";
}
static String resolvePrice(AppearancePatentResultRowDto resultRow) {
return resultRow == null || resultRow.getPrice() == null ? "" : resultRow.getPrice().trim();
}
static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) { static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
if (waitingForAssemble) { if (waitingForAssemble) {
return STATUS_RUNNING; return STATUS_RUNNING;
@@ -3048,57 +2856,6 @@ public class AppearancePatentTaskService {
return executionFailed ? STATUS_FAILED : STATUS_SUCCESS; return executionFailed ? STATUS_FAILED : STATUS_SUCCESS;
} }
private String userFacingLlmCellValue(AppearancePatentResultRowDto row, String value) {
String normalizedValue = normalize(value);
if (!normalizedValue.isBlank() && !isTechnicalLlmFailure(normalizedValue)) {
return value;
}
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
if (row != null && isTechnicalLlmFailure(row.getError())) {
return firstNonBlank(row.getError(), "");
}
return firstNonBlank(value, "");
}
private String userFacingConclusion(AppearancePatentResultRowDto row) {
if (row == null) {
return "";
}
String conclusion = normalize(row.getConclusion());
if (!conclusion.isBlank() && !isTechnicalLlmFailure(conclusion)) {
return row.getConclusion();
}
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
if (isTechnicalLlmFailure(row.getError())) {
return firstNonBlank(row.getError(), "");
}
return firstNonBlank(row.getConclusion(), "");
}
private String userFacingStatus(AppearancePatentResultRowDto row) {
if (row == null) {
return "";
}
String conclusion = userFacingConclusion(row);
return resolveResultStatus(conclusion);
}
static String resolveResultStatus(String conclusion) {
String normalized = conclusion == null ? "" : conclusion.trim();
return normalized.isBlank() ? "\u5931\u8d25" : "\u6210\u529f";
}
private boolean isTechnicalLlmFailure(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.contains("coze") || normalized.contains("llm")
|| normalized.contains("结果不完整")
|| normalized.contains("工作流节点执行超限")
|| normalized.contains("调用超时")
|| normalized.contains("timeout");
}
private String safeFileStem(String filename) { private String safeFileStem(String filename) {
String name = filename == null || filename.isBlank() ? "appearance-patent" : filename; String name = filename == null || filename.isBlank() ? "appearance-patent" : filename;
int idx = name.lastIndexOf('.'); int idx = name.lastIndexOf('.');
@@ -0,0 +1,225 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
/**
* 任务 93:AppearancePatentExcelParser 行解析器。
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 AppearancePatentTaskService.parseWorkbook
* 中对应段落一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、表头别名匹配(contains + 精确别名)、
* 空行跳过、必填表头缺失抛错。注意:appearancepatent 无 2000 字段截断、无错误值转空(与 similarasin 不同)。
*/
@Slf4j
public class AppearancePatentExcelParser {
public ParsedSheet parse(File input) {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (FileInputStream fis = new FileInputStream(input);
Workbook workbook = WorkbookFactory.create(fis)) {
return parseWorkbook(workbook);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[appearance-patent] parse failed file={} err={}", input.getName(), ex.getMessage());
throw new BusinessException("解析 Excel 失败");
}
}
public ParsedSheet parse(InputStream input) {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (Workbook workbook = WorkbookFactory.create(input)) {
return parseWorkbook(workbook);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[appearance-patent] parse failed input err={}", ex.getMessage());
throw new BusinessException("解析 Excel 失败");
}
}
private ParsedSheet parseWorkbook(Workbook workbook) {
DataFormatter formatter = new DataFormatter();
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int skuCol = findOptionalHeaderExact(headerMap,
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
List<AppearanceExcelRow> rows = new ArrayList<>();
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
String id = cell(row, idCol, formatter);
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
rows.add(new AppearanceExcelRow(
i + 1,
id,
asin,
country,
priceCol >= 0 ? cell(row, priceCol, formatter) : "",
skuCol >= 0 ? cell(row, skuCol, formatter) : "",
urlCol >= 0 ? cell(row, urlCol, formatter) : "",
titleCol >= 0 ? cell(row, titleCol, formatter) : "",
readRowValues(row, headers, formatter)));
}
return new ParsedSheet(headers, rows);
}
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
Map<String, Integer> map = new LinkedHashMap<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
return map;
}
private List<String> readHeaders(Row header, DataFormatter formatter) {
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(val.isBlank() ? "列" + (i + 1) : val);
}
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter));
}
return values;
}
private int findRequiredHeader(Map<String, Integer> map, String... names) {
int idx = findOptionalHeader(map, names);
if (idx < 0) {
throw new BusinessException("缺少必要表头: " + String.join("/", names));
}
return idx;
}
private int findOptionalHeader(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
for (String name : names) {
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
return entry.getValue();
}
}
}
return -1;
}
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
for (String name : names) {
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
return entry.getValue();
}
}
}
return -1;
}
private String normalizeHeaderAlias(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
}
private String cell(Row row, int col, DataFormatter formatter) {
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
}
private String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
public record ParsedSheet(List<String> headers, List<AppearanceExcelRow> rows) {
public ParsedSheet {
headers = headers == null ? List.of() : new ArrayList<>(headers);
rows = rows == null ? List.of() : new ArrayList<>(rows);
}
@Override
public List<String> headers() {
return java.util.Collections.unmodifiableList(headers);
}
@Override
public List<AppearanceExcelRow> rows() {
return java.util.Collections.unmodifiableList(rows);
}
}
public record AppearanceExcelRow(int rowIndex, String id, String asin, String country,
String price, String sku, String url, String title,
Map<String, String> values) {
public AppearanceExcelRow {
values = values == null ? Map.of() : new LinkedHashMap<>(values);
}
@Override
public boolean equals(Object o) {
if (!(o instanceof AppearanceExcelRow other)) {
return false;
}
return rowIndex == other.rowIndex
&& Objects.equals(id, other.id)
&& Objects.equals(asin, other.asin)
&& Objects.equals(country, other.country)
&& Objects.equals(price, other.price)
&& Objects.equals(sku, other.sku)
&& Objects.equals(url, other.url)
&& Objects.equals(title, other.title)
&& values.equals(other.values);
}
@Override
public int hashCode() {
return Objects.hash(rowIndex, id, asin, country, price, sku, url, title, values);
}
}
}
@@ -0,0 +1,161 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import java.time.LocalDateTime;
import java.util.function.Function;
/**
* 任务 97:AppearancePatentHistoryAssembler 历史查询组装器。
* 历史列表 VO 拼装(toHistoryItem + 排序辅助 historyPriority/historyActivityTime +
* 文件进度链)从 AppearancePatentTaskService 原样搬移;只读不落库;输出与现状逐字段一致。
* 进度百分比计算(calculateDisplayProgressPercent/extractSnapshotDisplayPercent)仍在
* 服务侧(与 saveFileBuildProgress 实时路径共享),以函数注入。
*/
public class AppearancePatentHistoryAssembler {
private static final String STATUS_PENDING = "PENDING";
private static final String STATUS_RUNNING = "RUNNING";
private static final String STATUS_SUCCESS = "SUCCESS";
private static final String STATUS_FAILED = "FAILED";
@FunctionalInterface
public interface ProgressPercentCalculator {
int calculate(int current, int total, TaskFileJobEntity job, LocalDateTime baseTime);
}
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final ProgressPercentCalculator calculateDisplayProgressPercent;
private final Function<TaskProgressSnapshotEntity, Integer> extractSnapshotDisplayPercent;
public AppearancePatentHistoryAssembler(TaskProgressSnapshotService taskProgressSnapshotService,
ProgressPercentCalculator calculateDisplayProgressPercent,
Function<TaskProgressSnapshotEntity, Integer> extractSnapshotDisplayPercent) {
this.taskProgressSnapshotService = taskProgressSnapshotService;
this.calculateDisplayProgressPercent = calculateDisplayProgressPercent;
this.extractSnapshotDisplayPercent = extractSnapshotDisplayPercent;
}
public AppearancePatentHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo();
vo.setResultId(row.getId());
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(null);
attachFileJobState(vo, row, job);
vo.setTaskStatus(task == null ? null : task.getStatus());
vo.setSuccess(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()
|| row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
vo.setCreatedAt(fmt(row.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
}
public int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
return 0;
}
return 1;
}
public LocalDateTime historyActivityTime(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
LocalDateTime latest = latestTime(
task == null ? null : task.getUpdatedAt(),
job == null ? null : job.getUpdatedAt(),
task == null ? null : task.getFinishedAt(),
row == null ? null : row.getCreatedAt(),
task == null ? null : task.getCreatedAt());
return latest == null && row != null ? row.getCreatedAt() : latest;
}
public boolean isHistoryFileBuilding(FileResultEntity row, String taskStatus, TaskFileJobEntity job) {
if (!STATUS_SUCCESS.equals(taskStatus)) {
return false;
}
boolean fileReady = row != null && row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank();
if (fileReady) {
return false;
}
String fileStatus = job == null ? null : job.getStatus();
return !STATUS_SUCCESS.equals(fileStatus) && !STATUS_FAILED.equals(fileStatus);
}
private void attachFileJobState(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
attachFileProgress(vo, row, job);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(STATUS_FAILED.equals(job.getStatus()) ? firstNonBlank(job.getErrorMessage(), null) : null);
attachFileProgress(vo, row, job);
}
private void attachFileProgress(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
if (row == null || row.getTaskId() == null) {
return;
}
if (Boolean.TRUE.equals(vo.getFileReady())) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(row.getTaskId(), AppearancePatentTaskService.MODULE_TYPE);
if (snapshot == null) {
return;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
if (total <= 0) {
return;
}
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
int percent = calculateDisplayProgressPercent.calculate(current, total, job, baseTime);
percent = Math.max(percent, extractSnapshotDisplayPercent.apply(snapshot));
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(snapshot.getMessage());
}
private String fmt(LocalDateTime t) {
return t == null ? null : t.toString();
}
private LocalDateTime latestTime(LocalDateTime... values) {
LocalDateTime latest = null;
if (values == null) {
return null;
}
for (LocalDateTime value : values) {
if (value == null) {
continue;
}
if (latest == null || value.isAfter(latest)) {
latest = value;
}
}
return latest;
}
private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
}
@@ -0,0 +1,31 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
/**
* 任务 93:AppearancePatentRowNormalizer 字段归一化器。
* 规则与 AppearancePatentTaskService.normalize / firstNonBlank / baseId / normalizeDisplayId
* 现状逐字节一致(BOM 剥离、全角空格转半角、trim、连续空白折叠为单空格;
* firstNonBlank 取首非空并 trim;baseId 取下划线前块基 id;displayId 仅 trim)。纯函数无状态。
*/
public final class AppearancePatentRowNormalizer {
private AppearancePatentRowNormalizer() {
}
public static String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
public static String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
public static String baseId(String id) {
String s = normalize(id);
int idx = s.indexOf('_');
return idx > 0 ? s.substring(0, idx) : s;
}
public static String normalizeDisplayId(String id) {
return id == null ? "" : id.trim();
}
}
@@ -0,0 +1,265 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Font;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
/**
* 任务 94:AppearancePatentSheetBuilder Sheet 构造器。
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式、行值派生)。
* 与现状 writeResultWorkbook / writeReasonSheet 语义一致:主 sheet 名"外观专利检测结果"、
* 13 列表头(RESULT_HEADERS 10 列 + 第 6/7/8 位插入标题/图片链接/sku)、加粗表头、
* 数据行从第 1 行;"原因"sheet 4 列按 ASIN 去重;LLM 技术性失败展示错误信息。
* 不落库、无 IO 依赖;rowKey/行值解析/LLM 失败判定均为静态纯函数。
*/
public final class AppearancePatentSheetBuilder {
public static final List<String> RESULT_HEADERS = List.of(
"id",
"asin",
"国家",
"卖家名称",
"品牌",
"价格",
"标题维度(商标)",
"外观维度(外观设计专利)",
"结论",
"状态"
);
private AppearancePatentSheetBuilder() {
}
public static void buildResultSheet(Workbook workbook, List<AppearancePatentParsedRowVo> rows,
Map<String, AppearancePatentResultRowDto> resultMap) {
buildResultSheet(workbook, rows, resultMap, null);
}
public static void buildResultSheet(Workbook workbook, List<AppearancePatentParsedRowVo> rows,
Map<String, AppearancePatentResultRowDto> resultMap,
java.util.function.Function<AppearancePatentParsedRowVo, AppearancePatentResultRowDto> findResultRowFn) {
java.util.function.Function<AppearancePatentParsedRowVo, AppearancePatentResultRowDto> finder =
findResultRowFn == null ? row -> findResultRow(row, resultMap) : findResultRowFn;
Sheet sheet = workbook.createSheet("外观专利检测结果");
CellStyle headerStyle = workbook.createCellStyle();
Font font = workbook.createFont();
font.setBold(true);
headerStyle.setFont(font);
List<String> resultHeaders = new ArrayList<>(RESULT_HEADERS);
resultHeaders.add(6, "标题");
resultHeaders.add(7, "图片链接");
resultHeaders.add(8, "sku");
Row header = sheet.createRow(0);
for (int i = 0; i < resultHeaders.size(); i++) {
Cell cell = header.createCell(i);
cell.setCellValue(resultHeaders.get(i));
cell.setCellStyle(headerStyle);
}
int rowIndex = 1;
List<AppearancePatentParsedRowVo> rowsToWrite = rows == null ? List.of() : rows;
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite) {
if (parsedRow == null) {
continue;
}
AppearancePatentResultRowDto resultRow = finder.apply(parsedRow);
Row row = sheet.createRow(rowIndex++);
int col = 0;
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
row.createCell(col++).setCellValue(resolveBrand(resultRow, parsedRow));
row.createCell(col++).setCellValue(resolvePrice(resultRow));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
}
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap, finder);
}
private static void writeReasonSheet(Workbook workbook, CellStyle headerStyle,
List<AppearancePatentParsedRowVo> rowsToWrite,
Map<String, AppearancePatentResultRowDto> resultMap,
java.util.function.Function<AppearancePatentParsedRowVo, AppearancePatentResultRowDto> finder) {
Sheet sheet = workbook.createSheet("原因");
Row header = sheet.createRow(0);
List<String> headers = List.of("ASIN", "外观原因", "专利原因", "标题原因");
for (int i = 0; i < headers.size(); i++) {
Cell cell = header.createCell(i);
cell.setCellValue(headers.get(i));
cell.setCellStyle(headerStyle);
}
Set<String> writtenAsins = new LinkedHashSet<>();
int rowIndex = 1;
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite == null ? List.<AppearancePatentParsedRowVo>of() : rowsToWrite) {
if (parsedRow == null) {
continue;
}
String asin = normalize(parsedRow.getAsin()).toUpperCase(Locale.ROOT);
if (asin.isBlank() || !writtenAsins.add(asin)) {
continue;
}
AppearancePatentResultRowDto resultRow = finder.apply(parsedRow);
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(asin);
row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), ""));
row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), ""));
row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), ""));
rowIndex++;
}
}
public static AppearancePatentResultRowDto findResultRow(AppearancePatentParsedRowVo parsedRow,
Map<String, AppearancePatentResultRowDto> resultMap) {
if (parsedRow == null || resultMap == null || resultMap.isEmpty()) {
return null;
}
AppearancePatentResultRowDto resultRow = resultMap.get(rowKey(parsedRow));
if (resultRow != null) {
return resultRow;
}
String legacyKey = legacyRowKey(parsedRow);
if (legacyKey.isBlank()) {
return null;
}
return resultMap.get(legacyKey);
}
public static String legacyRowKey(AppearancePatentParsedRowVo row) {
if (row == null) {
return "";
}
return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry());
}
public static String rowKey(AppearancePatentParsedRowVo row) {
if (row == null) {
return "";
}
return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry());
}
public static String rowKey(AppearancePatentResultRowDto row) {
if (row == null) {
return "";
}
return rowKey(row.getId(), row.getAsin(), row.getCountry());
}
public static String rowKey(String id, String asin, String country) {
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
public static String readValueByHeader(AppearancePatentParsedRowVo row, String... candidates) {
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
return "";
}
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT);
for (String candidate : candidates) {
String expected = normalize(candidate).toLowerCase(Locale.ROOT);
if (!expected.isBlank() && header.contains(expected)) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
}
return "";
}
public static String resolveBrand(AppearancePatentResultRowDto resultRow, AppearancePatentParsedRowVo parsedRow) {
String pythonBrand = resultRow == null ? "" : resultRow.getBrand();
if (pythonBrand != null && !pythonBrand.isBlank()) {
return pythonBrand.trim();
}
if (parsedRow == null || parsedRow.getValues() == null) {
return "";
}
for (Map.Entry<String, String> entry : parsedRow.getValues().entrySet()) {
String header = entry.getKey() == null ? "" : entry.getKey().trim().toLowerCase(Locale.ROOT);
if (header.contains("品牌") || header.contains("brand")) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
return "";
}
public static String resolvePrice(AppearancePatentResultRowDto resultRow) {
return resultRow == null || resultRow.getPrice() == null ? "" : resultRow.getPrice().trim();
}
public static String userFacingLlmCellValue(AppearancePatentResultRowDto row, String value) {
String normalizedValue = normalize(value);
if (!normalizedValue.isBlank() && !isTechnicalLlmFailure(normalizedValue)) {
return value;
}
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
if (row != null && isTechnicalLlmFailure(row.getError())) {
return firstNonBlank(row.getError(), "");
}
return firstNonBlank(value, "");
}
public static String userFacingConclusion(AppearancePatentResultRowDto row) {
if (row == null) {
return "";
}
String conclusion = normalize(row.getConclusion());
if (!conclusion.isBlank() && !isTechnicalLlmFailure(conclusion)) {
return row.getConclusion();
}
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
if (isTechnicalLlmFailure(row.getError())) {
return firstNonBlank(row.getError(), "");
}
return firstNonBlank(row.getConclusion(), "");
}
public static String userFacingStatus(AppearancePatentResultRowDto row) {
if (row == null) {
return "";
}
String conclusion = userFacingConclusion(row);
return resolveResultStatus(conclusion);
}
public static String resolveResultStatus(String conclusion) {
String normalized = conclusion == null ? "" : conclusion.trim();
return normalized.isBlank() ? "失败" : "成功";
}
public static boolean isTechnicalLlmFailure(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.contains("coze") || normalized.contains("llm")
|| normalized.contains("结果不完整")
|| normalized.contains("工作流节点执行超限")
|| normalized.contains("调用超时")
|| normalized.contains("timeout");
}
private static String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private static String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
}
@@ -27,6 +27,8 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -113,6 +115,14 @@ public class CollectDataController {
return ApiResponse.success(service.progressBatch(request.getTaskIds())); return ApiResponse.success(service.progressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(service.progressLight(request.getTaskIds()));
}
@DeleteMapping("/tasks/{taskId}") @DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除任务", description = "删除任务及其明细行、关联结果记录。") @Operation(summary = "删除任务", description = "删除任务及其明细行、关联结果记录。")
public ApiResponse<Void> deleteTask( public ApiResponse<Void> deleteTask(
@@ -87,6 +87,8 @@ import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -94,6 +96,10 @@ import java.util.regex.Pattern;
public class CollectDataService { public class CollectDataService {
public static final String MODULE_TYPE = "COLLECT_DATA"; public static final String MODULE_TYPE = "COLLECT_DATA";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
public static final int DEFAULT_PAGE_SIZE = 50; public static final int DEFAULT_PAGE_SIZE = 50;
/** /**
@@ -152,6 +158,7 @@ public class CollectDataService {
/** 结果明细 chunk 级读取器:生成结果文件时按 chunk 一次读取,替代逐行对象读取。 */ /** 结果明细 chunk 级读取器:生成结果文件时按 chunk 一次读取,替代逐行对象读取。 */
private final CollectDataResultDetailReader resultDetailReader; private final CollectDataResultDetailReader resultDetailReader;
private final TaskProgressLightAssembler taskProgressLightAssembler;
@Value("${aiimage.collect-data.stale-timeout-minutes:30}") @Value("${aiimage.collect-data.stale-timeout-minutes:30}")
private long staleTimeoutMinutes; private long staleTimeoutMinutes;
@@ -34,6 +34,8 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@@ -90,6 +92,14 @@ public class DeleteBrandRunController {
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds())); return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds()));
}
@GetMapping("/tasks/{taskId}/deletion-status") @GetMapping("/tasks/{taskId}/deletion-status")
@Operation(summary = "获取删除品牌任务删除状态", description = "供 Python/插件按 taskId 查询该删除品牌任务是否已被前端删空。") @Operation(summary = "获取删除品牌任务删除状态", description = "供 Python/插件按 taskId 查询该删除品牌任务是否已被前端删空。")
public ApiResponse<DeleteBrandTaskDeletionStatusVo> getTaskDeletionStatus( public ApiResponse<DeleteBrandTaskDeletionStatusVo> getTaskDeletionStatus(
@@ -68,6 +68,8 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -75,6 +77,10 @@ import java.util.Set;
public class DeleteBrandRunService { public class DeleteBrandRunService {
private static final String MODULE_TYPE = "DELETE_BRAND"; private static final String MODULE_TYPE = "DELETE_BRAND";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final Duration TASK_LOCK_TTL = TaskDistributedLockService.DEFAULT_LOCK_TTL; private static final Duration TASK_LOCK_TTL = TaskDistributedLockService.DEFAULT_LOCK_TTL;
private static final long TASK_LOCK_WAIT_MILLIS = TaskDistributedLockService.DEFAULT_WAIT_MILLIS; private static final long TASK_LOCK_WAIT_MILLIS = TaskDistributedLockService.DEFAULT_WAIT_MILLIS;
@@ -92,6 +98,7 @@ public class DeleteBrandRunService {
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService; private final TaskFileJobService taskFileJobService;
private final PlatformTransactionManager transactionManager; private final PlatformTransactionManager transactionManager;
private final TaskProgressLightAssembler taskProgressLightAssembler;
private TransactionTemplate newRequiresNewTemplate() { private TransactionTemplate newRequiresNewTemplate() {
TransactionTemplate template = new TransactionTemplate(transactionManager); TransactionTemplate template = new TransactionTemplate(transactionManager);
@@ -63,11 +63,13 @@ public class InvalidAsinDataController {
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page, @Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize, @Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword, @Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword,
@Parameter(description = "ASIN 模糊搜索") @RequestParam(required = false) String dataValue,
@Parameter(description = "品牌模糊搜索") @RequestParam(required = false) String brand,
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId, @Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
HttpServletRequest request) { HttpServletRequest request) {
RequestOperator operator = requireInvalidAsinDataAccess(request); RequestOperator operator = requireInvalidAsinDataAccess(request);
return ApiResponse.success(invalidAsinDataService.page( return ApiResponse.success(invalidAsinDataService.page(
page, pageSize, keyword, groupId, operator.id(), operator.superAdmin())); page, pageSize, keyword, dataValue, brand, groupId, operator.id(), operator.superAdmin()));
} }
@PostMapping @PostMapping
@@ -31,15 +31,19 @@ public class InvalidAsinDataService {
private final InvalidAsinDataMapper invalidAsinDataMapper; private final InvalidAsinDataMapper invalidAsinDataMapper;
private final ShopManageGroupService shopManageGroupService; private final ShopManageGroupService shopManageGroupService;
public InvalidAsinDataPageVo page(long page, long pageSize, String keyword, Long groupId, Long operatorId, boolean superAdmin) { public InvalidAsinDataPageVo page(long page, long pageSize, String keyword, String dataValue, String brand, Long groupId, Long operatorId, boolean superAdmin) {
long safePage = Math.max(page, 1); long safePage = Math.max(page, 1);
long safePageSize = Math.min(Math.max(pageSize, 1), 100); long safePageSize = Math.min(Math.max(pageSize, 1), 100);
String safeKeyword = keyword == null ? "" : keyword.trim(); String safeKeyword = keyword == null ? "" : keyword.trim();
String safeDataValue = dataValue == null ? "" : dataValue.trim();
String safeBrand = brand == null ? "" : brand.trim();
LambdaQueryWrapper<InvalidAsinDataEntity> query = new LambdaQueryWrapper<InvalidAsinDataEntity>() LambdaQueryWrapper<InvalidAsinDataEntity> query = new LambdaQueryWrapper<InvalidAsinDataEntity>()
.and(!safeKeyword.isEmpty(), wrapper -> wrapper .and(!safeKeyword.isEmpty(), wrapper -> wrapper
.like(InvalidAsinDataEntity::getDataValue, safeKeyword) .like(InvalidAsinDataEntity::getDataValue, safeKeyword)
.or() .or()
.like(InvalidAsinDataEntity::getBrand, safeKeyword)) .like(InvalidAsinDataEntity::getBrand, safeKeyword))
.like(!safeDataValue.isEmpty(), InvalidAsinDataEntity::getDataValue, safeDataValue)
.like(!safeBrand.isEmpty(), InvalidAsinDataEntity::getBrand, safeBrand)
.orderByDesc(InvalidAsinDataEntity::getId); .orderByDesc(InvalidAsinDataEntity::getId);
if (!superAdmin) { if (!superAdmin) {
Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId); Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId);
@@ -41,6 +41,8 @@ import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -167,6 +169,14 @@ public class PatrolDeleteController {
return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(patrolDeleteTaskService.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks") @PostMapping("/tasks")
@Operation(summary = "创建巡店删除任务", description = "根据已匹配的店铺创建任务和占位结果记录,后续由 Python 端处理并回传结果。") @Operation(summary = "创建巡店删除任务", description = "根据已匹配的店铺创建任务和占位结果记录,后续由 Python 端处理并回传结果。")
public ApiResponse<PatrolDeleteCreateTaskVo> createTask( public ApiResponse<PatrolDeleteCreateTaskVo> createTask(
@@ -43,6 +43,8 @@ import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -50,6 +52,10 @@ import java.util.Objects;
public class PatrolDeleteTaskService { public class PatrolDeleteTaskService {
private static final String MODULE_TYPE = "PATROL_DELETE"; private static final String MODULE_TYPE = "PATROL_DELETE";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1; private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0; private static final int RESULT_FAILED = 0;
@@ -70,6 +76,7 @@ public class PatrolDeleteTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -50,6 +50,7 @@ public class PermissionMenuSchemaInitializer {
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of( private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
new DefaultAdminMenu("用户管理", "admin_users", "users", 10), new DefaultAdminMenu("用户管理", "admin_users", "users", 10),
new DefaultAdminMenu("菜单权限配置", "admin_columns", "columns", 20), new DefaultAdminMenu("菜单权限配置", "admin_columns", "columns", 20),
new DefaultAdminMenu("分组管理", "admin_group_manage", "group-manage", 25),
new DefaultAdminMenu("去重数据汇总", "admin_dedupe_total_data", "dedupe-total-data", 30), new DefaultAdminMenu("去重数据汇总", "admin_dedupe_total_data", "dedupe-total-data", 30),
new DefaultAdminMenu("无效ASIN数据", "admin_invalid_asin_data", "invalid-asin-data", 35), new DefaultAdminMenu("无效ASIN数据", "admin_invalid_asin_data", "invalid-asin-data", 35),
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40), new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40),
@@ -49,6 +49,8 @@ import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -264,6 +266,14 @@ public class PriceTrackController {
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(priceTrackTaskService.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/result") @PostMapping("/tasks/{taskId}/result")
@Operation( @Operation(
summary = "Python 回传处理结果", summary = "Python 回传处理结果",
@@ -54,6 +54,8 @@ import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Locale; import java.util.Locale;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -61,6 +63,10 @@ import java.util.concurrent.atomic.AtomicReference;
public class PriceTrackTaskService { public class PriceTrackTaskService {
private static final String MODULE_TYPE = "PRICE_TRACK"; private static final String MODULE_TYPE = "PRICE_TRACK";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows"; private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
private static final String NO_USABLE_ROWS_ERROR = "未收到有效跟价数据,未生成结果文件"; private static final String NO_USABLE_ROWS_ERROR = "未收到有效跟价数据,未生成结果文件";
@@ -79,6 +85,7 @@ public class PriceTrackTaskService {
private final TaskResultPayloadService taskResultPayloadService; private final TaskResultPayloadService taskResultPayloadService;
private final TaskFileJobService taskFileJobService; private final TaskFileJobService taskFileJobService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -44,6 +44,8 @@ import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -165,6 +167,14 @@ public class ProductRiskResolveController {
return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(productRiskTaskService.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/result") @PostMapping("/tasks/{taskId}/result")
@Operation( @Operation(
summary = "Python 回传处理结果", summary = "Python 回传处理结果",
@@ -47,6 +47,8 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -54,6 +56,10 @@ import java.util.Objects;
public class ProductRiskTaskService { public class ProductRiskTaskService {
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE"; private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_ZIP = "application/zip"; private static final String CONTENT_TYPE_ZIP = "application/zip";
private final FileTaskMapper fileTaskMapper; private final FileTaskMapper fileTaskMapper;
@@ -70,6 +76,7 @@ public class ProductRiskTaskService {
private final TaskResultItemService taskResultItemService; private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -24,6 +24,8 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -104,6 +106,15 @@ public class PublishController {
return ApiResponse.success(publishTaskService.getTaskProgress(request.getUserId(), request.getTaskIds())); return ApiResponse.success(publishTaskService.getTaskProgress(request.getUserId(), request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。user_id 可省略,传入时仅返回该用户的任务。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(
@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(publishTaskService.progressLight(request.getUserId(), request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/result") @PostMapping("/tasks/{taskId}/result")
@Operation( @Operation(
summary = "Python 按文件分片回传上架结果", summary = "Python 按文件分片回传上架结果",
@@ -4,6 +4,7 @@ import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.IdUtil;
import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
@@ -70,6 +71,8 @@ import java.util.Set;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Slf4j @Slf4j
@Service @Service
@@ -77,6 +80,10 @@ import java.util.stream.Collectors;
public class PublishTaskService { public class PublishTaskService {
public static final String MODULE_TYPE = "PUBLISH"; public static final String MODULE_TYPE = "PUBLISH";
public TaskProgressLightBatchVo progressLight(Long userId, List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
public static final int DEFAULT_PAGE_SIZE = 50; public static final int DEFAULT_PAGE_SIZE = 50;
private static final int MAX_PAGE_SIZE = 200; private static final int MAX_PAGE_SIZE = 200;
private static final int INSERT_BATCH_SIZE = 500; private static final int INSERT_BATCH_SIZE = 500;
@@ -102,6 +109,7 @@ public class PublishTaskService {
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate; private final TransactionTemplate transactionTemplate;
private final InstanceMetadata instanceMetadata; private final InstanceMetadata instanceMetadata;
private final TaskProgressLightAssembler taskProgressLightAssembler;
@Value("${aiimage.publish.stale-timeout-minutes:30}") @Value("${aiimage.publish.stale-timeout-minutes:30}")
private int staleTimeoutMinutes; private int staleTimeoutMinutes;
@@ -340,10 +348,30 @@ public class PublishTaskService {
public PublishDashboardVo dashboard(Long userId) { public PublishDashboardVo dashboard(Long userId) {
validateUserId(userId); validateUserId(userId);
PublishDashboardVo response = new PublishDashboardVo(); PublishDashboardVo response = new PublishDashboardVo();
response.setPendingCount(countTasks(userId, STATUS_PENDING)); response.setPendingCount(0L);
response.setRunningCount(countTasks(userId, STATUS_RUNNING)); response.setRunningCount(0L);
response.setSuccessCount(countTasks(userId, STATUS_SUCCESS)); response.setSuccessCount(0L);
response.setFailedCount(countTasks(userId, STATUS_FAILED)); response.setFailedCount(0L);
for (Map<String, Object> row : fileTaskMapper.selectMaps(new QueryWrapper<FileTaskEntity>()
.select("status", "count(*) AS cnt")
.eq("module_type", MODULE_TYPE)
.eq("user_id", userId)
.groupBy("status"))) {
Object status = row.get("status");
Object count = row.get("cnt");
if (status == null || count == null) {
continue;
}
long n = ((Number) count).longValue();
switch (status.toString()) {
case STATUS_PENDING -> response.setPendingCount(n);
case STATUS_RUNNING -> response.setRunningCount(n);
case STATUS_SUCCESS -> response.setSuccessCount(n);
case STATUS_FAILED -> response.setFailedCount(n);
default -> {
}
}
}
response.setRecent(history(userId, 10).getItems()); response.setRecent(history(userId, 10).getItems());
return response; return response;
} }
@@ -1741,13 +1769,6 @@ public class PublishTaskService {
return STATUS_SUCCESS.equals(status) || STATUS_FAILED.equals(status); return STATUS_SUCCESS.equals(status) || STATUS_FAILED.equals(status);
} }
private long countTasks(Long userId, String status) {
return Objects.requireNonNullElse(fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.eq(FileTaskEntity::getStatus, status)), 0L);
}
private List<Long> normalizeTaskIds(List<Long> taskIds) { private List<Long> normalizeTaskIds(List<Long> taskIds) {
if (taskIds == null) { if (taskIds == null) {
return List.of(); return List.of();
@@ -39,6 +39,8 @@ import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -137,6 +139,14 @@ public class QueryAsinTaskController {
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(queryAsinTaskService.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks") @PostMapping("/tasks")
@Operation(summary = "创建查询 ASIN 任务", description = "根据已匹配的店铺创建任务和占位结果记录,并把后台维护的整张 ASIN 表数据随店铺项返回给 Python 端。") @Operation(summary = "创建查询 ASIN 任务", description = "根据已匹配的店铺创建任务和占位结果记录,并把后台维护的整张 ASIN 表数据随店铺项返回给 Python 端。")
public ApiResponse<QueryAsinCreateTaskVo> createTask( public ApiResponse<QueryAsinCreateTaskVo> createTask(
@@ -42,6 +42,8 @@ import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -49,6 +51,10 @@ import java.util.Objects;
public class QueryAsinTaskService { public class QueryAsinTaskService {
private static final String MODULE_TYPE = "QUERY_ASIN"; private static final String MODULE_TYPE = "QUERY_ASIN";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1; private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0; private static final int RESULT_FAILED = 0;
@@ -68,6 +74,7 @@ public class QueryAsinTaskService {
private final TaskResultItemService taskResultItemService; private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -33,6 +33,8 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.util.List; import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -151,6 +153,14 @@ public class ShopDataCrawlTaskController {
return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(taskService.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks") @PostMapping("/tasks")
@Operation( @Operation(
summary = "创建店铺数据抓取任务", summary = "创建店铺数据抓取任务",
@@ -1,14 +1,13 @@
package com.nanri.aiimage.modules.shopdatacrawl.service; package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache; import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache;
import com.nanri.aiimage.modules.shopdatacrawl.util.ShopDataCrawlPrefetchBudget; import com.nanri.aiimage.modules.shopdatacrawl.util.ShopDataCrawlPrefetchBudget;
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlSheetBuilder;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.ClientAnchor; import org.apache.poi.ss.usermodel.ClientAnchor;
import org.apache.poi.ss.usermodel.Drawing; import org.apache.poi.ss.usermodel.Drawing;
@@ -27,7 +26,6 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -35,16 +33,17 @@ import java.util.Map;
@Service @Service
@Slf4j @Slf4j
public class ShopDataCrawlExcelAssemblyService { public class ShopDataCrawlExcelAssemblyService {
static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT"); /** 常量已抽取到 ShopDataCrawlSheetBuilder,此处保留别名供既有调用方/测试引用。 */
static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利"); static final List<String> COUNTRIES = ShopDataCrawlSheetBuilder.COUNTRIES;
static final List<String> LEGACY_HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价"); static final List<String> SHEETS = ShopDataCrawlSheetBuilder.SHEETS;
static final List<String> HEADERS_WITHOUT_BRAND = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价"); static final List<String> LEGACY_HEADERS = ShopDataCrawlSheetBuilder.LEGACY_HEADERS;
static final List<String> HEADERS = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价", "品牌"); static final List<String> HEADERS_WITHOUT_BRAND = ShopDataCrawlSheetBuilder.HEADERS_WITHOUT_BRAND;
static final List<String> HEADERS = ShopDataCrawlSheetBuilder.HEADERS;
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx"; private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
private static final int IMAGE_COLUMN = 2; private static final int IMAGE_COLUMN = ShopDataCrawlSheetBuilder.IMAGE_COLUMN;
private static final int BRAND_COLUMN = HEADERS.size() - 1; private static final int BRAND_COLUMN = ShopDataCrawlSheetBuilder.BRAND_COLUMN;
private static final int IMAGE_COLUMN_WIDTH = 18 * 256; private static final int IMAGE_COLUMN_WIDTH = ShopDataCrawlSheetBuilder.IMAGE_COLUMN_WIDTH;
private static final float IMAGE_ROW_HEIGHT_POINTS = 80f; private static final float IMAGE_ROW_HEIGHT_POINTS = ShopDataCrawlSheetBuilder.IMAGE_ROW_HEIGHT_POINTS;
/** 图片缓存默认上限:64MB 字节预算 / 2000 条目,超过按 FIFO 淘汰,保证组装期内存有界。 */ /** 图片缓存默认上限:64MB 字节预算 / 2000 条目,超过按 FIFO 淘汰,保证组装期内存有界。 */
private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024; private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024;
private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000; private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000;
@@ -189,12 +188,7 @@ public class ShopDataCrawlExcelAssemblyService {
List<ShopDataCrawlRowDto> rows, List<ShopDataCrawlRowDto> rows,
BoundedImageCache imageCache, BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) { Map<String, Integer> pictureIndexes) {
Sheet sheet = workbook.createSheet(SHEETS.get(index)); Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(workbook, index);
Row header = sheet.createRow(0);
for (int column = 0; column < HEADERS.size(); column++) {
header.createCell(column).setCellValue(HEADERS.get(column));
}
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
int rowIndex = 1; int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) { for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
Row row = sheet.createRow(rowIndex++); Row row = sheet.createRow(rowIndex++);
@@ -203,27 +197,7 @@ public class ShopDataCrawlExcelAssemblyService {
} }
void validateTemplate(XSSFWorkbook workbook) { void validateTemplate(XSSFWorkbook workbook) {
if (workbook.getNumberOfSheets() != SHEETS.size()) { ShopDataCrawlSheetBuilder.validateTemplate(workbook);
throw new BusinessException("店铺数据抓取模板工作表数量不正确");
}
for (int i = 0; i < SHEETS.size(); i++) {
Sheet sheet = workbook.getSheetAt(i);
if (!SHEETS.get(i).equals(sheet.getSheetName())) {
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
}
Row header = sheet.getRow(0);
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
List<String> expectedHeaders = currentTemplate
? (templateHasBrand ? HEADERS : HEADERS_WITHOUT_BRAND)
: LEGACY_HEADERS;
for (int column = 0; column < expectedHeaders.size(); column++) {
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
if (!expectedHeaders.get(column).equals(actual)) {
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
}
}
}
} }
private void writeSheet(XSSFWorkbook workbook, private void writeSheet(XSSFWorkbook workbook,
@@ -232,32 +206,16 @@ public class ShopDataCrawlExcelAssemblyService {
BoundedImageCache imageCache, BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) { Map<String, Integer> pictureIndexes) {
Row header = sheet.getRow(0); Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1); boolean currentTemplate = header != null && "商品图片".equals(ShopDataCrawlSheetBuilder.cellText(header, IMAGE_COLUMN));
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN)); boolean templateHasBrand = "品牌".equals(ShopDataCrawlSheetBuilder.cellText(header, BRAND_COLUMN));
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN)); CellStyle[] styles = ShopDataCrawlSheetBuilder.templateStyles(sheet, currentTemplate, templateHasBrand);
CellStyle[] styles = new CellStyle[HEADERS.size()]; int[] columnWidths = ShopDataCrawlSheetBuilder.templateColumnWidths(sheet, currentTemplate, templateHasBrand);
for (int column = 0; column < styles.length; column++) { ShopDataCrawlSheetBuilder.writeHeaders(sheet, currentTemplate, templateHasBrand);
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
styles[column] = cell == null ? null : cell.getCellStyle();
}
int[] columnWidths = new int[HEADERS.size()];
for (int column = 0; column < columnWidths.length; column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
columnWidths[column] = sheet.getColumnWidth(sourceColumn);
}
writeHeaders(sheet, currentTemplate, templateHasBrand);
for (int column = 0; column < columnWidths.length; column++) { for (int column = 0; column < columnWidths.length; column++) {
sheet.setColumnWidth(column, columnWidths[column]); sheet.setColumnWidth(column, columnWidths[column]);
} }
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH); sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
int last = sheet.getLastRowNum(); ShopDataCrawlSheetBuilder.clearDataRows(sheet);
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
Row row = sheet.getRow(rowIndex);
if (row != null) {
sheet.removeRow(row);
}
}
clearSheetPictures(sheet, pictureIndexes); clearSheetPictures(sheet, pictureIndexes);
int rowIndex = 1; int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) { for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
@@ -273,43 +231,13 @@ public class ShopDataCrawlExcelAssemblyService {
CellStyle[] styles, CellStyle[] styles,
BoundedImageCache imageCache, BoundedImageCache imageCache,
Map<String, Integer> pictureIndexes) { Map<String, Integer> pictureIndexes) {
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(), ShopDataCrawlSheetBuilder.writeDataRowValues(row, value, styles);
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()}; if (!ShopDataCrawlSheetBuilder.blank(value.getCommodityImage())) {
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
if (!blank(value.getCommodityImage())) {
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS); row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes); embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
} }
} }
private void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
Row header = sheet.getRow(0);
if (header == null) header = sheet.createRow(0);
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < HEADERS.size(); column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell source = header.getCell(sourceColumn);
styles[column] = source == null ? null : source.getCellStyle();
}
for (int column = 0; column < HEADERS.size(); column++) {
Cell cell = header.getCell(column);
if (cell == null) cell = header.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(HEADERS.get(column));
}
}
private int templateColumnForOutput(int outputColumn, boolean currentTemplate, boolean templateHasBrand) {
if (outputColumn == BRAND_COLUMN) {
return templateHasBrand ? BRAND_COLUMN : 1;
}
return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1;
}
private void embedImage(Workbook workbook, private void embedImage(Workbook workbook,
Sheet sheet, Sheet sheet,
Row row, Row row,
@@ -357,26 +285,15 @@ public class ShopDataCrawlExcelAssemblyService {
} }
private String cellText(Row row, int column) { private String cellText(Row row, int column) {
Cell cell = row == null ? null : row.getCell(column); return ShopDataCrawlSheetBuilder.cellText(row, column);
return cell == null ? "" : cell.getStringCellValue().trim();
} }
private boolean blank(String value) { private boolean blank(String value) {
return value == null || value.isBlank(); return ShopDataCrawlSheetBuilder.blank(value);
} }
private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) { private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>(); return ShopDataCrawlSheetBuilder.rowsByCountry(items);
COUNTRIES.forEach(country -> result.put(country, new ArrayList<>()));
if (items == null) return result;
for (ShopDataCrawlResultItemVo item : items) {
if (item == null || Boolean.FALSE.equals(item.getSuccess()) || item.getCountryResults() == null) continue;
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) {
String country = countryResult == null || countryResult.getCountry() == null ? "" : countryResult.getCountry().trim().toUpperCase();
if (result.containsKey(country) && countryResult.getItems() != null) result.get(country).addAll(countryResult.getItems());
}
}
return result;
} }
private void clearSheetPictures(Sheet sheet, Map<String, Integer> pictureIndexes) { private void clearSheetPictures(Sheet sheet, Map<String, Integer> pictureIndexes) {
@@ -393,10 +310,6 @@ public class ShopDataCrawlExcelAssemblyService {
} }
private int totalDataRows(XSSFWorkbook workbook) { private int totalDataRows(XSSFWorkbook workbook) {
int total = 0; return ShopDataCrawlSheetBuilder.totalDataRows(workbook);
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
total += Math.max(0, workbook.getSheetAt(i).getLastRowNum());
}
return total;
} }
} }
@@ -16,6 +16,8 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlHistoryAssembler;
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlRowNormalizer;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity; import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity; import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
@@ -71,6 +73,8 @@ import java.util.Set;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.UUID; import java.util.UUID;
import java.util.function.Supplier; import java.util.function.Supplier;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -78,6 +82,10 @@ import java.util.function.Supplier;
public class ShopDataCrawlTaskService { public class ShopDataCrawlTaskService {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL"; private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1; private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0; private static final int RESULT_FAILED = 0;
@@ -91,8 +99,6 @@ public class ShopDataCrawlTaskService {
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断"; private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断,已保留已回传的部分数据"; private static final String PARTIAL_RESULT_MESSAGE = "Python 中断,已保留已回传的部分数据";
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:"; private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
/** 国家结果行去重键字段分隔符(控制字符,字段值 trim 后不可能包含)。 */
private static final String ROW_KEY_SEPARATOR = "";
private final FileTaskMapper fileTaskMapper; private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper; private final FileResultMapper fileResultMapper;
@@ -113,6 +119,7 @@ public class ShopDataCrawlTaskService {
private final InstanceMetadata instanceMetadata; private final InstanceMetadata instanceMetadata;
private final ShopDataCrawlDailyFileService dailyFileService; private final ShopDataCrawlDailyFileService dailyFileService;
private final PlatformTransactionManager transactionManager; private final PlatformTransactionManager transactionManager;
private final TaskProgressLightAssembler taskProgressLightAssembler;
@Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}") @Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}")
private long staleTimeoutMinutes; private long staleTimeoutMinutes;
@@ -999,11 +1006,7 @@ public class ShopDataCrawlTaskService {
} }
private String trimToNull(String value) { private String trimToNull(String value) {
if (value == null) { return ShopDataCrawlRowNormalizer.trimToNull(value);
return null;
}
String normalized = value.trim();
return normalized.isEmpty() ? null : normalized;
} }
private long countTasks(Long userId, List<String> statuses) { private long countTasks(Long userId, List<String> statuses) {
@@ -1108,26 +1111,11 @@ public class ShopDataCrawlTaskService {
} }
private ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, ShopDataCrawlResultItemVo snapshot, TaskFileJobEntity job) { private ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, ShopDataCrawlResultItemVo snapshot, TaskFileJobEntity job) {
ShopDataCrawlResultItemVo item = snapshot != null ? snapshot : new ShopDataCrawlResultItemVo(); return historyAssembler().toHistoryItem(entity, task, snapshot, job);
item.setResultId(entity.getId()); }
item.setTaskId(entity.getTaskId());
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename())); private ShopDataCrawlHistoryAssembler historyAssembler() {
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl())); return new ShopDataCrawlHistoryAssembler();
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
item.setCreatedAt(entity.getCreatedAt());
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(null);
attachFileJobState(item, entity, job);
if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>());
}
if (item.getCountryCodes() == null) {
item.setCountryCodes(new ArrayList<>());
}
return item;
} }
private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity) { private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity) {
@@ -1135,14 +1123,7 @@ public class ShopDataCrawlTaskService {
} }
private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) { private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
item.setFileReady(!blank(entity.getResultFileUrl())); historyAssembler().attachFileJobState(item, entity, job);
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
} }
private List<ShopDataCrawlTaskItemDto> dedupeItems(List<ShopDataCrawlTaskItemDto> items) { private List<ShopDataCrawlTaskItemDto> dedupeItems(List<ShopDataCrawlTaskItemDto> items) {
@@ -2817,18 +2798,7 @@ public class ShopDataCrawlTaskService {
} }
private ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) { private ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); return ShopDataCrawlRowNormalizer.copyRow(source);
row.setDate(trim(source.getDate()));
row.setAsin(trim(source.getAsin()));
row.setBrand(trim(source.getBrand()));
row.setCommodityImage(trim(source.getCommodityImage()));
row.setInventorySales(trim(source.getInventorySales()));
row.setSalesRank(trim(source.getSalesRank()));
row.setPageViews(trim(source.getPageViews()));
row.setUnitsSold(trim(source.getUnitsSold()));
row.setPrice(trim(source.getPrice()));
row.setRecommendedOffer(trim(source.getRecommendedOffer()));
return row;
} }
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) { public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
@@ -2856,39 +2826,20 @@ public class ShopDataCrawlTaskService {
} }
private boolean sameRow(ShopDataCrawlRowDto left, ShopDataCrawlRowDto right) { private boolean sameRow(ShopDataCrawlRowDto left, ShopDataCrawlRowDto right) {
return left != null && right != null return ShopDataCrawlRowNormalizer.sameRow(left, right);
&& Objects.equals(trim(left.getDate()), trim(right.getDate()))
&& Objects.equals(trim(left.getAsin()), trim(right.getAsin()))
&& Objects.equals(trim(left.getBrand()), trim(right.getBrand()))
&& Objects.equals(trim(left.getCommodityImage()), trim(right.getCommodityImage()))
&& Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales()))
&& Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank()))
&& Objects.equals(trim(left.getPageViews()), trim(right.getPageViews()))
&& Objects.equals(trim(left.getUnitsSold()), trim(right.getUnitsSold()))
&& Objects.equals(trim(left.getPrice()), trim(right.getPrice()))
&& Objects.equals(trim(left.getRecommendedOffer()), trim(right.getRecommendedOffer()));
} }
/** 国家结果行稳定去重键:与 sameRow 的 10 字段 trim 比较语义等价,用于 O(1) 去重。 */ /** 国家结果行稳定去重键:与 sameRow 的 10 字段 trim 比较语义等价,用于 O(1) 去重。 */
static String rowDedupKey(ShopDataCrawlRowDto row) { static String rowDedupKey(ShopDataCrawlRowDto row) {
if (row == null) { return ShopDataCrawlRowNormalizer.rowDedupKey(row);
return null;
}
return trim(row.getDate()) + ROW_KEY_SEPARATOR + trim(row.getAsin()) + ROW_KEY_SEPARATOR + trim(row.getBrand())
+ ROW_KEY_SEPARATOR + trim(row.getCommodityImage()) + ROW_KEY_SEPARATOR + trim(row.getInventorySales())
+ ROW_KEY_SEPARATOR + trim(row.getSalesRank()) + ROW_KEY_SEPARATOR + trim(row.getPageViews())
+ ROW_KEY_SEPARATOR + trim(row.getUnitsSold()) + ROW_KEY_SEPARATOR + trim(row.getPrice())
+ ROW_KEY_SEPARATOR + trim(row.getRecommendedOffer());
} }
private boolean rowEmpty(ShopDataCrawlRowDto row) { private boolean rowEmpty(ShopDataCrawlRowDto row) {
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales()) return ShopDataCrawlRowNormalizer.rowEmpty(row);
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
} }
private static String trim(String value) { private static String trim(String value) {
return value == null ? "" : value.trim(); return ShopDataCrawlRowNormalizer.trim(value);
} }
void deleteResultObjectIfUnreferenced(String resultFileUrl) { void deleteResultObjectIfUnreferenced(String resultFileUrl) {
@@ -2964,18 +2915,7 @@ public class ShopDataCrawlTaskService {
} }
private String normalizeCountry(String country) { private String normalizeCountry(String country) {
if (country == null) { return ShopDataCrawlRowNormalizer.normalizeCountry(country);
return "";
}
String value = country.trim().toUpperCase();
return switch (value) {
case "德国" -> "DE";
case "英国" -> "UK";
case "法国" -> "FR";
case "意大利" -> "IT";
case "西班牙" -> "ES";
default -> value;
};
} }
private String buildTaskWorkbookFilename(FileTaskEntity task) { private String buildTaskWorkbookFilename(FileTaskEntity task) {
@@ -3000,7 +2940,7 @@ public class ShopDataCrawlTaskService {
} }
private String blankToNull(String value) { private String blankToNull(String value) {
return blank(value) ? null : value.trim(); return ShopDataCrawlRowNormalizer.blankToNull(value);
} }
private void validateUserId(Long userId) { private void validateUserId(Long userId) {
@@ -0,0 +1,70 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import java.util.ArrayList;
/**
* 任务 103:ShopDataCrawlHistoryAssembler 历史查询组装器。
* 历史列表 VO 拼装(toHistoryItem + 文件任务状态链 attachFileJobState)从
* ShopDataCrawlTaskService 原样搬移;只读不落库;输出与现状逐字段一致
* (快照优先、实体字段兜底、task/job 附加,downloadUrl 恒为空)。
*/
public class ShopDataCrawlHistoryAssembler {
private static final int RESULT_PENDING = -1;
private static final int RESULT_SUCCESS = 1;
public ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task,
ShopDataCrawlResultItemVo snapshot, TaskFileJobEntity job) {
ShopDataCrawlResultItemVo item = snapshot != null ? snapshot : new ShopDataCrawlResultItemVo();
item.setResultId(entity.getId());
item.setTaskId(entity.getTaskId());
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
item.setCreatedAt(entity.getCreatedAt());
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(null);
attachFileJobState(item, entity, job);
if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>());
}
if (item.getCountryCodes() == null) {
item.setCountryCodes(new ArrayList<>());
}
return item;
}
public void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
}
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
return fallback;
}
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
}
private String firstNonBlank(String first, String second) {
return !blank(first) ? first : second;
}
private boolean blank(String value) {
return value == null || value.isBlank();
}
}
@@ -0,0 +1,101 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
/**
* 任务 99:ShopDataCrawlRowNormalizer 行解析/归一化器。
* 规则与 ShopDataCrawlTaskService.trim / blank / blankToNull / trimToNull /
* normalizeCountry / copyRow / rowEmpty / sameRow / rowDedupKey 现状逐字节一致。
* rowDedupKey 与国家结果行稳定去重键语义(10 字段 trim 拼接)等价于 sameRow 的
* 逐字段 trim 比较。纯函数无状态。
*/
public final class ShopDataCrawlRowNormalizer {
/** 国家结果行去重键字段分隔符(控制字符,字段值 trim 后不可能包含)。 */
public static final String ROW_KEY_SEPARATOR = "";
private ShopDataCrawlRowNormalizer() {
}
public static String trim(String value) {
return value == null ? "" : value.trim();
}
public static boolean blank(String value) {
return value == null || value.isBlank();
}
public static String blankToNull(String value) {
return blank(value) ? null : value.trim();
}
public static String trimToNull(String value) {
if (value == null) {
return null;
}
String normalized = value.trim();
return normalized.isEmpty() ? null : normalized;
}
public static String normalizeCountry(String country) {
if (country == null) {
return "";
}
String value = country.trim().toUpperCase();
return switch (value) {
case "德国" -> "DE";
case "英国" -> "UK";
case "法国" -> "FR";
case "意大利" -> "IT";
case "西班牙" -> "ES";
default -> value;
};
}
public static ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(trim(source.getDate()));
row.setAsin(trim(source.getAsin()));
row.setBrand(trim(source.getBrand()));
row.setCommodityImage(trim(source.getCommodityImage()));
row.setInventorySales(trim(source.getInventorySales()));
row.setSalesRank(trim(source.getSalesRank()));
row.setPageViews(trim(source.getPageViews()));
row.setUnitsSold(trim(source.getUnitsSold()));
row.setPrice(trim(source.getPrice()));
row.setRecommendedOffer(trim(source.getRecommendedOffer()));
return row;
}
public static boolean rowEmpty(ShopDataCrawlRowDto row) {
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
}
public static boolean sameRow(ShopDataCrawlRowDto left, ShopDataCrawlRowDto right) {
return left != null && right != null
&& java.util.Objects.equals(trim(left.getDate()), trim(right.getDate()))
&& java.util.Objects.equals(trim(left.getAsin()), trim(right.getAsin()))
&& java.util.Objects.equals(trim(left.getBrand()), trim(right.getBrand()))
&& java.util.Objects.equals(trim(left.getCommodityImage()), trim(right.getCommodityImage()))
&& java.util.Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales()))
&& java.util.Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank()))
&& java.util.Objects.equals(trim(left.getPageViews()), trim(right.getPageViews()))
&& java.util.Objects.equals(trim(left.getUnitsSold()), trim(right.getUnitsSold()))
&& java.util.Objects.equals(trim(left.getPrice()), trim(right.getPrice()))
&& java.util.Objects.equals(trim(left.getRecommendedOffer()), trim(right.getRecommendedOffer()));
}
/** 国家结果行稳定去重键:与 sameRow 的 10 字段 trim 比较语义等价,用于 O(1) 去重。 */
public static String rowDedupKey(ShopDataCrawlRowDto row) {
if (row == null) {
return null;
}
return trim(row.getDate()) + ROW_KEY_SEPARATOR + trim(row.getAsin()) + ROW_KEY_SEPARATOR + trim(row.getBrand())
+ ROW_KEY_SEPARATOR + trim(row.getCommodityImage()) + ROW_KEY_SEPARATOR + trim(row.getInventorySales())
+ ROW_KEY_SEPARATOR + trim(row.getSalesRank()) + ROW_KEY_SEPARATOR + trim(row.getPageViews())
+ ROW_KEY_SEPARATOR + trim(row.getUnitsSold()) + ROW_KEY_SEPARATOR + trim(row.getPrice())
+ ROW_KEY_SEPARATOR + trim(row.getRecommendedOffer());
}
}
@@ -0,0 +1,177 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* 任务 100:ShopDataCrawlSheetBuilder Sheet 构造器。
* 结果 Workbook/Sheet 构造辅助(表头、列序、样式、行值派生、模板校验)。
* 与现状 ShopDataCrawlExcelAssemblyService 一致:5 个国家工作表(英国/德国/法国/西班牙/意大利)、
* 10 列表头(HEADERS,品牌为最后一列)、图片列 2 宽 18*256、行高 80pt、
* 模板列映射(legacy/current/带品牌变体)、数据行从第 1 行。
* 不落库、无 IO 依赖;纯函数无状态。
*/
public final class ShopDataCrawlSheetBuilder {
public static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
public static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
public static final List<String> LEGACY_HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
public static final List<String> HEADERS_WITHOUT_BRAND = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
public static final List<String> HEADERS = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价", "品牌");
public static final int IMAGE_COLUMN = 2;
public static final int BRAND_COLUMN = HEADERS.size() - 1;
public static final int IMAGE_COLUMN_WIDTH = 18 * 256;
public static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
private ShopDataCrawlSheetBuilder() {
}
/** 流式路径:空 workbook 上自建国家工作表与表头(数据行由调用方写入)。 */
public static Sheet createStreamingSheet(Workbook workbook, int index) {
Sheet sheet = workbook.createSheet(SHEETS.get(index));
Row header = sheet.createRow(0);
for (int column = 0; column < HEADERS.size(); column++) {
header.createCell(column).setCellValue(HEADERS.get(column));
}
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
return sheet;
}
public static void validateTemplate(XSSFWorkbook workbook) {
if (workbook.getNumberOfSheets() != SHEETS.size()) {
throw new BusinessException("店铺数据抓取模板工作表数量不正确");
}
for (int i = 0; i < SHEETS.size(); i++) {
Sheet sheet = workbook.getSheetAt(i);
if (!SHEETS.get(i).equals(sheet.getSheetName())) {
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
}
Row header = sheet.getRow(0);
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
List<String> expectedHeaders = currentTemplate
? (templateHasBrand ? HEADERS : HEADERS_WITHOUT_BRAND)
: LEGACY_HEADERS;
for (int column = 0; column < expectedHeaders.size(); column++) {
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
if (!expectedHeaders.get(column).equals(actual)) {
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
}
}
}
}
/** 模板路径:从模板第 1 行按列映射提取输出列样式。 */
public static CellStyle[] templateStyles(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
Row styleRow = sheet.getRow(1);
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
styles[column] = cell == null ? null : cell.getCellStyle();
}
return styles;
}
/** 模板路径:按列映射提取模板列宽。 */
public static int[] templateColumnWidths(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
int[] columnWidths = new int[HEADERS.size()];
for (int column = 0; column < columnWidths.length; column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
columnWidths[column] = sheet.getColumnWidth(sourceColumn);
}
return columnWidths;
}
/** 模板路径:覆写表头文本并保留模板表头样式。 */
public static void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
Row header = sheet.getRow(0);
if (header == null) header = sheet.createRow(0);
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < HEADERS.size(); column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell source = header.getCell(sourceColumn);
styles[column] = source == null ? null : source.getCellStyle();
}
for (int column = 0; column < HEADERS.size(); column++) {
Cell cell = header.getCell(column);
if (cell == null) cell = header.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(HEADERS.get(column));
}
}
/** 模板路径:清空第 1 行起的全部数据行(保留表头)。 */
public static void clearDataRows(Sheet sheet) {
int last = sheet.getLastRowNum();
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
Row row = sheet.getRow(rowIndex);
if (row != null) {
sheet.removeRow(row);
}
}
}
/** 数据行 10 列值布局 + 样式应用(图片嵌入由调用方负责)。 */
public static void writeDataRowValues(Row row, ShopDataCrawlRowDto value, CellStyle[] styles) {
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
}
public static int templateColumnForOutput(int outputColumn, boolean currentTemplate, boolean templateHasBrand) {
if (outputColumn == BRAND_COLUMN) {
return templateHasBrand ? BRAND_COLUMN : 1;
}
return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1;
}
public static String cellText(Row row, int column) {
Cell cell = row == null ? null : row.getCell(column);
return cell == null ? "" : cell.getStringCellValue().trim();
}
public static boolean blank(String value) {
return value == null || value.isBlank();
}
/** 结果条目 → 按国家分组的行列表(国家码 trim+UPPER,未命中 5 国列表的行丢弃)。 */
public static Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>();
COUNTRIES.forEach(country -> result.put(country, new ArrayList<>()));
if (items == null) return result;
for (ShopDataCrawlResultItemVo item : items) {
if (item == null || Boolean.FALSE.equals(item.getSuccess()) || item.getCountryResults() == null) continue;
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) {
String country = countryResult == null || countryResult.getCountry() == null ? "" : countryResult.getCountry().trim().toUpperCase();
if (result.containsKey(country) && countryResult.getItems() != null) result.get(country).addAll(countryResult.getItems());
}
}
return result;
}
public static int totalDataRows(XSSFWorkbook workbook) {
int total = 0;
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
total += Math.max(0, workbook.getSheetAt(i).getLastRowNum());
}
return total;
}
}
@@ -49,6 +49,7 @@ public class QueryAsinController {
@Parameter(description = "分组ID") @RequestParam(required = false) Long groupId, @Parameter(description = "分组ID") @RequestParam(required = false) Long groupId,
@Parameter(description = "店铺名") @RequestParam(required = false) String shopName, @Parameter(description = "店铺名") @RequestParam(required = false) String shopName,
@Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin, @Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin,
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(required = false) String country,
@Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId, @Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId,
@Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) { @Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) {
return ApiResponse.success(queryAsinService.page( return ApiResponse.success(queryAsinService.page(
@@ -57,6 +58,7 @@ public class QueryAsinController {
groupId, groupId,
shopName, shopName,
asin, asin,
country,
operatorId, operatorId,
Boolean.TRUE.equals(superAdmin))); Boolean.TRUE.equals(superAdmin)));
} }
@@ -67,9 +69,10 @@ public class QueryAsinController {
@Parameter(description = "分组ID") @RequestParam(required = false) Long groupId, @Parameter(description = "分组ID") @RequestParam(required = false) Long groupId,
@Parameter(description = "店铺名") @RequestParam(required = false) String shopName, @Parameter(description = "店铺名") @RequestParam(required = false) String shopName,
@Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin, @Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin,
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(required = false) String country,
@Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId, @Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId,
@Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) { @Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) {
byte[] bytes = queryAsinService.export(groupId, shopName, asin, operatorId, Boolean.TRUE.equals(superAdmin)); byte[] bytes = queryAsinService.export(groupId, shopName, asin, country, operatorId, Boolean.TRUE.equals(superAdmin));
String filename = "query-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx"; String filename = "query-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
return ResponseEntity.ok() return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename)) .header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
@@ -28,8 +28,10 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import java.math.BigDecimal;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.math.BigDecimal;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -49,6 +51,9 @@ public class SkipPriceAsinController {
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId, @Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
@Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName, @Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName,
@Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin, @Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin,
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(name = "country", required = false) String country,
@Parameter(description = "最低价下限") @RequestParam(name = "minimum_price_from", required = false) BigDecimal minimumPriceFrom,
@Parameter(description = "最低价上限") @RequestParam(name = "minimum_price_to", required = false) BigDecimal minimumPriceTo,
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId, @Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId,
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) { @Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) {
return ApiResponse.success(skipPriceAsinService.page( return ApiResponse.success(skipPriceAsinService.page(
@@ -57,6 +62,9 @@ public class SkipPriceAsinController {
groupId, groupId,
shopName, shopName,
asin, asin,
country,
minimumPriceFrom,
minimumPriceTo,
operatorId, operatorId,
Boolean.TRUE.equals(superAdmin))); Boolean.TRUE.equals(superAdmin)));
} }
@@ -67,9 +75,13 @@ public class SkipPriceAsinController {
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId, @Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
@Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName, @Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName,
@Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin, @Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin,
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(name = "country", required = false) String country,
@Parameter(description = "最低价下限") @RequestParam(name = "minimum_price_from", required = false) BigDecimal minimumPriceFrom,
@Parameter(description = "最低价上限") @RequestParam(name = "minimum_price_to", required = false) BigDecimal minimumPriceTo,
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId, @Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId,
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) { @Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) {
byte[] bytes = skipPriceAsinService.export(groupId, shopName, asin, operatorId, Boolean.TRUE.equals(superAdmin)); byte[] bytes = skipPriceAsinService.export(groupId, shopName, asin, country, minimumPriceFrom, minimumPriceTo,
operatorId, Boolean.TRUE.equals(superAdmin));
String filename = "skip-price-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx"; String filename = "skip-price-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
return ResponseEntity.ok() return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename)) .header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.shopkey.service; package com.nanri.aiimage.modules.shopkey.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.support.SFunction;
import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.IdUtil;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader; import com.nanri.aiimage.common.util.ExcelStreamReader;
@@ -56,14 +57,15 @@ public class QueryAsinService {
private final Map<String, QueryAsinImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>(); private final Map<String, QueryAsinImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
public QueryAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin, public QueryAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin,
Long operatorId, boolean superAdmin) { String country, Long operatorId, boolean superAdmin) {
long safePage = Math.max(page, 1); long safePage = Math.max(page, 1);
long safePageSize = Math.min(Math.max(pageSize, 1), 100); long safePageSize = Math.min(Math.max(pageSize, 1), 100);
Long total = countFilteredRows(groupId, shopName, asin, operatorId, superAdmin); Long total = countFilteredRows(groupId, shopName, asin, country, operatorId, superAdmin);
List<QueryAsinEntity> rows = listFilteredRows( List<QueryAsinEntity> rows = listFilteredRows(
groupId, groupId,
shopName, shopName,
asin, asin,
country,
operatorId, operatorId,
superAdmin, superAdmin,
(safePage - 1) * safePageSize, (safePage - 1) * safePageSize,
@@ -83,8 +85,8 @@ public class QueryAsinService {
return vo; return vo;
} }
public byte[] export(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) { public byte[] export(Long groupId, String shopName, String asin, String country, Long operatorId, boolean superAdmin) {
List<QueryAsinEntity> rows = listFilteredRows(groupId, shopName, asin, operatorId, superAdmin, null, null); List<QueryAsinEntity> rows = listFilteredRows(groupId, shopName, asin, country, operatorId, superAdmin, null, null);
Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream() Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream()
.map(QueryAsinEntity::getGroupId) .map(QueryAsinEntity::getGroupId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
@@ -106,35 +108,46 @@ public class QueryAsinService {
} }
} }
private Long countFilteredRows(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) { private Long countFilteredRows(Long groupId, String shopName, String asin, String country, Long operatorId, boolean superAdmin) {
return queryAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin)); return queryAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, country, operatorId, superAdmin));
} }
private List<QueryAsinEntity> listFilteredRows(Long groupId, String shopName, String asin, private List<QueryAsinEntity> listFilteredRows(Long groupId, String shopName, String asin, String country,
Long operatorId, boolean superAdmin, Long operatorId, boolean superAdmin,
Long offset, Long limit) { Long offset, Long limit) {
LambdaQueryWrapper<QueryAsinEntity> query = buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin); LambdaQueryWrapper<QueryAsinEntity> query = buildFilterQuery(groupId, shopName, asin, country, operatorId, superAdmin);
if (offset != null && limit != null) { if (offset != null && limit != null) {
query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit)); query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit));
} }
return queryAsinMapper.selectList(query); return queryAsinMapper.selectList(query);
} }
private LambdaQueryWrapper<QueryAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin, private LambdaQueryWrapper<QueryAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin, String country,
Long operatorId, boolean superAdmin) { Long operatorId, boolean superAdmin) {
String safeShopName = normalizeBlank(shopName); String safeShopName = normalizeBlank(shopName);
String safeAsin = normalizeBlank(asin); String safeAsin = normalizeBlank(asin);
String safeCountry = normalizeBlank(country).toUpperCase(Locale.ROOT);
Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false); Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false);
LambdaQueryWrapper<QueryAsinEntity> query = new LambdaQueryWrapper<QueryAsinEntity>() LambdaQueryWrapper<QueryAsinEntity> query = new LambdaQueryWrapper<QueryAsinEntity>()
.eq(groupId != null && groupId > 0, QueryAsinEntity::getGroupId, groupId) .eq(groupId != null && groupId > 0, QueryAsinEntity::getGroupId, groupId)
.like(!safeShopName.isEmpty(), QueryAsinEntity::getShopName, safeShopName) .like(!safeShopName.isEmpty(), QueryAsinEntity::getShopName, safeShopName)
.and(!safeAsin.isEmpty(), wrapper -> wrapper .orderByDesc(QueryAsinEntity::getId);
if (!safeAsin.isEmpty()) {
if (safeCountry.isEmpty()) {
query.and(wrapper -> wrapper
.like(QueryAsinEntity::getAsinDe, safeAsin) .like(QueryAsinEntity::getAsinDe, safeAsin)
.or().like(QueryAsinEntity::getAsinUk, safeAsin) .or().like(QueryAsinEntity::getAsinUk, safeAsin)
.or().like(QueryAsinEntity::getAsinFr, safeAsin) .or().like(QueryAsinEntity::getAsinFr, safeAsin)
.or().like(QueryAsinEntity::getAsinIt, safeAsin) .or().like(QueryAsinEntity::getAsinIt, safeAsin)
.or().like(QueryAsinEntity::getAsinEs, safeAsin)) .or().like(QueryAsinEntity::getAsinEs, safeAsin));
.orderByDesc(QueryAsinEntity::getId); } else {
query.like(queryAsinCountryColumn(safeCountry), safeAsin);
}
}
if (!safeCountry.isEmpty()) {
SFunction<QueryAsinEntity, ?> countryColumn = queryAsinCountryColumn(safeCountry);
query.isNotNull(countryColumn).ne(countryColumn, "");
}
if (!superAdmin) { if (!superAdmin) {
if (accessibleGroupIds.isEmpty()) { if (accessibleGroupIds.isEmpty()) {
query.eq(QueryAsinEntity::getId, -1L); query.eq(QueryAsinEntity::getId, -1L);
@@ -145,6 +158,17 @@ public class QueryAsinService {
return query; return query;
} }
private SFunction<QueryAsinEntity, ?> queryAsinCountryColumn(String country) {
return switch (country) {
case "DE" -> QueryAsinEntity::getAsinDe;
case "UK" -> QueryAsinEntity::getAsinUk;
case "FR" -> QueryAsinEntity::getAsinFr;
case "IT" -> QueryAsinEntity::getAsinIt;
case "ES" -> QueryAsinEntity::getAsinEs;
default -> throw new BusinessException("不支持的国家代码: " + country);
};
}
private void writeExportHeader(Sheet sheet) { private void writeExportHeader(Sheet sheet) {
Row header = sheet.createRow(0); Row header = sheet.createRow(0);
String[] headers = { String[] headers = {
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.shopkey.service; package com.nanri.aiimage.modules.shopkey.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.support.SFunction;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.IdUtil;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
@@ -61,14 +62,18 @@ public class SkipPriceAsinService {
private final Map<String, CachedSkipAsinLookup> skipAsinLookupCache = new ConcurrentHashMap<>(); private final Map<String, CachedSkipAsinLookup> skipAsinLookupCache = new ConcurrentHashMap<>();
public SkipPriceAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin, public SkipPriceAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin,
String country, BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
Long operatorId, boolean superAdmin) { Long operatorId, boolean superAdmin) {
long safePage = Math.max(page, 1); long safePage = Math.max(page, 1);
long safePageSize = Math.min(Math.max(pageSize, 1), 100); long safePageSize = Math.min(Math.max(pageSize, 1), 100);
Long total = countFilteredRows(groupId, shopName, asin, operatorId, superAdmin); Long total = countFilteredRows(groupId, shopName, asin, country, minimumPriceFrom, minimumPriceTo, operatorId, superAdmin);
List<SkipPriceAsinEntity> rows = listFilteredRows( List<SkipPriceAsinEntity> rows = listFilteredRows(
groupId, groupId,
shopName, shopName,
asin, asin,
country,
minimumPriceFrom,
minimumPriceTo,
operatorId, operatorId,
superAdmin, superAdmin,
(safePage - 1) * safePageSize, (safePage - 1) * safePageSize,
@@ -88,8 +93,11 @@ public class SkipPriceAsinService {
return vo; return vo;
} }
public byte[] export(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) { public byte[] export(Long groupId, String shopName, String asin, String country,
List<SkipPriceAsinEntity> rows = listFilteredRows(groupId, shopName, asin, operatorId, superAdmin, null, null); BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
Long operatorId, boolean superAdmin) {
List<SkipPriceAsinEntity> rows = listFilteredRows(groupId, shopName, asin, country,
minimumPriceFrom, minimumPriceTo, operatorId, superAdmin, null, null);
Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream() Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream()
.map(SkipPriceAsinEntity::getGroupId) .map(SkipPriceAsinEntity::getGroupId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
@@ -111,35 +119,84 @@ public class SkipPriceAsinService {
} }
} }
private Long countFilteredRows(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) { private Long countFilteredRows(Long groupId, String shopName, String asin, String country,
return skipPriceAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin)); BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
Long operatorId, boolean superAdmin) {
return skipPriceAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, country,
minimumPriceFrom, minimumPriceTo, operatorId, superAdmin));
} }
private List<SkipPriceAsinEntity> listFilteredRows(Long groupId, String shopName, String asin, private List<SkipPriceAsinEntity> listFilteredRows(Long groupId, String shopName, String asin, String country,
BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
Long operatorId, boolean superAdmin, Long operatorId, boolean superAdmin,
Long offset, Long limit) { Long offset, Long limit) {
LambdaQueryWrapper<SkipPriceAsinEntity> query = buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin); LambdaQueryWrapper<SkipPriceAsinEntity> query = buildFilterQuery(groupId, shopName, asin, country,
minimumPriceFrom, minimumPriceTo, operatorId, superAdmin);
if (offset != null && limit != null) { if (offset != null && limit != null) {
query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit)); query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit));
} }
return skipPriceAsinMapper.selectList(query); return skipPriceAsinMapper.selectList(query);
} }
private LambdaQueryWrapper<SkipPriceAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin, private LambdaQueryWrapper<SkipPriceAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin, String country,
BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
Long operatorId, boolean superAdmin) { Long operatorId, boolean superAdmin) {
String safeShopName = normalizeBlank(shopName); String safeShopName = normalizeBlank(shopName);
String safeAsin = normalizeBlank(asin); String safeAsin = normalizeBlank(asin);
String safeCountry = normalizeBlank(country).toUpperCase(Locale.ROOT);
Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false); Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false);
LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>() LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>()
.eq(groupId != null && groupId > 0, SkipPriceAsinEntity::getGroupId, groupId) .eq(groupId != null && groupId > 0, SkipPriceAsinEntity::getGroupId, groupId)
.like(!safeShopName.isEmpty(), SkipPriceAsinEntity::getShopName, safeShopName) .like(!safeShopName.isEmpty(), SkipPriceAsinEntity::getShopName, safeShopName)
.and(!safeAsin.isEmpty(), wrapper -> wrapper .orderByDesc(SkipPriceAsinEntity::getId);
if (!safeAsin.isEmpty()) {
if (safeCountry.isEmpty()) {
query.and(wrapper -> wrapper
.like(SkipPriceAsinEntity::getAsinDe, safeAsin) .like(SkipPriceAsinEntity::getAsinDe, safeAsin)
.or().like(SkipPriceAsinEntity::getAsinUk, safeAsin) .or().like(SkipPriceAsinEntity::getAsinUk, safeAsin)
.or().like(SkipPriceAsinEntity::getAsinFr, safeAsin) .or().like(SkipPriceAsinEntity::getAsinFr, safeAsin)
.or().like(SkipPriceAsinEntity::getAsinIt, safeAsin) .or().like(SkipPriceAsinEntity::getAsinIt, safeAsin)
.or().like(SkipPriceAsinEntity::getAsinEs, safeAsin)) .or().like(SkipPriceAsinEntity::getAsinEs, safeAsin));
.orderByDesc(SkipPriceAsinEntity::getId); } else {
query.like(skipPriceAsinCountryColumn(safeCountry), safeAsin);
}
}
if (!safeCountry.isEmpty()) {
SFunction<SkipPriceAsinEntity, ?> countryColumn = skipPriceAsinCountryColumn(safeCountry);
query.isNotNull(countryColumn).ne(countryColumn, "");
}
boolean hasMinimumPriceFrom = minimumPriceFrom != null;
boolean hasMinimumPriceTo = minimumPriceTo != null;
if (hasMinimumPriceFrom || hasMinimumPriceTo) {
if (safeCountry.isEmpty()) {
List<SFunction<SkipPriceAsinEntity, ?>> priceColumns = skipPriceAsinMinimumPriceColumns();
query.and(wrapper -> {
boolean first = true;
for (SFunction<SkipPriceAsinEntity, ?> column : priceColumns) {
if (!first) {
wrapper.or();
}
first = false;
wrapper.isNotNull(column);
if (hasMinimumPriceFrom) {
wrapper.ge(column, minimumPriceFrom);
}
if (hasMinimumPriceTo) {
wrapper.le(column, minimumPriceTo);
}
}
});
} else {
SFunction<SkipPriceAsinEntity, ?> priceColumn = skipPriceAsinMinimumPriceColumn(safeCountry);
query.isNotNull(priceColumn);
if (hasMinimumPriceFrom) {
query.ge(priceColumn, minimumPriceFrom);
}
if (hasMinimumPriceTo) {
query.le(priceColumn, minimumPriceTo);
}
}
}
if (!superAdmin) { if (!superAdmin) {
if (accessibleGroupIds.isEmpty()) { if (accessibleGroupIds.isEmpty()) {
query.eq(SkipPriceAsinEntity::getId, -1L); query.eq(SkipPriceAsinEntity::getId, -1L);
@@ -485,6 +542,37 @@ public class SkipPriceAsinService {
return findCountryAsin(groupId, shopName, country, asin) != null; return findCountryAsin(groupId, shopName, country, asin) != null;
} }
private SFunction<SkipPriceAsinEntity, ?> skipPriceAsinCountryColumn(String country) {
return switch (country) {
case "DE" -> SkipPriceAsinEntity::getAsinDe;
case "UK" -> SkipPriceAsinEntity::getAsinUk;
case "FR" -> SkipPriceAsinEntity::getAsinFr;
case "IT" -> SkipPriceAsinEntity::getAsinIt;
case "ES" -> SkipPriceAsinEntity::getAsinEs;
default -> throw new BusinessException("不支持的国家代码: " + country);
};
}
private SFunction<SkipPriceAsinEntity, ?> skipPriceAsinMinimumPriceColumn(String country) {
return switch (country) {
case "DE" -> SkipPriceAsinEntity::getMinimumPriceDe;
case "UK" -> SkipPriceAsinEntity::getMinimumPriceUk;
case "FR" -> SkipPriceAsinEntity::getMinimumPriceFr;
case "IT" -> SkipPriceAsinEntity::getMinimumPriceIt;
case "ES" -> SkipPriceAsinEntity::getMinimumPriceEs;
default -> throw new BusinessException("不支持的国家代码: " + country);
};
}
private List<SFunction<SkipPriceAsinEntity, ?>> skipPriceAsinMinimumPriceColumns() {
return List.of(
SkipPriceAsinEntity::getMinimumPriceDe,
SkipPriceAsinEntity::getMinimumPriceUk,
SkipPriceAsinEntity::getMinimumPriceFr,
SkipPriceAsinEntity::getMinimumPriceIt,
SkipPriceAsinEntity::getMinimumPriceEs);
}
private SkipPriceAsinEntity findCountryAsin(Long groupId, String shopName, String country, String asin) { private SkipPriceAsinEntity findCountryAsin(Long groupId, String shopName, String country, String asin) {
LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>() LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>()
.eq(SkipPriceAsinEntity::getGroupId, groupId) .eq(SkipPriceAsinEntity::getGroupId, groupId)
@@ -41,6 +41,8 @@ import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -189,6 +191,14 @@ public class ShopMatchController {
return ApiResponse.success(shopMatchTaskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(shopMatchTaskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(shopMatchTaskService.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/result") @PostMapping("/tasks/{taskId}/result")
@Operation(summary = "提交匹配结果", description = "Python 可多次分批回传结果;后端会组装文件并在超时场景自动补偿收尾。") @Operation(summary = "提交匹配结果", description = "Python 可多次分批回传结果;后端会组装文件并在超时场景自动补偿收尾。")
public ApiResponse<Void> submitResult( public ApiResponse<Void> submitResult(
@@ -55,6 +55,8 @@ import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -62,6 +64,10 @@ import java.util.Map;
public class ShopMatchTaskService { public class ShopMatchTaskService {
private static final String MODULE_TYPE = "SHOP_MATCH"; private static final String MODULE_TYPE = "SHOP_MATCH";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai"); private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
@@ -79,6 +85,7 @@ public class ShopMatchTaskService {
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final SkipPriceAsinService skipPriceAsinService; private final SkipPriceAsinService skipPriceAsinService;
private final QueryAsinMapper queryAsinMapper; private final QueryAsinMapper queryAsinMapper;
private final TaskProgressLightAssembler taskProgressLightAssembler;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -7,11 +7,13 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest; import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightRequest;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinFilterConditionVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinFilterConditionVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService; import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo; import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
@@ -120,6 +122,14 @@ public class SimilarAsinController {
return ApiResponse.success(service.progressBatch(request.getTaskIds())); return ApiResponse.success(service.progressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<SimilarAsinTaskLightBatchVo> progressLight(@Valid @RequestBody SimilarAsinTaskLightRequest request) {
return ApiResponse.success(service.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/activate") @PostMapping("/tasks/{taskId}/activate")
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING。") @Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING。")
public ApiResponse<Void> activate( public ApiResponse<Void> activate(
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Schema(description = "相似ASIN检测任务轻量进度查询请求")
public class SimilarAsinTaskLightRequest {
@NotNull
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测任务轻量进度响应")
public class SimilarAsinTaskLightBatchVo {
@Schema(description = "轻量进度项列表。顺序按请求 taskIds 处理。")
private List<SimilarAsinTaskLightVo> items = new ArrayList<>();
@Schema(description = "未找到或不属于相似ASIN检测模块的任务 ID 列表。", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}
@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测任务轻量进度项(前端轮询专用,不含明细/result 内容)")
public class SimilarAsinTaskLightVo {
@Schema(description = "任务 ID。", example = "3938")
private Long taskId;
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
private String status;
@Schema(description = "业务状态码(预留字段,当前恒为 null)。", example = "null")
private String statusCode;
@Schema(description = "结果文件组装 Job 状态:SUCCESS/RUNNING/FAILED;无 Job 时为空。", example = "SUCCESS")
private String fileStatus;
@Schema(description = "文件组装失败时的错误信息;无错误时为空。", example = "组装失败: 文件写入超时")
private String fileError;
@Schema(description = "结果文件是否已就绪。", example = "true")
private Boolean fileReady;
@Schema(description = "最后更新时间,ISO 本地时间字符串。", example = "2026-04-26T10:05:00")
private String updatedAt;
}
@@ -35,6 +35,11 @@ import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo; import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache; import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache;
import com.nanri.aiimage.modules.similarasin.util.ExcelCellImageWriter; import com.nanri.aiimage.modules.similarasin.util.ExcelCellImageWriter;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
@@ -59,12 +64,9 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Font; import org.apache.poi.ss.usermodel.Font;
import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
@@ -136,6 +138,7 @@ public class SimilarAsinTaskService {
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String CONTENT_TYPE_ZIP = "application/zip"; private static final String CONTENT_TYPE_ZIP = "application/zip";
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
private static final int MAX_LIGHT_TASK_IDS = 200;
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */ /** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L; private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
@@ -336,6 +339,20 @@ public class SimilarAsinTaskService {
* best-effort:service 内部所有异常都已吞掉,不影响主流程。 * best-effort:service 内部所有异常都已吞掉,不影响主流程。
*/ */
private final SimilarAsinImagePrefetchService imagePrefetchService; private final SimilarAsinImagePrefetchService imagePrefetchService;
/**
* Task 89:Excel 行解析器(表头/单元格读取、别名匹配、空行跳过、单字段截断)。
* 由 parseAndCreateTask 委托;解析语义与搬移前 parseWorkbook 完全一致。
*/
SimilarAsinExcelParser excelParser = new SimilarAsinExcelParser();
/**
* Task 91:历史查询 VO 拼装(toHistoryItem + 进度链)搬入独立组件;
* 由 history/progressBatch 委托,只读不落库,输出与现状一致。
*/
private SimilarAsinHistoryAssembler historyAssembler() {
return new SimilarAsinHistoryAssembler(
taskScopeStateMapper, taskChunkMapper, fileTaskMapper,
taskProgressSnapshotService, ossStorageService, transientPayloadStorageService, objectMapper);
}
@Autowired @Autowired
@Qualifier("taskQueueExecutor") @Qualifier("taskQueueExecutor")
private TaskExecutor taskQueueExecutor; private TaskExecutor taskQueueExecutor;
@@ -466,7 +483,7 @@ public class SimilarAsinTaskService {
+ " (" + input.length() + " bytes > " + maxBytes + " bytes)"); + " (" + input.length() + " bytes > " + maxBytes + " bytes)");
} }
ParsedWorkbook parsed = parseWorkbook(input, source); ParsedWorkbook parsed = parseWorkbookDelegated(input, source);
totalRows += parsed.totalRows(); totalRows += parsed.totalRows();
droppedRows += parsed.droppedRows(); droppedRows += parsed.droppedRows();
allRows.addAll(parsed.allRows()); allRows.addAll(parsed.allRows());
@@ -487,7 +504,7 @@ public class SimilarAsinTaskService {
sourceFiles.size(), allRows.size()); sourceFiles.size(), allRows.size());
} }
long parsedAt = System.nanoTime(); long parsedAt = System.nanoTime();
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows); List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(allRows);
long groupedAt = System.nanoTime(); long groupedAt = System.nanoTime();
FileTaskEntity task = new FileTaskEntity(); FileTaskEntity task = new FileTaskEntity();
@@ -640,13 +657,49 @@ public class SimilarAsinTaskService {
.map(FileResultEntity::getId) .map(FileResultEntity::getId)
.filter(Objects::nonNull) .filter(Objects::nonNull)
.toList()); .toList());
for (FileResultEntity row : rows) { vo.getItems().addAll(historyAssembler().buildHistoryItems(rows, taskMap, jobMap));
FileTaskEntity task = taskMap.get(row.getTaskId()); return vo;
String taskStatus = task == null ? null : task.getStatus(); }
if (STATUS_PENDING.equals(taskStatus)) {
public SimilarAsinTaskLightBatchVo progressLight(List<Long> taskIds) {
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.limit(MAX_LIGHT_TASK_IDS)
.toList();
if (normalizedIds.isEmpty()) {
return vo;
}
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
for (FileTaskEntity task : fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getUpdatedAt)
.in(FileTaskEntity::getId, normalizedIds))) {
if (task != null && MODULE_TYPE.equals(task.getModuleType())) {
taskMap.put(task.getId(), task);
}
}
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByTaskIds(MODULE_TYPE, new ArrayList<>(taskMap.keySet()));
for (Long taskId : normalizedIds) {
FileTaskEntity task = taskMap.get(taskId);
if (task == null) {
vo.getMissingTaskIds().add(taskId);
continue; continue;
} }
vo.getItems().add(toHistoryItem(row, task, jobMap.get(row.getId()))); SimilarAsinTaskLightVo item = new SimilarAsinTaskLightVo();
item.setTaskId(task.getId());
item.setStatus(resolveDisplayTaskStatus(task));
TaskFileJobEntity job = jobMap.get(taskId);
boolean fileReady = job != null
&& job.getResultFileUrl() != null
&& !job.getResultFileUrl().isBlank();
item.setFileReady(fileReady);
item.setFileStatus(job == null ? null : job.getStatus());
item.setFileError(job == null ? null : job.getErrorMessage());
item.setUpdatedAt(fmt(task.getUpdatedAt()));
vo.getItems().add(item);
} }
return vo; return vo;
} }
@@ -708,7 +761,8 @@ public class SimilarAsinTaskService {
detail.setTask(toTaskItem(task)); detail.setTask(toTaskItem(task));
FileResultEntity resultRow = resultByTaskId.get(taskId); FileResultEntity resultRow = resultByTaskId.get(taskId);
if (resultRow != null) { if (resultRow != null) {
detail.getItems().add(toHistoryItem(resultRow, task, jobMap.get(resultRow.getId()))); detail.getItems().addAll(historyAssembler().buildHistoryItems(
List.of(resultRow), taskMap, jobMap));
} }
vo.getItems().add(detail); vo.getItems().add(detail);
} }
@@ -1751,7 +1805,7 @@ public class SimilarAsinTaskService {
private Map<String, List<SimilarAsinParsedRowVo>> loadAllRowsByBaseId(FileTaskEntity task) { private Map<String, List<SimilarAsinParsedRowVo>> loadAllRowsByBaseId(FileTaskEntity task) {
try { try {
SimilarAsinParsedPayloadDto payload = readParsedPayload(task); SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
return groupRowsByBaseId(resolveAllRows(payload)); return SimilarAsinGroupingConverter.groupRowsByBaseId(resolveAllRows(payload));
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage()); log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage());
return new LinkedHashMap<>(); return new LinkedHashMap<>();
@@ -1791,36 +1845,6 @@ public class SimilarAsinTaskService {
return candidates; return candidates;
} }
/**
* Task 6:分组数据改为索引/范围引用。组内行在原 items 中必然连续
* (parseWorkbook 按行遍历、同 baseId 块连续收集),因此只需要
* [startIndex, endIndex) 半开区间即可唯一定位组内行,
* 行对象仅存在于 items 一次,避免 groups 嵌套复制完整行对象。
*/
private List<SimilarAsinParsedGroupVo> buildParsedGroups(List<SimilarAsinParsedRowVo> rows) {
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
int cursor = 0;
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
if (siblings == null || siblings.isEmpty()) {
continue;
}
SimilarAsinParsedRowVo first = siblings.getFirst();
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setSourceFileKey(first.getSourceFileKey());
group.setSourceFilename(first.getSourceFilename());
group.setGroupKey(firstNonBlank(first.getGroupKey(), buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
group.setBaseId(baseId(first.getDisplayId()));
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
group.setItemCount(siblings.size());
int end = cursor + siblings.size();
group.setStartIndex(cursor);
group.setEndIndex(end);
cursor = end;
groups.add(group);
}
return groups;
}
private List<SimilarAsinParsedGroupVo> buildResponsePreviewGroups(List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) { private List<SimilarAsinParsedGroupVo> buildResponsePreviewGroups(List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
if (groups == null || groups.isEmpty()) { if (groups == null || groups.isEmpty()) {
return List.of(); return List.of();
@@ -1887,18 +1911,6 @@ public class SimilarAsinTaskService {
return vo; return vo;
} }
private Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
if (rows == null) {
return result;
}
for (SimilarAsinParsedRowVo row : rows) {
String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId()));
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
}
return result;
}
private List<SimilarAsinResultRowDto> flattenSubmittedRows(SimilarAsinSubmitResultRequest request) { private List<SimilarAsinResultRowDto> flattenSubmittedRows(SimilarAsinSubmitResultRequest request) {
if (request == null) { if (request == null) {
return List.of(); return List.of();
@@ -4718,8 +4730,11 @@ public class SimilarAsinTaskService {
return firstNonBlank(resultRow.getUrl(), fallbackUrl); return firstNonBlank(resultRow.getUrl(), fallbackUrl);
} }
private ParsedWorkbook parseWorkbook(File input, SimilarAsinSourceFileDto source) { /**
DataFormatter formatter = new DataFormatter(); * Task 89:解析委托 SimilarAsinExcelParser(表头/单元格读取、别名匹配、空行跳过、字段截断),
* 保留服务侧编排语义:zip 探针、status 列过滤、类目重试判定、行号/分组键/rowToken 组装。
*/
private ParsedWorkbook parseWorkbookDelegated(File input, SimilarAsinSourceFileDto source) {
try { try {
// Task 8:受控读取——先探测 zip 条目数与解压体积,超限拒绝 // Task 8:受控读取——先探测 zip 条目数与解压体积,超限拒绝
probeWorkbookZipBounds(input, source.getOriginalFilename()); probeWorkbookZipBounds(input, source.getOriginalFilename());
@@ -4729,26 +4744,9 @@ public class SimilarAsinTaskService {
// 非 zip 或损坏文件:留给 WorkbookFactory 尝试后由下方 catch 转业务异常 // 非 zip 或损坏文件:留给 WorkbookFactory 尝试后由下方 catch 转业务异常
log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage()); log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage());
} }
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0); try {
Row header = sheet.getRow(0); SimilarAsinExcelParser.ParsedSheet parsedSheet = excelParser.parse(input, resolveMaxFieldLength());
if (header == null) { List<String> headers = parsedSheet.headers();
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
// 兼容旧 xlsx:RESULT_HEADERS 已不再写出 sku 列,但仍要支持旧版结果簿重新上传,
// 因此这里仍按可选列读取并保留到 DTO,enrichRowForLlm 仍可使用,仅导出阶段不再写出。
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers); int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
String statusHeader = statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : null; String statusHeader = statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : null;
@@ -4758,14 +4756,10 @@ public class SimilarAsinTaskService {
int validRows = 0; int validRows = 0;
String currentBlockBaseId = ""; String currentBlockBaseId = "";
String currentGroupKey = ""; String currentGroupKey = "";
for (int i = 1; i <= sheet.getLastRowNum(); i++) { for (SimilarAsinExcelParser.SimilarAsinExcelRow parsedRow : parsedSheet.rows()) {
Row row = sheet.getRow(i); String id = parsedRow.id();
if (row == null) { String asin = parsedRow.asin();
continue; String country = parsedRow.country();
}
String id = cell(row, idCol, formatter);
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter);
if (id.isBlank() && asin.isBlank() && country.isBlank()) { if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue; continue;
} }
@@ -4778,7 +4772,7 @@ public class SimilarAsinTaskService {
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo(); SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey(source.getFileKey()); vo.setSourceFileKey(source.getFileKey());
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName())); vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
vo.setRowIndex(i + 1); vo.setRowIndex(parsedRow.rowIndex());
vo.setSourceId(id); vo.setSourceId(id);
vo.setDisplayId(normalizeDisplayId(id)); vo.setDisplayId(normalizeDisplayId(id));
String rowBaseId = baseId(vo.getDisplayId()); String rowBaseId = baseId(vo.getDisplayId());
@@ -4790,11 +4784,11 @@ public class SimilarAsinTaskService {
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex())); vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
vo.setAsin(asin); vo.setAsin(asin);
vo.setCountry(country); vo.setCountry(country);
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : ""); vo.setSku(parsedRow.sku());
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : ""); vo.setPrice(parsedRow.price());
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : ""); vo.setUrl(parsedRow.url());
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : ""); vo.setTitle(parsedRow.title());
vo.setValues(readRowValues(row, headers, formatter)); vo.setValues(parsedRow.values());
allRows.add(vo); allRows.add(vo);
} }
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers); boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
@@ -4911,86 +4905,6 @@ public class SimilarAsinTaskService {
return !normalized.isBlank() && !isTechnicalLlmFailure(normalized); return !normalized.isBlank() && !isTechnicalLlmFailure(normalized);
} }
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
Map<String, Integer> map = new LinkedHashMap<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
return map;
}
private List<String> readHeaders(Row header, DataFormatter formatter) {
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(val.isBlank() ? "列" + (i + 1) : val);
}
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter));
}
return values;
}
private int findRequiredHeader(Map<String, Integer> map, String... names) {
int idx = findOptionalHeader(map, names);
if (idx < 0) {
throw new BusinessException("缺少必要表头: " + String.join("/", names));
}
return idx;
}
private int findOptionalHeader(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
for (String name : names) {
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
return entry.getValue();
}
}
}
return -1;
}
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
for (String name : names) {
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
return entry.getValue();
}
}
}
return -1;
}
private String normalizeHeaderAlias(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
}
private String cell(Row row, int col, DataFormatter formatter) {
if (col < 0) {
return "";
}
String value = normalize(formatter.formatCellValue(row.getCell(col)));
if (isSpreadsheetErrorValue(value)) {
return "";
}
// Task 7:单字段长度上限,防止超长单元格导致内存无界增长
int maxLen = resolveMaxFieldLength();
if (value.length() > maxLen) {
return value.substring(0, maxLen);
}
return value;
}
private static boolean isSpreadsheetErrorValue(String value) { private static boolean isSpreadsheetErrorValue(String value) {
String normalized = value == null ? "" : value.trim(); String normalized = value == null ? "" : value.trim();
return normalized.startsWith("#") && normalized.endsWith("!"); return normalized.startsWith("#") && normalized.endsWith("!");
@@ -5133,253 +5047,6 @@ public class SimilarAsinTaskService {
return vo; return vo;
} }
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
vo.setResultId(row.getId());
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(buildFreshDownloadUrl(row));
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
attachFileJobState(vo, row, job, task);
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
vo.setCreatedAt(fmt(row.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
}
private String buildFreshDownloadUrl(FileResultEntity row) {
String resultFileUrl = row == null ? null : row.getResultFileUrl();
if (resultFileUrl == null || resultFileUrl.isBlank()) {
return null;
}
try {
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
} catch (Exception ex) {
log.warn("[similar-asin] build public download url failed resultId={} err={}",
row.getId(), ex.getMessage());
return null;
}
}
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
attachFileProgress(vo, row, job, task);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
attachFileProgress(vo, row, job, task);
}
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
if (row == null || row.getTaskId() == null) {
return;
}
Long taskId = row.getTaskId();
if (Boolean.TRUE.equals(vo.getFileReady())) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
int llmCompleted = countCompletedLlmStates(taskId);
int llmPending = countPendingLlmStates(taskId);
boolean uploadComplete = isResultSubmissionComplete(taskId);
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
if (!uploadComplete) {
if (canTreatTaskAsCompleted(task, row, job)) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
return;
}
attachPythonUploadProgress(vo, taskId);
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
return;
}
if (snapshot == null) {
return;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
if (total <= 0) {
return;
}
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(snapshot.getMessage());
}
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
TaskFileJobEntity job,
TaskProgressSnapshotEntity snapshot,
int llmCompleted,
int llmPending,
boolean uploadComplete) {
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
int total = calculateLlmDisplayTotal(snapshot, observed);
if (!uploadComplete) {
total = Math.max(total, observed + 1);
}
int current = Math.max(0, Math.min(llmCompleted, total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
vo.setFileProgressMessage(uploadComplete
? (llmPending > 0
? buildLlmProgressMessage(current, total, llmPending)
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
: buildUploadingLlmProgressMessage(current, total, llmPending));
}
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
int current = Math.max(0, Math.min(progress.current(), total));
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
}
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
if (chunkProgress.total() > 0) {
return chunkProgress;
}
if (chunkProgress.current() > 0) {
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
}
int totalRows = 0;
try {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
totalRows = allRowCount(task);
} catch (Exception ignored) {
totalRows = 0;
}
int uploadedRows = countSubmittedRows(taskId);
if (totalRows <= 0 && uploadedRows > 0) {
totalRows = uploadedRows + 1;
}
return new PythonUploadProgress(uploadedRows, totalRows, "行");
}
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
if (taskId == null || taskId <= 0) {
return new PythonUploadProgress(0, 0, "分片");
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getLlmStatus)
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
if (states == null || states.isEmpty()) {
return resolvePythonChunkProgressFromChunks(taskId);
}
int current = 0;
int total = 0;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
if (stateTotal > total || total <= 0 && stateCurrent > current) {
total = stateTotal;
current = stateCurrent;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return new PythonUploadProgress(0, 0, "分片");
}
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
Map<String, Integer> totalByScope = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
if (chunk == null) {
continue;
}
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
receivedByScope.merge(scopeHash, 1, Integer::sum);
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
}
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
}
}
int current = 0;
int total = 0;
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
current = scopeCurrent;
total = scopeTotal;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private int countSubmittedRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
int total = 0;
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return 0;
}
for (TaskChunkEntity chunk : chunks) {
total += readChunkRows(chunk).size();
}
return total;
}
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
}
private String resolveDisplayTaskStatus(FileTaskEntity task) { private String resolveDisplayTaskStatus(FileTaskEntity task) {
return resolveDisplayTaskStatus(task, null); return resolveDisplayTaskStatus(task, null);
} }
@@ -5416,36 +5083,6 @@ public class SimilarAsinTaskService {
return true; return true;
} }
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
int observed = Math.max(0, observedLlmStates);
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
return Math.max(1, observed);
}
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
return Math.max(1, Math.max(observed, expectedFromSnapshot));
}
private String buildLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (safeCompleted <= 0 && submitted > 0) {
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
}
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
}
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (pending > 0) {
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
}
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
}
private String fmt(LocalDateTime t) { private String fmt(LocalDateTime t) {
return t == null ? null : t.toString(); return t == null ? null : t.toString();
} }
@@ -0,0 +1,245 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
/**
* 任务 83:SimilarAsinExcelParser 行解析器。
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 SimilarAsinTaskService.parseWorkbook
* 中对应段落逐字节一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、错误值转空、
* 单字段截断 2000、表头别名匹配、空行跳过。
*/
@Slf4j
public class SimilarAsinExcelParser {
private static final int DEFAULT_MAX_FIELD_LENGTH = 2000;
public ParsedSheet parse(File input) {
return parse(input, DEFAULT_MAX_FIELD_LENGTH);
}
public ParsedSheet parse(File input, int maxFieldLength) {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (FileInputStream fis = new FileInputStream(input);
Workbook workbook = WorkbookFactory.create(fis)) {
return parseWorkbook(workbook, maxFieldLength);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[similar-asin] parse failed file={} err={}", input.getName(), ex.getMessage());
throw new BusinessException("解析 Excel 失败");
}
}
public ParsedSheet parse(InputStream input) {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (Workbook workbook = WorkbookFactory.create(input)) {
return parseWorkbook(workbook, DEFAULT_MAX_FIELD_LENGTH);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[similar-asin] parse failed input err={}", ex.getMessage());
throw new BusinessException("解析 Excel 失败");
}
}
private ParsedSheet parseWorkbook(Workbook workbook, int maxFieldLength) {
DataFormatter formatter = new DataFormatter();
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
List<SimilarAsinExcelRow> rows = new ArrayList<>();
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
String id = cell(row, idCol, formatter, maxFieldLength);
String asin = cell(row, asinCol, formatter, maxFieldLength).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter, maxFieldLength);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
rows.add(new SimilarAsinExcelRow(
i + 1,
id,
asin,
country,
skuCol >= 0 ? cell(row, skuCol, formatter, maxFieldLength) : "",
priceCol >= 0 ? cell(row, priceCol, formatter, maxFieldLength) : "",
urlCol >= 0 ? cell(row, urlCol, formatter, maxFieldLength) : "",
titleCol >= 0 ? cell(row, titleCol, formatter, maxFieldLength) : "",
readRowValues(row, headers, formatter, maxFieldLength)));
}
return new ParsedSheet(headers, rows);
}
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
Map<String, Integer> map = new LinkedHashMap<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
return map;
}
private List<String> readHeaders(Row header, DataFormatter formatter) {
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(val.isBlank() ? "列" + (i + 1) : val);
}
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter, int maxFieldLength) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter, maxFieldLength));
}
return values;
}
private int findRequiredHeader(Map<String, Integer> map, String... names) {
int idx = findOptionalHeader(map, names);
if (idx < 0) {
throw new BusinessException("缺少必要表头: " + String.join("/", names));
}
return idx;
}
private int findOptionalHeader(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
for (String name : names) {
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
return entry.getValue();
}
}
}
return -1;
}
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
for (String name : names) {
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
return entry.getValue();
}
}
}
return -1;
}
private String normalizeHeaderAlias(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
}
private String cell(Row row, int col, DataFormatter formatter, int maxFieldLength) {
if (col < 0) {
return "";
}
String value = normalize(formatter.formatCellValue(row.getCell(col)));
if (isSpreadsheetErrorValue(value)) {
return "";
}
if (value.length() > maxFieldLength) {
return value.substring(0, maxFieldLength);
}
return value;
}
private static boolean isSpreadsheetErrorValue(String value) {
String normalized = value == null ? "" : value.trim();
return normalized.startsWith("#") && normalized.endsWith("!");
}
private String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
public record ParsedSheet(List<String> headers, List<SimilarAsinExcelRow> rows) {
public ParsedSheet {
headers = headers == null ? List.of() : new ArrayList<>(headers);
rows = rows == null ? List.of() : new ArrayList<>(rows);
}
@Override
public List<String> headers() {
return java.util.Collections.unmodifiableList(headers);
}
@Override
public List<SimilarAsinExcelRow> rows() {
return java.util.Collections.unmodifiableList(rows);
}
}
public record SimilarAsinExcelRow(int rowIndex, String id, String asin, String country,
String sku, String price, String url, String title,
Map<String, String> values) {
public SimilarAsinExcelRow {
values = values == null ? Map.of() : new LinkedHashMap<>(values);
}
@Override
public boolean equals(Object o) {
if (!(o instanceof SimilarAsinExcelRow other)) {
return false;
}
return rowIndex == other.rowIndex
&& Objects.equals(id, other.id)
&& Objects.equals(asin, other.asin)
&& Objects.equals(country, other.country)
&& Objects.equals(sku, other.sku)
&& Objects.equals(price, other.price)
&& Objects.equals(url, other.url)
&& Objects.equals(title, other.title)
&& values.equals(other.values);
}
@Override
public int hashCode() {
return Objects.hash(rowIndex, id, asin, country, sku, price, url, title, values);
}
}
}
@@ -0,0 +1,78 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* 任务 86:SimilarAsinGroupingConverter 分组转换器。
* 归一化行 → 分组 DTO;分组规则与现状 SimilarAsinTaskService.buildParsedGroups 完全一致:
* 按 groupKey(缺省兜底 displayId 的 baseId)分组、startIndex/endIndex 半开区间游标、
* 组内行数、顺序保持行首次出现顺序。纯函数无状态。
*/
public final class SimilarAsinGroupingConverter {
private SimilarAsinGroupingConverter() {
}
public static List<SimilarAsinParsedGroupVo> convert(List<SimilarAsinParsedRowVo> rows) {
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
if (rows == null || rows.isEmpty()) {
return groups;
}
int cursor = 0;
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
if (siblings == null || siblings.isEmpty()) {
continue;
}
SimilarAsinParsedRowVo first = siblings.getFirst();
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setSourceFileKey(first.getSourceFileKey());
group.setSourceFilename(first.getSourceFilename());
group.setGroupKey(firstNonBlank(first.getGroupKey(),
buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
group.setBaseId(baseId(first.getDisplayId()));
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
group.setItemCount(siblings.size());
int end = cursor + siblings.size();
group.setStartIndex(cursor);
group.setEndIndex(end);
cursor = end;
groups.add(group);
}
return groups;
}
/** 行 → baseId 分组映射(LinkedHashMap 保序)。供 convert 及服务侧 loadAllRowsByBaseId 复用。 */
public static Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
for (SimilarAsinParsedRowVo row : rows) {
if (row == null) {
continue;
}
String key = firstNonBlank(SimilarAsinRowNormalizer.normalize(row.getGroupKey()),
baseId(row.getDisplayId()));
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
}
return result;
}
private static String baseId(String id) {
String s = SimilarAsinRowNormalizer.normalize(id);
int idx = s.indexOf('_');
return idx > 0 ? s.substring(0, idx) : s;
}
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
return SimilarAsinRowNormalizer.normalize(sourceFileKey) + "::"
+ SimilarAsinRowNormalizer.normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
}
private static String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
}
@@ -0,0 +1,757 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.extern.slf4j.Slf4j;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
/**
* 任务 91:SimilarAsinHistoryAssembler 历史查询组装器。
* 历史列表 VO 拼装(toHistoryItem + 进度链:file job 状态、LLM 回流进度、Python 上传进度、
* 快照百分比)从 SimilarAsinTaskService 原样搬移;只读不落库;输出与现状逐字段一致。
*/
@Slf4j
public class SimilarAsinHistoryAssembler {
private static final String MODULE_TYPE = "SIMILAR_ASIN";
private static final String STATUS_PENDING = "PENDING";
private static final String STATUS_RUNNING = "RUNNING";
private static final String STATUS_SUCCESS = "SUCCESS";
private static final String LLM_STATUS_SUBMITTED = "SUBMITTED";
private static final String LLM_STATUS_RUNNING = "RUNNING";
private static final String LLM_STATUS_DONE = "DONE";
private static final String LLM_STATUS_FAILED = "FAILED";
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskChunkMapper taskChunkMapper;
private final FileTaskMapper fileTaskMapper;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final OssStorageService ossStorageService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final ObjectMapper objectMapper;
public SimilarAsinHistoryAssembler(TaskScopeStateMapper taskScopeStateMapper,
TaskChunkMapper taskChunkMapper,
FileTaskMapper fileTaskMapper,
TaskProgressSnapshotService taskProgressSnapshotService,
OssStorageService ossStorageService,
TransientPayloadStorageService transientPayloadStorageService,
ObjectMapper objectMapper) {
this.taskScopeStateMapper = taskScopeStateMapper;
this.taskChunkMapper = taskChunkMapper;
this.fileTaskMapper = fileTaskMapper;
this.taskProgressSnapshotService = taskProgressSnapshotService;
this.ossStorageService = ossStorageService;
this.transientPayloadStorageService = transientPayloadStorageService;
this.objectMapper = objectMapper;
}
/**
* 按 rows 顺序组装历史项;PENDING 任务(history 语义)跳过。
* taskMap/jobMap 均为调用方已查好的映射,本方法只读不改输入。
*/
public List<SimilarAsinHistoryItemVo> buildHistoryItems(List<FileResultEntity> rows,
Map<Long, FileTaskEntity> taskMap,
Map<Long, TaskFileJobEntity> jobMap) {
List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
if (rows == null || rows.isEmpty()) {
return items;
}
for (FileResultEntity row : rows) {
FileTaskEntity task = taskMap.get(row.getTaskId());
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_PENDING.equals(taskStatus)) {
continue;
}
items.add(toHistoryItem(row, task, jobMap.get(row.getId())));
}
return items;
}
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
vo.setResultId(row.getId());
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(buildFreshDownloadUrl(row));
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
attachFileJobState(vo, row, job, task);
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
vo.setCreatedAt(fmt(row.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
}
private String buildFreshDownloadUrl(FileResultEntity row) {
String resultFileUrl = row == null ? null : row.getResultFileUrl();
if (resultFileUrl == null || resultFileUrl.isBlank()) {
return null;
}
try {
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
} catch (Exception ex) {
log.warn("[similar-asin] build public download url failed resultId={} err={}",
row.getId(), ex.getMessage());
return null;
}
}
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
attachFileProgress(vo, row, job, task);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
attachFileProgress(vo, row, job, task);
}
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
if (row == null || row.getTaskId() == null) {
return;
}
Long taskId = row.getTaskId();
if (Boolean.TRUE.equals(vo.getFileReady())) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
int llmCompleted = countCompletedLlmStates(taskId);
int llmPending = countPendingLlmStates(taskId);
boolean uploadComplete = isResultSubmissionComplete(taskId);
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
if (!uploadComplete) {
if (canTreatTaskAsCompleted(task, row, job)) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
return;
}
attachPythonUploadProgress(vo, taskId);
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
return;
}
if (snapshot == null) {
return;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
if (total <= 0) {
return;
}
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(snapshot.getMessage());
}
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
TaskFileJobEntity job,
TaskProgressSnapshotEntity snapshot,
int llmCompleted,
int llmPending,
boolean uploadComplete) {
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
int total = calculateLlmDisplayTotal(snapshot, observed);
if (!uploadComplete) {
total = Math.max(total, observed + 1);
}
int current = Math.max(0, Math.min(llmCompleted, total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
vo.setFileProgressMessage(uploadComplete
? (llmPending > 0
? buildLlmProgressMessage(current, total, llmPending)
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
: buildUploadingLlmProgressMessage(current, total, llmPending));
}
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
int current = Math.max(0, Math.min(progress.current(), total));
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
}
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
if (chunkProgress.total() > 0) {
return chunkProgress;
}
if (chunkProgress.current() > 0) {
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
}
int totalRows = 0;
try {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
totalRows = allRowCount(task);
} catch (Exception ignored) {
totalRows = 0;
}
int uploadedRows = countSubmittedRows(taskId);
if (totalRows <= 0 && uploadedRows > 0) {
totalRows = uploadedRows + 1;
}
return new PythonUploadProgress(uploadedRows, totalRows, "行");
}
private int allRowCount(FileTaskEntity task) {
try {
return rowCount(readParsedPayload(task));
} catch (Exception ex) {
log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage());
return 0;
}
}
private int rowCount(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return 0;
}
return resolveAllRows(payload).size();
}
private SimilarAsinParsedPayloadDto readParsedPayload(FileTaskEntity task) {
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
String pointer = root.path("parsedPayloadRef").asText("");
if (!pointer.isBlank()) {
String json = transientPayloadStorageService.resolvePayload(pointer, "read similar ASIN parsed payload failed");
return hydrateParsedPayloadRows(objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class));
}
if (root.path("allItems").isArray()) {
return hydrateParsedPayloadRows(objectMapper.treeToValue(root, SimilarAsinParsedPayloadDto.class));
}
} catch (Exception ex) {
throw new BusinessException("读取解析载荷失败");
}
return hydrateParsedPayloadRows(new SimilarAsinParsedPayloadDto());
}
private SimilarAsinParsedPayloadDto hydrateParsedPayloadRows(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return new SimilarAsinParsedPayloadDto();
}
List<SimilarAsinParsedRowVo> rows = resolveAllRows(payload);
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
payload.setAllItems(rows);
}
if (payload.getItems() == null || payload.getItems().isEmpty()) {
payload.setItems(rows);
}
return payload;
}
/**
* 从解析载荷统一恢复全量行:优先 items(新规范结构),其次 allItems(旧结构),
* 最后 groups 展开(最旧结构)。新旧结构均可恢复完整行集合,供 rowCount、
* LLM 候选加载与结果文件组装复用。
*/
private static List<SimilarAsinParsedRowVo> resolveAllRows(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return List.of();
}
if (containsGroupRefs(payload)) {
// 新格式:分组携带索引引用,信任引用展开结果(越界/非法区间安全跳过)
return expandGroupRefs(payload);
}
List<SimilarAsinParsedRowVo> rows = payload.getItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getAllItems();
}
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
rows = payload.getGroups().stream()
.filter(Objects::nonNull)
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
.toList();
}
return rows == null ? List.of() : rows;
}
private static boolean containsGroupRefs(SimilarAsinParsedPayloadDto payload) {
if (payload.getGroups() == null) {
return false;
}
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
if (group != null && (group.getStartIndex() != null || group.getEndIndex() != null)) {
return true;
}
}
return false;
}
/**
* 按分组引用 [startIndex, endIndex) 从 items 展开组内行。
* 引用越界或区间非法时安全跳过,不抛异常;展开不修改 payload 内部状态。
*/
private static List<SimilarAsinParsedRowVo> expandGroupRefs(SimilarAsinParsedPayloadDto payload) {
if (payload == null || payload.getGroups() == null || payload.getGroups().isEmpty()) {
return List.of();
}
List<SimilarAsinParsedRowVo> rows = payload.getItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getAllItems();
}
if (rows == null || rows.isEmpty()) {
return List.of();
}
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>();
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
if (group == null) {
continue;
}
int start = group.getStartIndex() == null ? 0 : group.getStartIndex();
int end = group.getEndIndex() == null ? 0 : group.getEndIndex();
if (start < 0 || end <= start || end > rows.size()) {
continue;
}
expanded.addAll(rows.subList(start, end));
}
return expanded;
}
private int countSubmittedRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
int total = 0;
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return 0;
}
for (TaskChunkEntity chunk : chunks) {
total += readChunkRows(chunk).size();
}
return total;
}
private Map<String, SimilarAsinResultRowDto> readChunkRows(TaskChunkEntity chunk) {
Map<String, SimilarAsinResultRowDto> rows = new LinkedHashMap<>();
if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) {
return rows;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read similar ASIN chunk failed");
JsonNode array = objectMapper.readTree(payloadJson);
if (!array.isArray()) {
return rows;
}
for (JsonNode node : array) {
SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class);
rows.put(rowKey(row), row);
}
} catch (Exception ex) {
String msg = ex.getMessage() == null ? "" : ex.getMessage();
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
boolean crossInstance = msg.contains("only exists on instance=");
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
recordChunkReadFailure(chunk, crossInstance, msg);
throw new BusinessException("similar ASIN chunk payload read failed chunk="
+ chunk.getChunkIndex() + ": " + msg, ex);
}
return rows;
}
/**
* 把 chunk 读失败信息聚合到 task_scope_state.last_error,
* 形式 "chunk-read-failed[{idx}]" / "chunk-read-failed[{idx}@cross-instance]"。
* best-effort:查询 / 更新失败时仅记日志,不抛回主流程。
*/
private void recordChunkReadFailure(TaskChunkEntity chunk, boolean crossInstance, String msg) {
if (chunk == null || chunk.getTaskId() == null || chunk.getScopeHash() == null) {
return;
}
try {
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, chunk.getTaskId())
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, chunk.getScopeHash())
.last("limit 1"));
if (scope == null) {
return;
}
String existing = scope.getLastError() == null ? "" : scope.getLastError();
String tag = "chunk-read-failed[" + chunk.getChunkIndex() + (crossInstance ? "@cross-instance" : "") + "]";
if (existing.contains(tag)) {
return;
}
String updated = existing.isBlank() ? tag : existing + "; " + tag;
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, scope.getId())
.set(TaskScopeStateEntity::getLastError, updated)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
} catch (Exception ignored) {
// best-effort:失败不影响主流程
}
}
private String rowKey(SimilarAsinResultRowDto row) {
if (row == null) {
return "";
}
String rowToken = normalize(row.getRowToken());
if (!rowToken.isBlank()) {
return rowToken;
}
return legacyRowKey(row);
}
private String legacyRowKey(SimilarAsinResultRowDto row) {
if (row == null) {
return "";
}
return rowKey(row.getId(), row.getAsin(), row.getCountry());
}
private String rowKey(String id, String asin, String country) {
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
private String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
if (taskId == null || taskId <= 0) {
return new PythonUploadProgress(0, 0, "分片");
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getLlmStatus)
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
if (states == null || states.isEmpty()) {
return resolvePythonChunkProgressFromChunks(taskId);
}
int current = 0;
int total = 0;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
if (stateTotal > total || total <= 0 && stateCurrent > current) {
total = stateTotal;
current = stateCurrent;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return new PythonUploadProgress(0, 0, "分片");
}
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
Map<String, Integer> totalByScope = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
if (chunk == null) {
continue;
}
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
receivedByScope.merge(scopeHash, 1, Integer::sum);
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
}
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
}
}
int current = 0;
int total = 0;
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
current = scopeCurrent;
total = scopeTotal;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private int resolveReceivedChunkProgress(Long taskId, String scopeHash, Integer chunkTotal) {
int receivedCount = countChunks(taskId, scopeHash);
Integer maxChunkIndex = taskChunkMapper.selectObjs(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getChunkIndex)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByDesc(TaskChunkEntity::getChunkIndex)
.last("limit 1"))
.stream()
.filter(Objects::nonNull)
.map(value -> {
if (value instanceof Number number) {
return number.intValue();
}
try {
return Integer.parseInt(value.toString());
} catch (Exception ignored) {
return 0;
}
})
.findFirst()
.orElse(0);
int current = Math.max(receivedCount, maxChunkIndex == null ? 0 : maxChunkIndex);
if (chunkTotal != null && chunkTotal > 0) {
current = Math.min(current, chunkTotal);
}
return Math.max(0, current);
}
private int countChunks(Long taskId, String scopeHash) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
return count == null ? 0 : count.intValue();
}
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
}
private String resolveDisplayTaskStatus(FileTaskEntity task, FileResultEntity row) {
if (task == null) {
return null;
}
if (canTreatTaskAsCompleted(task, row, null)) {
return STATUS_SUCCESS;
}
return task.getStatus();
}
private boolean canTreatTaskAsCompleted(FileTaskEntity task, FileResultEntity row, TaskFileJobEntity job) {
if (task == null) {
return false;
}
if (STATUS_SUCCESS.equals(task.getStatus())) {
return true;
}
if (!STATUS_RUNNING.equals(task.getStatus())) {
return false;
}
if (row == null) {
return false;
}
if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
return false;
}
if (job != null && "FAILED".equals(job.getStatus())) {
return false;
}
return true;
}
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
int observed = Math.max(0, observedLlmStates);
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
return Math.max(1, observed);
}
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
return Math.max(1, Math.max(observed, expectedFromSnapshot));
}
private String buildLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (safeCompleted <= 0 && submitted > 0) {
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
}
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
}
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (pending > 0) {
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
}
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
}
private int calculateDisplayProgressPercent(int current,
int total,
TaskFileJobEntity job,
LocalDateTime baseTime) {
if (total <= 0) {
return 0;
}
current = Math.max(0, Math.min(current, total));
int percent = Math.max(1, Math.min(99, (int) Math.floor(current * 100.0 / total)));
if (job != null && STATUS_RUNNING.equals(job.getStatus())) {
long elapsedSeconds = baseTime == null ? 0 : Math.max(0, Duration.between(baseTime, LocalDateTime.now()).getSeconds());
if (current <= 0) {
int firstRealProgressPercent = Math.max(1, Math.min(99, (int) Math.floor(100.0 / total)));
int waitingCap = Math.max(8, Math.min(35, firstRealProgressPercent - 1));
percent = Math.max(percent, Math.min(waitingCap, 8 + (int) (elapsedSeconds / 6)));
} else if (current < total) {
percent = Math.max(percent, Math.min(92, percent + (int) (elapsedSeconds / 10)));
}
}
return percent;
}
private int calculateSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot, TaskFileJobEntity job) {
if (snapshot == null) {
return 0;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
int percent = total <= 0 ? 0 : calculateDisplayProgressPercent(current, total, job, snapshot.getUpdatedAt());
return Math.max(percent, extractSnapshotDisplayPercent(snapshot));
}
private int extractSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot) {
if (snapshot == null || snapshot.getSnapshotJson() == null || snapshot.getSnapshotJson().isBlank()) {
return 0;
}
try {
JsonNode displayPercent = objectMapper.readTree(snapshot.getSnapshotJson()).path("displayPercent");
if (!displayPercent.isNumber()) {
return 0;
}
return Math.max(0, Math.min(100, displayPercent.asInt()));
} catch (Exception ignored) {
return 0;
}
}
private int countPendingLlmStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)));
return count == null ? 0 : count.intValue();
}
private int countCompletedLlmStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_DONE, LLM_STATUS_FAILED)));
return count == null ? 0 : count.intValue();
}
private boolean isResultSubmissionComplete(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getLlmStatus)
.isNotNull(TaskScopeStateEntity::getLastChunkAt));
if (states == null || states.isEmpty()) {
return false;
}
boolean hasCompletedScope = false;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
if (Integer.valueOf(1).equals(state.getCompleted())) {
hasCompletedScope = true;
continue;
}
return false;
}
return hasCompletedScope;
}
private String fmt(LocalDateTime t) {
return t == null ? null : t.toString();
}
private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private record PythonUploadProgress(int current, int total, String unit) {
}
}
@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import java.util.Locale;
/**
* 任务 85:SimilarAsinRowNormalizer 字段归一化器。
* 规则与 SimilarAsinTaskService.normalize 现状逐字节一致(BOM 剥离、全角空格转半角、
* trim、连续空白折叠为单空格);normalizeAsin = normalize + 大写。纯函数无状态。
*/
public final class SimilarAsinRowNormalizer {
private SimilarAsinRowNormalizer() {
}
public static String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
public static String normalizeAsin(String val) {
return normalize(val).toUpperCase(Locale.ROOT);
}
}
@@ -0,0 +1,66 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
/**
* 任务 87:SimilarAsinRowValidator 纯校验器。
* 必填检查(id/asin/country)、ASIN 格式校验(仅允许字母数字,含校验前大写化)、
* 重复检测(asin+country 键,保留首次出现)。返回错误清单不抛异常,由调用方决定处理。
* 规则与现状 parseWorkbook 必填丢弃 / dedupeRowsByRowKey 保留首次语义一致。
*/
public final class SimilarAsinRowValidator {
public record RowError(int rowIndex, String column, String message) {
}
private SimilarAsinRowValidator() {
}
public static List<RowError> validate(List<SimilarAsinParsedRowVo> rows) {
List<RowError> errors = new ArrayList<>();
if (rows == null || rows.isEmpty()) {
return errors;
}
Set<String> seenKeys = new java.util.HashSet<>();
for (SimilarAsinParsedRowVo row : rows) {
if (row == null) {
continue;
}
String id = SimilarAsinRowNormalizer.normalize(row.getSourceId());
String asin = SimilarAsinRowNormalizer.normalizeAsin(row.getAsin());
String country = SimilarAsinRowNormalizer.normalize(row.getCountry());
if (id.isEmpty() && asin.isEmpty() && country.isEmpty()) {
continue;
}
int rowIndex = row.getRowIndex() == null ? 0 : row.getRowIndex();
if (id.isEmpty()) {
errors.add(new RowError(rowIndex, "id", "缺少必要字段: id"));
}
if (asin.isEmpty()) {
errors.add(new RowError(rowIndex, "asin", "缺少必要字段: asin"));
}
if (country.isEmpty()) {
errors.add(new RowError(rowIndex, "country", "缺少必要字段: 国家"));
}
if (asin.isEmpty()) {
continue;
}
if (!asin.matches("[A-Z0-9]+")) {
errors.add(new RowError(rowIndex, "asin",
"ASIN 格式不正确(仅允许字母数字): " + asin));
continue;
}
String dupKey = asin + "::" + country;
if (!seenKeys.add(dupKey)) {
errors.add(new RowError(rowIndex, "asin",
"重复 ASIN: " + asin + " 国家: " + country));
}
}
return errors;
}
}
@@ -0,0 +1,112 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Font;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* 任务 88:SimilarAsinSheetBuilder Sheet 构造器。
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式);输入数据 → 输出 workbook;不落库、无 IO 依赖。
* 表头/列序/样式与现状 assembleResultWorkbook 一致:sheet 名"相似asin检测"、15 列表头、
* 加粗表头、数据行从第 1 行、图片列 12-14 列宽、行高 409pt、值兜底与现状一致。
* 结果派生列(是否有货/相似度/是否符合类目/不符合理由/产品类目/状态)来自 LLM 结果行,
* 本 builder 输入只有解析行,这些列由调用方补写。
*/
public final class SimilarAsinSheetBuilder {
public static final List<String> RESULT_HEADERS = List.of(
"id",
"asin",
"国家",
"价格",
"卖家名称",
"品牌",
"是否有货",
"相似度",
"是否符合类目",
"不符合理由",
"产品类目",
"状态",
"主图",
"阿里巴巴图片1",
"阿里巴巴图片2"
);
public static final int IMG_COL_MAIN = 12;
public static final int IMG_COL_PUZZLE1 = 13;
public static final int IMG_COL_PUZZLE2 = 14;
private SimilarAsinSheetBuilder() {
}
public static void build(Workbook workbook, List<SimilarAsinParsedRowVo> rows) {
Sheet sheet = workbook.createSheet("相似asin检测");
CellStyle headerStyle = workbook.createCellStyle();
Font font = workbook.createFont();
font.setBold(true);
headerStyle.setFont(font);
sheet.setColumnWidth(IMG_COL_MAIN, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
sheet.setColumnWidth(IMG_COL_PUZZLE1, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
sheet.setColumnWidth(IMG_COL_PUZZLE2, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
Row header = sheet.createRow(0);
for (int i = 0; i < RESULT_HEADERS.size(); i++) {
Cell cell = header.createCell(i);
cell.setCellValue(RESULT_HEADERS.get(i));
cell.setCellStyle(headerStyle);
}
if (rows == null) {
return;
}
int rowIndex = 1;
for (SimilarAsinParsedRowVo parsedRow : rows) {
if (parsedRow == null) {
continue;
}
Row row = sheet.createRow(rowIndex);
int col = 0;
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
row.createCell(col++).setCellValue(firstNonBlank(
parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")));
row.createCell(col++).setCellValue(readValueByHeader(
parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name",
"seller-name", "sellername", "store name", "shop name"));
row.createCell(col).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
rowIndex++;
}
}
private static String readValueByHeader(SimilarAsinParsedRowVo row, String... candidates) {
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
return "";
}
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
String header = SimilarAsinRowNormalizer.normalize(entry.getKey()).toLowerCase(Locale.ROOT);
for (String candidate : candidates) {
String expected = SimilarAsinRowNormalizer.normalize(candidate).toLowerCase(Locale.ROOT);
if (!expected.isBlank() && header.contains(expected)) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
}
return "";
}
private static String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
}
@@ -0,0 +1,27 @@
package com.nanri.aiimage.modules.task.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
/**
* 共享轻量进度查询请求(各模块 progress/light 端点共用)。
* 最多处理 200 个任务 ID,超长部分截断丢弃(轮询端点保持响应快)。
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@Schema(description = "任务轻量进度查询请求")
public class TaskProgressLightRequest {
@NotNull
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端批量查询,最多 200 个。",
example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds = new ArrayList<>();
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务,owner-scoped 模块使用)。", example = "1")
private Long userId;
}
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.task.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* 共享轻量进度批量响应:items 按请求顺序返回,查不到的任务进 missingTaskIds。
*/
@Data
@Schema(description = "任务轻量进度批量响应")
public class TaskProgressLightBatchVo {
@Schema(description = "按请求顺序返回的轻量进度项。")
private List<TaskProgressLightVo> items = new ArrayList<>();
@Schema(description = "查询不到的任务 ID 列表。")
private List<Long> missingTaskIds = new ArrayList<>();
}
@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.task.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
/**
* 共享轻量进度项(11 模块 progress/light 轮询专用)。
* 白名单键与 similarasin 的 SimilarAsinTaskLightVo 保持一致:
* taskId/status/statusCode?/fileStatus?/fileError?/fileReady?/updatedAt。
*/
@Data
@Schema(description = "任务轻量进度项(前端轮询专用,不含明细/result 内容)")
public class TaskProgressLightVo {
@Schema(description = "任务 ID。", example = "3938")
private Long taskId;
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
private String status;
@Schema(description = "业务状态码(预留字段,当前恒为 null)。", example = "null")
private String statusCode;
@Schema(description = "结果文件组装 Job 状态:SUCCESS/RUNNING/FAILED;无 Job 时为空。", example = "SUCCESS")
private String fileStatus;
@Schema(description = "文件组装失败时的错误信息;无错误时为空。", example = "组装失败: 文件写入超时")
private String fileError;
@Schema(description = "结果文件是否已就绪。", example = "true")
private Boolean fileReady;
@Schema(description = "最后更新时间,ISO 本地时间字符串。", example = "2026-04-26T10:05:00")
private String updatedAt;
}
@@ -531,6 +531,28 @@ public class TaskFileJobService {
return jobMap; return jobMap;
} }
/** 按任务 ID 集合一次 IN 查询 ASSEMBLE_RESULT Job,返回 taskId→Job 的 Map(轻量进度接口用)。 */
public Map<Long, TaskFileJobEntity> findAssembleJobsByTaskIds(String moduleType, List<Long> taskIds) {
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalizedIds.isEmpty() || moduleType == null || moduleType.isBlank()) {
return Map.of();
}
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.in(TaskFileJobEntity::getTaskId, normalizedIds));
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
for (TaskFileJobEntity job : jobs) {
if (job != null && job.getTaskId() != null) {
jobMap.putIfAbsent(job.getTaskId(), job);
}
}
return jobMap;
}
public long countUnfinishedAssembleJobs(Long taskId, String moduleType) { public long countUnfinishedAssembleJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) { if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return 0L; return 0L;
@@ -0,0 +1,76 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightVo;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* 共享轻量进度装配器:11 模块 progress/light 统一实现。
* 语义与 similarasin 的 progressLight 一致——只查任务行 + 一次 ASSEMBLE Job IN 批量查询;
* 空/重复/非正数过滤;超过 MAX_TASK_IDS 截断丢弃;缺失任务进 missingTaskIds;
* fileStatus/fileError/fileReady 由 Job 推导;状态值语义与各模块 progress/batch 一致。
* userId 非空时按用户过滤(publish 等 owner-scoped 模块使用)。
*/
@Component
@RequiredArgsConstructor
public class TaskProgressLightAssembler {
public static final int MAX_TASK_IDS = 200;
private final FileTaskMapper fileTaskMapper;
private final TaskFileJobService taskFileJobService;
public TaskProgressLightBatchVo assemble(String moduleType, Long userId, List<Long> taskIds) {
TaskProgressLightBatchVo vo = new TaskProgressLightBatchVo();
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.limit(MAX_TASK_IDS)
.toList();
if (normalizedIds.isEmpty()) {
return vo;
}
LambdaQueryWrapper<FileTaskEntity> query = new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getUpdatedAt)
.in(FileTaskEntity::getId, normalizedIds);
if (userId != null) {
query.eq(FileTaskEntity::getUserId, userId);
}
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
for (FileTaskEntity task : fileTaskMapper.selectList(query)) {
if (task != null && moduleType.equals(task.getModuleType())) {
taskMap.put(task.getId(), task);
}
}
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByTaskIds(moduleType, new ArrayList<>(taskMap.keySet()));
for (Long taskId : normalizedIds) {
FileTaskEntity task = taskMap.get(taskId);
if (task == null) {
vo.getMissingTaskIds().add(taskId);
continue;
}
TaskFileJobEntity job = jobMap.get(taskId);
TaskProgressLightVo item = new TaskProgressLightVo();
item.setTaskId(task.getId());
item.setStatus(task.getStatus());
item.setFileReady(job != null && job.getResultFileUrl() != null && !job.getResultFileUrl().isBlank());
item.setFileStatus(job == null ? null : job.getStatus());
item.setFileError(job == null ? null : job.getErrorMessage());
item.setUpdatedAt(task.getUpdatedAt() == null ? null : task.getUpdatedAt().toString());
vo.getItems().add(item);
}
return vo;
}
}
@@ -37,6 +37,8 @@ import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -107,6 +109,14 @@ public class WithdrawTaskController {
return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(withdrawTaskService.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks") @PostMapping("/tasks")
@Operation(summary = "创建取款任务", description = "为一批已匹配店铺创建一个取款任务,写入任务记录和店铺结果占位记录,并返回 taskId 及初始店铺快照。") @Operation(summary = "创建取款任务", description = "为一批已匹配店铺创建一个取款任务,写入任务记录和店铺结果占位记录,并返回 taskId 及初始店铺快照。")
public ApiResponse<WithdrawCreateTaskVo> createTask(@Valid @RequestBody WithdrawCreateTaskRequest request) { public ApiResponse<WithdrawCreateTaskVo> createTask(@Valid @RequestBody WithdrawCreateTaskRequest request) {
@@ -42,6 +42,8 @@ import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -49,6 +51,10 @@ import java.util.Objects;
public class WithdrawTaskService { public class WithdrawTaskService {
private static final String MODULE_TYPE = "WITHDRAW"; private static final String MODULE_TYPE = "WITHDRAW";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1; private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0; private static final int RESULT_FAILED = 0;
@@ -68,6 +74,7 @@ public class WithdrawTaskService {
private final TaskResultItemService taskResultItemService; private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
public ProductRiskDashboardVo dashboard(Long userId) { public ProductRiskDashboardVo dashboard(Long userId) {
validateUserId(userId); validateUserId(userId);
@@ -0,0 +1,31 @@
-- Task 116(依赖 115 审计):biz_task_file_job 增加 (module_type, job_type, task_id) 复合索引。
-- 背景:findAssembleJobsByTaskIds(progress/light)以 module_type + job_type 等值 + task_id IN 查询,
-- 现有 idx_file_job_task (task_id, module_type) 最左前缀不匹配,导致 index merge/大范围扫描。
-- 风险:ALTER TABLE ADD INDEX 需申请元数据锁(MySQL 8 在线 DDL,INSTANT/INPLACE),
-- 建议在低峰窗口(凌晨 02:00-06:00)执行;表若极大,评估 pt-online-schema-change。
-- 回滚:DROP INDEX idx_file_job_module_type_task ON biz_task_file_job(见文件末尾注释块)。
SET @db_name = DATABASE();
SET @idx_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name
AND TABLE_NAME = 'biz_task_file_job'
AND INDEX_NAME = 'idx_file_job_module_type_task'
);
SET @sql := IF(@idx_exists = 0,
'ALTER TABLE biz_task_file_job ADD INDEX idx_file_job_module_type_task (module_type, job_type, task_id)',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 验证 SQL:应返回 1 行(索引已存在)
-- SELECT COUNT(*) FROM information_schema.STATISTICS
-- WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'biz_task_file_job'
-- AND INDEX_NAME = 'idx_file_job_module_type_task';
-- 回滚(低峰执行):
-- ALTER TABLE biz_task_file_job DROP INDEX idx_file_job_module_type_task;
@@ -0,0 +1,4 @@
-- 分组管理:独立管理后台菜单项,放入「账号与权限」菜单组(sort_order 25)
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '分组管理', 'admin_group_manage', 'admin', 'group-manage', 25
WHERE NOT EXISTS (SELECT 1 FROM columns WHERE column_key = 'admin_group_manage');
@@ -0,0 +1,122 @@
package com.nanri.aiimage.config;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 115:高频查询 EXPLAIN 索引审计(文档产出验证)。
* 审计文档必须覆盖 history/progress/dashboard 三类高频查询的执行计划结论、
* 候选索引清单(表/列/类型/收益/风险)、全表扫描标记、可重复执行步骤,
* 且本任务明确不做 DDL(只审计,不加索引)。
*/
class ExplainIndexAuditDocTest {
private static final Path AUDIT = Paths.get("src", "main", "resources", "..", "..", "..",
"docs", "explain-index-audit.md").normalize();
private static final Path FLYWAY_SPEC = Paths.get("src", "main", "resources", "..", "..", "..",
"docs", "specs", "12-flyway-and-inspection.md").normalize();
private static String read(Path path) throws IOException {
return new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
}
private static List<String> linesOf(Path path) throws IOException {
return List.of(read(path).split("\r?\n"));
}
private static String lineContaining(String content, String needle) {
for (String line : content.split("\r?\n")) {
if (line.contains(needle)) {
return line;
}
}
return null;
}
@Test
void test_explain_history_plan() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("history"), "审计文档覆盖 history 查询");
assertTrue(doc.contains("EXPLAIN"), "审计文档包含 EXPLAIN 执行计划记录");
}
@Test
void test_explain_progress_plan() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("progress"), "审计文档覆盖 progress 查询");
}
@Test
void test_explain_dashboard_plan() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("dashboard"), "审计文档覆盖 dashboard 查询");
assertTrue(doc.contains("GROUP BY"), "dashboard 聚合查询计划已记录");
}
@Test
void test_full_scan_detected() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("type=ALL") || doc.contains("全表扫描") || doc.contains("ALL")
|| doc.contains("possible_keys") || doc.contains("扫描"),
"审计文档记录执行计划扫描类型(full scan 标记)");
String line = lineContaining(doc, "idx_file_job_task");
assertTrue(line != null && line.contains("task_id"),
"审计文档指出 task_id 前缀索引对 (module_type, job_type, task_id IN) 的局限");
}
@Test
void test_index_candidates_listed() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("idx_file_job_module_type_task")
|| doc.contains("module_type") && doc.contains("job_type") && doc.contains("task_id"),
"候选索引清单含 (module_type, job_type, task_id) 组合");
assertTrue(doc.contains("收益") && doc.contains("风险"),
"候选索引清单含收益与风险说明");
}
@Test
void test_plan_documented() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("biz_task_file_job"), "审计文档引用目标表");
assertTrue(doc.contains("biz_file_task") || doc.contains("biz_file_result"),
"审计文档引用业务任务/结果表");
}
@Test
void test_plan_repeatable() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("EXPLAIN SELECT") || doc.contains("EXPLAIN")
&& doc.contains("手工执行") || doc.contains("执行"),
"审计文档包含可重复执行的 EXPLAIN SQL 步骤");
}
@Test
void test_no_ddl_this_task() throws Exception {
String audit = read(AUDIT);
String spec = read(FLYWAY_SPEC);
assertTrue(audit.contains("不做 DDL") || audit.contains("本任务无 DDL")
|| audit.contains("不新增索引") || audit.contains("只审计"),
"审计文档明确本任务不做 DDL");
assertFalse(audit.contains("ALTER TABLE"),
"审计文档不包含 ALTER TABLE(DDL 属于后续任务 116)");
assertTrue(spec.contains("只读") || spec.contains("巡检"),
"12 spec 巡检报表模式与审计文档对应");
}
@Test
void test_audit_reflects_batch_loading_queries() throws Exception {
String doc = read(AUDIT);
assertTrue(doc.contains("findAssembleJobsByTaskIds"), "审计覆盖轻量进度 Job 批量查询");
assertTrue(doc.contains("findAssembleJobsByResultIds"), "审计覆盖结果列表 Job 批量查询");
assertTrue(doc.contains("selectMaps") || doc.contains("GROUP BY"), "审计覆盖 dashboard 聚合查询");
}
}
@@ -0,0 +1,132 @@
package com.nanri.aiimage.config;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 116:索引迁移 V{N+1}(审计清单落地)。
* 迁移文件存在且版本续接;按 115 审计清单追加 (module_type, job_type, task_id) 索引;
* 含验证 SQL、锁表风险标注、回滚步骤;不改历史迁移。
*/
class IndexMigrationV101Test {
private static final Path DB_DIR = Paths.get("src", "main", "resources", "db");
private static final Path MIGRATION = DB_DIR.resolve("V101__task_file_job_module_type_task_index.sql");
private static final Path AUDIT = Paths.get("src", "main", "resources", "..", "..", "..",
"docs", "explain-index-audit.md").normalize();
private static String read(Path path) throws IOException {
return new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
}
private static List<Path> allMigrations() throws IOException {
try (Stream<Path> stream = Files.list(DB_DIR)) {
return stream.filter(p -> p.getFileName().toString().matches("V\\d+__.*\\.sql"))
.sorted()
.toList();
}
}
private static int versionOf(Path path) {
String name = path.getFileName().toString();
return Integer.parseInt(name.substring(1, name.indexOf('_')));
}
@Test
void test_migration_file_exists() {
assertTrue(Files.isRegularFile(MIGRATION), "迁移文件必须存在: " + MIGRATION);
}
@Test
void test_migration_version_sequential() throws Exception {
List<Path> all = allMigrations();
assertTrue(!all.isEmpty(), "存在迁移文件");
int maxVersion = all.stream().mapToInt(IndexMigrationV101Test::versionOf).max().orElse(0);
assertTrue(maxVersion >= 100, "现有迁移最大版本 >= V100: " + maxVersion);
assertTrue(versionOf(MIGRATION) == 101, "新迁移版本号 V101 续接 V100 之后");
}
@Test
void test_migration_index_created() throws Exception {
String sql = read(MIGRATION);
assertTrue(sql.contains("biz_task_file_job"), "迁移作用于 biz_task_file_job");
assertTrue(sql.contains("idx_file_job_module_type_task"), "索引名与审计清单一致");
assertTrue(sql.contains("module_type") && sql.contains("job_type") && sql.contains("task_id"),
"索引列为 (module_type, job_type, task_id)");
assertTrue(sql.contains("ADD INDEX"), "使用 ADD INDEX");
}
@Test
void test_migration_conditional_repeatable() throws Exception {
String sql = read(MIGRATION);
assertTrue(sql.contains("information_schema.STATISTICS"), "通过 information_schema 判存在");
assertTrue(sql.contains("INDEX_NAME"), "按索引名判断");
assertTrue(sql.contains("PREPARE") && sql.contains("EXECUTE"), "条件执行(幂等,可重复运行)");
}
@Test
void test_migration_validated() throws Exception {
String sql = read(MIGRATION);
assertTrue(sql.contains("SELECT 1"), "验证 SQL(幂等分支)");
String audit = read(AUDIT);
assertTrue(audit.contains("idx_file_job_module_type_task"), "审计清单已列出该索引");
assertTrue(audit.contains("收益") && audit.contains("风险"), "审计清单含收益/风险");
}
@Test
void test_explain_improved() throws Exception {
String audit = read(AUDIT);
assertTrue(audit.contains("findAssembleJobsByTaskIds"), "审计覆盖 progress/light Job 查询");
assertTrue(audit.contains("执行后重跑") || audit.contains("前后计划")
|| audit.contains("对比"), "审计含迁移前后 EXPLAIN 对比步骤");
}
@Test
void test_migration_no_alter_legacy() throws Exception {
List<Path> all = allMigrations();
List<Path> others = all.stream()
.filter(p -> versionOf(p) != 101)
.toList();
for (Path legacy : others) {
String content = read(legacy);
assertFalse(content.contains("idx_file_job_module_type_task"),
"历史迁移不得包含新索引: " + legacy.getFileName());
}
}
@Test
void test_migration_rollback_doc() throws Exception {
String sql = read(MIGRATION);
assertTrue(sql.contains("DROP INDEX") || sql.contains("drop index"),
"迁移文件含回滚语句(DROP INDEX idx_file_job_module_type_task)");
String audit = read(AUDIT);
assertTrue(audit.contains("回滚") || audit.contains("DROP"),
"审计文档含回滚步骤说明");
}
@Test
void test_migration_lock_window() throws Exception {
String sql = read(MIGRATION);
assertTrue(sql.contains("LOCK") || sql.contains("低峰") || sql.contains("lock")
|| sql.contains("窗口"), "迁移标注锁表风险/窗口");
}
@Test
void test_migration_backward_safe() throws Exception {
String sql = read(MIGRATION);
assertTrue(sql.contains("ALTER TABLE"), "普通 BTREE 索引(不重建表、不删列)");
assertFalse(sql.contains("DROP COLUMN"), "不删列");
assertFalse(sql.contains("RENAME"), "不重命名");
assertTrue(sql.contains("ADD INDEX"), "仅追加索引,向后兼容");
}
}
@@ -0,0 +1,449 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentSheetBuilder;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentRowNormalizer;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.transaction.PlatformTransactionManager;
import java.io.File;
import java.lang.reflect.Method;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 任务 95:AppearancePatent 门面改委托。
* writeResultWorkbook/writeReasonSheet 改为委托 AppearancePatentSheetBuilder.buildResultSheet
* (门面保留 SXSSF 写出、异常包装与 rowToken 优先的 findResultRow 语义);
* submitResult 双事务路径与 owner 检查不动;Sheet 链专属私有方法从门面移除。
* 私有门面方法经反射直接验证,不经过完整 job 链路。
*/
class AppearancePatentTaskServiceDelegationTest {
private FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
private FileResultMapper fileResultMapper = mock(FileResultMapper.class);
private TaskScopeStateMapper taskScopeStateMapper = mock(TaskScopeStateMapper.class);
private TaskChunkMapper taskChunkMapper = mock(TaskChunkMapper.class);
private LocalFileStorageService localFileStorageService = mock(LocalFileStorageService.class);
private AppearancePatentLlmClient llmClient = mock(AppearancePatentLlmClient.class);
private AppearancePatentTaskCacheService taskCacheService = mock(AppearancePatentTaskCacheService.class); private AppearancePatentProperties properties = mock(AppearancePatentProperties.class);
private StorageProperties storageProperties = mock(StorageProperties.class);
private TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
private TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
private TransientPayloadStorageService transientPayloadStorageService = mock(TransientPayloadStorageService.class);
private PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
private DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class);
private TaskDistributedLockService taskDistributedLockService = mock(TaskDistributedLockService.class);
private InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
private AppearancePatentTaskService service;
private final AtomicInteger txCount = new AtomicInteger();
@BeforeEach
void setUp() throws Exception {
lenient().when(properties.getLlmBatchSize()).thenReturn(50);
// TransactionTemplate.execute 需要 getTransaction 返回非 null status;commit/rollback 默认为 no-op
lenient().when(transactionManager.getTransaction(any()))
.thenReturn(mock(org.springframework.transaction.TransactionStatus.class));
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
service = newService();
}
private AppearancePatentTaskService newService() {
return new AppearancePatentTaskService(
localFileStorageService, null, storageProperties, fileTaskMapper, fileResultMapper,
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
properties, taskFileJobService, taskProgressSnapshotService,
transientPayloadStorageService, transactionManager, distributedJobLockService,
taskDistributedLockService, instanceMetadata,
mock(TaskProgressLightAssembler.class));
}
private AppearancePatentTaskService serviceWithoutTransactionManager() {
return new AppearancePatentTaskService(
localFileStorageService, null, storageProperties, fileTaskMapper, fileResultMapper,
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
properties, taskFileJobService, taskProgressSnapshotService,
transientPayloadStorageService, null, distributedJobLockService,
taskDistributedLockService, instanceMetadata,
mock(TaskProgressLightAssembler.class));
}
// ---------- 1 签名不变 ----------
@Test
void test_write_result_workbook_signature_unchanged() throws Exception {
Method method = AppearancePatentTaskService.class.getDeclaredMethod(
"writeResultWorkbook",
File.class,
AppearancePatentParsedPayloadDto.class,
List.class,
Map.class);
assertEquals(void.class, method.getReturnType(), "返回类型不变");
assertEquals(4, method.getParameterCount(), "参数个数不变");
}
// ---------- 2 委托各组件 ----------
@Test
void test_write_result_workbook_delegates_sheet_builder() throws Exception {
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
AppearancePatentResultRowDto resultRow = resultRow("2_1", "B01A", "US", "f1.xlsx::row::2");
File out = new File("target/appearance-patent-delegation-tmp", "delegate.xlsx");
if (out.exists()) {
out.delete();
}
invokeWriteResultWorkbook(out, row, resultRow);
assertTrue(out.exists(), "委托后结果文件已生成");
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
Sheet main = wb.getSheet("外观专利检测结果");
assertNotNull(main, "主 sheet 由 SheetBuilder 产出");
Row data = main.getRow(1);
assertEquals("2_1", data.getCell(0).getStringCellValue());
assertEquals("B01A", data.getCell(1).getStringCellValue());
assertEquals("US", data.getCell(2).getStringCellValue());
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌经委托解析");
assertEquals("19.90", data.getCell(5).getStringCellValue(), "价格经委托解析");
assertEquals("无风险", data.getCell(9).getStringCellValue(), "标题维度经委托");
assertEquals("无风险", data.getCell(10).getStringCellValue(), "外观维度经委托");
assertEquals("已侵权", data.getCell(11).getStringCellValue());
assertEquals("成功", data.getCell(12).getStringCellValue(), "状态经委托 resolveResultStatus");
assertNotNull(wb.getSheet("原因"), "原因 sheet 由 SheetBuilder 产出");
}
}
@Test
void test_row_token_matching_preserved_in_delegated_builder() throws Exception {
// 关键行为:resultMap 键为 rowToken(service rowKey(row) 优先 token),
// 委托查找函数必须保持 rowToken 优先匹配(SheetBuilder 默认 legacy-only,门面必须传入自己的 findResultRow)
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
AppearancePatentResultRowDto resultRow = resultRow("2_1", "B01A", "US", "f1.xlsx::row::2");
// 注意:displayId/asin/country 与 resultRow 相同,若用 legacy key 也能匹配;
// 构造一个 legacy 不同但 rowToken 相同的用例验证 token 优先
row.setDisplayId("x_other");
row.setSourceId("x_other");
row.setAsin("B999");
row.setCountry("FR");
File out = new File("target/appearance-patent-delegation-tmp", "token.xlsx");
if (out.exists()) {
out.delete();
}
invokeWriteResultWorkbook(out, row, resultRow);
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("已侵权", data.getCell(11).getStringCellValue(),
"rowToken 命中 resultRow(legacy 键不匹配仍能解析结论)");
assertEquals("成功", data.getCell(12).getStringCellValue());
}
}
@Test
void test_delegation_without_result_rows_uses_parsed_fallback() throws Exception {
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
File out = new File("target/appearance-patent-delegation-tmp", "nofallback.xlsx");
if (out.exists()) {
out.delete();
}
invokeWriteResultWorkbook(out, row, null);
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌仍从解析行 values 回退");
assertEquals("", data.getCell(5).getStringCellValue(), "无结果行价格留空(resolvePrice 仅读结果行,与现状一致)");
}
}
// ---------- 3 结果一致(多行 + 原因 sheet 去重) ----------
@Test
void test_multiple_rows_and_reason_dedup() throws Exception {
AppearancePatentParsedRowVo p1 = parsedRow("2_1", "B01A", "US", "f1.xlsx");
AppearancePatentParsedRowVo p2 = parsedRow("2_2", "B01A", "DE", "f1.xlsx");
p2.setRowToken("f1.xlsx::row::3");
p2.setRowIndex(3);
AppearancePatentResultRowDto r1 = resultRow("2_1", "B01A", "US", "f1.xlsx::row::2");
r1.setAppearanceReason("外观理由");
r1.setPatentReason("专利理由");
r1.setTitleReason("标题理由");
File out = new File("target/appearance-patent-delegation-tmp", "multi.xlsx");
if (out.exists()) {
out.delete();
}
invokeWriteResultWorkbook(out, List.of(p1, p2), Map.of(tokenKey(r1), r1));
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
Sheet main = wb.getSheet("外观专利检测结果");
assertEquals(2, main.getLastRowNum(), "两行数据");
Sheet reason = wb.getSheet("原因");
assertEquals("B01A", reason.getRow(1).getCell(0).getStringCellValue());
assertEquals("外观理由", reason.getRow(1).getCell(1).getStringCellValue());
assertEquals("专利理由", reason.getRow(1).getCell(2).getStringCellValue());
assertEquals("标题理由", reason.getRow(1).getCell(3).getStringCellValue());
assertNull(reason.getRow(2), "同 ASIN 去重");
}
}
// ---------- 4 异常一致 ----------
@Test
void test_write_exception_wrapped_with_original_message() throws Exception {
// 写出目标为已存在目录 → FileOutputStream 抛异常 → 门面包成"生成外观专利检测结果失败"
File badDir = new File("target/appearance-patent-delegation-tmp", "bad-dir");
if (!badDir.exists()) {
badDir.mkdirs();
}
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
Throwable thrown = assertThrows(Throwable.class,
() -> invokeWriteResultWorkbook(badDir, row, null));
Throwable cause = thrown instanceof java.lang.reflect.InvocationTargetException
? ((java.lang.reflect.InvocationTargetException) thrown).getCause()
: thrown;
assertTrue(cause instanceof BusinessException, "实际异常类型: " + cause.getClass());
assertEquals("生成外观专利检测结果失败", cause.getMessage(), "异常包装一致");
}
// ---------- 5 双事务路径不动 ----------
@Test
void test_submit_result_dual_transaction_kept() throws Exception {
// transactionManager 非空 → submitResultLocked 走 REQUIRES_NEW 双事务(persist + complete)
FileTaskEntity t = runningTask();
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
when(taskChunkMapper.selectOne(any())).thenReturn(null);
when(taskScopeStateMapper.selectOne(any())).thenReturn(null);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
when(transientPayloadStorageService.storeChunkPayload(anyString(), any(), any(), any(), anyString()))
.thenReturn("transient:stored");
// 提交完成后的调度查询链
when(taskChunkMapper.selectList(any())).thenReturn(List.of());
when(fileResultMapper.selectList(any())).thenReturn(List.of());
service.submitResult(90001L, submitRequest("submission-x"));
verify(taskChunkMapper, Mockito.atLeast(1)).insert(any(TaskChunkEntity.class));
verify(transactionManager, Mockito.atLeast(2)).getTransaction(any());
}
@Test
void test_submit_result_keeps_task_status_check() throws Exception {
// 非 RUNNING 任务 → 双事务路径内 persist 仍抛"任务不是运行中状态"
FileTaskEntity t = task("{\"allItems\":[]}");
t.setStatus("SUCCESS");
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
BusinessException ex = assertThrows(BusinessException.class,
() -> service.submitResult(90001L, submitRequest("submission-status")));
assertTrue(ex.getMessage().contains("任务不是运行中状态"), "实际: " + ex.getMessage());
}
// ---------- 6 owner 检查不动 ----------
@Test
void test_owner_check_rejects_foreign_owner() throws Exception {
service = serviceWithoutTransactionManager();
FileTaskEntity t = task("{\"allItems\":[],\"ownerInstanceId\":\"server-121\"}");
t.setStatus("RUNNING");
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
assertThrows(com.nanri.aiimage.common.exception.TaskOwnerMismatchException.class,
() -> service.submitResult(90001L, submitRequest("submission-owner")), "owner 检查仍在手动路径生效");
}
@Test
void test_owner_check_passes_for_matching_owner() throws Exception {
service = serviceWithoutTransactionManager();
FileTaskEntity t = task("{\"allItems\":[],\"ownerInstanceId\":\"server-110\"}");
t.setStatus("RUNNING");
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
when(taskChunkMapper.selectOne(any())).thenReturn(null);
when(taskScopeStateMapper.selectOne(any())).thenReturn(null);
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
when(transientPayloadStorageService.storeChunkPayload(anyString(), any(), any(), any(), anyString()))
.thenReturn("transient:stored");
service.submitResult(90001L, submitRequest("submission-owner-ok"));
verify(taskChunkMapper, Mockito.atLeast(1)).insert(any(TaskChunkEntity.class));
verify(fileTaskMapper, Mockito.atLeast(1)).updateById(any(FileTaskEntity.class));
}
// ---------- 7 行值解析语义由 SheetBuilder 承接 ----------
@Test
void test_resolve_helpers_semantics_kept_in_sheet_builder() {
AppearancePatentParsedRowVo parsedRow = new AppearancePatentParsedRowVo();
Map<String, String> values = new LinkedHashMap<>();
values.put("品牌", "Source Brand");
parsedRow.setValues(values);
AppearancePatentResultRowDto resultRow = new AppearancePatentResultRowDto();
resultRow.setBrand("Python Brand");
assertEquals("Python Brand", AppearancePatentSheetBuilder.resolveBrand(resultRow, parsedRow));
resultRow.setBrand(" ");
assertEquals("Source Brand", AppearancePatentSheetBuilder.resolveBrand(resultRow, parsedRow));
assertEquals("", AppearancePatentSheetBuilder.resolvePrice(null));
assertEquals("失败", AppearancePatentSheetBuilder.resolveResultStatus(" "));
assertEquals("成功", AppearancePatentSheetBuilder.resolveResultStatus("已侵权"));
}
@Test
void test_llm_failure_semantics_kept_in_sheet_builder() {
assertTrue(AppearancePatentSheetBuilder.isTechnicalLlmFailure("coze 工作流节点执行超限"));
assertFalse(AppearancePatentSheetBuilder.isTechnicalLlmFailure("无风险"));
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setError("coze 调用超时");
row.setTitleRisk("coze 调用超时");
row.setConclusion("");
assertEquals("coze 调用超时", AppearancePatentSheetBuilder.userFacingLlmCellValue(row, row.getTitleRisk()));
assertEquals("成功", AppearancePatentSheetBuilder.userFacingStatus(row));
}
@Test
void test_normalizer_semantics_unchanged() {
assertEquals("", AppearancePatentRowNormalizer.normalize(null));
assertEquals("b01x", AppearancePatentRowNormalizer.normalize("b01x  "));
assertEquals("A", AppearancePatentRowNormalizer.firstNonBlank(null, "A"));
}
// ---------- 私有门面反射调用 ----------
private void invokeWriteResultWorkbook(File out, AppearancePatentParsedRowVo row,
AppearancePatentResultRowDto resultRow) throws Exception {
invokeWriteResultWorkbook(out, row == null ? List.of() : List.of(row),
resultRow == null ? Map.of() : Map.of(tokenKey(resultRow), resultRow));
}
private void invokeWriteResultWorkbook(File out, List<AppearancePatentParsedRowVo> rows,
Map<String, AppearancePatentResultRowDto> resultMap) throws Exception {
File parent = out.getParentFile();
if (parent != null && !parent.exists()) {
parent.mkdirs();
}
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
payload.setAllItems(rows);
Method method = AppearancePatentTaskService.class.getDeclaredMethod(
"writeResultWorkbook", File.class, AppearancePatentParsedPayloadDto.class, List.class, Map.class);
method.setAccessible(true);
method.invoke(service, out, payload, rows, resultMap);
}
private static String tokenKey(AppearancePatentResultRowDto row) {
return AppearancePatentRowNormalizer.normalize(row.getRowToken());
}
private static FileTaskEntity runningTask() {
FileTaskEntity t = task("{\"allItems\":[]}");
t.setStatus("RUNNING");
return t;
}
private static FileTaskEntity task(String resultJson) {
FileTaskEntity t = new FileTaskEntity();
t.setId(90001L);
t.setModuleType("APPEARANCE_PATENT");
t.setResultJson(resultJson);
t.setUserId(7L);
t.setUpdatedAt(java.time.LocalDateTime.now());
return t;
}
private static AppearancePatentSubmitResultRequest submitRequest(String submissionId) {
AppearancePatentSubmitResultRequest request = new AppearancePatentSubmitResultRequest();
request.setChunkIndex(0);
request.setChunkTotal(1);
request.setSubmissionId(submissionId);
request.setItems(List.of());
return request;
}
private static AppearancePatentParsedRowVo parsedRow(String id, String asin, String country, String fileKey) {
AppearancePatentParsedRowVo row = new AppearancePatentParsedRowVo();
row.setDisplayId(id);
row.setSourceId(id);
row.setAsin(asin);
row.setCountry(country);
row.setSourceFileKey(fileKey);
row.setRowToken(fileKey + "::row::" + 2);
row.setRowIndex(2);
row.setSourceFilename("delegation-source.xlsx");
Map<String, String> values = new LinkedHashMap<>();
values.put("卖家名称", "seller-A");
values.put("品牌", "brand-A");
values.put("价格", "19.90");
row.setValues(values);
return row;
}
private static AppearancePatentResultRowDto resultRow(String id, String asin, String country, String rowToken) {
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setId(id);
row.setAsin(asin);
row.setCountry(country);
row.setRowToken(rowToken);
row.setTitleRisk("无风险");
row.setAppearanceRisk("无风险");
row.setConclusion("已侵权");
row.setStatus("成功");
row.setBrand("brand-A");
row.setPrice("19.90");
return row;
}
}
@@ -0,0 +1,370 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.transaction.PlatformTransactionManager;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
/**
* 任务 106:appearancepatent 历史列表批量加载(同 105 模式)。
* history() 的任务/结果/Job 关联数据走 IN 批量查询 + 按 ID Map 装配,
* 不允许逐条 N+1:结果 1 次、任务 1 次、Job 1 次(恒定 3 次查询,与结果行数无关);
* 排序(priority → activityTime → createdAt → id)、分页、过滤、输出与现状一致。
*/
class AppearancePatentTaskServiceHistoryBatchTest {
private static final String MODULE = "APPEARANCE_PATENT";
private final List<FileResultEntity> resultDb = new ArrayList<>();
private final List<FileTaskEntity> taskDb = new ArrayList<>();
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
private final AtomicInteger resultSelectCount = new AtomicInteger();
private final AtomicInteger taskSelectCount = new AtomicInteger();
private final AtomicInteger jobSelectCount = new AtomicInteger();
private final List<Long> lastResultTaskIds = new ArrayList<>();
private FileTaskMapper fileTaskMapper;
private FileResultMapper fileResultMapper;
private TaskScopeStateMapper taskScopeStateMapper;
private TaskChunkMapper taskChunkMapper;
private LocalFileStorageService localFileStorageService;
private AppearancePatentLlmClient llmClient;
private AppearancePatentTaskCacheService taskCacheService;
private AppearancePatentProperties properties;
private StorageProperties storageProperties;
private TaskFileJobService taskFileJobService;
private TaskProgressSnapshotService taskProgressSnapshotService;
private TransientPayloadStorageService transientPayloadStorageService;
private PlatformTransactionManager transactionManager;
private DistributedJobLockService distributedJobLockService;
private TaskDistributedLockService taskDistributedLockService;
private InstanceMetadata instanceMetadata;
private AppearancePatentTaskService service;
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
List<Object> values = new ArrayList<>();
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
Map<String, Object> params = q.getParamNameValuePairs();
while (m.find()) {
Object value = params.get(m.group(1));
if (value != null) {
values.add(value);
}
}
return values;
}
@BeforeEach
void setUp() throws Exception {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskChunkEntity.class);
fileTaskMapper = mock(FileTaskMapper.class);
fileResultMapper = mock(FileResultMapper.class);
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
taskChunkMapper = mock(TaskChunkMapper.class);
localFileStorageService = mock(LocalFileStorageService.class);
llmClient = mock(AppearancePatentLlmClient.class);
taskCacheService = mock(AppearancePatentTaskCacheService.class);
properties = mock(AppearancePatentProperties.class);
storageProperties = mock(StorageProperties.class);
taskFileJobService = mock(TaskFileJobService.class);
taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
transientPayloadStorageService = mock(TransientPayloadStorageService.class);
transactionManager = mock(PlatformTransactionManager.class);
distributedJobLockService = mock(DistributedJobLockService.class);
taskDistributedLockService = mock(TaskDistributedLockService.class);
instanceMetadata = mock(InstanceMetadata.class);
// 结果表:按 wrapper 的 user/module/limit 过滤并保持 createdAt 倒序
doAnswer(invocation -> {
resultSelectCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
List<FileResultEntity> filtered = new ArrayList<>(resultDb);
Long userId = params.stream().filter(Long.class::isInstance).map(Long.class::cast).findFirst().orElse(null);
if (userId != null) {
filtered.removeIf(r -> !userId.equals(r.getUserId()));
}
if (segment.contains("module_type")) {
filtered.removeIf(r -> !MODULE.equals(r.getModuleType()));
}
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
lastResultTaskIds.clear();
lastResultTaskIds.addAll(filtered.stream().map(FileResultEntity::getTaskId).toList());
if (segment.contains("limit")) {
int cap = 50;
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
if (m.find()) {
cap = Integer.parseInt(m.group(1));
}
return filtered.subList(0, Math.min(cap, filtered.size()));
}
return filtered;
}).when(fileResultMapper).selectList(any());
// 任务表:IN 查询返回 id 命中集合
doAnswer(invocation -> {
taskSelectCount.incrementAndGet();
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
List<Long> wanted = new ArrayList<>();
for (Object value : params) {
if (value instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Long id) {
wanted.add(id);
}
}
}
}
if (wanted.isEmpty()) {
wanted.addAll(lastResultTaskIds);
}
if (wanted.isEmpty()) {
return List.of();
}
return taskDb.stream().filter(t -> wanted.contains(t.getId())).toList();
}).when(fileTaskMapper).selectList(any());
lenient().when(taskFileJobService.findAssembleJobsByResultIds(eq(MODULE), anyList())).thenAnswer(invocation -> {
jobSelectCount.incrementAndGet();
List<Long> resultIds = invocation.getArgument(1);
Map<Long, TaskFileJobEntity> map = new HashMap<>();
if (resultIds != null) {
for (TaskFileJobEntity job : jobDb) {
if (job.getResultId() != null && resultIds.contains(job.getResultId())) {
map.put(job.getResultId(), job);
}
}
}
return map;
});
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
lenient().when(taskProgressSnapshotService.find(any(), any())).thenReturn(null);
service = new AppearancePatentTaskService(
localFileStorageService, null, storageProperties, fileTaskMapper, fileResultMapper,
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
properties, taskFileJobService, taskProgressSnapshotService,
transientPayloadStorageService, transactionManager, distributedJobLockService,
taskDistributedLockService, instanceMetadata,
mock(TaskProgressLightAssembler.class));
}
private static FileResultEntity result(Long id, Long taskId, Long userId, LocalDateTime createdAt) {
FileResultEntity row = new FileResultEntity();
row.setId(id);
row.setTaskId(taskId);
row.setUserId(userId);
row.setModuleType(MODULE);
row.setSourceFilename("s" + id + ".xlsx");
row.setResultFilename("s" + id + "-result.xlsx");
row.setResultFileUrl("result/appearance-patent/" + id + "/out.xlsx");
row.setSuccess(1);
row.setRowCount(3);
row.setCreatedAt(createdAt);
return row;
}
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType(MODULE);
task.setStatus(status);
task.setCreatedAt(createdAt);
task.setUpdatedAt(createdAt);
task.setFinishedAt(createdAt.plusMinutes(5));
return task;
}
private static TaskFileJobEntity job(Long id, Long resultId, String status) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setResultId(resultId);
job.setModuleType(MODULE);
job.setStatus(status);
return job;
}
private void seed(int rows, int tasks, int jobs) {
for (int i = 1; i <= tasks; i++) {
taskDb.add(task(100L + i, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, i)));
}
for (int i = 1; i <= rows; i++) {
resultDb.add(result(200L + i, 100L + (i % tasks) + 1, 1L, LocalDateTime.of(2026, 8, 1, 10, i)));
}
for (int i = 1; i <= jobs; i++) {
jobDb.add(job(300L + i, 200L + i, "SUCCESS"));
}
}
@Test
void historyBatchLoads() throws Exception {
seed(5, 3, 3);
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(5, vo.getItems().size(), "5 条结果全部装配");
for (AppearancePatentHistoryItemVo item : vo.getItems()) {
assertNotNull(item.getTaskStatus(), "task 状态从批量任务 Map 装配");
assertEquals("SUCCESS", item.getFileStatus(), "job 状态从批量 Job Map 装配");
assertTrue(item.getFileReady(), "结果文件就绪");
}
}
@Test
void historySingleRow() throws Exception {
seed(1, 1, 1);
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(1, vo.getItems().size());
assertEquals(201L, vo.getItems().getFirst().getResultId());
assertEquals("SUCCESS", vo.getItems().getFirst().getTaskStatus());
}
@Test
void historyMultipleTasks() throws Exception {
seed(4, 4, 4);
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(4, vo.getItems().size());
long distinctTasks = vo.getItems().stream().map(AppearancePatentHistoryItemVo::getTaskId).distinct().count();
assertEquals(4, distinctTasks, "多个任务分别按 ID Map 装配");
}
@Test
void historyNoJobs() throws Exception {
seed(3, 3, 0);
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(3, vo.getItems().size());
for (AppearancePatentHistoryItemVo item : vo.getItems()) {
assertNull(item.getFileJobId(), "无 Job 时不附加 jobId");
assertEquals("SUCCESS", item.getFileStatus(), "文件就绪无 Job 状态为 SUCCESS");
}
}
@Test
void historyOrderUnchanged() throws Exception {
seed(3, 1, 0);
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(List.of(203L, 202L, 201L),
vo.getItems().stream().map(AppearancePatentHistoryItemVo::getResultId).toList(),
"createdAt 倒序保持");
}
@Test
void historyPagination() throws Exception {
seed(12, 1, 0);
AppearancePatentHistoryVo vo = service.history(1L, 5);
assertEquals(5, vo.getItems().size(), "limit 5 只取前 5");
assertEquals(List.of(212L, 211L, 210L, 209L, 208L),
vo.getItems().stream().map(AppearancePatentHistoryItemVo::getResultId).toList());
}
@Test
void historyMissingRelated() throws Exception {
resultDb.add(result(201L, 9999L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
resultDb.add(result(202L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(2, vo.getItems().size());
assertNull(vo.getItems().getFirst().getTaskStatus(), "关联任务缺失时状态为 null");
assertEquals("SUCCESS", vo.getItems().get(1).getTaskStatus(), "有关联任务的正常装配");
}
@Test
void historyActiveRunningPrioritized() throws Exception {
// 结果 201 关联 RUNNING 任务(活跃优先),202 关联 SUCCESS 任务(已完成)
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
resultDb.add(result(202L, 1002L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
taskDb.add(task(1001L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 0)));
taskDb.add(task(1002L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1)));
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(2, vo.getItems().size());
assertEquals(201L, vo.getItems().getFirst().getResultId(), "RUNNING 任务记录排前");
assertEquals(202L, vo.getItems().get(1).getResultId());
}
@Test
void historyUserFiltered() throws Exception {
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
resultDb.add(result(202L, 1002L, 2L, LocalDateTime.of(2026, 8, 1, 10, 1)));
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
taskDb.add(task(1002L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1)));
AppearancePatentHistoryVo vo = service.history(2L, 50);
assertEquals(1, vo.getItems().size(), "只返回当前用户结果");
assertEquals(202L, vo.getItems().getFirst().getResultId());
}
@Test
void historySqlCountConstant() throws Exception {
seed(20, 4, 20);
service.history(1L, 50);
verify(fileResultMapper, times(1)).selectList(any());
verify(fileTaskMapper, times(1)).selectList(any());
verify(taskFileJobService, times(1)).findAssembleJobsByResultIds(eq(MODULE), anyList());
assertEquals(1, resultSelectCount.get(), "结果 1 次 IN 查询");
assertEquals(1, taskSelectCount.get(), "任务 1 次 IN 查询");
assertEquals(1, jobSelectCount.get(), "Job 1 次 IN 查询");
assertEquals(20, resultSelectCount.get() * 20, "查询次数与结果行数无关(无 N+1)");
}
}
@@ -1,11 +1,7 @@
package com.nanri.aiimage.modules.appearancepatent.service; package com.nanri.aiimage.modules.appearancepatent.service;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.util.LinkedHashMap;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
class AppearancePatentTaskServiceTest { class AppearancePatentTaskServiceTest {
@@ -16,41 +12,4 @@ class AppearancePatentTaskServiceTest {
assertEquals("SUCCESS", AppearancePatentTaskService.resolveTaskExecutionStatus(false, false)); assertEquals("SUCCESS", AppearancePatentTaskService.resolveTaskExecutionStatus(false, false));
assertEquals("FAILED", AppearancePatentTaskService.resolveTaskExecutionStatus(false, true)); assertEquals("FAILED", AppearancePatentTaskService.resolveTaskExecutionStatus(false, true));
} }
@Test
void resultStatusFailsOnlyWhenConclusionIsEmpty() {
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(null));
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(" "));
assertEquals("\u6210\u529f", AppearancePatentTaskService.resolveResultStatus("\u4fb5\u6743"));
assertEquals("\u6210\u529f", AppearancePatentTaskService.resolveResultStatus("\u65e0\u4fb5\u6743"));
}
@Test
void resultBrandPrefersPythonThenFallsBackToSourceFile() {
AppearancePatentParsedRowVo parsedRow = new AppearancePatentParsedRowVo();
parsedRow.setValues(new LinkedHashMap<>());
parsedRow.getValues().put("品牌", "Source Brand");
AppearancePatentResultRowDto resultRow = new AppearancePatentResultRowDto();
resultRow.setBrand("Python Brand");
assertEquals("Python Brand", AppearancePatentTaskService.resolveBrand(resultRow, parsedRow));
resultRow.setBrand(" ");
assertEquals("Source Brand", AppearancePatentTaskService.resolveBrand(resultRow, parsedRow));
parsedRow.getValues().clear();
assertEquals("", AppearancePatentTaskService.resolveBrand(resultRow, parsedRow));
}
@Test
void resultPriceUsesOnlyPythonSubmittedValue() {
assertEquals("", AppearancePatentTaskService.resolvePrice(null));
AppearancePatentResultRowDto resultRow = new AppearancePatentResultRowDto();
resultRow.setPrice(" 12.99 ");
assertEquals("12.99", AppearancePatentTaskService.resolvePrice(resultRow));
resultRow.setPrice(null);
assertEquals("", AppearancePatentTaskService.resolvePrice(resultRow));
}
} }
@@ -0,0 +1,167 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 93:AppearancePatentExcelParser 行解析器。
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 AppearancePatentTaskService.parseWorkbook
* 对应段落一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、表头别名匹配、空行跳过、
* 必填表头缺失抛错。注意:appearancepatent 无 2000 截断、无错误值转空(与 similarasin 不同)。
*/
class AppearancePatentExcelParserTest {
private static final String[] REQUIRED = {"id", "asin", "国家"};
private static File workbook(String[] headers, List<String[]> rows) throws Exception {
File file = File.createTempFile("appearance-patent-parse-", ".xlsx");
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream out = new FileOutputStream(file)) {
Sheet sheet = wb.createSheet("Sheet1");
Row header = sheet.createRow(0);
for (int i = 0; i < headers.length; i++) {
header.createCell(i).setCellValue(headers[i]);
}
for (int r = 0; r < rows.size(); r++) {
Row row = sheet.createRow(r + 1);
for (int c = 0; c < rows.get(r).length; c++) {
row.createCell(c).setCellValue(rows.get(r)[c]);
}
}
wb.write(out);
}
return file;
}
private static final String[] FULL_HEADERS = {"id", "asin", "国家", "价格", "seller sku", "图片链接", "标题"};
@Test
void test_parser_normal_rows() throws Exception {
File file = workbook(FULL_HEADERS, List.of(
new String[]{"2_1", "b01a", "US", "19.90", "SKU-1", "http://img/a.jpg", "title a"},
new String[]{"2_2", "b01b", "DE", "29.90", "SKU-2", "http://img/b.jpg", "title b"}));
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
assertEquals(7, parsed.headers().size(), "表头 7 列");
assertEquals(2, parsed.rows().size());
AppearancePatentExcelParser.AppearanceExcelRow first = parsed.rows().get(0);
assertEquals(2, first.rowIndex(), "行号从 2 开始(表头占 1)");
assertEquals("2_1", first.id());
assertEquals("B01A", first.asin(), "asin 归一化后大写");
assertEquals("US", first.country());
assertEquals("19.90", first.price());
assertEquals("SKU-1", first.sku());
assertEquals("http://img/a.jpg", first.url());
assertEquals("title a", first.title());
assertEquals("19.90", first.values().get("价格"), "values 按表头键取值");
}
@Test
void test_parser_blank_row_skipped() throws Exception {
File file = workbook(FULL_HEADERS, List.of(
new String[]{"", "", ""},
new String[]{"1", "B01X", "US"},
new String[]{"", "", ""}));
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
assertEquals(1, parsed.rows().size(), "全空行跳过");
assertEquals("1", parsed.rows().get(0).id());
}
@Test
void test_parser_empty_sheet() throws Exception {
File file = workbook(REQUIRED, List.of());
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
assertTrue(parsed.rows().isEmpty(), "无数据行返回空");
assertEquals(3, parsed.headers().size());
}
@Test
void test_parser_header_aliases() throws Exception {
// 国家用 country、sku 用 merchant_sku、url 用 主图链接 均可识别
File file = workbook(new String[]{"id", "asin", "country", "merchant_sku", "主图链接"},
List.<String[]>of(new String[]{"1", "B01X", "FR", "MS-1", "http://img/x.jpg"}));
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
assertEquals(1, parsed.rows().size());
AppearancePatentExcelParser.AppearanceExcelRow row = parsed.rows().get(0);
assertEquals("FR", row.country());
assertEquals("MS-1", row.sku());
assertEquals("http://img/x.jpg", row.url());
}
@Test
void test_parser_missing_required_header_throws() throws Exception {
File file = workbook(new String[]{"id", "asin"}, List.<String[]>of(new String[]{"1", "B01X"}));
BusinessException ex = assertThrows(BusinessException.class,
() -> new AppearancePatentExcelParser().parse(file));
assertTrue(ex.getMessage().contains("缺少必要表头"), "必填表头缺失抛错,实际: " + ex.getMessage());
}
@Test
void test_parser_empty_header_row_throws() throws Exception {
File file = workbook(new String[]{}, List.of());
BusinessException ex = assertThrows(BusinessException.class,
() -> new AppearancePatentExcelParser().parse(file));
assertTrue(ex.getMessage().contains("表头为空") || ex.getMessage().contains("缺少必要表头"),
"表头缺失抛错,实际: " + ex.getMessage());
}
@Test
void test_parser_cell_normalization() throws Exception {
File file = workbook(FULL_HEADERS, List.<String[]>of(
new String[]{"2", "b01x", " US ", " 19.90 ", " SKU 1 ", " http://img/x.jpg ", " t a "}));
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
AppearancePatentExcelParser.AppearanceExcelRow row = parsed.rows().get(0);
assertEquals("2", row.id(), "BOM 剥离");
assertEquals("B01X", row.asin(), "大写 + trim");
assertEquals("US", row.country(), "trim");
assertEquals("SKU 1", row.sku(), "全角空格转半角 + 折叠");
assertEquals("http://img/x.jpg", row.url(), "url trim");
assertEquals("t a", row.title(), "连续空白折叠");
}
@Test
void test_parser_no_2000_truncation() throws Exception {
// appearancepatent 无字段截断(与 similarasin 不同):超长字段原样保留
String longSku = "S".repeat(5000);
File file = workbook(FULL_HEADERS, List.<String[]>of(new String[]{"1", "B01X", "US", "", longSku, "", ""}));
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
assertEquals(longSku, parsed.rows().get(0).sku(), "超长字段不截断");
assertEquals(5000, parsed.rows().get(0).sku().length());
}
@Test
void test_parser_header_fallback_names() throws Exception {
// 空表头列名回退为 "列N";重复列取首列
File file = workbook(new String[]{"id", "", "asin", "asin", "国家"}, List.<String[]>of(new String[]{"1", "x", "B01A", "B01B", "US"}));
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
assertEquals("列2", parsed.headers().get(1), "空表头回退列N");
assertEquals("asin", parsed.headers().get(2));
assertEquals("asin", parsed.headers().get(3), "重复表头保留");
assertEquals("B01A", parsed.rows().get(0).asin(), "重复列取首个匹配列");
}
}
@@ -0,0 +1,221 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 任务 97:AppearancePatentHistoryAssembler 历史查询组装器。
* 历史列表 VO 拼装(toHistoryItem + 进度链)抽到独立组件;只读不落库;输出与现状一致。
*/
@ExtendWith(MockitoExtension.class)
class AppearancePatentHistoryAssemblerTest {
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
private AppearancePatentHistoryAssembler assembler;
@BeforeEach
void setUp() {
assembler = new AppearancePatentHistoryAssembler(
taskProgressSnapshotService,
(current, total, job, baseTime) -> current <= 0 ? 1 : Math.min(99, current * 100 / 2),
snapshot -> 42);
lenient().when(taskProgressSnapshotService.find(anyLong(), anyString())).thenReturn(null);
}
private static FileResultEntity result(Long id, Long taskId, String source, String resultFileUrl, Integer success) {
FileResultEntity row = new FileResultEntity();
row.setId(id);
row.setTaskId(taskId);
row.setSourceFilename(source);
row.setResultFilename(source == null ? null : source.replace(".xlsx", "-result.xlsx"));
row.setResultFileUrl(resultFileUrl);
row.setSuccess(success);
row.setErrorMessage(null);
row.setRowCount(12);
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
return row;
}
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setStatus(status);
task.setCreatedAt(createdAt);
task.setUpdatedAt(createdAt);
task.setFinishedAt(createdAt == null ? null : createdAt.plusMinutes(5));
return task;
}
private static TaskFileJobEntity job(Long id, String status) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setStatus(status);
job.setErrorMessage(null);
job.setUpdatedAt(LocalDateTime.of(2026, 8, 1, 9, 30));
return job;
}
@Test
void test_assembler_history_items() {
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(
result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1),
task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)),
null);
assertEquals(100L, item.getResultId());
assertEquals(10L, item.getTaskId());
assertEquals("a.xlsx", item.getSourceFilename());
assertEquals("a-result.xlsx", item.getResultFilename());
assertNull(item.getDownloadUrl(), "appearancepatent 无下载 URL 生成");
assertEquals("SUCCESS", item.getTaskStatus());
assertEquals(Boolean.TRUE, item.getSuccess(), "文件 URL 就绪即成功");
assertEquals(12, item.getRowCount());
assertEquals("2026-08-01T10:00", item.getCreatedAt());
assertEquals("2026-08-01T09:00", item.getStartedAt(), "任务开始时间复用 task.createdAt");
assertEquals("2026-08-01T09:05", item.getFinishedAt(), "任务结束时间取 task.finishedAt");
}
@Test
void test_assembler_row_with_null_fields() {
FileResultEntity row = result(100L, 10L, null, null, null);
row.setCreatedAt(null);
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, null, null);
assertNull(item.getSourceFilename(), "空文件名保留");
assertEquals(Boolean.FALSE, item.getSuccess(), "无文件 URL 且 success 为空视为失败");
assertNull(item.getStartedAt(), "task 与 createdAt 均缺时无开始时间");
assertFalse(Boolean.TRUE.equals(item.getFileReady()), "文件未就绪");
}
@Test
void test_assembler_file_state() {
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "SUCCESS", null), job(7L, "RUNNING"));
assertTrue(item.getFileReady(), "结果文件就绪");
assertEquals(7L, item.getFileJobId());
assertEquals("RUNNING", item.getFileStatus());
assertEquals(Integer.valueOf(100), item.getFileProgressPercent(), "就绪即 100%");
assertEquals("结果文件已生成", item.getFileProgressMessage());
}
@Test
void test_assembler_order_priority() {
// RUNNING 任务优先(priority 0)于 SUCCESS 且文件就绪(priority 1)
FileResultEntity pending = result(200L, 2L, "b.xlsx", null, 0);
FileResultEntity done = result(100L, 1L, "a.xlsx", "result/1/a.xlsx", 1);
assertEquals(0, assembler.historyPriority(pending, task(2L, "RUNNING", null), null));
assertEquals(1, assembler.historyPriority(done, task(1L, "SUCCESS", null), null));
// SUCCESS 但文件未生成 → 构建中 → priority 0
FileResultEntity building = result(300L, 3L, "c.xlsx", null, 0);
assertEquals(0, assembler.historyPriority(building, task(3L, "SUCCESS", null), job(5L, "RUNNING")));
// 活动时间取 latestTime(row.createdAt 10:00 最晚)
assertEquals(LocalDateTime.of(2026, 8, 1, 10, 0),
assembler.historyActivityTime(done, task(1L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)), job(3L, "RUNNING")));
}
@Test
void test_assembler_file_building() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
assertTrue(assembler.isHistoryFileBuilding(row, "SUCCESS", job(3L, "RUNNING")), "SUCCESS 任务文件未生成视为构建中");
assertFalse(assembler.isHistoryFileBuilding(row, "SUCCESS", job(4L, "FAILED")), "job 失败不算构建中");
assertFalse(assembler.isHistoryFileBuilding(row, "FAILED", null), "非 SUCCESS 任务不算构建中");
row.setResultFileUrl("result/10/a.xlsx");
assertFalse(assembler.isHistoryFileBuilding(row, "SUCCESS", null), "文件就绪不算构建中");
}
@Test
void test_assembler_null_task() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 1);
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 11, 0));
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, null, null);
assertNull(item.getTaskStatus(), "缺 task 状态为空");
assertEquals("2026-08-01T11:00", item.getStartedAt(), "缺 task 回退 result.createdAt");
assertNull(item.getFinishedAt(), "缺 task 无结束时间");
assertNull(item.getFileStatus(), "无 job 且文件未就绪时状态为空");
}
@Test
void test_assembler_file_error_attached() {
TaskFileJobEntity failedJob = job(9L, "FAILED");
failedJob.setErrorMessage("assemble boom");
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "RUNNING", null), failedJob);
assertEquals("FAILED", item.getFileStatus(), "job 失败状态附带");
assertEquals("assemble boom", item.getFileError(), "job 错误信息附带");
}
@Test
void test_assembler_snapshot_progress() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
TaskProgressSnapshotEntity snapshot = new TaskProgressSnapshotEntity();
snapshot.setTotalCount(10);
snapshot.setSuccessCount(5);
snapshot.setMessage("LLM 处理中");
when(taskProgressSnapshotService.find(10L, "APPEARANCE_PATENT")).thenReturn(snapshot);
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "RUNNING", null), null);
assertEquals(Integer.valueOf(5), item.getFileProgressCurrent(), "快照 current 附带");
assertEquals(Integer.valueOf(10), item.getFileProgressTotal(), "快照 total 附带");
assertEquals(Integer.valueOf(99), item.getFileProgressPercent(), "百分比 = max(注入计算值 99, 注入提取值 42)");
assertEquals("LLM 处理中", item.getFileProgressMessage());
verify(taskProgressSnapshotService).find(10L, "APPEARANCE_PATENT");
}
@Test
void test_assembler_immutable_input() {
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
FileTaskEntity t = task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0));
assembler.toHistoryItem(row, t, null);
assertEquals("a.xlsx", row.getSourceFilename(), "result 不被修改");
assertEquals("SUCCESS", t.getStatus(), "task 不被修改");
}
@Test
void test_assembler_consistency() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
row.setErrorMessage("python timeout");
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 30)), null);
assertEquals(Boolean.FALSE, item.getSuccess(), "无文件 URL 且 success=0 为失败");
assertEquals("python timeout", item.getError());
assertEquals("FAILED", item.getTaskStatus());
assertEquals(12, item.getRowCount());
}
}
@@ -0,0 +1,74 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 93:AppearancePatentRowNormalizer 字段归一化器。
* 规则与 AppearancePatentTaskService.normalize / firstNonBlank / baseId / normalizeDisplayId
* 现状逐字节一致:BOM 剥离、全角空格转半角、trim、连续空白折叠、firstNonBlank 取首非空。
*/
class AppearancePatentRowNormalizerTest {
@Test
void test_normalize_null_returns_empty() {
assertEquals("", AppearancePatentRowNormalizer.normalize(null), "null 归一化为空串");
}
@Test
void test_normalize_bom_stripped() {
assertEquals("B01X", AppearancePatentRowNormalizer.normalize("B01X"), "BOM 剥离");
}
@Test
void test_normalize_fullwidth_space_to_halfwidth() {
assertEquals("SKU 1", AppearancePatentRowNormalizer.normalize("SKU 1"), "全角空格转半角");
}
@Test
void test_normalize_trim_and_collapse_whitespace() {
assertEquals("a b c", AppearancePatentRowNormalizer.normalize(" a\t b \nc "), "trim + 连续空白折叠为单空格");
}
@Test
void test_first_non_blank_prefers_first() {
assertEquals("preferred", AppearancePatentRowNormalizer.firstNonBlank("preferred", "fallback"));
assertEquals("fallback", AppearancePatentRowNormalizer.firstNonBlank(" ", "fallback"), "首选空白回退");
assertEquals("fallback", AppearancePatentRowNormalizer.firstNonBlank(null, "fallback"), "首选 null 回退");
assertEquals("kept", AppearancePatentRowNormalizer.firstNonBlank(" kept ", "fallback"), "结果去首尾空白");
}
@Test
void test_base_id_splits_underscore_block() {
assertEquals("2", AppearancePatentRowNormalizer.baseId("2_1"), "块基 id 取下划线前");
assertEquals("3", AppearancePatentRowNormalizer.baseId("3"), "无下划线原样");
assertEquals("2", AppearancePatentRowNormalizer.baseId(" 2_1 "), "归一化后再取基 id");
}
@Test
void test_normalize_display_id_trims_only() {
assertEquals("2_1", AppearancePatentRowNormalizer.normalizeDisplayId(" 2_1 "), "displayId 只 trim 不折叠内部");
assertEquals("", AppearancePatentRowNormalizer.normalizeDisplayId(null));
assertEquals("", AppearancePatentRowNormalizer.normalizeDisplayId(" "));
}
@Test
void test_normalize_keeps_other_fullwidth_punct() {
assertEquals("(外观)", AppearancePatentRowNormalizer.normalize(" (外观) "), "非空格全角字符保留");
}
@Test
void test_normalize_not_mutating_shared_rules() {
String sample = " X Y ";
String once = AppearancePatentRowNormalizer.normalize(sample);
assertEquals(once, AppearancePatentRowNormalizer.normalize(sample), "幂等");
assertNotEquals(sample, once);
assertFalse(once.contains(""), "BOM 不残留");
assertFalse(once.contains(" "), "全角空格不残留");
assertTrue(once.equals("X Y"));
}
}
@@ -0,0 +1,227 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayOutputStream;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 94:AppearancePatentSheetBuilder Sheet 构造器。
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式、行值派生)。
* 与现状 writeResultWorkbook / writeReasonSheet 一致:主 sheet 名"外观专利检测结果"、
* 13 列表头(RESULT_HEADERS 10 列 + 第 6/7/8 位插入标题/图片链接/sku)、加粗表头、
* 数据行从第 1 行;"原因"sheet 4 列按 ASIN 去重。不落库、无 IO 依赖。
*/
class AppearancePatentSheetBuilderTest {
private static final String[] RESULT_HEADERS_13 = {
"id", "asin", "国家", "卖家名称", "品牌", "价格",
"标题", "图片链接", "sku",
"标题维度(商标)", "外观维度(外观设计专利)", "结论", "状态"
};
private static AppearancePatentParsedRowVo parsedRow(String id, String asin, String country,
String title, String url, String sku) {
AppearancePatentParsedRowVo row = new AppearancePatentParsedRowVo();
row.setDisplayId(id);
row.setSourceId(id);
row.setAsin(asin);
row.setCountry(country);
row.setTitle(title);
row.setUrl(url);
row.setSku(sku);
Map<String, String> values = new LinkedHashMap<>();
values.put("卖家名称", "seller-A");
values.put("品牌", "brand-A");
values.put("价格", "19.90");
row.setValues(values);
return row;
}
private static AppearancePatentResultRowDto resultRow(String id, String asin, String country) {
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setId(id);
row.setAsin(asin);
row.setCountry(country);
row.setTitleRisk("无风险");
row.setAppearanceRisk("无风险");
row.setConclusion("已侵权");
row.setStatus("成功");
row.setError("");
row.setPrice("19.90");
return row;
}
@Test
void test_sheet_builder_headers_and_sheet_names() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(), Map.of());
Sheet main = wb.getSheet("外观专利检测结果");
Sheet reason = wb.getSheet("原因");
assertTrue(main != null, "主 sheet 存在");
assertTrue(reason != null, "原因 sheet 存在");
Row header = main.getRow(0);
assertEquals(13, header.getLastCellNum(), "主 sheet 13 列");
for (int i = 0; i < RESULT_HEADERS_13.length; i++) {
assertEquals(RESULT_HEADERS_13[i], header.getCell(i).getStringCellValue(),
"第 " + i + " 列表头");
}
Row reasonHeader = reason.getRow(0);
assertEquals("ASIN", reasonHeader.getCell(0).getStringCellValue());
assertEquals("外观原因", reasonHeader.getCell(1).getStringCellValue());
assertEquals("专利原因", reasonHeader.getCell(2).getStringCellValue());
assertEquals("标题原因", reasonHeader.getCell(3).getStringCellValue());
}
}
@Test
void test_sheet_builder_header_bold_style() throws Exception {
try (XSSFWorkbook wb = new XSSFWorkbook()) {
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(), Map.of());
Cell headerCell = wb.getSheet("外观专利检测结果").getRow(0).getCell(0);
org.apache.poi.xssf.usermodel.XSSFCellStyle style =
(org.apache.poi.xssf.usermodel.XSSFCellStyle) headerCell.getCellStyle();
assertTrue(style.getFont().getBold(), "表头加粗");
}
}
@Test
void test_sheet_builder_data_rows() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "title-a", "http://img/a.jpg", "SKU-1");
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
AppearancePatentSheetBuilder.rowKey(r), r));
Sheet main = wb.getSheet("外观专利检测结果");
Row data = main.getRow(1);
assertEquals("2_1", data.getCell(0).getStringCellValue(), "id 列");
assertEquals("B01A", data.getCell(1).getStringCellValue(), "asin 列");
assertEquals("US", data.getCell(2).getStringCellValue(), "国家列");
assertEquals("seller-A", data.getCell(3).getStringCellValue(), "卖家名称列");
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌列");
assertEquals("19.90", data.getCell(5).getStringCellValue(), "价格列");
assertEquals("title-a", data.getCell(6).getStringCellValue(), "标题列");
assertEquals("http://img/a.jpg", data.getCell(7).getStringCellValue(), "图片链接列");
assertEquals("SKU-1", data.getCell(8).getStringCellValue(), "sku 列");
assertEquals("无风险", data.getCell(9).getStringCellValue(), "标题维度列");
assertEquals("无风险", data.getCell(10).getStringCellValue(), "外观维度列");
assertEquals("已侵权", data.getCell(11).getStringCellValue(), "结论列");
assertEquals("成功", data.getCell(12).getStringCellValue(), "状态列");
}
}
@Test
void test_sheet_builder_empty_data_rows() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(), Map.of());
Sheet main = wb.getSheet("外观专利检测结果");
assertNull(main.getRow(1), "无数据行时只有表头");
}
}
@Test
void test_sheet_builder_no_result_row_uses_parsed_fallback() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "title-a", "http://img/a.jpg", "SKU-1");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of());
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("title-a", data.getCell(6).getStringCellValue(), "无结果行回退解析行标题");
assertEquals("http://img/a.jpg", data.getCell(7).getStringCellValue(), "回退解析行 URL");
assertEquals("SKU-1", data.getCell(8).getStringCellValue(), "回退解析行 sku");
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
}
}
@Test
void test_sheet_builder_reason_sheet_dedup_by_asin() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p1 = parsedRow("2_1", "B01A", "US", "t", "", "");
AppearancePatentParsedRowVo p2 = parsedRow("2_2", "B01A", "DE", "t", "", "");
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
r.setAppearanceReason("外观理由");
r.setPatentReason("专利理由");
r.setTitleReason("标题理由");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p1, p2), Map.of(
AppearancePatentSheetBuilder.rowKey(r), r));
Sheet reason = wb.getSheet("原因");
Row row1 = reason.getRow(1);
assertEquals("B01A", row1.getCell(0).getStringCellValue());
assertEquals("外观理由", row1.getCell(1).getStringCellValue());
assertEquals("专利理由", row1.getCell(2).getStringCellValue());
assertEquals("标题理由", row1.getCell(3).getStringCellValue());
assertNull(reason.getRow(2), "同 ASIN 第二行去重");
}
}
@Test
void test_sheet_builder_writable_output_stream() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentSheetBuilder.buildResultSheet(wb,
List.of(parsedRow("1", "B01X", "US", "t", "u", "s")), Map.of());
ByteArrayOutputStream out = new ByteArrayOutputStream();
wb.write(out);
assertTrue(out.size() > 0, "workbook 可写出");
}
}
@Test
void test_sheet_builder_row_key_helpers() {
AppearancePatentResultRowDto r = resultRow("2_1", "b01a", "US");
assertEquals("2_1::B01A::US", AppearancePatentSheetBuilder.rowKey(r), "rowKey 归一化 + 大写 asin");
AppearancePatentParsedRowVo p = new AppearancePatentParsedRowVo();
p.setDisplayId("2_1");
p.setAsin("b01a");
p.setCountry("US");
assertEquals("2_1::B01A::US", AppearancePatentSheetBuilder.rowKey(p), "解析行 rowKey 同构");
}
@Test
void test_sheet_builder_llm_failure_user_facing_cells() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
r.setTitleRisk("coze 调用超时");
r.setError("coze 工作流节点执行超限");
r.setConclusion("coze 调用超时");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
AppearancePatentSheetBuilder.rowKey(r), r));
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("coze 工作流节点执行超限", data.getCell(9).getStringCellValue(),
"LLM 技术失败展示错误信息");
assertEquals("成功", data.getCell(12).getStringCellValue(),
"结论回退错误信息后非空即成功(与现状一致)");
}
}
@Test
void test_sheet_builder_blank_asin_reason_row_skipped() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p = parsedRow("2_1", " ", "US", "t", "", "");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of());
Sheet reason = wb.getSheet("原因");
assertNull(reason.getRow(1), "空白 ASIN 不入原因 sheet");
}
}
}
@@ -0,0 +1,323 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 96:AppearancePatent 快照对比测试。
* 夹具 Excel(appearancepatent 实际结构)→ 解析结果快照(golden 文件);
* 同一夹具输出完全一致;快照变更即失败(防行为漂移)。
* 管线与服务侧 parseWorkbook 对应段落一致:parser.parse → VO 组装(displayId/baseId/
* groupKey/rowToken)→ hydratePromptFields(组内 title/url/sku 回填);夹具无"状态"列,
* 不走 FailedStatusRowFilter。
* golden 文件:src/test/resources/appearancepatent/golden/parse-snapshot.txt
*/
class AppearancePatentSnapshotTest {
private static final File GOLDEN_PARSE =
new File("src/test/resources/appearancepatent/golden/parse-snapshot.txt");
private static final String[] HEADERS = {"id", "asin", "国家", "价格", "seller sku", "图片链接", "标题", "卖家名称", "品牌"};
// ---- 夹具 ----
private static File workbook(String[] headers, List<String[]> rows) throws Exception {
File file = File.createTempFile("appearance-patent-snapshot-", ".xlsx");
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream out = new FileOutputStream(file)) {
Sheet sheet = wb.createSheet("Sheet1");
Row header = sheet.createRow(0);
for (int i = 0; i < headers.length; i++) {
header.createCell(i).setCellValue(headers[i]);
}
for (int r = 0; r < rows.size(); r++) {
Row row = sheet.createRow(r + 1);
for (int c = 0; c < rows.get(r).length; c++) {
row.createCell(c).setCellValue(rows.get(r)[c]);
}
}
wb.write(out);
}
return file;
}
/** 5 有效行:2_1/2_2/2_3 同块(2_2/2_3 标题空缺待组内回填)、单块 5、单块 6。 */
private static File fixtureMain() throws Exception {
return workbook(HEADERS, List.of(
new String[]{"2_1", "B01A", "US", "19.90", "SKU-1", "http://img/a.jpg", "title a", "seller-A", "brand-A"},
new String[]{"2_2", "B01B", "DE", "29.90", "SKU-2", "http://img/b.jpg", "", "seller-B", "brand-B"},
new String[]{"2_3", "B01C", "FR", "", "", "http://img/c.jpg", "", "seller-C", "brand-C"},
new String[]{"5", "B01D", "UK", "9.90", "SKU-4", "http://img/d.jpg", "title d", "seller-D", "brand-D"},
new String[]{"6", "B01E", "US", "1.00", "SKU-5", "http://img/e.jpg", "title e", "seller-E", "brand-E"}));
}
/** 缺 id/asin/国家 各 1 行 + 1 全空行;解析后剩 3 有效行。 */
private static File fixtureError() throws Exception {
return workbook(HEADERS, List.of(
new String[]{"", "B02A", "US", "", "", "", "", "seller", "brand"},
new String[]{"9", "", "DE", "", "", "", "", "seller", "brand"},
new String[]{"10", "B02B", "", "", "", "", "", "seller", "brand"},
new String[]{"11", "B02C", "FR", "", "", "", "", "seller", "brand"},
new String[]{"12", "B02D", "FR", "", "", "", "", "seller", "brand"},
new String[]{"13", "B02E", "FR", "", "", "", "", "seller", "brand"},
new String[]{"", "", ""}));
}
// ---- 解析管线(与服务侧 parseWorkbook 对应段落语义一致) ----
private static List<AppearancePatentParsedRowVo> toRows(File file, String fileKey, String filename) throws Exception {
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
List<AppearancePatentParsedRowVo> rows = new ArrayList<>();
String currentBlockBaseId = "";
String currentGroupKey = "";
for (AppearancePatentExcelParser.AppearanceExcelRow parsedRow : parsed.rows()) {
String id = parsedRow.id();
String asin = parsedRow.asin();
String country = parsedRow.country();
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
if (id.isBlank() || asin.isBlank() || country.isBlank()) {
continue;
}
AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo();
vo.setSourceFileKey(fileKey);
vo.setSourceFilename(filename);
vo.setRowIndex(parsedRow.rowIndex());
vo.setSourceId(id);
vo.setDisplayId(normalizeDisplayId(id));
String rowBaseId = baseId(vo.getDisplayId());
if (!Objects.equals(currentBlockBaseId, rowBaseId)) {
currentBlockBaseId = rowBaseId;
currentGroupKey = buildGroupKey(fileKey, rowBaseId, vo.getRowIndex());
}
vo.setGroupKey(currentGroupKey);
vo.setRowToken(buildRowToken(fileKey, vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setPrice(parsedRow.price());
vo.setSku(parsedRow.sku());
vo.setUrl(parsedRow.url());
vo.setTitle(parsedRow.title());
vo.setValues(parsedRow.values());
rows.add(vo);
}
hydratePromptFields(rows);
return rows;
}
private static void hydratePromptFields(List<AppearancePatentParsedRowVo> rows) {
if (rows == null || rows.isEmpty()) {
return;
}
Map<String, List<AppearancePatentParsedRowVo>> rowsByBaseId = new LinkedHashMap<>();
for (AppearancePatentParsedRowVo row : rows) {
String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId()));
rowsByBaseId.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
}
for (List<AppearancePatentParsedRowVo> siblings : rowsByBaseId.values()) {
String title = "";
String url = "";
String sku = "";
for (AppearancePatentParsedRowVo sibling : siblings) {
title = firstNonBlank(title, sibling.getTitle());
url = firstNonBlank(url, sibling.getUrl());
sku = firstNonBlank(sku, sibling.getSku());
}
for (AppearancePatentParsedRowVo sibling : siblings) {
if (normalize(sibling.getTitle()).isBlank()) {
sibling.setTitle(title);
}
if (normalize(sibling.getUrl()).isBlank()) {
sibling.setUrl(url);
}
if (normalize(sibling.getSku()).isBlank()) {
sibling.setSku(sku);
}
}
}
}
private static String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private static String baseId(String id) {
String s = normalize(id);
int idx = s.indexOf('_');
return idx > 0 ? s.substring(0, idx) : s;
}
private static String normalizeDisplayId(String id) {
return id == null ? "" : id.trim();
}
private static String buildRowToken(String sourceFileKey, Integer rowIndex) {
return normalize(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex);
}
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
}
private static String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
// ---- 快照渲染 ----
private static String renderRow(AppearancePatentParsedRowVo row) {
return " row: idx=" + row.getRowIndex()
+ " sourceId=" + row.getSourceId()
+ " displayId=" + row.getDisplayId()
+ " asin=" + row.getAsin()
+ " country=" + row.getCountry()
+ " price=" + row.getPrice()
+ " sku=" + row.getSku()
+ " url=" + row.getUrl()
+ " title=" + row.getTitle()
+ " groupKey=" + row.getGroupKey()
+ " rowToken=" + row.getRowToken()
+ " values.size=" + (row.getValues() == null ? 0 : row.getValues().size());
}
private static String renderRows(List<AppearancePatentParsedRowVo> rows) {
StringBuilder sb = new StringBuilder();
sb.append("rows=").append(rows.size()).append('\n');
for (AppearancePatentParsedRowVo row : rows) {
sb.append(renderRow(row)).append('\n');
}
return sb.toString();
}
private static List<AppearancePatentParsedRowVo> mergeRows(List<List<AppearancePatentParsedRowVo>> all) {
List<AppearancePatentParsedRowVo> merged = new ArrayList<>();
for (List<AppearancePatentParsedRowVo> list : all) {
merged.addAll(list);
}
return merged;
}
public static String runAllParse() throws Exception {
return renderRows(mergeRows(List.of(
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx"),
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx"))));
}
private static String read(File file) throws Exception {
return Files.readString(file.toPath(), StandardCharsets.UTF_8);
}
// ---- 用例 ----
@Test
void test_snapshot_parse_output() throws Exception {
assertEquals(read(GOLDEN_PARSE), runAllParse(), "解析输出快照与 golden 一致");
}
@Test
void test_snapshot_rows() throws Exception {
List<AppearancePatentParsedRowVo> rows = mergeRows(List.of(
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx"),
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx")));
assertEquals(8, rows.size(), "有效行 5 + 3");
assertEquals("uploads/main.xlsx::2@2", rows.get(0).getGroupKey(), "首块 groupKey = fileKey::baseId@rowIndex");
assertEquals("uploads/main.xlsx::row::2", rows.get(0).getRowToken(), "rowToken = fileKey::row::rowIndex");
assertEquals("2_1", rows.get(0).getDisplayId(), "displayId 原样保留");
assertEquals("B01A", rows.get(0).getAsin(), "asin 大写");
assertEquals(9, rows.get(0).getValues().size(), "values 按 9 列表头键取值");
}
@Test
void test_snapshot_block_grouping() throws Exception {
List<AppearancePatentParsedRowVo> rows = toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx");
assertEquals(5, rows.size());
assertEquals(rows.get(0).getGroupKey(), rows.get(1).getGroupKey(), "2_1/2_2 同块");
assertEquals(rows.get(0).getGroupKey(), rows.get(2).getGroupKey(), "2_1/2_3 同块");
assertFalse(rows.get(0).getGroupKey().equals(rows.get(3).getGroupKey()), "2_1 与 5 不同块");
assertEquals("uploads/main.xlsx::5@5", rows.get(3).getGroupKey(), "新块 baseId=5");
assertEquals("uploads/main.xlsx::6@6", rows.get(4).getGroupKey(), "新块 baseId=6");
}
@Test
void test_snapshot_hydration() throws Exception {
List<AppearancePatentParsedRowVo> rows = toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx");
assertEquals("title a", rows.get(1).getTitle(), "2_2 标题从组内回填");
assertEquals("http://img/b.jpg", rows.get(1).getUrl(), "2_2 自身 url 保留");
assertEquals("SKU-1", rows.get(2).getSku(), "2_3 sku 从组内回填");
assertEquals("title a", rows.get(2).getTitle(), "2_3 标题从组内回填");
assertEquals("http://img/c.jpg", rows.get(2).getUrl(), "2_3 自身 url 保留");
assertEquals("", rows.get(2).getPrice(), "价格不回填(仅 title/url/sku)");
}
@Test
void test_snapshot_error_cases() throws Exception {
// 缺必填字段行与全空行均不进入结果;不抛异常
List<AppearancePatentParsedRowVo> rows = toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx");
assertEquals(3, rows.size(), "错误夹具剩 3 有效行");
assertEquals("11", rows.get(0).getSourceId());
assertEquals("12", rows.get(1).getSourceId());
assertEquals("13", rows.get(2).getSourceId());
}
@Test
void test_snapshot_reproducible() throws Exception {
assertEquals(runAllParse(), runAllParse(), "同一夹具跑两次结果一致");
}
@Test
void test_snapshot_golden_committed() {
assertTrue(GOLDEN_PARSE.isFile(), "golden 文件必须存在并入库: " + GOLDEN_PARSE.getAbsolutePath());
assertTrue(GOLDEN_PARSE.length() > 0, "golden 文件非空");
}
@Test
void test_snapshot_diff_detected() throws Exception {
String original = read(GOLDEN_PARSE);
assertTrue(original.contains("rows="), "golden 内容合法");
try {
Files.writeString(GOLDEN_PARSE.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
AssertionError failure = null;
try {
assertEquals(read(GOLDEN_PARSE), runAllParse(), "篡改后应与 golden 不一致");
} catch (AssertionError ex) {
failure = ex;
}
assertTrue(failure != null, "篡改 golden 后断言应失败");
} finally {
Files.writeString(GOLDEN_PARSE.toPath(), original, StandardCharsets.UTF_8);
}
assertEquals(original, read(GOLDEN_PARSE), "恢复原始 golden");
}
@Test
void test_snapshot_regression_all_parse() throws Exception {
assertEquals(read(GOLDEN_PARSE), runAllParse(), "全 parse 路径与 golden 一致");
List<AppearancePatentParsedRowVo> rows = mergeRows(List.of(
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx"),
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx")));
assertEquals(8, rows.size(), "总行数 5 + 3");
assertEquals("uploads/error.xlsx::row::5", rows.get(5).getRowToken(), "错误夹具首行 rowToken");
assertEquals(9, rows.get(5).getValues().size(), "错误夹具 values 仍有 9 列");
}
}
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity; import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -112,7 +113,8 @@ class CollectDataDeleteConsistencyTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class), mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class),
resultDetailCodec, resultDetailCodec,
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class), mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class));; mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class));
} }
private FileTaskEntity task(long id, long userId) { private FileTaskEntity task(long id, long userId) {
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -111,7 +112,8 @@ class CollectDataStorageCallCountTest {
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class), mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class),
resultDetailCodec, resultDetailCodec,
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class), mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class)); mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
mock(TaskProgressLightAssembler.class));
} }
private FileResultEntity result(long id, long taskId, long userId) { private FileResultEntity result(long id, long taskId, long userId) {
@@ -55,12 +55,12 @@ class InvalidAsinDataControllerTest {
when(adminAuthSupport.currentRole(operator)).thenReturn(null); when(adminAuthSupport.currentRole(operator)).thenReturn(null);
when(permissionMenuService.getUserColumnPermissions(7L, "admin")) when(permissionMenuService.getUserColumnPermissions(7L, "admin"))
.thenReturn(List.of(invalidAsinDataPermission())); .thenReturn(List.of(invalidAsinDataPermission()));
when(invalidAsinDataService.page(1L, 15L, "", 3L, 7L, false)) when(invalidAsinDataService.page(1L, 15L, "", "", "", 3L, 7L, false))
.thenReturn(new InvalidAsinDataPageVo()); .thenReturn(new InvalidAsinDataPageVo());
controller.page(1L, 15L, "", 3L, request); controller.page(1L, 15L, "", "", "", 3L, request);
verify(invalidAsinDataService).page(1L, 15L, "", 3L, 7L, false); verify(invalidAsinDataService).page(1L, 15L, "", "", "", 3L, 7L, false);
} }
@Test @Test
@@ -70,7 +70,7 @@ class InvalidAsinDataControllerTest {
request.addParameter("superAdmin", "true"); request.addParameter("superAdmin", "true");
when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录")); when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录"));
assertThrows(BusinessException.class, () -> controller.page(1L, 15L, "", 3L, request)); assertThrows(BusinessException.class, () -> controller.page(1L, 15L, "", "", "", 3L, request));
verifyNoInteractions(invalidAsinDataService); verifyNoInteractions(invalidAsinDataService);
verify(permissionMenuService, never()).requireUserOperator(any()); verify(permissionMenuService, never()).requireUserOperator(any());
@@ -83,12 +83,12 @@ class InvalidAsinDataControllerTest {
AdminUserEntity operator = user(1L, "super_admin"); AdminUserEntity operator = user(1L, "super_admin");
when(adminAuthSupport.requireUser(request)).thenReturn(operator); when(adminAuthSupport.requireUser(request)).thenReturn(operator);
when(adminAuthSupport.currentRole(operator)).thenReturn("super_admin"); when(adminAuthSupport.currentRole(operator)).thenReturn("super_admin");
when(invalidAsinDataService.page(1L, 15L, "", 3L, 1L, true)) when(invalidAsinDataService.page(1L, 15L, "", "", "", 3L, 1L, true))
.thenReturn(new InvalidAsinDataPageVo()); .thenReturn(new InvalidAsinDataPageVo());
controller.page(1L, 15L, "", 3L, request); controller.page(1L, 15L, "", "", "", 3L, request);
verify(invalidAsinDataService).page(1L, 15L, "", 3L, 1L, true); verify(invalidAsinDataService).page(1L, 15L, "", "", "", 3L, 1L, true);
verify(permissionMenuService, never()).getUserColumnPermissions(eq(1L), eq("admin")); verify(permissionMenuService, never()).getUserColumnPermissions(eq(1L), eq("admin"));
} }
@@ -85,7 +85,7 @@ class InvalidAsinDataServiceTest {
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(manual)); when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(manual));
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of(9L, "group-a")); when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of(9L, "group-a"));
InvalidAsinDataPageVo page = service.page(1, 15, "", 10L, 7L, false); InvalidAsinDataPageVo page = service.page(1, 15, "", "", "", 10L, 7L, false);
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class); ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
verify(invalidAsinDataMapper).selectCount(captor.capture()); verify(invalidAsinDataMapper).selectCount(captor.capture());
@@ -109,7 +109,7 @@ class InvalidAsinDataServiceTest {
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(automatic, orphan)); when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(automatic, orphan));
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of()); when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of());
InvalidAsinDataPageVo page = service.page(1, 15, "", null, 1L, true); InvalidAsinDataPageVo page = service.page(1, 15, "", "", "", null, 1L, true);
assertEquals(2, page.getItems().size()); assertEquals(2, page.getItems().size());
assertEquals("AUTO", page.getItems().getFirst().getRecordSource()); assertEquals("AUTO", page.getItems().getFirst().getRecordSource());
@@ -123,7 +123,7 @@ class InvalidAsinDataServiceTest {
when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L); when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L);
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of()); when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of());
service.page(1, 15, "", 9L, 1L, true); service.page(1, 15, "", "", "", 9L, 1L, true);
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class); ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
verify(invalidAsinDataMapper).selectCount(captor.capture()); verify(invalidAsinDataMapper).selectCount(captor.capture());
@@ -134,6 +134,26 @@ class InvalidAsinDataServiceTest {
assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L)); assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L));
} }
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void pageFiltersByDataValueAndBrandIndependently() {
when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L);
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of());
service.page(1, 15, "", "B01", "acme", null, 1L, true);
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
verify(invalidAsinDataMapper).selectCount(captor.capture());
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
InvalidAsinDataEntity.class);
String sql = captor.getValue().getSqlSegment();
assertTrue(sql.contains("data_value LIKE"));
assertTrue(sql.contains("brand LIKE"));
assertTrue(captor.getValue().getParamNameValuePairs().containsValue("%B01%"));
assertTrue(captor.getValue().getParamNameValuePairs().containsValue("%acme%"));
}
@Test @Test
void normalUserCannotDeleteAutoRecord() { void normalUserCannotDeleteAutoRecord() {
when(invalidAsinDataMapper.selectById(91L)).thenReturn(data(91L, "AUTO", null)); when(invalidAsinDataMapper.selectById(91L)).thenReturn(data(91L, "AUTO", null));
@@ -0,0 +1,552 @@
package com.nanri.aiimage.modules.publish.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper;
import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
import com.nanri.aiimage.modules.publish.model.vo.PublishDashboardVo;
import com.nanri.aiimage.modules.publish.model.vo.PublishFileVo;
import com.nanri.aiimage.modules.publish.model.vo.PublishHistoryVo;
import com.nanri.aiimage.modules.publish.model.vo.PublishResultVo;
import com.nanri.aiimage.modules.publish.model.vo.PublishTaskDetailVo;
import com.nanri.aiimage.modules.publish.model.vo.PublishTaskVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.transaction.support.TransactionTemplate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* 任务 114:批量加载等价快照测试。
* 同一夹具下,批量路径(history 批量 IN / dashboard 聚合 / progress 批量)与逐条基准
* (getTaskDetail 的 selectById + 按 taskId 过滤装配)输出完全一致。
* 快照可重复;篡改基准即失败;覆盖历史/结果/dashboard 三页;全量回归门禁。
*/
class PublishBatchLoadingSnapshotEquivTest {
private static final String MODULE = "PUBLISH";
private static final java.io.File GOLDEN =
new java.io.File("src/test/resources/publish/golden/batch-loading-snapshot.txt");
private final List<FileTaskEntity> taskDb = new ArrayList<>();
private final List<PublishFileEntity> fileDb = new ArrayList<>();
private final List<FileResultEntity> resultDb = new ArrayList<>();
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
private FileTaskMapper fileTaskMapper;
private PublishFileMapper publishFileMapper;
private FileResultMapper fileResultMapper;
private TaskFileJobService taskFileJobService;
private PublishTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskFileJobEntity.class);
}
private static String renderDashboard(PublishDashboardVo vo) {
StringBuilder sb = new StringBuilder();
sb.append("pending=").append(vo.getPendingCount())
.append(" running=").append(vo.getRunningCount())
.append(" success=").append(vo.getSuccessCount())
.append(" failed=").append(vo.getFailedCount()).append('\n');
for (PublishTaskDetailVo item : vo.getRecent()) {
sb.append(renderTaskDetail(item));
}
return sb.toString();
}
private static String renderTaskDetail(PublishTaskDetailVo detail) {
PublishTaskVo task = detail.getTask();
StringBuilder sb = new StringBuilder();
sb.append("task id=").append(task.getId())
.append(" no=").append(task.getTaskNo())
.append(" status=").append(task.getStatus())
.append(" src=").append(task.getSourceFileCount())
.append(" ok=").append(task.getSuccessFileCount())
.append(" fail=").append(task.getFailedFileCount())
.append(" comp=").append(task.getCompletedFileCount())
.append(" rows=").append(task.getTotalRows())
.append("/").append(task.getProcessedRows())
.append(" pct=").append(task.getPercent())
.append(" err=").append(task.getErrorMessage())
.append('\n');
for (PublishFileVo file : detail.getFiles()) {
sb.append(" file id=").append(file.getFileId())
.append(" key=").append(file.getFileKey())
.append(" shop=").append(file.getShopName())
.append("/").append(file.getShopId())
.append(" matched=").append(file.isMatched())
.append(" status=").append(file.getStatus())
.append(" rows=").append(file.getTotalRows())
.append("/").append(file.getProcessedRows())
.append(" pct=").append(file.getPercent())
.append('\n');
}
PublishResultVo result = detail.getResult();
if (result != null) {
sb.append(" result id=").append(result.getResultId())
.append(" ready=").append(result.getFileReady())
.append(" url=").append(result.getDownloadUrl())
.append(" job=").append(result.getFileJobId())
.append("/").append(result.getFileJobStatus())
.append(" retry=").append(result.getFileJobRetryCount())
.append(" joberr=").append(result.getFileJobError())
.append(" err=").append(result.getErrorMessage())
.append('\n');
}
return sb.toString();
}
private static String renderHistory(PublishHistoryVo vo) {
StringBuilder sb = new StringBuilder();
sb.append("total=").append(vo.getTotal()).append('\n');
for (PublishTaskDetailVo item : vo.getItems()) {
sb.append(renderTaskDetail(item));
}
return sb.toString();
}
private static String renderBatch(List<PublishTaskDetailVo> items) {
StringBuilder sb = new StringBuilder();
for (PublishTaskDetailVo item : items) {
sb.append(renderTaskDetail(item));
}
return sb.toString();
}
/** 从渲染后的 wrapper 提取 IN 查询引用的全部 id(.in() 每个元素一个 #{ew.paramNameValuePairs.x} 参数)。 */
private static List<Long> inParamIds(LambdaQueryWrapper<?> q) {
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Long> ids = new ArrayList<>();
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
Map<String, Object> params = q.getParamNameValuePairs();
while (m.find()) {
Object value = params.get(m.group(1));
if (value instanceof Number n) {
ids.add(n.longValue());
} else if (value instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Number n) {
ids.add(n.longValue());
}
}
}
}
return ids;
}
@BeforeEach
void setUp() {
fileTaskMapper = mock(FileTaskMapper.class);
publishFileMapper = mock(PublishFileMapper.class);
fileResultMapper = mock(FileResultMapper.class);
taskFileJobService = mock(TaskFileJobService.class);
OssStorageService ossStorageService = mock(OssStorageService.class);
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = new ArrayList<>(q.getParamNameValuePairs().values());
List<Long> wanted = new ArrayList<>();
for (Object value : params) {
if (value instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Number n) {
wanted.add(n.longValue());
}
}
}
}
List<FileTaskEntity> filtered = new ArrayList<>();
for (FileTaskEntity task : taskDb) {
if (!MODULE.equals(task.getModuleType())) {
continue;
}
if (wanted.isEmpty()) {
filtered.add(task);
} else if (wanted.contains(task.getId())) {
filtered.add(task);
}
}
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
if (segment.toLowerCase().contains("limit")) {
int cap = 50;
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
if (m.find()) {
cap = Integer.parseInt(m.group(1));
}
return filtered.subList(0, Math.min(cap, filtered.size()));
}
return filtered;
}).when(fileTaskMapper).selectList(any());
doAnswer(invocation -> {
Long id = invocation.getArgument(0);
return taskDb.stream().filter(t -> t.getId().equals(id)).findFirst().orElse(null);
}).when(fileTaskMapper).selectById(any(Long.class));
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
LambdaQueryWrapper<PublishFileEntity> q = invocation.getArgument(0);
List<Long> wanted = inParamIds(q);
return fileDb.stream()
.filter(f -> wanted.contains(f.getTaskId()))
.sorted((a, b) -> a.getId().compareTo(b.getId()))
.toList();
}).when(publishFileMapper).selectList(any());
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
List<Long> wanted = inParamIds(q);
return resultDb.stream()
.filter(r -> wanted.contains(r.getTaskId()))
.sorted((a, b) -> a.getId().compareTo(b.getId()))
.toList();
}).when(fileResultMapper).selectList(any());
lenient().when(taskFileJobService.findAssembleJobsByResultIds(any(), any())).thenAnswer(invocation -> {
List<?> ids = invocation.getArgument(1);
Map<Long, TaskFileJobEntity> map = new HashMap<>();
if (ids != null) {
for (TaskFileJobEntity job : jobDb) {
if (job.getResultId() != null && ids.contains(job.getResultId())) {
map.put(job.getResultId(), job);
}
}
}
return map;
});
when(fileTaskMapper.selectCount(any())).thenAnswer(invocation -> (long) taskDb.stream()
.filter(t -> MODULE.equals(t.getModuleType()) && Long.valueOf(1L).equals(t.getUserId())).count());
when(fileTaskMapper.selectMaps(any())).thenAnswer(invocation -> {
@SuppressWarnings("unchecked")
com.baomidou.mybatisplus.core.conditions.Wrapper<FileTaskEntity> wrapper = invocation.getArgument(0);
String segment = wrapper.getSqlSegment() == null ? "" : wrapper.getSqlSegment();
assertTrue(segment.toLowerCase().contains("group by"), "聚合查询必须带 GROUP BY: " + segment);
Map<String, Long> counts = new HashMap<>();
for (FileTaskEntity task : taskDb) {
if (!MODULE.equals(task.getModuleType()) || task.getUserId() == null || task.getStatus() == null) {
continue;
}
counts.merge(task.getStatus(), 1L, Long::sum);
}
List<Map<String, Object>> rows = new ArrayList<>();
for (Map.Entry<String, Long> entry : counts.entrySet()) {
Map<String, Object> row = new HashMap<>();
row.put("status", entry.getKey());
row.put("cnt", entry.getValue());
rows.add(row);
}
return rows;
});
when(ossStorageService.generateFreshDownloadUrl(any())).thenAnswer(
invocation -> "https://oss" + invocation.getArgument(0));
service = new PublishTaskService(
mock(LocalFileStorageService.class), mock(ZiniaoShopSwitchService.class),
mock(PublishWorkbookService.class), publishFileMapper, mock(PublishItemMapper.class),
fileTaskMapper, fileResultMapper, mock(TaskChunkMapper.class),
mock(TaskScopeStateMapper.class), taskFileJobService,
mock(TaskDistributedLockService.class), mock(TransientPayloadStorageService.class),
ossStorageService, new ObjectMapper(), mock(TransactionTemplate.class),
mock(InstanceMetadata.class),
mock(TaskProgressLightAssembler.class));
}
private static FileTaskEntity task(long id, String status, LocalDateTime createdAt, Integer src, Integer ok, Integer fail) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType(MODULE);
task.setUserId(1L);
task.setStatus(status);
task.setCreatedAt(createdAt);
task.setUpdatedAt(createdAt);
task.setFinishedAt(status.equals("SUCCESS") || status.equals("FAILED") ? createdAt.plusMinutes(5) : null);
task.setTaskNo("T" + id);
task.setSourceFileCount(src);
task.setSuccessFileCount(ok);
task.setFailedFileCount(fail);
return task;
}
private static PublishFileEntity file(Long id, Long taskId, String status, Integer total, Integer processed) {
PublishFileEntity file = new PublishFileEntity();
file.setId(id);
file.setTaskId(taskId);
file.setFileKey("key" + id);
file.setSourceFilename("s" + id + ".xlsx");
file.setShopName("Shop " + (id % 3));
file.setShopId("SHOP" + (id % 3));
file.setMatched(1);
file.setStatus(status);
file.setTotalRows(total);
file.setProcessedRows(processed);
file.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
return file;
}
private static FileResultEntity result(Long id, Long taskId, String error) {
FileResultEntity result = new FileResultEntity();
result.setId(id);
result.setTaskId(taskId);
result.setModuleType(MODULE);
result.setResultFilename("r" + id + ".xlsx");
result.setResultFileUrl("result/publish/" + id + "/out.xlsx");
result.setSuccess(1);
result.setErrorMessage(error);
result.setCreatedAt(LocalDateTime.of(2026, 8, 1, 11, 0));
return result;
}
private static TaskFileJobEntity job(Long id, Long resultId, String status, Integer retry, String error) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setResultId(resultId);
job.setModuleType(MODULE);
job.setStatus(status);
job.setRetryCount(retry);
job.setErrorMessage(error);
return job;
}
/** 固定夹具:5 任务 ×(成功多文件/运行中+失败文件/待命无结果/失败+Job 错误/成功单文件)+ 结果与 Job 关联。 */
private void seed() {
taskDb.add(task(1L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1), 2, 2, 0));
taskDb.add(task(2L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 2), 2, 0, 1));
taskDb.add(task(3L, "PENDING", LocalDateTime.of(2026, 8, 1, 9, 3), 1, 0, 0));
taskDb.add(task(4L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 4), 1, 0, 1));
taskDb.add(task(5L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 5), 1, 1, 0));
fileDb.add(file(101L, 1L, "SUCCESS", 10, 10));
fileDb.add(file(102L, 1L, "SUCCESS", 20, 20));
fileDb.add(file(201L, 2L, "RUNNING", 30, 12));
fileDb.add(file(202L, 2L, "FAILED", 5, 3));
fileDb.add(file(301L, 3L, "PENDING", 8, 0));
fileDb.add(file(401L, 4L, "FAILED", 9, 4));
fileDb.add(file(501L, 5L, "SUCCESS", 15, 15));
resultDb.add(result(901L, 1L, null));
resultDb.add(result(902L, 2L, "still running"));
resultDb.add(result(904L, 4L, "boom"));
jobDb.add(job(801L, 901L, "SUCCESS", 0, null));
jobDb.add(job(802L, 902L, "RUNNING", 1, null));
jobDb.add(job(804L, 904L, "FAILED", 2, "assemble failed"));
}
private List<FileTaskEntity> orderedTasks() {
return taskDb.stream()
.filter(t -> MODULE.equals(t.getModuleType()) && Long.valueOf(1L).equals(t.getUserId()))
.sorted((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()))
.toList();
}
/** 逐条基准:与批量路径相同的装配语义,但每任务一次 getTaskDetail(selectById 单条路径)。 */
private String baselineDetails() {
StringBuilder sb = new StringBuilder();
for (FileTaskEntity task : orderedTasks()) {
sb.append(renderTaskDetail(service.getTaskDetail(task.getId(), 1L)));
}
return sb.toString();
}
private String baselineHistoryText() {
return "total=" + taskDb.stream()
.filter(t -> MODULE.equals(t.getModuleType()) && Long.valueOf(1L).equals(t.getUserId())).count()
+ "\n" + baselineDetails();
}
private String historySnapshot() {
return renderHistory(service.history(1L, 50));
}
private String dashboardSnapshot() {
return renderDashboard(service.dashboard(1L));
}
private String resultSnapshot() {
StringBuilder sb = new StringBuilder();
for (FileTaskEntity task : orderedTasks()) {
sb.append(renderTaskDetail(service.getTaskDetail(task.getId(), 1L)));
}
return sb.toString();
}
/** progress 批量:请求顺序按 createdAt 倒序(与 history 语义一致),输出明细。 */
private String batchSnapshot() {
List<Long> ids = orderedTasks().stream().map(FileTaskEntity::getId).toList();
return renderBatch(service.getTaskProgress(1L, ids).getItems());
}
private static String readGolden() throws Exception {
return java.nio.file.Files.readString(GOLDEN.toPath(), java.nio.charset.StandardCharsets.UTF_8);
}
@Test
void test_history_snapshot_equiv() throws Exception {
seed();
assertEquals(baselineHistoryText(), historySnapshot(),
"history 批量装配输出与逐条基准完全一致");
}
@Test
void test_dashboard_snapshot_equiv() throws Exception {
seed();
PublishDashboardVo vo = service.dashboard(1L);
assertEquals(1L, vo.getPendingCount());
assertEquals(1L, vo.getRunningCount());
assertEquals(2L, vo.getSuccessCount());
assertEquals(1L, vo.getFailedCount());
String recent = "";
for (PublishTaskDetailVo item : vo.getRecent()) {
recent += renderTaskDetail(item);
}
assertEquals(baselineDetails(), recent.toString(),
"dashboard recent 明细与逐条基准完全一致");
}
@Test
void test_result_snapshot_equiv() throws Exception {
seed();
assertEquals(baselineDetails(), resultSnapshot(),
"结果页逐条装配与逐条基准完全一致");
}
@Test
void test_pagination_equiv() throws Exception {
seed();
PublishHistoryVo limited = service.history(1L, 3);
assertEquals(3, limited.getItems().size(), "limit 3 只取最近 3 条");
StringBuilder expected = new StringBuilder("total=5\n");
for (FileTaskEntity task : orderedTasks().subList(0, 3)) {
expected.append(renderTaskDetail(service.getTaskDetail(task.getId(), 1L)));
}
assertEquals(expected.toString(), renderHistory(limited), "分页输出与基准最近 3 条一致");
}
@Test
void test_batch_progress_equiv() throws Exception {
seed();
assertEquals(baselineDetails(), batchSnapshot(),
"progress 批量明细与逐条基准完全一致");
}
@Test
void test_snapshot_reproducible() throws Exception {
seed();
assertEquals(baselineDetails(), baselineDetails(), "基准可重复");
assertEquals(historySnapshot(), historySnapshot(), "批量快照可重复");
assertEquals(dashboardSnapshot(), dashboardSnapshot(), "dashboard 快照可重复");
assertEquals(resultSnapshot(), resultSnapshot(), "结果快照可重复");
}
@Test
void test_snapshot_golden_committed() {
assertTrue(GOLDEN.isFile(), "golden 文件必须存在并入库: " + GOLDEN.getAbsolutePath());
assertTrue(GOLDEN.length() > 0, "golden 非空");
}
@Test
void test_snapshot_covers_all() throws Exception {
seed();
String golden = readGolden();
assertTrue(golden.contains("task id=1"), "golden 覆盖任务 1(成功多文件)");
assertTrue(golden.contains("task id=2"), "golden 覆盖任务 2(运行中+失败文件)");
assertTrue(golden.contains("task id=3"), "golden 覆盖任务 3(无结果)");
assertTrue(golden.contains("task id=4"), "golden 覆盖任务 4(失败+Job 错误)");
assertTrue(golden.contains("task id=5"), "golden 覆盖任务 5(成功单文件)");
assertTrue(golden.contains("result id=901"), "golden 覆盖成功结果");
assertTrue(golden.contains("result id=902"), "golden 覆盖运行中结果");
assertTrue(golden.contains("result id=904"), "golden 覆盖失败结果");
assertTrue(golden.contains("joberr=assemble failed"), "golden 覆盖 Job 错误信息");
}
@Test
void test_snapshot_diff_fails() throws Exception {
seed();
String original = readGolden();
assertTrue(original.contains("task id=1"), "golden 内容合法");
try {
java.nio.file.Files.writeString(GOLDEN.toPath(), original + "\n# tampered",
java.nio.charset.StandardCharsets.UTF_8);
AssertionError failure = null;
try {
assertEquals(original, readGolden(), "篡改后 golden 与原始不一致");
} catch (AssertionError ex) {
failure = ex;
}
assertTrue(failure != null, "篡改 golden 后断言应失败");
} finally {
java.nio.file.Files.writeString(GOLDEN.toPath(), original,
java.nio.charset.StandardCharsets.UTF_8);
}
assertEquals(original, readGolden(), "恢复原始 golden");
}
@Test
void test_snapshot_regression_guard() throws Exception {
seed();
assertEquals(baselineHistoryText(), historySnapshot(), "历史快照与基准一致");
assertEquals(baselineDetails(), resultSnapshot(), "结果快照与基准一致");
assertEquals(baselineDetails(), batchSnapshot(), "progress 快照与基准一致");
assertEquals(readGolden(), historySnapshot(), "整体输出与 golden 快照一致");
}
@Test
void test_snapshot_equiv_fails_on_mutation() throws Exception {
seed();
String before = historySnapshot();
assertEquals(before, historySnapshot());
taskDb.getFirst().setStatus("RUNNING");
assertNotEquals(before, historySnapshot(), "数据变化后快照必须不同(等价门禁可感知改动)");
}
}
@@ -0,0 +1,321 @@
package com.nanri.aiimage.modules.publish.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.AbstractWrapper;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper;
import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
import com.nanri.aiimage.modules.publish.model.vo.PublishDashboardVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.transaction.support.TransactionTemplate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
/**
* 任务 108:dashboard 聚合查询。
* dashboard 状态统计由 4 次逐条 selectCount 改为 1 次 GROUP BY 聚合(selectMaps status→count);
* 输出结构(pendingCount/runningCount/successCount/failedCount/recent)与逐条统计完全一致;
* 聚合结果为空时各计数为 0;recent 列表语义不变。
*/
class PublishDashboardAggregateTest {
private static final String MODULE = "PUBLISH";
private final List<FileTaskEntity> taskDb = new ArrayList<>();
private final AtomicInteger aggregateCallCount = new AtomicInteger();
private final AtomicInteger recentListCallCount = new AtomicInteger();
private FileTaskMapper fileTaskMapper;
private PublishItemMapper publishItemMapper;
private PublishFileMapper publishFileMapper;
private FileResultMapper fileResultMapper;
private TaskFileJobService taskFileJobService;
private OssStorageService ossStorageService;
private PublishTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
/** 从渲染后的 wrapper 参数中提取 userId(先 getSqlSegment 强制渲染,再读 paramNameValuePairs)。 */
private static Long userIdOf(Wrapper<FileTaskEntity> wrapper, String segment) {
AbstractWrapper<FileTaskEntity, ?, ?> q = (AbstractWrapper<FileTaskEntity, ?, ?>) wrapper;
List<Long> numbers = new ArrayList<>();
for (Object value : q.getParamNameValuePairs().values()) {
if (value instanceof Number n) {
numbers.add(n.longValue());
} else if (value instanceof Iterable<?> iterable) {
for (Object item : iterable) {
if (item instanceof Number n) {
numbers.add(n.longValue());
}
}
}
}
return numbers.stream().filter(Long.class::isInstance).map(Long.class::cast).findFirst().orElse(null);
}
@BeforeEach
void setUp() {
fileTaskMapper = mock(FileTaskMapper.class);
publishItemMapper = mock(PublishItemMapper.class);
publishFileMapper = mock(PublishFileMapper.class);
fileResultMapper = mock(FileResultMapper.class);
taskFileJobService = mock(TaskFileJobService.class);
ossStorageService = mock(OssStorageService.class);
// 聚合查询:一次 selectMaps,按模块+用户过滤后 status→count
doAnswer(invocation -> {
aggregateCallCount.incrementAndGet();
Wrapper<FileTaskEntity> wrapper = invocation.getArgument(0);
String segment = wrapper.getSqlSegment() == null ? "" : wrapper.getSqlSegment();
assertTrue(segment.toLowerCase().contains("group by"), "聚合查询必须带 GROUP BY: " + segment);
Long userId = userIdOf(wrapper, segment);
Map<String, Long> counts = new HashMap<>();
for (FileTaskEntity task : taskDb) {
if (!MODULE.equals(task.getModuleType()) || task.getUserId() == null || task.getStatus() == null) {
continue;
}
if (userId != null && !userId.equals(task.getUserId())) {
continue;
}
counts.merge(task.getStatus(), 1L, Long::sum);
}
List<Map<String, Object>> rows = new ArrayList<>();
for (Map.Entry<String, Long> entry : counts.entrySet()) {
Map<String, Object> row = new HashMap<>();
row.put("status", entry.getKey());
row.put("cnt", entry.getValue());
rows.add(row);
}
return rows;
}).when(fileTaskMapper).selectMaps(any());
// recent 列表:history() 依赖的明细查询
doAnswer(invocation -> {
recentListCallCount.incrementAndGet();
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
Long userId = userIdOf(q, segment);
List<FileTaskEntity> filtered = new ArrayList<>();
for (FileTaskEntity task : taskDb) {
if (!MODULE.equals(task.getModuleType()) || task.getUserId() == null) {
continue;
}
if (userId != null && !userId.equals(task.getUserId())) {
continue;
}
filtered.add(task);
}
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
if (segment.toLowerCase().contains("limit")) {
int cap = 10;
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
if (m.find()) {
cap = Integer.parseInt(m.group(1));
}
return filtered.subList(0, Math.min(cap, filtered.size()));
}
return filtered;
}).when(fileTaskMapper).selectList(any());
lenient().when(publishFileMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskFileJobService.findAssembleJobsByResultIds(any(), any())).thenReturn(Map.of());
lenient().when(ossStorageService.generateFreshDownloadUrl(any())).thenReturn("https://oss/x");
service = new PublishTaskService(
mock(LocalFileStorageService.class), mock(ZiniaoShopSwitchService.class),
mock(PublishWorkbookService.class), publishFileMapper, publishItemMapper,
fileTaskMapper, fileResultMapper, mock(TaskChunkMapper.class),
mock(TaskScopeStateMapper.class), taskFileJobService,
mock(TaskDistributedLockService.class), mock(TransientPayloadStorageService.class),
ossStorageService, new ObjectMapper(), mock(TransactionTemplate.class),
mock(InstanceMetadata.class),
mock(TaskProgressLightAssembler.class));
}
private static FileTaskEntity task(long id, long userId, String status, LocalDateTime createdAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setUserId(userId);
task.setModuleType(MODULE);
task.setStatus(status);
task.setCreatedAt(createdAt);
return task;
}
private void seed() {
taskDb.add(task(1L, 7L, "PENDING", LocalDateTime.of(2026, 8, 1, 10, 1)));
taskDb.add(task(2L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 2)));
taskDb.add(task(3L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 3)));
taskDb.add(task(4L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 4)));
taskDb.add(task(5L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 5)));
taskDb.add(task(6L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 6)));
taskDb.add(task(7L, 7L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 7)));
taskDb.add(task(8L, 7L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 8)));
}
@Test
void dashboardCounts() {
seed();
PublishDashboardVo vo = service.dashboard(7L);
assertEquals(1L, vo.getPendingCount());
assertEquals(2L, vo.getRunningCount());
assertEquals(3L, vo.getSuccessCount());
assertEquals(2L, vo.getFailedCount());
}
@Test
void dashboardEmptyUser() {
PublishDashboardVo vo = service.dashboard(7L);
assertEquals(0L, vo.getPendingCount());
assertEquals(0L, vo.getRunningCount());
assertEquals(0L, vo.getSuccessCount());
assertEquals(0L, vo.getFailedCount());
assertTrue(vo.getRecent().isEmpty());
}
@Test
void dashboardAggregateSql() {
seed();
service.dashboard(7L);
assertEquals(1, aggregateCallCount.get(), "状态统计一次 GROUP BY 聚合,不逐条 COUNT");
verify(fileTaskMapper, times(1)).selectMaps(any());
}
@Test
void dashboardRecentTasks() {
seed();
taskDb.add(task(9L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 9)));
taskDb.add(task(10L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 10)));
taskDb.add(task(11L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 11)));
taskDb.add(task(12L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 12)));
PublishDashboardVo vo = service.dashboard(7L);
assertEquals(10, vo.getRecent().size(), "recent 固定最多 10 条");
assertEquals(12L, vo.getRecent().getFirst().getTask().getId(), "最近任务按 createdAt 倒序");
assertEquals(3L, vo.getRecent().getLast().getTask().getId(), "超出 10 条的最旧任务被截断");
}
@Test
void dashboardConsistency() {
seed();
PublishDashboardVo vo = service.dashboard(7L);
Map<String, Long> perStatus = new HashMap<>();
for (FileTaskEntity task : taskDb) {
perStatus.merge(task.getStatus(), 1L, Long::sum);
}
assertEquals(perStatus.getOrDefault("PENDING", 0L).longValue(), vo.getPendingCount(), "与逐条统计一致");
assertEquals(perStatus.getOrDefault("RUNNING", 0L).longValue(), vo.getRunningCount());
assertEquals(perStatus.getOrDefault("SUCCESS", 0L).longValue(), vo.getSuccessCount());
assertEquals(perStatus.getOrDefault("FAILED", 0L).longValue(), vo.getFailedCount());
}
@Test
void dashboardMixedStatus() {
taskDb.add(task(1L, 7L, "PENDING", LocalDateTime.of(2026, 8, 1, 9, 0)));
taskDb.add(task(2L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 1)));
taskDb.add(task(3L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 2)));
taskDb.add(task(4L, 7L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 3)));
PublishDashboardVo vo = service.dashboard(7L);
assertEquals(1L, vo.getPendingCount());
assertEquals(1L, vo.getRunningCount());
assertEquals(1L, vo.getSuccessCount());
assertEquals(1L, vo.getFailedCount());
assertEquals(4, vo.getRecent().size());
}
@Test
void dashboardUserFiltered() {
seed();
taskDb.add(task(9L, 8L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 11, 0)));
taskDb.add(task(10L, 8L, "FAILED", LocalDateTime.of(2026, 8, 1, 11, 1)));
PublishDashboardVo vo = service.dashboard(7L);
assertEquals(3L, vo.getSuccessCount(), "只统计当前用户任务");
assertEquals(2L, vo.getFailedCount());
assertEquals(8, vo.getRecent().size(), "recent 只含当前用户任务");
}
@Test
void dashboardBigData() {
for (int i = 1; i <= 500; i++) {
String status = i % 4 == 0 ? "SUCCESS" : (i % 4 == 1 ? "FAILED" : (i % 4 == 2 ? "RUNNING" : "PENDING"));
taskDb.add(task(1000L + i, 7L, status, LocalDateTime.of(2026, 8, 1, 0, 0).plusMinutes(i)));
}
PublishDashboardVo vo = service.dashboard(7L);
assertEquals(125L, vo.getPendingCount());
assertEquals(125L, vo.getRunningCount());
assertEquals(125L, vo.getSuccessCount());
assertEquals(125L, vo.getFailedCount());
assertEquals(1, aggregateCallCount.get(), "大数据量仍一次聚合");
}
@Test
void dashboardOtherModuleExcluded() {
seed();
taskDb.add(task(99L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 12, 0)));
taskDb.getLast().setModuleType("SIMILAR_ASIN");
PublishDashboardVo vo = service.dashboard(7L);
assertEquals(3L, vo.getSuccessCount(), "其他模块任务不计入");
assertEquals(8, vo.getRecent().size());
}
@Test
void dashboardRecentDetailAssembled() {
seed();
PublishDashboardVo vo = service.dashboard(7L);
assertNotNull(vo.getRecent().getFirst().getTask(), "recent 明细含 task 信息");
assertEquals("FAILED", vo.getRecent().getFirst().getTask().getStatus());
assertEquals(8L, vo.getRecent().getFirst().getTask().getId());
assertEquals(1, aggregateCallCount.get());
assertEquals(1, recentListCallCount.get(), "recent 列表一次明细查询");
}
}
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -135,7 +136,8 @@ class ShopDataCrawlChunkUpsertTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
storedChunks.clear(); storedChunks.clear();
storedScopes.clear(); storedScopes.clear();
@@ -28,6 +28,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -170,7 +171,8 @@ class ShopDataCrawlCleanupTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L); ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
dbResultRows.clear(); dbResultRows.clear();
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -144,7 +145,8 @@ class ShopDataCrawlDailyFileIncrementalTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
dbResultRows.clear(); dbResultRows.clear();
dbDailyFiles.clear(); dbDailyFiles.clear();
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -150,7 +151,8 @@ class ShopDataCrawlDailyFileJobSplitTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
dbResultRows.clear(); dbResultRows.clear();
dbDailyFiles.clear(); dbDailyFiles.clear();
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -150,7 +151,8 @@ class ShopDataCrawlDailyFileLockTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
dbResultRows.clear(); dbResultRows.clear();
dbDailyFiles.clear(); dbDailyFiles.clear();
@@ -0,0 +1,240 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlRowNormalizer;
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlSheetBuilder;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.FileInputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
/**
* 任务 101:ShopDataCrawl 门面改委托。
* 门面(ShopDataCrawlExcelAssemblyService)签名与行为不变:writeWorkbook / writeWorkbookStreaming /
* replaceCountriesWorkbook / countRows 仍产出与抽取前一致的 workbook;Sheet 构造与行分组
* 全部委托 ShopDataCrawlSheetBuilder(私有方法经反射直接验证,不经过完整任务链路)。
*/
class ShopDataCrawlExcelAssemblyServiceDelegationTest {
@TempDir Path tempDir;
// ---------- 1 签名不变 ----------
@Test
void test_write_workbook_signature_unchanged() throws Exception {
Method method = ShopDataCrawlExcelAssemblyService.class.getDeclaredMethod(
"writeWorkbook", File.class, List.class);
assertEquals(int.class, method.getReturnType(), "返回类型不变");
assertEquals(2, method.getParameterCount(), "参数个数不变");
}
@Test
void test_write_workbook_streaming_signature_unchanged() throws Exception {
Method method = ShopDataCrawlExcelAssemblyService.class.getDeclaredMethod(
"writeWorkbookStreaming", File.class, List.class, int.class);
assertEquals(int.class, method.getReturnType(), "返回类型不变");
assertEquals(3, method.getParameterCount(), "参数个数不变");
}
// ---------- 2 委托各组件 ----------
@Test
void test_streaming_sheet_headers_delegated_to_builder() throws Exception {
ShopDataCrawlResultItemVo item = item("UK", row("2026-07-25", "B012345678"));
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
File output = tempDir.resolve("stream.xlsx").toFile();
int count = new ShopDataCrawlExcelAssemblyService(imageEmbedder)
.writeWorkbookStreaming(output, List.of(item), 100);
assertEquals(1, count, "数据行数委托 rowsByCountry");
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals(ShopDataCrawlSheetBuilder.SHEETS, sheetNames(wb), "sheet 名来自 SheetBuilder");
for (int i = 0; i < wb.getNumberOfSheets(); i++) {
Row header = wb.getSheetAt(i).getRow(0);
for (int column = 0; column < ShopDataCrawlSheetBuilder.HEADERS.size(); column++) {
assertEquals(ShopDataCrawlSheetBuilder.HEADERS.get(column),
header.getCell(column).getStringCellValue(), "表头来自 SheetBuilder");
}
}
Row data = wb.getSheet("英国").getRow(1);
assertEquals("2026-07-25", data.getCell(0).getStringCellValue());
assertEquals("B012345678", data.getCell(1).getStringCellValue());
assertEquals("Example Brand", data.getCell(9).getStringCellValue(), "品牌末列");
}
}
@Test
void test_rows_by_country_delegates_builder_grouping() throws Exception {
ShopDataCrawlResultItemVo item = item("uk", row("2026-07-25", "B012345678"));
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
File output = tempDir.resolve("group.xlsx").toFile();
new ShopDataCrawlExcelAssemblyService(imageEmbedder).writeWorkbookStreaming(output, List.of(item), 100);
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals("B012345678", wb.getSheetAt(0).getRow(1).getCell(1).getStringCellValue(),
"小写国家码归一化后归入英国 sheet");
assertEquals(0, wb.getSheetAt(1).getLastRowNum(), "德国 sheet 无数据");
}
}
@Test
void test_write_workbook_template_path_kept_in_facade() throws Exception {
// 模板路径仍由门面加载:写出的 workbook 保留模板样式路径(图片列宽 18*256)
ShopDataCrawlResultItemVo item = item("DE", row("2026-07-25", "B012345678"));
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
File output = tempDir.resolve("tpl.xlsx").toFile();
int count = new ShopDataCrawlExcelAssemblyService(imageEmbedder).writeWorkbook(output, List.of(item));
assertEquals(1, count);
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals(18 * 256, wb.getSheetAt(1).getColumnWidth(2), "模板路径保留图片列宽");
assertEquals("B012345678", wb.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
}
}
@Test
void test_replace_countries_delegates_sheet_builder() throws Exception {
ShopDataCrawlRowDto ukRow = row("2026-07-25", "B012345678");
ShopDataCrawlRowDto deRow = row("2026-07-26", "B099999999");
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
File base = tempDir.resolve("base.xlsx").toFile();
File output = tempDir.resolve("daily.xlsx").toFile();
service.writeWorkbook(base, List.of(item("UK", ukRow), item("DE", deRow)));
int total = service.replaceCountriesWorkbook(base, output, List.of(item("DE", row("2026-07-27", "B099999998"))));
assertEquals(2, total, "替换后总行数(UK 1 + DE 1)");
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals("B012345678", wb.getSheetAt(0).getRow(1).getCell(1).getStringCellValue(), "未替换国家保留");
assertEquals("B099999998", wb.getSheetAt(1).getRow(1).getCell(1).getStringCellValue(), "替换国家新行");
assertEquals(0, wb.getSheetAt(2).getLastRowNum());
}
}
// ---------- 3 结果一致 ----------
@Test
void test_write_and_streaming_produce_same_row_count() throws Exception {
ShopDataCrawlResultItemVo item = item("UK", row("2026-07-25", "B012345678"));
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
File tplOutput = tempDir.resolve("tpl.xlsx").toFile();
File streamOutput = tempDir.resolve("stream.xlsx").toFile();
int tplCount = service.writeWorkbook(tplOutput, List.of(item));
int streamCount = service.writeWorkbookStreaming(streamOutput, List.of(item), 100);
assertEquals(tplCount, streamCount, "模板路径与流式路径行数一致");
try (XSSFWorkbook tpl = new XSSFWorkbook(new FileInputStream(tplOutput));
XSSFWorkbook stream = new XSSFWorkbook(new FileInputStream(streamOutput))) {
assertEquals(tpl.getSheet("英国").getRow(1).getCell(1).getStringCellValue(),
stream.getSheet("英国").getRow(1).getCell(1).getStringCellValue(), "数据内容一致");
}
}
@Test
void test_count_rows_matches_write_row_count() throws Exception {
ShopDataCrawlResultItemVo uk = item("UK", row("2026-07-25", "B01"));
ShopDataCrawlResultItemVo de = item("DE", row("2026-07-25", "B02"));
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
assertEquals(2, service.countRows(List.of(uk, de)), "countRows 委托分组计数");
}
// ---------- 4 异常一致 ----------
@Test
void test_streaming_null_output_throws_business_exception() {
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(mock(SimilarAsinImageEmbedder.class));
com.nanri.aiimage.common.exception.BusinessException ex =
org.junit.jupiter.api.Assertions.assertThrows(com.nanri.aiimage.common.exception.BusinessException.class,
() -> service.writeWorkbookStreaming(null, List.of(), 100));
assertTrue(ex.getMessage().contains("输出文件路径不能为空"), "实际: " + ex.getMessage());
}
@Test
void test_streaming_invalid_window_throws_illegal_argument() {
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(mock(SimilarAsinImageEmbedder.class));
org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
() -> service.writeWorkbookStreaming(tempDir.resolve("w.xlsx").toFile(), List.of(), 0),
"rowAccessWindow 校验不变");
}
// ---------- 5 行分组/归一化语义由组件承接 ----------
@Test
void test_grouping_semantics_kept_in_sheet_builder() {
ShopDataCrawlRowDto row = row("2026-07-25", "B012345678");
ShopDataCrawlResultItemVo item = item("DE", row);
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(List.of(item));
assertEquals(List.of("UK", "DE", "FR", "ES", "IT"), grouped.keySet().stream().toList(), "5 国固定顺序");
assertEquals(1, grouped.get("DE").size());
assertEquals(0, grouped.get("UK").size());
assertEquals(0, ShopDataCrawlSheetBuilder.rowsByCountry(null).get("UK").size(), "null 条目返回空分组");
assertEquals(0, ShopDataCrawlSheetBuilder.rowsByCountry(List.of(item("US", row))).get("UK").size(),
"未命中 5 国列表的国家被丢弃");
assertEquals(0, ShopDataCrawlSheetBuilder.rowsByCountry(List.of(item("DE", row))).get("FR").size());
}
@Test
void test_normalizer_semantics_unchanged() {
assertEquals("", ShopDataCrawlRowNormalizer.normalizeCountry(null));
assertEquals("DE", ShopDataCrawlRowNormalizer.normalizeCountry(" 德国 "));
assertEquals("B01", ShopDataCrawlRowNormalizer.trim(" B01 "));
ShopDataCrawlRowDto a = row("2026-07-25", "B01");
ShopDataCrawlRowDto b = row(" 2026-07-25 ", " B01 ");
assertTrue(ShopDataCrawlRowNormalizer.sameRow(a, b), "sameRow trim 语义");
}
// ---------- 辅助 ----------
private static ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
country.setCountry(countryCode);
country.setItems(List.of(row));
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
item.setSuccess(true);
item.setCountryResults(List.of(country));
return item;
}
private static ShopDataCrawlRowDto row(String date, String asin) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setBrand("Example Brand");
row.setInventorySales("11");
row.setSalesRank("22");
row.setPageViews("33");
row.setUnitsSold("44");
row.setPrice("12.50");
row.setRecommendedOffer("12.00");
return row;
}
private static List<String> sheetNames(XSSFWorkbook wb) {
return java.util.stream.IntStream.range(0, wb.getNumberOfSheets())
.mapToObj(i -> wb.getSheetAt(i).getSheetName()).toList();
}
}
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -132,7 +133,8 @@ class ShopDataCrawlLightweightProgressTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
storedChunks.clear(); storedChunks.clear();
storedScopes.clear(); storedScopes.clear();
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
@@ -131,7 +132,8 @@ class ShopDataCrawlOwnerColumnTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
dbTasks.clear(); dbTasks.clear();
lastScan.clear(); lastScan.clear();
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -144,7 +145,8 @@ class ShopDataCrawlProgressQueryTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
dbResultRows.clear(); dbResultRows.clear();
dbTasks.clear(); dbTasks.clear();
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -131,7 +132,8 @@ class ShopDataCrawlRowDedupKeyTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
storedChunks.clear(); storedChunks.clear();
storedScopes.clear(); storedScopes.clear();
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -132,7 +133,8 @@ class ShopDataCrawlScopeCounterTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
storedChunks.clear(); storedChunks.clear();
storedScopes.clear(); storedScopes.clear();
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -133,7 +134,8 @@ class ShopDataCrawlScopeMergeTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
storedChunks.clear(); storedChunks.clear();
storedScopes.clear(); storedScopes.clear();
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -127,7 +128,8 @@ class ShopDataCrawlTaskServiceChunkTest {
transientPayloadStorageService, transientPayloadStorageService,
instanceMetadata, instanceMetadata,
dailyFileService, dailyFileService,
null); null,
mock(TaskProgressLightAssembler.class));
storedChunks.clear(); storedChunks.clear();
storedScopes.clear(); storedScopes.clear();
@@ -0,0 +1,216 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 103:ShopDataCrawlHistoryAssembler 历史查询组装器。
* 历史列表 VO 拼装(toHistoryItem + 文件任务状态链)从 ShopDataCrawlTaskService 原样搬移;
* 只读不落库;输出与现状逐字段一致(快照优先,实体字段兜底,task/job 附加)。
*/
class ShopDataCrawlHistoryAssemblerTest {
private final ShopDataCrawlHistoryAssembler assembler = new ShopDataCrawlHistoryAssembler();
private static FileResultEntity result(Long id, Long taskId, String source, String sourceFileUrl, String resultFilename,
String resultFileUrl, Integer success, String errorMessage, LocalDateTime createdAt) {
FileResultEntity row = new FileResultEntity();
row.setId(id);
row.setTaskId(taskId);
row.setSourceFilename(source);
row.setSourceFileUrl(sourceFileUrl);
row.setResultFilename(resultFilename);
row.setResultFileUrl(resultFileUrl);
row.setSuccess(success);
row.setErrorMessage(errorMessage);
row.setCreatedAt(createdAt);
return row;
}
private static FileTaskEntity task(Long id, String status, LocalDateTime finishedAt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setStatus(status);
task.setFinishedAt(finishedAt);
return task;
}
private static TaskFileJobEntity job(Long id, String status, String errorMessage) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setStatus(status);
job.setErrorMessage(errorMessage);
return job;
}
private static ShopDataCrawlResultItemVo snapshot(String shopName, String shopId, String outputFilename,
String error, String taskStatus) {
ShopDataCrawlResultItemVo snapshot = new ShopDataCrawlResultItemVo();
snapshot.setShopName(shopName);
snapshot.setShopId(shopId);
snapshot.setOutputFilename(outputFilename);
snapshot.setError(error);
snapshot.setTaskStatus(taskStatus);
return snapshot;
}
// ---------- toHistoryItem 字段拼装 ----------
@Test
void test_assembler_history_items() {
FileResultEntity row = result(100L, 10L, "a.xlsx", "https://src/10/a.xlsx", "a-result.xlsx",
"result/10/a.xlsx", 1, null, LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, task(10L, "SUCCESS",
LocalDateTime.of(2026, 8, 1, 9, 5)), snapshot("快照店", "S1", "snap.xlsx", null, null), null);
assertEquals(100L, item.getResultId());
assertEquals(10L, item.getTaskId());
assertEquals("快照店", item.getShopName(), "快照 shopName 优先");
assertEquals("S1", item.getShopId(), "快照 shopId 优先");
assertEquals("SUCCESS", item.getTaskStatus(), "task 状态覆盖快照");
assertEquals(Boolean.TRUE, item.getSuccess(), "success=1 即成功");
assertEquals(LocalDateTime.of(2026, 8, 1, 10, 0), item.getCreatedAt(), "createdAt 取实体");
assertEquals(LocalDateTime.of(2026, 8, 1, 9, 5), item.getFinishedAt(), "finishedAt 取 task");
assertEquals("snap.xlsx", item.getOutputFilename(), "快照 outputFilename 优先");
assertNull(item.getDownloadUrl(), "downloadUrl 恒为空");
assertTrue(item.getFileReady(), "文件 URL 就绪");
}
@Test
void test_assembler_snapshot_fallback_fields() {
FileResultEntity row = result(100L, 10L, "a.xlsx", "https://src/10/a.xlsx", "a-result.xlsx",
null, 0, "python timeout", LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, snapshot(null, null, null, "old err", null), null);
assertEquals("a.xlsx", item.getShopName(), "快照缺 shopName 回退实体 sourceFilename");
assertEquals("https://src/10/a.xlsx", item.getShopId(), "快照缺 shopId 回退实体 sourceFileUrl");
assertEquals("python timeout", item.getError(), "实体 errorMessage 优先");
assertEquals("a-result.xlsx", item.getOutputFilename(), "快照缺 outputFilename 回退实体 resultFilename");
assertNull(item.getTaskStatus(), "缺 task 快照 taskStatus 为 null 时保留 null");
assertEquals(Boolean.FALSE, item.getSuccess(), "success=0 为失败");
assertNull(item.getFinishedAt(), "缺 task 无结束时间");
}
@Test
void test_assembler_null_snapshot_and_null_task() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, null, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, null);
assertEquals(100L, item.getResultId());
assertEquals("a.xlsx", item.getShopName(), "无快照回退实体");
assertNull(item.getShopId(), "无快照且实体缺 shopId 为空");
assertNull(item.getSuccess(), "success 缺省且快照缺省为 null");
assertNull(item.getError());
assertFalse(Boolean.TRUE.equals(item.getFileReady()), "无文件 URL 未就绪");
assertNull(item.getFileStatus(), "无 job 且文件未就绪状态为空");
assertTrue(item.getCountryResults() != null && item.getCountryResults().isEmpty(), "countryResults 非 null 空列表");
assertTrue(item.getCountryCodes() != null && item.getCountryCodes().isEmpty(), "countryCodes 非 null 空列表");
}
@Test
void test_assembler_snapshot_mutation_keeps_existing_values() {
// 快照实体字段存在时保留(countryResults 原样,不被清空)
ShopDataCrawlResultItemVo snapshot = snapshot("店A", "S1", "out.xlsx", null, "RUNNING");
snapshot.setCountryResults(new ArrayList<>(List.of()));
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 1, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, snapshot, null);
assertSame(snapshot, item, "快照非 null 时返回同一实例(原地填充)");
assertEquals("店A", item.getShopName());
assertEquals("RUNNING", item.getTaskStatus(), "快照 taskStatus 保留");
}
// ---------- 文件任务状态链 ----------
@Test
void test_assembler_file_state_job_attached() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 0, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, job(7L, "RUNNING", null));
assertEquals(7L, item.getFileJobId());
assertEquals("RUNNING", item.getFileStatus());
assertNull(item.getFileError(), "非 FAILED job 不附错误");
}
@Test
void test_assembler_file_state_failed_job_error_attached() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 0, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, job(9L, "FAILED", "assemble boom"));
assertEquals("FAILED", item.getFileStatus());
assertEquals("assemble boom", item.getFileError(), "job 错误信息附带");
}
@Test
void test_assembler_file_state_no_job_file_ready_success() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, "a-result.xlsx", "result/10/a.xlsx", 1, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, null);
assertTrue(item.getFileReady(), "文件 URL 就绪");
assertEquals("SUCCESS", item.getFileStatus(), "无 job 且文件就绪状态为 SUCCESS");
}
@Test
void test_assembler_country_lists_filled_when_snapshot_missing() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 0, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, null);
assertTrue(item.getCountryResults() != null && item.getCountryResults().isEmpty(), "countryResults 空列表");
assertTrue(item.getCountryCodes() != null && item.getCountryCodes().isEmpty(), "countryCodes 空列表");
}
@Test
void test_assembler_immutable_input() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 1, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
FileTaskEntity t = task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 5));
assembler.toHistoryItem(row, t, null, null);
assertEquals("a.xlsx", row.getSourceFilename(), "result 不被修改");
assertEquals("SUCCESS", t.getStatus(), "task 不被修改");
assertEquals(LocalDateTime.of(2026, 8, 1, 9, 5), t.getFinishedAt(), "task 不被修改");
}
@Test
void test_assembler_consistency_with_current_output() {
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, "result/10/a.xlsx", 1, null,
LocalDateTime.of(2026, 8, 1, 10, 0));
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, task(10L, "FAILED",
LocalDateTime.of(2026, 8, 1, 9, 30)), null, null);
assertEquals("a.xlsx", item.getShopName(), "无快照时 shopName 回退 sourceFilename");
assertEquals(Boolean.TRUE, item.getSuccess(), "文件 URL 就绪即成功");
assertEquals("FAILED", item.getTaskStatus());
assertEquals(LocalDateTime.of(2026, 8, 1, 9, 30), item.getFinishedAt());
assertTrue(item.getFileReady(), "文件 URL 就绪");
}
}
@@ -0,0 +1,182 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 99:ShopDataCrawlRowNormalizer 行解析/归一化器。
* 规则与 ShopDataCrawlTaskService.trim / blank / blankToNull / trimToNull /
* normalizeCountry / copyRow / rowEmpty / sameRow / rowDedupKey 现状逐字节一致。
* 纯函数无状态;rowDedupKey 与 sameRow 的 10 字段 trim 比较语义等价。
*/
class ShopDataCrawlRowNormalizerTest {
// ---- trim ----
@Test
void test_trim_null_returns_empty() {
assertEquals("", ShopDataCrawlRowNormalizer.trim(null), "null 归一化为空串");
}
@Test
void test_trim_plain_trimming() {
assertEquals("B001", ShopDataCrawlRowNormalizer.trim(" B001 "), "trim 去首尾空白");
assertEquals("", ShopDataCrawlRowNormalizer.trim(" "), "全空白归空串");
}
// ---- blank ----
@Test
void test_blank_detection() {
assertTrue(ShopDataCrawlRowNormalizer.blank(null), "null 视为空白");
assertTrue(ShopDataCrawlRowNormalizer.blank(""), "空串视为空白");
assertTrue(ShopDataCrawlRowNormalizer.blank(" "), "全空白视为空白");
assertFalse(ShopDataCrawlRowNormalizer.blank("DE"), "非空白不视为空白");
}
// ---- blankToNull / trimToNull ----
@Test
void test_blank_to_null() {
assertNull(ShopDataCrawlRowNormalizer.blankToNull(null));
assertNull(ShopDataCrawlRowNormalizer.blankToNull(" "));
assertEquals("DE", ShopDataCrawlRowNormalizer.blankToNull(" DE "), "非空白 trim 返回");
}
@Test
void test_trim_to_null() {
assertNull(ShopDataCrawlRowNormalizer.trimToNull(null));
assertNull(ShopDataCrawlRowNormalizer.trimToNull(" "), "trim 后为空返回 null");
assertEquals("B001", ShopDataCrawlRowNormalizer.trimToNull(" B001 "));
}
// ---- normalizeCountry ----
@Test
void test_normalize_country_chinese_aliases() {
assertEquals("DE", ShopDataCrawlRowNormalizer.normalizeCountry("德国"));
assertEquals("UK", ShopDataCrawlRowNormalizer.normalizeCountry("英国"));
assertEquals("FR", ShopDataCrawlRowNormalizer.normalizeCountry("法国"));
assertEquals("IT", ShopDataCrawlRowNormalizer.normalizeCountry("意大利"));
assertEquals("ES", ShopDataCrawlRowNormalizer.normalizeCountry("西班牙"));
}
@Test
void test_normalize_country_uppercase_and_fallback() {
assertEquals("DE", ShopDataCrawlRowNormalizer.normalizeCountry(" de "), "trim + 大写");
assertEquals("US", ShopDataCrawlRowNormalizer.normalizeCountry("us"), "未知国家原样大写");
assertEquals("", ShopDataCrawlRowNormalizer.normalizeCountry(null), "null 归空串");
}
// ---- copyRow ----
@Test
void test_copy_row_ten_fields_trimmed() {
ShopDataCrawlRowDto source = row(" 2026-07-25 ", " B001 ", " Brand ", " img ",
" 10 ", " 20 ", " 30 ", " 40 ", " 50 ", " 60 ");
ShopDataCrawlRowDto copy = ShopDataCrawlRowNormalizer.copyRow(source);
assertEquals("2026-07-25", copy.getDate());
assertEquals("B001", copy.getAsin());
assertEquals("Brand", copy.getBrand());
assertEquals("img", copy.getCommodityImage());
assertEquals("10", copy.getInventorySales());
assertEquals("20", copy.getSalesRank());
assertEquals("30", copy.getPageViews());
assertEquals("40", copy.getUnitsSold());
assertEquals("50", copy.getPrice());
assertEquals("60", copy.getRecommendedOffer());
}
@Test
void test_copy_row_does_not_mutate_source() {
ShopDataCrawlRowDto source = row(" 2026-07-25 ", " B001 ", " B ", " i ",
" 1 ", " 2 ", " 3 ", " 4 ", " 5 ", " 6 ");
ShopDataCrawlRowNormalizer.copyRow(source);
assertEquals(" 2026-07-25 ", source.getDate(), "source 不被修改");
assertEquals(" B001 ", source.getAsin());
}
// ---- rowEmpty ----
@Test
void test_row_empty_detection() {
ShopDataCrawlRowDto blank = new ShopDataCrawlRowDto();
assertTrue(ShopDataCrawlRowNormalizer.rowEmpty(null), "null 行视为空");
assertTrue(ShopDataCrawlRowNormalizer.rowEmpty(blank), "全字段空白视为空");
blank.setAsin(" ");
assertTrue(ShopDataCrawlRowNormalizer.rowEmpty(blank), "空白 asin 不算有内容");
blank.setAsin("B001");
assertFalse(ShopDataCrawlRowNormalizer.rowEmpty(blank), "任一字段有值即非空");
}
// ---- sameRow ----
@Test
void test_same_row_compares_ten_trimmed_fields() {
ShopDataCrawlRowDto a = row("2026-07-25", "B001", "Brand", "img",
"10", "20", "30", "40", "50", "60");
ShopDataCrawlRowDto b = row(" 2026-07-25 ", " B001 ", " Brand ", " img ",
" 10 ", " 20 ", " 30 ", " 40 ", " 50 ", " 60 ");
assertTrue(ShopDataCrawlRowNormalizer.sameRow(a, b), "trim 后语义相同");
b.setPrice("51");
assertFalse(ShopDataCrawlRowNormalizer.sameRow(a, b), "单字段差异即不同");
assertFalse(ShopDataCrawlRowNormalizer.sameRow(null, a), "null 行不同");
}
// ---- rowDedupKey ----
@Test
void test_row_dedup_key_semantics_equivalent_to_same_row() {
ShopDataCrawlRowDto a = row("2026-07-25", "B001", "Brand", "img",
"10", "20", "30", "40", "50", "60");
ShopDataCrawlRowDto b = row(" 2026-07-25 ", " B001 ", " Brand ", " img ",
" 10 ", " 20 ", " 30 ", " 40 ", " 50 ", " 60 ");
assertEquals(ShopDataCrawlRowNormalizer.rowDedupKey(a),
ShopDataCrawlRowNormalizer.rowDedupKey(b), "语义相同行键相等");
assertNull(ShopDataCrawlRowNormalizer.rowDedupKey(null), "null 行键为 null");
b.setUnitsSold("41");
assertNotEquals(ShopDataCrawlRowNormalizer.rowDedupKey(a),
ShopDataCrawlRowNormalizer.rowDedupKey(b), "任一字段差异键不同");
}
@Test
void test_row_dedup_key_stable_order() {
ShopDataCrawlRowDto a = row("2026-07-25", "B001", "Brand", "img",
"10", "20", "30", "40", "50", "60");
assertEquals(ShopDataCrawlRowNormalizer.rowDedupKey(a),
ShopDataCrawlRowNormalizer.rowDedupKey(a), "同一行键稳定");
assertTrue(ShopDataCrawlRowNormalizer.rowDedupKey(a).contains("B001"), "键含字段值");
}
private static ShopDataCrawlRowDto row(String date, String asin, String brand, String image,
String inventory, String rank, String views, String units,
String price, String offer) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setBrand(brand);
row.setCommodityImage(image);
row.setInventorySales(inventory);
row.setSalesRank(rank);
row.setPageViews(views);
row.setUnitsSold(units);
row.setPrice(price);
row.setRecommendedOffer(offer);
return row;
}
}
@@ -0,0 +1,200 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayOutputStream;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 100:ShopDataCrawlSheetBuilder Sheet 构造器。
* 结果 Workbook/Sheet 构造辅助(表头、列序、样式、行值派生、模板校验)。
* 与现状 ShopDataCrawlExcelAssemblyService 一致:5 个国家工作表、10 列表头(品牌末列)、
* 图片列 2 宽 18*256、模板列映射(legacy/current/带品牌变体)。不落库、无 IO 依赖。
*/
class ShopDataCrawlSheetBuilderTest {
private static ShopDataCrawlRowDto row(String date, String asin, String brand, String image) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setBrand(brand);
row.setCommodityImage(image);
row.setInventorySales("11");
row.setSalesRank("22");
row.setPageViews("33");
row.setUnitsSold("44");
row.setPrice("12.50");
row.setRecommendedOffer("12.00");
return row;
}
@Test
void test_headers_and_sheet_names() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 1);
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 2);
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 3);
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 4);
assertEquals(List.of("英国", "德国", "法国", "西班牙", "意大利"), sheetNames(wb), "5 个国家 sheet 顺序");
for (int i = 0; i < 5; i++) {
Row header = wb.getSheetAt(i).getRow(0);
assertEquals(ShopDataCrawlSheetBuilder.HEADERS.size(), header.getLastCellNum(), "第 " + i + " 个 sheet 10 列");
for (int column = 0; column < ShopDataCrawlSheetBuilder.HEADERS.size(); column++) {
assertEquals(ShopDataCrawlSheetBuilder.HEADERS.get(column),
header.getCell(column).getStringCellValue(), "第 " + i + " 个 sheet 第 " + column + " 列表头");
}
}
}
}
@Test
void test_column_order_brand_last() {
assertEquals("品牌", ShopDataCrawlSheetBuilder.HEADERS.get(9), "品牌为最后一列");
assertEquals(9, ShopDataCrawlSheetBuilder.BRAND_COLUMN);
assertEquals("商品图片", ShopDataCrawlSheetBuilder.HEADERS.get(2), "图片列为第 3 列");
assertEquals(2, ShopDataCrawlSheetBuilder.IMAGE_COLUMN);
}
@Test
void test_streaming_sheet_column_width() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
assertEquals(18 * 256, sheet.getColumnWidth(2), "图片列宽");
}
}
@Test
void test_validate_template_accepts_current_brand_template() throws Exception {
try (XSSFWorkbook wb = new XSSFWorkbook()) {
for (int i = 0; i < 5; i++) {
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, i);
}
ShopDataCrawlSheetBuilder.validateTemplate(wb);
}
}
@Test
void test_validate_template_rejects_wrong_sheet_count() throws Exception {
try (XSSFWorkbook wb = new XSSFWorkbook()) {
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
assertThrows(BusinessException.class, () -> ShopDataCrawlSheetBuilder.validateTemplate(wb),
"工作表数量不正确");
}
}
@Test
void test_validate_template_rejects_wrong_headers() throws Exception {
try (XSSFWorkbook wb = new XSSFWorkbook()) {
for (int i = 0; i < 5; i++) {
wb.createSheet(ShopDataCrawlSheetBuilder.SHEETS.get(i));
}
wb.getSheetAt(0).createRow(0).createCell(0).setCellValue("错误表头");
assertThrows(BusinessException.class, () -> ShopDataCrawlSheetBuilder.validateTemplate(wb),
"表头不正确");
}
}
@Test
void test_write_data_row_values_layout() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
Row row = sheet.createRow(1);
ShopDataCrawlSheetBuilder.writeDataRowValues(row, row("2026-07-25", "B012345678", "Brand-X", "https://img/x.jpg"),
new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
assertEquals("2026-07-25", row.getCell(0).getStringCellValue(), "日期列");
assertEquals("B012345678", row.getCell(1).getStringCellValue(), "asin 列");
assertEquals("", row.getCell(2).getStringCellValue(), "图片列留空由调用方嵌入");
assertEquals("11", row.getCell(3).getStringCellValue(), "库存销量列");
assertEquals("22", row.getCell(4).getStringCellValue(), "销售排名列");
assertEquals("33", row.getCell(5).getStringCellValue(), "页面浏览量列");
assertEquals("44", row.getCell(6).getStringCellValue(), "售出件数列");
assertEquals("12.50", row.getCell(7).getStringCellValue(), "价格列");
assertEquals("12.00", row.getCell(8).getStringCellValue(), "推荐报价列");
assertEquals("Brand-X", row.getCell(9).getStringCellValue(), "品牌末列");
}
}
@Test
void test_write_data_row_null_values_blank_cells() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
ShopDataCrawlRowDto rowDto = new ShopDataCrawlRowDto();
rowDto.setAsin("B01");
Row row = sheet.createRow(1);
ShopDataCrawlSheetBuilder.writeDataRowValues(row, rowDto, new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
assertEquals("", row.getCell(0).getStringCellValue(), "null 日期归空串");
assertEquals("B01", row.getCell(1).getStringCellValue());
assertEquals("", row.getCell(9).getStringCellValue(), "null 品牌归空串");
}
}
@Test
void test_template_column_mapping_variants() {
assertEquals(0, ShopDataCrawlSheetBuilder.templateColumnForOutput(0, true, true), "日期列恒为 0");
assertEquals(2, ShopDataCrawlSheetBuilder.templateColumnForOutput(2, true, true), "current 带品牌图片列不变");
assertEquals(3, ShopDataCrawlSheetBuilder.templateColumnForOutput(3, true, true), "current 库存列不变");
assertEquals(9, ShopDataCrawlSheetBuilder.templateColumnForOutput(9, true, true), "current 品牌列不变");
assertEquals(1, ShopDataCrawlSheetBuilder.templateColumnForOutput(9, true, false), "current 无品牌品牌列映射到 1");
assertEquals(2, ShopDataCrawlSheetBuilder.templateColumnForOutput(3, false, false), "legacy 无图片列库存列映射到 2");
assertEquals(2, ShopDataCrawlSheetBuilder.templateColumnForOutput(3, false, true), "legacy 带品牌库存列映射到 2");
assertEquals(1, ShopDataCrawlSheetBuilder.templateColumnForOutput(9, false, false), "legacy 无品牌品牌列映射到 1");
}
@Test
void test_clear_data_rows_keeps_header() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
Row row = sheet.createRow(1);
ShopDataCrawlSheetBuilder.writeDataRowValues(row, row("2026-07-25", "B01", "B", "u"), new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
ShopDataCrawlSheetBuilder.clearDataRows(sheet);
assertEquals(0, sheet.getLastRowNum(), "数据行清空、表头保留");
assertTrue(sheet.getRow(0) != null, "表头仍在");
}
}
@Test
void test_workbook_writable() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
ByteArrayOutputStream out = new ByteArrayOutputStream();
wb.write(out);
assertTrue(out.size() > 0, "workbook 可写出");
}
}
@Test
void test_total_data_rows() throws Exception {
try (XSSFWorkbook wb = new XSSFWorkbook()) {
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 1);
wb.getSheetAt(0).createRow(1);
wb.getSheetAt(0).createRow(2);
assertEquals(2, ShopDataCrawlSheetBuilder.totalDataRows(wb), "跨 sheet 数据行计数");
}
}
private static List<String> sheetNames(Workbook wb) {
return java.util.stream.IntStream.range(0, wb.getNumberOfSheets())
.mapToObj(i -> wb.getSheetAt(i).getSheetName()).toList();
}
}
@@ -0,0 +1,254 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 任务 102:ShopDataCrawl 快照对比测试。
* 夹具 items → 分组快照(rowsByCountry)+ 行布局快照(writeDataRowValues);
* golden 文件固定输出,快照变更即失败(防行为漂移)。
* 与 05 spec §5 一致:抽取前后同一夹具输出完全一致。
* golden 文件:src/test/resources/shopdatacrawl/golden/groups-snapshot.txt
* src/test/resources/shopdatacrawl/golden/rows-snapshot.txt
*/
class ShopDataCrawlSnapshotTest {
private static final java.io.File GOLDEN_GROUPS =
new java.io.File("src/test/resources/shopdatacrawl/golden/groups-snapshot.txt");
private static final java.io.File GOLDEN_ROWS =
new java.io.File("src/test/resources/shopdatacrawl/golden/rows-snapshot.txt");
// ---- 夹具 ----
/** 5 个国家各 1 行 + 未命中国家(US)行 + 失败条目行 + null 条目;items 顺序刻意打乱。 */
private static ShopDataCrawlRowDto row(String date, String asin, String brand) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setBrand(brand);
row.setInventorySales("11");
row.setSalesRank("22");
row.setPageViews("33");
row.setUnitsSold("44");
row.setPrice("12.50");
row.setRecommendedOffer("12.00");
return row;
}
private static ShopDataCrawlCountryResultDto country(String code, ShopDataCrawlRowDto... rows) {
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
country.setCountry(code);
country.setItems(List.of(rows));
return country;
}
private static ShopDataCrawlResultItemVo item(Boolean success, ShopDataCrawlCountryResultDto... countries) {
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
item.setSuccess(success);
item.setCountryResults(List.of(countries));
return item;
}
/** 含 null 字段行的全字段夹具:10 列值全部有值 + 部分为 null。 */
private static ShopDataCrawlRowDto fullRow() {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate("2026-07-25");
row.setAsin("B012345678");
row.setBrand("Example Brand");
row.setInventorySales("128");
row.setSalesRank("#1,245");
row.setPageViews("3560");
row.setUnitsSold("42");
row.setPrice("GBP 19.99");
row.setRecommendedOffer("GBP 18.99");
return row;
}
private static ShopDataCrawlRowDto sparseRow() {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setAsin("B099999999");
row.setBrand(" Brand With Spaces ");
return row;
}
private static List<ShopDataCrawlResultItemVo> fixtureMain() {
List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
items.add(item(Boolean.TRUE, country("DE", row("2026-07-26", "B02", "DE-Brand")),
country("US", row("2026-07-26", "B-US", "US-Brand"))));
items.add(item(Boolean.TRUE, country("uk", row("2026-07-25", "B01", "UK-Brand"))));
items.add(item(null, country("FR", row("2026-07-27", "B03", "FR-Brand"))));
items.add(item(Boolean.FALSE, country("ES", row("2026-07-28", "B04", "ES-Brand"))));
items.add(item(Boolean.TRUE, country("IT", row("2026-07-29", "B05", "IT-Brand"))));
items.add(null);
return items;
}
private static List<ShopDataCrawlResultItemVo> fixtureNullFields() {
return List.of(item(Boolean.TRUE, country("UK", fullRow(), sparseRow())));
}
// ---- 快照管线(与服务侧 writeWorkbook 语义一致:rowsByCountry → writeDataRowValues) ----
private static String renderGroupKey(String country, int count) {
return " group: country=" + country + " rows=" + count;
}
static String runGroupsOnly() throws Exception {
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureMain());
StringBuilder sb = new StringBuilder();
for (Map.Entry<String, List<ShopDataCrawlRowDto>> entry : grouped.entrySet()) {
sb.append(renderGroupKey(entry.getKey(), entry.getValue().size())).append('\n');
for (ShopDataCrawlRowDto row : entry.getValue()) {
sb.append(" asin=").append(row.getAsin()).append('\n');
}
}
return sb.toString();
}
static String runRowsOnly() throws Exception {
List<ShopDataCrawlRowDto> rows = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureNullFields()).get("UK");
List<String> rowTexts = new ArrayList<>();
try (XSSFWorkbook wb = new XSSFWorkbook()) {
Sheet sheet = wb.createSheet("UK");
for (ShopDataCrawlRowDto rowDto : rows) {
Row row = sheet.createRow(rowTexts.size());
ShopDataCrawlSheetBuilder.writeDataRowValues(row, rowDto, new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
StringBuilder sb = new StringBuilder();
for (int column = 0; column < ShopDataCrawlSheetBuilder.HEADERS.size(); column++) {
sb.append(ShopDataCrawlSheetBuilder.HEADERS.get(column)).append('=')
.append(row.getCell(column).getStringCellValue()).append(';');
}
rowTexts.add(sb.toString());
}
}
StringBuilder sb = new StringBuilder();
sb.append("rows=").append(rows.size()).append('\n');
for (String text : rowTexts) {
sb.append(" ").append(text).append('\n');
}
return sb.toString();
}
private static String read(java.io.File file) throws Exception {
return Files.readString(file.toPath(), StandardCharsets.UTF_8);
}
// ---- 用例 ----
@Test
void test_snapshot_groups_output() throws Exception {
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "分组快照与 golden 一致");
}
@Test
void test_snapshot_rows_output() throws Exception {
assertEquals(read(GOLDEN_ROWS), runRowsOnly(), "行布局快照与 golden 一致");
}
@Test
void test_snapshot_dtos() throws Exception {
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureMain());
assertEquals(List.of("UK", "DE", "FR", "ES", "IT"),
new ArrayList<>(grouped.keySet()), "5 国固定顺序");
assertEquals(1, grouped.get("UK").size(), "uk 小写归一化归入英国");
assertEquals(1, grouped.get("DE").size());
assertEquals(1, grouped.get("FR").size(), "success=null 条目仍计入");
assertEquals(0, grouped.get("ES").size(), "success=false 条目丢弃");
assertEquals(1, grouped.get("IT").size());
assertEquals("B01", grouped.get("UK").get(0).getAsin(), "小写国家码行内容不变");
assertEquals("B02", grouped.get("DE").get(0).getAsin(), "US 未命中 5 国被丢弃,不混入 DE");
assertEquals(1, grouped.get("DE").size(), "DE 仅含 B02");
}
@Test
void test_snapshot_error_cases() throws Exception {
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureMain());
assertEquals(0, grouped.get("ES").size(), "失败条目不产生行");
assertEquals(4, grouped.values().stream().mapToInt(List::size).sum(), "总行数 = 4 有效国(UK/DE/FR/IT)");
Map<String, List<ShopDataCrawlRowDto>> empty = ShopDataCrawlSheetBuilder.rowsByCountry(null);
assertEquals(0, empty.get("UK").size(), "null 输入返回空分组");
assertEquals(5, empty.size(), "null 输入仍返回 5 国分组");
}
@Test
void test_snapshot_reproducible() throws Exception {
assertEquals(runGroupsOnly(), runGroupsOnly(), "分组跑两次一致");
assertEquals(runRowsOnly(), runRowsOnly(), "行布局跑两次一致");
}
@Test
void test_snapshot_golden_committed() {
assertTrue(GOLDEN_GROUPS.isFile(), "golden 文件必须存在并入库: " + GOLDEN_GROUPS.getAbsolutePath());
assertTrue(GOLDEN_ROWS.isFile(), "golden 文件必须存在并入库: " + GOLDEN_ROWS.getAbsolutePath());
assertTrue(GOLDEN_GROUPS.length() > 0, "groups golden 非空");
assertTrue(GOLDEN_ROWS.length() > 0, "rows golden 非空");
}
@Test
void test_snapshot_diff_detected() throws Exception {
String original = read(GOLDEN_GROUPS);
assertTrue(original.contains("rows="), "golden 内容合法");
try {
Files.writeString(GOLDEN_GROUPS.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
AssertionError failure = null;
try {
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "篡改后应与 golden 不一致");
} catch (AssertionError ex) {
failure = ex;
}
assertTrue(failure != null, "篡改 golden 后断言应失败");
} finally {
Files.writeString(GOLDEN_GROUPS.toPath(), original, StandardCharsets.UTF_8);
}
assertEquals(original, read(GOLDEN_GROUPS), "恢复原始 golden");
}
@Test
void test_snapshot_regression_all_pipeline() throws Exception {
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "全分组路径与 golden 一致");
assertEquals(read(GOLDEN_ROWS), runRowsOnly(), "全行布局路径与 golden 一致");
List<ShopDataCrawlRowDto> rows = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureNullFields()).get("UK");
assertEquals(2, rows.size(), "全字段 + 稀疏行共 2 行");
assertEquals("Example Brand", rows.get(0).getBrand(), "全字段行品牌原样");
assertEquals(" Brand With Spaces ", rows.get(1).getBrand(), "稀疏行品牌保留原样(写入时归一为空/原样由列布局决定)");
}
/** 行布局写入值归一(null → 空串)通过 golden rows 快照覆盖;此处补充列序断言。 */
@Test
void test_snapshot_column_order_fixed() throws Exception {
try (XSSFWorkbook wb = new XSSFWorkbook()) {
Sheet sheet = wb.createSheet("UK");
Row row = sheet.createRow(0);
ShopDataCrawlSheetBuilder.writeDataRowValues(row, fullRow(), new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
assertEquals("2026-07-25", row.getCell(0).getStringCellValue(), "列 0 日期");
assertEquals("B012345678", row.getCell(1).getStringCellValue(), "列 1 ASIN");
assertEquals("", row.getCell(2).getStringCellValue(), "列 2 商品图片留空");
assertEquals("128", row.getCell(3).getStringCellValue(), "列 3 库存销量");
assertEquals("#1,245", row.getCell(4).getStringCellValue(), "列 4 销售排名");
assertEquals("3560", row.getCell(5).getStringCellValue(), "列 5 页面浏览量");
assertEquals("42", row.getCell(6).getStringCellValue(), "列 6 售出件数");
assertEquals("GBP 19.99", row.getCell(7).getStringCellValue(), "列 7 价格");
assertEquals("GBP 18.99", row.getCell(8).getStringCellValue(), "列 8 推荐报价");
assertEquals("Example Brand", row.getCell(9).getStringCellValue(), "列 9 品牌");
}
}
}
@@ -1,5 +1,10 @@
package com.nanri.aiimage.modules.shopkey.service; package com.nanri.aiimage.modules.shopkey.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.ibatis.session.Configuration;
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper; import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper;
import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper; import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper;
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest; import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest;
@@ -11,6 +16,7 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
@@ -46,6 +52,13 @@ class SkipPriceAsinServiceTest {
@InjectMocks @InjectMocks
private SkipPriceAsinService service; private SkipPriceAsinService service;
@BeforeAll
static void initMybatisPlusTableInfo() {
Configuration configuration = new MybatisConfiguration();
MapperBuilderAssistant assistant = new MapperBuilderAssistant(configuration, "test");
TableInfoHelper.initTableInfo(assistant, SkipPriceAsinEntity.class);
}
@Test @Test
void createSkipsDuplicateCountryAsinWithoutWriting() { void createSkipsDuplicateCountryAsinWithoutWriting() {
when(shopManageGroupService.getAccessibleById(10L, 7L, true)).thenReturn(group()); when(shopManageGroupService.getAccessibleById(10L, 7L, true)).thenReturn(group());
@@ -184,6 +197,45 @@ class SkipPriceAsinServiceTest {
return request; return request;
} }
@Test
void pageAppliesMinimumPriceRangeAcrossAnyCountryWhenCountryAbsent() {
when(skipPriceAsinMapper.selectCount(any())).thenReturn(0L);
when(skipPriceAsinMapper.selectList(any())).thenReturn(List.of());
service.page(1, 15, null, null, null, null,
new BigDecimal("10.00"), new BigDecimal("20.00"), null, true);
ArgumentCaptor<Wrapper<SkipPriceAsinEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
verify(skipPriceAsinMapper).selectCount(captor.capture());
String sql = captor.getValue().getSqlSegment();
org.assertj.core.api.Assertions.assertThat(sql)
.contains("minimum_price_de")
.contains("minimum_price_uk")
.contains("minimum_price_fr")
.contains("minimum_price_it")
.contains("minimum_price_es")
.contains(">= #{")
.contains("<= #{")
.contains("OR");
}
@Test
void pageAppliesMinimumPriceRangeOnSelectedCountryColumn() {
when(skipPriceAsinMapper.selectCount(any())).thenReturn(0L);
when(skipPriceAsinMapper.selectList(any())).thenReturn(List.of());
service.page(1, 15, null, null, null, "DE",
new BigDecimal("15.00"), null, null, true);
ArgumentCaptor<Wrapper<SkipPriceAsinEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
verify(skipPriceAsinMapper).selectCount(captor.capture());
String sql = captor.getValue().getSqlSegment();
org.assertj.core.api.Assertions.assertThat(sql)
.contains("minimum_price_de")
.doesNotContain("minimum_price_uk")
.contains(">=");
}
private File importWorkbook(String asin, String minimumPrice) throws Exception { private File importWorkbook(String asin, String minimumPrice) throws Exception {
File file = File.createTempFile("skip-price-asin-test-", ".xlsx"); File file = File.createTempFile("skip-price-asin-test-", ".xlsx");
try (Workbook workbook = new XSSFWorkbook(); try (Workbook workbook = new XSSFWorkbook();
@@ -1,85 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import javax.net.ssl.SNIHostName;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import java.net.InetAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.List;
public class LlmGatewayTlsProbe {
public static void main(String[] args) throws Exception {
String host = "ai.t8star.org";
String apiKey = args[0];
System.out.println("[probe] java=" + System.getProperty("java.version")
+ " tls=" + System.getProperty("java.vm.name"));
for (InetAddress a : InetAddress.getAllByName(host)) {
System.out.println("[probe] dns " + a);
}
rawHandshake(host, null, "default");
rawHandshake(host, "TLSv1.2", "tls12-only");
httpClientCall(host, apiKey, null, "jdk-http-default");
httpClientCall(host, apiKey, "TLSv1.2", "jdk-http-tls12");
}
private static void rawHandshake(String host, String protocol, String label) throws Exception {
SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
try (SSLSocket socket = (SSLSocket) factory.createSocket(host, 443)) {
socket.setSoTimeout(15000);
SSLParameters params = socket.getSSLParameters();
if (protocol != null) {
params.setProtocols(new String[]{protocol});
}
params.setServerNames(List.of(new SNIHostName(host)));
socket.setSSLParameters(params);
socket.startHandshake();
System.out.println("[probe] raw[" + label + "] OK proto=" + socket.getSession().getProtocol()
+ " cipher=" + socket.getSession().getCipherSuite());
} catch (Exception ex) {
System.out.println("[probe] raw[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
}
}
private static void httpClientCall(String host, String apiKey, String protocol, String label) throws Exception {
HttpClient.Builder builder = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(10))
.version(HttpClient.Version.HTTP_1_1);
if (protocol != null) {
SSLContext context = SSLContext.getInstance("TLS");
context.init(null, null, null);
builder.sslContext(context);
}
HttpClient client = builder.build();
try {
String body = "{\"model\":\"gemini-3.5-flash-lite\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":10}";
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("https://" + host + "/v1/chat/completions"))
.timeout(Duration.ofSeconds(30))
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(body))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
String text = response.body();
System.out.println("[probe] http[" + label + "] status=" + response.statusCode()
+ " body=" + text.substring(0, Math.min(160, text.length())));
} catch (Exception ex) {
System.out.println("[probe] http[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
Throwable cause = ex;
while (cause.getCause() != null) {
cause = cause.getCause();
System.out.println("[probe] cause " + cause.getClass().getSimpleName() + ": " + cause.getMessage());
}
}
}
}
@@ -1,226 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 15:图片缓存访问时间更新改为异步批量刷新,减少逐图 UPDATE。
* lookup 命中不再同步 touchLastUsed,而是进入内存 touch 缓冲(按 url_hash 去重),
* 由定时任务/阈值触发 flushPendingTouches 批量 touchLastUsedBatch 刷新;
* 缓冲有大小上限,超限立即刷新,不会无界增长;失败 best-effort 吞掉。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinImagePrefetchServiceAsyncTouchTest {
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private TaskImageCacheMapper taskImageCacheMapper;
@Mock private SimilarAsinProperties properties;
@InjectMocks private SimilarAsinImagePrefetchService service;
@BeforeEach
void setUp() {
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(1000);
}
@AfterEach
void shutdown() {
service.shutdown();
}
private static String sha256Hex(String value) throws Exception {
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(digest.length * 2);
for (byte b : digest) {
sb.append(String.format("%02x", b & 0xFF));
}
return sb.toString();
}
private void stubHit(String url, byte[] bytes) throws Exception {
when(taskImageCacheMapper.selectBytesByUrlHash(sha256Hex(url))).thenReturn(bytes);
}
@Test
void test_task_015_image_cache_normal_default_path() throws Exception {
// 正常输入:命中不立即写 DB,进入缓冲;flush 后批量 touch 一次。
String url = "https://img.example.com/hit.jpg";
String hash = sha256Hex(url);
byte[] bytes = new byte[]{1, 2, 3};
stubHit(url, bytes);
byte[] result = service.lookup(url);
assertEquals(bytes, result, "命中必须返回缓存字节");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_normal_multiple_items() throws Exception {
// 批量场景:多次命中进入同一缓冲,flush 合并为一次批量 touch,覆盖全部命中。
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
"https://img.example.com/c.jpg");
Set<String> hashes = new HashSet<>();
for (int i = 0; i < urls.size(); i++) {
hashes.add(sha256Hex(urls.get(i)));
stubHit(urls.get(i), new byte[]{(byte) (i + 1)});
}
for (String url : urls) {
assertNotNull(service.lookup(url), "命中返回字节");
}
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
service.flushPendingTouches();
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
assertEquals(hashes, new HashSet<>(captor.getValue()), "一次批量 touch 覆盖全部命中 hash");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:同一 url 多次命中只 touch 一次;重复 flush 无多余请求。
String url = "https://img.example.com/same.jpg";
String hash = sha256Hex(url);
stubHit(url, new byte[]{5});
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(new byte[]{5});
service.lookup(url);
service.lookup(url);
service.lookup(url);
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_boundary_empty_input() throws Exception {
// 空输入:null/空白 url 不进入缓冲;flush 空缓冲不产生任何数据库访问。
assertNull(service.lookup(null));
assertNull(service.lookup(""));
assertNull(service.lookup(" "));
service.flushPendingTouches();
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
}
@Test
void test_task_015_image_cache_boundary_single_item() throws Exception {
// 单条命中:flush 后单元素批量 touch,不依赖批量路径。
String url = "https://img.example.com/single.jpg";
String hash = sha256Hex(url);
stubHit(url, new byte[]{7});
assertNotNull(service.lookup(url));
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
}
@Test
void test_task_015_image_cache_boundary_limit_and_overflow() throws Exception {
// 缓冲达到阈值立即刷新,不无界增长;刷新后继续累积。
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(3);
List<String> urls = List.of("https://img.example.com/o1.jpg", "https://img.example.com/o2.jpg",
"https://img.example.com/o3.jpg", "https://img.example.com/o4.jpg",
"https://img.example.com/o5.jpg");
for (String url : urls) {
stubHit(url, new byte[]{1});
}
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
for (int i = 0; i < 3; i++) {
service.lookup(urls.get(i));
}
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
assertEquals(3, captor.getValue().size(), "第 3 条命中触发阈值立即刷新 3 条");
service.lookup(urls.get(3));
service.lookup(urls.get(4));
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
service.flushPendingTouches();
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
assertEquals(2, captor.getValue().size(), "剩余 2 条在 flush 时刷新,缓冲不残留、不无界增长");
}
@Test
void test_task_015_image_cache_invalid_input_rejected() {
// 非法输入:db cache 关闭时 lookup 直接返回 null,不进入缓冲、不访问数据库。
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
assertNull(service.lookup("https://img.example.com/a.jpg"));
service.flushPendingTouches();
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_015_image_cache_dependency_failure_releases_resources() throws Exception {
// 依赖失败:批量 touch 抛异常时吞掉不阻塞 lookup、缓冲已排空无残留;
// 恢复后重新入队 flush 成功。
String url = "https://img.example.com/fail.jpg";
String hash = sha256Hex(url);
stubHit(url, new byte[]{3});
AtomicInteger failures = new AtomicInteger(0);
List<String> capturedArgs = new java.util.ArrayList<>();
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
List<String> arg = (List<String>) invocation.getArgument(0);
capturedArgs.addAll(arg);
if (failures.getAndIncrement() == 0) {
throw new IllegalStateException("db down");
}
return 0;
}).when(taskImageCacheMapper).touchLastUsedBatch(any());
assertNotNull(service.lookup(url), "touch 失败不阻断 lookup 返回缓存字节");
assertThrows(Exception.class, () -> service.flushPendingTouches(), "首次 flush 抛错(由调用方吞掉)");
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
assertNotNull(service.lookup(url), "失败后再次命中重新入队");
service.flushPendingTouches();
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(any());
assertEquals(List.of(hash, hash), capturedArgs, "两次 touch 都覆盖命中 hash,失败后恢复成功");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
private static void assertNull(Object value) {
org.junit.jupiter.api.Assertions.assertNull(value);
}
}
@@ -1,297 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 14:图片 DB cache 改为批量读取缩略图,并只更新实际命中的 last_used_at。
* 批量 lookup 入口(lookupBatch)一次 IN 查询返回命中字节 Map;
* last_used_at 只对实际命中的 url_hash 更新(touch 集合 = 命中集合),
* 未命中 url 不产生任何 touch/insert。单 URL 旧入口 lookup 语义保持兼容。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinImagePrefetchServiceBatchTest {
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private TaskImageCacheMapper taskImageCacheMapper;
@Mock private SimilarAsinProperties properties;
@InjectMocks private SimilarAsinImagePrefetchService service;
@BeforeEach
void setUp() {
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
}
@AfterEach
void shutdown() {
service.shutdown();
}
private static String sha256Hex(String value) throws Exception {
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(digest.length * 2);
for (byte b : digest) {
sb.append(String.format("%02x", b & 0xFF));
}
return sb.toString();
}
/** stub 批量命中读取:只对 hits 集合内的 hash 返回字节。 */
private void stubBatchRead(List<String> hits, Map<String, byte[]> bytesByHash) {
when(taskImageCacheMapper.selectBytesByUrlHashes(any())).thenAnswer(invocation -> {
List<String> hashes = invocation.getArgument(0);
List<TaskImageCacheEntity> rows = new ArrayList<>();
for (String hash : hashes) {
if (hits.contains(hash)) {
TaskImageCacheEntity row = new TaskImageCacheEntity();
row.setUrlHash(hash);
row.setImageBytes(bytesByHash.get(hash));
rows.add(row);
}
}
return rows;
});
}
/** 调用批量入口,按 url 顺序返回字节(未命中为 null)。 */
@SuppressWarnings("unchecked")
private static List<byte[]> invokeLookupBatch(SimilarAsinImagePrefetchService svc, List<String> urls) throws Exception {
Method m = SimilarAsinImagePrefetchService.class.getDeclaredMethod("lookupBatch", List.class);
m.setAccessible(true);
return (List<byte[]>) m.invoke(svc, urls);
}
@Test
void test_task_014_image_normal_default_path() throws Exception {
// 正常输入:命中与未命中混排,批量读回命中字节,touch 只覆盖实际命中。
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
List<String> hits = List.of(sha256Hex(urls.get(0)));
byte[] bytesA = new byte[]{1, 2, 3};
stubBatchRead(hits, Map.of(sha256Hex(urls.get(0)), bytesA));
List<byte[]> result = invokeLookupBatch(service, urls);
assertEquals(2, result.size(), "批量入口必须按输入顺序返回");
assertEquals(bytesA, result.get(0), "命中行返回缓存字节");
assertNull(result.get(1), "未命中行返回 null,不虚构缓存内容");
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(sha256Hex(urls.get(0))));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
}
@Test
void test_task_014_image_normal_multiple_items() throws Exception {
// 批量场景:全命中多 url,一次 IN 查询返回全部字节,touch 覆盖全部命中。
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
"https://img.example.com/c.jpg");
List<String> hits = new ArrayList<>();
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
for (int i = 0; i < urls.size(); i++) {
hits.add(sha256Hex(urls.get(i)));
bytesByHash.put(hits.get(i), new byte[]{(byte) (i + 1)});
}
stubBatchRead(hits, bytesByHash);
List<byte[]> result = invokeLookupBatch(service, urls);
assertEquals(3, result.size());
for (int i = 0; i < urls.size(); i++) {
assertEquals(bytesByHash.get(sha256Hex(urls.get(i))), result.get(i), "顺序稳定、字节不丢失");
}
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(new ArrayList<>(hits));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:每次行为一致,不产生重复请求/重复 touch。
List<String> urls = List.of("https://img.example.com/a.jpg");
String hash = sha256Hex(urls.get(0));
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{9}));
invokeLookupBatch(service, urls);
invokeLookupBatch(service, urls);
verify(taskImageCacheMapper, times(2)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(List.of(hash));
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_boundary_empty_input() throws Exception {
// 空输入:null/空列表安全返回空结果,不产生任何数据库访问。
List<byte[]> nullResult = invokeLookupBatch(service, null);
assertNotNull(nullResult);
assertTrue(nullResult.isEmpty());
List<byte[]> emptyResult = invokeLookupBatch(service, List.of());
assertNotNull(emptyResult);
assertTrue(emptyResult.isEmpty());
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_boundary_single_item() throws Exception {
// 单 url:不依赖批量路径,命中时单次查询 + 单次 touch。
String url = "https://img.example.com/single.jpg";
String hash = sha256Hex(url);
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{7}));
List<byte[]> result = invokeLookupBatch(service, List.of(url));
assertEquals(1, result.size());
assertEquals(7, result.get(0)[0]);
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
}
@Test
void test_task_014_image_boundary_limit_and_overflow() throws Exception {
// 大批量(超过单批上限 500):分片查询,命中 touch 只覆盖命中集合。
List<String> urls = new ArrayList<>();
for (int i = 0; i < 1200; i++) {
urls.add("https://img.example.com/overflow-" + i + ".jpg");
}
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
List<String> hits = new ArrayList<>();
for (int i = 0; i < 1200; i += 2) {
String hash = sha256Hex(urls.get(i));
hits.add(hash);
bytesByHash.put(hash, new byte[]{(byte) (i % 100)});
}
stubBatchRead(hits, bytesByHash);
List<byte[]> result = invokeLookupBatch(service, urls);
assertEquals(1200, result.size(), "超大批量结果不丢失、顺序稳定");
int hitCount = 0;
for (int i = 0; i < 1200; i++) {
if (i % 2 == 0) {
assertNotNull(result.get(i), "偶数下标命中必须返回字节");
hitCount++;
} else {
assertNull(result.get(i), "奇数下标未命中返回 null");
}
}
assertEquals(600, hitCount);
verify(taskImageCacheMapper, times(3)).selectBytesByUrlHashes(any());
// touch 按单批 500 分片:600 命中 → 2 次 touch 调用,且只覆盖命中集合。
var captor = org.mockito.ArgumentCaptor.forClass(List.class);
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
List<List<String>> touchCalls = new ArrayList<>();
for (Object call : captor.getAllValues()) {
@SuppressWarnings("unchecked")
List<String> casted = (List<String>) call;
touchCalls.add(casted);
}
assertEquals(2, touchCalls.size());
assertEquals(500, touchCalls.get(0).size(), "第一批 touch 500 个命中");
assertEquals(100, touchCalls.get(1).size(), "第二批 touch 剩余 100 个命中");
assertEquals(hits.subList(0, 500), touchCalls.get(0), "touch 只覆盖实际命中集合");
assertEquals(hits.subList(500, 600), touchCalls.get(1));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
@Test
void test_task_014_image_invalid_input_rejected() throws Exception {
// 非法输入:db cache 关闭时批量入口直接返回空,不访问数据库。
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
List<String> urls = List.of("https://img.example.com/a.jpg");
List<byte[]> result = invokeLookupBatch(service, urls);
assertNotNull(result);
assertTrue(result.isEmpty(), "db cache 关闭时必须直接返回空结果");
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
}
@Test
void test_task_014_image_dependency_failure_releases_resources() throws Exception {
// 依赖失败:查询抛异常时批量入口返回空结果、无任何 touch/insert 残留;
// 恢复后重试成功。
List<String> urls = List.of("https://img.example.com/a.jpg");
String hash = sha256Hex(urls.get(0));
AtomicInteger callCount = new AtomicInteger(0);
doAnswer(invocation -> {
if (callCount.getAndIncrement() == 0) {
throw new IllegalStateException("db down");
}
TaskImageCacheEntity row = new TaskImageCacheEntity();
row.setUrlHash(hash);
row.setImageBytes(new byte[]{5});
return List.of(row);
}).when(taskImageCacheMapper).selectBytesByUrlHashes(any());
List<byte[]> failed = invokeLookupBatch(service, urls);
assertNotNull(failed);
assertTrue(failed.isEmpty(), "查询失败必须返回空结果而不是抛错阻断组装");
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
List<byte[]> recovered = invokeLookupBatch(service, urls);
assertEquals(1, recovered.size());
assertEquals(5, recovered.get(0)[0], "依赖恢复后重试成功");
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
}
@Test
void test_task_014_image_normal_single_lookup_legacy_compat() throws Exception {
// 兼容性:单 URL 旧入口 lookup 保持返回字节语义;Task 15 起 touch 改为
// 异步批量缓冲,flush 后批量 touch 一次,命中才入缓冲。
String url = "https://img.example.com/legacy.jpg";
String hash = sha256Hex(url);
byte[] bytes = new byte[]{6};
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(bytes);
byte[] result = service.lookup(url);
assertEquals(bytes, result, "lookup 命中必须返回缓存字节");
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
assertNull(service.lookup("https://img.example.com/missing.jpg"), "未命中返回 null");
service.flushPendingTouches();
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
}
}
@@ -1,167 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper;
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import org.mockito.Mockito;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
/**
* 本地验证入口:用生产真实批次数据 + 生产 LLM 网关跑 SimilarAsinLlmService 完整链路。
* 用法:mvn compile test-compile 后执行
* java -cp target/classes;target/test-classes;$(cat cp.txt) com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]
*/
public class SimilarAsinLlmLocalVerify {
public static void main(String[] args) throws Exception {
System.setOut(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.out), true, "UTF-8"));
System.setErr(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.err), true, "UTF-8"));
if (args.length < 2) {
System.err.println("usage: SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]");
System.exit(1);
}
String dataFile = args[0];
String apiKey = args[1];
boolean imgSwitch = args.length > 2 && Boolean.parseBoolean(args[2]);
boolean categorySwitch = args.length > 3 && Boolean.parseBoolean(args[3]);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
JsonNode root = objectMapper.readTree(new File(dataFile));
List<SimilarAsinResultRowDto> rows = new ArrayList<>();
if (root.isArray()) {
for (JsonNode node : root) {
rows.add(fromJson(objectMapper, node));
}
} else {
rows.add(fromJson(objectMapper, root));
}
System.out.println("[verify] loaded rows=" + rows.size() + " imgSwitch=" + imgSwitch
+ " categorySwitch=" + categorySwitch);
if (imgSwitch && categorySwitch) {
System.out.println("[verify] raw first row alibaba[0].url="
+ (rows.isEmpty() || rows.get(0).getAlibaba().isEmpty() ? "null"
: rows.get(0).getAlibaba().get(0).getUrl()));
}
SimilarAsinProperties props = new SimilarAsinProperties();
props.setLlmApiKey(apiKey);
props.setLlmRowConcurrency(2);
props.setLlmImageDownloadTimeoutSeconds(10);
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null);
OssProperties ossProps = new OssProperties();
ossProps.setEndpoint("https://oss.aishufu.top");
ossProps.setPublicEndpoint("https://oss.aishufu.top");
ossProps.setBucket("nanri-ai-images");
ossProps.setAccessKeyId("appuser");
ossProps.setAccessKeySecret("AppUser@2024SecureKey");
OssStorageService oss = new OssStorageService(ossProps);
PuzzleImageMerger merger = new PuzzleImageMerger(props);
// 生产真实类目数据(导出自 biz_product_category),spy 类目服务按 parentId 过滤。
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> categories = loadCategories(objectMapper);
ProductCategoryService categoryService = Mockito.spy(new ProductCategoryService(
Mockito.mock(com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper.class)));
Mockito.doAnswer(invocation -> {
Long parentId = invocation.getArgument(0);
List<com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo> items = categories.stream()
.filter(c -> parentId == null ? c.getParentId() == null : parentId.equals(c.getParentId()))
.map(c -> {
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo item =
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo();
item.setId(c.getId());
item.setParentId(c.getParentId());
item.setName(c.getName());
item.setCategoryKey(c.getCategoryKey());
return item;
})
.toList();
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo vo =
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo();
vo.setItems(items);
vo.setTree(List.of());
vo.setTotal((long) items.size());
vo.setPage(1L);
vo.setPageSize((long) Math.max(1, items.size()));
vo.setHasMore(false);
return vo;
}).when(categoryService).children(Mockito.any(), Mockito.anyLong(), Mockito.anyLong());
SimilarAsinLlmService service = new SimilarAsinLlmService(
client, props, ossProps, categoryService, merger, oss);
long start = System.currentTimeMillis();
List<SimilarAsinResultRowDto> result = service.inspectRows(rows, null, apiKey, imgSwitch, categorySwitch);
long elapsed = System.currentTimeMillis() - start;
System.out.println("[verify] done rows=" + result.size() + " elapsedMs=" + elapsed);
for (SimilarAsinResultRowDto row : result) {
System.out.println(String.format(
"asin=%s | status=%s | isConform=%s | category=%s | reason=%s | isStock=%s | similarity=%s | mainUrl=%s | puzzle1=%s | puzzle2=%s",
row.getAsin(), row.getStatus(), row.getIsConform(), row.getCategory(),
row.getReason(), row.getIsStock(), row.getSimilarity(),
shorten(row.getMainUrl()), shorten(row.getPuzzleImg1()), shorten(row.getPuzzleImg2())));
}
}
private static List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> loadCategories(ObjectMapper objectMapper) throws Exception {
com.fasterxml.jackson.databind.JsonNode root = objectMapper.readTree(
SimilarAsinLlmLocalVerify.class.getResourceAsStream("/categories.json"));
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> list = new ArrayList<>();
for (com.fasterxml.jackson.databind.JsonNode node : root) {
com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity entity =
new com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity();
entity.setId(node.get("id").asLong());
if (!node.get("parent_id").isNull()) {
entity.setParentId(node.get("parent_id").asLong());
}
entity.setName(node.get("name").asText());
entity.setCategoryKey(node.get("category_key").asText());
entity.setSortOrder(node.get("sort_order").isNull() ? null : node.get("sort_order").asInt());
entity.setDescription(node.get("description").isNull() ? null : node.get("description").asText());
entity.setIsBuiltin(node.get("is_builtin") != null && node.get("is_builtin").asBoolean());
list.add(entity);
}
return list;
}
private static SimilarAsinResultRowDto fromJson(ObjectMapper objectMapper, JsonNode node) throws Exception {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin(text(node, "asin"));
row.setTitle(text(node, "title"));
row.setSku(text(node, "sku"));
row.setCountry(text(node, "country"));
row.setUrl(text(node, "url"));
JsonNode alibaba = node.get("alibaba");
if (alibaba != null && alibaba.isArray()) {
List<SimilarAsinResultRowDto.AlibabaItem> items = new ArrayList<>();
for (JsonNode item : alibaba) {
items.add(objectMapper.treeToValue(item, SimilarAsinResultRowDto.AlibabaItem.class));
}
row.setAlibaba(items);
}
return row;
}
private static String text(JsonNode node, String field) {
JsonNode value = node.get(field);
return value == null || value.isNull() ? null : value.asText();
}
private static String shorten(String value) {
if (value == null) {
return "null";
}
return value.length() <= 70 ? value : value.substring(0, 70) + "...";
}
}
@@ -1,279 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo;
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo;
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.mockito.Mockito;
class SimilarAsinLlmServiceTest {
private static byte[] jpegBytes() {
try {
BufferedImage image = new BufferedImage(10, 10, BufferedImage.TYPE_INT_RGB);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", baos);
return baos.toByteArray();
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
}
private SimilarAsinProperties properties() {
SimilarAsinProperties props = new SimilarAsinProperties();
props.setLlmApiKey("test-key");
return props;
}
private SimilarAsinLlmService service(SimilarAsinLlmClient llmClient) {
OssStorageService ossStorage = mock(OssStorageService.class);
when(ossStorage.getPublicUrl(anyString())).thenAnswer(invocation -> "https://oss.aishufu.top/nanri-ai-images/" + invocation.getArgument(0));
PuzzleImageMerger merger = mock(PuzzleImageMerger.class);
when(merger.merge(anyList(), Mockito.<SimilarAsinResultRowDto>any())).thenReturn(jpegBytes());
ProductCategoryService categoryService = mock(ProductCategoryService.class);
when(categoryService.children(any(), anyLong(), anyLong()))
.thenAnswer(invocation -> {
Long parentId = invocation.getArgument(0);
if (parentId == null) {
return categoryPage("类目A", 1L);
}
if (parentId == 1L) {
return categoryPage("类目B", 2L);
}
if (parentId == 2L) {
return categoryPage("类目C", 3L);
}
return emptyCategoryPage();
});
SimilarAsinLlmService svc = new SimilarAsinLlmService(
llmClient,
properties(),
mock(OssProperties.class),
categoryService,
merger,
ossStorage);
svc.setDownloadHttpClientForTest(mockHttpClient());
return svc;
}
private static ProductCategoryListVo categoryPage(String name, long id) {
ProductCategoryItemVo item = new ProductCategoryItemVo();
item.setId(id);
item.setName(name);
ProductCategoryListVo vo = new ProductCategoryListVo();
vo.setItems(List.of(item));
vo.setTotal(1L);
vo.setPage(1L);
vo.setPageSize(1L);
vo.setHasMore(false);
return vo;
}
private static ProductCategoryListVo emptyCategoryPage() {
ProductCategoryListVo vo = new ProductCategoryListVo();
vo.setItems(List.of());
vo.setTotal(0L);
vo.setPage(1L);
vo.setPageSize(1L);
vo.setHasMore(false);
return vo;
}
private static HttpClient mockHttpClient() {
HttpClient client = mock(HttpClient.class);
try {
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
.thenAnswer(invocation -> {
byte[] body = jpegBytes();
HttpRequest request = invocation.getArgument(0);
return new TestHttpResponse(200, body, request);
});
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
return client;
}
private static final class TestHttpResponse implements HttpResponse<byte[]> {
private final int statusCode;
private final byte[] body;
private final HttpRequest request;
TestHttpResponse(int statusCode, byte[] body, HttpRequest request) {
this.statusCode = statusCode;
this.body = body;
this.request = request;
}
@Override
public int statusCode() {
return statusCode;
}
@Override
public HttpRequest request() {
return request;
}
@Override
public java.util.Optional<HttpResponse<byte[]>> previousResponse() {
return java.util.Optional.empty();
}
@Override
public java.net.http.HttpHeaders headers() {
return java.net.http.HttpHeaders.of(Map.of(), (a, b) -> true);
}
@Override
public byte[] body() {
return body;
}
@Override
public java.net.URI uri() {
return java.net.URI.create("http://test");
}
@Override
public java.net.http.HttpClient.Version version() {
return HttpClient.Version.HTTP_1_1;
}
@Override
public java.util.Optional<javax.net.ssl.SSLSession> sslSession() {
return java.util.Optional.empty();
}
}
private SimilarAsinLlmClient llmClient(String apiKey, Map<String, String> responses) {
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
when(client.resolveApiKey(anyString())).thenReturn(apiKey == null ? "" : apiKey);
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
.thenAnswer(invocation -> {
String response = responses.get("chat");
if (response == null) {
throw new IllegalStateException("unexpected chat call");
}
return response;
});
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
String response = responses.get("images");
if (response == null) {
throw new IllegalStateException("unexpected images call");
}
return response;
});
return client;
}
private static final com.fasterxml.jackson.databind.ObjectMapper OBJECT_MAPPER =
new com.fasterxml.jackson.databind.ObjectMapper();
/** 类目匹配(按序号返回不同类目名)与合规检查(后续)返回不同 JSON。 */
private SimilarAsinLlmClient llmClientStaged(String apiKey, List<String> categoryJsons,
String conformJson, String imagesJson) {
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
int[] categoryIndex = {0};
when(client.resolveApiKey(anyString())).thenReturn(apiKey);
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
.thenAnswer(invocation -> {
if (categoryIndex[0] < categoryJsons.size()) {
return categoryJsons.get(categoryIndex[0]++);
}
return conformJson;
});
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
.thenAnswer(invocation -> imagesJson);
when(client.parseJsonContent(anyString()))
.thenAnswer(invocation -> parseJson(invocation.getArgument(0)));
return client;
}
private static com.fasterxml.jackson.databind.JsonNode parseJson(String content) {
try {
return OBJECT_MAPPER.readTree(content);
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
}
@Test
void noApiKeyKeepsRawRows() {
SimilarAsinLlmService service = service(llmClient("", Map.of()));
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0TEST");
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "", true, true);
assertEquals(1, result.size());
assertEquals("B0TEST", result.get(0).getAsin());
assertNull(result.get(0).getStatus());
}
@Test
void categorySwitchOffOnlyPreparesImagesAndMarksNotExistsWhenNoMainUrl() {
SimilarAsinLlmService service = service(llmClient("k", Map.of()));
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0TEST");
row.setTitle("Test product");
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", false, false);
assertEquals(1, result.size());
assertEquals("不存在", result.get(0).getStatus());
assertNull(result.get(0).getIsConform());
assertNull(result.get(0).getPuzzleImg1());
}
@Test
void imageCompareStopsOnStockAndFillsFields() {
// 前 2 次 chat:一级/二级类目匹配返回名称(Java 按名回查候选取真实 ID);第 3 次:合规检查。
SimilarAsinLlmClient client = llmClientStaged("k", List.of("{\"name\":\"类目A\"}", "{\"name\":\"类目B\"}"),
"{\"asin\":\"B0TEST\",\"is_conform\":\"符合\",\"reason\":\"无\",\"category\":\"类目A->类目B->类目C\"}",
"{\"asin\":\"B0TEST\",\"is_stock\":\"有货\",\"similarity\":\"95%\",\"status\":\"成功\",\"is_conform\":\"符合\",\"category\":\"类目A->类目B->类目C\"}");
SimilarAsinLlmService service = service(client);
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0TEST");
row.setTitle("Test product");
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
SimilarAsinResultRowDto.AlibabaItem item = new SimilarAsinResultRowDto.AlibabaItem();
item.setUrl("https://cbu01.alicdn.com/img/1.jpg");
row.setAlibaba(List.of(item));
// 一级/二级都匹配(按名回查 ID),三级候选可用,合规符合 → 图片对比,有货即停。
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", true, true);
assertEquals(1, result.size());
SimilarAsinResultRowDto out = result.get(0);
assertEquals("成功", out.getStatus());
assertEquals("有货", out.getIsStock());
assertEquals("95%", out.getSimilarity());
assertEquals("符合", out.getIsConform());
}
}
@@ -1,204 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 9:chunk 查询从单行分页改为批量 keyset 分页,保持低内存读取。
* loadChunksKeyset 按 id 递增分批拉取(每批 pageSize),最后按 chunkIndex 升序合并,
* 避免超大任务一次 selectList 全量载入 chunk 元数据。
* mock 分页由 wrapper 中 gt("id", lastId) 的 keyset 值驱动,保证重复调用可复现(幂等)。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceChunkKeysetTest {
private static final String MODULE = "similar-asin";
@Mock private TaskChunkMapper taskChunkMapper;
private static TaskChunkEntity chunk(long id, int chunkIndex) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7004L);
chunk.setModuleType(MODULE);
chunk.setChunkIndex(chunkIndex);
return chunk;
}
private static List<TaskChunkEntity> chunks(long... idsAndIndexes) {
List<TaskChunkEntity> result = new ArrayList<>();
for (int i = 0; i < idsAndIndexes.length; i += 2) {
result.add(chunk(idsAndIndexes[i], (int) idsAndIndexes[i + 1]));
}
return result;
}
/** 从 wrapper 的 SQL 片段解析 keyset:匹配 "id > #{ew.paramNameValuePairs.键}" 后从参数表取值。 */
private static long keysetOf(QueryWrapper<TaskChunkEntity> wrapper) {
java.util.regex.Matcher m = java.util.regex.Pattern
.compile("id\\s*>\\s*#\\{ew\\.paramNameValuePairs\\.(\\w+)\\}", java.util.regex.Pattern.CASE_INSENSITIVE)
.matcher(wrapper.getSqlSegment());
if (m.find()) {
Object value = wrapper.getParamNameValuePairs().get(m.group(1));
if (value instanceof Number number) {
return number.longValue();
}
}
return 0L;
}
/** 按 keyset 驱动分页:每次 selectList 返回 id > keyset 的下一批,天然支持重复调用。 */
private void stubKeysetPages(List<TaskChunkEntity> all, int pageSize) {
int batch = pageSize > 0 ? pageSize : 500;
doAnswer(invocation -> {
long lastId = keysetOf(invocation.getArgument(0));
return all.stream().filter(c -> c.getId() > lastId).limit(batch).toList();
}).when(taskChunkMapper).selectList(any());
}
private List<Long> keysetIdsFromRounds(int rounds) {
ArgumentCaptor<QueryWrapper<TaskChunkEntity>> captor =
ArgumentCaptor.forClass(QueryWrapper.class);
verify(taskChunkMapper, times(rounds)).selectList(captor.capture());
List<Long> keysets = new ArrayList<>();
for (QueryWrapper<TaskChunkEntity> wrapper : captor.getAllValues()) {
keysets.add(keysetOf(wrapper));
}
return keysets;
}
@Test
void test_task_009_chunk_normal_default_path() {
// 正常输入:chunk 数小于 pageSize,一轮拉完,结果全且按 chunkIndex 有序
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
stubKeysetPages(all, 500);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
assertEquals(3, result.size());
assertEquals(List.of(1, 2, 3), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
assertEquals(List.of(0L), keysetIdsFromRounds(1), "首轮 keyset 为 0");
}
@Test
void test_task_009_chunk_normal_multiple_items() {
// 超过 pageSize:多轮拉取,keyset 逐轮推进,全部合并且按 chunkIndex 升序、无重复
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7);
stubKeysetPages(all, 3);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
assertEquals(7, result.size());
assertEquals(List.of(1, 2, 3, 4, 5, 6, 7), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
long distinctIds = result.stream().map(TaskChunkEntity::getId).distinct().count();
assertEquals(7, distinctIds, "keyset 分页不能产生重复 chunk");
assertEquals(List.of(0L, 3L, 6L), keysetIdsFromRounds(3), "keyset 逐轮推进,不足一批即止");
}
@Test
void test_task_009_chunk_normal_repeated_operation_is_idempotent() {
// 重复执行同一输入:每轮都从 keyset=0 开始,结果一致
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
stubKeysetPages(all, 500);
List<TaskChunkEntity> first = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
List<TaskChunkEntity> second = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getId(), second.get(i).getId());
}
assertEquals(List.of(0L, 0L), keysetIdsFromRounds(2), "重复执行每轮都从 keyset=0 开始");
}
@Test
void test_task_009_chunk_boundary_empty_input() {
// 空集合:返回空列表,不创建无效资源,且只查一轮
stubKeysetPages(List.of(), 500);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
assertNotNull(result);
assertEquals(0, result.size());
verify(taskChunkMapper, times(1)).selectList(any());
}
@Test
void test_task_009_chunk_boundary_single_item() {
// 单 chunk:一轮返回后 keyset 推进即拉空,不依赖批量路径
List<TaskChunkEntity> all = chunks(42, 9);
stubKeysetPages(all, 1);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 1);
assertEquals(1, result.size());
assertEquals(9, result.get(0).getChunkIndex());
assertEquals(42L, result.get(0).getId());
}
@Test
void test_task_009_chunk_boundary_limit_and_overflow() {
// chunk 数恰好等于 pageSize 的倍数:最后一轮仍返回非空才继续,全部取回
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6);
stubKeysetPages(all, 3);
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
assertEquals(6, result.size());
// pageSize 为 0/负数:回退默认 500,不抛异常
stubKeysetPages(all, 0);
List<TaskChunkEntity> fallback = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 0);
assertEquals(6, fallback.size());
stubKeysetPages(all, -5);
List<TaskChunkEntity> negative = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, -5);
assertEquals(6, negative.size());
}
@Test
void test_task_009_chunk_invalid_input_rejected() {
// taskId 为 null:安全返回空列表,不发起查询
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, null, MODULE, 500);
assertNotNull(result);
assertEquals(0, result.size());
verify(taskChunkMapper, times(0)).selectList(any());
// mapper 查询抛异常:转项目约定异常,消息可识别
when(taskChunkMapper.selectList(any())).thenThrow(new IllegalStateException("db down"));
BusinessException ex = assertThrows(BusinessException.class,
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
"异常消息必须可识别,实际: " + ex.getMessage());
}
@Test
void test_task_009_chunk_dependency_failure_releases_resources() {
// 第二轮查询失败:抛异常不返回半截结果;恢复后重试可完整返回
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5);
doAnswer(invocation -> {
long lastId = keysetOf(invocation.getArgument(0));
if (lastId == 0L) {
return all.subList(0, 3);
}
if (lastId == 3L) {
throw new IllegalStateException("db down mid-page");
}
return all.subList(3, 5);
}).when(taskChunkMapper).selectList(any());
BusinessException ex = assertThrows(BusinessException.class,
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
"异常消息必须可识别,实际: " + ex.getMessage());
// 恢复后重试成功:5 个 chunk 全部取回
stubKeysetPages(all, 3);
List<TaskChunkEntity> recovered = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
assertEquals(5, recovered.size());
assertEquals(List.of(1, 2, 3, 4, 5), recovered.stream().map(TaskChunkEntity::getChunkIndex).toList());
}
}
@@ -1,399 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 13:chunk 合并增加单次最大行数与 payload 字节上限。
* 校验点位于 mergeChunkPayload 单次合并入口:合并后总行数超过 chunkMergeMaxRows、
* 或 payload 字节超过 chunkMergePayloadMaxBytes 时,从最旧行开始降级到
* orphan 兜底(assemble 阶段 putIfAbsent 合并回结果,不丢数据);
* 单行本身超过字节上限时抛可识别异常拒绝合并。低于上限的行为与旧路径完全一致。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceChunkMergeLimitTest {
private static final AtomicLong NEXT_ID = new AtomicLong(90000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(16L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/90000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String asin, String title) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(rowToken);
r.setAsin(asin);
r.setCountry("英国");
r.setTitle(title);
return r;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(9004L);
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(payloadJson);
return chunk;
}
/** 记录每次 storeChunkPayloadVersioned 收到的 payload 字符串。 */
private void stubChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicReference<String> storedPayload) throws Exception {
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
.thenReturn(payloadJson);
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storedPayload.set(invocation.getArgument(4));
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
}
private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task,
String scopeHash, Integer chunkIndex,
List<SimilarAsinResultRowDto> llmRows) throws Exception {
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, scopeHash, chunkIndex, llmRows, Map.of());
}
@Test
void test_task_013_payload_row_count_chunk_normal_default_path() throws Exception {
// 正常输入:行数与 payload 字节均在上限内,合并走原路径,结果完整保留。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicReference<String> storedPayload = new AtomicReference<>("");
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> llmRows = List.of(
row("r1", "B0A0000001", "标题1"),
row("r2", "B0A0000002", "标题2"),
row("r3", "B0A0000003", "标题3"));
invokeMerge(service, task, "hashA", 1, llmRows);
assertNotNull(storedPayload.get());
assertTrue(storedPayload.get().contains("\"r1\"") && storedPayload.get().contains("\"r3\""),
"上限内合并必须完整保留存量行与新增行,实际: " + storedPayload.get());
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(1)).update(any(), any());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_normal_multiple_items() throws Exception {
// 批量场景:多 chunk 一次 merge,全部在上限内,各 chunk 分别写回、结果不丢失。
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r1", "B0A0000001", "标题1"))));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
.thenReturn(rowsJson(List.of(row("r2", "B0A0000002", "标题2"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
AtomicLong selectOneRound = new AtomicLong(0);
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, null, null, List.of(
row("r1", "B0A0000001", "标题1-新"),
row("r2", "B0A0000002", "标题2-新")));
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(2)).update(any(), any());
}
@Test
void test_task_013_payload_row_count_chunk_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:每次 merge 恰好写一次,不产生重复对象、重复状态。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storeCalls = new AtomicLong(0);
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storeCalls.incrementAndGet();
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "B0A0000001", "标题1"));
invokeMerge(service, task, "hashA", 1, llmRows);
invokeMerge(service, task, "hashA", 1, llmRows);
assertEquals(2, storeCalls.get(), "重复执行同一输入:每次 merge 恰好写回一次,无多余请求");
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_boundary_empty_input() throws Exception {
// 空输入:null/空列表安全跳过,不读取 chunk、不写存储、不创建资源。
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, "hashA", 1, null);
invokeMerge(service, task, "hashA", 1, List.of());
verify(taskChunkMapper, times(0)).selectList(any());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_boundary_single_item() throws Exception {
// 单行:不依赖批量路径,合并后结果正确。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicReference<String> storedPayload = new AtomicReference<>("");
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
assertTrue(storedPayload.get().contains("\"r1\""), "单行合并也必须写回 chunk payload,实际: " + storedPayload.get());
verify(taskChunkMapper, times(1)).update(any(), any());
}
@Test
void test_task_013_payload_row_count_chunk_boundary_limit_and_overflow() throws Exception {
// 超限场景三连:行数超限降级、字节超限降级、单行超字节上限拒绝。
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(2);
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storeCalls = new AtomicLong(0);
AtomicReference<String> lastStoredPayload = new AtomicReference<>("");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storeCalls.incrementAndGet();
lastStoredPayload.set(invocation.getArgument(4));
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
for (int i = 0; i < 5; i++) {
llmRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
}
// Phase A:行数超限(上限 2,存量 1 + 新增 5)→ 只保留上限内最新行,超限部分转 orphan。
invokeMerge(service, task, "hashA", 1, llmRows);
assertTrue(lastStoredPayload.get().contains("\"r4\"") && lastStoredPayload.get().contains("\"r5\""),
"行数超限时保留上限内的最新行,实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "行数超限时最旧行被降级,实际: " + lastStoredPayload.get());
assertEquals(1, storeCalls.get());
verify(taskScopeStateMapper, times(1)).insert(any(TaskScopeStateEntity.class));
verify(transientPayloadStorageService, times(1))
.storeParsedPayloadEntry(anyString(), any(), anyString(), anyString(), anyString(), eq(true));
// Phase B:字节超限(行数放开)→ 从最旧行降级到字节上限内,保留最新结果。
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
long oneRowBytes = rowsJson(List.of(row("r9", "B0A0000099", "样本行"))).getBytes(StandardCharsets.UTF_8).length;
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(oneRowBytes + 5L);
invokeMerge(service, task, "hashA", 1, llmRows);
assertTrue(lastStoredPayload.get().contains("\"r5\""), "字节超限时保留最新行,实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "字节超限时最旧行被降级,实际: " + lastStoredPayload.get());
assertEquals(2, storeCalls.get());
verify(taskScopeStateMapper, times(2)).insert(any(TaskScopeStateEntity.class));
// Phase C:单行本身超过字节上限 → 抛可识别异常拒绝合并,不写 chunk。
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(5L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, llmRows);
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertNotNull(ex.getMessage());
assertTrue(ex.getMessage().contains("字节上限"),
"超字节上限必须抛可识别异常,实际: " + ex.getMessage());
assertEquals(2, storeCalls.get(), "拒绝合并时不写 chunk");
}
@Test
void test_task_013_payload_row_count_chunk_invalid_input_rejected() {
// 非法输入:chunk 载荷加载失败(resolve 抛异常)时抛出可识别异常且不写 chunk。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenThrow(new IllegalStateException("rustfs down"));
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertNotNull(ex.getMessage());
assertTrue(ex.getMessage().contains("chunk"),
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_dependency_failure_releases_resources() throws Exception {
// 依赖失败:payload 存储失败时抛带上下文的可识别异常、无残留状态;恢复后重试成功。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
AtomicLong storeCalls = new AtomicLong(0);
doAnswer(invocation -> {
if (storeCalls.getAndIncrement() == 0) {
throw new IllegalStateException("rustfs down");
}
return "stored:" + invocation.getArgument(2);
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertTrue(ex.getMessage() != null && ex.getMessage().contains("相似ASIN分片载荷"),
"存储失败必须抛带上下文的可识别异常,实际: " + ex.getMessage());
assertEquals(1, storeCalls.get(), "失败时只尝试一次即抛出,不静默吞错");
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
// 依赖恢复后重试成功:结果正确、无残留状态。
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
assertEquals(2, storeCalls.get(), "恢复后重试成功");
verify(taskChunkMapper, times(1)).update(any(), any());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
}
@@ -1,416 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 12:扩展 Coze 结果缓冲覆盖范围,减少频繁读写完整 chunk payload。
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1";Task 12 扩展为:
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
* 3) 统一走 bufferLlmRowsOrMerge:缓冲失败回退立即 merge,结果不丢失。
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceCozeBufferScopeTest {
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
private static final String CREDENTIAL = "cred-1";
@Mock private LocalFileStorageService localFileStorageService;
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinLlmService similarAsinLlmService;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() throws Exception {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(properties.getLlmBatchSize()).thenReturn(5);
lenient().when(properties.getLlmTextOnlyBatchSize()).thenReturn(10);
lenient().when(properties.isLlmResultBufferEnabled()).thenReturn(true);
lenient().when(properties.getDbJobTouchIntervalMillis()).thenReturn(2_000L);
lenient().when(properties.getDbTaskTouchIntervalMillis()).thenReturn(2_000L);
lenient().when(properties.getLlmFlushPendingMinutes()).thenReturn(10);
lenient().when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(
mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
.thenAnswer(invocation -> "rustfs:coze-result/" + NEXT_ID.incrementAndGet());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country, String title) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(id);
r.setAsin(asin);
r.setCountry(country);
r.setTitle(title);
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
return r;
}
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7104L);
chunk.setModuleType(MODULE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(payloadJson);
chunk.setPayloadHash("h-" + id);
return chunk;
}
private static FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(7104L);
task.setModuleType(MODULE);
task.setStatus("RUNNING");
task.setResultJson("{\"categorySwitch\":true}");
return task;
}
private static TaskScopeStateEntity state(FileTaskEntity task, long id, String status, int batchTotal) {
TaskScopeStateEntity state = new TaskScopeStateEntity();
state.setId(id);
state.setTaskId(task.getId());
state.setModuleType(MODULE);
state.setScopeHash("scope-" + id);
state.setLlmStatus(status);
state.setParsedPayloadJson("ptr:batch-" + id);
state.setStateJson("{\"jobId\":7101,\"resultId\":7201,\"chunkScopeHash\":null,\"chunkIndex\":null,"
+ "\"batchIndex\":1,\"batchTotal\":" + batchTotal + ",\"ownerInstanceId\":\"test-instance\","
+ "\"submitRetryCount\":0,\"credentialName\":\"" + CREDENTIAL + "\",\"resultPayloadPointer\":\"ptr:buffer-" + id + "\"}");
return state;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private SimilarAsinTaskService.LlmBatchContext context(int batchTotal) {
return new SimilarAsinTaskService.LlmBatchContext(
7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null);
}
private void stubChunkMerge(String payloadJson) throws Exception {
TaskChunkEntity chunk = chunk(1L, "scope-1", 1, "ptr:chunk-1");
// loadSubmittedChunks 只保留非空 chunk,chunk payload 必须能解析出至少一行。
// 全部 lenient:缓冲成功路径不触达 merge,仅缓冲失败/flush 合并路径消费。
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
lenient().when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
return payloadJson;
}
return "[]";
});
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
}
private static String chunkRowsJson() throws Exception {
// chunk-1 已含 r1 行:loadSubmittedChunks 只保留非空 chunk,且 rowKey 索引能命中缓冲行。
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
}
@Test
void test_task_012_payload_chunk_normal_default_path() throws Exception {
// 正常输入:DONE 结果(batchTotal=1 单 batch)经 bufferLlmRowsOrMerge 走缓冲,
// 不立即写 chunk;缓冲失败回退立即 merge 结果不丢失。
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_normal_multiple_items() throws Exception {
// 批量场景:多个 DONE state(单 batch)全部缓冲;flush 后按 chunk 分组一次合并
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
when(taskScopeStateMapper.selectList(any())).thenReturn(
List.of(state(task, 1L, "DONE", 1), state(task, 2L, "DONE", 1)));
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
}
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
return chunkRowsJson();
}
return "[]";
});
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(1L, "scope-1", 1, "ptr:chunk-1"));
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1")));
when(taskChunkMapper.update(any(), any())).thenReturn(1);
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
flush.setAccessible(true);
flush.invoke(service, 7104L);
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
// pointer 清理:每个缓冲 state 都更新
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入:缓冲写幂等(同一 state 不产生重复 buffer/merge)
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
TaskScopeStateEntity state = state(task, 1L, "DONE", 2);
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
// 缓冲 2 次(每次重新写 pointer 是幂等语义:同一 state 覆盖写,无重复行)
verify(transientPayloadStorageService, times(2)).storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_boundary_empty_input() throws Exception {
// 空输入:无行时缓冲与 merge 都不发生,不创建无效资源
FileTaskEntity task = task();
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of());
bufferOrMerge.invoke(service, state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of());
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_boundary_single_item() throws Exception {
// 单 batch(batchTotal=1):原 P0-3 例外,现在也缓冲
FileTaskEntity task = task();
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
verify(taskChunkMapper, never()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_boundary_limit_and_overflow() throws Exception {
// 缓冲开关关闭:回退立即 merge,DONE 结果仍落 chunk 不丢失
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
when(properties.isLlmResultBufferEnabled()).thenReturn(false);
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
}
@Test
void test_task_012_payload_chunk_invalid_input_rejected() throws Exception {
// 缓冲写失败(storeParsedPayloadEntry 抛异常):回退立即 merge,结果不丢失
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
when(transientPayloadStorageService.storeParsedPayloadEntry(
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)))
.thenThrow(new IllegalStateException("rustfs full"));
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
TaskScopeStateEntity.class,
SimilarAsinTaskService.LlmBatchContext.class,
List.class, FileTaskEntity.class, Map.class);
bufferOrMerge.setAccessible(true);
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
}
@Test
void test_task_012_payload_chunk_dependency_failure_releases_resources() throws Exception {
// flush 时 chunk 写失败:抛可识别业务异常且不清 pointer(保留待重试);
// 依赖恢复后重试 flush 成功,chunk 合并一次、pointer 清理。
FileTaskEntity task = task();
stubChunkMerge(chunkRowsJson());
TaskScopeStateEntity s = state(task, 1L, "DONE", 1);
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(s));
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
}
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
return chunkRowsJson();
}
return "[]";
});
java.util.concurrent.atomic.AtomicInteger storeCalls = new java.util.concurrent.atomic.AtomicInteger(0);
doAnswer(invocation -> {
if (storeCalls.incrementAndGet() == 1) {
throw new IllegalStateException("rustfs write failed");
}
return "ptr:stored-" + invocation.getArgument(3);
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
flush.setAccessible(true);
Exception ex = assertThrows(Exception.class, () -> {
try {
flush.invoke(service, 7104L);
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertTrue(ex.getMessage() != null && ex.getMessage().contains("刷新缓冲区"),
"flush 失败消息必须可识别, 实际: " + ex.getMessage());
// 失败分组不清 pointer:buffer 未被删除、stateJson 未更新,留待重试
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
verify(taskScopeStateMapper, never()).update(any(), any());
// 恢复后重试 flush:chunk 合并成功一次,pointer 清理
flush.invoke(service, 7104L);
assertEquals(2, storeCalls.get(), "恢复后重试应再次写 chunk");
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
}
}
@@ -1,378 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 6:分组数据改为索引/范围引用,避免 groups 嵌套复制完整行对象。
* 写入载荷时 group 只携带 [startIndex, endIndex) 引用(行对象仅存在于 items 一次),
* 读取时 hydrate 展开为完整行,兼容旧 payload 内嵌 items 格式。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceGroupRefTest {
private static final AtomicLong NEXT_ID = new AtomicLong(30000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/30000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
File file = Files.createTempFile("similar-asin-group-ref-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(String.format("B0GRP%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("group-ref.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
private String storedPayloadJson() {
// 捕获最近一次存储的 payload JSON
return "rustfs:task-parsed/similar-asin/30000/payload.json";
}
private SimilarAsinParsedPayloadDto readPayload(String json) throws Exception {
return objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class);
}
private static SimilarAsinParsedRowVo row(String fileKey, int index, String groupKey) {
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
row.setSourceFileKey(fileKey);
row.setSourceFilename("group-ref.xlsx");
row.setRowIndex(index);
row.setSourceId(String.valueOf(index));
row.setDisplayId(String.valueOf(index));
row.setRowToken(fileKey + "::row::" + index);
row.setGroupKey(groupKey);
row.setAsin(String.format("B0GRP%05d", index));
row.setCountry("英国");
row.setValues(new java.util.LinkedHashMap<>());
return row;
}
@Test
void test_task_006_group_normal_default_path() throws Exception {
// 正常多行文件:groups 写入为索引引用,行对象只出现在 items 一次
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenAnswer(invocation -> {
String json = invocation.getArgument(3);
return "rustfs:task-parsed/similar-asin/30000/payload.json::" + json;
});
File workbook = buildWorkbook(150);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-default.xlsx");
assertEquals(150, vo.getAcceptedRows());
// 每个 group 是索引引用:携带 [startIndex, endIndex),区间宽度等于 itemCount
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertNotNull(group.getStartIndex());
assertNotNull(group.getEndIndex());
assertTrue(group.getStartIndex() < group.getEndIndex());
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItemCount());
}
// 响应 groups 按预览上限裁剪(默认 100),引用区间覆盖全部行、不重叠
int coverage = 0;
int prevEnd = -1;
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertTrue(group.getStartIndex() >= prevEnd, "组区间不能重叠且必须顺序递增");
coverage += group.getEndIndex() - group.getStartIndex();
prevEnd = group.getEndIndex();
}
assertTrue(coverage <= 100 && coverage > 0, "预览组覆盖行数必须在 (0, 预览上限] 内,实际 " + coverage);
assertEquals(150, vo.getAcceptedRows());
// 响应组内嵌预览行(前端兼容):每个组 items 与引用区间宽度一致
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertNotNull(group.getItems());
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItems().size(),
"响应组内嵌预览行数量必须与引用区间宽度一致");
}
}
@Test
void test_task_006_group_normal_multiple_items() throws Exception {
// 多组批量:每组行数不同,引用与 items 严格对应且顺序稳定
String json = groupRefJson(3, new int[][]{{0, 3}, {3, 8}, {8, 10}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
assertEquals(10, payload.getItems().size());
assertEquals(3, payload.getGroups().size());
for (int g = 0; g < payload.getGroups().size(); g++) {
SimilarAsinParsedGroupVo group = payload.getGroups().get(g);
int start = group.getStartIndex();
int end = group.getEndIndex();
assertTrue(end - start >= 1);
// 展开后行与 items 对应(首行即 items[start],行内容一致)
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(start, end));
assertEquals(end - start, expanded.size());
assertEquals("t" + (start + 1), expanded.get(0).getRowToken(), "展开首行必须是 items[start]");
assertEquals("B0GRP" + String.format("%05d", start + 1), expanded.get(0).getAsin());
}
}
@Test
void test_task_006_group_normal_repeated_operation_is_idempotent() throws Exception {
// 重复展开同一 payload:结果一致,且不修改 items
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 5}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
List<SimilarAsinParsedRowVo> first = hydrateForTest(payload);
List<SimilarAsinParsedRowVo> second = hydrateForTest(payload);
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
}
assertEquals(5, payload.getItems().size(), "展开不能修改 payload 内部状态");
}
@Test
void test_task_006_group_boundary_empty_input() throws Exception {
// 空 groups:引用列表为空,不创建无效引用
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setItems(List.of());
payload.setGroups(List.of());
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
assertNotNull(restored);
assertEquals(0, restored.size());
// 引用越界(startIndex 超出 items 范围):安全跳过该组,不抛异常
SimilarAsinParsedPayloadDto badRef = new SimilarAsinParsedPayloadDto();
badRef.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setGroupKey("f.xlsx::1");
group.setStartIndex(5);
group.setEndIndex(7);
badRef.setGroups(List.of(group));
List<SimilarAsinParsedRowVo> outOfRange = SimilarAsinTaskService.resolveAllRows(badRef);
assertEquals(0, outOfRange.size(), "越界引用必须安全跳过");
}
@Test
void test_task_006_group_boundary_single_item() throws Exception {
// 单行单组:区间为 [0,1),单行不依赖批量路径
String json = groupRefJson(1, new int[][]{{0, 1}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
assertEquals(1, payload.getGroups().size());
SimilarAsinParsedGroupVo group = payload.getGroups().get(0);
assertEquals(0, group.getStartIndex());
assertEquals(1, group.getEndIndex());
assertEquals(1, group.getItemCount());
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(0, 1));
assertEquals(1, expanded.size());
assertEquals("B0GRP00001", expanded.get(0).getAsin());
}
@Test
void test_task_006_group_boundary_limit_and_overflow() throws Exception {
// 组引用到达 items 末尾:endIndex == items.size(),不越界
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 6}});
SimilarAsinParsedPayloadDto payload = readPayload(json);
assertEquals(6, payload.getItems().size());
SimilarAsinParsedGroupVo last = payload.getGroups().get(1);
assertEquals(6, last.getEndIndex());
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(last.getStartIndex(), last.getEndIndex()));
assertEquals(4, expanded.size());
// 未携带 items 的旧 payload 走 allItems 兜底
String legacy = "{\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0OLD00001\"}],"
+ "\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":1,\"itemCount\":1}]}";
SimilarAsinParsedPayloadDto legacyPayload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class);
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacyPayload);
assertEquals(1, restored.size());
assertEquals("B0OLD00001", restored.get(0).getAsin());
}
@Test
void test_task_006_group_invalid_input_rejected() throws Exception {
// 非法区间:endIndex <= startIndex,安全跳过
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1"), row("f.xlsx", 2, "f.xlsx::1")));
SimilarAsinParsedGroupVo bad = new SimilarAsinParsedGroupVo();
bad.setGroupKey("f.xlsx::1");
bad.setStartIndex(1);
bad.setEndIndex(1);
payload.setGroups(List.of(bad));
assertEquals(0, SimilarAsinTaskService.resolveAllRows(payload).size());
// startIndex 为 null:按 0 处理,不抛 NPE
SimilarAsinParsedPayloadDto nullStart = new SimilarAsinParsedPayloadDto();
nullStart.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
SimilarAsinParsedGroupVo g = new SimilarAsinParsedGroupVo();
g.setGroupKey("f.xlsx::1");
g.setStartIndex(null);
g.setEndIndex(1);
nullStart.setGroups(List.of(g));
assertEquals(1, SimilarAsinTaskService.resolveAllRows(nullStart).size());
}
@Test
void test_task_006_group_dependency_failure_releases_resources() throws Exception {
// RustFS 存储失败:解析抛异常;恢复后重试成功,groups 引用与行一致
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(80);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/gr-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/30001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/gr-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-recovered.xlsx");
assertEquals(80, vo.getAcceptedRows());
int coverage = 0;
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
assertTrue(group.getStartIndex() < group.getEndIndex());
coverage += group.getEndIndex() - group.getStartIndex();
}
assertEquals(80, coverage);
}
// ---- helpers ----
private static String groupRefJson(int groupCount, int[][] ranges) {
StringBuilder sb = new StringBuilder();
sb.append("{\"items\":[");
// 计算总行数
int maxEnd = 0;
for (int[] r : ranges) {
maxEnd = Math.max(maxEnd, r[1]);
}
for (int i = 1; i <= maxEnd; i++) {
if (i > 1) {
sb.append(",");
}
sb.append("{\"rowToken\":\"t").append(i).append("\",\"asin\":\"B0GRP")
.append(String.format("%05d", i)).append("\",\"sourceFileKey\":\"f.xlsx\",\"rowIndex\":")
.append(i).append("}");
}
sb.append("],\"groups\":[");
for (int g = 0; g < groupCount; g++) {
if (g > 0) {
sb.append(",");
}
sb.append("{\"groupKey\":\"g").append(g + 1).append("\",\"startIndex\":")
.append(ranges[g][0]).append(",\"endIndex\":").append(ranges[g][1])
.append(",\"itemCount\":").append(ranges[g][1] - ranges[g][0]).append("}");
}
sb.append("]}");
return sb.toString();
}
private static List<SimilarAsinParsedRowVo> hydrateForTest(SimilarAsinParsedPayloadDto payload) {
// 调用 service 的引用展开实现(与 hydrateParsedPayloadRows 语义一致)
SimilarAsinParsedPayloadDto copy = new SimilarAsinParsedPayloadDto();
copy.setItems(payload.getItems());
copy.setAllItems(payload.getAllItems());
copy.setGroups(payload.getGroups());
return SimilarAsinTaskService.expandGroupRefs(copy);
}
}
@@ -1,275 +0,0 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 7:限制单文件大小、最大行数和最大字段长度,防止解析任务无界增长。
* 超限输入在解析入口被拒绝或截断;mock 依赖 + 真实 xlsx 验证边界行为。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceParseLimitsTest {
private static final AtomicLong NEXT_ID = new AtomicLong(40000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/40000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
return buildWorkbookWithAsin(rowCount, null);
}
private File buildWorkbookWithAsin(int rowCount, String asinValue) throws Exception {
File file = Files.createTempFile("similar-asin-parse-limits-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(asinValue != null ? asinValue : String.format("B0LIM%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("limits.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_007_file_size_row_count_normal_default_path() throws Exception {
// 默认配置(50MB/50000 行/2000 字符):正常文件解析成功,行数不丢失
File workbook = buildWorkbook(120);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-default.xlsx");
assertEquals(120, vo.getAcceptedRows());
assertEquals(120, vo.getTotalRows());
assertEquals(100, vo.getItems().size());
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
// 源文件大小在限制内
assertTrue(workbook.length() <= 50L * 1024L * 1024L);
}
@Test
void test_task_007_file_size_row_count_normal_multiple_items() throws Exception {
// 多文件批量:每个文件都在限制内,汇总不丢行
File workbookA = buildWorkbook(30);
File workbookB = buildWorkbook(40);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-a.xlsx")).thenReturn(workbookA);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-b.xlsx")).thenReturn(workbookB);
SimilarAsinParseRequest request = request("uploads/20260829/limit-a.xlsx");
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
sourceB.setFileKey("uploads/20260829/limit-b.xlsx");
sourceB.setOriginalFilename("limits-b.xlsx");
request.setFiles(List.of(request.getFiles().get(0), sourceB));
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
assertEquals(70, vo.getAcceptedRows());
assertEquals(70, vo.getTotalRows());
assertNotNull(vo.getTaskId());
}
@Test
void test_task_007_file_size_row_count_normal_repeated_operation_is_idempotent() throws Exception {
// 重复解析同一文件:结果一致,不产生重复状态
File workbook = buildWorkbook(60);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/limit-idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/limit-idem.xlsx");
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_007_file_size_row_count_boundary_empty_input() throws Exception {
// 空文件(无有效数据行):抛业务异常,不创建任务
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-empty.xlsx")).thenReturn(workbook);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/limit-empty.xlsx"));
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
}
@Test
void test_task_007_file_size_row_count_boundary_single_item() throws Exception {
// 单行小文件:不依赖批量路径,结果正确
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-single.xlsx");
assertEquals(1, vo.getAcceptedRows());
assertEquals(1, vo.getItems().size());
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
assertEquals(1, vo.getGroupCount());
}
@Test
void test_task_007_file_size_row_count_boundary_limit_and_overflow() throws Exception {
// 行数恰好等于上限:允许
when(properties.getMaxParseRows()).thenReturn(8);
File workbook = buildWorkbook(8);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-exact.xlsx");
assertEquals(8, vo.getAcceptedRows());
// 行数超过上限:拒绝,且不创建任务
when(properties.getMaxParseRows()).thenReturn(3);
File workbookOver = buildWorkbook(4);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-over.xlsx")).thenReturn(workbookOver);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbookOver, "uploads/20260829/limit-over.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("行数"),
"超行数异常消息必须可识别,实际: " + ex.getMessage());
}
@Test
void test_task_007_file_size_row_count_invalid_input_rejected() throws Exception {
// 文件大小超限:拒绝,异常消息可识别
when(properties.getMaxSourceFileBytes()).thenReturn(64L);
File workbook = buildWorkbook(5);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-bigfile.xlsx")).thenReturn(workbook);
assertTrue(workbook.length() > 64L, "测试文件必须超过 64 字节限制,实际 " + workbook.length());
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/limit-bigfile.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("大小限制"),
"超文件大小异常消息必须可识别,实际: " + ex.getMessage());
// 字段长度超限:截断而非拒绝,字段仍非空
when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
when(properties.getMaxFieldLength()).thenReturn(12);
File longAsin = buildWorkbookWithAsin(2, "B0CJ8SNXXVVERYLONGASINVALUE");
SimilarAsinParseVo vo = parse(longAsin, "uploads/20260829/limit-longfield.xlsx");
assertEquals(2, vo.getAcceptedRows());
for (var item : vo.getItems()) {
assertTrue(item.getAsin().length() <= 12, "超长字段必须截断到配置上限");
assertTrue(!item.getAsin().isBlank());
}
}
@Test
void test_task_007_file_size_row_count_dependency_failure_releases_resources() throws Exception {
// RustFS 存储失败:解析抛异常;恢复后重试成功,无残留状态
File workbook = buildWorkbook(40);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/40001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/limit-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-recovered.xlsx");
assertEquals(40, vo.getAcceptedRows());
assertEquals(40, vo.getItems().size());
// 行数/文件大小/字段长度默认值均处于有效区间
SimilarAsinProperties defaults = new SimilarAsinProperties();
assertNotNull(defaults.getMaxParseRows());
assertNotNull(defaults.getMaxSourceFileBytes());
assertNotNull(defaults.getMaxFieldLength());
assertTrue(defaults.getMaxParseRows() >= 1000, "默认最大行数至少 1000");
assertTrue(defaults.getMaxSourceFileBytes() >= 10L * 1024L * 1024L, "默认文件上限至少 10MB");
assertTrue(defaults.getMaxFieldLength() >= 500, "默认字段上限至少 500 字符");
}
}

Some files were not shown because too many files have changed in this diff Show More