task-116: 后台管理新增/导入表单弹窗化 + 分组管理独立菜单与UI优化 + 轻量进度端点
Build Backend JAR / build (push) Has been cancelled
Build Backend JAR / build (push) Has been cancelled
- 后台管理页(admin)所有面板的新增/导入表单改为弹窗操作,原有字段 ID 全部保留、提交逻辑不变;导入删除入口不再触发二次确认拦截 - 分组管理升级为独立菜单(V102 + schema initializer),移除 5 个面板内的管理分组按钮;分组列表改为蓝白主题、增加权限分组横幅 - Python 侧 group-manage 权限守卫(_ensure_backend_menu_access 补充 group-manage) - 引入 V101(biz_task_file_job 复合索引)+ 新增 TaskProgressLight/TaskFileJob 轻量端点与进度聚合作 - 前端 progress-light / page-separated-loads / dispatch-guard 共享模块及单元测试
This commit is contained in:
@@ -114,6 +114,11 @@ OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
|
|||||||
# ===== 本地规划文档(不入库)=====
|
# ===== 本地规划文档(不入库)=====
|
||||||
docs/plans/
|
docs/plans/
|
||||||
docs/*plan*.md
|
docs/*plan*.md
|
||||||
|
backend-java/docs/plans/
|
||||||
|
backend-java/docs/*plan*.md
|
||||||
|
backend-java/docs/specs/
|
||||||
|
backend-java/docs/*audit*.md
|
||||||
|
*.broken/
|
||||||
|
|
||||||
# ===== 进度/规划工具(本地使用,不入库)=====
|
# ===== 进度/规划工具(本地使用,不入库)=====
|
||||||
check_progress.py
|
check_progress.py
|
||||||
|
|||||||
+10
@@ -31,6 +31,8 @@ import org.springframework.web.server.ResponseStatusException;
|
|||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -98,6 +100,14 @@ public class AppearancePatentController {
|
|||||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks/{taskId}/activate")
|
@PostMapping("/tasks/{taskId}/activate")
|
||||||
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING,并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新,不需要单独心跳接口。")
|
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING,并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新,不需要单独心跳接口。")
|
||||||
public ApiResponse<Void> activate(
|
public ApiResponse<Void> activate(
|
||||||
|
|||||||
+23
-266
@@ -24,6 +24,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSour
|
|||||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentHistoryAssembler;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||||
@@ -31,6 +32,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParse
|
|||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskDetailVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskDetailVo;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskItemVo;
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskItemVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentSheetBuilder;
|
||||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
@@ -88,6 +90,8 @@ import java.util.UUID;
|
|||||||
import java.util.function.Supplier;
|
import java.util.function.Supplier;
|
||||||
import java.util.zip.ZipEntry;
|
import java.util.zip.ZipEntry;
|
||||||
import java.util.zip.ZipOutputStream;
|
import java.util.zip.ZipOutputStream;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -95,6 +99,10 @@ import java.util.zip.ZipOutputStream;
|
|||||||
public class AppearancePatentTaskService {
|
public class AppearancePatentTaskService {
|
||||||
|
|
||||||
public static final String MODULE_TYPE = "APPEARANCE_PATENT";
|
public static final String MODULE_TYPE = "APPEARANCE_PATENT";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String STATUS_PENDING = "PENDING";
|
private static final String STATUS_PENDING = "PENDING";
|
||||||
private static final String STATUS_RUNNING = "RUNNING";
|
private static final String STATUS_RUNNING = "RUNNING";
|
||||||
private static final String STATUS_SUCCESS = "SUCCESS";
|
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||||
@@ -107,18 +115,6 @@ public class AppearancePatentTaskService {
|
|||||||
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
||||||
private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
|
private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
|
||||||
private static final long TASK_LOCK_RETRY_DELAY_MILLIS = 200L;
|
private static final long TASK_LOCK_RETRY_DELAY_MILLIS = 200L;
|
||||||
private static final List<String> RESULT_HEADERS = List.of(
|
|
||||||
"id",
|
|
||||||
"asin",
|
|
||||||
"国家",
|
|
||||||
"卖家名称",
|
|
||||||
"品牌",
|
|
||||||
"价格",
|
|
||||||
"标题维度(商标)",
|
|
||||||
"外观维度(外观设计专利)",
|
|
||||||
"结论",
|
|
||||||
"状态"
|
|
||||||
);
|
|
||||||
|
|
||||||
private final LocalFileStorageService localFileStorageService;
|
private final LocalFileStorageService localFileStorageService;
|
||||||
private final OssStorageService ossStorageService;
|
private final OssStorageService ossStorageService;
|
||||||
@@ -138,6 +134,7 @@ public class AppearancePatentTaskService {
|
|||||||
private final DistributedJobLockService distributedJobLockService;
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
private final InstanceMetadata instanceMetadata;
|
private final InstanceMetadata instanceMetadata;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) {
|
public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) {
|
||||||
long startedAt = System.nanoTime();
|
long startedAt = System.nanoTime();
|
||||||
@@ -2199,44 +2196,8 @@ public class AppearancePatentTaskService {
|
|||||||
List<AppearancePatentParsedRowVo> receivedRows,
|
List<AppearancePatentParsedRowVo> receivedRows,
|
||||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
|
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
|
||||||
Sheet sheet = workbook.createSheet("外观专利检测结果");
|
AppearancePatentSheetBuilder.buildResultSheet(workbook, receivedRows, resultMap,
|
||||||
CellStyle headerStyle = workbook.createCellStyle();
|
parsedRow -> findResultRow(parsedRow, resultMap));
|
||||||
Font font = workbook.createFont();
|
|
||||||
font.setBold(true);
|
|
||||||
headerStyle.setFont(font);
|
|
||||||
|
|
||||||
List<String> resultHeaders = new ArrayList<>(RESULT_HEADERS);
|
|
||||||
resultHeaders.add(6, "标题");
|
|
||||||
resultHeaders.add(7, "图片链接");
|
|
||||||
resultHeaders.add(8, "sku");
|
|
||||||
Row header = sheet.createRow(0);
|
|
||||||
for (int i = 0; i < resultHeaders.size(); i++) {
|
|
||||||
Cell cell = header.createCell(i);
|
|
||||||
cell.setCellValue(resultHeaders.get(i));
|
|
||||||
cell.setCellStyle(headerStyle);
|
|
||||||
}
|
|
||||||
|
|
||||||
int rowIndex = 1;
|
|
||||||
List<AppearancePatentParsedRowVo> rowsToWrite = receivedRows == null ? List.of() : receivedRows;
|
|
||||||
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite) {
|
|
||||||
AppearancePatentResultRowDto resultRow = findResultRow(parsedRow, resultMap);
|
|
||||||
Row row = sheet.createRow(rowIndex++);
|
|
||||||
int col = 0;
|
|
||||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
|
|
||||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
|
|
||||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
|
|
||||||
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
|
|
||||||
row.createCell(col++).setCellValue(resolveBrand(resultRow, parsedRow));
|
|
||||||
row.createCell(col++).setCellValue(resolvePrice(resultRow));
|
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
|
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
|
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
|
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()));
|
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()));
|
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
|
|
||||||
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
|
|
||||||
}
|
|
||||||
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap);
|
|
||||||
workbook.write(fos);
|
workbook.write(fos);
|
||||||
workbook.dispose();
|
workbook.dispose();
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
@@ -2244,36 +2205,6 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeReasonSheet(SXSSFWorkbook workbook,
|
|
||||||
CellStyle headerStyle,
|
|
||||||
List<AppearancePatentParsedRowVo> rowsToWrite,
|
|
||||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
|
||||||
Sheet sheet = workbook.createSheet("原因");
|
|
||||||
Row header = sheet.createRow(0);
|
|
||||||
List<String> headers = List.of("ASIN", "外观原因", "专利原因", "标题原因");
|
|
||||||
for (int i = 0; i < headers.size(); i++) {
|
|
||||||
Cell cell = header.createCell(i);
|
|
||||||
cell.setCellValue(headers.get(i));
|
|
||||||
cell.setCellStyle(headerStyle);
|
|
||||||
}
|
|
||||||
|
|
||||||
Set<String> writtenAsins = new LinkedHashSet<>();
|
|
||||||
int rowIndex = 1;
|
|
||||||
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite == null ? List.<AppearancePatentParsedRowVo>of() : rowsToWrite) {
|
|
||||||
String asin = normalize(parsedRow.getAsin()).toUpperCase(Locale.ROOT);
|
|
||||||
if (asin.isBlank() || !writtenAsins.add(asin)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
AppearancePatentResultRowDto resultRow = findResultRow(parsedRow, resultMap);
|
|
||||||
Row row = sheet.createRow(rowIndex++);
|
|
||||||
row.createCell(0).setCellValue(asin);
|
|
||||||
row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), ""));
|
|
||||||
row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), ""));
|
|
||||||
row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), ""));
|
|
||||||
rowIndex++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private AppearancePatentResultRowDto findResultRowByAsin(String asin, Map<String, AppearancePatentResultRowDto> resultMap) {
|
private AppearancePatentResultRowDto findResultRowByAsin(String asin, Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||||
String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT);
|
String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT);
|
||||||
if (normalizedAsin.isBlank() || resultMap == null || resultMap.isEmpty()) {
|
if (normalizedAsin.isBlank() || resultMap == null || resultMap.isEmpty()) {
|
||||||
@@ -2496,7 +2427,7 @@ public class AppearancePatentTaskService {
|
|||||||
|
|
||||||
private boolean hasUsableLlmField(String value) {
|
private boolean hasUsableLlmField(String value) {
|
||||||
String normalized = normalize(value);
|
String normalized = normalize(value);
|
||||||
return !normalized.isBlank() && !isTechnicalLlmFailure(normalized);
|
return !normalized.isBlank() && !AppearancePatentSheetBuilder.isTechnicalLlmFailure(normalized);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||||
@@ -2641,119 +2572,33 @@ public class AppearancePatentTaskService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private AppearancePatentHistoryAssembler historyAssembler() {
|
||||||
|
return new AppearancePatentHistoryAssembler(
|
||||||
|
taskProgressSnapshotService,
|
||||||
|
this::calculateDisplayProgressPercent,
|
||||||
|
this::extractSnapshotDisplayPercent);
|
||||||
|
}
|
||||||
|
|
||||||
private AppearancePatentHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
private AppearancePatentHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||||
AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo();
|
return historyAssembler().toHistoryItem(row, task, job);
|
||||||
vo.setResultId(row.getId());
|
|
||||||
vo.setTaskId(row.getTaskId());
|
|
||||||
vo.setSourceFilename(row.getSourceFilename());
|
|
||||||
vo.setResultFilename(row.getResultFilename());
|
|
||||||
vo.setDownloadUrl(null);
|
|
||||||
attachFileJobState(vo, row, job);
|
|
||||||
vo.setTaskStatus(task == null ? null : task.getStatus());
|
|
||||||
vo.setSuccess(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()
|
|
||||||
|| row.getSuccess() != null && row.getSuccess() == 1);
|
|
||||||
vo.setError(row.getErrorMessage());
|
|
||||||
vo.setRowCount(row.getRowCount());
|
|
||||||
vo.setCreatedAt(fmt(row.getCreatedAt()));
|
|
||||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
|
||||||
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
|
|
||||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
|
||||||
return vo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
private int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||||
String taskStatus = task == null ? null : task.getStatus();
|
return historyAssembler().historyPriority(row, task, job);
|
||||||
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private LocalDateTime historyActivityTime(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
private LocalDateTime historyActivityTime(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||||
LocalDateTime latest = latestTime(
|
return historyAssembler().historyActivityTime(row, task, job);
|
||||||
task == null ? null : task.getUpdatedAt(),
|
|
||||||
job == null ? null : job.getUpdatedAt(),
|
|
||||||
task == null ? null : task.getFinishedAt(),
|
|
||||||
row == null ? null : row.getCreatedAt(),
|
|
||||||
task == null ? null : task.getCreatedAt());
|
|
||||||
return latest == null && row != null ? row.getCreatedAt() : latest;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isHistoryFileBuilding(FileResultEntity row, String taskStatus, TaskFileJobEntity job) {
|
private boolean isHistoryFileBuilding(FileResultEntity row, String taskStatus, TaskFileJobEntity job) {
|
||||||
if (!STATUS_SUCCESS.equals(taskStatus)) {
|
return historyAssembler().isHistoryFileBuilding(row, taskStatus, job);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
boolean fileReady = row != null && row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank();
|
|
||||||
if (fileReady) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
String fileStatus = job == null ? null : job.getStatus();
|
|
||||||
return !STATUS_SUCCESS.equals(fileStatus) && !STATUS_FAILED.equals(fileStatus);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void attachFileJobState(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
|
|
||||||
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
|
|
||||||
if (job == null) {
|
|
||||||
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
|
||||||
attachFileProgress(vo, row, job);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
vo.setFileJobId(job.getId());
|
|
||||||
vo.setFileStatus(job.getStatus());
|
|
||||||
vo.setFileError(STATUS_FAILED.equals(job.getStatus()) ? firstNonBlank(job.getErrorMessage(), null) : null);
|
|
||||||
attachFileProgress(vo, row, job);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void attachFileProgress(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
|
|
||||||
if (row == null || row.getTaskId() == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (Boolean.TRUE.equals(vo.getFileReady())) {
|
|
||||||
vo.setFileProgressCurrent(1);
|
|
||||||
vo.setFileProgressTotal(1);
|
|
||||||
vo.setFileProgressPercent(100);
|
|
||||||
vo.setFileProgressMessage("结果文件已生成");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(row.getTaskId(), MODULE_TYPE);
|
|
||||||
if (snapshot == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
|
||||||
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
|
||||||
if (total <= 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
|
|
||||||
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
|
|
||||||
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
|
|
||||||
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
|
|
||||||
vo.setFileProgressCurrent(current);
|
|
||||||
vo.setFileProgressTotal(total);
|
|
||||||
vo.setFileProgressPercent(percent);
|
|
||||||
vo.setFileProgressMessage(snapshot.getMessage());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String fmt(LocalDateTime t) {
|
private String fmt(LocalDateTime t) {
|
||||||
return t == null ? null : t.toString();
|
return t == null ? null : t.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
private LocalDateTime latestTime(LocalDateTime... values) {
|
|
||||||
LocalDateTime latest = null;
|
|
||||||
if (values == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
for (LocalDateTime value : values) {
|
|
||||||
if (value == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (latest == null || value.isAfter(latest)) {
|
|
||||||
latest = value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return latest;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String firstNonBlank(String preferred, String fallback) {
|
private String firstNonBlank(String preferred, String fallback) {
|
||||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
}
|
}
|
||||||
@@ -3004,43 +2849,6 @@ public class AppearancePatentTaskService {
|
|||||||
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
|
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String readValueByHeader(AppearancePatentParsedRowVo row, String... candidates) {
|
|
||||||
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
|
|
||||||
String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT);
|
|
||||||
for (String candidate : candidates) {
|
|
||||||
String expected = normalize(candidate).toLowerCase(Locale.ROOT);
|
|
||||||
if (!expected.isBlank() && header.contains(expected)) {
|
|
||||||
return entry.getValue() == null ? "" : entry.getValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
|
|
||||||
static String resolveBrand(AppearancePatentResultRowDto resultRow, AppearancePatentParsedRowVo parsedRow) {
|
|
||||||
String pythonBrand = resultRow == null ? "" : resultRow.getBrand();
|
|
||||||
if (pythonBrand != null && !pythonBrand.isBlank()) {
|
|
||||||
return pythonBrand.trim();
|
|
||||||
}
|
|
||||||
if (parsedRow == null || parsedRow.getValues() == null) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
for (Map.Entry<String, String> entry : parsedRow.getValues().entrySet()) {
|
|
||||||
String header = entry.getKey() == null ? "" : entry.getKey().trim().toLowerCase(Locale.ROOT);
|
|
||||||
if (header.contains("品牌") || header.contains("brand")) {
|
|
||||||
return entry.getValue() == null ? "" : entry.getValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
|
|
||||||
static String resolvePrice(AppearancePatentResultRowDto resultRow) {
|
|
||||||
return resultRow == null || resultRow.getPrice() == null ? "" : resultRow.getPrice().trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
|
static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
|
||||||
if (waitingForAssemble) {
|
if (waitingForAssemble) {
|
||||||
return STATUS_RUNNING;
|
return STATUS_RUNNING;
|
||||||
@@ -3048,57 +2856,6 @@ public class AppearancePatentTaskService {
|
|||||||
return executionFailed ? STATUS_FAILED : STATUS_SUCCESS;
|
return executionFailed ? STATUS_FAILED : STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String userFacingLlmCellValue(AppearancePatentResultRowDto row, String value) {
|
|
||||||
String normalizedValue = normalize(value);
|
|
||||||
if (!normalizedValue.isBlank() && !isTechnicalLlmFailure(normalizedValue)) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
|
|
||||||
if (row != null && isTechnicalLlmFailure(row.getError())) {
|
|
||||||
return firstNonBlank(row.getError(), "");
|
|
||||||
}
|
|
||||||
return firstNonBlank(value, "");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String userFacingConclusion(AppearancePatentResultRowDto row) {
|
|
||||||
if (row == null) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
String conclusion = normalize(row.getConclusion());
|
|
||||||
if (!conclusion.isBlank() && !isTechnicalLlmFailure(conclusion)) {
|
|
||||||
return row.getConclusion();
|
|
||||||
}
|
|
||||||
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
|
|
||||||
if (isTechnicalLlmFailure(row.getError())) {
|
|
||||||
return firstNonBlank(row.getError(), "");
|
|
||||||
}
|
|
||||||
return firstNonBlank(row.getConclusion(), "");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private String userFacingStatus(AppearancePatentResultRowDto row) {
|
|
||||||
if (row == null) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
String conclusion = userFacingConclusion(row);
|
|
||||||
return resolveResultStatus(conclusion);
|
|
||||||
}
|
|
||||||
|
|
||||||
static String resolveResultStatus(String conclusion) {
|
|
||||||
String normalized = conclusion == null ? "" : conclusion.trim();
|
|
||||||
return normalized.isBlank() ? "\u5931\u8d25" : "\u6210\u529f";
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isTechnicalLlmFailure(String value) {
|
|
||||||
String normalized = normalize(value).toLowerCase(Locale.ROOT);
|
|
||||||
return normalized.contains("coze") || normalized.contains("llm")
|
|
||||||
|| normalized.contains("结果不完整")
|
|
||||||
|| normalized.contains("工作流节点执行超限")
|
|
||||||
|| normalized.contains("调用超时")
|
|
||||||
|| normalized.contains("timeout");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private String safeFileStem(String filename) {
|
private String safeFileStem(String filename) {
|
||||||
String name = filename == null || filename.isBlank() ? "appearance-patent" : filename;
|
String name = filename == null || filename.isBlank() ? "appearance-patent" : filename;
|
||||||
int idx = name.lastIndexOf('.');
|
int idx = name.lastIndexOf('.');
|
||||||
|
|||||||
+225
@@ -0,0 +1,225 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 93:AppearancePatentExcelParser 行解析器。
|
||||||
|
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 AppearancePatentTaskService.parseWorkbook
|
||||||
|
* 中对应段落一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、表头别名匹配(contains + 精确别名)、
|
||||||
|
* 空行跳过、必填表头缺失抛错。注意:appearancepatent 无 2000 字段截断、无错误值转空(与 similarasin 不同)。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class AppearancePatentExcelParser {
|
||||||
|
|
||||||
|
public ParsedSheet parse(File input) {
|
||||||
|
if (input == null) {
|
||||||
|
throw new IllegalArgumentException("input must not be null");
|
||||||
|
}
|
||||||
|
try (FileInputStream fis = new FileInputStream(input);
|
||||||
|
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||||
|
return parseWorkbook(workbook);
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
throw ex;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent] parse failed file={} err={}", input.getName(), ex.getMessage());
|
||||||
|
throw new BusinessException("解析 Excel 失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ParsedSheet parse(InputStream input) {
|
||||||
|
if (input == null) {
|
||||||
|
throw new IllegalArgumentException("input must not be null");
|
||||||
|
}
|
||||||
|
try (Workbook workbook = WorkbookFactory.create(input)) {
|
||||||
|
return parseWorkbook(workbook);
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
throw ex;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent] parse failed input err={}", ex.getMessage());
|
||||||
|
throw new BusinessException("解析 Excel 失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ParsedSheet parseWorkbook(Workbook workbook) {
|
||||||
|
DataFormatter formatter = new DataFormatter();
|
||||||
|
Sheet sheet = workbook.getSheetAt(0);
|
||||||
|
Row header = sheet.getRow(0);
|
||||||
|
if (header == null) {
|
||||||
|
throw new BusinessException("Excel 表头为空");
|
||||||
|
}
|
||||||
|
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
||||||
|
List<String> headers = readHeaders(header, formatter);
|
||||||
|
int idCol = findRequiredHeader(headerMap, "id");
|
||||||
|
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||||
|
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||||
|
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||||
|
int skuCol = findOptionalHeaderExact(headerMap,
|
||||||
|
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
|
||||||
|
int urlCol = findOptionalHeaderExact(headerMap,
|
||||||
|
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||||
|
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||||
|
int titleCol = findOptionalHeaderExact(headerMap,
|
||||||
|
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||||
|
|
||||||
|
List<AppearanceExcelRow> rows = new ArrayList<>();
|
||||||
|
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||||
|
Row row = sheet.getRow(i);
|
||||||
|
if (row == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String id = cell(row, idCol, formatter);
|
||||||
|
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
|
||||||
|
String country = cell(row, countryCol, formatter);
|
||||||
|
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rows.add(new AppearanceExcelRow(
|
||||||
|
i + 1,
|
||||||
|
id,
|
||||||
|
asin,
|
||||||
|
country,
|
||||||
|
priceCol >= 0 ? cell(row, priceCol, formatter) : "",
|
||||||
|
skuCol >= 0 ? cell(row, skuCol, formatter) : "",
|
||||||
|
urlCol >= 0 ? cell(row, urlCol, formatter) : "",
|
||||||
|
titleCol >= 0 ? cell(row, titleCol, formatter) : "",
|
||||||
|
readRowValues(row, headers, formatter)));
|
||||||
|
}
|
||||||
|
return new ParsedSheet(headers, rows);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||||
|
Map<String, Integer> map = new LinkedHashMap<>();
|
||||||
|
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||||
|
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||||
|
if (!val.isBlank()) {
|
||||||
|
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> readHeaders(Row header, DataFormatter formatter) {
|
||||||
|
List<String> headers = new ArrayList<>();
|
||||||
|
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||||
|
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||||
|
headers.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||||
|
}
|
||||||
|
return headers;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
for (int i = 0; i < headers.size(); i++) {
|
||||||
|
values.put(headers.get(i), cell(row, i, formatter));
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int findRequiredHeader(Map<String, Integer> map, String... names) {
|
||||||
|
int idx = findOptionalHeader(map, names);
|
||||||
|
if (idx < 0) {
|
||||||
|
throw new BusinessException("缺少必要表头: " + String.join("/", names));
|
||||||
|
}
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int findOptionalHeader(Map<String, Integer> map, String... names) {
|
||||||
|
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||||
|
for (String name : names) {
|
||||||
|
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
|
||||||
|
return entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
|
||||||
|
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||||
|
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
|
||||||
|
for (String name : names) {
|
||||||
|
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
|
||||||
|
return entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeHeaderAlias(String value) {
|
||||||
|
String normalized = normalize(value).toLowerCase(Locale.ROOT);
|
||||||
|
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String cell(Row row, int col, DataFormatter formatter) {
|
||||||
|
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String val) {
|
||||||
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
public record ParsedSheet(List<String> headers, List<AppearanceExcelRow> rows) {
|
||||||
|
|
||||||
|
public ParsedSheet {
|
||||||
|
headers = headers == null ? List.of() : new ArrayList<>(headers);
|
||||||
|
rows = rows == null ? List.of() : new ArrayList<>(rows);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<String> headers() {
|
||||||
|
return java.util.Collections.unmodifiableList(headers);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<AppearanceExcelRow> rows() {
|
||||||
|
return java.util.Collections.unmodifiableList(rows);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public record AppearanceExcelRow(int rowIndex, String id, String asin, String country,
|
||||||
|
String price, String sku, String url, String title,
|
||||||
|
Map<String, String> values) {
|
||||||
|
|
||||||
|
public AppearanceExcelRow {
|
||||||
|
values = values == null ? Map.of() : new LinkedHashMap<>(values);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (!(o instanceof AppearanceExcelRow other)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return rowIndex == other.rowIndex
|
||||||
|
&& Objects.equals(id, other.id)
|
||||||
|
&& Objects.equals(asin, other.asin)
|
||||||
|
&& Objects.equals(country, other.country)
|
||||||
|
&& Objects.equals(price, other.price)
|
||||||
|
&& Objects.equals(sku, other.sku)
|
||||||
|
&& Objects.equals(url, other.url)
|
||||||
|
&& Objects.equals(title, other.title)
|
||||||
|
&& values.equals(other.values);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(rowIndex, id, asin, country, price, sku, url, title, values);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+161
@@ -0,0 +1,161 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.function.Function;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 97:AppearancePatentHistoryAssembler 历史查询组装器。
|
||||||
|
* 历史列表 VO 拼装(toHistoryItem + 排序辅助 historyPriority/historyActivityTime +
|
||||||
|
* 文件进度链)从 AppearancePatentTaskService 原样搬移;只读不落库;输出与现状逐字段一致。
|
||||||
|
* 进度百分比计算(calculateDisplayProgressPercent/extractSnapshotDisplayPercent)仍在
|
||||||
|
* 服务侧(与 saveFileBuildProgress 实时路径共享),以函数注入。
|
||||||
|
*/
|
||||||
|
public class AppearancePatentHistoryAssembler {
|
||||||
|
|
||||||
|
private static final String STATUS_PENDING = "PENDING";
|
||||||
|
private static final String STATUS_RUNNING = "RUNNING";
|
||||||
|
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||||
|
private static final String STATUS_FAILED = "FAILED";
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface ProgressPercentCalculator {
|
||||||
|
int calculate(int current, int total, TaskFileJobEntity job, LocalDateTime baseTime);
|
||||||
|
}
|
||||||
|
|
||||||
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
private final ProgressPercentCalculator calculateDisplayProgressPercent;
|
||||||
|
private final Function<TaskProgressSnapshotEntity, Integer> extractSnapshotDisplayPercent;
|
||||||
|
|
||||||
|
public AppearancePatentHistoryAssembler(TaskProgressSnapshotService taskProgressSnapshotService,
|
||||||
|
ProgressPercentCalculator calculateDisplayProgressPercent,
|
||||||
|
Function<TaskProgressSnapshotEntity, Integer> extractSnapshotDisplayPercent) {
|
||||||
|
this.taskProgressSnapshotService = taskProgressSnapshotService;
|
||||||
|
this.calculateDisplayProgressPercent = calculateDisplayProgressPercent;
|
||||||
|
this.extractSnapshotDisplayPercent = extractSnapshotDisplayPercent;
|
||||||
|
}
|
||||||
|
|
||||||
|
public AppearancePatentHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||||
|
AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo();
|
||||||
|
vo.setResultId(row.getId());
|
||||||
|
vo.setTaskId(row.getTaskId());
|
||||||
|
vo.setSourceFilename(row.getSourceFilename());
|
||||||
|
vo.setResultFilename(row.getResultFilename());
|
||||||
|
vo.setDownloadUrl(null);
|
||||||
|
attachFileJobState(vo, row, job);
|
||||||
|
vo.setTaskStatus(task == null ? null : task.getStatus());
|
||||||
|
vo.setSuccess(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()
|
||||||
|
|| row.getSuccess() != null && row.getSuccess() == 1);
|
||||||
|
vo.setError(row.getErrorMessage());
|
||||||
|
vo.setRowCount(row.getRowCount());
|
||||||
|
vo.setCreatedAt(fmt(row.getCreatedAt()));
|
||||||
|
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||||
|
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
|
||||||
|
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||||
|
String taskStatus = task == null ? null : task.getStatus();
|
||||||
|
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public LocalDateTime historyActivityTime(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||||
|
LocalDateTime latest = latestTime(
|
||||||
|
task == null ? null : task.getUpdatedAt(),
|
||||||
|
job == null ? null : job.getUpdatedAt(),
|
||||||
|
task == null ? null : task.getFinishedAt(),
|
||||||
|
row == null ? null : row.getCreatedAt(),
|
||||||
|
task == null ? null : task.getCreatedAt());
|
||||||
|
return latest == null && row != null ? row.getCreatedAt() : latest;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isHistoryFileBuilding(FileResultEntity row, String taskStatus, TaskFileJobEntity job) {
|
||||||
|
if (!STATUS_SUCCESS.equals(taskStatus)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
boolean fileReady = row != null && row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank();
|
||||||
|
if (fileReady) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String fileStatus = job == null ? null : job.getStatus();
|
||||||
|
return !STATUS_SUCCESS.equals(fileStatus) && !STATUS_FAILED.equals(fileStatus);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
|
||||||
|
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
|
||||||
|
if (job == null) {
|
||||||
|
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
attachFileProgress(vo, row, job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vo.setFileJobId(job.getId());
|
||||||
|
vo.setFileStatus(job.getStatus());
|
||||||
|
vo.setFileError(STATUS_FAILED.equals(job.getStatus()) ? firstNonBlank(job.getErrorMessage(), null) : null);
|
||||||
|
attachFileProgress(vo, row, job);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void attachFileProgress(AppearancePatentHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
|
||||||
|
if (row == null || row.getTaskId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (Boolean.TRUE.equals(vo.getFileReady())) {
|
||||||
|
vo.setFileProgressCurrent(1);
|
||||||
|
vo.setFileProgressTotal(1);
|
||||||
|
vo.setFileProgressPercent(100);
|
||||||
|
vo.setFileProgressMessage("结果文件已生成");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(row.getTaskId(), AppearancePatentTaskService.MODULE_TYPE);
|
||||||
|
if (snapshot == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
||||||
|
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
||||||
|
if (total <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
|
||||||
|
int percent = calculateDisplayProgressPercent.calculate(current, total, job, baseTime);
|
||||||
|
percent = Math.max(percent, extractSnapshotDisplayPercent.apply(snapshot));
|
||||||
|
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
|
||||||
|
vo.setFileProgressCurrent(current);
|
||||||
|
vo.setFileProgressTotal(total);
|
||||||
|
vo.setFileProgressPercent(percent);
|
||||||
|
vo.setFileProgressMessage(snapshot.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
private String fmt(LocalDateTime t) {
|
||||||
|
return t == null ? null : t.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private LocalDateTime latestTime(LocalDateTime... values) {
|
||||||
|
LocalDateTime latest = null;
|
||||||
|
if (values == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
for (LocalDateTime value : values) {
|
||||||
|
if (value == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (latest == null || value.isAfter(latest)) {
|
||||||
|
latest = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return latest;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String firstNonBlank(String preferred, String fallback) {
|
||||||
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
+31
@@ -0,0 +1,31 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 93:AppearancePatentRowNormalizer 字段归一化器。
|
||||||
|
* 规则与 AppearancePatentTaskService.normalize / firstNonBlank / baseId / normalizeDisplayId
|
||||||
|
* 现状逐字节一致(BOM 剥离、全角空格转半角、trim、连续空白折叠为单空格;
|
||||||
|
* firstNonBlank 取首非空并 trim;baseId 取下划线前块基 id;displayId 仅 trim)。纯函数无状态。
|
||||||
|
*/
|
||||||
|
public final class AppearancePatentRowNormalizer {
|
||||||
|
|
||||||
|
private AppearancePatentRowNormalizer() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String normalize(String val) {
|
||||||
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String firstNonBlank(String preferred, String fallback) {
|
||||||
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String baseId(String id) {
|
||||||
|
String s = normalize(id);
|
||||||
|
int idx = s.indexOf('_');
|
||||||
|
return idx > 0 ? s.substring(0, idx) : s;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String normalizeDisplayId(String id) {
|
||||||
|
return id == null ? "" : id.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
+265
@@ -0,0 +1,265 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||||
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
|
import org.apache.poi.ss.usermodel.CellStyle;
|
||||||
|
import org.apache.poi.ss.usermodel.Font;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 94:AppearancePatentSheetBuilder Sheet 构造器。
|
||||||
|
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式、行值派生)。
|
||||||
|
* 与现状 writeResultWorkbook / writeReasonSheet 语义一致:主 sheet 名"外观专利检测结果"、
|
||||||
|
* 13 列表头(RESULT_HEADERS 10 列 + 第 6/7/8 位插入标题/图片链接/sku)、加粗表头、
|
||||||
|
* 数据行从第 1 行;"原因"sheet 4 列按 ASIN 去重;LLM 技术性失败展示错误信息。
|
||||||
|
* 不落库、无 IO 依赖;rowKey/行值解析/LLM 失败判定均为静态纯函数。
|
||||||
|
*/
|
||||||
|
public final class AppearancePatentSheetBuilder {
|
||||||
|
|
||||||
|
public static final List<String> RESULT_HEADERS = List.of(
|
||||||
|
"id",
|
||||||
|
"asin",
|
||||||
|
"国家",
|
||||||
|
"卖家名称",
|
||||||
|
"品牌",
|
||||||
|
"价格",
|
||||||
|
"标题维度(商标)",
|
||||||
|
"外观维度(外观设计专利)",
|
||||||
|
"结论",
|
||||||
|
"状态"
|
||||||
|
);
|
||||||
|
|
||||||
|
private AppearancePatentSheetBuilder() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void buildResultSheet(Workbook workbook, List<AppearancePatentParsedRowVo> rows,
|
||||||
|
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||||
|
buildResultSheet(workbook, rows, resultMap, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void buildResultSheet(Workbook workbook, List<AppearancePatentParsedRowVo> rows,
|
||||||
|
Map<String, AppearancePatentResultRowDto> resultMap,
|
||||||
|
java.util.function.Function<AppearancePatentParsedRowVo, AppearancePatentResultRowDto> findResultRowFn) {
|
||||||
|
java.util.function.Function<AppearancePatentParsedRowVo, AppearancePatentResultRowDto> finder =
|
||||||
|
findResultRowFn == null ? row -> findResultRow(row, resultMap) : findResultRowFn;
|
||||||
|
Sheet sheet = workbook.createSheet("外观专利检测结果");
|
||||||
|
CellStyle headerStyle = workbook.createCellStyle();
|
||||||
|
Font font = workbook.createFont();
|
||||||
|
font.setBold(true);
|
||||||
|
headerStyle.setFont(font);
|
||||||
|
|
||||||
|
List<String> resultHeaders = new ArrayList<>(RESULT_HEADERS);
|
||||||
|
resultHeaders.add(6, "标题");
|
||||||
|
resultHeaders.add(7, "图片链接");
|
||||||
|
resultHeaders.add(8, "sku");
|
||||||
|
Row header = sheet.createRow(0);
|
||||||
|
for (int i = 0; i < resultHeaders.size(); i++) {
|
||||||
|
Cell cell = header.createCell(i);
|
||||||
|
cell.setCellValue(resultHeaders.get(i));
|
||||||
|
cell.setCellStyle(headerStyle);
|
||||||
|
}
|
||||||
|
|
||||||
|
int rowIndex = 1;
|
||||||
|
List<AppearancePatentParsedRowVo> rowsToWrite = rows == null ? List.of() : rows;
|
||||||
|
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite) {
|
||||||
|
if (parsedRow == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
AppearancePatentResultRowDto resultRow = finder.apply(parsedRow);
|
||||||
|
Row row = sheet.createRow(rowIndex++);
|
||||||
|
int col = 0;
|
||||||
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
|
||||||
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
|
||||||
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
|
||||||
|
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
|
||||||
|
row.createCell(col++).setCellValue(resolveBrand(resultRow, parsedRow));
|
||||||
|
row.createCell(col++).setCellValue(resolvePrice(resultRow));
|
||||||
|
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
|
||||||
|
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
|
||||||
|
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
|
||||||
|
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getTitleRisk()));
|
||||||
|
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getAppearanceRisk()));
|
||||||
|
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
|
||||||
|
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
|
||||||
|
}
|
||||||
|
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap, finder);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void writeReasonSheet(Workbook workbook, CellStyle headerStyle,
|
||||||
|
List<AppearancePatentParsedRowVo> rowsToWrite,
|
||||||
|
Map<String, AppearancePatentResultRowDto> resultMap,
|
||||||
|
java.util.function.Function<AppearancePatentParsedRowVo, AppearancePatentResultRowDto> finder) {
|
||||||
|
Sheet sheet = workbook.createSheet("原因");
|
||||||
|
Row header = sheet.createRow(0);
|
||||||
|
List<String> headers = List.of("ASIN", "外观原因", "专利原因", "标题原因");
|
||||||
|
for (int i = 0; i < headers.size(); i++) {
|
||||||
|
Cell cell = header.createCell(i);
|
||||||
|
cell.setCellValue(headers.get(i));
|
||||||
|
cell.setCellStyle(headerStyle);
|
||||||
|
}
|
||||||
|
|
||||||
|
Set<String> writtenAsins = new LinkedHashSet<>();
|
||||||
|
int rowIndex = 1;
|
||||||
|
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite == null ? List.<AppearancePatentParsedRowVo>of() : rowsToWrite) {
|
||||||
|
if (parsedRow == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String asin = normalize(parsedRow.getAsin()).toUpperCase(Locale.ROOT);
|
||||||
|
if (asin.isBlank() || !writtenAsins.add(asin)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
AppearancePatentResultRowDto resultRow = finder.apply(parsedRow);
|
||||||
|
Row row = sheet.createRow(rowIndex++);
|
||||||
|
row.createCell(0).setCellValue(asin);
|
||||||
|
row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), ""));
|
||||||
|
row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), ""));
|
||||||
|
row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), ""));
|
||||||
|
rowIndex++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static AppearancePatentResultRowDto findResultRow(AppearancePatentParsedRowVo parsedRow,
|
||||||
|
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||||
|
if (parsedRow == null || resultMap == null || resultMap.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
AppearancePatentResultRowDto resultRow = resultMap.get(rowKey(parsedRow));
|
||||||
|
if (resultRow != null) {
|
||||||
|
return resultRow;
|
||||||
|
}
|
||||||
|
String legacyKey = legacyRowKey(parsedRow);
|
||||||
|
if (legacyKey.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return resultMap.get(legacyKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String legacyRowKey(AppearancePatentParsedRowVo row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String rowKey(AppearancePatentParsedRowVo row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String rowKey(AppearancePatentResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return rowKey(row.getId(), row.getAsin(), row.getCountry());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String rowKey(String id, String asin, String country) {
|
||||||
|
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String readValueByHeader(AppearancePatentParsedRowVo row, String... candidates) {
|
||||||
|
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
|
||||||
|
String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT);
|
||||||
|
for (String candidate : candidates) {
|
||||||
|
String expected = normalize(candidate).toLowerCase(Locale.ROOT);
|
||||||
|
if (!expected.isBlank() && header.contains(expected)) {
|
||||||
|
return entry.getValue() == null ? "" : entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String resolveBrand(AppearancePatentResultRowDto resultRow, AppearancePatentParsedRowVo parsedRow) {
|
||||||
|
String pythonBrand = resultRow == null ? "" : resultRow.getBrand();
|
||||||
|
if (pythonBrand != null && !pythonBrand.isBlank()) {
|
||||||
|
return pythonBrand.trim();
|
||||||
|
}
|
||||||
|
if (parsedRow == null || parsedRow.getValues() == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
for (Map.Entry<String, String> entry : parsedRow.getValues().entrySet()) {
|
||||||
|
String header = entry.getKey() == null ? "" : entry.getKey().trim().toLowerCase(Locale.ROOT);
|
||||||
|
if (header.contains("品牌") || header.contains("brand")) {
|
||||||
|
return entry.getValue() == null ? "" : entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String resolvePrice(AppearancePatentResultRowDto resultRow) {
|
||||||
|
return resultRow == null || resultRow.getPrice() == null ? "" : resultRow.getPrice().trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String userFacingLlmCellValue(AppearancePatentResultRowDto row, String value) {
|
||||||
|
String normalizedValue = normalize(value);
|
||||||
|
if (!normalizedValue.isBlank() && !isTechnicalLlmFailure(normalizedValue)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
|
||||||
|
if (row != null && isTechnicalLlmFailure(row.getError())) {
|
||||||
|
return firstNonBlank(row.getError(), "");
|
||||||
|
}
|
||||||
|
return firstNonBlank(value, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String userFacingConclusion(AppearancePatentResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String conclusion = normalize(row.getConclusion());
|
||||||
|
if (!conclusion.isBlank() && !isTechnicalLlmFailure(conclusion)) {
|
||||||
|
return row.getConclusion();
|
||||||
|
}
|
||||||
|
// llm 技术性失败:有错误信息则放入错误信息,没有则留空
|
||||||
|
if (isTechnicalLlmFailure(row.getError())) {
|
||||||
|
return firstNonBlank(row.getError(), "");
|
||||||
|
}
|
||||||
|
return firstNonBlank(row.getConclusion(), "");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String userFacingStatus(AppearancePatentResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String conclusion = userFacingConclusion(row);
|
||||||
|
return resolveResultStatus(conclusion);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String resolveResultStatus(String conclusion) {
|
||||||
|
String normalized = conclusion == null ? "" : conclusion.trim();
|
||||||
|
return normalized.isBlank() ? "失败" : "成功";
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean isTechnicalLlmFailure(String value) {
|
||||||
|
String normalized = normalize(value).toLowerCase(Locale.ROOT);
|
||||||
|
return normalized.contains("coze") || normalized.contains("llm")
|
||||||
|
|| normalized.contains("结果不完整")
|
||||||
|
|| normalized.contains("工作流节点执行超限")
|
||||||
|
|| normalized.contains("调用超时")
|
||||||
|
|| normalized.contains("timeout");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String firstNonBlank(String preferred, String fallback) {
|
||||||
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String normalize(String val) {
|
||||||
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
|
}
|
||||||
|
}
|
||||||
+10
@@ -27,6 +27,8 @@ import org.springframework.web.bind.annotation.RequestBody;
|
|||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -113,6 +115,14 @@ public class CollectDataController {
|
|||||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@DeleteMapping("/tasks/{taskId}")
|
@DeleteMapping("/tasks/{taskId}")
|
||||||
@Operation(summary = "删除任务", description = "删除任务及其明细行、关联结果记录。")
|
@Operation(summary = "删除任务", description = "删除任务及其明细行、关联结果记录。")
|
||||||
public ApiResponse<Void> deleteTask(
|
public ApiResponse<Void> deleteTask(
|
||||||
|
|||||||
+7
@@ -87,6 +87,8 @@ import java.util.Map;
|
|||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -94,6 +96,10 @@ import java.util.regex.Pattern;
|
|||||||
public class CollectDataService {
|
public class CollectDataService {
|
||||||
|
|
||||||
public static final String MODULE_TYPE = "COLLECT_DATA";
|
public static final String MODULE_TYPE = "COLLECT_DATA";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
public static final int DEFAULT_PAGE_SIZE = 50;
|
public static final int DEFAULT_PAGE_SIZE = 50;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -152,6 +158,7 @@ public class CollectDataService {
|
|||||||
|
|
||||||
/** 结果明细 chunk 级读取器:生成结果文件时按 chunk 一次读取,替代逐行对象读取。 */
|
/** 结果明细 chunk 级读取器:生成结果文件时按 chunk 一次读取,替代逐行对象读取。 */
|
||||||
private final CollectDataResultDetailReader resultDetailReader;
|
private final CollectDataResultDetailReader resultDetailReader;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
||||||
private long staleTimeoutMinutes;
|
private long staleTimeoutMinutes;
|
||||||
|
|||||||
+10
@@ -34,6 +34,8 @@ import org.springframework.web.server.ResponseStatusException;
|
|||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@@ -90,6 +92,14 @@ public class DeleteBrandRunController {
|
|||||||
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
|
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@GetMapping("/tasks/{taskId}/deletion-status")
|
@GetMapping("/tasks/{taskId}/deletion-status")
|
||||||
@Operation(summary = "获取删除品牌任务删除状态", description = "供 Python/插件按 taskId 查询该删除品牌任务是否已被前端删空。")
|
@Operation(summary = "获取删除品牌任务删除状态", description = "供 Python/插件按 taskId 查询该删除品牌任务是否已被前端删空。")
|
||||||
public ApiResponse<DeleteBrandTaskDeletionStatusVo> getTaskDeletionStatus(
|
public ApiResponse<DeleteBrandTaskDeletionStatusVo> getTaskDeletionStatus(
|
||||||
|
|||||||
+7
@@ -68,6 +68,8 @@ import java.util.List;
|
|||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -75,6 +77,10 @@ import java.util.Set;
|
|||||||
public class DeleteBrandRunService {
|
public class DeleteBrandRunService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "DELETE_BRAND";
|
private static final String MODULE_TYPE = "DELETE_BRAND";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final Duration TASK_LOCK_TTL = TaskDistributedLockService.DEFAULT_LOCK_TTL;
|
private static final Duration TASK_LOCK_TTL = TaskDistributedLockService.DEFAULT_LOCK_TTL;
|
||||||
private static final long TASK_LOCK_WAIT_MILLIS = TaskDistributedLockService.DEFAULT_WAIT_MILLIS;
|
private static final long TASK_LOCK_WAIT_MILLIS = TaskDistributedLockService.DEFAULT_WAIT_MILLIS;
|
||||||
@@ -92,6 +98,7 @@ public class DeleteBrandRunService {
|
|||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
private final TaskFileJobService taskFileJobService;
|
private final TaskFileJobService taskFileJobService;
|
||||||
private final PlatformTransactionManager transactionManager;
|
private final PlatformTransactionManager transactionManager;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
private TransactionTemplate newRequiresNewTemplate() {
|
private TransactionTemplate newRequiresNewTemplate() {
|
||||||
TransactionTemplate template = new TransactionTemplate(transactionManager);
|
TransactionTemplate template = new TransactionTemplate(transactionManager);
|
||||||
|
|||||||
+3
-1
@@ -63,11 +63,13 @@ public class InvalidAsinDataController {
|
|||||||
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
||||||
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
|
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
|
||||||
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword,
|
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword,
|
||||||
|
@Parameter(description = "ASIN 模糊搜索") @RequestParam(required = false) String dataValue,
|
||||||
|
@Parameter(description = "品牌模糊搜索") @RequestParam(required = false) String brand,
|
||||||
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
|
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
|
||||||
HttpServletRequest request) {
|
HttpServletRequest request) {
|
||||||
RequestOperator operator = requireInvalidAsinDataAccess(request);
|
RequestOperator operator = requireInvalidAsinDataAccess(request);
|
||||||
return ApiResponse.success(invalidAsinDataService.page(
|
return ApiResponse.success(invalidAsinDataService.page(
|
||||||
page, pageSize, keyword, groupId, operator.id(), operator.superAdmin()));
|
page, pageSize, keyword, dataValue, brand, groupId, operator.id(), operator.superAdmin()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping
|
@PostMapping
|
||||||
|
|||||||
+5
-1
@@ -31,15 +31,19 @@ public class InvalidAsinDataService {
|
|||||||
private final InvalidAsinDataMapper invalidAsinDataMapper;
|
private final InvalidAsinDataMapper invalidAsinDataMapper;
|
||||||
private final ShopManageGroupService shopManageGroupService;
|
private final ShopManageGroupService shopManageGroupService;
|
||||||
|
|
||||||
public InvalidAsinDataPageVo page(long page, long pageSize, String keyword, Long groupId, Long operatorId, boolean superAdmin) {
|
public InvalidAsinDataPageVo page(long page, long pageSize, String keyword, String dataValue, String brand, Long groupId, Long operatorId, boolean superAdmin) {
|
||||||
long safePage = Math.max(page, 1);
|
long safePage = Math.max(page, 1);
|
||||||
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
||||||
String safeKeyword = keyword == null ? "" : keyword.trim();
|
String safeKeyword = keyword == null ? "" : keyword.trim();
|
||||||
|
String safeDataValue = dataValue == null ? "" : dataValue.trim();
|
||||||
|
String safeBrand = brand == null ? "" : brand.trim();
|
||||||
LambdaQueryWrapper<InvalidAsinDataEntity> query = new LambdaQueryWrapper<InvalidAsinDataEntity>()
|
LambdaQueryWrapper<InvalidAsinDataEntity> query = new LambdaQueryWrapper<InvalidAsinDataEntity>()
|
||||||
.and(!safeKeyword.isEmpty(), wrapper -> wrapper
|
.and(!safeKeyword.isEmpty(), wrapper -> wrapper
|
||||||
.like(InvalidAsinDataEntity::getDataValue, safeKeyword)
|
.like(InvalidAsinDataEntity::getDataValue, safeKeyword)
|
||||||
.or()
|
.or()
|
||||||
.like(InvalidAsinDataEntity::getBrand, safeKeyword))
|
.like(InvalidAsinDataEntity::getBrand, safeKeyword))
|
||||||
|
.like(!safeDataValue.isEmpty(), InvalidAsinDataEntity::getDataValue, safeDataValue)
|
||||||
|
.like(!safeBrand.isEmpty(), InvalidAsinDataEntity::getBrand, safeBrand)
|
||||||
.orderByDesc(InvalidAsinDataEntity::getId);
|
.orderByDesc(InvalidAsinDataEntity::getId);
|
||||||
if (!superAdmin) {
|
if (!superAdmin) {
|
||||||
Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId);
|
Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId);
|
||||||
|
|||||||
+10
@@ -41,6 +41,8 @@ import java.io.InputStream;
|
|||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -167,6 +169,14 @@ public class PatrolDeleteController {
|
|||||||
return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(request.getTaskIds()));
|
return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(patrolDeleteTaskService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks")
|
@PostMapping("/tasks")
|
||||||
@Operation(summary = "创建巡店删除任务", description = "根据已匹配的店铺创建任务和占位结果记录,后续由 Python 端处理并回传结果。")
|
@Operation(summary = "创建巡店删除任务", description = "根据已匹配的店铺创建任务和占位结果记录,后续由 Python 端处理并回传结果。")
|
||||||
public ApiResponse<PatrolDeleteCreateTaskVo> createTask(
|
public ApiResponse<PatrolDeleteCreateTaskVo> createTask(
|
||||||
|
|||||||
+7
@@ -43,6 +43,8 @@ import java.util.LinkedHashMap;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -50,6 +52,10 @@ import java.util.Objects;
|
|||||||
public class PatrolDeleteTaskService {
|
public class PatrolDeleteTaskService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "PATROL_DELETE";
|
private static final String MODULE_TYPE = "PATROL_DELETE";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final int RESULT_PENDING = -1;
|
private static final int RESULT_PENDING = -1;
|
||||||
private static final int RESULT_FAILED = 0;
|
private static final int RESULT_FAILED = 0;
|
||||||
@@ -70,6 +76,7 @@ public class PatrolDeleteTaskService {
|
|||||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
|
|||||||
+1
@@ -50,6 +50,7 @@ public class PermissionMenuSchemaInitializer {
|
|||||||
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
|
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
|
||||||
new DefaultAdminMenu("用户管理", "admin_users", "users", 10),
|
new DefaultAdminMenu("用户管理", "admin_users", "users", 10),
|
||||||
new DefaultAdminMenu("菜单权限配置", "admin_columns", "columns", 20),
|
new DefaultAdminMenu("菜单权限配置", "admin_columns", "columns", 20),
|
||||||
|
new DefaultAdminMenu("分组管理", "admin_group_manage", "group-manage", 25),
|
||||||
new DefaultAdminMenu("去重数据汇总", "admin_dedupe_total_data", "dedupe-total-data", 30),
|
new DefaultAdminMenu("去重数据汇总", "admin_dedupe_total_data", "dedupe-total-data", 30),
|
||||||
new DefaultAdminMenu("无效ASIN数据", "admin_invalid_asin_data", "invalid-asin-data", 35),
|
new DefaultAdminMenu("无效ASIN数据", "admin_invalid_asin_data", "invalid-asin-data", 35),
|
||||||
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40),
|
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40),
|
||||||
|
|||||||
+10
@@ -49,6 +49,8 @@ import java.io.InputStream;
|
|||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -264,6 +266,14 @@ public class PriceTrackController {
|
|||||||
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds()));
|
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(priceTrackTaskService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks/{taskId}/result")
|
@PostMapping("/tasks/{taskId}/result")
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "Python 回传处理结果",
|
summary = "Python 回传处理结果",
|
||||||
|
|||||||
+7
@@ -54,6 +54,8 @@ import java.util.Map;
|
|||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -61,6 +63,10 @@ import java.util.concurrent.atomic.AtomicReference;
|
|||||||
public class PriceTrackTaskService {
|
public class PriceTrackTaskService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "PRICE_TRACK";
|
private static final String MODULE_TYPE = "PRICE_TRACK";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
|
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
|
||||||
private static final String NO_USABLE_ROWS_ERROR = "未收到有效跟价数据,未生成结果文件";
|
private static final String NO_USABLE_ROWS_ERROR = "未收到有效跟价数据,未生成结果文件";
|
||||||
@@ -79,6 +85,7 @@ public class PriceTrackTaskService {
|
|||||||
private final TaskResultPayloadService taskResultPayloadService;
|
private final TaskResultPayloadService taskResultPayloadService;
|
||||||
private final TaskFileJobService taskFileJobService;
|
private final TaskFileJobService taskFileJobService;
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
|
|||||||
+10
@@ -44,6 +44,8 @@ import java.io.InputStream;
|
|||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -165,6 +167,14 @@ public class ProductRiskResolveController {
|
|||||||
return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(request.getTaskIds()));
|
return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(productRiskTaskService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks/{taskId}/result")
|
@PostMapping("/tasks/{taskId}/result")
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "Python 回传处理结果",
|
summary = "Python 回传处理结果",
|
||||||
|
|||||||
+7
@@ -47,6 +47,8 @@ import java.util.List;
|
|||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -54,6 +56,10 @@ import java.util.Objects;
|
|||||||
public class ProductRiskTaskService {
|
public class ProductRiskTaskService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
|
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||||
|
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
@@ -70,6 +76,7 @@ public class ProductRiskTaskService {
|
|||||||
private final TaskResultItemService taskResultItemService;
|
private final TaskResultItemService taskResultItemService;
|
||||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
|
|||||||
+11
@@ -24,6 +24,8 @@ import org.springframework.web.bind.annotation.RequestBody;
|
|||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -104,6 +106,15 @@ public class PublishController {
|
|||||||
return ApiResponse.success(publishTaskService.getTaskProgress(request.getUserId(), request.getTaskIds()));
|
return ApiResponse.success(publishTaskService.getTaskProgress(request.getUserId(), request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。user_id 可省略,传入时仅返回该用户的任务。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(
|
||||||
|
@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(publishTaskService.progressLight(request.getUserId(), request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks/{taskId}/result")
|
@PostMapping("/tasks/{taskId}/result")
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "Python 按文件分片回传上架结果",
|
summary = "Python 按文件分片回传上架结果",
|
||||||
|
|||||||
+32
-11
@@ -4,6 +4,7 @@ import cn.hutool.core.io.FileUtil;
|
|||||||
import cn.hutool.core.util.IdUtil;
|
import cn.hutool.core.util.IdUtil;
|
||||||
import cn.hutool.crypto.digest.DigestUtil;
|
import cn.hutool.crypto.digest.DigestUtil;
|
||||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
import com.fasterxml.jackson.core.type.TypeReference;
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
@@ -70,6 +71,8 @@ import java.util.Set;
|
|||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
@@ -77,6 +80,10 @@ import java.util.stream.Collectors;
|
|||||||
public class PublishTaskService {
|
public class PublishTaskService {
|
||||||
|
|
||||||
public static final String MODULE_TYPE = "PUBLISH";
|
public static final String MODULE_TYPE = "PUBLISH";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(Long userId, List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||||
|
}
|
||||||
public static final int DEFAULT_PAGE_SIZE = 50;
|
public static final int DEFAULT_PAGE_SIZE = 50;
|
||||||
private static final int MAX_PAGE_SIZE = 200;
|
private static final int MAX_PAGE_SIZE = 200;
|
||||||
private static final int INSERT_BATCH_SIZE = 500;
|
private static final int INSERT_BATCH_SIZE = 500;
|
||||||
@@ -102,6 +109,7 @@ public class PublishTaskService {
|
|||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TransactionTemplate transactionTemplate;
|
private final TransactionTemplate transactionTemplate;
|
||||||
private final InstanceMetadata instanceMetadata;
|
private final InstanceMetadata instanceMetadata;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
@Value("${aiimage.publish.stale-timeout-minutes:30}")
|
@Value("${aiimage.publish.stale-timeout-minutes:30}")
|
||||||
private int staleTimeoutMinutes;
|
private int staleTimeoutMinutes;
|
||||||
@@ -340,10 +348,30 @@ public class PublishTaskService {
|
|||||||
public PublishDashboardVo dashboard(Long userId) {
|
public PublishDashboardVo dashboard(Long userId) {
|
||||||
validateUserId(userId);
|
validateUserId(userId);
|
||||||
PublishDashboardVo response = new PublishDashboardVo();
|
PublishDashboardVo response = new PublishDashboardVo();
|
||||||
response.setPendingCount(countTasks(userId, STATUS_PENDING));
|
response.setPendingCount(0L);
|
||||||
response.setRunningCount(countTasks(userId, STATUS_RUNNING));
|
response.setRunningCount(0L);
|
||||||
response.setSuccessCount(countTasks(userId, STATUS_SUCCESS));
|
response.setSuccessCount(0L);
|
||||||
response.setFailedCount(countTasks(userId, STATUS_FAILED));
|
response.setFailedCount(0L);
|
||||||
|
for (Map<String, Object> row : fileTaskMapper.selectMaps(new QueryWrapper<FileTaskEntity>()
|
||||||
|
.select("status", "count(*) AS cnt")
|
||||||
|
.eq("module_type", MODULE_TYPE)
|
||||||
|
.eq("user_id", userId)
|
||||||
|
.groupBy("status"))) {
|
||||||
|
Object status = row.get("status");
|
||||||
|
Object count = row.get("cnt");
|
||||||
|
if (status == null || count == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
long n = ((Number) count).longValue();
|
||||||
|
switch (status.toString()) {
|
||||||
|
case STATUS_PENDING -> response.setPendingCount(n);
|
||||||
|
case STATUS_RUNNING -> response.setRunningCount(n);
|
||||||
|
case STATUS_SUCCESS -> response.setSuccessCount(n);
|
||||||
|
case STATUS_FAILED -> response.setFailedCount(n);
|
||||||
|
default -> {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
response.setRecent(history(userId, 10).getItems());
|
response.setRecent(history(userId, 10).getItems());
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
@@ -1741,13 +1769,6 @@ public class PublishTaskService {
|
|||||||
return STATUS_SUCCESS.equals(status) || STATUS_FAILED.equals(status);
|
return STATUS_SUCCESS.equals(status) || STATUS_FAILED.equals(status);
|
||||||
}
|
}
|
||||||
|
|
||||||
private long countTasks(Long userId, String status) {
|
|
||||||
return Objects.requireNonNullElse(fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
|
||||||
.eq(FileTaskEntity::getUserId, userId)
|
|
||||||
.eq(FileTaskEntity::getStatus, status)), 0L);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<Long> normalizeTaskIds(List<Long> taskIds) {
|
private List<Long> normalizeTaskIds(List<Long> taskIds) {
|
||||||
if (taskIds == null) {
|
if (taskIds == null) {
|
||||||
return List.of();
|
return List.of();
|
||||||
|
|||||||
+10
@@ -39,6 +39,8 @@ import java.io.InputStream;
|
|||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -137,6 +139,14 @@ public class QueryAsinTaskController {
|
|||||||
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds()));
|
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(queryAsinTaskService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks")
|
@PostMapping("/tasks")
|
||||||
@Operation(summary = "创建查询 ASIN 任务", description = "根据已匹配的店铺创建任务和占位结果记录,并把后台维护的整张 ASIN 表数据随店铺项返回给 Python 端。")
|
@Operation(summary = "创建查询 ASIN 任务", description = "根据已匹配的店铺创建任务和占位结果记录,并把后台维护的整张 ASIN 表数据随店铺项返回给 Python 端。")
|
||||||
public ApiResponse<QueryAsinCreateTaskVo> createTask(
|
public ApiResponse<QueryAsinCreateTaskVo> createTask(
|
||||||
|
|||||||
+7
@@ -42,6 +42,8 @@ import java.util.LinkedHashMap;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -49,6 +51,10 @@ import java.util.Objects;
|
|||||||
public class QueryAsinTaskService {
|
public class QueryAsinTaskService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "QUERY_ASIN";
|
private static final String MODULE_TYPE = "QUERY_ASIN";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final int RESULT_PENDING = -1;
|
private static final int RESULT_PENDING = -1;
|
||||||
private static final int RESULT_FAILED = 0;
|
private static final int RESULT_FAILED = 0;
|
||||||
@@ -68,6 +74,7 @@ public class QueryAsinTaskService {
|
|||||||
private final TaskResultItemService taskResultItemService;
|
private final TaskResultItemService taskResultItemService;
|
||||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
|
|||||||
+10
@@ -33,6 +33,8 @@ import org.springframework.web.server.ResponseStatusException;
|
|||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -151,6 +153,14 @@ public class ShopDataCrawlTaskController {
|
|||||||
return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds()));
|
return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(taskService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks")
|
@PostMapping("/tasks")
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "创建店铺数据抓取任务",
|
summary = "创建店铺数据抓取任务",
|
||||||
|
|||||||
+25
-112
@@ -1,14 +1,13 @@
|
|||||||
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||||
|
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache;
|
import com.nanri.aiimage.modules.shopdatacrawl.util.BoundedImageCache;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.util.ShopDataCrawlPrefetchBudget;
|
import com.nanri.aiimage.modules.shopdatacrawl.util.ShopDataCrawlPrefetchBudget;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlSheetBuilder;
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.poi.ss.usermodel.Cell;
|
|
||||||
import org.apache.poi.ss.usermodel.CellStyle;
|
import org.apache.poi.ss.usermodel.CellStyle;
|
||||||
import org.apache.poi.ss.usermodel.ClientAnchor;
|
import org.apache.poi.ss.usermodel.ClientAnchor;
|
||||||
import org.apache.poi.ss.usermodel.Drawing;
|
import org.apache.poi.ss.usermodel.Drawing;
|
||||||
@@ -27,7 +26,6 @@ import java.io.File;
|
|||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -35,16 +33,17 @@ import java.util.Map;
|
|||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class ShopDataCrawlExcelAssemblyService {
|
public class ShopDataCrawlExcelAssemblyService {
|
||||||
static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
|
/** 常量已抽取到 ShopDataCrawlSheetBuilder,此处保留别名供既有调用方/测试引用。 */
|
||||||
static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
|
static final List<String> COUNTRIES = ShopDataCrawlSheetBuilder.COUNTRIES;
|
||||||
static final List<String> LEGACY_HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
|
static final List<String> SHEETS = ShopDataCrawlSheetBuilder.SHEETS;
|
||||||
static final List<String> HEADERS_WITHOUT_BRAND = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
|
static final List<String> LEGACY_HEADERS = ShopDataCrawlSheetBuilder.LEGACY_HEADERS;
|
||||||
static final List<String> HEADERS = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价", "品牌");
|
static final List<String> HEADERS_WITHOUT_BRAND = ShopDataCrawlSheetBuilder.HEADERS_WITHOUT_BRAND;
|
||||||
|
static final List<String> HEADERS = ShopDataCrawlSheetBuilder.HEADERS;
|
||||||
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
|
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
|
||||||
private static final int IMAGE_COLUMN = 2;
|
private static final int IMAGE_COLUMN = ShopDataCrawlSheetBuilder.IMAGE_COLUMN;
|
||||||
private static final int BRAND_COLUMN = HEADERS.size() - 1;
|
private static final int BRAND_COLUMN = ShopDataCrawlSheetBuilder.BRAND_COLUMN;
|
||||||
private static final int IMAGE_COLUMN_WIDTH = 18 * 256;
|
private static final int IMAGE_COLUMN_WIDTH = ShopDataCrawlSheetBuilder.IMAGE_COLUMN_WIDTH;
|
||||||
private static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
|
private static final float IMAGE_ROW_HEIGHT_POINTS = ShopDataCrawlSheetBuilder.IMAGE_ROW_HEIGHT_POINTS;
|
||||||
/** 图片缓存默认上限:64MB 字节预算 / 2000 条目,超过按 FIFO 淘汰,保证组装期内存有界。 */
|
/** 图片缓存默认上限:64MB 字节预算 / 2000 条目,超过按 FIFO 淘汰,保证组装期内存有界。 */
|
||||||
private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024;
|
private static final long DEFAULT_IMAGE_CACHE_MAX_BYTES = 64L * 1024 * 1024;
|
||||||
private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000;
|
private static final int DEFAULT_IMAGE_CACHE_MAX_ENTRIES = 2000;
|
||||||
@@ -189,12 +188,7 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
List<ShopDataCrawlRowDto> rows,
|
List<ShopDataCrawlRowDto> rows,
|
||||||
BoundedImageCache imageCache,
|
BoundedImageCache imageCache,
|
||||||
Map<String, Integer> pictureIndexes) {
|
Map<String, Integer> pictureIndexes) {
|
||||||
Sheet sheet = workbook.createSheet(SHEETS.get(index));
|
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(workbook, index);
|
||||||
Row header = sheet.createRow(0);
|
|
||||||
for (int column = 0; column < HEADERS.size(); column++) {
|
|
||||||
header.createCell(column).setCellValue(HEADERS.get(column));
|
|
||||||
}
|
|
||||||
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
|
|
||||||
int rowIndex = 1;
|
int rowIndex = 1;
|
||||||
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
|
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
|
||||||
Row row = sheet.createRow(rowIndex++);
|
Row row = sheet.createRow(rowIndex++);
|
||||||
@@ -203,27 +197,7 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void validateTemplate(XSSFWorkbook workbook) {
|
void validateTemplate(XSSFWorkbook workbook) {
|
||||||
if (workbook.getNumberOfSheets() != SHEETS.size()) {
|
ShopDataCrawlSheetBuilder.validateTemplate(workbook);
|
||||||
throw new BusinessException("店铺数据抓取模板工作表数量不正确");
|
|
||||||
}
|
|
||||||
for (int i = 0; i < SHEETS.size(); i++) {
|
|
||||||
Sheet sheet = workbook.getSheetAt(i);
|
|
||||||
if (!SHEETS.get(i).equals(sheet.getSheetName())) {
|
|
||||||
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
|
|
||||||
}
|
|
||||||
Row header = sheet.getRow(0);
|
|
||||||
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
|
|
||||||
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
|
|
||||||
List<String> expectedHeaders = currentTemplate
|
|
||||||
? (templateHasBrand ? HEADERS : HEADERS_WITHOUT_BRAND)
|
|
||||||
: LEGACY_HEADERS;
|
|
||||||
for (int column = 0; column < expectedHeaders.size(); column++) {
|
|
||||||
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
|
|
||||||
if (!expectedHeaders.get(column).equals(actual)) {
|
|
||||||
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeSheet(XSSFWorkbook workbook,
|
private void writeSheet(XSSFWorkbook workbook,
|
||||||
@@ -232,32 +206,16 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
BoundedImageCache imageCache,
|
BoundedImageCache imageCache,
|
||||||
Map<String, Integer> pictureIndexes) {
|
Map<String, Integer> pictureIndexes) {
|
||||||
Row header = sheet.getRow(0);
|
Row header = sheet.getRow(0);
|
||||||
Row styleRow = sheet.getRow(1);
|
boolean currentTemplate = header != null && "商品图片".equals(ShopDataCrawlSheetBuilder.cellText(header, IMAGE_COLUMN));
|
||||||
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
|
boolean templateHasBrand = "品牌".equals(ShopDataCrawlSheetBuilder.cellText(header, BRAND_COLUMN));
|
||||||
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
|
CellStyle[] styles = ShopDataCrawlSheetBuilder.templateStyles(sheet, currentTemplate, templateHasBrand);
|
||||||
CellStyle[] styles = new CellStyle[HEADERS.size()];
|
int[] columnWidths = ShopDataCrawlSheetBuilder.templateColumnWidths(sheet, currentTemplate, templateHasBrand);
|
||||||
for (int column = 0; column < styles.length; column++) {
|
ShopDataCrawlSheetBuilder.writeHeaders(sheet, currentTemplate, templateHasBrand);
|
||||||
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
|
||||||
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
|
|
||||||
styles[column] = cell == null ? null : cell.getCellStyle();
|
|
||||||
}
|
|
||||||
int[] columnWidths = new int[HEADERS.size()];
|
|
||||||
for (int column = 0; column < columnWidths.length; column++) {
|
|
||||||
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
|
||||||
columnWidths[column] = sheet.getColumnWidth(sourceColumn);
|
|
||||||
}
|
|
||||||
writeHeaders(sheet, currentTemplate, templateHasBrand);
|
|
||||||
for (int column = 0; column < columnWidths.length; column++) {
|
for (int column = 0; column < columnWidths.length; column++) {
|
||||||
sheet.setColumnWidth(column, columnWidths[column]);
|
sheet.setColumnWidth(column, columnWidths[column]);
|
||||||
}
|
}
|
||||||
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
|
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
|
||||||
int last = sheet.getLastRowNum();
|
ShopDataCrawlSheetBuilder.clearDataRows(sheet);
|
||||||
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
|
|
||||||
Row row = sheet.getRow(rowIndex);
|
|
||||||
if (row != null) {
|
|
||||||
sheet.removeRow(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
clearSheetPictures(sheet, pictureIndexes);
|
clearSheetPictures(sheet, pictureIndexes);
|
||||||
int rowIndex = 1;
|
int rowIndex = 1;
|
||||||
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
|
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
|
||||||
@@ -273,43 +231,13 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
CellStyle[] styles,
|
CellStyle[] styles,
|
||||||
BoundedImageCache imageCache,
|
BoundedImageCache imageCache,
|
||||||
Map<String, Integer> pictureIndexes) {
|
Map<String, Integer> pictureIndexes) {
|
||||||
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
|
ShopDataCrawlSheetBuilder.writeDataRowValues(row, value, styles);
|
||||||
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
|
if (!ShopDataCrawlSheetBuilder.blank(value.getCommodityImage())) {
|
||||||
for (int column = 0; column < values.length; column++) {
|
|
||||||
Cell cell = row.createCell(column);
|
|
||||||
if (styles[column] != null) cell.setCellStyle(styles[column]);
|
|
||||||
cell.setCellValue(values[column] == null ? "" : values[column]);
|
|
||||||
}
|
|
||||||
if (!blank(value.getCommodityImage())) {
|
|
||||||
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
|
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
|
||||||
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
|
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
|
|
||||||
Row header = sheet.getRow(0);
|
|
||||||
if (header == null) header = sheet.createRow(0);
|
|
||||||
CellStyle[] styles = new CellStyle[HEADERS.size()];
|
|
||||||
for (int column = 0; column < HEADERS.size(); column++) {
|
|
||||||
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
|
||||||
Cell source = header.getCell(sourceColumn);
|
|
||||||
styles[column] = source == null ? null : source.getCellStyle();
|
|
||||||
}
|
|
||||||
for (int column = 0; column < HEADERS.size(); column++) {
|
|
||||||
Cell cell = header.getCell(column);
|
|
||||||
if (cell == null) cell = header.createCell(column);
|
|
||||||
if (styles[column] != null) cell.setCellStyle(styles[column]);
|
|
||||||
cell.setCellValue(HEADERS.get(column));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private int templateColumnForOutput(int outputColumn, boolean currentTemplate, boolean templateHasBrand) {
|
|
||||||
if (outputColumn == BRAND_COLUMN) {
|
|
||||||
return templateHasBrand ? BRAND_COLUMN : 1;
|
|
||||||
}
|
|
||||||
return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void embedImage(Workbook workbook,
|
private void embedImage(Workbook workbook,
|
||||||
Sheet sheet,
|
Sheet sheet,
|
||||||
Row row,
|
Row row,
|
||||||
@@ -357,26 +285,15 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String cellText(Row row, int column) {
|
private String cellText(Row row, int column) {
|
||||||
Cell cell = row == null ? null : row.getCell(column);
|
return ShopDataCrawlSheetBuilder.cellText(row, column);
|
||||||
return cell == null ? "" : cell.getStringCellValue().trim();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean blank(String value) {
|
private boolean blank(String value) {
|
||||||
return value == null || value.isBlank();
|
return ShopDataCrawlSheetBuilder.blank(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
|
private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
|
||||||
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>();
|
return ShopDataCrawlSheetBuilder.rowsByCountry(items);
|
||||||
COUNTRIES.forEach(country -> result.put(country, new ArrayList<>()));
|
|
||||||
if (items == null) return result;
|
|
||||||
for (ShopDataCrawlResultItemVo item : items) {
|
|
||||||
if (item == null || Boolean.FALSE.equals(item.getSuccess()) || item.getCountryResults() == null) continue;
|
|
||||||
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) {
|
|
||||||
String country = countryResult == null || countryResult.getCountry() == null ? "" : countryResult.getCountry().trim().toUpperCase();
|
|
||||||
if (result.containsKey(country) && countryResult.getItems() != null) result.get(country).addAll(countryResult.getItems());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void clearSheetPictures(Sheet sheet, Map<String, Integer> pictureIndexes) {
|
private void clearSheetPictures(Sheet sheet, Map<String, Integer> pictureIndexes) {
|
||||||
@@ -393,10 +310,6 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private int totalDataRows(XSSFWorkbook workbook) {
|
private int totalDataRows(XSSFWorkbook workbook) {
|
||||||
int total = 0;
|
return ShopDataCrawlSheetBuilder.totalDataRows(workbook);
|
||||||
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
|
|
||||||
total += Math.max(0, workbook.getSheetAt(i).getLastRowNum());
|
|
||||||
}
|
|
||||||
return total;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+23
-83
@@ -16,6 +16,8 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlHistoryAssembler;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlRowNormalizer;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||||
@@ -71,6 +73,8 @@ import java.util.Set;
|
|||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.function.Supplier;
|
import java.util.function.Supplier;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -78,6 +82,10 @@ import java.util.function.Supplier;
|
|||||||
public class ShopDataCrawlTaskService {
|
public class ShopDataCrawlTaskService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
|
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final int RESULT_PENDING = -1;
|
private static final int RESULT_PENDING = -1;
|
||||||
private static final int RESULT_FAILED = 0;
|
private static final int RESULT_FAILED = 0;
|
||||||
@@ -91,8 +99,6 @@ public class ShopDataCrawlTaskService {
|
|||||||
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
|
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
|
||||||
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断,已保留已回传的部分数据";
|
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断,已保留已回传的部分数据";
|
||||||
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
|
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
|
||||||
/** 国家结果行去重键字段分隔符(控制字符,字段值 trim 后不可能包含)。 */
|
|
||||||
private static final String ROW_KEY_SEPARATOR = "";
|
|
||||||
|
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
private final FileResultMapper fileResultMapper;
|
private final FileResultMapper fileResultMapper;
|
||||||
@@ -113,6 +119,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
private final InstanceMetadata instanceMetadata;
|
private final InstanceMetadata instanceMetadata;
|
||||||
private final ShopDataCrawlDailyFileService dailyFileService;
|
private final ShopDataCrawlDailyFileService dailyFileService;
|
||||||
private final PlatformTransactionManager transactionManager;
|
private final PlatformTransactionManager transactionManager;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
@Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}")
|
@Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}")
|
||||||
private long staleTimeoutMinutes;
|
private long staleTimeoutMinutes;
|
||||||
@@ -999,11 +1006,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String trimToNull(String value) {
|
private String trimToNull(String value) {
|
||||||
if (value == null) {
|
return ShopDataCrawlRowNormalizer.trimToNull(value);
|
||||||
return null;
|
|
||||||
}
|
|
||||||
String normalized = value.trim();
|
|
||||||
return normalized.isEmpty() ? null : normalized;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private long countTasks(Long userId, List<String> statuses) {
|
private long countTasks(Long userId, List<String> statuses) {
|
||||||
@@ -1108,26 +1111,11 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, ShopDataCrawlResultItemVo snapshot, TaskFileJobEntity job) {
|
private ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, ShopDataCrawlResultItemVo snapshot, TaskFileJobEntity job) {
|
||||||
ShopDataCrawlResultItemVo item = snapshot != null ? snapshot : new ShopDataCrawlResultItemVo();
|
return historyAssembler().toHistoryItem(entity, task, snapshot, job);
|
||||||
item.setResultId(entity.getId());
|
}
|
||||||
item.setTaskId(entity.getTaskId());
|
|
||||||
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
|
private ShopDataCrawlHistoryAssembler historyAssembler() {
|
||||||
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
|
return new ShopDataCrawlHistoryAssembler();
|
||||||
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
|
|
||||||
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
|
|
||||||
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
|
|
||||||
item.setCreatedAt(entity.getCreatedAt());
|
|
||||||
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
|
|
||||||
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
|
|
||||||
item.setDownloadUrl(null);
|
|
||||||
attachFileJobState(item, entity, job);
|
|
||||||
if (item.getCountryResults() == null) {
|
|
||||||
item.setCountryResults(new ArrayList<>());
|
|
||||||
}
|
|
||||||
if (item.getCountryCodes() == null) {
|
|
||||||
item.setCountryCodes(new ArrayList<>());
|
|
||||||
}
|
|
||||||
return item;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity) {
|
private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity) {
|
||||||
@@ -1135,14 +1123,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
|
private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
|
||||||
item.setFileReady(!blank(entity.getResultFileUrl()));
|
historyAssembler().attachFileJobState(item, entity, job);
|
||||||
if (job == null) {
|
|
||||||
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
item.setFileJobId(job.getId());
|
|
||||||
item.setFileStatus(job.getStatus());
|
|
||||||
item.setFileError(job.getErrorMessage());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<ShopDataCrawlTaskItemDto> dedupeItems(List<ShopDataCrawlTaskItemDto> items) {
|
private List<ShopDataCrawlTaskItemDto> dedupeItems(List<ShopDataCrawlTaskItemDto> items) {
|
||||||
@@ -2817,18 +2798,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) {
|
private ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) {
|
||||||
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
return ShopDataCrawlRowNormalizer.copyRow(source);
|
||||||
row.setDate(trim(source.getDate()));
|
|
||||||
row.setAsin(trim(source.getAsin()));
|
|
||||||
row.setBrand(trim(source.getBrand()));
|
|
||||||
row.setCommodityImage(trim(source.getCommodityImage()));
|
|
||||||
row.setInventorySales(trim(source.getInventorySales()));
|
|
||||||
row.setSalesRank(trim(source.getSalesRank()));
|
|
||||||
row.setPageViews(trim(source.getPageViews()));
|
|
||||||
row.setUnitsSold(trim(source.getUnitsSold()));
|
|
||||||
row.setPrice(trim(source.getPrice()));
|
|
||||||
row.setRecommendedOffer(trim(source.getRecommendedOffer()));
|
|
||||||
return row;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
|
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
|
||||||
@@ -2856,39 +2826,20 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private boolean sameRow(ShopDataCrawlRowDto left, ShopDataCrawlRowDto right) {
|
private boolean sameRow(ShopDataCrawlRowDto left, ShopDataCrawlRowDto right) {
|
||||||
return left != null && right != null
|
return ShopDataCrawlRowNormalizer.sameRow(left, right);
|
||||||
&& Objects.equals(trim(left.getDate()), trim(right.getDate()))
|
|
||||||
&& Objects.equals(trim(left.getAsin()), trim(right.getAsin()))
|
|
||||||
&& Objects.equals(trim(left.getBrand()), trim(right.getBrand()))
|
|
||||||
&& Objects.equals(trim(left.getCommodityImage()), trim(right.getCommodityImage()))
|
|
||||||
&& Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales()))
|
|
||||||
&& Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank()))
|
|
||||||
&& Objects.equals(trim(left.getPageViews()), trim(right.getPageViews()))
|
|
||||||
&& Objects.equals(trim(left.getUnitsSold()), trim(right.getUnitsSold()))
|
|
||||||
&& Objects.equals(trim(left.getPrice()), trim(right.getPrice()))
|
|
||||||
&& Objects.equals(trim(left.getRecommendedOffer()), trim(right.getRecommendedOffer()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 国家结果行稳定去重键:与 sameRow 的 10 字段 trim 比较语义等价,用于 O(1) 去重。 */
|
/** 国家结果行稳定去重键:与 sameRow 的 10 字段 trim 比较语义等价,用于 O(1) 去重。 */
|
||||||
static String rowDedupKey(ShopDataCrawlRowDto row) {
|
static String rowDedupKey(ShopDataCrawlRowDto row) {
|
||||||
if (row == null) {
|
return ShopDataCrawlRowNormalizer.rowDedupKey(row);
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return trim(row.getDate()) + ROW_KEY_SEPARATOR + trim(row.getAsin()) + ROW_KEY_SEPARATOR + trim(row.getBrand())
|
|
||||||
+ ROW_KEY_SEPARATOR + trim(row.getCommodityImage()) + ROW_KEY_SEPARATOR + trim(row.getInventorySales())
|
|
||||||
+ ROW_KEY_SEPARATOR + trim(row.getSalesRank()) + ROW_KEY_SEPARATOR + trim(row.getPageViews())
|
|
||||||
+ ROW_KEY_SEPARATOR + trim(row.getUnitsSold()) + ROW_KEY_SEPARATOR + trim(row.getPrice())
|
|
||||||
+ ROW_KEY_SEPARATOR + trim(row.getRecommendedOffer());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean rowEmpty(ShopDataCrawlRowDto row) {
|
private boolean rowEmpty(ShopDataCrawlRowDto row) {
|
||||||
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
|
return ShopDataCrawlRowNormalizer.rowEmpty(row);
|
||||||
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
|
|
||||||
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String trim(String value) {
|
private static String trim(String value) {
|
||||||
return value == null ? "" : value.trim();
|
return ShopDataCrawlRowNormalizer.trim(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
void deleteResultObjectIfUnreferenced(String resultFileUrl) {
|
void deleteResultObjectIfUnreferenced(String resultFileUrl) {
|
||||||
@@ -2964,18 +2915,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String normalizeCountry(String country) {
|
private String normalizeCountry(String country) {
|
||||||
if (country == null) {
|
return ShopDataCrawlRowNormalizer.normalizeCountry(country);
|
||||||
return "";
|
|
||||||
}
|
|
||||||
String value = country.trim().toUpperCase();
|
|
||||||
return switch (value) {
|
|
||||||
case "德国" -> "DE";
|
|
||||||
case "英国" -> "UK";
|
|
||||||
case "法国" -> "FR";
|
|
||||||
case "意大利" -> "IT";
|
|
||||||
case "西班牙" -> "ES";
|
|
||||||
default -> value;
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildTaskWorkbookFilename(FileTaskEntity task) {
|
private String buildTaskWorkbookFilename(FileTaskEntity task) {
|
||||||
@@ -3000,7 +2940,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String blankToNull(String value) {
|
private String blankToNull(String value) {
|
||||||
return blank(value) ? null : value.trim();
|
return ShopDataCrawlRowNormalizer.blankToNull(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void validateUserId(Long userId) {
|
private void validateUserId(Long userId) {
|
||||||
|
|||||||
+70
@@ -0,0 +1,70 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 103:ShopDataCrawlHistoryAssembler 历史查询组装器。
|
||||||
|
* 历史列表 VO 拼装(toHistoryItem + 文件任务状态链 attachFileJobState)从
|
||||||
|
* ShopDataCrawlTaskService 原样搬移;只读不落库;输出与现状逐字段一致
|
||||||
|
* (快照优先、实体字段兜底、task/job 附加,downloadUrl 恒为空)。
|
||||||
|
*/
|
||||||
|
public class ShopDataCrawlHistoryAssembler {
|
||||||
|
|
||||||
|
private static final int RESULT_PENDING = -1;
|
||||||
|
private static final int RESULT_SUCCESS = 1;
|
||||||
|
|
||||||
|
public ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task,
|
||||||
|
ShopDataCrawlResultItemVo snapshot, TaskFileJobEntity job) {
|
||||||
|
ShopDataCrawlResultItemVo item = snapshot != null ? snapshot : new ShopDataCrawlResultItemVo();
|
||||||
|
item.setResultId(entity.getId());
|
||||||
|
item.setTaskId(entity.getTaskId());
|
||||||
|
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
|
||||||
|
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
|
||||||
|
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
|
||||||
|
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
|
||||||
|
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
|
||||||
|
item.setCreatedAt(entity.getCreatedAt());
|
||||||
|
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
|
||||||
|
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
|
||||||
|
item.setDownloadUrl(null);
|
||||||
|
attachFileJobState(item, entity, job);
|
||||||
|
if (item.getCountryResults() == null) {
|
||||||
|
item.setCountryResults(new ArrayList<>());
|
||||||
|
}
|
||||||
|
if (item.getCountryCodes() == null) {
|
||||||
|
item.setCountryCodes(new ArrayList<>());
|
||||||
|
}
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
|
||||||
|
item.setFileReady(!blank(entity.getResultFileUrl()));
|
||||||
|
if (job == null) {
|
||||||
|
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
item.setFileJobId(job.getId());
|
||||||
|
item.setFileStatus(job.getStatus());
|
||||||
|
item.setFileError(job.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
|
||||||
|
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String firstNonBlank(String first, String second) {
|
||||||
|
return !blank(first) ? first : second;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean blank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
}
|
||||||
+101
@@ -0,0 +1,101 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 99:ShopDataCrawlRowNormalizer 行解析/归一化器。
|
||||||
|
* 规则与 ShopDataCrawlTaskService.trim / blank / blankToNull / trimToNull /
|
||||||
|
* normalizeCountry / copyRow / rowEmpty / sameRow / rowDedupKey 现状逐字节一致。
|
||||||
|
* rowDedupKey 与国家结果行稳定去重键语义(10 字段 trim 拼接)等价于 sameRow 的
|
||||||
|
* 逐字段 trim 比较。纯函数无状态。
|
||||||
|
*/
|
||||||
|
public final class ShopDataCrawlRowNormalizer {
|
||||||
|
|
||||||
|
/** 国家结果行去重键字段分隔符(控制字符,字段值 trim 后不可能包含)。 */
|
||||||
|
public static final String ROW_KEY_SEPARATOR = "";
|
||||||
|
|
||||||
|
private ShopDataCrawlRowNormalizer() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String trim(String value) {
|
||||||
|
return value == null ? "" : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean blank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String blankToNull(String value) {
|
||||||
|
return blank(value) ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String trimToNull(String value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String normalized = value.trim();
|
||||||
|
return normalized.isEmpty() ? null : normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String normalizeCountry(String country) {
|
||||||
|
if (country == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String value = country.trim().toUpperCase();
|
||||||
|
return switch (value) {
|
||||||
|
case "德国" -> "DE";
|
||||||
|
case "英国" -> "UK";
|
||||||
|
case "法国" -> "FR";
|
||||||
|
case "意大利" -> "IT";
|
||||||
|
case "西班牙" -> "ES";
|
||||||
|
default -> value;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) {
|
||||||
|
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||||
|
row.setDate(trim(source.getDate()));
|
||||||
|
row.setAsin(trim(source.getAsin()));
|
||||||
|
row.setBrand(trim(source.getBrand()));
|
||||||
|
row.setCommodityImage(trim(source.getCommodityImage()));
|
||||||
|
row.setInventorySales(trim(source.getInventorySales()));
|
||||||
|
row.setSalesRank(trim(source.getSalesRank()));
|
||||||
|
row.setPageViews(trim(source.getPageViews()));
|
||||||
|
row.setUnitsSold(trim(source.getUnitsSold()));
|
||||||
|
row.setPrice(trim(source.getPrice()));
|
||||||
|
row.setRecommendedOffer(trim(source.getRecommendedOffer()));
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean rowEmpty(ShopDataCrawlRowDto row) {
|
||||||
|
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
|
||||||
|
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
|
||||||
|
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean sameRow(ShopDataCrawlRowDto left, ShopDataCrawlRowDto right) {
|
||||||
|
return left != null && right != null
|
||||||
|
&& java.util.Objects.equals(trim(left.getDate()), trim(right.getDate()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getAsin()), trim(right.getAsin()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getBrand()), trim(right.getBrand()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getCommodityImage()), trim(right.getCommodityImage()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getPageViews()), trim(right.getPageViews()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getUnitsSold()), trim(right.getUnitsSold()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getPrice()), trim(right.getPrice()))
|
||||||
|
&& java.util.Objects.equals(trim(left.getRecommendedOffer()), trim(right.getRecommendedOffer()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 国家结果行稳定去重键:与 sameRow 的 10 字段 trim 比较语义等价,用于 O(1) 去重。 */
|
||||||
|
public static String rowDedupKey(ShopDataCrawlRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return trim(row.getDate()) + ROW_KEY_SEPARATOR + trim(row.getAsin()) + ROW_KEY_SEPARATOR + trim(row.getBrand())
|
||||||
|
+ ROW_KEY_SEPARATOR + trim(row.getCommodityImage()) + ROW_KEY_SEPARATOR + trim(row.getInventorySales())
|
||||||
|
+ ROW_KEY_SEPARATOR + trim(row.getSalesRank()) + ROW_KEY_SEPARATOR + trim(row.getPageViews())
|
||||||
|
+ ROW_KEY_SEPARATOR + trim(row.getUnitsSold()) + ROW_KEY_SEPARATOR + trim(row.getPrice())
|
||||||
|
+ ROW_KEY_SEPARATOR + trim(row.getRecommendedOffer());
|
||||||
|
}
|
||||||
|
}
|
||||||
+177
@@ -0,0 +1,177 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
|
import org.apache.poi.ss.usermodel.CellStyle;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 100:ShopDataCrawlSheetBuilder Sheet 构造器。
|
||||||
|
* 结果 Workbook/Sheet 构造辅助(表头、列序、样式、行值派生、模板校验)。
|
||||||
|
* 与现状 ShopDataCrawlExcelAssemblyService 一致:5 个国家工作表(英国/德国/法国/西班牙/意大利)、
|
||||||
|
* 10 列表头(HEADERS,品牌为最后一列)、图片列 2 宽 18*256、行高 80pt、
|
||||||
|
* 模板列映射(legacy/current/带品牌变体)、数据行从第 1 行。
|
||||||
|
* 不落库、无 IO 依赖;纯函数无状态。
|
||||||
|
*/
|
||||||
|
public final class ShopDataCrawlSheetBuilder {
|
||||||
|
|
||||||
|
public static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
|
||||||
|
public static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
|
||||||
|
public static final List<String> LEGACY_HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
|
||||||
|
public static final List<String> HEADERS_WITHOUT_BRAND = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
|
||||||
|
public static final List<String> HEADERS = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价", "品牌");
|
||||||
|
public static final int IMAGE_COLUMN = 2;
|
||||||
|
public static final int BRAND_COLUMN = HEADERS.size() - 1;
|
||||||
|
public static final int IMAGE_COLUMN_WIDTH = 18 * 256;
|
||||||
|
public static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
|
||||||
|
|
||||||
|
private ShopDataCrawlSheetBuilder() {
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 流式路径:空 workbook 上自建国家工作表与表头(数据行由调用方写入)。 */
|
||||||
|
public static Sheet createStreamingSheet(Workbook workbook, int index) {
|
||||||
|
Sheet sheet = workbook.createSheet(SHEETS.get(index));
|
||||||
|
Row header = sheet.createRow(0);
|
||||||
|
for (int column = 0; column < HEADERS.size(); column++) {
|
||||||
|
header.createCell(column).setCellValue(HEADERS.get(column));
|
||||||
|
}
|
||||||
|
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
|
||||||
|
return sheet;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void validateTemplate(XSSFWorkbook workbook) {
|
||||||
|
if (workbook.getNumberOfSheets() != SHEETS.size()) {
|
||||||
|
throw new BusinessException("店铺数据抓取模板工作表数量不正确");
|
||||||
|
}
|
||||||
|
for (int i = 0; i < SHEETS.size(); i++) {
|
||||||
|
Sheet sheet = workbook.getSheetAt(i);
|
||||||
|
if (!SHEETS.get(i).equals(sheet.getSheetName())) {
|
||||||
|
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
|
||||||
|
}
|
||||||
|
Row header = sheet.getRow(0);
|
||||||
|
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
|
||||||
|
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
|
||||||
|
List<String> expectedHeaders = currentTemplate
|
||||||
|
? (templateHasBrand ? HEADERS : HEADERS_WITHOUT_BRAND)
|
||||||
|
: LEGACY_HEADERS;
|
||||||
|
for (int column = 0; column < expectedHeaders.size(); column++) {
|
||||||
|
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
|
||||||
|
if (!expectedHeaders.get(column).equals(actual)) {
|
||||||
|
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 模板路径:从模板第 1 行按列映射提取输出列样式。 */
|
||||||
|
public static CellStyle[] templateStyles(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
|
||||||
|
Row styleRow = sheet.getRow(1);
|
||||||
|
CellStyle[] styles = new CellStyle[HEADERS.size()];
|
||||||
|
for (int column = 0; column < styles.length; column++) {
|
||||||
|
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
||||||
|
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
|
||||||
|
styles[column] = cell == null ? null : cell.getCellStyle();
|
||||||
|
}
|
||||||
|
return styles;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 模板路径:按列映射提取模板列宽。 */
|
||||||
|
public static int[] templateColumnWidths(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
|
||||||
|
int[] columnWidths = new int[HEADERS.size()];
|
||||||
|
for (int column = 0; column < columnWidths.length; column++) {
|
||||||
|
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
||||||
|
columnWidths[column] = sheet.getColumnWidth(sourceColumn);
|
||||||
|
}
|
||||||
|
return columnWidths;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 模板路径:覆写表头文本并保留模板表头样式。 */
|
||||||
|
public static void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
|
||||||
|
Row header = sheet.getRow(0);
|
||||||
|
if (header == null) header = sheet.createRow(0);
|
||||||
|
CellStyle[] styles = new CellStyle[HEADERS.size()];
|
||||||
|
for (int column = 0; column < HEADERS.size(); column++) {
|
||||||
|
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
||||||
|
Cell source = header.getCell(sourceColumn);
|
||||||
|
styles[column] = source == null ? null : source.getCellStyle();
|
||||||
|
}
|
||||||
|
for (int column = 0; column < HEADERS.size(); column++) {
|
||||||
|
Cell cell = header.getCell(column);
|
||||||
|
if (cell == null) cell = header.createCell(column);
|
||||||
|
if (styles[column] != null) cell.setCellStyle(styles[column]);
|
||||||
|
cell.setCellValue(HEADERS.get(column));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 模板路径:清空第 1 行起的全部数据行(保留表头)。 */
|
||||||
|
public static void clearDataRows(Sheet sheet) {
|
||||||
|
int last = sheet.getLastRowNum();
|
||||||
|
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
|
||||||
|
Row row = sheet.getRow(rowIndex);
|
||||||
|
if (row != null) {
|
||||||
|
sheet.removeRow(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 数据行 10 列值布局 + 样式应用(图片嵌入由调用方负责)。 */
|
||||||
|
public static void writeDataRowValues(Row row, ShopDataCrawlRowDto value, CellStyle[] styles) {
|
||||||
|
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
|
||||||
|
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
|
||||||
|
for (int column = 0; column < values.length; column++) {
|
||||||
|
Cell cell = row.createCell(column);
|
||||||
|
if (styles[column] != null) cell.setCellStyle(styles[column]);
|
||||||
|
cell.setCellValue(values[column] == null ? "" : values[column]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int templateColumnForOutput(int outputColumn, boolean currentTemplate, boolean templateHasBrand) {
|
||||||
|
if (outputColumn == BRAND_COLUMN) {
|
||||||
|
return templateHasBrand ? BRAND_COLUMN : 1;
|
||||||
|
}
|
||||||
|
return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String cellText(Row row, int column) {
|
||||||
|
Cell cell = row == null ? null : row.getCell(column);
|
||||||
|
return cell == null ? "" : cell.getStringCellValue().trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean blank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 结果条目 → 按国家分组的行列表(国家码 trim+UPPER,未命中 5 国列表的行丢弃)。 */
|
||||||
|
public static Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
|
||||||
|
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>();
|
||||||
|
COUNTRIES.forEach(country -> result.put(country, new ArrayList<>()));
|
||||||
|
if (items == null) return result;
|
||||||
|
for (ShopDataCrawlResultItemVo item : items) {
|
||||||
|
if (item == null || Boolean.FALSE.equals(item.getSuccess()) || item.getCountryResults() == null) continue;
|
||||||
|
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) {
|
||||||
|
String country = countryResult == null || countryResult.getCountry() == null ? "" : countryResult.getCountry().trim().toUpperCase();
|
||||||
|
if (result.containsKey(country) && countryResult.getItems() != null) result.get(country).addAll(countryResult.getItems());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int totalDataRows(XSSFWorkbook workbook) {
|
||||||
|
int total = 0;
|
||||||
|
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
|
||||||
|
total += Math.max(0, workbook.getSheetAt(i).getLastRowNum());
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
}
|
||||||
+4
-1
@@ -49,6 +49,7 @@ public class QueryAsinController {
|
|||||||
@Parameter(description = "分组ID") @RequestParam(required = false) Long groupId,
|
@Parameter(description = "分组ID") @RequestParam(required = false) Long groupId,
|
||||||
@Parameter(description = "店铺名") @RequestParam(required = false) String shopName,
|
@Parameter(description = "店铺名") @RequestParam(required = false) String shopName,
|
||||||
@Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin,
|
@Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin,
|
||||||
|
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(required = false) String country,
|
||||||
@Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId,
|
@Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId,
|
||||||
@Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) {
|
@Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) {
|
||||||
return ApiResponse.success(queryAsinService.page(
|
return ApiResponse.success(queryAsinService.page(
|
||||||
@@ -57,6 +58,7 @@ public class QueryAsinController {
|
|||||||
groupId,
|
groupId,
|
||||||
shopName,
|
shopName,
|
||||||
asin,
|
asin,
|
||||||
|
country,
|
||||||
operatorId,
|
operatorId,
|
||||||
Boolean.TRUE.equals(superAdmin)));
|
Boolean.TRUE.equals(superAdmin)));
|
||||||
}
|
}
|
||||||
@@ -67,9 +69,10 @@ public class QueryAsinController {
|
|||||||
@Parameter(description = "分组ID") @RequestParam(required = false) Long groupId,
|
@Parameter(description = "分组ID") @RequestParam(required = false) Long groupId,
|
||||||
@Parameter(description = "店铺名") @RequestParam(required = false) String shopName,
|
@Parameter(description = "店铺名") @RequestParam(required = false) String shopName,
|
||||||
@Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin,
|
@Parameter(description = "商品 ASIN") @RequestParam(required = false) String asin,
|
||||||
|
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(required = false) String country,
|
||||||
@Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId,
|
@Parameter(description = "当前操作人用户ID") @RequestParam(required = false) Long operatorId,
|
||||||
@Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) {
|
@Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) {
|
||||||
byte[] bytes = queryAsinService.export(groupId, shopName, asin, operatorId, Boolean.TRUE.equals(superAdmin));
|
byte[] bytes = queryAsinService.export(groupId, shopName, asin, country, operatorId, Boolean.TRUE.equals(superAdmin));
|
||||||
String filename = "query-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
|
String filename = "query-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
|
||||||
return ResponseEntity.ok()
|
return ResponseEntity.ok()
|
||||||
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
|
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
|
||||||
|
|||||||
+13
-1
@@ -28,8 +28,10 @@ import org.springframework.web.bind.annotation.RequestParam;
|
|||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -49,6 +51,9 @@ public class SkipPriceAsinController {
|
|||||||
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
|
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
|
||||||
@Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName,
|
@Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName,
|
||||||
@Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin,
|
@Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin,
|
||||||
|
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(name = "country", required = false) String country,
|
||||||
|
@Parameter(description = "最低价下限") @RequestParam(name = "minimum_price_from", required = false) BigDecimal minimumPriceFrom,
|
||||||
|
@Parameter(description = "最低价上限") @RequestParam(name = "minimum_price_to", required = false) BigDecimal minimumPriceTo,
|
||||||
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId,
|
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId,
|
||||||
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) {
|
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) {
|
||||||
return ApiResponse.success(skipPriceAsinService.page(
|
return ApiResponse.success(skipPriceAsinService.page(
|
||||||
@@ -57,6 +62,9 @@ public class SkipPriceAsinController {
|
|||||||
groupId,
|
groupId,
|
||||||
shopName,
|
shopName,
|
||||||
asin,
|
asin,
|
||||||
|
country,
|
||||||
|
minimumPriceFrom,
|
||||||
|
minimumPriceTo,
|
||||||
operatorId,
|
operatorId,
|
||||||
Boolean.TRUE.equals(superAdmin)));
|
Boolean.TRUE.equals(superAdmin)));
|
||||||
}
|
}
|
||||||
@@ -67,9 +75,13 @@ public class SkipPriceAsinController {
|
|||||||
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
|
@Parameter(description = "分组 ID") @RequestParam(name = "group_id", required = false) Long groupId,
|
||||||
@Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName,
|
@Parameter(description = "店铺名称") @RequestParam(name = "shop_name", required = false) String shopName,
|
||||||
@Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin,
|
@Parameter(description = "商品 ASIN") @RequestParam(name = "asin", required = false) String asin,
|
||||||
|
@Parameter(description = "国家代码(DE/UK/FR/IT/ES)") @RequestParam(name = "country", required = false) String country,
|
||||||
|
@Parameter(description = "最低价下限") @RequestParam(name = "minimum_price_from", required = false) BigDecimal minimumPriceFrom,
|
||||||
|
@Parameter(description = "最低价上限") @RequestParam(name = "minimum_price_to", required = false) BigDecimal minimumPriceTo,
|
||||||
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId,
|
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id", required = false) Long operatorId,
|
||||||
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) {
|
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin) {
|
||||||
byte[] bytes = skipPriceAsinService.export(groupId, shopName, asin, operatorId, Boolean.TRUE.equals(superAdmin));
|
byte[] bytes = skipPriceAsinService.export(groupId, shopName, asin, country, minimumPriceFrom, minimumPriceTo,
|
||||||
|
operatorId, Boolean.TRUE.equals(superAdmin));
|
||||||
String filename = "skip-price-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
|
String filename = "skip-price-asin-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
|
||||||
return ResponseEntity.ok()
|
return ResponseEntity.ok()
|
||||||
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
|
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
|
||||||
|
|||||||
+36
-12
@@ -1,6 +1,7 @@
|
|||||||
package com.nanri.aiimage.modules.shopkey.service;
|
package com.nanri.aiimage.modules.shopkey.service;
|
||||||
|
|
||||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.toolkit.support.SFunction;
|
||||||
import cn.hutool.core.util.IdUtil;
|
import cn.hutool.core.util.IdUtil;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||||
@@ -56,14 +57,15 @@ public class QueryAsinService {
|
|||||||
private final Map<String, QueryAsinImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
|
private final Map<String, QueryAsinImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public QueryAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin,
|
public QueryAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin,
|
||||||
Long operatorId, boolean superAdmin) {
|
String country, Long operatorId, boolean superAdmin) {
|
||||||
long safePage = Math.max(page, 1);
|
long safePage = Math.max(page, 1);
|
||||||
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
||||||
Long total = countFilteredRows(groupId, shopName, asin, operatorId, superAdmin);
|
Long total = countFilteredRows(groupId, shopName, asin, country, operatorId, superAdmin);
|
||||||
List<QueryAsinEntity> rows = listFilteredRows(
|
List<QueryAsinEntity> rows = listFilteredRows(
|
||||||
groupId,
|
groupId,
|
||||||
shopName,
|
shopName,
|
||||||
asin,
|
asin,
|
||||||
|
country,
|
||||||
operatorId,
|
operatorId,
|
||||||
superAdmin,
|
superAdmin,
|
||||||
(safePage - 1) * safePageSize,
|
(safePage - 1) * safePageSize,
|
||||||
@@ -83,8 +85,8 @@ public class QueryAsinService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
public byte[] export(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) {
|
public byte[] export(Long groupId, String shopName, String asin, String country, Long operatorId, boolean superAdmin) {
|
||||||
List<QueryAsinEntity> rows = listFilteredRows(groupId, shopName, asin, operatorId, superAdmin, null, null);
|
List<QueryAsinEntity> rows = listFilteredRows(groupId, shopName, asin, country, operatorId, superAdmin, null, null);
|
||||||
Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream()
|
Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream()
|
||||||
.map(QueryAsinEntity::getGroupId)
|
.map(QueryAsinEntity::getGroupId)
|
||||||
.filter(id -> id != null && id > 0)
|
.filter(id -> id != null && id > 0)
|
||||||
@@ -106,35 +108,46 @@ public class QueryAsinService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Long countFilteredRows(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) {
|
private Long countFilteredRows(Long groupId, String shopName, String asin, String country, Long operatorId, boolean superAdmin) {
|
||||||
return queryAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin));
|
return queryAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, country, operatorId, superAdmin));
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<QueryAsinEntity> listFilteredRows(Long groupId, String shopName, String asin,
|
private List<QueryAsinEntity> listFilteredRows(Long groupId, String shopName, String asin, String country,
|
||||||
Long operatorId, boolean superAdmin,
|
Long operatorId, boolean superAdmin,
|
||||||
Long offset, Long limit) {
|
Long offset, Long limit) {
|
||||||
LambdaQueryWrapper<QueryAsinEntity> query = buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin);
|
LambdaQueryWrapper<QueryAsinEntity> query = buildFilterQuery(groupId, shopName, asin, country, operatorId, superAdmin);
|
||||||
if (offset != null && limit != null) {
|
if (offset != null && limit != null) {
|
||||||
query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit));
|
query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit));
|
||||||
}
|
}
|
||||||
return queryAsinMapper.selectList(query);
|
return queryAsinMapper.selectList(query);
|
||||||
}
|
}
|
||||||
|
|
||||||
private LambdaQueryWrapper<QueryAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin,
|
private LambdaQueryWrapper<QueryAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin, String country,
|
||||||
Long operatorId, boolean superAdmin) {
|
Long operatorId, boolean superAdmin) {
|
||||||
String safeShopName = normalizeBlank(shopName);
|
String safeShopName = normalizeBlank(shopName);
|
||||||
String safeAsin = normalizeBlank(asin);
|
String safeAsin = normalizeBlank(asin);
|
||||||
|
String safeCountry = normalizeBlank(country).toUpperCase(Locale.ROOT);
|
||||||
Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false);
|
Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false);
|
||||||
LambdaQueryWrapper<QueryAsinEntity> query = new LambdaQueryWrapper<QueryAsinEntity>()
|
LambdaQueryWrapper<QueryAsinEntity> query = new LambdaQueryWrapper<QueryAsinEntity>()
|
||||||
.eq(groupId != null && groupId > 0, QueryAsinEntity::getGroupId, groupId)
|
.eq(groupId != null && groupId > 0, QueryAsinEntity::getGroupId, groupId)
|
||||||
.like(!safeShopName.isEmpty(), QueryAsinEntity::getShopName, safeShopName)
|
.like(!safeShopName.isEmpty(), QueryAsinEntity::getShopName, safeShopName)
|
||||||
.and(!safeAsin.isEmpty(), wrapper -> wrapper
|
.orderByDesc(QueryAsinEntity::getId);
|
||||||
|
if (!safeAsin.isEmpty()) {
|
||||||
|
if (safeCountry.isEmpty()) {
|
||||||
|
query.and(wrapper -> wrapper
|
||||||
.like(QueryAsinEntity::getAsinDe, safeAsin)
|
.like(QueryAsinEntity::getAsinDe, safeAsin)
|
||||||
.or().like(QueryAsinEntity::getAsinUk, safeAsin)
|
.or().like(QueryAsinEntity::getAsinUk, safeAsin)
|
||||||
.or().like(QueryAsinEntity::getAsinFr, safeAsin)
|
.or().like(QueryAsinEntity::getAsinFr, safeAsin)
|
||||||
.or().like(QueryAsinEntity::getAsinIt, safeAsin)
|
.or().like(QueryAsinEntity::getAsinIt, safeAsin)
|
||||||
.or().like(QueryAsinEntity::getAsinEs, safeAsin))
|
.or().like(QueryAsinEntity::getAsinEs, safeAsin));
|
||||||
.orderByDesc(QueryAsinEntity::getId);
|
} else {
|
||||||
|
query.like(queryAsinCountryColumn(safeCountry), safeAsin);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!safeCountry.isEmpty()) {
|
||||||
|
SFunction<QueryAsinEntity, ?> countryColumn = queryAsinCountryColumn(safeCountry);
|
||||||
|
query.isNotNull(countryColumn).ne(countryColumn, "");
|
||||||
|
}
|
||||||
if (!superAdmin) {
|
if (!superAdmin) {
|
||||||
if (accessibleGroupIds.isEmpty()) {
|
if (accessibleGroupIds.isEmpty()) {
|
||||||
query.eq(QueryAsinEntity::getId, -1L);
|
query.eq(QueryAsinEntity::getId, -1L);
|
||||||
@@ -145,6 +158,17 @@ public class QueryAsinService {
|
|||||||
return query;
|
return query;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private SFunction<QueryAsinEntity, ?> queryAsinCountryColumn(String country) {
|
||||||
|
return switch (country) {
|
||||||
|
case "DE" -> QueryAsinEntity::getAsinDe;
|
||||||
|
case "UK" -> QueryAsinEntity::getAsinUk;
|
||||||
|
case "FR" -> QueryAsinEntity::getAsinFr;
|
||||||
|
case "IT" -> QueryAsinEntity::getAsinIt;
|
||||||
|
case "ES" -> QueryAsinEntity::getAsinEs;
|
||||||
|
default -> throw new BusinessException("不支持的国家代码: " + country);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private void writeExportHeader(Sheet sheet) {
|
private void writeExportHeader(Sheet sheet) {
|
||||||
Row header = sheet.createRow(0);
|
Row header = sheet.createRow(0);
|
||||||
String[] headers = {
|
String[] headers = {
|
||||||
|
|||||||
+99
-11
@@ -1,6 +1,7 @@
|
|||||||
package com.nanri.aiimage.modules.shopkey.service;
|
package com.nanri.aiimage.modules.shopkey.service;
|
||||||
|
|
||||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.toolkit.support.SFunction;
|
||||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||||
import cn.hutool.core.util.IdUtil;
|
import cn.hutool.core.util.IdUtil;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
@@ -61,14 +62,18 @@ public class SkipPriceAsinService {
|
|||||||
private final Map<String, CachedSkipAsinLookup> skipAsinLookupCache = new ConcurrentHashMap<>();
|
private final Map<String, CachedSkipAsinLookup> skipAsinLookupCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public SkipPriceAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin,
|
public SkipPriceAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin,
|
||||||
|
String country, BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
|
||||||
Long operatorId, boolean superAdmin) {
|
Long operatorId, boolean superAdmin) {
|
||||||
long safePage = Math.max(page, 1);
|
long safePage = Math.max(page, 1);
|
||||||
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
||||||
Long total = countFilteredRows(groupId, shopName, asin, operatorId, superAdmin);
|
Long total = countFilteredRows(groupId, shopName, asin, country, minimumPriceFrom, minimumPriceTo, operatorId, superAdmin);
|
||||||
List<SkipPriceAsinEntity> rows = listFilteredRows(
|
List<SkipPriceAsinEntity> rows = listFilteredRows(
|
||||||
groupId,
|
groupId,
|
||||||
shopName,
|
shopName,
|
||||||
asin,
|
asin,
|
||||||
|
country,
|
||||||
|
minimumPriceFrom,
|
||||||
|
minimumPriceTo,
|
||||||
operatorId,
|
operatorId,
|
||||||
superAdmin,
|
superAdmin,
|
||||||
(safePage - 1) * safePageSize,
|
(safePage - 1) * safePageSize,
|
||||||
@@ -88,8 +93,11 @@ public class SkipPriceAsinService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
public byte[] export(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) {
|
public byte[] export(Long groupId, String shopName, String asin, String country,
|
||||||
List<SkipPriceAsinEntity> rows = listFilteredRows(groupId, shopName, asin, operatorId, superAdmin, null, null);
|
BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
|
||||||
|
Long operatorId, boolean superAdmin) {
|
||||||
|
List<SkipPriceAsinEntity> rows = listFilteredRows(groupId, shopName, asin, country,
|
||||||
|
minimumPriceFrom, minimumPriceTo, operatorId, superAdmin, null, null);
|
||||||
Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream()
|
Map<Long, String> groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream()
|
||||||
.map(SkipPriceAsinEntity::getGroupId)
|
.map(SkipPriceAsinEntity::getGroupId)
|
||||||
.filter(id -> id != null && id > 0)
|
.filter(id -> id != null && id > 0)
|
||||||
@@ -111,35 +119,84 @@ public class SkipPriceAsinService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Long countFilteredRows(Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) {
|
private Long countFilteredRows(Long groupId, String shopName, String asin, String country,
|
||||||
return skipPriceAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin));
|
BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
|
||||||
|
Long operatorId, boolean superAdmin) {
|
||||||
|
return skipPriceAsinMapper.selectCount(buildFilterQuery(groupId, shopName, asin, country,
|
||||||
|
minimumPriceFrom, minimumPriceTo, operatorId, superAdmin));
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<SkipPriceAsinEntity> listFilteredRows(Long groupId, String shopName, String asin,
|
private List<SkipPriceAsinEntity> listFilteredRows(Long groupId, String shopName, String asin, String country,
|
||||||
|
BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
|
||||||
Long operatorId, boolean superAdmin,
|
Long operatorId, boolean superAdmin,
|
||||||
Long offset, Long limit) {
|
Long offset, Long limit) {
|
||||||
LambdaQueryWrapper<SkipPriceAsinEntity> query = buildFilterQuery(groupId, shopName, asin, operatorId, superAdmin);
|
LambdaQueryWrapper<SkipPriceAsinEntity> query = buildFilterQuery(groupId, shopName, asin, country,
|
||||||
|
minimumPriceFrom, minimumPriceTo, operatorId, superAdmin);
|
||||||
if (offset != null && limit != null) {
|
if (offset != null && limit != null) {
|
||||||
query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit));
|
query.last("LIMIT " + Math.max(0L, offset) + ", " + Math.max(1L, limit));
|
||||||
}
|
}
|
||||||
return skipPriceAsinMapper.selectList(query);
|
return skipPriceAsinMapper.selectList(query);
|
||||||
}
|
}
|
||||||
|
|
||||||
private LambdaQueryWrapper<SkipPriceAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin,
|
private LambdaQueryWrapper<SkipPriceAsinEntity> buildFilterQuery(Long groupId, String shopName, String asin, String country,
|
||||||
|
BigDecimal minimumPriceFrom, BigDecimal minimumPriceTo,
|
||||||
Long operatorId, boolean superAdmin) {
|
Long operatorId, boolean superAdmin) {
|
||||||
String safeShopName = normalizeBlank(shopName);
|
String safeShopName = normalizeBlank(shopName);
|
||||||
String safeAsin = normalizeBlank(asin);
|
String safeAsin = normalizeBlank(asin);
|
||||||
|
String safeCountry = normalizeBlank(country).toUpperCase(Locale.ROOT);
|
||||||
Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false);
|
Set<Long> accessibleGroupIds = superAdmin ? Set.of() : shopManageGroupService.listAccessibleGroupIds(operatorId, false);
|
||||||
LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
||||||
.eq(groupId != null && groupId > 0, SkipPriceAsinEntity::getGroupId, groupId)
|
.eq(groupId != null && groupId > 0, SkipPriceAsinEntity::getGroupId, groupId)
|
||||||
.like(!safeShopName.isEmpty(), SkipPriceAsinEntity::getShopName, safeShopName)
|
.like(!safeShopName.isEmpty(), SkipPriceAsinEntity::getShopName, safeShopName)
|
||||||
.and(!safeAsin.isEmpty(), wrapper -> wrapper
|
.orderByDesc(SkipPriceAsinEntity::getId);
|
||||||
|
if (!safeAsin.isEmpty()) {
|
||||||
|
if (safeCountry.isEmpty()) {
|
||||||
|
query.and(wrapper -> wrapper
|
||||||
.like(SkipPriceAsinEntity::getAsinDe, safeAsin)
|
.like(SkipPriceAsinEntity::getAsinDe, safeAsin)
|
||||||
.or().like(SkipPriceAsinEntity::getAsinUk, safeAsin)
|
.or().like(SkipPriceAsinEntity::getAsinUk, safeAsin)
|
||||||
.or().like(SkipPriceAsinEntity::getAsinFr, safeAsin)
|
.or().like(SkipPriceAsinEntity::getAsinFr, safeAsin)
|
||||||
.or().like(SkipPriceAsinEntity::getAsinIt, safeAsin)
|
.or().like(SkipPriceAsinEntity::getAsinIt, safeAsin)
|
||||||
.or().like(SkipPriceAsinEntity::getAsinEs, safeAsin))
|
.or().like(SkipPriceAsinEntity::getAsinEs, safeAsin));
|
||||||
.orderByDesc(SkipPriceAsinEntity::getId);
|
} else {
|
||||||
|
query.like(skipPriceAsinCountryColumn(safeCountry), safeAsin);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!safeCountry.isEmpty()) {
|
||||||
|
SFunction<SkipPriceAsinEntity, ?> countryColumn = skipPriceAsinCountryColumn(safeCountry);
|
||||||
|
query.isNotNull(countryColumn).ne(countryColumn, "");
|
||||||
|
}
|
||||||
|
boolean hasMinimumPriceFrom = minimumPriceFrom != null;
|
||||||
|
boolean hasMinimumPriceTo = minimumPriceTo != null;
|
||||||
|
if (hasMinimumPriceFrom || hasMinimumPriceTo) {
|
||||||
|
if (safeCountry.isEmpty()) {
|
||||||
|
List<SFunction<SkipPriceAsinEntity, ?>> priceColumns = skipPriceAsinMinimumPriceColumns();
|
||||||
|
query.and(wrapper -> {
|
||||||
|
boolean first = true;
|
||||||
|
for (SFunction<SkipPriceAsinEntity, ?> column : priceColumns) {
|
||||||
|
if (!first) {
|
||||||
|
wrapper.or();
|
||||||
|
}
|
||||||
|
first = false;
|
||||||
|
wrapper.isNotNull(column);
|
||||||
|
if (hasMinimumPriceFrom) {
|
||||||
|
wrapper.ge(column, minimumPriceFrom);
|
||||||
|
}
|
||||||
|
if (hasMinimumPriceTo) {
|
||||||
|
wrapper.le(column, minimumPriceTo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
SFunction<SkipPriceAsinEntity, ?> priceColumn = skipPriceAsinMinimumPriceColumn(safeCountry);
|
||||||
|
query.isNotNull(priceColumn);
|
||||||
|
if (hasMinimumPriceFrom) {
|
||||||
|
query.ge(priceColumn, minimumPriceFrom);
|
||||||
|
}
|
||||||
|
if (hasMinimumPriceTo) {
|
||||||
|
query.le(priceColumn, minimumPriceTo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!superAdmin) {
|
if (!superAdmin) {
|
||||||
if (accessibleGroupIds.isEmpty()) {
|
if (accessibleGroupIds.isEmpty()) {
|
||||||
query.eq(SkipPriceAsinEntity::getId, -1L);
|
query.eq(SkipPriceAsinEntity::getId, -1L);
|
||||||
@@ -485,6 +542,37 @@ public class SkipPriceAsinService {
|
|||||||
return findCountryAsin(groupId, shopName, country, asin) != null;
|
return findCountryAsin(groupId, shopName, country, asin) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private SFunction<SkipPriceAsinEntity, ?> skipPriceAsinCountryColumn(String country) {
|
||||||
|
return switch (country) {
|
||||||
|
case "DE" -> SkipPriceAsinEntity::getAsinDe;
|
||||||
|
case "UK" -> SkipPriceAsinEntity::getAsinUk;
|
||||||
|
case "FR" -> SkipPriceAsinEntity::getAsinFr;
|
||||||
|
case "IT" -> SkipPriceAsinEntity::getAsinIt;
|
||||||
|
case "ES" -> SkipPriceAsinEntity::getAsinEs;
|
||||||
|
default -> throw new BusinessException("不支持的国家代码: " + country);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private SFunction<SkipPriceAsinEntity, ?> skipPriceAsinMinimumPriceColumn(String country) {
|
||||||
|
return switch (country) {
|
||||||
|
case "DE" -> SkipPriceAsinEntity::getMinimumPriceDe;
|
||||||
|
case "UK" -> SkipPriceAsinEntity::getMinimumPriceUk;
|
||||||
|
case "FR" -> SkipPriceAsinEntity::getMinimumPriceFr;
|
||||||
|
case "IT" -> SkipPriceAsinEntity::getMinimumPriceIt;
|
||||||
|
case "ES" -> SkipPriceAsinEntity::getMinimumPriceEs;
|
||||||
|
default -> throw new BusinessException("不支持的国家代码: " + country);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<SFunction<SkipPriceAsinEntity, ?>> skipPriceAsinMinimumPriceColumns() {
|
||||||
|
return List.of(
|
||||||
|
SkipPriceAsinEntity::getMinimumPriceDe,
|
||||||
|
SkipPriceAsinEntity::getMinimumPriceUk,
|
||||||
|
SkipPriceAsinEntity::getMinimumPriceFr,
|
||||||
|
SkipPriceAsinEntity::getMinimumPriceIt,
|
||||||
|
SkipPriceAsinEntity::getMinimumPriceEs);
|
||||||
|
}
|
||||||
|
|
||||||
private SkipPriceAsinEntity findCountryAsin(Long groupId, String shopName, String country, String asin) {
|
private SkipPriceAsinEntity findCountryAsin(Long groupId, String shopName, String country, String asin) {
|
||||||
LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
LambdaQueryWrapper<SkipPriceAsinEntity> query = new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
||||||
.eq(SkipPriceAsinEntity::getGroupId, groupId)
|
.eq(SkipPriceAsinEntity::getGroupId, groupId)
|
||||||
|
|||||||
+10
@@ -41,6 +41,8 @@ import java.io.InputStream;
|
|||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -189,6 +191,14 @@ public class ShopMatchController {
|
|||||||
return ApiResponse.success(shopMatchTaskService.getTaskProgressBatch(request.getTaskIds()));
|
return ApiResponse.success(shopMatchTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(shopMatchTaskService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks/{taskId}/result")
|
@PostMapping("/tasks/{taskId}/result")
|
||||||
@Operation(summary = "提交匹配结果", description = "Python 可多次分批回传结果;后端会组装文件并在超时场景自动补偿收尾。")
|
@Operation(summary = "提交匹配结果", description = "Python 可多次分批回传结果;后端会组装文件并在超时场景自动补偿收尾。")
|
||||||
public ApiResponse<Void> submitResult(
|
public ApiResponse<Void> submitResult(
|
||||||
|
|||||||
+7
@@ -55,6 +55,8 @@ import java.util.LinkedHashSet;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -62,6 +64,10 @@ import java.util.Map;
|
|||||||
public class ShopMatchTaskService {
|
public class ShopMatchTaskService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "SHOP_MATCH";
|
private static final String MODULE_TYPE = "SHOP_MATCH";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
|
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
|
||||||
|
|
||||||
@@ -79,6 +85,7 @@ public class ShopMatchTaskService {
|
|||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
private final SkipPriceAsinService skipPriceAsinService;
|
private final SkipPriceAsinService skipPriceAsinService;
|
||||||
private final QueryAsinMapper queryAsinMapper;
|
private final QueryAsinMapper queryAsinMapper;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
|
|||||||
+10
@@ -7,11 +7,13 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
|||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest;
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightRequest;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinFilterConditionVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinFilterConditionVo;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo;
|
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo;
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
@@ -120,6 +122,14 @@ public class SimilarAsinController {
|
|||||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<SimilarAsinTaskLightBatchVo> progressLight(@Valid @RequestBody SimilarAsinTaskLightRequest request) {
|
||||||
|
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks/{taskId}/activate")
|
@PostMapping("/tasks/{taskId}/activate")
|
||||||
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING。")
|
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING。")
|
||||||
public ApiResponse<Void> activate(
|
public ApiResponse<Void> activate(
|
||||||
|
|||||||
+19
@@ -0,0 +1,19 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.model.dto;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import jakarta.validation.constraints.NotNull;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Schema(description = "相似ASIN检测任务轻量进度查询请求")
|
||||||
|
public class SimilarAsinTaskLightRequest {
|
||||||
|
@NotNull
|
||||||
|
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
private List<Long> taskIds;
|
||||||
|
}
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "相似ASIN检测任务轻量进度响应")
|
||||||
|
public class SimilarAsinTaskLightBatchVo {
|
||||||
|
@Schema(description = "轻量进度项列表。顺序按请求 taskIds 处理。")
|
||||||
|
private List<SimilarAsinTaskLightVo> items = new ArrayList<>();
|
||||||
|
@Schema(description = "未找到或不属于相似ASIN检测模块的任务 ID 列表。", example = "[99999]")
|
||||||
|
private List<Long> missingTaskIds = new ArrayList<>();
|
||||||
|
}
|
||||||
+23
@@ -0,0 +1,23 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "相似ASIN检测任务轻量进度项(前端轮询专用,不含明细/result 内容)")
|
||||||
|
public class SimilarAsinTaskLightVo {
|
||||||
|
@Schema(description = "任务 ID。", example = "3938")
|
||||||
|
private Long taskId;
|
||||||
|
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
|
||||||
|
private String status;
|
||||||
|
@Schema(description = "业务状态码(预留字段,当前恒为 null)。", example = "null")
|
||||||
|
private String statusCode;
|
||||||
|
@Schema(description = "结果文件组装 Job 状态:SUCCESS/RUNNING/FAILED;无 Job 时为空。", example = "SUCCESS")
|
||||||
|
private String fileStatus;
|
||||||
|
@Schema(description = "文件组装失败时的错误信息;无错误时为空。", example = "组装失败: 文件写入超时")
|
||||||
|
private String fileError;
|
||||||
|
@Schema(description = "结果文件是否已就绪。", example = "true")
|
||||||
|
private Boolean fileReady;
|
||||||
|
@Schema(description = "最后更新时间,ISO 本地时间字符串。", example = "2026-04-26T10:05:00")
|
||||||
|
private String updatedAt;
|
||||||
|
}
|
||||||
+84
-447
@@ -35,6 +35,11 @@ import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
|||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo;
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
|
||||||
import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache;
|
import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache;
|
||||||
import com.nanri.aiimage.modules.similarasin.util.ExcelCellImageWriter;
|
import com.nanri.aiimage.modules.similarasin.util.ExcelCellImageWriter;
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||||
@@ -59,12 +64,9 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.poi.ss.usermodel.Cell;
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
import org.apache.poi.ss.usermodel.CellStyle;
|
import org.apache.poi.ss.usermodel.CellStyle;
|
||||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
|
||||||
import org.apache.poi.ss.usermodel.Font;
|
import org.apache.poi.ss.usermodel.Font;
|
||||||
import org.apache.poi.ss.usermodel.Row;
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.ss.usermodel.Workbook;
|
|
||||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
|
||||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
@@ -136,6 +138,7 @@ public class SimilarAsinTaskService {
|
|||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||||
|
private static final int MAX_LIGHT_TASK_IDS = 200;
|
||||||
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
|
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
|
||||||
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
|
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
|
||||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||||
@@ -336,6 +339,20 @@ public class SimilarAsinTaskService {
|
|||||||
* best-effort:service 内部所有异常都已吞掉,不影响主流程。
|
* best-effort:service 内部所有异常都已吞掉,不影响主流程。
|
||||||
*/
|
*/
|
||||||
private final SimilarAsinImagePrefetchService imagePrefetchService;
|
private final SimilarAsinImagePrefetchService imagePrefetchService;
|
||||||
|
/**
|
||||||
|
* Task 89:Excel 行解析器(表头/单元格读取、别名匹配、空行跳过、单字段截断)。
|
||||||
|
* 由 parseAndCreateTask 委托;解析语义与搬移前 parseWorkbook 完全一致。
|
||||||
|
*/
|
||||||
|
SimilarAsinExcelParser excelParser = new SimilarAsinExcelParser();
|
||||||
|
/**
|
||||||
|
* Task 91:历史查询 VO 拼装(toHistoryItem + 进度链)搬入独立组件;
|
||||||
|
* 由 history/progressBatch 委托,只读不落库,输出与现状一致。
|
||||||
|
*/
|
||||||
|
private SimilarAsinHistoryAssembler historyAssembler() {
|
||||||
|
return new SimilarAsinHistoryAssembler(
|
||||||
|
taskScopeStateMapper, taskChunkMapper, fileTaskMapper,
|
||||||
|
taskProgressSnapshotService, ossStorageService, transientPayloadStorageService, objectMapper);
|
||||||
|
}
|
||||||
@Autowired
|
@Autowired
|
||||||
@Qualifier("taskQueueExecutor")
|
@Qualifier("taskQueueExecutor")
|
||||||
private TaskExecutor taskQueueExecutor;
|
private TaskExecutor taskQueueExecutor;
|
||||||
@@ -466,7 +483,7 @@ public class SimilarAsinTaskService {
|
|||||||
+ " (" + input.length() + " bytes > " + maxBytes + " bytes)");
|
+ " (" + input.length() + " bytes > " + maxBytes + " bytes)");
|
||||||
}
|
}
|
||||||
|
|
||||||
ParsedWorkbook parsed = parseWorkbook(input, source);
|
ParsedWorkbook parsed = parseWorkbookDelegated(input, source);
|
||||||
totalRows += parsed.totalRows();
|
totalRows += parsed.totalRows();
|
||||||
droppedRows += parsed.droppedRows();
|
droppedRows += parsed.droppedRows();
|
||||||
allRows.addAll(parsed.allRows());
|
allRows.addAll(parsed.allRows());
|
||||||
@@ -487,7 +504,7 @@ public class SimilarAsinTaskService {
|
|||||||
sourceFiles.size(), allRows.size());
|
sourceFiles.size(), allRows.size());
|
||||||
}
|
}
|
||||||
long parsedAt = System.nanoTime();
|
long parsedAt = System.nanoTime();
|
||||||
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows);
|
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(allRows);
|
||||||
long groupedAt = System.nanoTime();
|
long groupedAt = System.nanoTime();
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
@@ -640,13 +657,49 @@ public class SimilarAsinTaskService {
|
|||||||
.map(FileResultEntity::getId)
|
.map(FileResultEntity::getId)
|
||||||
.filter(Objects::nonNull)
|
.filter(Objects::nonNull)
|
||||||
.toList());
|
.toList());
|
||||||
for (FileResultEntity row : rows) {
|
vo.getItems().addAll(historyAssembler().buildHistoryItems(rows, taskMap, jobMap));
|
||||||
FileTaskEntity task = taskMap.get(row.getTaskId());
|
return vo;
|
||||||
String taskStatus = task == null ? null : task.getStatus();
|
}
|
||||||
if (STATUS_PENDING.equals(taskStatus)) {
|
|
||||||
|
public SimilarAsinTaskLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
|
||||||
|
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||||
|
.filter(id -> id != null && id > 0)
|
||||||
|
.distinct()
|
||||||
|
.limit(MAX_LIGHT_TASK_IDS)
|
||||||
|
.toList();
|
||||||
|
if (normalizedIds.isEmpty()) {
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||||
|
for (FileTaskEntity task : fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.select(FileTaskEntity::getId,
|
||||||
|
FileTaskEntity::getStatus,
|
||||||
|
FileTaskEntity::getUpdatedAt)
|
||||||
|
.in(FileTaskEntity::getId, normalizedIds))) {
|
||||||
|
if (task != null && MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
|
taskMap.put(task.getId(), task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByTaskIds(MODULE_TYPE, new ArrayList<>(taskMap.keySet()));
|
||||||
|
for (Long taskId : normalizedIds) {
|
||||||
|
FileTaskEntity task = taskMap.get(taskId);
|
||||||
|
if (task == null) {
|
||||||
|
vo.getMissingTaskIds().add(taskId);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
vo.getItems().add(toHistoryItem(row, task, jobMap.get(row.getId())));
|
SimilarAsinTaskLightVo item = new SimilarAsinTaskLightVo();
|
||||||
|
item.setTaskId(task.getId());
|
||||||
|
item.setStatus(resolveDisplayTaskStatus(task));
|
||||||
|
TaskFileJobEntity job = jobMap.get(taskId);
|
||||||
|
boolean fileReady = job != null
|
||||||
|
&& job.getResultFileUrl() != null
|
||||||
|
&& !job.getResultFileUrl().isBlank();
|
||||||
|
item.setFileReady(fileReady);
|
||||||
|
item.setFileStatus(job == null ? null : job.getStatus());
|
||||||
|
item.setFileError(job == null ? null : job.getErrorMessage());
|
||||||
|
item.setUpdatedAt(fmt(task.getUpdatedAt()));
|
||||||
|
vo.getItems().add(item);
|
||||||
}
|
}
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
@@ -708,7 +761,8 @@ public class SimilarAsinTaskService {
|
|||||||
detail.setTask(toTaskItem(task));
|
detail.setTask(toTaskItem(task));
|
||||||
FileResultEntity resultRow = resultByTaskId.get(taskId);
|
FileResultEntity resultRow = resultByTaskId.get(taskId);
|
||||||
if (resultRow != null) {
|
if (resultRow != null) {
|
||||||
detail.getItems().add(toHistoryItem(resultRow, task, jobMap.get(resultRow.getId())));
|
detail.getItems().addAll(historyAssembler().buildHistoryItems(
|
||||||
|
List.of(resultRow), taskMap, jobMap));
|
||||||
}
|
}
|
||||||
vo.getItems().add(detail);
|
vo.getItems().add(detail);
|
||||||
}
|
}
|
||||||
@@ -1751,7 +1805,7 @@ public class SimilarAsinTaskService {
|
|||||||
private Map<String, List<SimilarAsinParsedRowVo>> loadAllRowsByBaseId(FileTaskEntity task) {
|
private Map<String, List<SimilarAsinParsedRowVo>> loadAllRowsByBaseId(FileTaskEntity task) {
|
||||||
try {
|
try {
|
||||||
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
|
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
|
||||||
return groupRowsByBaseId(resolveAllRows(payload));
|
return SimilarAsinGroupingConverter.groupRowsByBaseId(resolveAllRows(payload));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage());
|
log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||||
return new LinkedHashMap<>();
|
return new LinkedHashMap<>();
|
||||||
@@ -1791,36 +1845,6 @@ public class SimilarAsinTaskService {
|
|||||||
return candidates;
|
return candidates;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 6:分组数据改为索引/范围引用。组内行在原 items 中必然连续
|
|
||||||
* (parseWorkbook 按行遍历、同 baseId 块连续收集),因此只需要
|
|
||||||
* [startIndex, endIndex) 半开区间即可唯一定位组内行,
|
|
||||||
* 行对象仅存在于 items 一次,避免 groups 嵌套复制完整行对象。
|
|
||||||
*/
|
|
||||||
private List<SimilarAsinParsedGroupVo> buildParsedGroups(List<SimilarAsinParsedRowVo> rows) {
|
|
||||||
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
|
|
||||||
int cursor = 0;
|
|
||||||
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
|
|
||||||
if (siblings == null || siblings.isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
SimilarAsinParsedRowVo first = siblings.getFirst();
|
|
||||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
|
||||||
group.setSourceFileKey(first.getSourceFileKey());
|
|
||||||
group.setSourceFilename(first.getSourceFilename());
|
|
||||||
group.setGroupKey(firstNonBlank(first.getGroupKey(), buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
|
|
||||||
group.setBaseId(baseId(first.getDisplayId()));
|
|
||||||
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
|
|
||||||
group.setItemCount(siblings.size());
|
|
||||||
int end = cursor + siblings.size();
|
|
||||||
group.setStartIndex(cursor);
|
|
||||||
group.setEndIndex(end);
|
|
||||||
cursor = end;
|
|
||||||
groups.add(group);
|
|
||||||
}
|
|
||||||
return groups;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<SimilarAsinParsedGroupVo> buildResponsePreviewGroups(List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
|
private List<SimilarAsinParsedGroupVo> buildResponsePreviewGroups(List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
|
||||||
if (groups == null || groups.isEmpty()) {
|
if (groups == null || groups.isEmpty()) {
|
||||||
return List.of();
|
return List.of();
|
||||||
@@ -1887,18 +1911,6 @@ public class SimilarAsinTaskService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
|
|
||||||
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
|
|
||||||
if (rows == null) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
for (SimilarAsinParsedRowVo row : rows) {
|
|
||||||
String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId()));
|
|
||||||
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<SimilarAsinResultRowDto> flattenSubmittedRows(SimilarAsinSubmitResultRequest request) {
|
private List<SimilarAsinResultRowDto> flattenSubmittedRows(SimilarAsinSubmitResultRequest request) {
|
||||||
if (request == null) {
|
if (request == null) {
|
||||||
return List.of();
|
return List.of();
|
||||||
@@ -4718,8 +4730,11 @@ public class SimilarAsinTaskService {
|
|||||||
return firstNonBlank(resultRow.getUrl(), fallbackUrl);
|
return firstNonBlank(resultRow.getUrl(), fallbackUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ParsedWorkbook parseWorkbook(File input, SimilarAsinSourceFileDto source) {
|
/**
|
||||||
DataFormatter formatter = new DataFormatter();
|
* Task 89:解析委托 SimilarAsinExcelParser(表头/单元格读取、别名匹配、空行跳过、字段截断),
|
||||||
|
* 保留服务侧编排语义:zip 探针、status 列过滤、类目重试判定、行号/分组键/rowToken 组装。
|
||||||
|
*/
|
||||||
|
private ParsedWorkbook parseWorkbookDelegated(File input, SimilarAsinSourceFileDto source) {
|
||||||
try {
|
try {
|
||||||
// Task 8:受控读取——先探测 zip 条目数与解压体积,超限拒绝
|
// Task 8:受控读取——先探测 zip 条目数与解压体积,超限拒绝
|
||||||
probeWorkbookZipBounds(input, source.getOriginalFilename());
|
probeWorkbookZipBounds(input, source.getOriginalFilename());
|
||||||
@@ -4729,26 +4744,9 @@ public class SimilarAsinTaskService {
|
|||||||
// 非 zip 或损坏文件:留给 WorkbookFactory 尝试后由下方 catch 转业务异常
|
// 非 zip 或损坏文件:留给 WorkbookFactory 尝试后由下方 catch 转业务异常
|
||||||
log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage());
|
log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage());
|
||||||
}
|
}
|
||||||
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0);
|
try {
|
||||||
Row header = sheet.getRow(0);
|
SimilarAsinExcelParser.ParsedSheet parsedSheet = excelParser.parse(input, resolveMaxFieldLength());
|
||||||
if (header == null) {
|
List<String> headers = parsedSheet.headers();
|
||||||
throw new BusinessException("Excel 表头为空");
|
|
||||||
}
|
|
||||||
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
|
||||||
List<String> headers = readHeaders(header, formatter);
|
|
||||||
int idCol = findRequiredHeader(headerMap, "id");
|
|
||||||
int asinCol = findRequiredHeader(headerMap, "asin");
|
|
||||||
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
|
||||||
// 兼容旧 xlsx:RESULT_HEADERS 已不再写出 sku 列,但仍要支持旧版结果簿重新上传,
|
|
||||||
// 因此这里仍按可选列读取并保留到 DTO,enrichRowForLlm 仍可使用,仅导出阶段不再写出。
|
|
||||||
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
|
|
||||||
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
|
||||||
int urlCol = findOptionalHeaderExact(headerMap,
|
|
||||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
|
||||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
|
||||||
int titleCol = findOptionalHeaderExact(headerMap,
|
|
||||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
|
||||||
|
|
||||||
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
|
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
|
||||||
String statusHeader = statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : null;
|
String statusHeader = statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : null;
|
||||||
|
|
||||||
@@ -4758,14 +4756,10 @@ public class SimilarAsinTaskService {
|
|||||||
int validRows = 0;
|
int validRows = 0;
|
||||||
String currentBlockBaseId = "";
|
String currentBlockBaseId = "";
|
||||||
String currentGroupKey = "";
|
String currentGroupKey = "";
|
||||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
for (SimilarAsinExcelParser.SimilarAsinExcelRow parsedRow : parsedSheet.rows()) {
|
||||||
Row row = sheet.getRow(i);
|
String id = parsedRow.id();
|
||||||
if (row == null) {
|
String asin = parsedRow.asin();
|
||||||
continue;
|
String country = parsedRow.country();
|
||||||
}
|
|
||||||
String id = cell(row, idCol, formatter);
|
|
||||||
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
|
|
||||||
String country = cell(row, countryCol, formatter);
|
|
||||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -4778,7 +4772,7 @@ public class SimilarAsinTaskService {
|
|||||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||||
vo.setSourceFileKey(source.getFileKey());
|
vo.setSourceFileKey(source.getFileKey());
|
||||||
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
|
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
|
||||||
vo.setRowIndex(i + 1);
|
vo.setRowIndex(parsedRow.rowIndex());
|
||||||
vo.setSourceId(id);
|
vo.setSourceId(id);
|
||||||
vo.setDisplayId(normalizeDisplayId(id));
|
vo.setDisplayId(normalizeDisplayId(id));
|
||||||
String rowBaseId = baseId(vo.getDisplayId());
|
String rowBaseId = baseId(vo.getDisplayId());
|
||||||
@@ -4790,11 +4784,11 @@ public class SimilarAsinTaskService {
|
|||||||
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
|
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
|
||||||
vo.setAsin(asin);
|
vo.setAsin(asin);
|
||||||
vo.setCountry(country);
|
vo.setCountry(country);
|
||||||
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : "");
|
vo.setSku(parsedRow.sku());
|
||||||
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : "");
|
vo.setPrice(parsedRow.price());
|
||||||
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
|
vo.setUrl(parsedRow.url());
|
||||||
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
|
vo.setTitle(parsedRow.title());
|
||||||
vo.setValues(readRowValues(row, headers, formatter));
|
vo.setValues(parsedRow.values());
|
||||||
allRows.add(vo);
|
allRows.add(vo);
|
||||||
}
|
}
|
||||||
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
|
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
|
||||||
@@ -4911,86 +4905,6 @@ public class SimilarAsinTaskService {
|
|||||||
return !normalized.isBlank() && !isTechnicalLlmFailure(normalized);
|
return !normalized.isBlank() && !isTechnicalLlmFailure(normalized);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
|
||||||
Map<String, Integer> map = new LinkedHashMap<>();
|
|
||||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
|
||||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
|
||||||
if (!val.isBlank()) {
|
|
||||||
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return map;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<String> readHeaders(Row header, DataFormatter formatter) {
|
|
||||||
List<String> headers = new ArrayList<>();
|
|
||||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
|
||||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
|
||||||
headers.add(val.isBlank() ? "列" + (i + 1) : val);
|
|
||||||
}
|
|
||||||
return headers;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
|
|
||||||
Map<String, String> values = new LinkedHashMap<>();
|
|
||||||
for (int i = 0; i < headers.size(); i++) {
|
|
||||||
values.put(headers.get(i), cell(row, i, formatter));
|
|
||||||
}
|
|
||||||
return values;
|
|
||||||
}
|
|
||||||
|
|
||||||
private int findRequiredHeader(Map<String, Integer> map, String... names) {
|
|
||||||
int idx = findOptionalHeader(map, names);
|
|
||||||
if (idx < 0) {
|
|
||||||
throw new BusinessException("缺少必要表头: " + String.join("/", names));
|
|
||||||
}
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
|
|
||||||
private int findOptionalHeader(Map<String, Integer> map, String... names) {
|
|
||||||
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
|
||||||
for (String name : names) {
|
|
||||||
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
|
|
||||||
return entry.getValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
|
|
||||||
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
|
||||||
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
|
|
||||||
for (String name : names) {
|
|
||||||
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
|
|
||||||
return entry.getValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String normalizeHeaderAlias(String value) {
|
|
||||||
String normalized = normalize(value).toLowerCase(Locale.ROOT);
|
|
||||||
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String cell(Row row, int col, DataFormatter formatter) {
|
|
||||||
if (col < 0) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
String value = normalize(formatter.formatCellValue(row.getCell(col)));
|
|
||||||
if (isSpreadsheetErrorValue(value)) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
// Task 7:单字段长度上限,防止超长单元格导致内存无界增长
|
|
||||||
int maxLen = resolveMaxFieldLength();
|
|
||||||
if (value.length() > maxLen) {
|
|
||||||
return value.substring(0, maxLen);
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static boolean isSpreadsheetErrorValue(String value) {
|
private static boolean isSpreadsheetErrorValue(String value) {
|
||||||
String normalized = value == null ? "" : value.trim();
|
String normalized = value == null ? "" : value.trim();
|
||||||
return normalized.startsWith("#") && normalized.endsWith("!");
|
return normalized.startsWith("#") && normalized.endsWith("!");
|
||||||
@@ -5133,253 +5047,6 @@ public class SimilarAsinTaskService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
|
||||||
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
|
|
||||||
vo.setResultId(row.getId());
|
|
||||||
vo.setTaskId(row.getTaskId());
|
|
||||||
vo.setSourceFilename(row.getSourceFilename());
|
|
||||||
vo.setResultFilename(row.getResultFilename());
|
|
||||||
vo.setDownloadUrl(buildFreshDownloadUrl(row));
|
|
||||||
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
|
|
||||||
attachFileJobState(vo, row, job, task);
|
|
||||||
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
|
||||||
vo.setError(row.getErrorMessage());
|
|
||||||
vo.setRowCount(row.getRowCount());
|
|
||||||
vo.setCreatedAt(fmt(row.getCreatedAt()));
|
|
||||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
|
||||||
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
|
|
||||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
|
||||||
return vo;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildFreshDownloadUrl(FileResultEntity row) {
|
|
||||||
String resultFileUrl = row == null ? null : row.getResultFileUrl();
|
|
||||||
if (resultFileUrl == null || resultFileUrl.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
|
|
||||||
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
|
|
||||||
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
log.warn("[similar-asin] build public download url failed resultId={} err={}",
|
|
||||||
row.getId(), ex.getMessage());
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
|
||||||
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
|
|
||||||
if (job == null) {
|
|
||||||
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
|
||||||
attachFileProgress(vo, row, job, task);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
vo.setFileJobId(job.getId());
|
|
||||||
vo.setFileStatus(job.getStatus());
|
|
||||||
vo.setFileError(job.getErrorMessage());
|
|
||||||
attachFileProgress(vo, row, job, task);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
|
||||||
if (row == null || row.getTaskId() == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Long taskId = row.getTaskId();
|
|
||||||
if (Boolean.TRUE.equals(vo.getFileReady())) {
|
|
||||||
vo.setFileProgressCurrent(1);
|
|
||||||
vo.setFileProgressTotal(1);
|
|
||||||
vo.setFileProgressPercent(100);
|
|
||||||
vo.setFileProgressMessage("结果文件已生成");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
int llmCompleted = countCompletedLlmStates(taskId);
|
|
||||||
int llmPending = countPendingLlmStates(taskId);
|
|
||||||
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
|
||||||
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
|
|
||||||
if (!uploadComplete) {
|
|
||||||
if (canTreatTaskAsCompleted(task, row, job)) {
|
|
||||||
vo.setFileProgressCurrent(1);
|
|
||||||
vo.setFileProgressTotal(1);
|
|
||||||
vo.setFileProgressPercent(100);
|
|
||||||
vo.setFileProgressMessage("结果文件已生成");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (llmCompleted + llmPending > 0) {
|
|
||||||
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
attachPythonUploadProgress(vo, taskId);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (llmCompleted + llmPending > 0) {
|
|
||||||
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (snapshot == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
|
||||||
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
|
||||||
if (total <= 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
|
|
||||||
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
|
|
||||||
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
|
|
||||||
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
|
|
||||||
vo.setFileProgressCurrent(current);
|
|
||||||
vo.setFileProgressTotal(total);
|
|
||||||
vo.setFileProgressPercent(percent);
|
|
||||||
vo.setFileProgressMessage(snapshot.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
|
|
||||||
TaskFileJobEntity job,
|
|
||||||
TaskProgressSnapshotEntity snapshot,
|
|
||||||
int llmCompleted,
|
|
||||||
int llmPending,
|
|
||||||
boolean uploadComplete) {
|
|
||||||
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
|
|
||||||
int total = calculateLlmDisplayTotal(snapshot, observed);
|
|
||||||
if (!uploadComplete) {
|
|
||||||
total = Math.max(total, observed + 1);
|
|
||||||
}
|
|
||||||
int current = Math.max(0, Math.min(llmCompleted, total));
|
|
||||||
vo.setFileProgressCurrent(current);
|
|
||||||
vo.setFileProgressTotal(total);
|
|
||||||
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
|
|
||||||
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
|
|
||||||
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
|
|
||||||
vo.setFileProgressMessage(uploadComplete
|
|
||||||
? (llmPending > 0
|
|
||||||
? buildLlmProgressMessage(current, total, llmPending)
|
|
||||||
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
|
|
||||||
: buildUploadingLlmProgressMessage(current, total, llmPending));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
|
|
||||||
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
|
|
||||||
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
|
|
||||||
int current = Math.max(0, Math.min(progress.current(), total));
|
|
||||||
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
|
|
||||||
vo.setFileProgressCurrent(current);
|
|
||||||
vo.setFileProgressTotal(total);
|
|
||||||
vo.setFileProgressPercent(percent);
|
|
||||||
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
|
|
||||||
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
|
|
||||||
if (chunkProgress.total() > 0) {
|
|
||||||
return chunkProgress;
|
|
||||||
}
|
|
||||||
if (chunkProgress.current() > 0) {
|
|
||||||
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
|
|
||||||
}
|
|
||||||
int totalRows = 0;
|
|
||||||
try {
|
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
|
||||||
totalRows = allRowCount(task);
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
totalRows = 0;
|
|
||||||
}
|
|
||||||
int uploadedRows = countSubmittedRows(taskId);
|
|
||||||
if (totalRows <= 0 && uploadedRows > 0) {
|
|
||||||
totalRows = uploadedRows + 1;
|
|
||||||
}
|
|
||||||
return new PythonUploadProgress(uploadedRows, totalRows, "行");
|
|
||||||
}
|
|
||||||
|
|
||||||
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
|
|
||||||
if (taskId == null || taskId <= 0) {
|
|
||||||
return new PythonUploadProgress(0, 0, "分片");
|
|
||||||
}
|
|
||||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
|
||||||
.isNull(TaskScopeStateEntity::getLlmStatus)
|
|
||||||
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
|
|
||||||
if (states == null || states.isEmpty()) {
|
|
||||||
return resolvePythonChunkProgressFromChunks(taskId);
|
|
||||||
}
|
|
||||||
int current = 0;
|
|
||||||
int total = 0;
|
|
||||||
for (TaskScopeStateEntity state : states) {
|
|
||||||
if (state == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
|
|
||||||
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
|
|
||||||
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
|
|
||||||
if (stateTotal > total || total <= 0 && stateCurrent > current) {
|
|
||||||
total = stateTotal;
|
|
||||||
current = stateCurrent;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new PythonUploadProgress(current, total, "分片");
|
|
||||||
}
|
|
||||||
|
|
||||||
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
|
|
||||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
|
||||||
if (chunks == null || chunks.isEmpty()) {
|
|
||||||
return new PythonUploadProgress(0, 0, "分片");
|
|
||||||
}
|
|
||||||
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
|
|
||||||
Map<String, Integer> totalByScope = new LinkedHashMap<>();
|
|
||||||
for (TaskChunkEntity chunk : chunks) {
|
|
||||||
if (chunk == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
|
|
||||||
receivedByScope.merge(scopeHash, 1, Integer::sum);
|
|
||||||
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
|
|
||||||
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
|
|
||||||
}
|
|
||||||
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
|
|
||||||
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
int current = 0;
|
|
||||||
int total = 0;
|
|
||||||
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
|
|
||||||
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
|
|
||||||
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
|
|
||||||
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
|
|
||||||
current = scopeCurrent;
|
|
||||||
total = scopeTotal;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new PythonUploadProgress(current, total, "分片");
|
|
||||||
}
|
|
||||||
|
|
||||||
private int countSubmittedRows(Long taskId) {
|
|
||||||
if (taskId == null || taskId <= 0) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
int total = 0;
|
|
||||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
|
||||||
if (chunks == null || chunks.isEmpty()) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
for (TaskChunkEntity chunk : chunks) {
|
|
||||||
total += readChunkRows(chunk).size();
|
|
||||||
}
|
|
||||||
return total;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
|
|
||||||
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
|
|
||||||
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String resolveDisplayTaskStatus(FileTaskEntity task) {
|
private String resolveDisplayTaskStatus(FileTaskEntity task) {
|
||||||
return resolveDisplayTaskStatus(task, null);
|
return resolveDisplayTaskStatus(task, null);
|
||||||
}
|
}
|
||||||
@@ -5416,36 +5083,6 @@ public class SimilarAsinTaskService {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
|
|
||||||
int observed = Math.max(0, observedLlmStates);
|
|
||||||
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
|
|
||||||
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
|
|
||||||
return Math.max(1, observed);
|
|
||||||
}
|
|
||||||
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
|
|
||||||
return Math.max(1, Math.max(observed, expectedFromSnapshot));
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildLlmProgressMessage(int completed, int total, int pending) {
|
|
||||||
int safeTotal = Math.max(1, total);
|
|
||||||
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
|
||||||
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
|
||||||
if (safeCompleted <= 0 && submitted > 0) {
|
|
||||||
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
|
|
||||||
}
|
|
||||||
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
|
|
||||||
int safeTotal = Math.max(1, total);
|
|
||||||
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
|
||||||
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
|
||||||
if (pending > 0) {
|
|
||||||
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
|
|
||||||
}
|
|
||||||
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
|
|
||||||
}
|
|
||||||
|
|
||||||
private String fmt(LocalDateTime t) {
|
private String fmt(LocalDateTime t) {
|
||||||
return t == null ? null : t.toString();
|
return t == null ? null : t.toString();
|
||||||
}
|
}
|
||||||
|
|||||||
+245
@@ -0,0 +1,245 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 83:SimilarAsinExcelParser 行解析器。
|
||||||
|
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 SimilarAsinTaskService.parseWorkbook
|
||||||
|
* 中对应段落逐字节一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、错误值转空、
|
||||||
|
* 单字段截断 2000、表头别名匹配、空行跳过。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class SimilarAsinExcelParser {
|
||||||
|
|
||||||
|
private static final int DEFAULT_MAX_FIELD_LENGTH = 2000;
|
||||||
|
|
||||||
|
public ParsedSheet parse(File input) {
|
||||||
|
return parse(input, DEFAULT_MAX_FIELD_LENGTH);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ParsedSheet parse(File input, int maxFieldLength) {
|
||||||
|
if (input == null) {
|
||||||
|
throw new IllegalArgumentException("input must not be null");
|
||||||
|
}
|
||||||
|
try (FileInputStream fis = new FileInputStream(input);
|
||||||
|
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||||
|
return parseWorkbook(workbook, maxFieldLength);
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
throw ex;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[similar-asin] parse failed file={} err={}", input.getName(), ex.getMessage());
|
||||||
|
throw new BusinessException("解析 Excel 失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ParsedSheet parse(InputStream input) {
|
||||||
|
if (input == null) {
|
||||||
|
throw new IllegalArgumentException("input must not be null");
|
||||||
|
}
|
||||||
|
try (Workbook workbook = WorkbookFactory.create(input)) {
|
||||||
|
return parseWorkbook(workbook, DEFAULT_MAX_FIELD_LENGTH);
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
throw ex;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[similar-asin] parse failed input err={}", ex.getMessage());
|
||||||
|
throw new BusinessException("解析 Excel 失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ParsedSheet parseWorkbook(Workbook workbook, int maxFieldLength) {
|
||||||
|
DataFormatter formatter = new DataFormatter();
|
||||||
|
Sheet sheet = workbook.getSheetAt(0);
|
||||||
|
Row header = sheet.getRow(0);
|
||||||
|
if (header == null) {
|
||||||
|
throw new BusinessException("Excel 表头为空");
|
||||||
|
}
|
||||||
|
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
||||||
|
List<String> headers = readHeaders(header, formatter);
|
||||||
|
int idCol = findRequiredHeader(headerMap, "id");
|
||||||
|
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||||
|
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||||
|
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
|
||||||
|
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||||
|
int urlCol = findOptionalHeaderExact(headerMap,
|
||||||
|
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||||
|
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||||
|
int titleCol = findOptionalHeaderExact(headerMap,
|
||||||
|
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||||
|
|
||||||
|
List<SimilarAsinExcelRow> rows = new ArrayList<>();
|
||||||
|
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||||
|
Row row = sheet.getRow(i);
|
||||||
|
if (row == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String id = cell(row, idCol, formatter, maxFieldLength);
|
||||||
|
String asin = cell(row, asinCol, formatter, maxFieldLength).toUpperCase(Locale.ROOT);
|
||||||
|
String country = cell(row, countryCol, formatter, maxFieldLength);
|
||||||
|
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rows.add(new SimilarAsinExcelRow(
|
||||||
|
i + 1,
|
||||||
|
id,
|
||||||
|
asin,
|
||||||
|
country,
|
||||||
|
skuCol >= 0 ? cell(row, skuCol, formatter, maxFieldLength) : "",
|
||||||
|
priceCol >= 0 ? cell(row, priceCol, formatter, maxFieldLength) : "",
|
||||||
|
urlCol >= 0 ? cell(row, urlCol, formatter, maxFieldLength) : "",
|
||||||
|
titleCol >= 0 ? cell(row, titleCol, formatter, maxFieldLength) : "",
|
||||||
|
readRowValues(row, headers, formatter, maxFieldLength)));
|
||||||
|
}
|
||||||
|
return new ParsedSheet(headers, rows);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||||
|
Map<String, Integer> map = new LinkedHashMap<>();
|
||||||
|
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||||
|
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||||
|
if (!val.isBlank()) {
|
||||||
|
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> readHeaders(Row header, DataFormatter formatter) {
|
||||||
|
List<String> headers = new ArrayList<>();
|
||||||
|
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||||
|
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||||
|
headers.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||||
|
}
|
||||||
|
return headers;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter, int maxFieldLength) {
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
for (int i = 0; i < headers.size(); i++) {
|
||||||
|
values.put(headers.get(i), cell(row, i, formatter, maxFieldLength));
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int findRequiredHeader(Map<String, Integer> map, String... names) {
|
||||||
|
int idx = findOptionalHeader(map, names);
|
||||||
|
if (idx < 0) {
|
||||||
|
throw new BusinessException("缺少必要表头: " + String.join("/", names));
|
||||||
|
}
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int findOptionalHeader(Map<String, Integer> map, String... names) {
|
||||||
|
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||||
|
for (String name : names) {
|
||||||
|
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
|
||||||
|
return entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
|
||||||
|
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||||
|
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
|
||||||
|
for (String name : names) {
|
||||||
|
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
|
||||||
|
return entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeHeaderAlias(String value) {
|
||||||
|
String normalized = normalize(value).toLowerCase(Locale.ROOT);
|
||||||
|
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String cell(Row row, int col, DataFormatter formatter, int maxFieldLength) {
|
||||||
|
if (col < 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String value = normalize(formatter.formatCellValue(row.getCell(col)));
|
||||||
|
if (isSpreadsheetErrorValue(value)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (value.length() > maxFieldLength) {
|
||||||
|
return value.substring(0, maxFieldLength);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean isSpreadsheetErrorValue(String value) {
|
||||||
|
String normalized = value == null ? "" : value.trim();
|
||||||
|
return normalized.startsWith("#") && normalized.endsWith("!");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String val) {
|
||||||
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
public record ParsedSheet(List<String> headers, List<SimilarAsinExcelRow> rows) {
|
||||||
|
|
||||||
|
public ParsedSheet {
|
||||||
|
headers = headers == null ? List.of() : new ArrayList<>(headers);
|
||||||
|
rows = rows == null ? List.of() : new ArrayList<>(rows);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<String> headers() {
|
||||||
|
return java.util.Collections.unmodifiableList(headers);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<SimilarAsinExcelRow> rows() {
|
||||||
|
return java.util.Collections.unmodifiableList(rows);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public record SimilarAsinExcelRow(int rowIndex, String id, String asin, String country,
|
||||||
|
String sku, String price, String url, String title,
|
||||||
|
Map<String, String> values) {
|
||||||
|
|
||||||
|
public SimilarAsinExcelRow {
|
||||||
|
values = values == null ? Map.of() : new LinkedHashMap<>(values);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (!(o instanceof SimilarAsinExcelRow other)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return rowIndex == other.rowIndex
|
||||||
|
&& Objects.equals(id, other.id)
|
||||||
|
&& Objects.equals(asin, other.asin)
|
||||||
|
&& Objects.equals(country, other.country)
|
||||||
|
&& Objects.equals(sku, other.sku)
|
||||||
|
&& Objects.equals(price, other.price)
|
||||||
|
&& Objects.equals(url, other.url)
|
||||||
|
&& Objects.equals(title, other.title)
|
||||||
|
&& values.equals(other.values);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(rowIndex, id, asin, country, sku, price, url, title, values);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+78
@@ -0,0 +1,78 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 86:SimilarAsinGroupingConverter 分组转换器。
|
||||||
|
* 归一化行 → 分组 DTO;分组规则与现状 SimilarAsinTaskService.buildParsedGroups 完全一致:
|
||||||
|
* 按 groupKey(缺省兜底 displayId 的 baseId)分组、startIndex/endIndex 半开区间游标、
|
||||||
|
* 组内行数、顺序保持行首次出现顺序。纯函数无状态。
|
||||||
|
*/
|
||||||
|
public final class SimilarAsinGroupingConverter {
|
||||||
|
|
||||||
|
private SimilarAsinGroupingConverter() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<SimilarAsinParsedGroupVo> convert(List<SimilarAsinParsedRowVo> rows) {
|
||||||
|
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
int cursor = 0;
|
||||||
|
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
|
||||||
|
if (siblings == null || siblings.isEmpty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
SimilarAsinParsedRowVo first = siblings.getFirst();
|
||||||
|
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||||
|
group.setSourceFileKey(first.getSourceFileKey());
|
||||||
|
group.setSourceFilename(first.getSourceFilename());
|
||||||
|
group.setGroupKey(firstNonBlank(first.getGroupKey(),
|
||||||
|
buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
|
||||||
|
group.setBaseId(baseId(first.getDisplayId()));
|
||||||
|
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
|
||||||
|
group.setItemCount(siblings.size());
|
||||||
|
int end = cursor + siblings.size();
|
||||||
|
group.setStartIndex(cursor);
|
||||||
|
group.setEndIndex(end);
|
||||||
|
cursor = end;
|
||||||
|
groups.add(group);
|
||||||
|
}
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 行 → baseId 分组映射(LinkedHashMap 保序)。供 convert 及服务侧 loadAllRowsByBaseId 复用。 */
|
||||||
|
public static Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
|
||||||
|
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
|
||||||
|
for (SimilarAsinParsedRowVo row : rows) {
|
||||||
|
if (row == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String key = firstNonBlank(SimilarAsinRowNormalizer.normalize(row.getGroupKey()),
|
||||||
|
baseId(row.getDisplayId()));
|
||||||
|
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String baseId(String id) {
|
||||||
|
String s = SimilarAsinRowNormalizer.normalize(id);
|
||||||
|
int idx = s.indexOf('_');
|
||||||
|
return idx > 0 ? s.substring(0, idx) : s;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
|
||||||
|
return SimilarAsinRowNormalizer.normalize(sourceFileKey) + "::"
|
||||||
|
+ SimilarAsinRowNormalizer.normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String firstNonBlank(String preferred, String fallback) {
|
||||||
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
+757
@@ -0,0 +1,757 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 91:SimilarAsinHistoryAssembler 历史查询组装器。
|
||||||
|
* 历史列表 VO 拼装(toHistoryItem + 进度链:file job 状态、LLM 回流进度、Python 上传进度、
|
||||||
|
* 快照百分比)从 SimilarAsinTaskService 原样搬移;只读不落库;输出与现状逐字段一致。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class SimilarAsinHistoryAssembler {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "SIMILAR_ASIN";
|
||||||
|
private static final String STATUS_PENDING = "PENDING";
|
||||||
|
private static final String STATUS_RUNNING = "RUNNING";
|
||||||
|
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||||
|
private static final String LLM_STATUS_SUBMITTED = "SUBMITTED";
|
||||||
|
private static final String LLM_STATUS_RUNNING = "RUNNING";
|
||||||
|
private static final String LLM_STATUS_DONE = "DONE";
|
||||||
|
private static final String LLM_STATUS_FAILED = "FAILED";
|
||||||
|
|
||||||
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private final TaskChunkMapper taskChunkMapper;
|
||||||
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
private final OssStorageService ossStorageService;
|
||||||
|
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
public SimilarAsinHistoryAssembler(TaskScopeStateMapper taskScopeStateMapper,
|
||||||
|
TaskChunkMapper taskChunkMapper,
|
||||||
|
FileTaskMapper fileTaskMapper,
|
||||||
|
TaskProgressSnapshotService taskProgressSnapshotService,
|
||||||
|
OssStorageService ossStorageService,
|
||||||
|
TransientPayloadStorageService transientPayloadStorageService,
|
||||||
|
ObjectMapper objectMapper) {
|
||||||
|
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||||
|
this.taskChunkMapper = taskChunkMapper;
|
||||||
|
this.fileTaskMapper = fileTaskMapper;
|
||||||
|
this.taskProgressSnapshotService = taskProgressSnapshotService;
|
||||||
|
this.ossStorageService = ossStorageService;
|
||||||
|
this.transientPayloadStorageService = transientPayloadStorageService;
|
||||||
|
this.objectMapper = objectMapper;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按 rows 顺序组装历史项;PENDING 任务(history 语义)跳过。
|
||||||
|
* taskMap/jobMap 均为调用方已查好的映射,本方法只读不改输入。
|
||||||
|
*/
|
||||||
|
public List<SimilarAsinHistoryItemVo> buildHistoryItems(List<FileResultEntity> rows,
|
||||||
|
Map<Long, FileTaskEntity> taskMap,
|
||||||
|
Map<Long, TaskFileJobEntity> jobMap) {
|
||||||
|
List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
return items;
|
||||||
|
}
|
||||||
|
for (FileResultEntity row : rows) {
|
||||||
|
FileTaskEntity task = taskMap.get(row.getTaskId());
|
||||||
|
String taskStatus = task == null ? null : task.getStatus();
|
||||||
|
if (STATUS_PENDING.equals(taskStatus)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
items.add(toHistoryItem(row, task, jobMap.get(row.getId())));
|
||||||
|
}
|
||||||
|
return items;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||||
|
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
|
||||||
|
vo.setResultId(row.getId());
|
||||||
|
vo.setTaskId(row.getTaskId());
|
||||||
|
vo.setSourceFilename(row.getSourceFilename());
|
||||||
|
vo.setResultFilename(row.getResultFilename());
|
||||||
|
vo.setDownloadUrl(buildFreshDownloadUrl(row));
|
||||||
|
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
|
||||||
|
attachFileJobState(vo, row, job, task);
|
||||||
|
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
||||||
|
vo.setError(row.getErrorMessage());
|
||||||
|
vo.setRowCount(row.getRowCount());
|
||||||
|
vo.setCreatedAt(fmt(row.getCreatedAt()));
|
||||||
|
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||||
|
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
|
||||||
|
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildFreshDownloadUrl(FileResultEntity row) {
|
||||||
|
String resultFileUrl = row == null ? null : row.getResultFileUrl();
|
||||||
|
if (resultFileUrl == null || resultFileUrl.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
|
||||||
|
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
|
||||||
|
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[similar-asin] build public download url failed resultId={} err={}",
|
||||||
|
row.getId(), ex.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
||||||
|
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
|
||||||
|
if (job == null) {
|
||||||
|
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
attachFileProgress(vo, row, job, task);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vo.setFileJobId(job.getId());
|
||||||
|
vo.setFileStatus(job.getStatus());
|
||||||
|
vo.setFileError(job.getErrorMessage());
|
||||||
|
attachFileProgress(vo, row, job, task);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
||||||
|
if (row == null || row.getTaskId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Long taskId = row.getTaskId();
|
||||||
|
if (Boolean.TRUE.equals(vo.getFileReady())) {
|
||||||
|
vo.setFileProgressCurrent(1);
|
||||||
|
vo.setFileProgressTotal(1);
|
||||||
|
vo.setFileProgressPercent(100);
|
||||||
|
vo.setFileProgressMessage("结果文件已生成");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int llmCompleted = countCompletedLlmStates(taskId);
|
||||||
|
int llmPending = countPendingLlmStates(taskId);
|
||||||
|
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
||||||
|
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
|
||||||
|
if (!uploadComplete) {
|
||||||
|
if (canTreatTaskAsCompleted(task, row, job)) {
|
||||||
|
vo.setFileProgressCurrent(1);
|
||||||
|
vo.setFileProgressTotal(1);
|
||||||
|
vo.setFileProgressPercent(100);
|
||||||
|
vo.setFileProgressMessage("结果文件已生成");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (llmCompleted + llmPending > 0) {
|
||||||
|
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
attachPythonUploadProgress(vo, taskId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (llmCompleted + llmPending > 0) {
|
||||||
|
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (snapshot == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
||||||
|
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
||||||
|
if (total <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
|
||||||
|
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
|
||||||
|
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
|
||||||
|
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
|
||||||
|
vo.setFileProgressCurrent(current);
|
||||||
|
vo.setFileProgressTotal(total);
|
||||||
|
vo.setFileProgressPercent(percent);
|
||||||
|
vo.setFileProgressMessage(snapshot.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
|
||||||
|
TaskFileJobEntity job,
|
||||||
|
TaskProgressSnapshotEntity snapshot,
|
||||||
|
int llmCompleted,
|
||||||
|
int llmPending,
|
||||||
|
boolean uploadComplete) {
|
||||||
|
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
|
||||||
|
int total = calculateLlmDisplayTotal(snapshot, observed);
|
||||||
|
if (!uploadComplete) {
|
||||||
|
total = Math.max(total, observed + 1);
|
||||||
|
}
|
||||||
|
int current = Math.max(0, Math.min(llmCompleted, total));
|
||||||
|
vo.setFileProgressCurrent(current);
|
||||||
|
vo.setFileProgressTotal(total);
|
||||||
|
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
|
||||||
|
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
|
||||||
|
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
|
||||||
|
vo.setFileProgressMessage(uploadComplete
|
||||||
|
? (llmPending > 0
|
||||||
|
? buildLlmProgressMessage(current, total, llmPending)
|
||||||
|
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
|
||||||
|
: buildUploadingLlmProgressMessage(current, total, llmPending));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
|
||||||
|
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
|
||||||
|
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
|
||||||
|
int current = Math.max(0, Math.min(progress.current(), total));
|
||||||
|
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
|
||||||
|
vo.setFileProgressCurrent(current);
|
||||||
|
vo.setFileProgressTotal(total);
|
||||||
|
vo.setFileProgressPercent(percent);
|
||||||
|
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
|
||||||
|
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
|
||||||
|
if (chunkProgress.total() > 0) {
|
||||||
|
return chunkProgress;
|
||||||
|
}
|
||||||
|
if (chunkProgress.current() > 0) {
|
||||||
|
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
|
||||||
|
}
|
||||||
|
int totalRows = 0;
|
||||||
|
try {
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
|
totalRows = allRowCount(task);
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
totalRows = 0;
|
||||||
|
}
|
||||||
|
int uploadedRows = countSubmittedRows(taskId);
|
||||||
|
if (totalRows <= 0 && uploadedRows > 0) {
|
||||||
|
totalRows = uploadedRows + 1;
|
||||||
|
}
|
||||||
|
return new PythonUploadProgress(uploadedRows, totalRows, "行");
|
||||||
|
}
|
||||||
|
|
||||||
|
private int allRowCount(FileTaskEntity task) {
|
||||||
|
try {
|
||||||
|
return rowCount(readParsedPayload(task));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private int rowCount(SimilarAsinParsedPayloadDto payload) {
|
||||||
|
if (payload == null) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return resolveAllRows(payload).size();
|
||||||
|
}
|
||||||
|
|
||||||
|
private SimilarAsinParsedPayloadDto readParsedPayload(FileTaskEntity task) {
|
||||||
|
try {
|
||||||
|
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||||
|
String pointer = root.path("parsedPayloadRef").asText("");
|
||||||
|
if (!pointer.isBlank()) {
|
||||||
|
String json = transientPayloadStorageService.resolvePayload(pointer, "read similar ASIN parsed payload failed");
|
||||||
|
return hydrateParsedPayloadRows(objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class));
|
||||||
|
}
|
||||||
|
if (root.path("allItems").isArray()) {
|
||||||
|
return hydrateParsedPayloadRows(objectMapper.treeToValue(root, SimilarAsinParsedPayloadDto.class));
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取解析载荷失败");
|
||||||
|
}
|
||||||
|
return hydrateParsedPayloadRows(new SimilarAsinParsedPayloadDto());
|
||||||
|
}
|
||||||
|
|
||||||
|
private SimilarAsinParsedPayloadDto hydrateParsedPayloadRows(SimilarAsinParsedPayloadDto payload) {
|
||||||
|
if (payload == null) {
|
||||||
|
return new SimilarAsinParsedPayloadDto();
|
||||||
|
}
|
||||||
|
List<SimilarAsinParsedRowVo> rows = resolveAllRows(payload);
|
||||||
|
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
|
||||||
|
payload.setAllItems(rows);
|
||||||
|
}
|
||||||
|
if (payload.getItems() == null || payload.getItems().isEmpty()) {
|
||||||
|
payload.setItems(rows);
|
||||||
|
}
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从解析载荷统一恢复全量行:优先 items(新规范结构),其次 allItems(旧结构),
|
||||||
|
* 最后 groups 展开(最旧结构)。新旧结构均可恢复完整行集合,供 rowCount、
|
||||||
|
* LLM 候选加载与结果文件组装复用。
|
||||||
|
*/
|
||||||
|
private static List<SimilarAsinParsedRowVo> resolveAllRows(SimilarAsinParsedPayloadDto payload) {
|
||||||
|
if (payload == null) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
if (containsGroupRefs(payload)) {
|
||||||
|
// 新格式:分组携带索引引用,信任引用展开结果(越界/非法区间安全跳过)
|
||||||
|
return expandGroupRefs(payload);
|
||||||
|
}
|
||||||
|
List<SimilarAsinParsedRowVo> rows = payload.getItems();
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
rows = payload.getAllItems();
|
||||||
|
}
|
||||||
|
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
|
||||||
|
rows = payload.getGroups().stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
return rows == null ? List.of() : rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean containsGroupRefs(SimilarAsinParsedPayloadDto payload) {
|
||||||
|
if (payload.getGroups() == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
|
||||||
|
if (group != null && (group.getStartIndex() != null || group.getEndIndex() != null)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按分组引用 [startIndex, endIndex) 从 items 展开组内行。
|
||||||
|
* 引用越界或区间非法时安全跳过,不抛异常;展开不修改 payload 内部状态。
|
||||||
|
*/
|
||||||
|
private static List<SimilarAsinParsedRowVo> expandGroupRefs(SimilarAsinParsedPayloadDto payload) {
|
||||||
|
if (payload == null || payload.getGroups() == null || payload.getGroups().isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<SimilarAsinParsedRowVo> rows = payload.getItems();
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
rows = payload.getAllItems();
|
||||||
|
}
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>();
|
||||||
|
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
|
||||||
|
if (group == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int start = group.getStartIndex() == null ? 0 : group.getStartIndex();
|
||||||
|
int end = group.getEndIndex() == null ? 0 : group.getEndIndex();
|
||||||
|
if (start < 0 || end <= start || end > rows.size()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
expanded.addAll(rows.subList(start, end));
|
||||||
|
}
|
||||||
|
return expanded;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int countSubmittedRows(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
int total = 0;
|
||||||
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
|
if (chunks == null || chunks.isEmpty()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
|
total += readChunkRows(chunk).size();
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, SimilarAsinResultRowDto> readChunkRows(TaskChunkEntity chunk) {
|
||||||
|
Map<String, SimilarAsinResultRowDto> rows = new LinkedHashMap<>();
|
||||||
|
if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) {
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read similar ASIN chunk failed");
|
||||||
|
JsonNode array = objectMapper.readTree(payloadJson);
|
||||||
|
if (!array.isArray()) {
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
for (JsonNode node : array) {
|
||||||
|
SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class);
|
||||||
|
rows.put(rowKey(row), row);
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||||
|
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||||
|
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||||
|
boolean crossInstance = msg.contains("only exists on instance=");
|
||||||
|
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||||
|
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||||
|
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||||
|
throw new BusinessException("similar ASIN chunk payload read failed chunk="
|
||||||
|
+ chunk.getChunkIndex() + ": " + msg, ex);
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 把 chunk 读失败信息聚合到 task_scope_state.last_error,
|
||||||
|
* 形式 "chunk-read-failed[{idx}]" / "chunk-read-failed[{idx}@cross-instance]"。
|
||||||
|
* best-effort:查询 / 更新失败时仅记日志,不抛回主流程。
|
||||||
|
*/
|
||||||
|
private void recordChunkReadFailure(TaskChunkEntity chunk, boolean crossInstance, String msg) {
|
||||||
|
if (chunk == null || chunk.getTaskId() == null || chunk.getScopeHash() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, chunk.getTaskId())
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskScopeStateEntity::getScopeHash, chunk.getScopeHash())
|
||||||
|
.last("limit 1"));
|
||||||
|
if (scope == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String existing = scope.getLastError() == null ? "" : scope.getLastError();
|
||||||
|
String tag = "chunk-read-failed[" + chunk.getChunkIndex() + (crossInstance ? "@cross-instance" : "") + "]";
|
||||||
|
if (existing.contains(tag)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String updated = existing.isBlank() ? tag : existing + "; " + tag;
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, scope.getId())
|
||||||
|
.set(TaskScopeStateEntity::getLastError, updated)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
// best-effort:失败不影响主流程
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String rowKey(SimilarAsinResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String rowToken = normalize(row.getRowToken());
|
||||||
|
if (!rowToken.isBlank()) {
|
||||||
|
return rowToken;
|
||||||
|
}
|
||||||
|
return legacyRowKey(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String legacyRowKey(SimilarAsinResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return rowKey(row.getId(), row.getAsin(), row.getCountry());
|
||||||
|
}
|
||||||
|
|
||||||
|
private String rowKey(String id, String asin, String country) {
|
||||||
|
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String val) {
|
||||||
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return new PythonUploadProgress(0, 0, "分片");
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.isNull(TaskScopeStateEntity::getLlmStatus)
|
||||||
|
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return resolvePythonChunkProgressFromChunks(taskId);
|
||||||
|
}
|
||||||
|
int current = 0;
|
||||||
|
int total = 0;
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (state == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
|
||||||
|
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
|
||||||
|
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
|
||||||
|
if (stateTotal > total || total <= 0 && stateCurrent > current) {
|
||||||
|
total = stateTotal;
|
||||||
|
current = stateCurrent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new PythonUploadProgress(current, total, "分片");
|
||||||
|
}
|
||||||
|
|
||||||
|
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
|
||||||
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
|
if (chunks == null || chunks.isEmpty()) {
|
||||||
|
return new PythonUploadProgress(0, 0, "分片");
|
||||||
|
}
|
||||||
|
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
|
||||||
|
Map<String, Integer> totalByScope = new LinkedHashMap<>();
|
||||||
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
|
if (chunk == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
|
||||||
|
receivedByScope.merge(scopeHash, 1, Integer::sum);
|
||||||
|
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
|
||||||
|
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
|
||||||
|
}
|
||||||
|
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
|
||||||
|
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int current = 0;
|
||||||
|
int total = 0;
|
||||||
|
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
|
||||||
|
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
|
||||||
|
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
|
||||||
|
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
|
||||||
|
current = scopeCurrent;
|
||||||
|
total = scopeTotal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new PythonUploadProgress(current, total, "分片");
|
||||||
|
}
|
||||||
|
|
||||||
|
private int resolveReceivedChunkProgress(Long taskId, String scopeHash, Integer chunkTotal) {
|
||||||
|
int receivedCount = countChunks(taskId, scopeHash);
|
||||||
|
Integer maxChunkIndex = taskChunkMapper.selectObjs(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.select(TaskChunkEntity::getChunkIndex)
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
|
.orderByDesc(TaskChunkEntity::getChunkIndex)
|
||||||
|
.last("limit 1"))
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.map(value -> {
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return number.intValue();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return Integer.parseInt(value.toString());
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.findFirst()
|
||||||
|
.orElse(0);
|
||||||
|
int current = Math.max(receivedCount, maxChunkIndex == null ? 0 : maxChunkIndex);
|
||||||
|
if (chunkTotal != null && chunkTotal > 0) {
|
||||||
|
current = Math.min(current, chunkTotal);
|
||||||
|
}
|
||||||
|
return Math.max(0, current);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int countChunks(Long taskId, String scopeHash) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
|
||||||
|
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
|
||||||
|
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveDisplayTaskStatus(FileTaskEntity task, FileResultEntity row) {
|
||||||
|
if (task == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (canTreatTaskAsCompleted(task, row, null)) {
|
||||||
|
return STATUS_SUCCESS;
|
||||||
|
}
|
||||||
|
return task.getStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean canTreatTaskAsCompleted(FileTaskEntity task, FileResultEntity row, TaskFileJobEntity job) {
|
||||||
|
if (task == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (STATUS_SUCCESS.equals(task.getStatus())) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (row == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (job != null && "FAILED".equals(job.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
|
||||||
|
int observed = Math.max(0, observedLlmStates);
|
||||||
|
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
|
||||||
|
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
|
||||||
|
return Math.max(1, observed);
|
||||||
|
}
|
||||||
|
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
|
||||||
|
return Math.max(1, Math.max(observed, expectedFromSnapshot));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildLlmProgressMessage(int completed, int total, int pending) {
|
||||||
|
int safeTotal = Math.max(1, total);
|
||||||
|
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
||||||
|
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
||||||
|
if (safeCompleted <= 0 && submitted > 0) {
|
||||||
|
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
|
||||||
|
}
|
||||||
|
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
|
||||||
|
int safeTotal = Math.max(1, total);
|
||||||
|
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
||||||
|
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
||||||
|
if (pending > 0) {
|
||||||
|
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
|
||||||
|
}
|
||||||
|
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
|
||||||
|
}
|
||||||
|
|
||||||
|
private int calculateDisplayProgressPercent(int current,
|
||||||
|
int total,
|
||||||
|
TaskFileJobEntity job,
|
||||||
|
LocalDateTime baseTime) {
|
||||||
|
if (total <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
current = Math.max(0, Math.min(current, total));
|
||||||
|
int percent = Math.max(1, Math.min(99, (int) Math.floor(current * 100.0 / total)));
|
||||||
|
if (job != null && STATUS_RUNNING.equals(job.getStatus())) {
|
||||||
|
long elapsedSeconds = baseTime == null ? 0 : Math.max(0, Duration.between(baseTime, LocalDateTime.now()).getSeconds());
|
||||||
|
if (current <= 0) {
|
||||||
|
int firstRealProgressPercent = Math.max(1, Math.min(99, (int) Math.floor(100.0 / total)));
|
||||||
|
int waitingCap = Math.max(8, Math.min(35, firstRealProgressPercent - 1));
|
||||||
|
percent = Math.max(percent, Math.min(waitingCap, 8 + (int) (elapsedSeconds / 6)));
|
||||||
|
} else if (current < total) {
|
||||||
|
percent = Math.max(percent, Math.min(92, percent + (int) (elapsedSeconds / 10)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return percent;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int calculateSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot, TaskFileJobEntity job) {
|
||||||
|
if (snapshot == null) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
||||||
|
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
||||||
|
int percent = total <= 0 ? 0 : calculateDisplayProgressPercent(current, total, job, snapshot.getUpdatedAt());
|
||||||
|
return Math.max(percent, extractSnapshotDisplayPercent(snapshot));
|
||||||
|
}
|
||||||
|
|
||||||
|
private int extractSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot) {
|
||||||
|
if (snapshot == null || snapshot.getSnapshotJson() == null || snapshot.getSnapshotJson().isBlank()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
JsonNode displayPercent = objectMapper.readTree(snapshot.getSnapshotJson()).path("displayPercent");
|
||||||
|
if (!displayPercent.isNumber()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return Math.max(0, Math.min(100, displayPercent.asInt()));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private int countPendingLlmStates(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
private int countCompletedLlmStates(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_DONE, LLM_STATUS_FAILED)));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isResultSubmissionComplete(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.isNull(TaskScopeStateEntity::getLlmStatus)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getLastChunkAt));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
boolean hasCompletedScope = false;
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (state == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (Integer.valueOf(1).equals(state.getCompleted())) {
|
||||||
|
hasCompletedScope = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return hasCompletedScope;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String fmt(LocalDateTime t) {
|
||||||
|
return t == null ? null : t.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String firstNonBlank(String preferred, String fallback) {
|
||||||
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private record PythonUploadProgress(int current, int total, String unit) {
|
||||||
|
}
|
||||||
|
}
|
||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import java.util.Locale;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 85:SimilarAsinRowNormalizer 字段归一化器。
|
||||||
|
* 规则与 SimilarAsinTaskService.normalize 现状逐字节一致(BOM 剥离、全角空格转半角、
|
||||||
|
* trim、连续空白折叠为单空格);normalizeAsin = normalize + 大写。纯函数无状态。
|
||||||
|
*/
|
||||||
|
public final class SimilarAsinRowNormalizer {
|
||||||
|
|
||||||
|
private SimilarAsinRowNormalizer() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String normalize(String val) {
|
||||||
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String normalizeAsin(String val) {
|
||||||
|
return normalize(val).toUpperCase(Locale.ROOT);
|
||||||
|
}
|
||||||
|
}
|
||||||
+66
@@ -0,0 +1,66 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 87:SimilarAsinRowValidator 纯校验器。
|
||||||
|
* 必填检查(id/asin/country)、ASIN 格式校验(仅允许字母数字,含校验前大写化)、
|
||||||
|
* 重复检测(asin+country 键,保留首次出现)。返回错误清单不抛异常,由调用方决定处理。
|
||||||
|
* 规则与现状 parseWorkbook 必填丢弃 / dedupeRowsByRowKey 保留首次语义一致。
|
||||||
|
*/
|
||||||
|
public final class SimilarAsinRowValidator {
|
||||||
|
|
||||||
|
public record RowError(int rowIndex, String column, String message) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private SimilarAsinRowValidator() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<RowError> validate(List<SimilarAsinParsedRowVo> rows) {
|
||||||
|
List<RowError> errors = new ArrayList<>();
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
Set<String> seenKeys = new java.util.HashSet<>();
|
||||||
|
for (SimilarAsinParsedRowVo row : rows) {
|
||||||
|
if (row == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String id = SimilarAsinRowNormalizer.normalize(row.getSourceId());
|
||||||
|
String asin = SimilarAsinRowNormalizer.normalizeAsin(row.getAsin());
|
||||||
|
String country = SimilarAsinRowNormalizer.normalize(row.getCountry());
|
||||||
|
if (id.isEmpty() && asin.isEmpty() && country.isEmpty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int rowIndex = row.getRowIndex() == null ? 0 : row.getRowIndex();
|
||||||
|
if (id.isEmpty()) {
|
||||||
|
errors.add(new RowError(rowIndex, "id", "缺少必要字段: id"));
|
||||||
|
}
|
||||||
|
if (asin.isEmpty()) {
|
||||||
|
errors.add(new RowError(rowIndex, "asin", "缺少必要字段: asin"));
|
||||||
|
}
|
||||||
|
if (country.isEmpty()) {
|
||||||
|
errors.add(new RowError(rowIndex, "country", "缺少必要字段: 国家"));
|
||||||
|
}
|
||||||
|
if (asin.isEmpty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!asin.matches("[A-Z0-9]+")) {
|
||||||
|
errors.add(new RowError(rowIndex, "asin",
|
||||||
|
"ASIN 格式不正确(仅允许字母数字): " + asin));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String dupKey = asin + "::" + country;
|
||||||
|
if (!seenKeys.add(dupKey)) {
|
||||||
|
errors.add(new RowError(rowIndex, "asin",
|
||||||
|
"重复 ASIN: " + asin + " 国家: " + country));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
}
|
||||||
+112
@@ -0,0 +1,112 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||||
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
|
import org.apache.poi.ss.usermodel.CellStyle;
|
||||||
|
import org.apache.poi.ss.usermodel.Font;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 88:SimilarAsinSheetBuilder Sheet 构造器。
|
||||||
|
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式);输入数据 → 输出 workbook;不落库、无 IO 依赖。
|
||||||
|
* 表头/列序/样式与现状 assembleResultWorkbook 一致:sheet 名"相似asin检测"、15 列表头、
|
||||||
|
* 加粗表头、数据行从第 1 行、图片列 12-14 列宽、行高 409pt、值兜底与现状一致。
|
||||||
|
* 结果派生列(是否有货/相似度/是否符合类目/不符合理由/产品类目/状态)来自 LLM 结果行,
|
||||||
|
* 本 builder 输入只有解析行,这些列由调用方补写。
|
||||||
|
*/
|
||||||
|
public final class SimilarAsinSheetBuilder {
|
||||||
|
|
||||||
|
public static final List<String> RESULT_HEADERS = List.of(
|
||||||
|
"id",
|
||||||
|
"asin",
|
||||||
|
"国家",
|
||||||
|
"价格",
|
||||||
|
"卖家名称",
|
||||||
|
"品牌",
|
||||||
|
"是否有货",
|
||||||
|
"相似度",
|
||||||
|
"是否符合类目",
|
||||||
|
"不符合理由",
|
||||||
|
"产品类目",
|
||||||
|
"状态",
|
||||||
|
"主图",
|
||||||
|
"阿里巴巴图片1",
|
||||||
|
"阿里巴巴图片2"
|
||||||
|
);
|
||||||
|
|
||||||
|
public static final int IMG_COL_MAIN = 12;
|
||||||
|
public static final int IMG_COL_PUZZLE1 = 13;
|
||||||
|
public static final int IMG_COL_PUZZLE2 = 14;
|
||||||
|
|
||||||
|
private SimilarAsinSheetBuilder() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void build(Workbook workbook, List<SimilarAsinParsedRowVo> rows) {
|
||||||
|
Sheet sheet = workbook.createSheet("相似asin检测");
|
||||||
|
CellStyle headerStyle = workbook.createCellStyle();
|
||||||
|
Font font = workbook.createFont();
|
||||||
|
font.setBold(true);
|
||||||
|
headerStyle.setFont(font);
|
||||||
|
|
||||||
|
sheet.setColumnWidth(IMG_COL_MAIN, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
|
||||||
|
sheet.setColumnWidth(IMG_COL_PUZZLE1, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
|
||||||
|
sheet.setColumnWidth(IMG_COL_PUZZLE2, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
|
||||||
|
|
||||||
|
Row header = sheet.createRow(0);
|
||||||
|
for (int i = 0; i < RESULT_HEADERS.size(); i++) {
|
||||||
|
Cell cell = header.createCell(i);
|
||||||
|
cell.setCellValue(RESULT_HEADERS.get(i));
|
||||||
|
cell.setCellStyle(headerStyle);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rows == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int rowIndex = 1;
|
||||||
|
for (SimilarAsinParsedRowVo parsedRow : rows) {
|
||||||
|
if (parsedRow == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Row row = sheet.createRow(rowIndex);
|
||||||
|
int col = 0;
|
||||||
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
|
||||||
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
|
||||||
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
|
||||||
|
row.createCell(col++).setCellValue(firstNonBlank(
|
||||||
|
parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")));
|
||||||
|
row.createCell(col++).setCellValue(readValueByHeader(
|
||||||
|
parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name",
|
||||||
|
"seller-name", "sellername", "store name", "shop name"));
|
||||||
|
row.createCell(col).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
|
||||||
|
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
|
||||||
|
rowIndex++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String readValueByHeader(SimilarAsinParsedRowVo row, String... candidates) {
|
||||||
|
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
|
||||||
|
String header = SimilarAsinRowNormalizer.normalize(entry.getKey()).toLowerCase(Locale.ROOT);
|
||||||
|
for (String candidate : candidates) {
|
||||||
|
String expected = SimilarAsinRowNormalizer.normalize(candidate).toLowerCase(Locale.ROOT);
|
||||||
|
if (!expected.isBlank() && header.contains(expected)) {
|
||||||
|
return entry.getValue() == null ? "" : entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String firstNonBlank(String preferred, String fallback) {
|
||||||
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
+27
@@ -0,0 +1,27 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.dto;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import jakarta.validation.constraints.NotNull;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 共享轻量进度查询请求(各模块 progress/light 端点共用)。
|
||||||
|
* 最多处理 200 个任务 ID,超长部分截断丢弃(轮询端点保持响应快)。
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Schema(description = "任务轻量进度查询请求")
|
||||||
|
public class TaskProgressLightRequest {
|
||||||
|
@NotNull
|
||||||
|
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端批量查询,最多 200 个。",
|
||||||
|
example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
private List<Long> taskIds = new ArrayList<>();
|
||||||
|
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务,owner-scoped 模块使用)。", example = "1")
|
||||||
|
private Long userId;
|
||||||
|
}
|
||||||
+19
@@ -0,0 +1,19 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 共享轻量进度批量响应:items 按请求顺序返回,查不到的任务进 missingTaskIds。
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@Schema(description = "任务轻量进度批量响应")
|
||||||
|
public class TaskProgressLightBatchVo {
|
||||||
|
@Schema(description = "按请求顺序返回的轻量进度项。")
|
||||||
|
private List<TaskProgressLightVo> items = new ArrayList<>();
|
||||||
|
@Schema(description = "查询不到的任务 ID 列表。")
|
||||||
|
private List<Long> missingTaskIds = new ArrayList<>();
|
||||||
|
}
|
||||||
+28
@@ -0,0 +1,28 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 共享轻量进度项(11 模块 progress/light 轮询专用)。
|
||||||
|
* 白名单键与 similarasin 的 SimilarAsinTaskLightVo 保持一致:
|
||||||
|
* taskId/status/statusCode?/fileStatus?/fileError?/fileReady?/updatedAt。
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@Schema(description = "任务轻量进度项(前端轮询专用,不含明细/result 内容)")
|
||||||
|
public class TaskProgressLightVo {
|
||||||
|
@Schema(description = "任务 ID。", example = "3938")
|
||||||
|
private Long taskId;
|
||||||
|
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
|
||||||
|
private String status;
|
||||||
|
@Schema(description = "业务状态码(预留字段,当前恒为 null)。", example = "null")
|
||||||
|
private String statusCode;
|
||||||
|
@Schema(description = "结果文件组装 Job 状态:SUCCESS/RUNNING/FAILED;无 Job 时为空。", example = "SUCCESS")
|
||||||
|
private String fileStatus;
|
||||||
|
@Schema(description = "文件组装失败时的错误信息;无错误时为空。", example = "组装失败: 文件写入超时")
|
||||||
|
private String fileError;
|
||||||
|
@Schema(description = "结果文件是否已就绪。", example = "true")
|
||||||
|
private Boolean fileReady;
|
||||||
|
@Schema(description = "最后更新时间,ISO 本地时间字符串。", example = "2026-04-26T10:05:00")
|
||||||
|
private String updatedAt;
|
||||||
|
}
|
||||||
+22
@@ -531,6 +531,28 @@ public class TaskFileJobService {
|
|||||||
return jobMap;
|
return jobMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 按任务 ID 集合一次 IN 查询 ASSEMBLE_RESULT Job,返回 taskId→Job 的 Map(轻量进度接口用)。 */
|
||||||
|
public Map<Long, TaskFileJobEntity> findAssembleJobsByTaskIds(String moduleType, List<Long> taskIds) {
|
||||||
|
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||||
|
.filter(id -> id != null && id > 0)
|
||||||
|
.distinct()
|
||||||
|
.toList();
|
||||||
|
if (normalizedIds.isEmpty() || moduleType == null || moduleType.isBlank()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||||
|
.in(TaskFileJobEntity::getTaskId, normalizedIds));
|
||||||
|
Map<Long, TaskFileJobEntity> jobMap = new LinkedHashMap<>();
|
||||||
|
for (TaskFileJobEntity job : jobs) {
|
||||||
|
if (job != null && job.getTaskId() != null) {
|
||||||
|
jobMap.putIfAbsent(job.getTaskId(), job);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return jobMap;
|
||||||
|
}
|
||||||
|
|
||||||
public long countUnfinishedAssembleJobs(Long taskId, String moduleType) {
|
public long countUnfinishedAssembleJobs(Long taskId, String moduleType) {
|
||||||
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||||
return 0L;
|
return 0L;
|
||||||
|
|||||||
+76
@@ -0,0 +1,76 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightVo;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 共享轻量进度装配器:11 模块 progress/light 统一实现。
|
||||||
|
* 语义与 similarasin 的 progressLight 一致——只查任务行 + 一次 ASSEMBLE Job IN 批量查询;
|
||||||
|
* 空/重复/非正数过滤;超过 MAX_TASK_IDS 截断丢弃;缺失任务进 missingTaskIds;
|
||||||
|
* fileStatus/fileError/fileReady 由 Job 推导;状态值语义与各模块 progress/batch 一致。
|
||||||
|
* userId 非空时按用户过滤(publish 等 owner-scoped 模块使用)。
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskProgressLightAssembler {
|
||||||
|
|
||||||
|
public static final int MAX_TASK_IDS = 200;
|
||||||
|
|
||||||
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo assemble(String moduleType, Long userId, List<Long> taskIds) {
|
||||||
|
TaskProgressLightBatchVo vo = new TaskProgressLightBatchVo();
|
||||||
|
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||||
|
.filter(id -> id != null && id > 0)
|
||||||
|
.distinct()
|
||||||
|
.limit(MAX_TASK_IDS)
|
||||||
|
.toList();
|
||||||
|
if (normalizedIds.isEmpty()) {
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
LambdaQueryWrapper<FileTaskEntity> query = new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.select(FileTaskEntity::getId,
|
||||||
|
FileTaskEntity::getStatus,
|
||||||
|
FileTaskEntity::getUpdatedAt)
|
||||||
|
.in(FileTaskEntity::getId, normalizedIds);
|
||||||
|
if (userId != null) {
|
||||||
|
query.eq(FileTaskEntity::getUserId, userId);
|
||||||
|
}
|
||||||
|
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||||
|
for (FileTaskEntity task : fileTaskMapper.selectList(query)) {
|
||||||
|
if (task != null && moduleType.equals(task.getModuleType())) {
|
||||||
|
taskMap.put(task.getId(), task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByTaskIds(moduleType, new ArrayList<>(taskMap.keySet()));
|
||||||
|
for (Long taskId : normalizedIds) {
|
||||||
|
FileTaskEntity task = taskMap.get(taskId);
|
||||||
|
if (task == null) {
|
||||||
|
vo.getMissingTaskIds().add(taskId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
TaskFileJobEntity job = jobMap.get(taskId);
|
||||||
|
TaskProgressLightVo item = new TaskProgressLightVo();
|
||||||
|
item.setTaskId(task.getId());
|
||||||
|
item.setStatus(task.getStatus());
|
||||||
|
item.setFileReady(job != null && job.getResultFileUrl() != null && !job.getResultFileUrl().isBlank());
|
||||||
|
item.setFileStatus(job == null ? null : job.getStatus());
|
||||||
|
item.setFileError(job == null ? null : job.getErrorMessage());
|
||||||
|
item.setUpdatedAt(task.getUpdatedAt() == null ? null : task.getUpdatedAt().toString());
|
||||||
|
vo.getItems().add(item);
|
||||||
|
}
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
}
|
||||||
+10
@@ -37,6 +37,8 @@ import java.io.InputStream;
|
|||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -107,6 +109,14 @@ public class WithdrawTaskController {
|
|||||||
return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(request.getTaskIds()));
|
return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/light")
|
||||||
|
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||||
|
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||||
|
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||||
|
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||||
|
return ApiResponse.success(withdrawTaskService.progressLight(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/tasks")
|
@PostMapping("/tasks")
|
||||||
@Operation(summary = "创建取款任务", description = "为一批已匹配店铺创建一个取款任务,写入任务记录和店铺结果占位记录,并返回 taskId 及初始店铺快照。")
|
@Operation(summary = "创建取款任务", description = "为一批已匹配店铺创建一个取款任务,写入任务记录和店铺结果占位记录,并返回 taskId 及初始店铺快照。")
|
||||||
public ApiResponse<WithdrawCreateTaskVo> createTask(@Valid @RequestBody WithdrawCreateTaskRequest request) {
|
public ApiResponse<WithdrawCreateTaskVo> createTask(@Valid @RequestBody WithdrawCreateTaskRequest request) {
|
||||||
|
|||||||
+7
@@ -42,6 +42,8 @@ import java.util.LinkedHashMap;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -49,6 +51,10 @@ import java.util.Objects;
|
|||||||
public class WithdrawTaskService {
|
public class WithdrawTaskService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "WITHDRAW";
|
private static final String MODULE_TYPE = "WITHDRAW";
|
||||||
|
|
||||||
|
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||||
|
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||||
|
}
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final int RESULT_PENDING = -1;
|
private static final int RESULT_PENDING = -1;
|
||||||
private static final int RESULT_FAILED = 0;
|
private static final int RESULT_FAILED = 0;
|
||||||
@@ -68,6 +74,7 @@ public class WithdrawTaskService {
|
|||||||
private final TaskResultItemService taskResultItemService;
|
private final TaskResultItemService taskResultItemService;
|
||||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
public ProductRiskDashboardVo dashboard(Long userId) {
|
public ProductRiskDashboardVo dashboard(Long userId) {
|
||||||
validateUserId(userId);
|
validateUserId(userId);
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
-- Task 116(依赖 115 审计):biz_task_file_job 增加 (module_type, job_type, task_id) 复合索引。
|
||||||
|
-- 背景:findAssembleJobsByTaskIds(progress/light)以 module_type + job_type 等值 + task_id IN 查询,
|
||||||
|
-- 现有 idx_file_job_task (task_id, module_type) 最左前缀不匹配,导致 index merge/大范围扫描。
|
||||||
|
-- 风险:ALTER TABLE ADD INDEX 需申请元数据锁(MySQL 8 在线 DDL,INSTANT/INPLACE),
|
||||||
|
-- 建议在低峰窗口(凌晨 02:00-06:00)执行;表若极大,评估 pt-online-schema-change。
|
||||||
|
-- 回滚:DROP INDEX idx_file_job_module_type_task ON biz_task_file_job(见文件末尾注释块)。
|
||||||
|
|
||||||
|
SET @db_name = DATABASE();
|
||||||
|
|
||||||
|
SET @idx_exists := (
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM information_schema.STATISTICS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name
|
||||||
|
AND TABLE_NAME = 'biz_task_file_job'
|
||||||
|
AND INDEX_NAME = 'idx_file_job_module_type_task'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@idx_exists = 0,
|
||||||
|
'ALTER TABLE biz_task_file_job ADD INDEX idx_file_job_module_type_task (module_type, job_type, task_id)',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
-- 验证 SQL:应返回 1 行(索引已存在)
|
||||||
|
-- SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||||
|
-- WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'biz_task_file_job'
|
||||||
|
-- AND INDEX_NAME = 'idx_file_job_module_type_task';
|
||||||
|
|
||||||
|
-- 回滚(低峰执行):
|
||||||
|
-- ALTER TABLE biz_task_file_job DROP INDEX idx_file_job_module_type_task;
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
-- 分组管理:独立管理后台菜单项,放入「账号与权限」菜单组(sort_order 25)
|
||||||
|
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
|
||||||
|
SELECT '分组管理', 'admin_group_manage', 'admin', 'group-manage', 25
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM columns WHERE column_key = 'admin_group_manage');
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 115:高频查询 EXPLAIN 索引审计(文档产出验证)。
|
||||||
|
* 审计文档必须覆盖 history/progress/dashboard 三类高频查询的执行计划结论、
|
||||||
|
* 候选索引清单(表/列/类型/收益/风险)、全表扫描标记、可重复执行步骤,
|
||||||
|
* 且本任务明确不做 DDL(只审计,不加索引)。
|
||||||
|
*/
|
||||||
|
class ExplainIndexAuditDocTest {
|
||||||
|
|
||||||
|
private static final Path AUDIT = Paths.get("src", "main", "resources", "..", "..", "..",
|
||||||
|
"docs", "explain-index-audit.md").normalize();
|
||||||
|
private static final Path FLYWAY_SPEC = Paths.get("src", "main", "resources", "..", "..", "..",
|
||||||
|
"docs", "specs", "12-flyway-and-inspection.md").normalize();
|
||||||
|
|
||||||
|
private static String read(Path path) throws IOException {
|
||||||
|
return new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<String> linesOf(Path path) throws IOException {
|
||||||
|
return List.of(read(path).split("\r?\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String lineContaining(String content, String needle) {
|
||||||
|
for (String line : content.split("\r?\n")) {
|
||||||
|
if (line.contains(needle)) {
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_explain_history_plan() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("history"), "审计文档覆盖 history 查询");
|
||||||
|
assertTrue(doc.contains("EXPLAIN"), "审计文档包含 EXPLAIN 执行计划记录");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_explain_progress_plan() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("progress"), "审计文档覆盖 progress 查询");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_explain_dashboard_plan() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("dashboard"), "审计文档覆盖 dashboard 查询");
|
||||||
|
assertTrue(doc.contains("GROUP BY"), "dashboard 聚合查询计划已记录");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_full_scan_detected() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("type=ALL") || doc.contains("全表扫描") || doc.contains("ALL")
|
||||||
|
|| doc.contains("possible_keys") || doc.contains("扫描"),
|
||||||
|
"审计文档记录执行计划扫描类型(full scan 标记)");
|
||||||
|
String line = lineContaining(doc, "idx_file_job_task");
|
||||||
|
assertTrue(line != null && line.contains("task_id"),
|
||||||
|
"审计文档指出 task_id 前缀索引对 (module_type, job_type, task_id IN) 的局限");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_index_candidates_listed() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("idx_file_job_module_type_task")
|
||||||
|
|| doc.contains("module_type") && doc.contains("job_type") && doc.contains("task_id"),
|
||||||
|
"候选索引清单含 (module_type, job_type, task_id) 组合");
|
||||||
|
assertTrue(doc.contains("收益") && doc.contains("风险"),
|
||||||
|
"候选索引清单含收益与风险说明");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_plan_documented() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("biz_task_file_job"), "审计文档引用目标表");
|
||||||
|
assertTrue(doc.contains("biz_file_task") || doc.contains("biz_file_result"),
|
||||||
|
"审计文档引用业务任务/结果表");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_plan_repeatable() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("EXPLAIN SELECT") || doc.contains("EXPLAIN")
|
||||||
|
&& doc.contains("手工执行") || doc.contains("执行"),
|
||||||
|
"审计文档包含可重复执行的 EXPLAIN SQL 步骤");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_no_ddl_this_task() throws Exception {
|
||||||
|
String audit = read(AUDIT);
|
||||||
|
String spec = read(FLYWAY_SPEC);
|
||||||
|
assertTrue(audit.contains("不做 DDL") || audit.contains("本任务无 DDL")
|
||||||
|
|| audit.contains("不新增索引") || audit.contains("只审计"),
|
||||||
|
"审计文档明确本任务不做 DDL");
|
||||||
|
assertFalse(audit.contains("ALTER TABLE"),
|
||||||
|
"审计文档不包含 ALTER TABLE(DDL 属于后续任务 116)");
|
||||||
|
assertTrue(spec.contains("只读") || spec.contains("巡检"),
|
||||||
|
"12 spec 巡检报表模式与审计文档对应");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_audit_reflects_batch_loading_queries() throws Exception {
|
||||||
|
String doc = read(AUDIT);
|
||||||
|
assertTrue(doc.contains("findAssembleJobsByTaskIds"), "审计覆盖轻量进度 Job 批量查询");
|
||||||
|
assertTrue(doc.contains("findAssembleJobsByResultIds"), "审计覆盖结果列表 Job 批量查询");
|
||||||
|
assertTrue(doc.contains("selectMaps") || doc.contains("GROUP BY"), "审计覆盖 dashboard 聚合查询");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 116:索引迁移 V{N+1}(审计清单落地)。
|
||||||
|
* 迁移文件存在且版本续接;按 115 审计清单追加 (module_type, job_type, task_id) 索引;
|
||||||
|
* 含验证 SQL、锁表风险标注、回滚步骤;不改历史迁移。
|
||||||
|
*/
|
||||||
|
class IndexMigrationV101Test {
|
||||||
|
|
||||||
|
private static final Path DB_DIR = Paths.get("src", "main", "resources", "db");
|
||||||
|
private static final Path MIGRATION = DB_DIR.resolve("V101__task_file_job_module_type_task_index.sql");
|
||||||
|
private static final Path AUDIT = Paths.get("src", "main", "resources", "..", "..", "..",
|
||||||
|
"docs", "explain-index-audit.md").normalize();
|
||||||
|
|
||||||
|
private static String read(Path path) throws IOException {
|
||||||
|
return new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<Path> allMigrations() throws IOException {
|
||||||
|
try (Stream<Path> stream = Files.list(DB_DIR)) {
|
||||||
|
return stream.filter(p -> p.getFileName().toString().matches("V\\d+__.*\\.sql"))
|
||||||
|
.sorted()
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int versionOf(Path path) {
|
||||||
|
String name = path.getFileName().toString();
|
||||||
|
return Integer.parseInt(name.substring(1, name.indexOf('_')));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_file_exists() {
|
||||||
|
assertTrue(Files.isRegularFile(MIGRATION), "迁移文件必须存在: " + MIGRATION);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_version_sequential() throws Exception {
|
||||||
|
List<Path> all = allMigrations();
|
||||||
|
assertTrue(!all.isEmpty(), "存在迁移文件");
|
||||||
|
int maxVersion = all.stream().mapToInt(IndexMigrationV101Test::versionOf).max().orElse(0);
|
||||||
|
assertTrue(maxVersion >= 100, "现有迁移最大版本 >= V100: " + maxVersion);
|
||||||
|
assertTrue(versionOf(MIGRATION) == 101, "新迁移版本号 V101 续接 V100 之后");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_index_created() throws Exception {
|
||||||
|
String sql = read(MIGRATION);
|
||||||
|
assertTrue(sql.contains("biz_task_file_job"), "迁移作用于 biz_task_file_job");
|
||||||
|
assertTrue(sql.contains("idx_file_job_module_type_task"), "索引名与审计清单一致");
|
||||||
|
assertTrue(sql.contains("module_type") && sql.contains("job_type") && sql.contains("task_id"),
|
||||||
|
"索引列为 (module_type, job_type, task_id)");
|
||||||
|
assertTrue(sql.contains("ADD INDEX"), "使用 ADD INDEX");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_conditional_repeatable() throws Exception {
|
||||||
|
String sql = read(MIGRATION);
|
||||||
|
assertTrue(sql.contains("information_schema.STATISTICS"), "通过 information_schema 判存在");
|
||||||
|
assertTrue(sql.contains("INDEX_NAME"), "按索引名判断");
|
||||||
|
assertTrue(sql.contains("PREPARE") && sql.contains("EXECUTE"), "条件执行(幂等,可重复运行)");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_validated() throws Exception {
|
||||||
|
String sql = read(MIGRATION);
|
||||||
|
assertTrue(sql.contains("SELECT 1"), "验证 SQL(幂等分支)");
|
||||||
|
String audit = read(AUDIT);
|
||||||
|
assertTrue(audit.contains("idx_file_job_module_type_task"), "审计清单已列出该索引");
|
||||||
|
assertTrue(audit.contains("收益") && audit.contains("风险"), "审计清单含收益/风险");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_explain_improved() throws Exception {
|
||||||
|
String audit = read(AUDIT);
|
||||||
|
assertTrue(audit.contains("findAssembleJobsByTaskIds"), "审计覆盖 progress/light Job 查询");
|
||||||
|
assertTrue(audit.contains("执行后重跑") || audit.contains("前后计划")
|
||||||
|
|| audit.contains("对比"), "审计含迁移前后 EXPLAIN 对比步骤");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_no_alter_legacy() throws Exception {
|
||||||
|
List<Path> all = allMigrations();
|
||||||
|
List<Path> others = all.stream()
|
||||||
|
.filter(p -> versionOf(p) != 101)
|
||||||
|
.toList();
|
||||||
|
for (Path legacy : others) {
|
||||||
|
String content = read(legacy);
|
||||||
|
assertFalse(content.contains("idx_file_job_module_type_task"),
|
||||||
|
"历史迁移不得包含新索引: " + legacy.getFileName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_rollback_doc() throws Exception {
|
||||||
|
String sql = read(MIGRATION);
|
||||||
|
assertTrue(sql.contains("DROP INDEX") || sql.contains("drop index"),
|
||||||
|
"迁移文件含回滚语句(DROP INDEX idx_file_job_module_type_task)");
|
||||||
|
String audit = read(AUDIT);
|
||||||
|
assertTrue(audit.contains("回滚") || audit.contains("DROP"),
|
||||||
|
"审计文档含回滚步骤说明");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_lock_window() throws Exception {
|
||||||
|
String sql = read(MIGRATION);
|
||||||
|
assertTrue(sql.contains("LOCK") || sql.contains("低峰") || sql.contains("lock")
|
||||||
|
|| sql.contains("窗口"), "迁移标注锁表风险/窗口");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_migration_backward_safe() throws Exception {
|
||||||
|
String sql = read(MIGRATION);
|
||||||
|
assertTrue(sql.contains("ALTER TABLE"), "普通 BTREE 索引(不重建表、不删列)");
|
||||||
|
assertFalse(sql.contains("DROP COLUMN"), "不删列");
|
||||||
|
assertFalse(sql.contains("RENAME"), "不重命名");
|
||||||
|
assertTrue(sql.contains("ADD INDEX"), "仅追加索引,向后兼容");
|
||||||
|
}
|
||||||
|
}
|
||||||
+449
@@ -0,0 +1,449 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentSheetBuilder;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentRowNormalizer;
|
||||||
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.Mockito.doAnswer;
|
||||||
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 95:AppearancePatent 门面改委托。
|
||||||
|
* writeResultWorkbook/writeReasonSheet 改为委托 AppearancePatentSheetBuilder.buildResultSheet
|
||||||
|
* (门面保留 SXSSF 写出、异常包装与 rowToken 优先的 findResultRow 语义);
|
||||||
|
* submitResult 双事务路径与 owner 检查不动;Sheet 链专属私有方法从门面移除。
|
||||||
|
* 私有门面方法经反射直接验证,不经过完整 job 链路。
|
||||||
|
*/
|
||||||
|
class AppearancePatentTaskServiceDelegationTest {
|
||||||
|
|
||||||
|
private FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
private FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||||
|
private TaskScopeStateMapper taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||||
|
private TaskChunkMapper taskChunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
private LocalFileStorageService localFileStorageService = mock(LocalFileStorageService.class);
|
||||||
|
private AppearancePatentLlmClient llmClient = mock(AppearancePatentLlmClient.class);
|
||||||
|
private AppearancePatentTaskCacheService taskCacheService = mock(AppearancePatentTaskCacheService.class); private AppearancePatentProperties properties = mock(AppearancePatentProperties.class);
|
||||||
|
private StorageProperties storageProperties = mock(StorageProperties.class);
|
||||||
|
private TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||||
|
private TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
|
||||||
|
private TransientPayloadStorageService transientPayloadStorageService = mock(TransientPayloadStorageService.class);
|
||||||
|
private PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||||
|
private DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class);
|
||||||
|
private TaskDistributedLockService taskDistributedLockService = mock(TaskDistributedLockService.class);
|
||||||
|
private InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
|
||||||
|
|
||||||
|
private AppearancePatentTaskService service;
|
||||||
|
private final AtomicInteger txCount = new AtomicInteger();
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() throws Exception {
|
||||||
|
lenient().when(properties.getLlmBatchSize()).thenReturn(50);
|
||||||
|
// TransactionTemplate.execute 需要 getTransaction 返回非 null status;commit/rollback 默认为 no-op
|
||||||
|
lenient().when(transactionManager.getTransaction(any()))
|
||||||
|
.thenReturn(mock(org.springframework.transaction.TransactionStatus.class));
|
||||||
|
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||||
|
service = newService();
|
||||||
|
}
|
||||||
|
|
||||||
|
private AppearancePatentTaskService newService() {
|
||||||
|
return new AppearancePatentTaskService(
|
||||||
|
localFileStorageService, null, storageProperties, fileTaskMapper, fileResultMapper,
|
||||||
|
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||||
|
properties, taskFileJobService, taskProgressSnapshotService,
|
||||||
|
transientPayloadStorageService, transactionManager, distributedJobLockService,
|
||||||
|
taskDistributedLockService, instanceMetadata,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
private AppearancePatentTaskService serviceWithoutTransactionManager() {
|
||||||
|
return new AppearancePatentTaskService(
|
||||||
|
localFileStorageService, null, storageProperties, fileTaskMapper, fileResultMapper,
|
||||||
|
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||||
|
properties, taskFileJobService, taskProgressSnapshotService,
|
||||||
|
transientPayloadStorageService, null, distributedJobLockService,
|
||||||
|
taskDistributedLockService, instanceMetadata,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 1 签名不变 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_result_workbook_signature_unchanged() throws Exception {
|
||||||
|
Method method = AppearancePatentTaskService.class.getDeclaredMethod(
|
||||||
|
"writeResultWorkbook",
|
||||||
|
File.class,
|
||||||
|
AppearancePatentParsedPayloadDto.class,
|
||||||
|
List.class,
|
||||||
|
Map.class);
|
||||||
|
assertEquals(void.class, method.getReturnType(), "返回类型不变");
|
||||||
|
assertEquals(4, method.getParameterCount(), "参数个数不变");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 2 委托各组件 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_result_workbook_delegates_sheet_builder() throws Exception {
|
||||||
|
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
|
||||||
|
AppearancePatentResultRowDto resultRow = resultRow("2_1", "B01A", "US", "f1.xlsx::row::2");
|
||||||
|
File out = new File("target/appearance-patent-delegation-tmp", "delegate.xlsx");
|
||||||
|
if (out.exists()) {
|
||||||
|
out.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
invokeWriteResultWorkbook(out, row, resultRow);
|
||||||
|
|
||||||
|
assertTrue(out.exists(), "委托后结果文件已生成");
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
|
||||||
|
Sheet main = wb.getSheet("外观专利检测结果");
|
||||||
|
assertNotNull(main, "主 sheet 由 SheetBuilder 产出");
|
||||||
|
Row data = main.getRow(1);
|
||||||
|
assertEquals("2_1", data.getCell(0).getStringCellValue());
|
||||||
|
assertEquals("B01A", data.getCell(1).getStringCellValue());
|
||||||
|
assertEquals("US", data.getCell(2).getStringCellValue());
|
||||||
|
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌经委托解析");
|
||||||
|
assertEquals("19.90", data.getCell(5).getStringCellValue(), "价格经委托解析");
|
||||||
|
assertEquals("无风险", data.getCell(9).getStringCellValue(), "标题维度经委托");
|
||||||
|
assertEquals("无风险", data.getCell(10).getStringCellValue(), "外观维度经委托");
|
||||||
|
assertEquals("已侵权", data.getCell(11).getStringCellValue());
|
||||||
|
assertEquals("成功", data.getCell(12).getStringCellValue(), "状态经委托 resolveResultStatus");
|
||||||
|
assertNotNull(wb.getSheet("原因"), "原因 sheet 由 SheetBuilder 产出");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_row_token_matching_preserved_in_delegated_builder() throws Exception {
|
||||||
|
// 关键行为:resultMap 键为 rowToken(service rowKey(row) 优先 token),
|
||||||
|
// 委托查找函数必须保持 rowToken 优先匹配(SheetBuilder 默认 legacy-only,门面必须传入自己的 findResultRow)
|
||||||
|
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
|
||||||
|
AppearancePatentResultRowDto resultRow = resultRow("2_1", "B01A", "US", "f1.xlsx::row::2");
|
||||||
|
// 注意:displayId/asin/country 与 resultRow 相同,若用 legacy key 也能匹配;
|
||||||
|
// 构造一个 legacy 不同但 rowToken 相同的用例验证 token 优先
|
||||||
|
row.setDisplayId("x_other");
|
||||||
|
row.setSourceId("x_other");
|
||||||
|
row.setAsin("B999");
|
||||||
|
row.setCountry("FR");
|
||||||
|
File out = new File("target/appearance-patent-delegation-tmp", "token.xlsx");
|
||||||
|
if (out.exists()) {
|
||||||
|
out.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
invokeWriteResultWorkbook(out, row, resultRow);
|
||||||
|
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
|
||||||
|
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||||
|
assertEquals("已侵权", data.getCell(11).getStringCellValue(),
|
||||||
|
"rowToken 命中 resultRow(legacy 键不匹配仍能解析结论)");
|
||||||
|
assertEquals("成功", data.getCell(12).getStringCellValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_delegation_without_result_rows_uses_parsed_fallback() throws Exception {
|
||||||
|
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
|
||||||
|
File out = new File("target/appearance-patent-delegation-tmp", "nofallback.xlsx");
|
||||||
|
if (out.exists()) {
|
||||||
|
out.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
invokeWriteResultWorkbook(out, row, null);
|
||||||
|
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
|
||||||
|
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||||
|
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
|
||||||
|
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
|
||||||
|
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌仍从解析行 values 回退");
|
||||||
|
assertEquals("", data.getCell(5).getStringCellValue(), "无结果行价格留空(resolvePrice 仅读结果行,与现状一致)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 3 结果一致(多行 + 原因 sheet 去重) ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_multiple_rows_and_reason_dedup() throws Exception {
|
||||||
|
AppearancePatentParsedRowVo p1 = parsedRow("2_1", "B01A", "US", "f1.xlsx");
|
||||||
|
AppearancePatentParsedRowVo p2 = parsedRow("2_2", "B01A", "DE", "f1.xlsx");
|
||||||
|
p2.setRowToken("f1.xlsx::row::3");
|
||||||
|
p2.setRowIndex(3);
|
||||||
|
AppearancePatentResultRowDto r1 = resultRow("2_1", "B01A", "US", "f1.xlsx::row::2");
|
||||||
|
r1.setAppearanceReason("外观理由");
|
||||||
|
r1.setPatentReason("专利理由");
|
||||||
|
r1.setTitleReason("标题理由");
|
||||||
|
File out = new File("target/appearance-patent-delegation-tmp", "multi.xlsx");
|
||||||
|
if (out.exists()) {
|
||||||
|
out.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
invokeWriteResultWorkbook(out, List.of(p1, p2), Map.of(tokenKey(r1), r1));
|
||||||
|
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
|
||||||
|
Sheet main = wb.getSheet("外观专利检测结果");
|
||||||
|
assertEquals(2, main.getLastRowNum(), "两行数据");
|
||||||
|
Sheet reason = wb.getSheet("原因");
|
||||||
|
assertEquals("B01A", reason.getRow(1).getCell(0).getStringCellValue());
|
||||||
|
assertEquals("外观理由", reason.getRow(1).getCell(1).getStringCellValue());
|
||||||
|
assertEquals("专利理由", reason.getRow(1).getCell(2).getStringCellValue());
|
||||||
|
assertEquals("标题理由", reason.getRow(1).getCell(3).getStringCellValue());
|
||||||
|
assertNull(reason.getRow(2), "同 ASIN 去重");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 4 异常一致 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_exception_wrapped_with_original_message() throws Exception {
|
||||||
|
// 写出目标为已存在目录 → FileOutputStream 抛异常 → 门面包成"生成外观专利检测结果失败"
|
||||||
|
File badDir = new File("target/appearance-patent-delegation-tmp", "bad-dir");
|
||||||
|
if (!badDir.exists()) {
|
||||||
|
badDir.mkdirs();
|
||||||
|
}
|
||||||
|
AppearancePatentParsedRowVo row = parsedRow("2_1", "B01A", "US", "f1.xlsx");
|
||||||
|
|
||||||
|
Throwable thrown = assertThrows(Throwable.class,
|
||||||
|
() -> invokeWriteResultWorkbook(badDir, row, null));
|
||||||
|
Throwable cause = thrown instanceof java.lang.reflect.InvocationTargetException
|
||||||
|
? ((java.lang.reflect.InvocationTargetException) thrown).getCause()
|
||||||
|
: thrown;
|
||||||
|
assertTrue(cause instanceof BusinessException, "实际异常类型: " + cause.getClass());
|
||||||
|
assertEquals("生成外观专利检测结果失败", cause.getMessage(), "异常包装一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 5 双事务路径不动 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_submit_result_dual_transaction_kept() throws Exception {
|
||||||
|
// transactionManager 非空 → submitResultLocked 走 REQUIRES_NEW 双事务(persist + complete)
|
||||||
|
FileTaskEntity t = runningTask();
|
||||||
|
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
|
||||||
|
when(taskChunkMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(taskScopeStateMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
|
||||||
|
when(transientPayloadStorageService.storeChunkPayload(anyString(), any(), any(), any(), anyString()))
|
||||||
|
.thenReturn("transient:stored");
|
||||||
|
// 提交完成后的调度查询链
|
||||||
|
when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
|
service.submitResult(90001L, submitRequest("submission-x"));
|
||||||
|
|
||||||
|
verify(taskChunkMapper, Mockito.atLeast(1)).insert(any(TaskChunkEntity.class));
|
||||||
|
verify(transactionManager, Mockito.atLeast(2)).getTransaction(any());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_submit_result_keeps_task_status_check() throws Exception {
|
||||||
|
// 非 RUNNING 任务 → 双事务路径内 persist 仍抛"任务不是运行中状态"
|
||||||
|
FileTaskEntity t = task("{\"allItems\":[]}");
|
||||||
|
t.setStatus("SUCCESS");
|
||||||
|
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
|
||||||
|
|
||||||
|
BusinessException ex = assertThrows(BusinessException.class,
|
||||||
|
() -> service.submitResult(90001L, submitRequest("submission-status")));
|
||||||
|
assertTrue(ex.getMessage().contains("任务不是运行中状态"), "实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 6 owner 检查不动 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_owner_check_rejects_foreign_owner() throws Exception {
|
||||||
|
service = serviceWithoutTransactionManager();
|
||||||
|
FileTaskEntity t = task("{\"allItems\":[],\"ownerInstanceId\":\"server-121\"}");
|
||||||
|
t.setStatus("RUNNING");
|
||||||
|
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
|
||||||
|
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
|
||||||
|
|
||||||
|
assertThrows(com.nanri.aiimage.common.exception.TaskOwnerMismatchException.class,
|
||||||
|
() -> service.submitResult(90001L, submitRequest("submission-owner")), "owner 检查仍在手动路径生效");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_owner_check_passes_for_matching_owner() throws Exception {
|
||||||
|
service = serviceWithoutTransactionManager();
|
||||||
|
FileTaskEntity t = task("{\"allItems\":[],\"ownerInstanceId\":\"server-110\"}");
|
||||||
|
t.setStatus("RUNNING");
|
||||||
|
when(fileTaskMapper.selectById(90001L)).thenReturn(t);
|
||||||
|
when(instanceMetadata.getInstanceId()).thenReturn("server-110");
|
||||||
|
when(taskChunkMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(taskScopeStateMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
when(transientPayloadStorageService.storeChunkPayload(anyString(), any(), any(), any(), anyString()))
|
||||||
|
.thenReturn("transient:stored");
|
||||||
|
|
||||||
|
service.submitResult(90001L, submitRequest("submission-owner-ok"));
|
||||||
|
|
||||||
|
verify(taskChunkMapper, Mockito.atLeast(1)).insert(any(TaskChunkEntity.class));
|
||||||
|
verify(fileTaskMapper, Mockito.atLeast(1)).updateById(any(FileTaskEntity.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 7 行值解析语义由 SheetBuilder 承接 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_resolve_helpers_semantics_kept_in_sheet_builder() {
|
||||||
|
AppearancePatentParsedRowVo parsedRow = new AppearancePatentParsedRowVo();
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
values.put("品牌", "Source Brand");
|
||||||
|
parsedRow.setValues(values);
|
||||||
|
|
||||||
|
AppearancePatentResultRowDto resultRow = new AppearancePatentResultRowDto();
|
||||||
|
resultRow.setBrand("Python Brand");
|
||||||
|
assertEquals("Python Brand", AppearancePatentSheetBuilder.resolveBrand(resultRow, parsedRow));
|
||||||
|
resultRow.setBrand(" ");
|
||||||
|
assertEquals("Source Brand", AppearancePatentSheetBuilder.resolveBrand(resultRow, parsedRow));
|
||||||
|
assertEquals("", AppearancePatentSheetBuilder.resolvePrice(null));
|
||||||
|
assertEquals("失败", AppearancePatentSheetBuilder.resolveResultStatus(" "));
|
||||||
|
assertEquals("成功", AppearancePatentSheetBuilder.resolveResultStatus("已侵权"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_llm_failure_semantics_kept_in_sheet_builder() {
|
||||||
|
assertTrue(AppearancePatentSheetBuilder.isTechnicalLlmFailure("coze 工作流节点执行超限"));
|
||||||
|
assertFalse(AppearancePatentSheetBuilder.isTechnicalLlmFailure("无风险"));
|
||||||
|
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||||
|
row.setError("coze 调用超时");
|
||||||
|
row.setTitleRisk("coze 调用超时");
|
||||||
|
row.setConclusion("");
|
||||||
|
assertEquals("coze 调用超时", AppearancePatentSheetBuilder.userFacingLlmCellValue(row, row.getTitleRisk()));
|
||||||
|
assertEquals("成功", AppearancePatentSheetBuilder.userFacingStatus(row));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalizer_semantics_unchanged() {
|
||||||
|
assertEquals("", AppearancePatentRowNormalizer.normalize(null));
|
||||||
|
assertEquals("b01x", AppearancePatentRowNormalizer.normalize("b01x "));
|
||||||
|
assertEquals("A", AppearancePatentRowNormalizer.firstNonBlank(null, "A"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 私有门面反射调用 ----------
|
||||||
|
|
||||||
|
private void invokeWriteResultWorkbook(File out, AppearancePatentParsedRowVo row,
|
||||||
|
AppearancePatentResultRowDto resultRow) throws Exception {
|
||||||
|
invokeWriteResultWorkbook(out, row == null ? List.of() : List.of(row),
|
||||||
|
resultRow == null ? Map.of() : Map.of(tokenKey(resultRow), resultRow));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeWriteResultWorkbook(File out, List<AppearancePatentParsedRowVo> rows,
|
||||||
|
Map<String, AppearancePatentResultRowDto> resultMap) throws Exception {
|
||||||
|
File parent = out.getParentFile();
|
||||||
|
if (parent != null && !parent.exists()) {
|
||||||
|
parent.mkdirs();
|
||||||
|
}
|
||||||
|
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
|
||||||
|
payload.setAllItems(rows);
|
||||||
|
Method method = AppearancePatentTaskService.class.getDeclaredMethod(
|
||||||
|
"writeResultWorkbook", File.class, AppearancePatentParsedPayloadDto.class, List.class, Map.class);
|
||||||
|
method.setAccessible(true);
|
||||||
|
method.invoke(service, out, payload, rows, resultMap);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String tokenKey(AppearancePatentResultRowDto row) {
|
||||||
|
return AppearancePatentRowNormalizer.normalize(row.getRowToken());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity runningTask() {
|
||||||
|
FileTaskEntity t = task("{\"allItems\":[]}");
|
||||||
|
t.setStatus("RUNNING");
|
||||||
|
return t;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity task(String resultJson) {
|
||||||
|
FileTaskEntity t = new FileTaskEntity();
|
||||||
|
t.setId(90001L);
|
||||||
|
t.setModuleType("APPEARANCE_PATENT");
|
||||||
|
t.setResultJson(resultJson);
|
||||||
|
t.setUserId(7L);
|
||||||
|
t.setUpdatedAt(java.time.LocalDateTime.now());
|
||||||
|
return t;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AppearancePatentSubmitResultRequest submitRequest(String submissionId) {
|
||||||
|
AppearancePatentSubmitResultRequest request = new AppearancePatentSubmitResultRequest();
|
||||||
|
request.setChunkIndex(0);
|
||||||
|
request.setChunkTotal(1);
|
||||||
|
request.setSubmissionId(submissionId);
|
||||||
|
request.setItems(List.of());
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AppearancePatentParsedRowVo parsedRow(String id, String asin, String country, String fileKey) {
|
||||||
|
AppearancePatentParsedRowVo row = new AppearancePatentParsedRowVo();
|
||||||
|
row.setDisplayId(id);
|
||||||
|
row.setSourceId(id);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setCountry(country);
|
||||||
|
row.setSourceFileKey(fileKey);
|
||||||
|
row.setRowToken(fileKey + "::row::" + 2);
|
||||||
|
row.setRowIndex(2);
|
||||||
|
row.setSourceFilename("delegation-source.xlsx");
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
values.put("卖家名称", "seller-A");
|
||||||
|
values.put("品牌", "brand-A");
|
||||||
|
values.put("价格", "19.90");
|
||||||
|
row.setValues(values);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AppearancePatentResultRowDto resultRow(String id, String asin, String country, String rowToken) {
|
||||||
|
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||||
|
row.setId(id);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setCountry(country);
|
||||||
|
row.setRowToken(rowToken);
|
||||||
|
row.setTitleRisk("无风险");
|
||||||
|
row.setAppearanceRisk("无风险");
|
||||||
|
row.setConclusion("已侵权");
|
||||||
|
row.setStatus("成功");
|
||||||
|
row.setBrand("brand-A");
|
||||||
|
row.setPrice("19.90");
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
}
|
||||||
+370
@@ -0,0 +1,370 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyList;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.doAnswer;
|
||||||
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 106:appearancepatent 历史列表批量加载(同 105 模式)。
|
||||||
|
* history() 的任务/结果/Job 关联数据走 IN 批量查询 + 按 ID Map 装配,
|
||||||
|
* 不允许逐条 N+1:结果 1 次、任务 1 次、Job 1 次(恒定 3 次查询,与结果行数无关);
|
||||||
|
* 排序(priority → activityTime → createdAt → id)、分页、过滤、输出与现状一致。
|
||||||
|
*/
|
||||||
|
class AppearancePatentTaskServiceHistoryBatchTest {
|
||||||
|
|
||||||
|
private static final String MODULE = "APPEARANCE_PATENT";
|
||||||
|
|
||||||
|
private final List<FileResultEntity> resultDb = new ArrayList<>();
|
||||||
|
private final List<FileTaskEntity> taskDb = new ArrayList<>();
|
||||||
|
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
|
||||||
|
|
||||||
|
private final AtomicInteger resultSelectCount = new AtomicInteger();
|
||||||
|
private final AtomicInteger taskSelectCount = new AtomicInteger();
|
||||||
|
private final AtomicInteger jobSelectCount = new AtomicInteger();
|
||||||
|
private final List<Long> lastResultTaskIds = new ArrayList<>();
|
||||||
|
|
||||||
|
private FileTaskMapper fileTaskMapper;
|
||||||
|
private FileResultMapper fileResultMapper;
|
||||||
|
private TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private TaskChunkMapper taskChunkMapper;
|
||||||
|
private LocalFileStorageService localFileStorageService;
|
||||||
|
private AppearancePatentLlmClient llmClient;
|
||||||
|
private AppearancePatentTaskCacheService taskCacheService;
|
||||||
|
private AppearancePatentProperties properties;
|
||||||
|
private StorageProperties storageProperties;
|
||||||
|
private TaskFileJobService taskFileJobService;
|
||||||
|
private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
private TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
private PlatformTransactionManager transactionManager;
|
||||||
|
private DistributedJobLockService distributedJobLockService;
|
||||||
|
private TaskDistributedLockService taskDistributedLockService;
|
||||||
|
private InstanceMetadata instanceMetadata;
|
||||||
|
|
||||||
|
private AppearancePatentTaskService service;
|
||||||
|
|
||||||
|
/** 从 wrapper SQL 片段解析所有 #{ew.paramNameValuePairs.<key>} 引用的参数值。 */
|
||||||
|
private static List<Object> paramValuesOf(LambdaQueryWrapper<?> q, String segment) {
|
||||||
|
List<Object> values = new ArrayList<>();
|
||||||
|
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||||
|
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
|
||||||
|
Map<String, Object> params = q.getParamNameValuePairs();
|
||||||
|
while (m.find()) {
|
||||||
|
Object value = params.get(m.group(1));
|
||||||
|
if (value != null) {
|
||||||
|
values.add(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() throws Exception {
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileResultEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskScopeStateEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskFileJobEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), TaskChunkEntity.class);
|
||||||
|
|
||||||
|
fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
fileResultMapper = mock(FileResultMapper.class);
|
||||||
|
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||||
|
taskChunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
localFileStorageService = mock(LocalFileStorageService.class);
|
||||||
|
llmClient = mock(AppearancePatentLlmClient.class);
|
||||||
|
taskCacheService = mock(AppearancePatentTaskCacheService.class);
|
||||||
|
properties = mock(AppearancePatentProperties.class);
|
||||||
|
storageProperties = mock(StorageProperties.class);
|
||||||
|
taskFileJobService = mock(TaskFileJobService.class);
|
||||||
|
taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
|
||||||
|
transientPayloadStorageService = mock(TransientPayloadStorageService.class);
|
||||||
|
transactionManager = mock(PlatformTransactionManager.class);
|
||||||
|
distributedJobLockService = mock(DistributedJobLockService.class);
|
||||||
|
taskDistributedLockService = mock(TaskDistributedLockService.class);
|
||||||
|
instanceMetadata = mock(InstanceMetadata.class);
|
||||||
|
|
||||||
|
// 结果表:按 wrapper 的 user/module/limit 过滤并保持 createdAt 倒序
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
resultSelectCount.incrementAndGet();
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
|
||||||
|
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||||
|
List<Object> params = paramValuesOf(q, segment);
|
||||||
|
List<FileResultEntity> filtered = new ArrayList<>(resultDb);
|
||||||
|
Long userId = params.stream().filter(Long.class::isInstance).map(Long.class::cast).findFirst().orElse(null);
|
||||||
|
if (userId != null) {
|
||||||
|
filtered.removeIf(r -> !userId.equals(r.getUserId()));
|
||||||
|
}
|
||||||
|
if (segment.contains("module_type")) {
|
||||||
|
filtered.removeIf(r -> !MODULE.equals(r.getModuleType()));
|
||||||
|
}
|
||||||
|
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
|
||||||
|
lastResultTaskIds.clear();
|
||||||
|
lastResultTaskIds.addAll(filtered.stream().map(FileResultEntity::getTaskId).toList());
|
||||||
|
if (segment.contains("limit")) {
|
||||||
|
int cap = 50;
|
||||||
|
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||||
|
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
|
||||||
|
if (m.find()) {
|
||||||
|
cap = Integer.parseInt(m.group(1));
|
||||||
|
}
|
||||||
|
return filtered.subList(0, Math.min(cap, filtered.size()));
|
||||||
|
}
|
||||||
|
return filtered;
|
||||||
|
}).when(fileResultMapper).selectList(any());
|
||||||
|
|
||||||
|
// 任务表:IN 查询返回 id 命中集合
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
taskSelectCount.incrementAndGet();
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
|
||||||
|
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||||
|
List<Object> params = paramValuesOf(q, segment);
|
||||||
|
List<Long> wanted = new ArrayList<>();
|
||||||
|
for (Object value : params) {
|
||||||
|
if (value instanceof List<?> list) {
|
||||||
|
for (Object item : list) {
|
||||||
|
if (item instanceof Long id) {
|
||||||
|
wanted.add(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (wanted.isEmpty()) {
|
||||||
|
wanted.addAll(lastResultTaskIds);
|
||||||
|
}
|
||||||
|
if (wanted.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
return taskDb.stream().filter(t -> wanted.contains(t.getId())).toList();
|
||||||
|
}).when(fileTaskMapper).selectList(any());
|
||||||
|
|
||||||
|
lenient().when(taskFileJobService.findAssembleJobsByResultIds(eq(MODULE), anyList())).thenAnswer(invocation -> {
|
||||||
|
jobSelectCount.incrementAndGet();
|
||||||
|
List<Long> resultIds = invocation.getArgument(1);
|
||||||
|
Map<Long, TaskFileJobEntity> map = new HashMap<>();
|
||||||
|
if (resultIds != null) {
|
||||||
|
for (TaskFileJobEntity job : jobDb) {
|
||||||
|
if (job.getResultId() != null && resultIds.contains(job.getResultId())) {
|
||||||
|
map.put(job.getResultId(), job);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
});
|
||||||
|
|
||||||
|
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
lenient().when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
lenient().when(taskProgressSnapshotService.find(any(), any())).thenReturn(null);
|
||||||
|
|
||||||
|
service = new AppearancePatentTaskService(
|
||||||
|
localFileStorageService, null, storageProperties, fileTaskMapper, fileResultMapper,
|
||||||
|
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||||
|
properties, taskFileJobService, taskProgressSnapshotService,
|
||||||
|
transientPayloadStorageService, transactionManager, distributedJobLockService,
|
||||||
|
taskDistributedLockService, instanceMetadata,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileResultEntity result(Long id, Long taskId, Long userId, LocalDateTime createdAt) {
|
||||||
|
FileResultEntity row = new FileResultEntity();
|
||||||
|
row.setId(id);
|
||||||
|
row.setTaskId(taskId);
|
||||||
|
row.setUserId(userId);
|
||||||
|
row.setModuleType(MODULE);
|
||||||
|
row.setSourceFilename("s" + id + ".xlsx");
|
||||||
|
row.setResultFilename("s" + id + "-result.xlsx");
|
||||||
|
row.setResultFileUrl("result/appearance-patent/" + id + "/out.xlsx");
|
||||||
|
row.setSuccess(1);
|
||||||
|
row.setRowCount(3);
|
||||||
|
row.setCreatedAt(createdAt);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(id);
|
||||||
|
task.setModuleType(MODULE);
|
||||||
|
task.setStatus(status);
|
||||||
|
task.setCreatedAt(createdAt);
|
||||||
|
task.setUpdatedAt(createdAt);
|
||||||
|
task.setFinishedAt(createdAt.plusMinutes(5));
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskFileJobEntity job(Long id, Long resultId, String status) {
|
||||||
|
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||||
|
job.setId(id);
|
||||||
|
job.setResultId(resultId);
|
||||||
|
job.setModuleType(MODULE);
|
||||||
|
job.setStatus(status);
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void seed(int rows, int tasks, int jobs) {
|
||||||
|
for (int i = 1; i <= tasks; i++) {
|
||||||
|
taskDb.add(task(100L + i, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, i)));
|
||||||
|
}
|
||||||
|
for (int i = 1; i <= rows; i++) {
|
||||||
|
resultDb.add(result(200L + i, 100L + (i % tasks) + 1, 1L, LocalDateTime.of(2026, 8, 1, 10, i)));
|
||||||
|
}
|
||||||
|
for (int i = 1; i <= jobs; i++) {
|
||||||
|
jobDb.add(job(300L + i, 200L + i, "SUCCESS"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyBatchLoads() throws Exception {
|
||||||
|
seed(5, 3, 3);
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 50);
|
||||||
|
assertEquals(5, vo.getItems().size(), "5 条结果全部装配");
|
||||||
|
for (AppearancePatentHistoryItemVo item : vo.getItems()) {
|
||||||
|
assertNotNull(item.getTaskStatus(), "task 状态从批量任务 Map 装配");
|
||||||
|
assertEquals("SUCCESS", item.getFileStatus(), "job 状态从批量 Job Map 装配");
|
||||||
|
assertTrue(item.getFileReady(), "结果文件就绪");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historySingleRow() throws Exception {
|
||||||
|
seed(1, 1, 1);
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 50);
|
||||||
|
assertEquals(1, vo.getItems().size());
|
||||||
|
assertEquals(201L, vo.getItems().getFirst().getResultId());
|
||||||
|
assertEquals("SUCCESS", vo.getItems().getFirst().getTaskStatus());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyMultipleTasks() throws Exception {
|
||||||
|
seed(4, 4, 4);
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 50);
|
||||||
|
assertEquals(4, vo.getItems().size());
|
||||||
|
long distinctTasks = vo.getItems().stream().map(AppearancePatentHistoryItemVo::getTaskId).distinct().count();
|
||||||
|
assertEquals(4, distinctTasks, "多个任务分别按 ID Map 装配");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyNoJobs() throws Exception {
|
||||||
|
seed(3, 3, 0);
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 50);
|
||||||
|
assertEquals(3, vo.getItems().size());
|
||||||
|
for (AppearancePatentHistoryItemVo item : vo.getItems()) {
|
||||||
|
assertNull(item.getFileJobId(), "无 Job 时不附加 jobId");
|
||||||
|
assertEquals("SUCCESS", item.getFileStatus(), "文件就绪无 Job 状态为 SUCCESS");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyOrderUnchanged() throws Exception {
|
||||||
|
seed(3, 1, 0);
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 50);
|
||||||
|
assertEquals(List.of(203L, 202L, 201L),
|
||||||
|
vo.getItems().stream().map(AppearancePatentHistoryItemVo::getResultId).toList(),
|
||||||
|
"createdAt 倒序保持");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyPagination() throws Exception {
|
||||||
|
seed(12, 1, 0);
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 5);
|
||||||
|
assertEquals(5, vo.getItems().size(), "limit 5 只取前 5");
|
||||||
|
assertEquals(List.of(212L, 211L, 210L, 209L, 208L),
|
||||||
|
vo.getItems().stream().map(AppearancePatentHistoryItemVo::getResultId).toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyMissingRelated() throws Exception {
|
||||||
|
resultDb.add(result(201L, 9999L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
|
||||||
|
resultDb.add(result(202L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||||
|
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 50);
|
||||||
|
assertEquals(2, vo.getItems().size());
|
||||||
|
assertNull(vo.getItems().getFirst().getTaskStatus(), "关联任务缺失时状态为 null");
|
||||||
|
assertEquals("SUCCESS", vo.getItems().get(1).getTaskStatus(), "有关联任务的正常装配");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyActiveRunningPrioritized() throws Exception {
|
||||||
|
// 结果 201 关联 RUNNING 任务(活跃优先),202 关联 SUCCESS 任务(已完成)
|
||||||
|
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||||
|
resultDb.add(result(202L, 1002L, 1L, LocalDateTime.of(2026, 8, 1, 10, 1)));
|
||||||
|
taskDb.add(task(1001L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||||
|
taskDb.add(task(1002L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1)));
|
||||||
|
AppearancePatentHistoryVo vo = service.history(1L, 50);
|
||||||
|
assertEquals(2, vo.getItems().size());
|
||||||
|
assertEquals(201L, vo.getItems().getFirst().getResultId(), "RUNNING 任务记录排前");
|
||||||
|
assertEquals(202L, vo.getItems().get(1).getResultId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historyUserFiltered() throws Exception {
|
||||||
|
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
|
||||||
|
resultDb.add(result(202L, 1002L, 2L, LocalDateTime.of(2026, 8, 1, 10, 1)));
|
||||||
|
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||||
|
taskDb.add(task(1002L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1)));
|
||||||
|
AppearancePatentHistoryVo vo = service.history(2L, 50);
|
||||||
|
assertEquals(1, vo.getItems().size(), "只返回当前用户结果");
|
||||||
|
assertEquals(202L, vo.getItems().getFirst().getResultId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void historySqlCountConstant() throws Exception {
|
||||||
|
seed(20, 4, 20);
|
||||||
|
service.history(1L, 50);
|
||||||
|
verify(fileResultMapper, times(1)).selectList(any());
|
||||||
|
verify(fileTaskMapper, times(1)).selectList(any());
|
||||||
|
verify(taskFileJobService, times(1)).findAssembleJobsByResultIds(eq(MODULE), anyList());
|
||||||
|
assertEquals(1, resultSelectCount.get(), "结果 1 次 IN 查询");
|
||||||
|
assertEquals(1, taskSelectCount.get(), "任务 1 次 IN 查询");
|
||||||
|
assertEquals(1, jobSelectCount.get(), "Job 1 次 IN 查询");
|
||||||
|
assertEquals(20, resultSelectCount.get() * 20, "查询次数与结果行数无关(无 N+1)");
|
||||||
|
}
|
||||||
|
}
|
||||||
-41
@@ -1,11 +1,7 @@
|
|||||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
class AppearancePatentTaskServiceTest {
|
class AppearancePatentTaskServiceTest {
|
||||||
@@ -16,41 +12,4 @@ class AppearancePatentTaskServiceTest {
|
|||||||
assertEquals("SUCCESS", AppearancePatentTaskService.resolveTaskExecutionStatus(false, false));
|
assertEquals("SUCCESS", AppearancePatentTaskService.resolveTaskExecutionStatus(false, false));
|
||||||
assertEquals("FAILED", AppearancePatentTaskService.resolveTaskExecutionStatus(false, true));
|
assertEquals("FAILED", AppearancePatentTaskService.resolveTaskExecutionStatus(false, true));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
void resultStatusFailsOnlyWhenConclusionIsEmpty() {
|
|
||||||
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(null));
|
|
||||||
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(" "));
|
|
||||||
assertEquals("\u6210\u529f", AppearancePatentTaskService.resolveResultStatus("\u4fb5\u6743"));
|
|
||||||
assertEquals("\u6210\u529f", AppearancePatentTaskService.resolveResultStatus("\u65e0\u4fb5\u6743"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void resultBrandPrefersPythonThenFallsBackToSourceFile() {
|
|
||||||
AppearancePatentParsedRowVo parsedRow = new AppearancePatentParsedRowVo();
|
|
||||||
parsedRow.setValues(new LinkedHashMap<>());
|
|
||||||
parsedRow.getValues().put("品牌", "Source Brand");
|
|
||||||
|
|
||||||
AppearancePatentResultRowDto resultRow = new AppearancePatentResultRowDto();
|
|
||||||
resultRow.setBrand("Python Brand");
|
|
||||||
assertEquals("Python Brand", AppearancePatentTaskService.resolveBrand(resultRow, parsedRow));
|
|
||||||
|
|
||||||
resultRow.setBrand(" ");
|
|
||||||
assertEquals("Source Brand", AppearancePatentTaskService.resolveBrand(resultRow, parsedRow));
|
|
||||||
|
|
||||||
parsedRow.getValues().clear();
|
|
||||||
assertEquals("", AppearancePatentTaskService.resolveBrand(resultRow, parsedRow));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void resultPriceUsesOnlyPythonSubmittedValue() {
|
|
||||||
assertEquals("", AppearancePatentTaskService.resolvePrice(null));
|
|
||||||
|
|
||||||
AppearancePatentResultRowDto resultRow = new AppearancePatentResultRowDto();
|
|
||||||
resultRow.setPrice(" 12.99 ");
|
|
||||||
assertEquals("12.99", AppearancePatentTaskService.resolvePrice(resultRow));
|
|
||||||
|
|
||||||
resultRow.setPrice(null);
|
|
||||||
assertEquals("", AppearancePatentTaskService.resolvePrice(resultRow));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
+167
@@ -0,0 +1,167 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 93:AppearancePatentExcelParser 行解析器。
|
||||||
|
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 AppearancePatentTaskService.parseWorkbook
|
||||||
|
* 对应段落一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、表头别名匹配、空行跳过、
|
||||||
|
* 必填表头缺失抛错。注意:appearancepatent 无 2000 截断、无错误值转空(与 similarasin 不同)。
|
||||||
|
*/
|
||||||
|
class AppearancePatentExcelParserTest {
|
||||||
|
|
||||||
|
private static final String[] REQUIRED = {"id", "asin", "国家"};
|
||||||
|
|
||||||
|
private static File workbook(String[] headers, List<String[]> rows) throws Exception {
|
||||||
|
File file = File.createTempFile("appearance-patent-parse-", ".xlsx");
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream out = new FileOutputStream(file)) {
|
||||||
|
Sheet sheet = wb.createSheet("Sheet1");
|
||||||
|
Row header = sheet.createRow(0);
|
||||||
|
for (int i = 0; i < headers.length; i++) {
|
||||||
|
header.createCell(i).setCellValue(headers[i]);
|
||||||
|
}
|
||||||
|
for (int r = 0; r < rows.size(); r++) {
|
||||||
|
Row row = sheet.createRow(r + 1);
|
||||||
|
for (int c = 0; c < rows.get(r).length; c++) {
|
||||||
|
row.createCell(c).setCellValue(rows.get(r)[c]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wb.write(out);
|
||||||
|
}
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final String[] FULL_HEADERS = {"id", "asin", "国家", "价格", "seller sku", "图片链接", "标题"};
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_normal_rows() throws Exception {
|
||||||
|
File file = workbook(FULL_HEADERS, List.of(
|
||||||
|
new String[]{"2_1", "b01a", "US", "19.90", "SKU-1", "http://img/a.jpg", "title a"},
|
||||||
|
new String[]{"2_2", "b01b", "DE", "29.90", "SKU-2", "http://img/b.jpg", "title b"}));
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
|
||||||
|
assertEquals(7, parsed.headers().size(), "表头 7 列");
|
||||||
|
assertEquals(2, parsed.rows().size());
|
||||||
|
AppearancePatentExcelParser.AppearanceExcelRow first = parsed.rows().get(0);
|
||||||
|
assertEquals(2, first.rowIndex(), "行号从 2 开始(表头占 1)");
|
||||||
|
assertEquals("2_1", first.id());
|
||||||
|
assertEquals("B01A", first.asin(), "asin 归一化后大写");
|
||||||
|
assertEquals("US", first.country());
|
||||||
|
assertEquals("19.90", first.price());
|
||||||
|
assertEquals("SKU-1", first.sku());
|
||||||
|
assertEquals("http://img/a.jpg", first.url());
|
||||||
|
assertEquals("title a", first.title());
|
||||||
|
assertEquals("19.90", first.values().get("价格"), "values 按表头键取值");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_blank_row_skipped() throws Exception {
|
||||||
|
File file = workbook(FULL_HEADERS, List.of(
|
||||||
|
new String[]{"", "", ""},
|
||||||
|
new String[]{"1", "B01X", "US"},
|
||||||
|
new String[]{"", "", ""}));
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
|
||||||
|
assertEquals(1, parsed.rows().size(), "全空行跳过");
|
||||||
|
assertEquals("1", parsed.rows().get(0).id());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_empty_sheet() throws Exception {
|
||||||
|
File file = workbook(REQUIRED, List.of());
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
|
||||||
|
assertTrue(parsed.rows().isEmpty(), "无数据行返回空");
|
||||||
|
assertEquals(3, parsed.headers().size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_header_aliases() throws Exception {
|
||||||
|
// 国家用 country、sku 用 merchant_sku、url 用 主图链接 均可识别
|
||||||
|
File file = workbook(new String[]{"id", "asin", "country", "merchant_sku", "主图链接"},
|
||||||
|
List.<String[]>of(new String[]{"1", "B01X", "FR", "MS-1", "http://img/x.jpg"}));
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
|
||||||
|
assertEquals(1, parsed.rows().size());
|
||||||
|
AppearancePatentExcelParser.AppearanceExcelRow row = parsed.rows().get(0);
|
||||||
|
assertEquals("FR", row.country());
|
||||||
|
assertEquals("MS-1", row.sku());
|
||||||
|
assertEquals("http://img/x.jpg", row.url());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_missing_required_header_throws() throws Exception {
|
||||||
|
File file = workbook(new String[]{"id", "asin"}, List.<String[]>of(new String[]{"1", "B01X"}));
|
||||||
|
|
||||||
|
BusinessException ex = assertThrows(BusinessException.class,
|
||||||
|
() -> new AppearancePatentExcelParser().parse(file));
|
||||||
|
assertTrue(ex.getMessage().contains("缺少必要表头"), "必填表头缺失抛错,实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_empty_header_row_throws() throws Exception {
|
||||||
|
File file = workbook(new String[]{}, List.of());
|
||||||
|
|
||||||
|
BusinessException ex = assertThrows(BusinessException.class,
|
||||||
|
() -> new AppearancePatentExcelParser().parse(file));
|
||||||
|
assertTrue(ex.getMessage().contains("表头为空") || ex.getMessage().contains("缺少必要表头"),
|
||||||
|
"表头缺失抛错,实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_cell_normalization() throws Exception {
|
||||||
|
File file = workbook(FULL_HEADERS, List.<String[]>of(
|
||||||
|
new String[]{"2", "b01x", " US ", " 19.90 ", " SKU 1 ", " http://img/x.jpg ", " t a "}));
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.AppearanceExcelRow row = parsed.rows().get(0);
|
||||||
|
assertEquals("2", row.id(), "BOM 剥离");
|
||||||
|
assertEquals("B01X", row.asin(), "大写 + trim");
|
||||||
|
assertEquals("US", row.country(), "trim");
|
||||||
|
assertEquals("SKU 1", row.sku(), "全角空格转半角 + 折叠");
|
||||||
|
assertEquals("http://img/x.jpg", row.url(), "url trim");
|
||||||
|
assertEquals("t a", row.title(), "连续空白折叠");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_no_2000_truncation() throws Exception {
|
||||||
|
// appearancepatent 无字段截断(与 similarasin 不同):超长字段原样保留
|
||||||
|
String longSku = "S".repeat(5000);
|
||||||
|
File file = workbook(FULL_HEADERS, List.<String[]>of(new String[]{"1", "B01X", "US", "", longSku, "", ""}));
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
|
||||||
|
assertEquals(longSku, parsed.rows().get(0).sku(), "超长字段不截断");
|
||||||
|
assertEquals(5000, parsed.rows().get(0).sku().length());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_parser_header_fallback_names() throws Exception {
|
||||||
|
// 空表头列名回退为 "列N";重复列取首列
|
||||||
|
File file = workbook(new String[]{"id", "", "asin", "asin", "国家"}, List.<String[]>of(new String[]{"1", "x", "B01A", "B01B", "US"}));
|
||||||
|
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
|
||||||
|
assertEquals("列2", parsed.headers().get(1), "空表头回退列N");
|
||||||
|
assertEquals("asin", parsed.headers().get(2));
|
||||||
|
assertEquals("asin", parsed.headers().get(3), "重复表头保留");
|
||||||
|
assertEquals("B01A", parsed.rows().get(0).asin(), "重复列取首个匹配列");
|
||||||
|
}
|
||||||
|
}
|
||||||
+221
@@ -0,0 +1,221 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 97:AppearancePatentHistoryAssembler 历史查询组装器。
|
||||||
|
* 历史列表 VO 拼装(toHistoryItem + 进度链)抽到独立组件;只读不落库;输出与现状一致。
|
||||||
|
*/
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class AppearancePatentHistoryAssemblerTest {
|
||||||
|
|
||||||
|
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
|
||||||
|
private AppearancePatentHistoryAssembler assembler;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
assembler = new AppearancePatentHistoryAssembler(
|
||||||
|
taskProgressSnapshotService,
|
||||||
|
(current, total, job, baseTime) -> current <= 0 ? 1 : Math.min(99, current * 100 / 2),
|
||||||
|
snapshot -> 42);
|
||||||
|
lenient().when(taskProgressSnapshotService.find(anyLong(), anyString())).thenReturn(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileResultEntity result(Long id, Long taskId, String source, String resultFileUrl, Integer success) {
|
||||||
|
FileResultEntity row = new FileResultEntity();
|
||||||
|
row.setId(id);
|
||||||
|
row.setTaskId(taskId);
|
||||||
|
row.setSourceFilename(source);
|
||||||
|
row.setResultFilename(source == null ? null : source.replace(".xlsx", "-result.xlsx"));
|
||||||
|
row.setResultFileUrl(resultFileUrl);
|
||||||
|
row.setSuccess(success);
|
||||||
|
row.setErrorMessage(null);
|
||||||
|
row.setRowCount(12);
|
||||||
|
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity task(Long id, String status, LocalDateTime createdAt) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(id);
|
||||||
|
task.setStatus(status);
|
||||||
|
task.setCreatedAt(createdAt);
|
||||||
|
task.setUpdatedAt(createdAt);
|
||||||
|
task.setFinishedAt(createdAt == null ? null : createdAt.plusMinutes(5));
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskFileJobEntity job(Long id, String status) {
|
||||||
|
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||||
|
job.setId(id);
|
||||||
|
job.setStatus(status);
|
||||||
|
job.setErrorMessage(null);
|
||||||
|
job.setUpdatedAt(LocalDateTime.of(2026, 8, 1, 9, 30));
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_history_items() {
|
||||||
|
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(
|
||||||
|
result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1),
|
||||||
|
task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)),
|
||||||
|
null);
|
||||||
|
|
||||||
|
assertEquals(100L, item.getResultId());
|
||||||
|
assertEquals(10L, item.getTaskId());
|
||||||
|
assertEquals("a.xlsx", item.getSourceFilename());
|
||||||
|
assertEquals("a-result.xlsx", item.getResultFilename());
|
||||||
|
assertNull(item.getDownloadUrl(), "appearancepatent 无下载 URL 生成");
|
||||||
|
assertEquals("SUCCESS", item.getTaskStatus());
|
||||||
|
assertEquals(Boolean.TRUE, item.getSuccess(), "文件 URL 就绪即成功");
|
||||||
|
assertEquals(12, item.getRowCount());
|
||||||
|
assertEquals("2026-08-01T10:00", item.getCreatedAt());
|
||||||
|
assertEquals("2026-08-01T09:00", item.getStartedAt(), "任务开始时间复用 task.createdAt");
|
||||||
|
assertEquals("2026-08-01T09:05", item.getFinishedAt(), "任务结束时间取 task.finishedAt");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_row_with_null_fields() {
|
||||||
|
FileResultEntity row = result(100L, 10L, null, null, null);
|
||||||
|
row.setCreatedAt(null);
|
||||||
|
|
||||||
|
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, null, null);
|
||||||
|
|
||||||
|
assertNull(item.getSourceFilename(), "空文件名保留");
|
||||||
|
assertEquals(Boolean.FALSE, item.getSuccess(), "无文件 URL 且 success 为空视为失败");
|
||||||
|
assertNull(item.getStartedAt(), "task 与 createdAt 均缺时无开始时间");
|
||||||
|
assertFalse(Boolean.TRUE.equals(item.getFileReady()), "文件未就绪");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_file_state() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
|
||||||
|
|
||||||
|
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "SUCCESS", null), job(7L, "RUNNING"));
|
||||||
|
|
||||||
|
assertTrue(item.getFileReady(), "结果文件就绪");
|
||||||
|
assertEquals(7L, item.getFileJobId());
|
||||||
|
assertEquals("RUNNING", item.getFileStatus());
|
||||||
|
assertEquals(Integer.valueOf(100), item.getFileProgressPercent(), "就绪即 100%");
|
||||||
|
assertEquals("结果文件已生成", item.getFileProgressMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_order_priority() {
|
||||||
|
// RUNNING 任务优先(priority 0)于 SUCCESS 且文件就绪(priority 1)
|
||||||
|
FileResultEntity pending = result(200L, 2L, "b.xlsx", null, 0);
|
||||||
|
FileResultEntity done = result(100L, 1L, "a.xlsx", "result/1/a.xlsx", 1);
|
||||||
|
|
||||||
|
assertEquals(0, assembler.historyPriority(pending, task(2L, "RUNNING", null), null));
|
||||||
|
assertEquals(1, assembler.historyPriority(done, task(1L, "SUCCESS", null), null));
|
||||||
|
|
||||||
|
// SUCCESS 但文件未生成 → 构建中 → priority 0
|
||||||
|
FileResultEntity building = result(300L, 3L, "c.xlsx", null, 0);
|
||||||
|
assertEquals(0, assembler.historyPriority(building, task(3L, "SUCCESS", null), job(5L, "RUNNING")));
|
||||||
|
|
||||||
|
// 活动时间取 latestTime(row.createdAt 10:00 最晚)
|
||||||
|
assertEquals(LocalDateTime.of(2026, 8, 1, 10, 0),
|
||||||
|
assembler.historyActivityTime(done, task(1L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)), job(3L, "RUNNING")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_file_building() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||||
|
assertTrue(assembler.isHistoryFileBuilding(row, "SUCCESS", job(3L, "RUNNING")), "SUCCESS 任务文件未生成视为构建中");
|
||||||
|
assertFalse(assembler.isHistoryFileBuilding(row, "SUCCESS", job(4L, "FAILED")), "job 失败不算构建中");
|
||||||
|
assertFalse(assembler.isHistoryFileBuilding(row, "FAILED", null), "非 SUCCESS 任务不算构建中");
|
||||||
|
row.setResultFileUrl("result/10/a.xlsx");
|
||||||
|
assertFalse(assembler.isHistoryFileBuilding(row, "SUCCESS", null), "文件就绪不算构建中");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_null_task() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 1);
|
||||||
|
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 11, 0));
|
||||||
|
|
||||||
|
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, null, null);
|
||||||
|
|
||||||
|
assertNull(item.getTaskStatus(), "缺 task 状态为空");
|
||||||
|
assertEquals("2026-08-01T11:00", item.getStartedAt(), "缺 task 回退 result.createdAt");
|
||||||
|
assertNull(item.getFinishedAt(), "缺 task 无结束时间");
|
||||||
|
assertNull(item.getFileStatus(), "无 job 且文件未就绪时状态为空");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_file_error_attached() {
|
||||||
|
TaskFileJobEntity failedJob = job(9L, "FAILED");
|
||||||
|
failedJob.setErrorMessage("assemble boom");
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||||
|
|
||||||
|
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "RUNNING", null), failedJob);
|
||||||
|
|
||||||
|
assertEquals("FAILED", item.getFileStatus(), "job 失败状态附带");
|
||||||
|
assertEquals("assemble boom", item.getFileError(), "job 错误信息附带");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_snapshot_progress() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||||
|
TaskProgressSnapshotEntity snapshot = new TaskProgressSnapshotEntity();
|
||||||
|
snapshot.setTotalCount(10);
|
||||||
|
snapshot.setSuccessCount(5);
|
||||||
|
snapshot.setMessage("LLM 处理中");
|
||||||
|
when(taskProgressSnapshotService.find(10L, "APPEARANCE_PATENT")).thenReturn(snapshot);
|
||||||
|
|
||||||
|
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "RUNNING", null), null);
|
||||||
|
|
||||||
|
assertEquals(Integer.valueOf(5), item.getFileProgressCurrent(), "快照 current 附带");
|
||||||
|
assertEquals(Integer.valueOf(10), item.getFileProgressTotal(), "快照 total 附带");
|
||||||
|
assertEquals(Integer.valueOf(99), item.getFileProgressPercent(), "百分比 = max(注入计算值 99, 注入提取值 42)");
|
||||||
|
assertEquals("LLM 处理中", item.getFileProgressMessage());
|
||||||
|
verify(taskProgressSnapshotService).find(10L, "APPEARANCE_PATENT");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_immutable_input() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", "result/10/a.xlsx", 1);
|
||||||
|
FileTaskEntity t = task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0));
|
||||||
|
|
||||||
|
assembler.toHistoryItem(row, t, null);
|
||||||
|
|
||||||
|
assertEquals("a.xlsx", row.getSourceFilename(), "result 不被修改");
|
||||||
|
assertEquals("SUCCESS", t.getStatus(), "task 不被修改");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_consistency() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||||
|
row.setErrorMessage("python timeout");
|
||||||
|
row.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
AppearancePatentHistoryItemVo item = assembler.toHistoryItem(row, task(10L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 30)), null);
|
||||||
|
|
||||||
|
assertEquals(Boolean.FALSE, item.getSuccess(), "无文件 URL 且 success=0 为失败");
|
||||||
|
assertEquals("python timeout", item.getError());
|
||||||
|
assertEquals("FAILED", item.getTaskStatus());
|
||||||
|
assertEquals(12, item.getRowCount());
|
||||||
|
}
|
||||||
|
}
|
||||||
+74
@@ -0,0 +1,74 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 93:AppearancePatentRowNormalizer 字段归一化器。
|
||||||
|
* 规则与 AppearancePatentTaskService.normalize / firstNonBlank / baseId / normalizeDisplayId
|
||||||
|
* 现状逐字节一致:BOM 剥离、全角空格转半角、trim、连续空白折叠、firstNonBlank 取首非空。
|
||||||
|
*/
|
||||||
|
class AppearancePatentRowNormalizerTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_null_returns_empty() {
|
||||||
|
assertEquals("", AppearancePatentRowNormalizer.normalize(null), "null 归一化为空串");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_bom_stripped() {
|
||||||
|
assertEquals("B01X", AppearancePatentRowNormalizer.normalize("B01X"), "BOM 剥离");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_fullwidth_space_to_halfwidth() {
|
||||||
|
assertEquals("SKU 1", AppearancePatentRowNormalizer.normalize("SKU 1"), "全角空格转半角");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_trim_and_collapse_whitespace() {
|
||||||
|
assertEquals("a b c", AppearancePatentRowNormalizer.normalize(" a\t b \nc "), "trim + 连续空白折叠为单空格");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_first_non_blank_prefers_first() {
|
||||||
|
assertEquals("preferred", AppearancePatentRowNormalizer.firstNonBlank("preferred", "fallback"));
|
||||||
|
assertEquals("fallback", AppearancePatentRowNormalizer.firstNonBlank(" ", "fallback"), "首选空白回退");
|
||||||
|
assertEquals("fallback", AppearancePatentRowNormalizer.firstNonBlank(null, "fallback"), "首选 null 回退");
|
||||||
|
assertEquals("kept", AppearancePatentRowNormalizer.firstNonBlank(" kept ", "fallback"), "结果去首尾空白");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_base_id_splits_underscore_block() {
|
||||||
|
assertEquals("2", AppearancePatentRowNormalizer.baseId("2_1"), "块基 id 取下划线前");
|
||||||
|
assertEquals("3", AppearancePatentRowNormalizer.baseId("3"), "无下划线原样");
|
||||||
|
assertEquals("2", AppearancePatentRowNormalizer.baseId(" 2_1 "), "归一化后再取基 id");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_display_id_trims_only() {
|
||||||
|
assertEquals("2_1", AppearancePatentRowNormalizer.normalizeDisplayId(" 2_1 "), "displayId 只 trim 不折叠内部");
|
||||||
|
assertEquals("", AppearancePatentRowNormalizer.normalizeDisplayId(null));
|
||||||
|
assertEquals("", AppearancePatentRowNormalizer.normalizeDisplayId(" "));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_keeps_other_fullwidth_punct() {
|
||||||
|
assertEquals("(外观)", AppearancePatentRowNormalizer.normalize(" (外观) "), "非空格全角字符保留");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_not_mutating_shared_rules() {
|
||||||
|
String sample = " X Y ";
|
||||||
|
String once = AppearancePatentRowNormalizer.normalize(sample);
|
||||||
|
assertEquals(once, AppearancePatentRowNormalizer.normalize(sample), "幂等");
|
||||||
|
assertNotEquals(sample, once);
|
||||||
|
assertFalse(once.contains(""), "BOM 不残留");
|
||||||
|
assertFalse(once.contains(" "), "全角空格不残留");
|
||||||
|
assertTrue(once.equals("X Y"));
|
||||||
|
}
|
||||||
|
}
|
||||||
+227
@@ -0,0 +1,227 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 94:AppearancePatentSheetBuilder Sheet 构造器。
|
||||||
|
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式、行值派生)。
|
||||||
|
* 与现状 writeResultWorkbook / writeReasonSheet 一致:主 sheet 名"外观专利检测结果"、
|
||||||
|
* 13 列表头(RESULT_HEADERS 10 列 + 第 6/7/8 位插入标题/图片链接/sku)、加粗表头、
|
||||||
|
* 数据行从第 1 行;"原因"sheet 4 列按 ASIN 去重。不落库、无 IO 依赖。
|
||||||
|
*/
|
||||||
|
class AppearancePatentSheetBuilderTest {
|
||||||
|
|
||||||
|
private static final String[] RESULT_HEADERS_13 = {
|
||||||
|
"id", "asin", "国家", "卖家名称", "品牌", "价格",
|
||||||
|
"标题", "图片链接", "sku",
|
||||||
|
"标题维度(商标)", "外观维度(外观设计专利)", "结论", "状态"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static AppearancePatentParsedRowVo parsedRow(String id, String asin, String country,
|
||||||
|
String title, String url, String sku) {
|
||||||
|
AppearancePatentParsedRowVo row = new AppearancePatentParsedRowVo();
|
||||||
|
row.setDisplayId(id);
|
||||||
|
row.setSourceId(id);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setCountry(country);
|
||||||
|
row.setTitle(title);
|
||||||
|
row.setUrl(url);
|
||||||
|
row.setSku(sku);
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
values.put("卖家名称", "seller-A");
|
||||||
|
values.put("品牌", "brand-A");
|
||||||
|
values.put("价格", "19.90");
|
||||||
|
row.setValues(values);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AppearancePatentResultRowDto resultRow(String id, String asin, String country) {
|
||||||
|
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||||
|
row.setId(id);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setCountry(country);
|
||||||
|
row.setTitleRisk("无风险");
|
||||||
|
row.setAppearanceRisk("无风险");
|
||||||
|
row.setConclusion("已侵权");
|
||||||
|
row.setStatus("成功");
|
||||||
|
row.setError("");
|
||||||
|
row.setPrice("19.90");
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_headers_and_sheet_names() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(), Map.of());
|
||||||
|
|
||||||
|
Sheet main = wb.getSheet("外观专利检测结果");
|
||||||
|
Sheet reason = wb.getSheet("原因");
|
||||||
|
assertTrue(main != null, "主 sheet 存在");
|
||||||
|
assertTrue(reason != null, "原因 sheet 存在");
|
||||||
|
|
||||||
|
Row header = main.getRow(0);
|
||||||
|
assertEquals(13, header.getLastCellNum(), "主 sheet 13 列");
|
||||||
|
for (int i = 0; i < RESULT_HEADERS_13.length; i++) {
|
||||||
|
assertEquals(RESULT_HEADERS_13[i], header.getCell(i).getStringCellValue(),
|
||||||
|
"第 " + i + " 列表头");
|
||||||
|
}
|
||||||
|
Row reasonHeader = reason.getRow(0);
|
||||||
|
assertEquals("ASIN", reasonHeader.getCell(0).getStringCellValue());
|
||||||
|
assertEquals("外观原因", reasonHeader.getCell(1).getStringCellValue());
|
||||||
|
assertEquals("专利原因", reasonHeader.getCell(2).getStringCellValue());
|
||||||
|
assertEquals("标题原因", reasonHeader.getCell(3).getStringCellValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_header_bold_style() throws Exception {
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(), Map.of());
|
||||||
|
Cell headerCell = wb.getSheet("外观专利检测结果").getRow(0).getCell(0);
|
||||||
|
org.apache.poi.xssf.usermodel.XSSFCellStyle style =
|
||||||
|
(org.apache.poi.xssf.usermodel.XSSFCellStyle) headerCell.getCellStyle();
|
||||||
|
assertTrue(style.getFont().getBold(), "表头加粗");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_data_rows() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "title-a", "http://img/a.jpg", "SKU-1");
|
||||||
|
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
|
||||||
|
AppearancePatentSheetBuilder.rowKey(r), r));
|
||||||
|
|
||||||
|
Sheet main = wb.getSheet("外观专利检测结果");
|
||||||
|
Row data = main.getRow(1);
|
||||||
|
assertEquals("2_1", data.getCell(0).getStringCellValue(), "id 列");
|
||||||
|
assertEquals("B01A", data.getCell(1).getStringCellValue(), "asin 列");
|
||||||
|
assertEquals("US", data.getCell(2).getStringCellValue(), "国家列");
|
||||||
|
assertEquals("seller-A", data.getCell(3).getStringCellValue(), "卖家名称列");
|
||||||
|
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌列");
|
||||||
|
assertEquals("19.90", data.getCell(5).getStringCellValue(), "价格列");
|
||||||
|
assertEquals("title-a", data.getCell(6).getStringCellValue(), "标题列");
|
||||||
|
assertEquals("http://img/a.jpg", data.getCell(7).getStringCellValue(), "图片链接列");
|
||||||
|
assertEquals("SKU-1", data.getCell(8).getStringCellValue(), "sku 列");
|
||||||
|
assertEquals("无风险", data.getCell(9).getStringCellValue(), "标题维度列");
|
||||||
|
assertEquals("无风险", data.getCell(10).getStringCellValue(), "外观维度列");
|
||||||
|
assertEquals("已侵权", data.getCell(11).getStringCellValue(), "结论列");
|
||||||
|
assertEquals("成功", data.getCell(12).getStringCellValue(), "状态列");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_empty_data_rows() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(), Map.of());
|
||||||
|
Sheet main = wb.getSheet("外观专利检测结果");
|
||||||
|
assertNull(main.getRow(1), "无数据行时只有表头");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_no_result_row_uses_parsed_fallback() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "title-a", "http://img/a.jpg", "SKU-1");
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of());
|
||||||
|
|
||||||
|
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||||
|
assertEquals("title-a", data.getCell(6).getStringCellValue(), "无结果行回退解析行标题");
|
||||||
|
assertEquals("http://img/a.jpg", data.getCell(7).getStringCellValue(), "回退解析行 URL");
|
||||||
|
assertEquals("SKU-1", data.getCell(8).getStringCellValue(), "回退解析行 sku");
|
||||||
|
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
|
||||||
|
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_reason_sheet_dedup_by_asin() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentParsedRowVo p1 = parsedRow("2_1", "B01A", "US", "t", "", "");
|
||||||
|
AppearancePatentParsedRowVo p2 = parsedRow("2_2", "B01A", "DE", "t", "", "");
|
||||||
|
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
|
||||||
|
r.setAppearanceReason("外观理由");
|
||||||
|
r.setPatentReason("专利理由");
|
||||||
|
r.setTitleReason("标题理由");
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p1, p2), Map.of(
|
||||||
|
AppearancePatentSheetBuilder.rowKey(r), r));
|
||||||
|
|
||||||
|
Sheet reason = wb.getSheet("原因");
|
||||||
|
Row row1 = reason.getRow(1);
|
||||||
|
assertEquals("B01A", row1.getCell(0).getStringCellValue());
|
||||||
|
assertEquals("外观理由", row1.getCell(1).getStringCellValue());
|
||||||
|
assertEquals("专利理由", row1.getCell(2).getStringCellValue());
|
||||||
|
assertEquals("标题理由", row1.getCell(3).getStringCellValue());
|
||||||
|
assertNull(reason.getRow(2), "同 ASIN 第二行去重");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_writable_output_stream() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb,
|
||||||
|
List.of(parsedRow("1", "B01X", "US", "t", "u", "s")), Map.of());
|
||||||
|
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||||
|
wb.write(out);
|
||||||
|
assertTrue(out.size() > 0, "workbook 可写出");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_row_key_helpers() {
|
||||||
|
AppearancePatentResultRowDto r = resultRow("2_1", "b01a", "US");
|
||||||
|
assertEquals("2_1::B01A::US", AppearancePatentSheetBuilder.rowKey(r), "rowKey 归一化 + 大写 asin");
|
||||||
|
|
||||||
|
AppearancePatentParsedRowVo p = new AppearancePatentParsedRowVo();
|
||||||
|
p.setDisplayId("2_1");
|
||||||
|
p.setAsin("b01a");
|
||||||
|
p.setCountry("US");
|
||||||
|
assertEquals("2_1::B01A::US", AppearancePatentSheetBuilder.rowKey(p), "解析行 rowKey 同构");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_llm_failure_user_facing_cells() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
|
||||||
|
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
|
||||||
|
r.setTitleRisk("coze 调用超时");
|
||||||
|
r.setError("coze 工作流节点执行超限");
|
||||||
|
r.setConclusion("coze 调用超时");
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
|
||||||
|
AppearancePatentSheetBuilder.rowKey(r), r));
|
||||||
|
|
||||||
|
Row data = wb.getSheet("外观专利检测结果").getRow(1);
|
||||||
|
assertEquals("coze 工作流节点执行超限", data.getCell(9).getStringCellValue(),
|
||||||
|
"LLM 技术失败展示错误信息");
|
||||||
|
assertEquals("成功", data.getCell(12).getStringCellValue(),
|
||||||
|
"结论回退错误信息后非空即成功(与现状一致)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_sheet_builder_blank_asin_reason_row_skipped() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
AppearancePatentParsedRowVo p = parsedRow("2_1", " ", "US", "t", "", "");
|
||||||
|
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of());
|
||||||
|
|
||||||
|
Sheet reason = wb.getSheet("原因");
|
||||||
|
assertNull(reason.getRow(1), "空白 ASIN 不入原因 sheet");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+323
@@ -0,0 +1,323 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 96:AppearancePatent 快照对比测试。
|
||||||
|
* 夹具 Excel(appearancepatent 实际结构)→ 解析结果快照(golden 文件);
|
||||||
|
* 同一夹具输出完全一致;快照变更即失败(防行为漂移)。
|
||||||
|
* 管线与服务侧 parseWorkbook 对应段落一致:parser.parse → VO 组装(displayId/baseId/
|
||||||
|
* groupKey/rowToken)→ hydratePromptFields(组内 title/url/sku 回填);夹具无"状态"列,
|
||||||
|
* 不走 FailedStatusRowFilter。
|
||||||
|
* golden 文件:src/test/resources/appearancepatent/golden/parse-snapshot.txt
|
||||||
|
*/
|
||||||
|
class AppearancePatentSnapshotTest {
|
||||||
|
|
||||||
|
private static final File GOLDEN_PARSE =
|
||||||
|
new File("src/test/resources/appearancepatent/golden/parse-snapshot.txt");
|
||||||
|
|
||||||
|
private static final String[] HEADERS = {"id", "asin", "国家", "价格", "seller sku", "图片链接", "标题", "卖家名称", "品牌"};
|
||||||
|
|
||||||
|
// ---- 夹具 ----
|
||||||
|
|
||||||
|
private static File workbook(String[] headers, List<String[]> rows) throws Exception {
|
||||||
|
File file = File.createTempFile("appearance-patent-snapshot-", ".xlsx");
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream out = new FileOutputStream(file)) {
|
||||||
|
Sheet sheet = wb.createSheet("Sheet1");
|
||||||
|
Row header = sheet.createRow(0);
|
||||||
|
for (int i = 0; i < headers.length; i++) {
|
||||||
|
header.createCell(i).setCellValue(headers[i]);
|
||||||
|
}
|
||||||
|
for (int r = 0; r < rows.size(); r++) {
|
||||||
|
Row row = sheet.createRow(r + 1);
|
||||||
|
for (int c = 0; c < rows.get(r).length; c++) {
|
||||||
|
row.createCell(c).setCellValue(rows.get(r)[c]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wb.write(out);
|
||||||
|
}
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 5 有效行:2_1/2_2/2_3 同块(2_2/2_3 标题空缺待组内回填)、单块 5、单块 6。 */
|
||||||
|
private static File fixtureMain() throws Exception {
|
||||||
|
return workbook(HEADERS, List.of(
|
||||||
|
new String[]{"2_1", "B01A", "US", "19.90", "SKU-1", "http://img/a.jpg", "title a", "seller-A", "brand-A"},
|
||||||
|
new String[]{"2_2", "B01B", "DE", "29.90", "SKU-2", "http://img/b.jpg", "", "seller-B", "brand-B"},
|
||||||
|
new String[]{"2_3", "B01C", "FR", "", "", "http://img/c.jpg", "", "seller-C", "brand-C"},
|
||||||
|
new String[]{"5", "B01D", "UK", "9.90", "SKU-4", "http://img/d.jpg", "title d", "seller-D", "brand-D"},
|
||||||
|
new String[]{"6", "B01E", "US", "1.00", "SKU-5", "http://img/e.jpg", "title e", "seller-E", "brand-E"}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 缺 id/asin/国家 各 1 行 + 1 全空行;解析后剩 3 有效行。 */
|
||||||
|
private static File fixtureError() throws Exception {
|
||||||
|
return workbook(HEADERS, List.of(
|
||||||
|
new String[]{"", "B02A", "US", "", "", "", "", "seller", "brand"},
|
||||||
|
new String[]{"9", "", "DE", "", "", "", "", "seller", "brand"},
|
||||||
|
new String[]{"10", "B02B", "", "", "", "", "", "seller", "brand"},
|
||||||
|
new String[]{"11", "B02C", "FR", "", "", "", "", "seller", "brand"},
|
||||||
|
new String[]{"12", "B02D", "FR", "", "", "", "", "seller", "brand"},
|
||||||
|
new String[]{"13", "B02E", "FR", "", "", "", "", "seller", "brand"},
|
||||||
|
new String[]{"", "", ""}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 解析管线(与服务侧 parseWorkbook 对应段落语义一致) ----
|
||||||
|
|
||||||
|
private static List<AppearancePatentParsedRowVo> toRows(File file, String fileKey, String filename) throws Exception {
|
||||||
|
AppearancePatentExcelParser.ParsedSheet parsed = new AppearancePatentExcelParser().parse(file);
|
||||||
|
List<AppearancePatentParsedRowVo> rows = new ArrayList<>();
|
||||||
|
String currentBlockBaseId = "";
|
||||||
|
String currentGroupKey = "";
|
||||||
|
for (AppearancePatentExcelParser.AppearanceExcelRow parsedRow : parsed.rows()) {
|
||||||
|
String id = parsedRow.id();
|
||||||
|
String asin = parsedRow.asin();
|
||||||
|
String country = parsedRow.country();
|
||||||
|
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (id.isBlank() || asin.isBlank() || country.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo();
|
||||||
|
vo.setSourceFileKey(fileKey);
|
||||||
|
vo.setSourceFilename(filename);
|
||||||
|
vo.setRowIndex(parsedRow.rowIndex());
|
||||||
|
vo.setSourceId(id);
|
||||||
|
vo.setDisplayId(normalizeDisplayId(id));
|
||||||
|
String rowBaseId = baseId(vo.getDisplayId());
|
||||||
|
if (!Objects.equals(currentBlockBaseId, rowBaseId)) {
|
||||||
|
currentBlockBaseId = rowBaseId;
|
||||||
|
currentGroupKey = buildGroupKey(fileKey, rowBaseId, vo.getRowIndex());
|
||||||
|
}
|
||||||
|
vo.setGroupKey(currentGroupKey);
|
||||||
|
vo.setRowToken(buildRowToken(fileKey, vo.getRowIndex()));
|
||||||
|
vo.setAsin(asin);
|
||||||
|
vo.setCountry(country);
|
||||||
|
vo.setPrice(parsedRow.price());
|
||||||
|
vo.setSku(parsedRow.sku());
|
||||||
|
vo.setUrl(parsedRow.url());
|
||||||
|
vo.setTitle(parsedRow.title());
|
||||||
|
vo.setValues(parsedRow.values());
|
||||||
|
rows.add(vo);
|
||||||
|
}
|
||||||
|
hydratePromptFields(rows);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void hydratePromptFields(List<AppearancePatentParsedRowVo> rows) {
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Map<String, List<AppearancePatentParsedRowVo>> rowsByBaseId = new LinkedHashMap<>();
|
||||||
|
for (AppearancePatentParsedRowVo row : rows) {
|
||||||
|
String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId()));
|
||||||
|
rowsByBaseId.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
|
||||||
|
}
|
||||||
|
for (List<AppearancePatentParsedRowVo> siblings : rowsByBaseId.values()) {
|
||||||
|
String title = "";
|
||||||
|
String url = "";
|
||||||
|
String sku = "";
|
||||||
|
for (AppearancePatentParsedRowVo sibling : siblings) {
|
||||||
|
title = firstNonBlank(title, sibling.getTitle());
|
||||||
|
url = firstNonBlank(url, sibling.getUrl());
|
||||||
|
sku = firstNonBlank(sku, sibling.getSku());
|
||||||
|
}
|
||||||
|
for (AppearancePatentParsedRowVo sibling : siblings) {
|
||||||
|
if (normalize(sibling.getTitle()).isBlank()) {
|
||||||
|
sibling.setTitle(title);
|
||||||
|
}
|
||||||
|
if (normalize(sibling.getUrl()).isBlank()) {
|
||||||
|
sibling.setUrl(url);
|
||||||
|
}
|
||||||
|
if (normalize(sibling.getSku()).isBlank()) {
|
||||||
|
sibling.setSku(sku);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String normalize(String val) {
|
||||||
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String baseId(String id) {
|
||||||
|
String s = normalize(id);
|
||||||
|
int idx = s.indexOf('_');
|
||||||
|
return idx > 0 ? s.substring(0, idx) : s;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String normalizeDisplayId(String id) {
|
||||||
|
return id == null ? "" : id.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String buildRowToken(String sourceFileKey, Integer rowIndex) {
|
||||||
|
return normalize(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
|
||||||
|
return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String firstNonBlank(String preferred, String fallback) {
|
||||||
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 快照渲染 ----
|
||||||
|
|
||||||
|
private static String renderRow(AppearancePatentParsedRowVo row) {
|
||||||
|
return " row: idx=" + row.getRowIndex()
|
||||||
|
+ " sourceId=" + row.getSourceId()
|
||||||
|
+ " displayId=" + row.getDisplayId()
|
||||||
|
+ " asin=" + row.getAsin()
|
||||||
|
+ " country=" + row.getCountry()
|
||||||
|
+ " price=" + row.getPrice()
|
||||||
|
+ " sku=" + row.getSku()
|
||||||
|
+ " url=" + row.getUrl()
|
||||||
|
+ " title=" + row.getTitle()
|
||||||
|
+ " groupKey=" + row.getGroupKey()
|
||||||
|
+ " rowToken=" + row.getRowToken()
|
||||||
|
+ " values.size=" + (row.getValues() == null ? 0 : row.getValues().size());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String renderRows(List<AppearancePatentParsedRowVo> rows) {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("rows=").append(rows.size()).append('\n');
|
||||||
|
for (AppearancePatentParsedRowVo row : rows) {
|
||||||
|
sb.append(renderRow(row)).append('\n');
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<AppearancePatentParsedRowVo> mergeRows(List<List<AppearancePatentParsedRowVo>> all) {
|
||||||
|
List<AppearancePatentParsedRowVo> merged = new ArrayList<>();
|
||||||
|
for (List<AppearancePatentParsedRowVo> list : all) {
|
||||||
|
merged.addAll(list);
|
||||||
|
}
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String runAllParse() throws Exception {
|
||||||
|
return renderRows(mergeRows(List.of(
|
||||||
|
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx"),
|
||||||
|
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx"))));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String read(File file) throws Exception {
|
||||||
|
return Files.readString(file.toPath(), StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 用例 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_parse_output() throws Exception {
|
||||||
|
assertEquals(read(GOLDEN_PARSE), runAllParse(), "解析输出快照与 golden 一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_rows() throws Exception {
|
||||||
|
List<AppearancePatentParsedRowVo> rows = mergeRows(List.of(
|
||||||
|
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx"),
|
||||||
|
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx")));
|
||||||
|
|
||||||
|
assertEquals(8, rows.size(), "有效行 5 + 3");
|
||||||
|
assertEquals("uploads/main.xlsx::2@2", rows.get(0).getGroupKey(), "首块 groupKey = fileKey::baseId@rowIndex");
|
||||||
|
assertEquals("uploads/main.xlsx::row::2", rows.get(0).getRowToken(), "rowToken = fileKey::row::rowIndex");
|
||||||
|
assertEquals("2_1", rows.get(0).getDisplayId(), "displayId 原样保留");
|
||||||
|
assertEquals("B01A", rows.get(0).getAsin(), "asin 大写");
|
||||||
|
assertEquals(9, rows.get(0).getValues().size(), "values 按 9 列表头键取值");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_block_grouping() throws Exception {
|
||||||
|
List<AppearancePatentParsedRowVo> rows = toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx");
|
||||||
|
|
||||||
|
assertEquals(5, rows.size());
|
||||||
|
assertEquals(rows.get(0).getGroupKey(), rows.get(1).getGroupKey(), "2_1/2_2 同块");
|
||||||
|
assertEquals(rows.get(0).getGroupKey(), rows.get(2).getGroupKey(), "2_1/2_3 同块");
|
||||||
|
assertFalse(rows.get(0).getGroupKey().equals(rows.get(3).getGroupKey()), "2_1 与 5 不同块");
|
||||||
|
assertEquals("uploads/main.xlsx::5@5", rows.get(3).getGroupKey(), "新块 baseId=5");
|
||||||
|
assertEquals("uploads/main.xlsx::6@6", rows.get(4).getGroupKey(), "新块 baseId=6");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_hydration() throws Exception {
|
||||||
|
List<AppearancePatentParsedRowVo> rows = toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx");
|
||||||
|
|
||||||
|
assertEquals("title a", rows.get(1).getTitle(), "2_2 标题从组内回填");
|
||||||
|
assertEquals("http://img/b.jpg", rows.get(1).getUrl(), "2_2 自身 url 保留");
|
||||||
|
assertEquals("SKU-1", rows.get(2).getSku(), "2_3 sku 从组内回填");
|
||||||
|
assertEquals("title a", rows.get(2).getTitle(), "2_3 标题从组内回填");
|
||||||
|
assertEquals("http://img/c.jpg", rows.get(2).getUrl(), "2_3 自身 url 保留");
|
||||||
|
assertEquals("", rows.get(2).getPrice(), "价格不回填(仅 title/url/sku)");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_error_cases() throws Exception {
|
||||||
|
// 缺必填字段行与全空行均不进入结果;不抛异常
|
||||||
|
List<AppearancePatentParsedRowVo> rows = toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx");
|
||||||
|
|
||||||
|
assertEquals(3, rows.size(), "错误夹具剩 3 有效行");
|
||||||
|
assertEquals("11", rows.get(0).getSourceId());
|
||||||
|
assertEquals("12", rows.get(1).getSourceId());
|
||||||
|
assertEquals("13", rows.get(2).getSourceId());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_reproducible() throws Exception {
|
||||||
|
assertEquals(runAllParse(), runAllParse(), "同一夹具跑两次结果一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_golden_committed() {
|
||||||
|
assertTrue(GOLDEN_PARSE.isFile(), "golden 文件必须存在并入库: " + GOLDEN_PARSE.getAbsolutePath());
|
||||||
|
assertTrue(GOLDEN_PARSE.length() > 0, "golden 文件非空");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_diff_detected() throws Exception {
|
||||||
|
String original = read(GOLDEN_PARSE);
|
||||||
|
assertTrue(original.contains("rows="), "golden 内容合法");
|
||||||
|
try {
|
||||||
|
Files.writeString(GOLDEN_PARSE.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
|
||||||
|
AssertionError failure = null;
|
||||||
|
try {
|
||||||
|
assertEquals(read(GOLDEN_PARSE), runAllParse(), "篡改后应与 golden 不一致");
|
||||||
|
} catch (AssertionError ex) {
|
||||||
|
failure = ex;
|
||||||
|
}
|
||||||
|
assertTrue(failure != null, "篡改 golden 后断言应失败");
|
||||||
|
} finally {
|
||||||
|
Files.writeString(GOLDEN_PARSE.toPath(), original, StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
assertEquals(original, read(GOLDEN_PARSE), "恢复原始 golden");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_regression_all_parse() throws Exception {
|
||||||
|
assertEquals(read(GOLDEN_PARSE), runAllParse(), "全 parse 路径与 golden 一致");
|
||||||
|
List<AppearancePatentParsedRowVo> rows = mergeRows(List.of(
|
||||||
|
toRows(fixtureMain(), "uploads/main.xlsx", "main.xlsx"),
|
||||||
|
toRows(fixtureError(), "uploads/error.xlsx", "error.xlsx")));
|
||||||
|
assertEquals(8, rows.size(), "总行数 5 + 3");
|
||||||
|
assertEquals("uploads/error.xlsx::row::5", rows.get(5).getRowToken(), "错误夹具首行 rowToken");
|
||||||
|
assertEquals(9, rows.get(5).getValues().size(), "错误夹具 values 仍有 9 列");
|
||||||
|
}
|
||||||
|
}
|
||||||
+3
-1
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
|||||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
@@ -112,7 +113,8 @@ class CollectDataDeleteConsistencyTest {
|
|||||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class),
|
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class),
|
||||||
resultDetailCodec,
|
resultDetailCodec,
|
||||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
||||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class));;
|
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
private FileTaskEntity task(long id, long userId) {
|
private FileTaskEntity task(long id, long userId) {
|
||||||
|
|||||||
+3
-1
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
|||||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
@@ -111,7 +112,8 @@ class CollectDataStorageCallCountTest {
|
|||||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class),
|
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter.class),
|
||||||
resultDetailCodec,
|
resultDetailCodec,
|
||||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter.class),
|
||||||
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class));
|
mock(com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader.class),
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
private FileResultEntity result(long id, long taskId, long userId) {
|
private FileResultEntity result(long id, long taskId, long userId) {
|
||||||
|
|||||||
+7
-7
@@ -55,12 +55,12 @@ class InvalidAsinDataControllerTest {
|
|||||||
when(adminAuthSupport.currentRole(operator)).thenReturn(null);
|
when(adminAuthSupport.currentRole(operator)).thenReturn(null);
|
||||||
when(permissionMenuService.getUserColumnPermissions(7L, "admin"))
|
when(permissionMenuService.getUserColumnPermissions(7L, "admin"))
|
||||||
.thenReturn(List.of(invalidAsinDataPermission()));
|
.thenReturn(List.of(invalidAsinDataPermission()));
|
||||||
when(invalidAsinDataService.page(1L, 15L, "", 3L, 7L, false))
|
when(invalidAsinDataService.page(1L, 15L, "", "", "", 3L, 7L, false))
|
||||||
.thenReturn(new InvalidAsinDataPageVo());
|
.thenReturn(new InvalidAsinDataPageVo());
|
||||||
|
|
||||||
controller.page(1L, 15L, "", 3L, request);
|
controller.page(1L, 15L, "", "", "", 3L, request);
|
||||||
|
|
||||||
verify(invalidAsinDataService).page(1L, 15L, "", 3L, 7L, false);
|
verify(invalidAsinDataService).page(1L, 15L, "", "", "", 3L, 7L, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -70,7 +70,7 @@ class InvalidAsinDataControllerTest {
|
|||||||
request.addParameter("superAdmin", "true");
|
request.addParameter("superAdmin", "true");
|
||||||
when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录"));
|
when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录"));
|
||||||
|
|
||||||
assertThrows(BusinessException.class, () -> controller.page(1L, 15L, "", 3L, request));
|
assertThrows(BusinessException.class, () -> controller.page(1L, 15L, "", "", "", 3L, request));
|
||||||
|
|
||||||
verifyNoInteractions(invalidAsinDataService);
|
verifyNoInteractions(invalidAsinDataService);
|
||||||
verify(permissionMenuService, never()).requireUserOperator(any());
|
verify(permissionMenuService, never()).requireUserOperator(any());
|
||||||
@@ -83,12 +83,12 @@ class InvalidAsinDataControllerTest {
|
|||||||
AdminUserEntity operator = user(1L, "super_admin");
|
AdminUserEntity operator = user(1L, "super_admin");
|
||||||
when(adminAuthSupport.requireUser(request)).thenReturn(operator);
|
when(adminAuthSupport.requireUser(request)).thenReturn(operator);
|
||||||
when(adminAuthSupport.currentRole(operator)).thenReturn("super_admin");
|
when(adminAuthSupport.currentRole(operator)).thenReturn("super_admin");
|
||||||
when(invalidAsinDataService.page(1L, 15L, "", 3L, 1L, true))
|
when(invalidAsinDataService.page(1L, 15L, "", "", "", 3L, 1L, true))
|
||||||
.thenReturn(new InvalidAsinDataPageVo());
|
.thenReturn(new InvalidAsinDataPageVo());
|
||||||
|
|
||||||
controller.page(1L, 15L, "", 3L, request);
|
controller.page(1L, 15L, "", "", "", 3L, request);
|
||||||
|
|
||||||
verify(invalidAsinDataService).page(1L, 15L, "", 3L, 1L, true);
|
verify(invalidAsinDataService).page(1L, 15L, "", "", "", 3L, 1L, true);
|
||||||
verify(permissionMenuService, never()).getUserColumnPermissions(eq(1L), eq("admin"));
|
verify(permissionMenuService, never()).getUserColumnPermissions(eq(1L), eq("admin"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+23
-3
@@ -85,7 +85,7 @@ class InvalidAsinDataServiceTest {
|
|||||||
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(manual));
|
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(manual));
|
||||||
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of(9L, "group-a"));
|
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of(9L, "group-a"));
|
||||||
|
|
||||||
InvalidAsinDataPageVo page = service.page(1, 15, "", 10L, 7L, false);
|
InvalidAsinDataPageVo page = service.page(1, 15, "", "", "", 10L, 7L, false);
|
||||||
|
|
||||||
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
||||||
verify(invalidAsinDataMapper).selectCount(captor.capture());
|
verify(invalidAsinDataMapper).selectCount(captor.capture());
|
||||||
@@ -109,7 +109,7 @@ class InvalidAsinDataServiceTest {
|
|||||||
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(automatic, orphan));
|
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(automatic, orphan));
|
||||||
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of());
|
when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of());
|
||||||
|
|
||||||
InvalidAsinDataPageVo page = service.page(1, 15, "", null, 1L, true);
|
InvalidAsinDataPageVo page = service.page(1, 15, "", "", "", null, 1L, true);
|
||||||
|
|
||||||
assertEquals(2, page.getItems().size());
|
assertEquals(2, page.getItems().size());
|
||||||
assertEquals("AUTO", page.getItems().getFirst().getRecordSource());
|
assertEquals("AUTO", page.getItems().getFirst().getRecordSource());
|
||||||
@@ -123,7 +123,7 @@ class InvalidAsinDataServiceTest {
|
|||||||
when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L);
|
when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L);
|
||||||
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of());
|
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
service.page(1, 15, "", 9L, 1L, true);
|
service.page(1, 15, "", "", "", 9L, 1L, true);
|
||||||
|
|
||||||
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
||||||
verify(invalidAsinDataMapper).selectCount(captor.capture());
|
verify(invalidAsinDataMapper).selectCount(captor.capture());
|
||||||
@@ -134,6 +134,26 @@ class InvalidAsinDataServiceTest {
|
|||||||
assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L));
|
assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||||
|
void pageFiltersByDataValueAndBrandIndependently() {
|
||||||
|
when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
|
service.page(1, 15, "", "B01", "acme", null, 1L, true);
|
||||||
|
|
||||||
|
ArgumentCaptor<LambdaQueryWrapper<InvalidAsinDataEntity>> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
||||||
|
verify(invalidAsinDataMapper).selectCount(captor.capture());
|
||||||
|
TableInfoHelper.initTableInfo(
|
||||||
|
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||||
|
InvalidAsinDataEntity.class);
|
||||||
|
String sql = captor.getValue().getSqlSegment();
|
||||||
|
assertTrue(sql.contains("data_value LIKE"));
|
||||||
|
assertTrue(sql.contains("brand LIKE"));
|
||||||
|
assertTrue(captor.getValue().getParamNameValuePairs().containsValue("%B01%"));
|
||||||
|
assertTrue(captor.getValue().getParamNameValuePairs().containsValue("%acme%"));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void normalUserCannotDeleteAutoRecord() {
|
void normalUserCannotDeleteAutoRecord() {
|
||||||
when(invalidAsinDataMapper.selectById(91L)).thenReturn(data(91L, "AUTO", null));
|
when(invalidAsinDataMapper.selectById(91L)).thenReturn(data(91L, "AUTO", null));
|
||||||
|
|||||||
+552
@@ -0,0 +1,552 @@
|
|||||||
|
package com.nanri.aiimage.modules.publish.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
|
||||||
|
import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.vo.PublishDashboardVo;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.vo.PublishFileVo;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.vo.PublishHistoryVo;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.vo.PublishResultVo;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.vo.PublishTaskDetailVo;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.vo.PublishTaskVo;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.doAnswer;
|
||||||
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 114:批量加载等价快照测试。
|
||||||
|
* 同一夹具下,批量路径(history 批量 IN / dashboard 聚合 / progress 批量)与逐条基准
|
||||||
|
* (getTaskDetail 的 selectById + 按 taskId 过滤装配)输出完全一致。
|
||||||
|
* 快照可重复;篡改基准即失败;覆盖历史/结果/dashboard 三页;全量回归门禁。
|
||||||
|
*/
|
||||||
|
class PublishBatchLoadingSnapshotEquivTest {
|
||||||
|
|
||||||
|
private static final String MODULE = "PUBLISH";
|
||||||
|
private static final java.io.File GOLDEN =
|
||||||
|
new java.io.File("src/test/resources/publish/golden/batch-loading-snapshot.txt");
|
||||||
|
|
||||||
|
private final List<FileTaskEntity> taskDb = new ArrayList<>();
|
||||||
|
private final List<PublishFileEntity> fileDb = new ArrayList<>();
|
||||||
|
private final List<FileResultEntity> resultDb = new ArrayList<>();
|
||||||
|
private final List<TaskFileJobEntity> jobDb = new ArrayList<>();
|
||||||
|
|
||||||
|
private FileTaskMapper fileTaskMapper;
|
||||||
|
private PublishFileMapper publishFileMapper;
|
||||||
|
private FileResultMapper fileResultMapper;
|
||||||
|
private TaskFileJobService taskFileJobService;
|
||||||
|
|
||||||
|
private PublishTaskService service;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initializeMybatisMetadata() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskFileJobEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String renderDashboard(PublishDashboardVo vo) {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("pending=").append(vo.getPendingCount())
|
||||||
|
.append(" running=").append(vo.getRunningCount())
|
||||||
|
.append(" success=").append(vo.getSuccessCount())
|
||||||
|
.append(" failed=").append(vo.getFailedCount()).append('\n');
|
||||||
|
for (PublishTaskDetailVo item : vo.getRecent()) {
|
||||||
|
sb.append(renderTaskDetail(item));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String renderTaskDetail(PublishTaskDetailVo detail) {
|
||||||
|
PublishTaskVo task = detail.getTask();
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("task id=").append(task.getId())
|
||||||
|
.append(" no=").append(task.getTaskNo())
|
||||||
|
.append(" status=").append(task.getStatus())
|
||||||
|
.append(" src=").append(task.getSourceFileCount())
|
||||||
|
.append(" ok=").append(task.getSuccessFileCount())
|
||||||
|
.append(" fail=").append(task.getFailedFileCount())
|
||||||
|
.append(" comp=").append(task.getCompletedFileCount())
|
||||||
|
.append(" rows=").append(task.getTotalRows())
|
||||||
|
.append("/").append(task.getProcessedRows())
|
||||||
|
.append(" pct=").append(task.getPercent())
|
||||||
|
.append(" err=").append(task.getErrorMessage())
|
||||||
|
.append('\n');
|
||||||
|
for (PublishFileVo file : detail.getFiles()) {
|
||||||
|
sb.append(" file id=").append(file.getFileId())
|
||||||
|
.append(" key=").append(file.getFileKey())
|
||||||
|
.append(" shop=").append(file.getShopName())
|
||||||
|
.append("/").append(file.getShopId())
|
||||||
|
.append(" matched=").append(file.isMatched())
|
||||||
|
.append(" status=").append(file.getStatus())
|
||||||
|
.append(" rows=").append(file.getTotalRows())
|
||||||
|
.append("/").append(file.getProcessedRows())
|
||||||
|
.append(" pct=").append(file.getPercent())
|
||||||
|
.append('\n');
|
||||||
|
}
|
||||||
|
PublishResultVo result = detail.getResult();
|
||||||
|
if (result != null) {
|
||||||
|
sb.append(" result id=").append(result.getResultId())
|
||||||
|
.append(" ready=").append(result.getFileReady())
|
||||||
|
.append(" url=").append(result.getDownloadUrl())
|
||||||
|
.append(" job=").append(result.getFileJobId())
|
||||||
|
.append("/").append(result.getFileJobStatus())
|
||||||
|
.append(" retry=").append(result.getFileJobRetryCount())
|
||||||
|
.append(" joberr=").append(result.getFileJobError())
|
||||||
|
.append(" err=").append(result.getErrorMessage())
|
||||||
|
.append('\n');
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String renderHistory(PublishHistoryVo vo) {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("total=").append(vo.getTotal()).append('\n');
|
||||||
|
for (PublishTaskDetailVo item : vo.getItems()) {
|
||||||
|
sb.append(renderTaskDetail(item));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String renderBatch(List<PublishTaskDetailVo> items) {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (PublishTaskDetailVo item : items) {
|
||||||
|
sb.append(renderTaskDetail(item));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 从渲染后的 wrapper 提取 IN 查询引用的全部 id(.in() 每个元素一个 #{ew.paramNameValuePairs.x} 参数)。 */
|
||||||
|
private static List<Long> inParamIds(LambdaQueryWrapper<?> q) {
|
||||||
|
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||||
|
List<Long> ids = new ArrayList<>();
|
||||||
|
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||||
|
.compile("#\\{ew\\.paramNameValuePairs\\.(\\w+)}").matcher(segment);
|
||||||
|
Map<String, Object> params = q.getParamNameValuePairs();
|
||||||
|
while (m.find()) {
|
||||||
|
Object value = params.get(m.group(1));
|
||||||
|
if (value instanceof Number n) {
|
||||||
|
ids.add(n.longValue());
|
||||||
|
} else if (value instanceof List<?> list) {
|
||||||
|
for (Object item : list) {
|
||||||
|
if (item instanceof Number n) {
|
||||||
|
ids.add(n.longValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ids;
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
publishFileMapper = mock(PublishFileMapper.class);
|
||||||
|
fileResultMapper = mock(FileResultMapper.class);
|
||||||
|
taskFileJobService = mock(TaskFileJobService.class);
|
||||||
|
OssStorageService ossStorageService = mock(OssStorageService.class);
|
||||||
|
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
|
||||||
|
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||||
|
List<Object> params = new ArrayList<>(q.getParamNameValuePairs().values());
|
||||||
|
List<Long> wanted = new ArrayList<>();
|
||||||
|
for (Object value : params) {
|
||||||
|
if (value instanceof List<?> list) {
|
||||||
|
for (Object item : list) {
|
||||||
|
if (item instanceof Number n) {
|
||||||
|
wanted.add(n.longValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
List<FileTaskEntity> filtered = new ArrayList<>();
|
||||||
|
for (FileTaskEntity task : taskDb) {
|
||||||
|
if (!MODULE.equals(task.getModuleType())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (wanted.isEmpty()) {
|
||||||
|
filtered.add(task);
|
||||||
|
} else if (wanted.contains(task.getId())) {
|
||||||
|
filtered.add(task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
|
||||||
|
if (segment.toLowerCase().contains("limit")) {
|
||||||
|
int cap = 50;
|
||||||
|
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||||
|
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
|
||||||
|
if (m.find()) {
|
||||||
|
cap = Integer.parseInt(m.group(1));
|
||||||
|
}
|
||||||
|
return filtered.subList(0, Math.min(cap, filtered.size()));
|
||||||
|
}
|
||||||
|
return filtered;
|
||||||
|
}).when(fileTaskMapper).selectList(any());
|
||||||
|
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
Long id = invocation.getArgument(0);
|
||||||
|
return taskDb.stream().filter(t -> t.getId().equals(id)).findFirst().orElse(null);
|
||||||
|
}).when(fileTaskMapper).selectById(any(Long.class));
|
||||||
|
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
LambdaQueryWrapper<PublishFileEntity> q = invocation.getArgument(0);
|
||||||
|
List<Long> wanted = inParamIds(q);
|
||||||
|
return fileDb.stream()
|
||||||
|
.filter(f -> wanted.contains(f.getTaskId()))
|
||||||
|
.sorted((a, b) -> a.getId().compareTo(b.getId()))
|
||||||
|
.toList();
|
||||||
|
}).when(publishFileMapper).selectList(any());
|
||||||
|
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
LambdaQueryWrapper<FileResultEntity> q = invocation.getArgument(0);
|
||||||
|
List<Long> wanted = inParamIds(q);
|
||||||
|
return resultDb.stream()
|
||||||
|
.filter(r -> wanted.contains(r.getTaskId()))
|
||||||
|
.sorted((a, b) -> a.getId().compareTo(b.getId()))
|
||||||
|
.toList();
|
||||||
|
}).when(fileResultMapper).selectList(any());
|
||||||
|
|
||||||
|
lenient().when(taskFileJobService.findAssembleJobsByResultIds(any(), any())).thenAnswer(invocation -> {
|
||||||
|
List<?> ids = invocation.getArgument(1);
|
||||||
|
Map<Long, TaskFileJobEntity> map = new HashMap<>();
|
||||||
|
if (ids != null) {
|
||||||
|
for (TaskFileJobEntity job : jobDb) {
|
||||||
|
if (job.getResultId() != null && ids.contains(job.getResultId())) {
|
||||||
|
map.put(job.getResultId(), job);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
});
|
||||||
|
|
||||||
|
when(fileTaskMapper.selectCount(any())).thenAnswer(invocation -> (long) taskDb.stream()
|
||||||
|
.filter(t -> MODULE.equals(t.getModuleType()) && Long.valueOf(1L).equals(t.getUserId())).count());
|
||||||
|
|
||||||
|
when(fileTaskMapper.selectMaps(any())).thenAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
com.baomidou.mybatisplus.core.conditions.Wrapper<FileTaskEntity> wrapper = invocation.getArgument(0);
|
||||||
|
String segment = wrapper.getSqlSegment() == null ? "" : wrapper.getSqlSegment();
|
||||||
|
assertTrue(segment.toLowerCase().contains("group by"), "聚合查询必须带 GROUP BY: " + segment);
|
||||||
|
Map<String, Long> counts = new HashMap<>();
|
||||||
|
for (FileTaskEntity task : taskDb) {
|
||||||
|
if (!MODULE.equals(task.getModuleType()) || task.getUserId() == null || task.getStatus() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
counts.merge(task.getStatus(), 1L, Long::sum);
|
||||||
|
}
|
||||||
|
List<Map<String, Object>> rows = new ArrayList<>();
|
||||||
|
for (Map.Entry<String, Long> entry : counts.entrySet()) {
|
||||||
|
Map<String, Object> row = new HashMap<>();
|
||||||
|
row.put("status", entry.getKey());
|
||||||
|
row.put("cnt", entry.getValue());
|
||||||
|
rows.add(row);
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
});
|
||||||
|
|
||||||
|
when(ossStorageService.generateFreshDownloadUrl(any())).thenAnswer(
|
||||||
|
invocation -> "https://oss" + invocation.getArgument(0));
|
||||||
|
|
||||||
|
service = new PublishTaskService(
|
||||||
|
mock(LocalFileStorageService.class), mock(ZiniaoShopSwitchService.class),
|
||||||
|
mock(PublishWorkbookService.class), publishFileMapper, mock(PublishItemMapper.class),
|
||||||
|
fileTaskMapper, fileResultMapper, mock(TaskChunkMapper.class),
|
||||||
|
mock(TaskScopeStateMapper.class), taskFileJobService,
|
||||||
|
mock(TaskDistributedLockService.class), mock(TransientPayloadStorageService.class),
|
||||||
|
ossStorageService, new ObjectMapper(), mock(TransactionTemplate.class),
|
||||||
|
mock(InstanceMetadata.class),
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity task(long id, String status, LocalDateTime createdAt, Integer src, Integer ok, Integer fail) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(id);
|
||||||
|
task.setModuleType(MODULE);
|
||||||
|
task.setUserId(1L);
|
||||||
|
task.setStatus(status);
|
||||||
|
task.setCreatedAt(createdAt);
|
||||||
|
task.setUpdatedAt(createdAt);
|
||||||
|
task.setFinishedAt(status.equals("SUCCESS") || status.equals("FAILED") ? createdAt.plusMinutes(5) : null);
|
||||||
|
task.setTaskNo("T" + id);
|
||||||
|
task.setSourceFileCount(src);
|
||||||
|
task.setSuccessFileCount(ok);
|
||||||
|
task.setFailedFileCount(fail);
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static PublishFileEntity file(Long id, Long taskId, String status, Integer total, Integer processed) {
|
||||||
|
PublishFileEntity file = new PublishFileEntity();
|
||||||
|
file.setId(id);
|
||||||
|
file.setTaskId(taskId);
|
||||||
|
file.setFileKey("key" + id);
|
||||||
|
file.setSourceFilename("s" + id + ".xlsx");
|
||||||
|
file.setShopName("Shop " + (id % 3));
|
||||||
|
file.setShopId("SHOP" + (id % 3));
|
||||||
|
file.setMatched(1);
|
||||||
|
file.setStatus(status);
|
||||||
|
file.setTotalRows(total);
|
||||||
|
file.setProcessedRows(processed);
|
||||||
|
file.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileResultEntity result(Long id, Long taskId, String error) {
|
||||||
|
FileResultEntity result = new FileResultEntity();
|
||||||
|
result.setId(id);
|
||||||
|
result.setTaskId(taskId);
|
||||||
|
result.setModuleType(MODULE);
|
||||||
|
result.setResultFilename("r" + id + ".xlsx");
|
||||||
|
result.setResultFileUrl("result/publish/" + id + "/out.xlsx");
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(error);
|
||||||
|
result.setCreatedAt(LocalDateTime.of(2026, 8, 1, 11, 0));
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskFileJobEntity job(Long id, Long resultId, String status, Integer retry, String error) {
|
||||||
|
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||||
|
job.setId(id);
|
||||||
|
job.setResultId(resultId);
|
||||||
|
job.setModuleType(MODULE);
|
||||||
|
job.setStatus(status);
|
||||||
|
job.setRetryCount(retry);
|
||||||
|
job.setErrorMessage(error);
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 固定夹具:5 任务 ×(成功多文件/运行中+失败文件/待命无结果/失败+Job 错误/成功单文件)+ 结果与 Job 关联。 */
|
||||||
|
private void seed() {
|
||||||
|
taskDb.add(task(1L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 1), 2, 2, 0));
|
||||||
|
taskDb.add(task(2L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 2), 2, 0, 1));
|
||||||
|
taskDb.add(task(3L, "PENDING", LocalDateTime.of(2026, 8, 1, 9, 3), 1, 0, 0));
|
||||||
|
taskDb.add(task(4L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 4), 1, 0, 1));
|
||||||
|
taskDb.add(task(5L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 5), 1, 1, 0));
|
||||||
|
fileDb.add(file(101L, 1L, "SUCCESS", 10, 10));
|
||||||
|
fileDb.add(file(102L, 1L, "SUCCESS", 20, 20));
|
||||||
|
fileDb.add(file(201L, 2L, "RUNNING", 30, 12));
|
||||||
|
fileDb.add(file(202L, 2L, "FAILED", 5, 3));
|
||||||
|
fileDb.add(file(301L, 3L, "PENDING", 8, 0));
|
||||||
|
fileDb.add(file(401L, 4L, "FAILED", 9, 4));
|
||||||
|
fileDb.add(file(501L, 5L, "SUCCESS", 15, 15));
|
||||||
|
resultDb.add(result(901L, 1L, null));
|
||||||
|
resultDb.add(result(902L, 2L, "still running"));
|
||||||
|
resultDb.add(result(904L, 4L, "boom"));
|
||||||
|
jobDb.add(job(801L, 901L, "SUCCESS", 0, null));
|
||||||
|
jobDb.add(job(802L, 902L, "RUNNING", 1, null));
|
||||||
|
jobDb.add(job(804L, 904L, "FAILED", 2, "assemble failed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<FileTaskEntity> orderedTasks() {
|
||||||
|
return taskDb.stream()
|
||||||
|
.filter(t -> MODULE.equals(t.getModuleType()) && Long.valueOf(1L).equals(t.getUserId()))
|
||||||
|
.sorted((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()))
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 逐条基准:与批量路径相同的装配语义,但每任务一次 getTaskDetail(selectById 单条路径)。 */
|
||||||
|
private String baselineDetails() {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (FileTaskEntity task : orderedTasks()) {
|
||||||
|
sb.append(renderTaskDetail(service.getTaskDetail(task.getId(), 1L)));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String baselineHistoryText() {
|
||||||
|
return "total=" + taskDb.stream()
|
||||||
|
.filter(t -> MODULE.equals(t.getModuleType()) && Long.valueOf(1L).equals(t.getUserId())).count()
|
||||||
|
+ "\n" + baselineDetails();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String historySnapshot() {
|
||||||
|
return renderHistory(service.history(1L, 50));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String dashboardSnapshot() {
|
||||||
|
return renderDashboard(service.dashboard(1L));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resultSnapshot() {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (FileTaskEntity task : orderedTasks()) {
|
||||||
|
sb.append(renderTaskDetail(service.getTaskDetail(task.getId(), 1L)));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** progress 批量:请求顺序按 createdAt 倒序(与 history 语义一致),输出明细。 */
|
||||||
|
private String batchSnapshot() {
|
||||||
|
List<Long> ids = orderedTasks().stream().map(FileTaskEntity::getId).toList();
|
||||||
|
return renderBatch(service.getTaskProgress(1L, ids).getItems());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String readGolden() throws Exception {
|
||||||
|
return java.nio.file.Files.readString(GOLDEN.toPath(), java.nio.charset.StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_history_snapshot_equiv() throws Exception {
|
||||||
|
seed();
|
||||||
|
assertEquals(baselineHistoryText(), historySnapshot(),
|
||||||
|
"history 批量装配输出与逐条基准完全一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_dashboard_snapshot_equiv() throws Exception {
|
||||||
|
seed();
|
||||||
|
PublishDashboardVo vo = service.dashboard(1L);
|
||||||
|
assertEquals(1L, vo.getPendingCount());
|
||||||
|
assertEquals(1L, vo.getRunningCount());
|
||||||
|
assertEquals(2L, vo.getSuccessCount());
|
||||||
|
assertEquals(1L, vo.getFailedCount());
|
||||||
|
String recent = "";
|
||||||
|
for (PublishTaskDetailVo item : vo.getRecent()) {
|
||||||
|
recent += renderTaskDetail(item);
|
||||||
|
}
|
||||||
|
assertEquals(baselineDetails(), recent.toString(),
|
||||||
|
"dashboard recent 明细与逐条基准完全一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_result_snapshot_equiv() throws Exception {
|
||||||
|
seed();
|
||||||
|
assertEquals(baselineDetails(), resultSnapshot(),
|
||||||
|
"结果页逐条装配与逐条基准完全一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_pagination_equiv() throws Exception {
|
||||||
|
seed();
|
||||||
|
PublishHistoryVo limited = service.history(1L, 3);
|
||||||
|
assertEquals(3, limited.getItems().size(), "limit 3 只取最近 3 条");
|
||||||
|
StringBuilder expected = new StringBuilder("total=5\n");
|
||||||
|
for (FileTaskEntity task : orderedTasks().subList(0, 3)) {
|
||||||
|
expected.append(renderTaskDetail(service.getTaskDetail(task.getId(), 1L)));
|
||||||
|
}
|
||||||
|
assertEquals(expected.toString(), renderHistory(limited), "分页输出与基准最近 3 条一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_batch_progress_equiv() throws Exception {
|
||||||
|
seed();
|
||||||
|
assertEquals(baselineDetails(), batchSnapshot(),
|
||||||
|
"progress 批量明细与逐条基准完全一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_reproducible() throws Exception {
|
||||||
|
seed();
|
||||||
|
assertEquals(baselineDetails(), baselineDetails(), "基准可重复");
|
||||||
|
assertEquals(historySnapshot(), historySnapshot(), "批量快照可重复");
|
||||||
|
assertEquals(dashboardSnapshot(), dashboardSnapshot(), "dashboard 快照可重复");
|
||||||
|
assertEquals(resultSnapshot(), resultSnapshot(), "结果快照可重复");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_golden_committed() {
|
||||||
|
assertTrue(GOLDEN.isFile(), "golden 文件必须存在并入库: " + GOLDEN.getAbsolutePath());
|
||||||
|
assertTrue(GOLDEN.length() > 0, "golden 非空");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_covers_all() throws Exception {
|
||||||
|
seed();
|
||||||
|
String golden = readGolden();
|
||||||
|
assertTrue(golden.contains("task id=1"), "golden 覆盖任务 1(成功多文件)");
|
||||||
|
assertTrue(golden.contains("task id=2"), "golden 覆盖任务 2(运行中+失败文件)");
|
||||||
|
assertTrue(golden.contains("task id=3"), "golden 覆盖任务 3(无结果)");
|
||||||
|
assertTrue(golden.contains("task id=4"), "golden 覆盖任务 4(失败+Job 错误)");
|
||||||
|
assertTrue(golden.contains("task id=5"), "golden 覆盖任务 5(成功单文件)");
|
||||||
|
assertTrue(golden.contains("result id=901"), "golden 覆盖成功结果");
|
||||||
|
assertTrue(golden.contains("result id=902"), "golden 覆盖运行中结果");
|
||||||
|
assertTrue(golden.contains("result id=904"), "golden 覆盖失败结果");
|
||||||
|
assertTrue(golden.contains("joberr=assemble failed"), "golden 覆盖 Job 错误信息");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_diff_fails() throws Exception {
|
||||||
|
seed();
|
||||||
|
String original = readGolden();
|
||||||
|
assertTrue(original.contains("task id=1"), "golden 内容合法");
|
||||||
|
try {
|
||||||
|
java.nio.file.Files.writeString(GOLDEN.toPath(), original + "\n# tampered",
|
||||||
|
java.nio.charset.StandardCharsets.UTF_8);
|
||||||
|
AssertionError failure = null;
|
||||||
|
try {
|
||||||
|
assertEquals(original, readGolden(), "篡改后 golden 与原始不一致");
|
||||||
|
} catch (AssertionError ex) {
|
||||||
|
failure = ex;
|
||||||
|
}
|
||||||
|
assertTrue(failure != null, "篡改 golden 后断言应失败");
|
||||||
|
} finally {
|
||||||
|
java.nio.file.Files.writeString(GOLDEN.toPath(), original,
|
||||||
|
java.nio.charset.StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
assertEquals(original, readGolden(), "恢复原始 golden");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_regression_guard() throws Exception {
|
||||||
|
seed();
|
||||||
|
assertEquals(baselineHistoryText(), historySnapshot(), "历史快照与基准一致");
|
||||||
|
assertEquals(baselineDetails(), resultSnapshot(), "结果快照与基准一致");
|
||||||
|
assertEquals(baselineDetails(), batchSnapshot(), "progress 快照与基准一致");
|
||||||
|
assertEquals(readGolden(), historySnapshot(), "整体输出与 golden 快照一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_equiv_fails_on_mutation() throws Exception {
|
||||||
|
seed();
|
||||||
|
String before = historySnapshot();
|
||||||
|
assertEquals(before, historySnapshot());
|
||||||
|
taskDb.getFirst().setStatus("RUNNING");
|
||||||
|
assertNotEquals(before, historySnapshot(), "数据变化后快照必须不同(等价门禁可感知改动)");
|
||||||
|
}
|
||||||
|
}
|
||||||
+321
@@ -0,0 +1,321 @@
|
|||||||
|
package com.nanri.aiimage.modules.publish.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.AbstractWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
|
||||||
|
import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
|
||||||
|
import com.nanri.aiimage.modules.publish.model.vo.PublishDashboardVo;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.doAnswer;
|
||||||
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 108:dashboard 聚合查询。
|
||||||
|
* dashboard 状态统计由 4 次逐条 selectCount 改为 1 次 GROUP BY 聚合(selectMaps status→count);
|
||||||
|
* 输出结构(pendingCount/runningCount/successCount/failedCount/recent)与逐条统计完全一致;
|
||||||
|
* 聚合结果为空时各计数为 0;recent 列表语义不变。
|
||||||
|
*/
|
||||||
|
class PublishDashboardAggregateTest {
|
||||||
|
|
||||||
|
private static final String MODULE = "PUBLISH";
|
||||||
|
|
||||||
|
private final List<FileTaskEntity> taskDb = new ArrayList<>();
|
||||||
|
private final AtomicInteger aggregateCallCount = new AtomicInteger();
|
||||||
|
private final AtomicInteger recentListCallCount = new AtomicInteger();
|
||||||
|
|
||||||
|
private FileTaskMapper fileTaskMapper;
|
||||||
|
private PublishItemMapper publishItemMapper;
|
||||||
|
private PublishFileMapper publishFileMapper;
|
||||||
|
private FileResultMapper fileResultMapper;
|
||||||
|
private TaskFileJobService taskFileJobService;
|
||||||
|
private OssStorageService ossStorageService;
|
||||||
|
|
||||||
|
private PublishTaskService service;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initializeMybatisMetadata() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 从渲染后的 wrapper 参数中提取 userId(先 getSqlSegment 强制渲染,再读 paramNameValuePairs)。 */
|
||||||
|
private static Long userIdOf(Wrapper<FileTaskEntity> wrapper, String segment) {
|
||||||
|
AbstractWrapper<FileTaskEntity, ?, ?> q = (AbstractWrapper<FileTaskEntity, ?, ?>) wrapper;
|
||||||
|
List<Long> numbers = new ArrayList<>();
|
||||||
|
for (Object value : q.getParamNameValuePairs().values()) {
|
||||||
|
if (value instanceof Number n) {
|
||||||
|
numbers.add(n.longValue());
|
||||||
|
} else if (value instanceof Iterable<?> iterable) {
|
||||||
|
for (Object item : iterable) {
|
||||||
|
if (item instanceof Number n) {
|
||||||
|
numbers.add(n.longValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return numbers.stream().filter(Long.class::isInstance).map(Long.class::cast).findFirst().orElse(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
publishItemMapper = mock(PublishItemMapper.class);
|
||||||
|
publishFileMapper = mock(PublishFileMapper.class);
|
||||||
|
fileResultMapper = mock(FileResultMapper.class);
|
||||||
|
taskFileJobService = mock(TaskFileJobService.class);
|
||||||
|
ossStorageService = mock(OssStorageService.class);
|
||||||
|
|
||||||
|
// 聚合查询:一次 selectMaps,按模块+用户过滤后 status→count
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
aggregateCallCount.incrementAndGet();
|
||||||
|
Wrapper<FileTaskEntity> wrapper = invocation.getArgument(0);
|
||||||
|
String segment = wrapper.getSqlSegment() == null ? "" : wrapper.getSqlSegment();
|
||||||
|
assertTrue(segment.toLowerCase().contains("group by"), "聚合查询必须带 GROUP BY: " + segment);
|
||||||
|
Long userId = userIdOf(wrapper, segment);
|
||||||
|
Map<String, Long> counts = new HashMap<>();
|
||||||
|
for (FileTaskEntity task : taskDb) {
|
||||||
|
if (!MODULE.equals(task.getModuleType()) || task.getUserId() == null || task.getStatus() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (userId != null && !userId.equals(task.getUserId())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
counts.merge(task.getStatus(), 1L, Long::sum);
|
||||||
|
}
|
||||||
|
List<Map<String, Object>> rows = new ArrayList<>();
|
||||||
|
for (Map.Entry<String, Long> entry : counts.entrySet()) {
|
||||||
|
Map<String, Object> row = new HashMap<>();
|
||||||
|
row.put("status", entry.getKey());
|
||||||
|
row.put("cnt", entry.getValue());
|
||||||
|
rows.add(row);
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}).when(fileTaskMapper).selectMaps(any());
|
||||||
|
|
||||||
|
// recent 列表:history() 依赖的明细查询
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
recentListCallCount.incrementAndGet();
|
||||||
|
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
|
||||||
|
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
|
||||||
|
Long userId = userIdOf(q, segment);
|
||||||
|
List<FileTaskEntity> filtered = new ArrayList<>();
|
||||||
|
for (FileTaskEntity task : taskDb) {
|
||||||
|
if (!MODULE.equals(task.getModuleType()) || task.getUserId() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (userId != null && !userId.equals(task.getUserId())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
filtered.add(task);
|
||||||
|
}
|
||||||
|
filtered.sort((a, b) -> b.getCreatedAt().compareTo(a.getCreatedAt()));
|
||||||
|
if (segment.toLowerCase().contains("limit")) {
|
||||||
|
int cap = 10;
|
||||||
|
java.util.regex.Matcher m = java.util.regex.Pattern
|
||||||
|
.compile("limit\\s+(\\d+)", java.util.regex.Pattern.CASE_INSENSITIVE).matcher(segment);
|
||||||
|
if (m.find()) {
|
||||||
|
cap = Integer.parseInt(m.group(1));
|
||||||
|
}
|
||||||
|
return filtered.subList(0, Math.min(cap, filtered.size()));
|
||||||
|
}
|
||||||
|
return filtered;
|
||||||
|
}).when(fileTaskMapper).selectList(any());
|
||||||
|
|
||||||
|
lenient().when(publishFileMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
lenient().when(taskFileJobService.findAssembleJobsByResultIds(any(), any())).thenReturn(Map.of());
|
||||||
|
lenient().when(ossStorageService.generateFreshDownloadUrl(any())).thenReturn("https://oss/x");
|
||||||
|
|
||||||
|
service = new PublishTaskService(
|
||||||
|
mock(LocalFileStorageService.class), mock(ZiniaoShopSwitchService.class),
|
||||||
|
mock(PublishWorkbookService.class), publishFileMapper, publishItemMapper,
|
||||||
|
fileTaskMapper, fileResultMapper, mock(TaskChunkMapper.class),
|
||||||
|
mock(TaskScopeStateMapper.class), taskFileJobService,
|
||||||
|
mock(TaskDistributedLockService.class), mock(TransientPayloadStorageService.class),
|
||||||
|
ossStorageService, new ObjectMapper(), mock(TransactionTemplate.class),
|
||||||
|
mock(InstanceMetadata.class),
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity task(long id, long userId, String status, LocalDateTime createdAt) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(id);
|
||||||
|
task.setUserId(userId);
|
||||||
|
task.setModuleType(MODULE);
|
||||||
|
task.setStatus(status);
|
||||||
|
task.setCreatedAt(createdAt);
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void seed() {
|
||||||
|
taskDb.add(task(1L, 7L, "PENDING", LocalDateTime.of(2026, 8, 1, 10, 1)));
|
||||||
|
taskDb.add(task(2L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 2)));
|
||||||
|
taskDb.add(task(3L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 3)));
|
||||||
|
taskDb.add(task(4L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 4)));
|
||||||
|
taskDb.add(task(5L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 5)));
|
||||||
|
taskDb.add(task(6L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 6)));
|
||||||
|
taskDb.add(task(7L, 7L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 7)));
|
||||||
|
taskDb.add(task(8L, 7L, "FAILED", LocalDateTime.of(2026, 8, 1, 10, 8)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardCounts() {
|
||||||
|
seed();
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertEquals(1L, vo.getPendingCount());
|
||||||
|
assertEquals(2L, vo.getRunningCount());
|
||||||
|
assertEquals(3L, vo.getSuccessCount());
|
||||||
|
assertEquals(2L, vo.getFailedCount());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardEmptyUser() {
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertEquals(0L, vo.getPendingCount());
|
||||||
|
assertEquals(0L, vo.getRunningCount());
|
||||||
|
assertEquals(0L, vo.getSuccessCount());
|
||||||
|
assertEquals(0L, vo.getFailedCount());
|
||||||
|
assertTrue(vo.getRecent().isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardAggregateSql() {
|
||||||
|
seed();
|
||||||
|
service.dashboard(7L);
|
||||||
|
assertEquals(1, aggregateCallCount.get(), "状态统计一次 GROUP BY 聚合,不逐条 COUNT");
|
||||||
|
verify(fileTaskMapper, times(1)).selectMaps(any());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardRecentTasks() {
|
||||||
|
seed();
|
||||||
|
taskDb.add(task(9L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 9)));
|
||||||
|
taskDb.add(task(10L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 10)));
|
||||||
|
taskDb.add(task(11L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 11)));
|
||||||
|
taskDb.add(task(12L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 12)));
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertEquals(10, vo.getRecent().size(), "recent 固定最多 10 条");
|
||||||
|
assertEquals(12L, vo.getRecent().getFirst().getTask().getId(), "最近任务按 createdAt 倒序");
|
||||||
|
assertEquals(3L, vo.getRecent().getLast().getTask().getId(), "超出 10 条的最旧任务被截断");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardConsistency() {
|
||||||
|
seed();
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
Map<String, Long> perStatus = new HashMap<>();
|
||||||
|
for (FileTaskEntity task : taskDb) {
|
||||||
|
perStatus.merge(task.getStatus(), 1L, Long::sum);
|
||||||
|
}
|
||||||
|
assertEquals(perStatus.getOrDefault("PENDING", 0L).longValue(), vo.getPendingCount(), "与逐条统计一致");
|
||||||
|
assertEquals(perStatus.getOrDefault("RUNNING", 0L).longValue(), vo.getRunningCount());
|
||||||
|
assertEquals(perStatus.getOrDefault("SUCCESS", 0L).longValue(), vo.getSuccessCount());
|
||||||
|
assertEquals(perStatus.getOrDefault("FAILED", 0L).longValue(), vo.getFailedCount());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardMixedStatus() {
|
||||||
|
taskDb.add(task(1L, 7L, "PENDING", LocalDateTime.of(2026, 8, 1, 9, 0)));
|
||||||
|
taskDb.add(task(2L, 7L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 1)));
|
||||||
|
taskDb.add(task(3L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 2)));
|
||||||
|
taskDb.add(task(4L, 7L, "FAILED", LocalDateTime.of(2026, 8, 1, 9, 3)));
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertEquals(1L, vo.getPendingCount());
|
||||||
|
assertEquals(1L, vo.getRunningCount());
|
||||||
|
assertEquals(1L, vo.getSuccessCount());
|
||||||
|
assertEquals(1L, vo.getFailedCount());
|
||||||
|
assertEquals(4, vo.getRecent().size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardUserFiltered() {
|
||||||
|
seed();
|
||||||
|
taskDb.add(task(9L, 8L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 11, 0)));
|
||||||
|
taskDb.add(task(10L, 8L, "FAILED", LocalDateTime.of(2026, 8, 1, 11, 1)));
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertEquals(3L, vo.getSuccessCount(), "只统计当前用户任务");
|
||||||
|
assertEquals(2L, vo.getFailedCount());
|
||||||
|
assertEquals(8, vo.getRecent().size(), "recent 只含当前用户任务");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardBigData() {
|
||||||
|
for (int i = 1; i <= 500; i++) {
|
||||||
|
String status = i % 4 == 0 ? "SUCCESS" : (i % 4 == 1 ? "FAILED" : (i % 4 == 2 ? "RUNNING" : "PENDING"));
|
||||||
|
taskDb.add(task(1000L + i, 7L, status, LocalDateTime.of(2026, 8, 1, 0, 0).plusMinutes(i)));
|
||||||
|
}
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertEquals(125L, vo.getPendingCount());
|
||||||
|
assertEquals(125L, vo.getRunningCount());
|
||||||
|
assertEquals(125L, vo.getSuccessCount());
|
||||||
|
assertEquals(125L, vo.getFailedCount());
|
||||||
|
assertEquals(1, aggregateCallCount.get(), "大数据量仍一次聚合");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardOtherModuleExcluded() {
|
||||||
|
seed();
|
||||||
|
taskDb.add(task(99L, 7L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 12, 0)));
|
||||||
|
taskDb.getLast().setModuleType("SIMILAR_ASIN");
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertEquals(3L, vo.getSuccessCount(), "其他模块任务不计入");
|
||||||
|
assertEquals(8, vo.getRecent().size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void dashboardRecentDetailAssembled() {
|
||||||
|
seed();
|
||||||
|
PublishDashboardVo vo = service.dashboard(7L);
|
||||||
|
assertNotNull(vo.getRecent().getFirst().getTask(), "recent 明细含 task 信息");
|
||||||
|
assertEquals("FAILED", vo.getRecent().getFirst().getTask().getStatus());
|
||||||
|
assertEquals(8L, vo.getRecent().getFirst().getTask().getId());
|
||||||
|
assertEquals(1, aggregateCallCount.get());
|
||||||
|
assertEquals(1, recentListCallCount.get(), "recent 列表一次明细查询");
|
||||||
|
}
|
||||||
|
}
|
||||||
+3
-1
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -135,7 +136,8 @@ class ShopDataCrawlChunkUpsertTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
storedChunks.clear();
|
storedChunks.clear();
|
||||||
storedScopes.clear();
|
storedScopes.clear();
|
||||||
|
|||||||
+3
-1
@@ -28,6 +28,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -170,7 +171,8 @@ class ShopDataCrawlCleanupTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||||
|
|
||||||
dbResultRows.clear();
|
dbResultRows.clear();
|
||||||
|
|||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -144,7 +145,8 @@ class ShopDataCrawlDailyFileIncrementalTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
dbResultRows.clear();
|
dbResultRows.clear();
|
||||||
dbDailyFiles.clear();
|
dbDailyFiles.clear();
|
||||||
|
|||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -150,7 +151,8 @@ class ShopDataCrawlDailyFileJobSplitTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
dbResultRows.clear();
|
dbResultRows.clear();
|
||||||
dbDailyFiles.clear();
|
dbDailyFiles.clear();
|
||||||
|
|||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -150,7 +151,8 @@ class ShopDataCrawlDailyFileLockTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
dbResultRows.clear();
|
dbResultRows.clear();
|
||||||
dbDailyFiles.clear();
|
dbDailyFiles.clear();
|
||||||
|
|||||||
+240
@@ -0,0 +1,240 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlRowNormalizer;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.service.support.ShopDataCrawlSheetBuilder;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.lang.reflect.InvocationTargetException;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 101:ShopDataCrawl 门面改委托。
|
||||||
|
* 门面(ShopDataCrawlExcelAssemblyService)签名与行为不变:writeWorkbook / writeWorkbookStreaming /
|
||||||
|
* replaceCountriesWorkbook / countRows 仍产出与抽取前一致的 workbook;Sheet 构造与行分组
|
||||||
|
* 全部委托 ShopDataCrawlSheetBuilder(私有方法经反射直接验证,不经过完整任务链路)。
|
||||||
|
*/
|
||||||
|
class ShopDataCrawlExcelAssemblyServiceDelegationTest {
|
||||||
|
@TempDir Path tempDir;
|
||||||
|
|
||||||
|
// ---------- 1 签名不变 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_workbook_signature_unchanged() throws Exception {
|
||||||
|
Method method = ShopDataCrawlExcelAssemblyService.class.getDeclaredMethod(
|
||||||
|
"writeWorkbook", File.class, List.class);
|
||||||
|
assertEquals(int.class, method.getReturnType(), "返回类型不变");
|
||||||
|
assertEquals(2, method.getParameterCount(), "参数个数不变");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_workbook_streaming_signature_unchanged() throws Exception {
|
||||||
|
Method method = ShopDataCrawlExcelAssemblyService.class.getDeclaredMethod(
|
||||||
|
"writeWorkbookStreaming", File.class, List.class, int.class);
|
||||||
|
assertEquals(int.class, method.getReturnType(), "返回类型不变");
|
||||||
|
assertEquals(3, method.getParameterCount(), "参数个数不变");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 2 委托各组件 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_streaming_sheet_headers_delegated_to_builder() throws Exception {
|
||||||
|
ShopDataCrawlResultItemVo item = item("UK", row("2026-07-25", "B012345678"));
|
||||||
|
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||||
|
File output = tempDir.resolve("stream.xlsx").toFile();
|
||||||
|
|
||||||
|
int count = new ShopDataCrawlExcelAssemblyService(imageEmbedder)
|
||||||
|
.writeWorkbookStreaming(output, List.of(item), 100);
|
||||||
|
|
||||||
|
assertEquals(1, count, "数据行数委托 rowsByCountry");
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
|
||||||
|
assertEquals(ShopDataCrawlSheetBuilder.SHEETS, sheetNames(wb), "sheet 名来自 SheetBuilder");
|
||||||
|
for (int i = 0; i < wb.getNumberOfSheets(); i++) {
|
||||||
|
Row header = wb.getSheetAt(i).getRow(0);
|
||||||
|
for (int column = 0; column < ShopDataCrawlSheetBuilder.HEADERS.size(); column++) {
|
||||||
|
assertEquals(ShopDataCrawlSheetBuilder.HEADERS.get(column),
|
||||||
|
header.getCell(column).getStringCellValue(), "表头来自 SheetBuilder");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Row data = wb.getSheet("英国").getRow(1);
|
||||||
|
assertEquals("2026-07-25", data.getCell(0).getStringCellValue());
|
||||||
|
assertEquals("B012345678", data.getCell(1).getStringCellValue());
|
||||||
|
assertEquals("Example Brand", data.getCell(9).getStringCellValue(), "品牌末列");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_rows_by_country_delegates_builder_grouping() throws Exception {
|
||||||
|
ShopDataCrawlResultItemVo item = item("uk", row("2026-07-25", "B012345678"));
|
||||||
|
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||||
|
File output = tempDir.resolve("group.xlsx").toFile();
|
||||||
|
|
||||||
|
new ShopDataCrawlExcelAssemblyService(imageEmbedder).writeWorkbookStreaming(output, List.of(item), 100);
|
||||||
|
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
|
||||||
|
assertEquals("B012345678", wb.getSheetAt(0).getRow(1).getCell(1).getStringCellValue(),
|
||||||
|
"小写国家码归一化后归入英国 sheet");
|
||||||
|
assertEquals(0, wb.getSheetAt(1).getLastRowNum(), "德国 sheet 无数据");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_workbook_template_path_kept_in_facade() throws Exception {
|
||||||
|
// 模板路径仍由门面加载:写出的 workbook 保留模板样式路径(图片列宽 18*256)
|
||||||
|
ShopDataCrawlResultItemVo item = item("DE", row("2026-07-25", "B012345678"));
|
||||||
|
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||||
|
File output = tempDir.resolve("tpl.xlsx").toFile();
|
||||||
|
|
||||||
|
int count = new ShopDataCrawlExcelAssemblyService(imageEmbedder).writeWorkbook(output, List.of(item));
|
||||||
|
|
||||||
|
assertEquals(1, count);
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
|
||||||
|
assertEquals(18 * 256, wb.getSheetAt(1).getColumnWidth(2), "模板路径保留图片列宽");
|
||||||
|
assertEquals("B012345678", wb.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_replace_countries_delegates_sheet_builder() throws Exception {
|
||||||
|
ShopDataCrawlRowDto ukRow = row("2026-07-25", "B012345678");
|
||||||
|
ShopDataCrawlRowDto deRow = row("2026-07-26", "B099999999");
|
||||||
|
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||||
|
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
|
||||||
|
File base = tempDir.resolve("base.xlsx").toFile();
|
||||||
|
File output = tempDir.resolve("daily.xlsx").toFile();
|
||||||
|
service.writeWorkbook(base, List.of(item("UK", ukRow), item("DE", deRow)));
|
||||||
|
|
||||||
|
int total = service.replaceCountriesWorkbook(base, output, List.of(item("DE", row("2026-07-27", "B099999998"))));
|
||||||
|
|
||||||
|
assertEquals(2, total, "替换后总行数(UK 1 + DE 1)");
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook(new FileInputStream(output))) {
|
||||||
|
assertEquals("B012345678", wb.getSheetAt(0).getRow(1).getCell(1).getStringCellValue(), "未替换国家保留");
|
||||||
|
assertEquals("B099999998", wb.getSheetAt(1).getRow(1).getCell(1).getStringCellValue(), "替换国家新行");
|
||||||
|
assertEquals(0, wb.getSheetAt(2).getLastRowNum());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 3 结果一致 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_and_streaming_produce_same_row_count() throws Exception {
|
||||||
|
ShopDataCrawlResultItemVo item = item("UK", row("2026-07-25", "B012345678"));
|
||||||
|
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||||
|
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
|
||||||
|
File tplOutput = tempDir.resolve("tpl.xlsx").toFile();
|
||||||
|
File streamOutput = tempDir.resolve("stream.xlsx").toFile();
|
||||||
|
|
||||||
|
int tplCount = service.writeWorkbook(tplOutput, List.of(item));
|
||||||
|
int streamCount = service.writeWorkbookStreaming(streamOutput, List.of(item), 100);
|
||||||
|
|
||||||
|
assertEquals(tplCount, streamCount, "模板路径与流式路径行数一致");
|
||||||
|
try (XSSFWorkbook tpl = new XSSFWorkbook(new FileInputStream(tplOutput));
|
||||||
|
XSSFWorkbook stream = new XSSFWorkbook(new FileInputStream(streamOutput))) {
|
||||||
|
assertEquals(tpl.getSheet("英国").getRow(1).getCell(1).getStringCellValue(),
|
||||||
|
stream.getSheet("英国").getRow(1).getCell(1).getStringCellValue(), "数据内容一致");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_count_rows_matches_write_row_count() throws Exception {
|
||||||
|
ShopDataCrawlResultItemVo uk = item("UK", row("2026-07-25", "B01"));
|
||||||
|
ShopDataCrawlResultItemVo de = item("DE", row("2026-07-25", "B02"));
|
||||||
|
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||||
|
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
|
||||||
|
|
||||||
|
assertEquals(2, service.countRows(List.of(uk, de)), "countRows 委托分组计数");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 4 异常一致 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_streaming_null_output_throws_business_exception() {
|
||||||
|
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(mock(SimilarAsinImageEmbedder.class));
|
||||||
|
com.nanri.aiimage.common.exception.BusinessException ex =
|
||||||
|
org.junit.jupiter.api.Assertions.assertThrows(com.nanri.aiimage.common.exception.BusinessException.class,
|
||||||
|
() -> service.writeWorkbookStreaming(null, List.of(), 100));
|
||||||
|
assertTrue(ex.getMessage().contains("输出文件路径不能为空"), "实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_streaming_invalid_window_throws_illegal_argument() {
|
||||||
|
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(mock(SimilarAsinImageEmbedder.class));
|
||||||
|
org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> service.writeWorkbookStreaming(tempDir.resolve("w.xlsx").toFile(), List.of(), 0),
|
||||||
|
"rowAccessWindow 校验不变");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 5 行分组/归一化语义由组件承接 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_grouping_semantics_kept_in_sheet_builder() {
|
||||||
|
ShopDataCrawlRowDto row = row("2026-07-25", "B012345678");
|
||||||
|
ShopDataCrawlResultItemVo item = item("DE", row);
|
||||||
|
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(List.of(item));
|
||||||
|
|
||||||
|
assertEquals(List.of("UK", "DE", "FR", "ES", "IT"), grouped.keySet().stream().toList(), "5 国固定顺序");
|
||||||
|
assertEquals(1, grouped.get("DE").size());
|
||||||
|
assertEquals(0, grouped.get("UK").size());
|
||||||
|
assertEquals(0, ShopDataCrawlSheetBuilder.rowsByCountry(null).get("UK").size(), "null 条目返回空分组");
|
||||||
|
assertEquals(0, ShopDataCrawlSheetBuilder.rowsByCountry(List.of(item("US", row))).get("UK").size(),
|
||||||
|
"未命中 5 国列表的国家被丢弃");
|
||||||
|
assertEquals(0, ShopDataCrawlSheetBuilder.rowsByCountry(List.of(item("DE", row))).get("FR").size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalizer_semantics_unchanged() {
|
||||||
|
assertEquals("", ShopDataCrawlRowNormalizer.normalizeCountry(null));
|
||||||
|
assertEquals("DE", ShopDataCrawlRowNormalizer.normalizeCountry(" 德国 "));
|
||||||
|
assertEquals("B01", ShopDataCrawlRowNormalizer.trim(" B01 "));
|
||||||
|
ShopDataCrawlRowDto a = row("2026-07-25", "B01");
|
||||||
|
ShopDataCrawlRowDto b = row(" 2026-07-25 ", " B01 ");
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.sameRow(a, b), "sameRow trim 语义");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 辅助 ----------
|
||||||
|
|
||||||
|
private static ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
|
||||||
|
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
|
||||||
|
country.setCountry(countryCode);
|
||||||
|
country.setItems(List.of(row));
|
||||||
|
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
|
||||||
|
item.setSuccess(true);
|
||||||
|
item.setCountryResults(List.of(country));
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ShopDataCrawlRowDto row(String date, String asin) {
|
||||||
|
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||||
|
row.setDate(date);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setBrand("Example Brand");
|
||||||
|
row.setInventorySales("11");
|
||||||
|
row.setSalesRank("22");
|
||||||
|
row.setPageViews("33");
|
||||||
|
row.setUnitsSold("44");
|
||||||
|
row.setPrice("12.50");
|
||||||
|
row.setRecommendedOffer("12.00");
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<String> sheetNames(XSSFWorkbook wb) {
|
||||||
|
return java.util.stream.IntStream.range(0, wb.getNumberOfSheets())
|
||||||
|
.mapToObj(i -> wb.getSheetAt(i).getSheetName()).toList();
|
||||||
|
}
|
||||||
|
}
|
||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -132,7 +133,8 @@ class ShopDataCrawlLightweightProgressTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
storedChunks.clear();
|
storedChunks.clear();
|
||||||
storedScopes.clear();
|
storedScopes.clear();
|
||||||
|
|||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||||
@@ -131,7 +132,8 @@ class ShopDataCrawlOwnerColumnTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
dbTasks.clear();
|
dbTasks.clear();
|
||||||
lastScan.clear();
|
lastScan.clear();
|
||||||
|
|||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -144,7 +145,8 @@ class ShopDataCrawlProgressQueryTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
dbResultRows.clear();
|
dbResultRows.clear();
|
||||||
dbTasks.clear();
|
dbTasks.clear();
|
||||||
|
|||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -131,7 +132,8 @@ class ShopDataCrawlRowDedupKeyTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
storedChunks.clear();
|
storedChunks.clear();
|
||||||
storedScopes.clear();
|
storedScopes.clear();
|
||||||
|
|||||||
+3
-1
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -132,7 +133,8 @@ class ShopDataCrawlScopeCounterTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
storedChunks.clear();
|
storedChunks.clear();
|
||||||
storedScopes.clear();
|
storedScopes.clear();
|
||||||
|
|||||||
+3
-1
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -133,7 +134,8 @@ class ShopDataCrawlScopeMergeTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
storedChunks.clear();
|
storedChunks.clear();
|
||||||
storedScopes.clear();
|
storedScopes.clear();
|
||||||
|
|||||||
+3
-1
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -127,7 +128,8 @@ class ShopDataCrawlTaskServiceChunkTest {
|
|||||||
transientPayloadStorageService,
|
transientPayloadStorageService,
|
||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
null);
|
null,
|
||||||
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
storedChunks.clear();
|
storedChunks.clear();
|
||||||
storedScopes.clear();
|
storedScopes.clear();
|
||||||
|
|||||||
+216
@@ -0,0 +1,216 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 103:ShopDataCrawlHistoryAssembler 历史查询组装器。
|
||||||
|
* 历史列表 VO 拼装(toHistoryItem + 文件任务状态链)从 ShopDataCrawlTaskService 原样搬移;
|
||||||
|
* 只读不落库;输出与现状逐字段一致(快照优先,实体字段兜底,task/job 附加)。
|
||||||
|
*/
|
||||||
|
class ShopDataCrawlHistoryAssemblerTest {
|
||||||
|
|
||||||
|
private final ShopDataCrawlHistoryAssembler assembler = new ShopDataCrawlHistoryAssembler();
|
||||||
|
|
||||||
|
private static FileResultEntity result(Long id, Long taskId, String source, String sourceFileUrl, String resultFilename,
|
||||||
|
String resultFileUrl, Integer success, String errorMessage, LocalDateTime createdAt) {
|
||||||
|
FileResultEntity row = new FileResultEntity();
|
||||||
|
row.setId(id);
|
||||||
|
row.setTaskId(taskId);
|
||||||
|
row.setSourceFilename(source);
|
||||||
|
row.setSourceFileUrl(sourceFileUrl);
|
||||||
|
row.setResultFilename(resultFilename);
|
||||||
|
row.setResultFileUrl(resultFileUrl);
|
||||||
|
row.setSuccess(success);
|
||||||
|
row.setErrorMessage(errorMessage);
|
||||||
|
row.setCreatedAt(createdAt);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity task(Long id, String status, LocalDateTime finishedAt) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(id);
|
||||||
|
task.setStatus(status);
|
||||||
|
task.setFinishedAt(finishedAt);
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskFileJobEntity job(Long id, String status, String errorMessage) {
|
||||||
|
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||||
|
job.setId(id);
|
||||||
|
job.setStatus(status);
|
||||||
|
job.setErrorMessage(errorMessage);
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ShopDataCrawlResultItemVo snapshot(String shopName, String shopId, String outputFilename,
|
||||||
|
String error, String taskStatus) {
|
||||||
|
ShopDataCrawlResultItemVo snapshot = new ShopDataCrawlResultItemVo();
|
||||||
|
snapshot.setShopName(shopName);
|
||||||
|
snapshot.setShopId(shopId);
|
||||||
|
snapshot.setOutputFilename(outputFilename);
|
||||||
|
snapshot.setError(error);
|
||||||
|
snapshot.setTaskStatus(taskStatus);
|
||||||
|
return snapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- toHistoryItem 字段拼装 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_history_items() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", "https://src/10/a.xlsx", "a-result.xlsx",
|
||||||
|
"result/10/a.xlsx", 1, null, LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, task(10L, "SUCCESS",
|
||||||
|
LocalDateTime.of(2026, 8, 1, 9, 5)), snapshot("快照店", "S1", "snap.xlsx", null, null), null);
|
||||||
|
|
||||||
|
assertEquals(100L, item.getResultId());
|
||||||
|
assertEquals(10L, item.getTaskId());
|
||||||
|
assertEquals("快照店", item.getShopName(), "快照 shopName 优先");
|
||||||
|
assertEquals("S1", item.getShopId(), "快照 shopId 优先");
|
||||||
|
assertEquals("SUCCESS", item.getTaskStatus(), "task 状态覆盖快照");
|
||||||
|
assertEquals(Boolean.TRUE, item.getSuccess(), "success=1 即成功");
|
||||||
|
assertEquals(LocalDateTime.of(2026, 8, 1, 10, 0), item.getCreatedAt(), "createdAt 取实体");
|
||||||
|
assertEquals(LocalDateTime.of(2026, 8, 1, 9, 5), item.getFinishedAt(), "finishedAt 取 task");
|
||||||
|
assertEquals("snap.xlsx", item.getOutputFilename(), "快照 outputFilename 优先");
|
||||||
|
assertNull(item.getDownloadUrl(), "downloadUrl 恒为空");
|
||||||
|
assertTrue(item.getFileReady(), "文件 URL 就绪");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_snapshot_fallback_fields() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", "https://src/10/a.xlsx", "a-result.xlsx",
|
||||||
|
null, 0, "python timeout", LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, snapshot(null, null, null, "old err", null), null);
|
||||||
|
|
||||||
|
assertEquals("a.xlsx", item.getShopName(), "快照缺 shopName 回退实体 sourceFilename");
|
||||||
|
assertEquals("https://src/10/a.xlsx", item.getShopId(), "快照缺 shopId 回退实体 sourceFileUrl");
|
||||||
|
assertEquals("python timeout", item.getError(), "实体 errorMessage 优先");
|
||||||
|
assertEquals("a-result.xlsx", item.getOutputFilename(), "快照缺 outputFilename 回退实体 resultFilename");
|
||||||
|
assertNull(item.getTaskStatus(), "缺 task 快照 taskStatus 为 null 时保留 null");
|
||||||
|
assertEquals(Boolean.FALSE, item.getSuccess(), "success=0 为失败");
|
||||||
|
assertNull(item.getFinishedAt(), "缺 task 无结束时间");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_null_snapshot_and_null_task() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, null, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, null);
|
||||||
|
|
||||||
|
assertEquals(100L, item.getResultId());
|
||||||
|
assertEquals("a.xlsx", item.getShopName(), "无快照回退实体");
|
||||||
|
assertNull(item.getShopId(), "无快照且实体缺 shopId 为空");
|
||||||
|
assertNull(item.getSuccess(), "success 缺省且快照缺省为 null");
|
||||||
|
assertNull(item.getError());
|
||||||
|
assertFalse(Boolean.TRUE.equals(item.getFileReady()), "无文件 URL 未就绪");
|
||||||
|
assertNull(item.getFileStatus(), "无 job 且文件未就绪状态为空");
|
||||||
|
assertTrue(item.getCountryResults() != null && item.getCountryResults().isEmpty(), "countryResults 非 null 空列表");
|
||||||
|
assertTrue(item.getCountryCodes() != null && item.getCountryCodes().isEmpty(), "countryCodes 非 null 空列表");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_snapshot_mutation_keeps_existing_values() {
|
||||||
|
// 快照实体字段存在时保留(countryResults 原样,不被清空)
|
||||||
|
ShopDataCrawlResultItemVo snapshot = snapshot("店A", "S1", "out.xlsx", null, "RUNNING");
|
||||||
|
snapshot.setCountryResults(new ArrayList<>(List.of()));
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 1, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, snapshot, null);
|
||||||
|
|
||||||
|
assertSame(snapshot, item, "快照非 null 时返回同一实例(原地填充)");
|
||||||
|
assertEquals("店A", item.getShopName());
|
||||||
|
assertEquals("RUNNING", item.getTaskStatus(), "快照 taskStatus 保留");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 文件任务状态链 ----------
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_file_state_job_attached() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 0, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, job(7L, "RUNNING", null));
|
||||||
|
|
||||||
|
assertEquals(7L, item.getFileJobId());
|
||||||
|
assertEquals("RUNNING", item.getFileStatus());
|
||||||
|
assertNull(item.getFileError(), "非 FAILED job 不附错误");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_file_state_failed_job_error_attached() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 0, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, job(9L, "FAILED", "assemble boom"));
|
||||||
|
|
||||||
|
assertEquals("FAILED", item.getFileStatus());
|
||||||
|
assertEquals("assemble boom", item.getFileError(), "job 错误信息附带");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_file_state_no_job_file_ready_success() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, "a-result.xlsx", "result/10/a.xlsx", 1, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, null);
|
||||||
|
|
||||||
|
assertTrue(item.getFileReady(), "文件 URL 就绪");
|
||||||
|
assertEquals("SUCCESS", item.getFileStatus(), "无 job 且文件就绪状态为 SUCCESS");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_country_lists_filled_when_snapshot_missing() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 0, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, null, null, null);
|
||||||
|
|
||||||
|
assertTrue(item.getCountryResults() != null && item.getCountryResults().isEmpty(), "countryResults 空列表");
|
||||||
|
assertTrue(item.getCountryCodes() != null && item.getCountryCodes().isEmpty(), "countryCodes 空列表");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_immutable_input() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, null, 1, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
FileTaskEntity t = task(10L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 5));
|
||||||
|
|
||||||
|
assembler.toHistoryItem(row, t, null, null);
|
||||||
|
|
||||||
|
assertEquals("a.xlsx", row.getSourceFilename(), "result 不被修改");
|
||||||
|
assertEquals("SUCCESS", t.getStatus(), "task 不被修改");
|
||||||
|
assertEquals(LocalDateTime.of(2026, 8, 1, 9, 5), t.getFinishedAt(), "task 不被修改");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_assembler_consistency_with_current_output() {
|
||||||
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, null, "result/10/a.xlsx", 1, null,
|
||||||
|
LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||||
|
|
||||||
|
ShopDataCrawlResultItemVo item = assembler.toHistoryItem(row, task(10L, "FAILED",
|
||||||
|
LocalDateTime.of(2026, 8, 1, 9, 30)), null, null);
|
||||||
|
|
||||||
|
assertEquals("a.xlsx", item.getShopName(), "无快照时 shopName 回退 sourceFilename");
|
||||||
|
assertEquals(Boolean.TRUE, item.getSuccess(), "文件 URL 就绪即成功");
|
||||||
|
assertEquals("FAILED", item.getTaskStatus());
|
||||||
|
assertEquals(LocalDateTime.of(2026, 8, 1, 9, 30), item.getFinishedAt());
|
||||||
|
assertTrue(item.getFileReady(), "文件 URL 就绪");
|
||||||
|
}
|
||||||
|
}
|
||||||
+182
@@ -0,0 +1,182 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 99:ShopDataCrawlRowNormalizer 行解析/归一化器。
|
||||||
|
* 规则与 ShopDataCrawlTaskService.trim / blank / blankToNull / trimToNull /
|
||||||
|
* normalizeCountry / copyRow / rowEmpty / sameRow / rowDedupKey 现状逐字节一致。
|
||||||
|
* 纯函数无状态;rowDedupKey 与 sameRow 的 10 字段 trim 比较语义等价。
|
||||||
|
*/
|
||||||
|
class ShopDataCrawlRowNormalizerTest {
|
||||||
|
|
||||||
|
// ---- trim ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_trim_null_returns_empty() {
|
||||||
|
assertEquals("", ShopDataCrawlRowNormalizer.trim(null), "null 归一化为空串");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_trim_plain_trimming() {
|
||||||
|
assertEquals("B001", ShopDataCrawlRowNormalizer.trim(" B001 "), "trim 去首尾空白");
|
||||||
|
assertEquals("", ShopDataCrawlRowNormalizer.trim(" "), "全空白归空串");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- blank ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_blank_detection() {
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.blank(null), "null 视为空白");
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.blank(""), "空串视为空白");
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.blank(" "), "全空白视为空白");
|
||||||
|
assertFalse(ShopDataCrawlRowNormalizer.blank("DE"), "非空白不视为空白");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- blankToNull / trimToNull ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_blank_to_null() {
|
||||||
|
assertNull(ShopDataCrawlRowNormalizer.blankToNull(null));
|
||||||
|
assertNull(ShopDataCrawlRowNormalizer.blankToNull(" "));
|
||||||
|
assertEquals("DE", ShopDataCrawlRowNormalizer.blankToNull(" DE "), "非空白 trim 返回");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_trim_to_null() {
|
||||||
|
assertNull(ShopDataCrawlRowNormalizer.trimToNull(null));
|
||||||
|
assertNull(ShopDataCrawlRowNormalizer.trimToNull(" "), "trim 后为空返回 null");
|
||||||
|
assertEquals("B001", ShopDataCrawlRowNormalizer.trimToNull(" B001 "));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- normalizeCountry ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_country_chinese_aliases() {
|
||||||
|
assertEquals("DE", ShopDataCrawlRowNormalizer.normalizeCountry("德国"));
|
||||||
|
assertEquals("UK", ShopDataCrawlRowNormalizer.normalizeCountry("英国"));
|
||||||
|
assertEquals("FR", ShopDataCrawlRowNormalizer.normalizeCountry("法国"));
|
||||||
|
assertEquals("IT", ShopDataCrawlRowNormalizer.normalizeCountry("意大利"));
|
||||||
|
assertEquals("ES", ShopDataCrawlRowNormalizer.normalizeCountry("西班牙"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_normalize_country_uppercase_and_fallback() {
|
||||||
|
assertEquals("DE", ShopDataCrawlRowNormalizer.normalizeCountry(" de "), "trim + 大写");
|
||||||
|
assertEquals("US", ShopDataCrawlRowNormalizer.normalizeCountry("us"), "未知国家原样大写");
|
||||||
|
assertEquals("", ShopDataCrawlRowNormalizer.normalizeCountry(null), "null 归空串");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- copyRow ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_copy_row_ten_fields_trimmed() {
|
||||||
|
ShopDataCrawlRowDto source = row(" 2026-07-25 ", " B001 ", " Brand ", " img ",
|
||||||
|
" 10 ", " 20 ", " 30 ", " 40 ", " 50 ", " 60 ");
|
||||||
|
|
||||||
|
ShopDataCrawlRowDto copy = ShopDataCrawlRowNormalizer.copyRow(source);
|
||||||
|
|
||||||
|
assertEquals("2026-07-25", copy.getDate());
|
||||||
|
assertEquals("B001", copy.getAsin());
|
||||||
|
assertEquals("Brand", copy.getBrand());
|
||||||
|
assertEquals("img", copy.getCommodityImage());
|
||||||
|
assertEquals("10", copy.getInventorySales());
|
||||||
|
assertEquals("20", copy.getSalesRank());
|
||||||
|
assertEquals("30", copy.getPageViews());
|
||||||
|
assertEquals("40", copy.getUnitsSold());
|
||||||
|
assertEquals("50", copy.getPrice());
|
||||||
|
assertEquals("60", copy.getRecommendedOffer());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_copy_row_does_not_mutate_source() {
|
||||||
|
ShopDataCrawlRowDto source = row(" 2026-07-25 ", " B001 ", " B ", " i ",
|
||||||
|
" 1 ", " 2 ", " 3 ", " 4 ", " 5 ", " 6 ");
|
||||||
|
|
||||||
|
ShopDataCrawlRowNormalizer.copyRow(source);
|
||||||
|
|
||||||
|
assertEquals(" 2026-07-25 ", source.getDate(), "source 不被修改");
|
||||||
|
assertEquals(" B001 ", source.getAsin());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- rowEmpty ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_row_empty_detection() {
|
||||||
|
ShopDataCrawlRowDto blank = new ShopDataCrawlRowDto();
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.rowEmpty(null), "null 行视为空");
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.rowEmpty(blank), "全字段空白视为空");
|
||||||
|
|
||||||
|
blank.setAsin(" ");
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.rowEmpty(blank), "空白 asin 不算有内容");
|
||||||
|
|
||||||
|
blank.setAsin("B001");
|
||||||
|
assertFalse(ShopDataCrawlRowNormalizer.rowEmpty(blank), "任一字段有值即非空");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- sameRow ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_same_row_compares_ten_trimmed_fields() {
|
||||||
|
ShopDataCrawlRowDto a = row("2026-07-25", "B001", "Brand", "img",
|
||||||
|
"10", "20", "30", "40", "50", "60");
|
||||||
|
ShopDataCrawlRowDto b = row(" 2026-07-25 ", " B001 ", " Brand ", " img ",
|
||||||
|
" 10 ", " 20 ", " 30 ", " 40 ", " 50 ", " 60 ");
|
||||||
|
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.sameRow(a, b), "trim 后语义相同");
|
||||||
|
b.setPrice("51");
|
||||||
|
assertFalse(ShopDataCrawlRowNormalizer.sameRow(a, b), "单字段差异即不同");
|
||||||
|
assertFalse(ShopDataCrawlRowNormalizer.sameRow(null, a), "null 行不同");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- rowDedupKey ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_row_dedup_key_semantics_equivalent_to_same_row() {
|
||||||
|
ShopDataCrawlRowDto a = row("2026-07-25", "B001", "Brand", "img",
|
||||||
|
"10", "20", "30", "40", "50", "60");
|
||||||
|
ShopDataCrawlRowDto b = row(" 2026-07-25 ", " B001 ", " Brand ", " img ",
|
||||||
|
" 10 ", " 20 ", " 30 ", " 40 ", " 50 ", " 60 ");
|
||||||
|
|
||||||
|
assertEquals(ShopDataCrawlRowNormalizer.rowDedupKey(a),
|
||||||
|
ShopDataCrawlRowNormalizer.rowDedupKey(b), "语义相同行键相等");
|
||||||
|
assertNull(ShopDataCrawlRowNormalizer.rowDedupKey(null), "null 行键为 null");
|
||||||
|
|
||||||
|
b.setUnitsSold("41");
|
||||||
|
assertNotEquals(ShopDataCrawlRowNormalizer.rowDedupKey(a),
|
||||||
|
ShopDataCrawlRowNormalizer.rowDedupKey(b), "任一字段差异键不同");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_row_dedup_key_stable_order() {
|
||||||
|
ShopDataCrawlRowDto a = row("2026-07-25", "B001", "Brand", "img",
|
||||||
|
"10", "20", "30", "40", "50", "60");
|
||||||
|
assertEquals(ShopDataCrawlRowNormalizer.rowDedupKey(a),
|
||||||
|
ShopDataCrawlRowNormalizer.rowDedupKey(a), "同一行键稳定");
|
||||||
|
assertTrue(ShopDataCrawlRowNormalizer.rowDedupKey(a).contains("B001"), "键含字段值");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ShopDataCrawlRowDto row(String date, String asin, String brand, String image,
|
||||||
|
String inventory, String rank, String views, String units,
|
||||||
|
String price, String offer) {
|
||||||
|
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||||
|
row.setDate(date);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setBrand(brand);
|
||||||
|
row.setCommodityImage(image);
|
||||||
|
row.setInventorySales(inventory);
|
||||||
|
row.setSalesRank(rank);
|
||||||
|
row.setPageViews(views);
|
||||||
|
row.setUnitsSold(units);
|
||||||
|
row.setPrice(price);
|
||||||
|
row.setRecommendedOffer(offer);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
}
|
||||||
+200
@@ -0,0 +1,200 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
|
import org.apache.poi.ss.usermodel.CellStyle;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 100:ShopDataCrawlSheetBuilder Sheet 构造器。
|
||||||
|
* 结果 Workbook/Sheet 构造辅助(表头、列序、样式、行值派生、模板校验)。
|
||||||
|
* 与现状 ShopDataCrawlExcelAssemblyService 一致:5 个国家工作表、10 列表头(品牌末列)、
|
||||||
|
* 图片列 2 宽 18*256、模板列映射(legacy/current/带品牌变体)。不落库、无 IO 依赖。
|
||||||
|
*/
|
||||||
|
class ShopDataCrawlSheetBuilderTest {
|
||||||
|
|
||||||
|
private static ShopDataCrawlRowDto row(String date, String asin, String brand, String image) {
|
||||||
|
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||||
|
row.setDate(date);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setBrand(brand);
|
||||||
|
row.setCommodityImage(image);
|
||||||
|
row.setInventorySales("11");
|
||||||
|
row.setSalesRank("22");
|
||||||
|
row.setPageViews("33");
|
||||||
|
row.setUnitsSold("44");
|
||||||
|
row.setPrice("12.50");
|
||||||
|
row.setRecommendedOffer("12.00");
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_headers_and_sheet_names() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 1);
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 2);
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 3);
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 4);
|
||||||
|
|
||||||
|
assertEquals(List.of("英国", "德国", "法国", "西班牙", "意大利"), sheetNames(wb), "5 个国家 sheet 顺序");
|
||||||
|
for (int i = 0; i < 5; i++) {
|
||||||
|
Row header = wb.getSheetAt(i).getRow(0);
|
||||||
|
assertEquals(ShopDataCrawlSheetBuilder.HEADERS.size(), header.getLastCellNum(), "第 " + i + " 个 sheet 10 列");
|
||||||
|
for (int column = 0; column < ShopDataCrawlSheetBuilder.HEADERS.size(); column++) {
|
||||||
|
assertEquals(ShopDataCrawlSheetBuilder.HEADERS.get(column),
|
||||||
|
header.getCell(column).getStringCellValue(), "第 " + i + " 个 sheet 第 " + column + " 列表头");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_column_order_brand_last() {
|
||||||
|
assertEquals("品牌", ShopDataCrawlSheetBuilder.HEADERS.get(9), "品牌为最后一列");
|
||||||
|
assertEquals(9, ShopDataCrawlSheetBuilder.BRAND_COLUMN);
|
||||||
|
assertEquals("商品图片", ShopDataCrawlSheetBuilder.HEADERS.get(2), "图片列为第 3 列");
|
||||||
|
assertEquals(2, ShopDataCrawlSheetBuilder.IMAGE_COLUMN);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_streaming_sheet_column_width() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
assertEquals(18 * 256, sheet.getColumnWidth(2), "图片列宽");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_validate_template_accepts_current_brand_template() throws Exception {
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||||
|
for (int i = 0; i < 5; i++) {
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, i);
|
||||||
|
}
|
||||||
|
ShopDataCrawlSheetBuilder.validateTemplate(wb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_validate_template_rejects_wrong_sheet_count() throws Exception {
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
assertThrows(BusinessException.class, () -> ShopDataCrawlSheetBuilder.validateTemplate(wb),
|
||||||
|
"工作表数量不正确");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_validate_template_rejects_wrong_headers() throws Exception {
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||||
|
for (int i = 0; i < 5; i++) {
|
||||||
|
wb.createSheet(ShopDataCrawlSheetBuilder.SHEETS.get(i));
|
||||||
|
}
|
||||||
|
wb.getSheetAt(0).createRow(0).createCell(0).setCellValue("错误表头");
|
||||||
|
assertThrows(BusinessException.class, () -> ShopDataCrawlSheetBuilder.validateTemplate(wb),
|
||||||
|
"表头不正确");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_data_row_values_layout() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
Row row = sheet.createRow(1);
|
||||||
|
ShopDataCrawlSheetBuilder.writeDataRowValues(row, row("2026-07-25", "B012345678", "Brand-X", "https://img/x.jpg"),
|
||||||
|
new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
|
||||||
|
|
||||||
|
assertEquals("2026-07-25", row.getCell(0).getStringCellValue(), "日期列");
|
||||||
|
assertEquals("B012345678", row.getCell(1).getStringCellValue(), "asin 列");
|
||||||
|
assertEquals("", row.getCell(2).getStringCellValue(), "图片列留空由调用方嵌入");
|
||||||
|
assertEquals("11", row.getCell(3).getStringCellValue(), "库存销量列");
|
||||||
|
assertEquals("22", row.getCell(4).getStringCellValue(), "销售排名列");
|
||||||
|
assertEquals("33", row.getCell(5).getStringCellValue(), "页面浏览量列");
|
||||||
|
assertEquals("44", row.getCell(6).getStringCellValue(), "售出件数列");
|
||||||
|
assertEquals("12.50", row.getCell(7).getStringCellValue(), "价格列");
|
||||||
|
assertEquals("12.00", row.getCell(8).getStringCellValue(), "推荐报价列");
|
||||||
|
assertEquals("Brand-X", row.getCell(9).getStringCellValue(), "品牌末列");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_write_data_row_null_values_blank_cells() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
ShopDataCrawlRowDto rowDto = new ShopDataCrawlRowDto();
|
||||||
|
rowDto.setAsin("B01");
|
||||||
|
Row row = sheet.createRow(1);
|
||||||
|
ShopDataCrawlSheetBuilder.writeDataRowValues(row, rowDto, new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
|
||||||
|
|
||||||
|
assertEquals("", row.getCell(0).getStringCellValue(), "null 日期归空串");
|
||||||
|
assertEquals("B01", row.getCell(1).getStringCellValue());
|
||||||
|
assertEquals("", row.getCell(9).getStringCellValue(), "null 品牌归空串");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_template_column_mapping_variants() {
|
||||||
|
assertEquals(0, ShopDataCrawlSheetBuilder.templateColumnForOutput(0, true, true), "日期列恒为 0");
|
||||||
|
assertEquals(2, ShopDataCrawlSheetBuilder.templateColumnForOutput(2, true, true), "current 带品牌图片列不变");
|
||||||
|
assertEquals(3, ShopDataCrawlSheetBuilder.templateColumnForOutput(3, true, true), "current 库存列不变");
|
||||||
|
assertEquals(9, ShopDataCrawlSheetBuilder.templateColumnForOutput(9, true, true), "current 品牌列不变");
|
||||||
|
assertEquals(1, ShopDataCrawlSheetBuilder.templateColumnForOutput(9, true, false), "current 无品牌品牌列映射到 1");
|
||||||
|
assertEquals(2, ShopDataCrawlSheetBuilder.templateColumnForOutput(3, false, false), "legacy 无图片列库存列映射到 2");
|
||||||
|
assertEquals(2, ShopDataCrawlSheetBuilder.templateColumnForOutput(3, false, true), "legacy 带品牌库存列映射到 2");
|
||||||
|
assertEquals(1, ShopDataCrawlSheetBuilder.templateColumnForOutput(9, false, false), "legacy 无品牌品牌列映射到 1");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_clear_data_rows_keeps_header() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
Sheet sheet = ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
Row row = sheet.createRow(1);
|
||||||
|
ShopDataCrawlSheetBuilder.writeDataRowValues(row, row("2026-07-25", "B01", "B", "u"), new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
|
||||||
|
|
||||||
|
ShopDataCrawlSheetBuilder.clearDataRows(sheet);
|
||||||
|
|
||||||
|
assertEquals(0, sheet.getLastRowNum(), "数据行清空、表头保留");
|
||||||
|
assertTrue(sheet.getRow(0) != null, "表头仍在");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_workbook_writable() throws Exception {
|
||||||
|
try (Workbook wb = new XSSFWorkbook()) {
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||||
|
wb.write(out);
|
||||||
|
assertTrue(out.size() > 0, "workbook 可写出");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_total_data_rows() throws Exception {
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 0);
|
||||||
|
ShopDataCrawlSheetBuilder.createStreamingSheet(wb, 1);
|
||||||
|
wb.getSheetAt(0).createRow(1);
|
||||||
|
wb.getSheetAt(0).createRow(2);
|
||||||
|
assertEquals(2, ShopDataCrawlSheetBuilder.totalDataRows(wb), "跨 sheet 数据行计数");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<String> sheetNames(Workbook wb) {
|
||||||
|
return java.util.stream.IntStream.range(0, wb.getNumberOfSheets())
|
||||||
|
.mapToObj(i -> wb.getSheetAt(i).getSheetName()).toList();
|
||||||
|
}
|
||||||
|
}
|
||||||
+254
@@ -0,0 +1,254 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import org.apache.poi.ss.usermodel.CellStyle;
|
||||||
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 任务 102:ShopDataCrawl 快照对比测试。
|
||||||
|
* 夹具 items → 分组快照(rowsByCountry)+ 行布局快照(writeDataRowValues);
|
||||||
|
* golden 文件固定输出,快照变更即失败(防行为漂移)。
|
||||||
|
* 与 05 spec §5 一致:抽取前后同一夹具输出完全一致。
|
||||||
|
* golden 文件:src/test/resources/shopdatacrawl/golden/groups-snapshot.txt
|
||||||
|
* src/test/resources/shopdatacrawl/golden/rows-snapshot.txt
|
||||||
|
*/
|
||||||
|
class ShopDataCrawlSnapshotTest {
|
||||||
|
|
||||||
|
private static final java.io.File GOLDEN_GROUPS =
|
||||||
|
new java.io.File("src/test/resources/shopdatacrawl/golden/groups-snapshot.txt");
|
||||||
|
private static final java.io.File GOLDEN_ROWS =
|
||||||
|
new java.io.File("src/test/resources/shopdatacrawl/golden/rows-snapshot.txt");
|
||||||
|
|
||||||
|
// ---- 夹具 ----
|
||||||
|
|
||||||
|
/** 5 个国家各 1 行 + 未命中国家(US)行 + 失败条目行 + null 条目;items 顺序刻意打乱。 */
|
||||||
|
private static ShopDataCrawlRowDto row(String date, String asin, String brand) {
|
||||||
|
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||||
|
row.setDate(date);
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setBrand(brand);
|
||||||
|
row.setInventorySales("11");
|
||||||
|
row.setSalesRank("22");
|
||||||
|
row.setPageViews("33");
|
||||||
|
row.setUnitsSold("44");
|
||||||
|
row.setPrice("12.50");
|
||||||
|
row.setRecommendedOffer("12.00");
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ShopDataCrawlCountryResultDto country(String code, ShopDataCrawlRowDto... rows) {
|
||||||
|
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
|
||||||
|
country.setCountry(code);
|
||||||
|
country.setItems(List.of(rows));
|
||||||
|
return country;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ShopDataCrawlResultItemVo item(Boolean success, ShopDataCrawlCountryResultDto... countries) {
|
||||||
|
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
|
||||||
|
item.setSuccess(success);
|
||||||
|
item.setCountryResults(List.of(countries));
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 含 null 字段行的全字段夹具:10 列值全部有值 + 部分为 null。 */
|
||||||
|
private static ShopDataCrawlRowDto fullRow() {
|
||||||
|
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||||
|
row.setDate("2026-07-25");
|
||||||
|
row.setAsin("B012345678");
|
||||||
|
row.setBrand("Example Brand");
|
||||||
|
row.setInventorySales("128");
|
||||||
|
row.setSalesRank("#1,245");
|
||||||
|
row.setPageViews("3560");
|
||||||
|
row.setUnitsSold("42");
|
||||||
|
row.setPrice("GBP 19.99");
|
||||||
|
row.setRecommendedOffer("GBP 18.99");
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ShopDataCrawlRowDto sparseRow() {
|
||||||
|
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||||
|
row.setAsin("B099999999");
|
||||||
|
row.setBrand(" Brand With Spaces ");
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<ShopDataCrawlResultItemVo> fixtureMain() {
|
||||||
|
List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
|
||||||
|
items.add(item(Boolean.TRUE, country("DE", row("2026-07-26", "B02", "DE-Brand")),
|
||||||
|
country("US", row("2026-07-26", "B-US", "US-Brand"))));
|
||||||
|
items.add(item(Boolean.TRUE, country("uk", row("2026-07-25", "B01", "UK-Brand"))));
|
||||||
|
items.add(item(null, country("FR", row("2026-07-27", "B03", "FR-Brand"))));
|
||||||
|
items.add(item(Boolean.FALSE, country("ES", row("2026-07-28", "B04", "ES-Brand"))));
|
||||||
|
items.add(item(Boolean.TRUE, country("IT", row("2026-07-29", "B05", "IT-Brand"))));
|
||||||
|
items.add(null);
|
||||||
|
return items;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<ShopDataCrawlResultItemVo> fixtureNullFields() {
|
||||||
|
return List.of(item(Boolean.TRUE, country("UK", fullRow(), sparseRow())));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 快照管线(与服务侧 writeWorkbook 语义一致:rowsByCountry → writeDataRowValues) ----
|
||||||
|
|
||||||
|
private static String renderGroupKey(String country, int count) {
|
||||||
|
return " group: country=" + country + " rows=" + count;
|
||||||
|
}
|
||||||
|
|
||||||
|
static String runGroupsOnly() throws Exception {
|
||||||
|
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureMain());
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (Map.Entry<String, List<ShopDataCrawlRowDto>> entry : grouped.entrySet()) {
|
||||||
|
sb.append(renderGroupKey(entry.getKey(), entry.getValue().size())).append('\n');
|
||||||
|
for (ShopDataCrawlRowDto row : entry.getValue()) {
|
||||||
|
sb.append(" asin=").append(row.getAsin()).append('\n');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
static String runRowsOnly() throws Exception {
|
||||||
|
List<ShopDataCrawlRowDto> rows = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureNullFields()).get("UK");
|
||||||
|
List<String> rowTexts = new ArrayList<>();
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||||
|
Sheet sheet = wb.createSheet("UK");
|
||||||
|
for (ShopDataCrawlRowDto rowDto : rows) {
|
||||||
|
Row row = sheet.createRow(rowTexts.size());
|
||||||
|
ShopDataCrawlSheetBuilder.writeDataRowValues(row, rowDto, new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (int column = 0; column < ShopDataCrawlSheetBuilder.HEADERS.size(); column++) {
|
||||||
|
sb.append(ShopDataCrawlSheetBuilder.HEADERS.get(column)).append('=')
|
||||||
|
.append(row.getCell(column).getStringCellValue()).append(';');
|
||||||
|
}
|
||||||
|
rowTexts.add(sb.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("rows=").append(rows.size()).append('\n');
|
||||||
|
for (String text : rowTexts) {
|
||||||
|
sb.append(" ").append(text).append('\n');
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String read(java.io.File file) throws Exception {
|
||||||
|
return Files.readString(file.toPath(), StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 用例 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_groups_output() throws Exception {
|
||||||
|
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "分组快照与 golden 一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_rows_output() throws Exception {
|
||||||
|
assertEquals(read(GOLDEN_ROWS), runRowsOnly(), "行布局快照与 golden 一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_dtos() throws Exception {
|
||||||
|
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureMain());
|
||||||
|
|
||||||
|
assertEquals(List.of("UK", "DE", "FR", "ES", "IT"),
|
||||||
|
new ArrayList<>(grouped.keySet()), "5 国固定顺序");
|
||||||
|
assertEquals(1, grouped.get("UK").size(), "uk 小写归一化归入英国");
|
||||||
|
assertEquals(1, grouped.get("DE").size());
|
||||||
|
assertEquals(1, grouped.get("FR").size(), "success=null 条目仍计入");
|
||||||
|
assertEquals(0, grouped.get("ES").size(), "success=false 条目丢弃");
|
||||||
|
assertEquals(1, grouped.get("IT").size());
|
||||||
|
assertEquals("B01", grouped.get("UK").get(0).getAsin(), "小写国家码行内容不变");
|
||||||
|
assertEquals("B02", grouped.get("DE").get(0).getAsin(), "US 未命中 5 国被丢弃,不混入 DE");
|
||||||
|
assertEquals(1, grouped.get("DE").size(), "DE 仅含 B02");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_error_cases() throws Exception {
|
||||||
|
Map<String, List<ShopDataCrawlRowDto>> grouped = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureMain());
|
||||||
|
|
||||||
|
assertEquals(0, grouped.get("ES").size(), "失败条目不产生行");
|
||||||
|
assertEquals(4, grouped.values().stream().mapToInt(List::size).sum(), "总行数 = 4 有效国(UK/DE/FR/IT)");
|
||||||
|
Map<String, List<ShopDataCrawlRowDto>> empty = ShopDataCrawlSheetBuilder.rowsByCountry(null);
|
||||||
|
assertEquals(0, empty.get("UK").size(), "null 输入返回空分组");
|
||||||
|
assertEquals(5, empty.size(), "null 输入仍返回 5 国分组");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_reproducible() throws Exception {
|
||||||
|
assertEquals(runGroupsOnly(), runGroupsOnly(), "分组跑两次一致");
|
||||||
|
assertEquals(runRowsOnly(), runRowsOnly(), "行布局跑两次一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_golden_committed() {
|
||||||
|
assertTrue(GOLDEN_GROUPS.isFile(), "golden 文件必须存在并入库: " + GOLDEN_GROUPS.getAbsolutePath());
|
||||||
|
assertTrue(GOLDEN_ROWS.isFile(), "golden 文件必须存在并入库: " + GOLDEN_ROWS.getAbsolutePath());
|
||||||
|
assertTrue(GOLDEN_GROUPS.length() > 0, "groups golden 非空");
|
||||||
|
assertTrue(GOLDEN_ROWS.length() > 0, "rows golden 非空");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_diff_detected() throws Exception {
|
||||||
|
String original = read(GOLDEN_GROUPS);
|
||||||
|
assertTrue(original.contains("rows="), "golden 内容合法");
|
||||||
|
try {
|
||||||
|
Files.writeString(GOLDEN_GROUPS.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
|
||||||
|
AssertionError failure = null;
|
||||||
|
try {
|
||||||
|
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "篡改后应与 golden 不一致");
|
||||||
|
} catch (AssertionError ex) {
|
||||||
|
failure = ex;
|
||||||
|
}
|
||||||
|
assertTrue(failure != null, "篡改 golden 后断言应失败");
|
||||||
|
} finally {
|
||||||
|
Files.writeString(GOLDEN_GROUPS.toPath(), original, StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
assertEquals(original, read(GOLDEN_GROUPS), "恢复原始 golden");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_snapshot_regression_all_pipeline() throws Exception {
|
||||||
|
assertEquals(read(GOLDEN_GROUPS), runGroupsOnly(), "全分组路径与 golden 一致");
|
||||||
|
assertEquals(read(GOLDEN_ROWS), runRowsOnly(), "全行布局路径与 golden 一致");
|
||||||
|
|
||||||
|
List<ShopDataCrawlRowDto> rows = ShopDataCrawlSheetBuilder.rowsByCountry(fixtureNullFields()).get("UK");
|
||||||
|
assertEquals(2, rows.size(), "全字段 + 稀疏行共 2 行");
|
||||||
|
assertEquals("Example Brand", rows.get(0).getBrand(), "全字段行品牌原样");
|
||||||
|
assertEquals(" Brand With Spaces ", rows.get(1).getBrand(), "稀疏行品牌保留原样(写入时归一为空/原样由列布局决定)");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 行布局写入值归一(null → 空串)通过 golden rows 快照覆盖;此处补充列序断言。 */
|
||||||
|
@Test
|
||||||
|
void test_snapshot_column_order_fixed() throws Exception {
|
||||||
|
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||||
|
Sheet sheet = wb.createSheet("UK");
|
||||||
|
Row row = sheet.createRow(0);
|
||||||
|
ShopDataCrawlSheetBuilder.writeDataRowValues(row, fullRow(), new CellStyle[ShopDataCrawlSheetBuilder.HEADERS.size()]);
|
||||||
|
|
||||||
|
assertEquals("2026-07-25", row.getCell(0).getStringCellValue(), "列 0 日期");
|
||||||
|
assertEquals("B012345678", row.getCell(1).getStringCellValue(), "列 1 ASIN");
|
||||||
|
assertEquals("", row.getCell(2).getStringCellValue(), "列 2 商品图片留空");
|
||||||
|
assertEquals("128", row.getCell(3).getStringCellValue(), "列 3 库存销量");
|
||||||
|
assertEquals("#1,245", row.getCell(4).getStringCellValue(), "列 4 销售排名");
|
||||||
|
assertEquals("3560", row.getCell(5).getStringCellValue(), "列 5 页面浏览量");
|
||||||
|
assertEquals("42", row.getCell(6).getStringCellValue(), "列 6 售出件数");
|
||||||
|
assertEquals("GBP 19.99", row.getCell(7).getStringCellValue(), "列 7 价格");
|
||||||
|
assertEquals("GBP 18.99", row.getCell(8).getStringCellValue(), "列 8 推荐报价");
|
||||||
|
assertEquals("Example Brand", row.getCell(9).getStringCellValue(), "列 9 品牌");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+52
@@ -1,5 +1,10 @@
|
|||||||
package com.nanri.aiimage.modules.shopkey.service;
|
package com.nanri.aiimage.modules.shopkey.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.apache.ibatis.session.Configuration;
|
||||||
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper;
|
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper;
|
||||||
import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper;
|
import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper;
|
||||||
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest;
|
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest;
|
||||||
@@ -11,6 +16,7 @@ import org.apache.poi.ss.usermodel.Sheet;
|
|||||||
import org.apache.poi.ss.usermodel.Workbook;
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.mockito.ArgumentCaptor;
|
import org.mockito.ArgumentCaptor;
|
||||||
import org.mockito.InjectMocks;
|
import org.mockito.InjectMocks;
|
||||||
@@ -46,6 +52,13 @@ class SkipPriceAsinServiceTest {
|
|||||||
@InjectMocks
|
@InjectMocks
|
||||||
private SkipPriceAsinService service;
|
private SkipPriceAsinService service;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initMybatisPlusTableInfo() {
|
||||||
|
Configuration configuration = new MybatisConfiguration();
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(configuration, "test");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, SkipPriceAsinEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void createSkipsDuplicateCountryAsinWithoutWriting() {
|
void createSkipsDuplicateCountryAsinWithoutWriting() {
|
||||||
when(shopManageGroupService.getAccessibleById(10L, 7L, true)).thenReturn(group());
|
when(shopManageGroupService.getAccessibleById(10L, 7L, true)).thenReturn(group());
|
||||||
@@ -184,6 +197,45 @@ class SkipPriceAsinServiceTest {
|
|||||||
return request;
|
return request;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void pageAppliesMinimumPriceRangeAcrossAnyCountryWhenCountryAbsent() {
|
||||||
|
when(skipPriceAsinMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
when(skipPriceAsinMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
|
service.page(1, 15, null, null, null, null,
|
||||||
|
new BigDecimal("10.00"), new BigDecimal("20.00"), null, true);
|
||||||
|
|
||||||
|
ArgumentCaptor<Wrapper<SkipPriceAsinEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(skipPriceAsinMapper).selectCount(captor.capture());
|
||||||
|
String sql = captor.getValue().getSqlSegment();
|
||||||
|
org.assertj.core.api.Assertions.assertThat(sql)
|
||||||
|
.contains("minimum_price_de")
|
||||||
|
.contains("minimum_price_uk")
|
||||||
|
.contains("minimum_price_fr")
|
||||||
|
.contains("minimum_price_it")
|
||||||
|
.contains("minimum_price_es")
|
||||||
|
.contains(">= #{")
|
||||||
|
.contains("<= #{")
|
||||||
|
.contains("OR");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void pageAppliesMinimumPriceRangeOnSelectedCountryColumn() {
|
||||||
|
when(skipPriceAsinMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
when(skipPriceAsinMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
|
service.page(1, 15, null, null, null, "DE",
|
||||||
|
new BigDecimal("15.00"), null, null, true);
|
||||||
|
|
||||||
|
ArgumentCaptor<Wrapper<SkipPriceAsinEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(skipPriceAsinMapper).selectCount(captor.capture());
|
||||||
|
String sql = captor.getValue().getSqlSegment();
|
||||||
|
org.assertj.core.api.Assertions.assertThat(sql)
|
||||||
|
.contains("minimum_price_de")
|
||||||
|
.doesNotContain("minimum_price_uk")
|
||||||
|
.contains(">=");
|
||||||
|
}
|
||||||
|
|
||||||
private File importWorkbook(String asin, String minimumPrice) throws Exception {
|
private File importWorkbook(String asin, String minimumPrice) throws Exception {
|
||||||
File file = File.createTempFile("skip-price-asin-test-", ".xlsx");
|
File file = File.createTempFile("skip-price-asin-test-", ".xlsx");
|
||||||
try (Workbook workbook = new XSSFWorkbook();
|
try (Workbook workbook = new XSSFWorkbook();
|
||||||
|
|||||||
-85
@@ -1,85 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import javax.net.ssl.SNIHostName;
|
|
||||||
import javax.net.ssl.SSLContext;
|
|
||||||
import javax.net.ssl.SSLParameters;
|
|
||||||
import javax.net.ssl.SSLSocket;
|
|
||||||
import javax.net.ssl.SSLSocketFactory;
|
|
||||||
import java.net.InetAddress;
|
|
||||||
import java.net.URI;
|
|
||||||
import java.net.http.HttpClient;
|
|
||||||
import java.net.http.HttpRequest;
|
|
||||||
import java.net.http.HttpResponse;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public class LlmGatewayTlsProbe {
|
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
String host = "ai.t8star.org";
|
|
||||||
String apiKey = args[0];
|
|
||||||
|
|
||||||
System.out.println("[probe] java=" + System.getProperty("java.version")
|
|
||||||
+ " tls=" + System.getProperty("java.vm.name"));
|
|
||||||
for (InetAddress a : InetAddress.getAllByName(host)) {
|
|
||||||
System.out.println("[probe] dns " + a);
|
|
||||||
}
|
|
||||||
|
|
||||||
rawHandshake(host, null, "default");
|
|
||||||
rawHandshake(host, "TLSv1.2", "tls12-only");
|
|
||||||
|
|
||||||
httpClientCall(host, apiKey, null, "jdk-http-default");
|
|
||||||
httpClientCall(host, apiKey, "TLSv1.2", "jdk-http-tls12");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void rawHandshake(String host, String protocol, String label) throws Exception {
|
|
||||||
SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
|
|
||||||
try (SSLSocket socket = (SSLSocket) factory.createSocket(host, 443)) {
|
|
||||||
socket.setSoTimeout(15000);
|
|
||||||
SSLParameters params = socket.getSSLParameters();
|
|
||||||
if (protocol != null) {
|
|
||||||
params.setProtocols(new String[]{protocol});
|
|
||||||
}
|
|
||||||
params.setServerNames(List.of(new SNIHostName(host)));
|
|
||||||
socket.setSSLParameters(params);
|
|
||||||
socket.startHandshake();
|
|
||||||
System.out.println("[probe] raw[" + label + "] OK proto=" + socket.getSession().getProtocol()
|
|
||||||
+ " cipher=" + socket.getSession().getCipherSuite());
|
|
||||||
} catch (Exception ex) {
|
|
||||||
System.out.println("[probe] raw[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void httpClientCall(String host, String apiKey, String protocol, String label) throws Exception {
|
|
||||||
HttpClient.Builder builder = HttpClient.newBuilder()
|
|
||||||
.connectTimeout(Duration.ofSeconds(10))
|
|
||||||
.version(HttpClient.Version.HTTP_1_1);
|
|
||||||
if (protocol != null) {
|
|
||||||
SSLContext context = SSLContext.getInstance("TLS");
|
|
||||||
context.init(null, null, null);
|
|
||||||
builder.sslContext(context);
|
|
||||||
}
|
|
||||||
HttpClient client = builder.build();
|
|
||||||
try {
|
|
||||||
String body = "{\"model\":\"gemini-3.5-flash-lite\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":10}";
|
|
||||||
HttpRequest request = HttpRequest.newBuilder()
|
|
||||||
.uri(URI.create("https://" + host + "/v1/chat/completions"))
|
|
||||||
.timeout(Duration.ofSeconds(30))
|
|
||||||
.header("Authorization", "Bearer " + apiKey)
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.POST(HttpRequest.BodyPublishers.ofString(body))
|
|
||||||
.build();
|
|
||||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
|
||||||
String text = response.body();
|
|
||||||
System.out.println("[probe] http[" + label + "] status=" + response.statusCode()
|
|
||||||
+ " body=" + text.substring(0, Math.min(160, text.length())));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
System.out.println("[probe] http[" + label + "] FAIL " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
|
|
||||||
Throwable cause = ex;
|
|
||||||
while (cause.getCause() != null) {
|
|
||||||
cause = cause.getCause();
|
|
||||||
System.out.println("[probe] cause " + cause.getClass().getSimpleName() + ": " + cause.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-226
@@ -1,226 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
|
||||||
import org.junit.jupiter.api.AfterEach;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.mockito.InjectMocks;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
|
||||||
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.Mockito.doAnswer;
|
|
||||||
import static org.mockito.Mockito.lenient;
|
|
||||||
import static org.mockito.Mockito.never;
|
|
||||||
import static org.mockito.Mockito.times;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 15:图片缓存访问时间更新改为异步批量刷新,减少逐图 UPDATE。
|
|
||||||
* lookup 命中不再同步 touchLastUsed,而是进入内存 touch 缓冲(按 url_hash 去重),
|
|
||||||
* 由定时任务/阈值触发 flushPendingTouches 批量 touchLastUsedBatch 刷新;
|
|
||||||
* 缓冲有大小上限,超限立即刷新,不会无界增长;失败 best-effort 吞掉。
|
|
||||||
*/
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
|
||||||
class SimilarAsinImagePrefetchServiceAsyncTouchTest {
|
|
||||||
|
|
||||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
|
||||||
@Mock private TaskImageCacheMapper taskImageCacheMapper;
|
|
||||||
@Mock private SimilarAsinProperties properties;
|
|
||||||
|
|
||||||
@InjectMocks private SimilarAsinImagePrefetchService service;
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
|
|
||||||
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(1000);
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterEach
|
|
||||||
void shutdown() {
|
|
||||||
service.shutdown();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String sha256Hex(String value) throws Exception {
|
|
||||||
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
|
|
||||||
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
|
||||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
|
||||||
for (byte b : digest) {
|
|
||||||
sb.append(String.format("%02x", b & 0xFF));
|
|
||||||
}
|
|
||||||
return sb.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void stubHit(String url, byte[] bytes) throws Exception {
|
|
||||||
when(taskImageCacheMapper.selectBytesByUrlHash(sha256Hex(url))).thenReturn(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_normal_default_path() throws Exception {
|
|
||||||
// 正常输入:命中不立即写 DB,进入缓冲;flush 后批量 touch 一次。
|
|
||||||
String url = "https://img.example.com/hit.jpg";
|
|
||||||
String hash = sha256Hex(url);
|
|
||||||
byte[] bytes = new byte[]{1, 2, 3};
|
|
||||||
stubHit(url, bytes);
|
|
||||||
|
|
||||||
byte[] result = service.lookup(url);
|
|
||||||
assertEquals(bytes, result, "命中必须返回缓存字节");
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
|
||||||
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_normal_multiple_items() throws Exception {
|
|
||||||
// 批量场景:多次命中进入同一缓冲,flush 合并为一次批量 touch,覆盖全部命中。
|
|
||||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
|
|
||||||
"https://img.example.com/c.jpg");
|
|
||||||
Set<String> hashes = new HashSet<>();
|
|
||||||
for (int i = 0; i < urls.size(); i++) {
|
|
||||||
hashes.add(sha256Hex(urls.get(i)));
|
|
||||||
stubHit(urls.get(i), new byte[]{(byte) (i + 1)});
|
|
||||||
}
|
|
||||||
for (String url : urls) {
|
|
||||||
assertNotNull(service.lookup(url), "命中返回字节");
|
|
||||||
}
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
|
||||||
|
|
||||||
service.flushPendingTouches();
|
|
||||||
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
|
|
||||||
assertEquals(hashes, new HashSet<>(captor.getValue()), "一次批量 touch 覆盖全部命中 hash");
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_normal_repeated_operation_is_idempotent() throws Exception {
|
|
||||||
// 重复执行同一输入:同一 url 多次命中只 touch 一次;重复 flush 无多余请求。
|
|
||||||
String url = "https://img.example.com/same.jpg";
|
|
||||||
String hash = sha256Hex(url);
|
|
||||||
stubHit(url, new byte[]{5});
|
|
||||||
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(new byte[]{5});
|
|
||||||
|
|
||||||
service.lookup(url);
|
|
||||||
service.lookup(url);
|
|
||||||
service.lookup(url);
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
|
||||||
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_boundary_empty_input() throws Exception {
|
|
||||||
// 空输入:null/空白 url 不进入缓冲;flush 空缓冲不产生任何数据库访问。
|
|
||||||
assertNull(service.lookup(null));
|
|
||||||
assertNull(service.lookup(""));
|
|
||||||
assertNull(service.lookup(" "));
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_boundary_single_item() throws Exception {
|
|
||||||
// 单条命中:flush 后单元素批量 touch,不依赖批量路径。
|
|
||||||
String url = "https://img.example.com/single.jpg";
|
|
||||||
String hash = sha256Hex(url);
|
|
||||||
stubHit(url, new byte[]{7});
|
|
||||||
|
|
||||||
assertNotNull(service.lookup(url));
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_boundary_limit_and_overflow() throws Exception {
|
|
||||||
// 缓冲达到阈值立即刷新,不无界增长;刷新后继续累积。
|
|
||||||
lenient().when(properties.getImageCacheTouchFlushThreshold()).thenReturn(3);
|
|
||||||
List<String> urls = List.of("https://img.example.com/o1.jpg", "https://img.example.com/o2.jpg",
|
|
||||||
"https://img.example.com/o3.jpg", "https://img.example.com/o4.jpg",
|
|
||||||
"https://img.example.com/o5.jpg");
|
|
||||||
for (String url : urls) {
|
|
||||||
stubHit(url, new byte[]{1});
|
|
||||||
}
|
|
||||||
org.mockito.ArgumentCaptor<List<String>> captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
|
||||||
|
|
||||||
for (int i = 0; i < 3; i++) {
|
|
||||||
service.lookup(urls.get(i));
|
|
||||||
}
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(captor.capture());
|
|
||||||
assertEquals(3, captor.getValue().size(), "第 3 条命中触发阈值立即刷新 3 条");
|
|
||||||
|
|
||||||
service.lookup(urls.get(3));
|
|
||||||
service.lookup(urls.get(4));
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
|
||||||
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
|
|
||||||
assertEquals(2, captor.getValue().size(), "剩余 2 条在 flush 时刷新,缓冲不残留、不无界增长");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_invalid_input_rejected() {
|
|
||||||
// 非法输入:db cache 关闭时 lookup 直接返回 null,不进入缓冲、不访问数据库。
|
|
||||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
|
|
||||||
assertNull(service.lookup("https://img.example.com/a.jpg"));
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHash(anyString());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_015_image_cache_dependency_failure_releases_resources() throws Exception {
|
|
||||||
// 依赖失败:批量 touch 抛异常时吞掉不阻塞 lookup、缓冲已排空无残留;
|
|
||||||
// 恢复后重新入队 flush 成功。
|
|
||||||
String url = "https://img.example.com/fail.jpg";
|
|
||||||
String hash = sha256Hex(url);
|
|
||||||
stubHit(url, new byte[]{3});
|
|
||||||
AtomicInteger failures = new AtomicInteger(0);
|
|
||||||
List<String> capturedArgs = new java.util.ArrayList<>();
|
|
||||||
doAnswer(invocation -> {
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
List<String> arg = (List<String>) invocation.getArgument(0);
|
|
||||||
capturedArgs.addAll(arg);
|
|
||||||
if (failures.getAndIncrement() == 0) {
|
|
||||||
throw new IllegalStateException("db down");
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}).when(taskImageCacheMapper).touchLastUsedBatch(any());
|
|
||||||
|
|
||||||
assertNotNull(service.lookup(url), "touch 失败不阻断 lookup 返回缓存字节");
|
|
||||||
assertThrows(Exception.class, () -> service.flushPendingTouches(), "首次 flush 抛错(由调用方吞掉)");
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
|
||||||
|
|
||||||
assertNotNull(service.lookup(url), "失败后再次命中重新入队");
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(any());
|
|
||||||
assertEquals(List.of(hash, hash), capturedArgs, "两次 touch 都覆盖命中 hash,失败后恢复成功");
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void assertNull(Object value) {
|
|
||||||
org.junit.jupiter.api.Assertions.assertNull(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-297
@@ -1,297 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
|
|
||||||
import org.junit.jupiter.api.AfterEach;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.mockito.InjectMocks;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
|
||||||
|
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
|
||||||
import static org.mockito.Mockito.doAnswer;
|
|
||||||
import static org.mockito.Mockito.lenient;
|
|
||||||
import static org.mockito.Mockito.never;
|
|
||||||
import static org.mockito.Mockito.times;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 14:图片 DB cache 改为批量读取缩略图,并只更新实际命中的 last_used_at。
|
|
||||||
* 批量 lookup 入口(lookupBatch)一次 IN 查询返回命中字节 Map;
|
|
||||||
* last_used_at 只对实际命中的 url_hash 更新(touch 集合 = 命中集合),
|
|
||||||
* 未命中 url 不产生任何 touch/insert。单 URL 旧入口 lookup 语义保持兼容。
|
|
||||||
*/
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
|
||||||
class SimilarAsinImagePrefetchServiceBatchTest {
|
|
||||||
|
|
||||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
|
||||||
@Mock private TaskImageCacheMapper taskImageCacheMapper;
|
|
||||||
@Mock private SimilarAsinProperties properties;
|
|
||||||
|
|
||||||
@InjectMocks private SimilarAsinImagePrefetchService service;
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterEach
|
|
||||||
void shutdown() {
|
|
||||||
service.shutdown();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String sha256Hex(String value) throws Exception {
|
|
||||||
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
|
|
||||||
byte[] digest = md.digest(value.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
|
||||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
|
||||||
for (byte b : digest) {
|
|
||||||
sb.append(String.format("%02x", b & 0xFF));
|
|
||||||
}
|
|
||||||
return sb.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** stub 批量命中读取:只对 hits 集合内的 hash 返回字节。 */
|
|
||||||
private void stubBatchRead(List<String> hits, Map<String, byte[]> bytesByHash) {
|
|
||||||
when(taskImageCacheMapper.selectBytesByUrlHashes(any())).thenAnswer(invocation -> {
|
|
||||||
List<String> hashes = invocation.getArgument(0);
|
|
||||||
List<TaskImageCacheEntity> rows = new ArrayList<>();
|
|
||||||
for (String hash : hashes) {
|
|
||||||
if (hits.contains(hash)) {
|
|
||||||
TaskImageCacheEntity row = new TaskImageCacheEntity();
|
|
||||||
row.setUrlHash(hash);
|
|
||||||
row.setImageBytes(bytesByHash.get(hash));
|
|
||||||
rows.add(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return rows;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 调用批量入口,按 url 顺序返回字节(未命中为 null)。 */
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
private static List<byte[]> invokeLookupBatch(SimilarAsinImagePrefetchService svc, List<String> urls) throws Exception {
|
|
||||||
Method m = SimilarAsinImagePrefetchService.class.getDeclaredMethod("lookupBatch", List.class);
|
|
||||||
m.setAccessible(true);
|
|
||||||
return (List<byte[]>) m.invoke(svc, urls);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_normal_default_path() throws Exception {
|
|
||||||
// 正常输入:命中与未命中混排,批量读回命中字节,touch 只覆盖实际命中。
|
|
||||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg");
|
|
||||||
List<String> hits = List.of(sha256Hex(urls.get(0)));
|
|
||||||
byte[] bytesA = new byte[]{1, 2, 3};
|
|
||||||
stubBatchRead(hits, Map.of(sha256Hex(urls.get(0)), bytesA));
|
|
||||||
|
|
||||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
|
||||||
|
|
||||||
assertEquals(2, result.size(), "批量入口必须按输入顺序返回");
|
|
||||||
assertEquals(bytesA, result.get(0), "命中行返回缓存字节");
|
|
||||||
assertNull(result.get(1), "未命中行返回 null,不虚构缓存内容");
|
|
||||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(sha256Hex(urls.get(0))));
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_normal_multiple_items() throws Exception {
|
|
||||||
// 批量场景:全命中多 url,一次 IN 查询返回全部字节,touch 覆盖全部命中。
|
|
||||||
List<String> urls = List.of("https://img.example.com/a.jpg", "https://img.example.com/b.jpg",
|
|
||||||
"https://img.example.com/c.jpg");
|
|
||||||
List<String> hits = new ArrayList<>();
|
|
||||||
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
|
|
||||||
for (int i = 0; i < urls.size(); i++) {
|
|
||||||
hits.add(sha256Hex(urls.get(i)));
|
|
||||||
bytesByHash.put(hits.get(i), new byte[]{(byte) (i + 1)});
|
|
||||||
}
|
|
||||||
stubBatchRead(hits, bytesByHash);
|
|
||||||
|
|
||||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
|
||||||
|
|
||||||
assertEquals(3, result.size());
|
|
||||||
for (int i = 0; i < urls.size(); i++) {
|
|
||||||
assertEquals(bytesByHash.get(sha256Hex(urls.get(i))), result.get(i), "顺序稳定、字节不丢失");
|
|
||||||
}
|
|
||||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(new ArrayList<>(hits));
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_normal_repeated_operation_is_idempotent() throws Exception {
|
|
||||||
// 重复执行同一输入:每次行为一致,不产生重复请求/重复 touch。
|
|
||||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
|
||||||
String hash = sha256Hex(urls.get(0));
|
|
||||||
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{9}));
|
|
||||||
|
|
||||||
invokeLookupBatch(service, urls);
|
|
||||||
invokeLookupBatch(service, urls);
|
|
||||||
|
|
||||||
verify(taskImageCacheMapper, times(2)).selectBytesByUrlHashes(any());
|
|
||||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(List.of(hash));
|
|
||||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_boundary_empty_input() throws Exception {
|
|
||||||
// 空输入:null/空列表安全返回空结果,不产生任何数据库访问。
|
|
||||||
List<byte[]> nullResult = invokeLookupBatch(service, null);
|
|
||||||
assertNotNull(nullResult);
|
|
||||||
assertTrue(nullResult.isEmpty());
|
|
||||||
|
|
||||||
List<byte[]> emptyResult = invokeLookupBatch(service, List.of());
|
|
||||||
assertNotNull(emptyResult);
|
|
||||||
assertTrue(emptyResult.isEmpty());
|
|
||||||
|
|
||||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_boundary_single_item() throws Exception {
|
|
||||||
// 单 url:不依赖批量路径,命中时单次查询 + 单次 touch。
|
|
||||||
String url = "https://img.example.com/single.jpg";
|
|
||||||
String hash = sha256Hex(url);
|
|
||||||
stubBatchRead(List.of(hash), Map.of(hash, new byte[]{7}));
|
|
||||||
|
|
||||||
List<byte[]> result = invokeLookupBatch(service, List.of(url));
|
|
||||||
|
|
||||||
assertEquals(1, result.size());
|
|
||||||
assertEquals(7, result.get(0)[0]);
|
|
||||||
verify(taskImageCacheMapper, times(1)).selectBytesByUrlHashes(any());
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_boundary_limit_and_overflow() throws Exception {
|
|
||||||
// 大批量(超过单批上限 500):分片查询,命中 touch 只覆盖命中集合。
|
|
||||||
List<String> urls = new ArrayList<>();
|
|
||||||
for (int i = 0; i < 1200; i++) {
|
|
||||||
urls.add("https://img.example.com/overflow-" + i + ".jpg");
|
|
||||||
}
|
|
||||||
Map<String, byte[]> bytesByHash = new java.util.LinkedHashMap<>();
|
|
||||||
List<String> hits = new ArrayList<>();
|
|
||||||
for (int i = 0; i < 1200; i += 2) {
|
|
||||||
String hash = sha256Hex(urls.get(i));
|
|
||||||
hits.add(hash);
|
|
||||||
bytesByHash.put(hash, new byte[]{(byte) (i % 100)});
|
|
||||||
}
|
|
||||||
stubBatchRead(hits, bytesByHash);
|
|
||||||
|
|
||||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
|
||||||
|
|
||||||
assertEquals(1200, result.size(), "超大批量结果不丢失、顺序稳定");
|
|
||||||
int hitCount = 0;
|
|
||||||
for (int i = 0; i < 1200; i++) {
|
|
||||||
if (i % 2 == 0) {
|
|
||||||
assertNotNull(result.get(i), "偶数下标命中必须返回字节");
|
|
||||||
hitCount++;
|
|
||||||
} else {
|
|
||||||
assertNull(result.get(i), "奇数下标未命中返回 null");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assertEquals(600, hitCount);
|
|
||||||
verify(taskImageCacheMapper, times(3)).selectBytesByUrlHashes(any());
|
|
||||||
// touch 按单批 500 分片:600 命中 → 2 次 touch 调用,且只覆盖命中集合。
|
|
||||||
var captor = org.mockito.ArgumentCaptor.forClass(List.class);
|
|
||||||
verify(taskImageCacheMapper, times(2)).touchLastUsedBatch(captor.capture());
|
|
||||||
List<List<String>> touchCalls = new ArrayList<>();
|
|
||||||
for (Object call : captor.getAllValues()) {
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
List<String> casted = (List<String>) call;
|
|
||||||
touchCalls.add(casted);
|
|
||||||
}
|
|
||||||
assertEquals(2, touchCalls.size());
|
|
||||||
assertEquals(500, touchCalls.get(0).size(), "第一批 touch 500 个命中");
|
|
||||||
assertEquals(100, touchCalls.get(1).size(), "第二批 touch 剩余 100 个命中");
|
|
||||||
assertEquals(hits.subList(0, 500), touchCalls.get(0), "touch 只覆盖实际命中集合");
|
|
||||||
assertEquals(hits.subList(500, 600), touchCalls.get(1));
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_invalid_input_rejected() throws Exception {
|
|
||||||
// 非法输入:db cache 关闭时批量入口直接返回空,不访问数据库。
|
|
||||||
lenient().when(properties.isImageDbCacheEnabled()).thenReturn(false);
|
|
||||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
|
||||||
List<byte[]> result = invokeLookupBatch(service, urls);
|
|
||||||
assertNotNull(result);
|
|
||||||
assertTrue(result.isEmpty(), "db cache 关闭时必须直接返回空结果");
|
|
||||||
verify(taskImageCacheMapper, never()).selectBytesByUrlHashes(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_dependency_failure_releases_resources() throws Exception {
|
|
||||||
// 依赖失败:查询抛异常时批量入口返回空结果、无任何 touch/insert 残留;
|
|
||||||
// 恢复后重试成功。
|
|
||||||
List<String> urls = List.of("https://img.example.com/a.jpg");
|
|
||||||
String hash = sha256Hex(urls.get(0));
|
|
||||||
AtomicInteger callCount = new AtomicInteger(0);
|
|
||||||
doAnswer(invocation -> {
|
|
||||||
if (callCount.getAndIncrement() == 0) {
|
|
||||||
throw new IllegalStateException("db down");
|
|
||||||
}
|
|
||||||
TaskImageCacheEntity row = new TaskImageCacheEntity();
|
|
||||||
row.setUrlHash(hash);
|
|
||||||
row.setImageBytes(new byte[]{5});
|
|
||||||
return List.of(row);
|
|
||||||
}).when(taskImageCacheMapper).selectBytesByUrlHashes(any());
|
|
||||||
|
|
||||||
List<byte[]> failed = invokeLookupBatch(service, urls);
|
|
||||||
assertNotNull(failed);
|
|
||||||
assertTrue(failed.isEmpty(), "查询失败必须返回空结果而不是抛错阻断组装");
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsedBatch(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
verify(taskImageCacheMapper, never()).insert(any(TaskImageCacheEntity.class));
|
|
||||||
|
|
||||||
List<byte[]> recovered = invokeLookupBatch(service, urls);
|
|
||||||
assertEquals(1, recovered.size());
|
|
||||||
assertEquals(5, recovered.get(0)[0], "依赖恢复后重试成功");
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_014_image_normal_single_lookup_legacy_compat() throws Exception {
|
|
||||||
// 兼容性:单 URL 旧入口 lookup 保持返回字节语义;Task 15 起 touch 改为
|
|
||||||
// 异步批量缓冲,flush 后批量 touch 一次,命中才入缓冲。
|
|
||||||
String url = "https://img.example.com/legacy.jpg";
|
|
||||||
String hash = sha256Hex(url);
|
|
||||||
byte[] bytes = new byte[]{6};
|
|
||||||
when(taskImageCacheMapper.selectBytesByUrlHash(hash)).thenReturn(bytes);
|
|
||||||
|
|
||||||
byte[] result = service.lookup(url);
|
|
||||||
assertEquals(bytes, result, "lookup 命中必须返回缓存字节");
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(List.of(hash));
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
|
|
||||||
assertNull(service.lookup("https://img.example.com/missing.jpg"), "未命中返回 null");
|
|
||||||
service.flushPendingTouches();
|
|
||||||
verify(taskImageCacheMapper, times(1)).touchLastUsedBatch(any());
|
|
||||||
verify(taskImageCacheMapper, never()).touchLastUsed(anyString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-167
@@ -1,167 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.nanri.aiimage.config.OssProperties;
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|
||||||
import com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper;
|
|
||||||
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
|
|
||||||
import org.mockito.Mockito;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 本地验证入口:用生产真实批次数据 + 生产 LLM 网关跑 SimilarAsinLlmService 完整链路。
|
|
||||||
* 用法:mvn compile test-compile 后执行
|
|
||||||
* java -cp target/classes;target/test-classes;$(cat cp.txt) com.nanri.aiimage.modules.similarasin.service.SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]
|
|
||||||
*/
|
|
||||||
public class SimilarAsinLlmLocalVerify {
|
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
System.setOut(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.out), true, "UTF-8"));
|
|
||||||
System.setErr(new java.io.PrintStream(new java.io.FileOutputStream(java.io.FileDescriptor.err), true, "UTF-8"));
|
|
||||||
if (args.length < 2) {
|
|
||||||
System.err.println("usage: SimilarAsinLlmLocalVerify <data.json> <apiKey> [imgSwitch] [categorySwitch]");
|
|
||||||
System.exit(1);
|
|
||||||
}
|
|
||||||
String dataFile = args[0];
|
|
||||||
String apiKey = args[1];
|
|
||||||
boolean imgSwitch = args.length > 2 && Boolean.parseBoolean(args[2]);
|
|
||||||
boolean categorySwitch = args.length > 3 && Boolean.parseBoolean(args[3]);
|
|
||||||
|
|
||||||
ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
objectMapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
||||||
JsonNode root = objectMapper.readTree(new File(dataFile));
|
|
||||||
List<SimilarAsinResultRowDto> rows = new ArrayList<>();
|
|
||||||
if (root.isArray()) {
|
|
||||||
for (JsonNode node : root) {
|
|
||||||
rows.add(fromJson(objectMapper, node));
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
rows.add(fromJson(objectMapper, root));
|
|
||||||
}
|
|
||||||
System.out.println("[verify] loaded rows=" + rows.size() + " imgSwitch=" + imgSwitch
|
|
||||||
+ " categorySwitch=" + categorySwitch);
|
|
||||||
if (imgSwitch && categorySwitch) {
|
|
||||||
System.out.println("[verify] raw first row alibaba[0].url="
|
|
||||||
+ (rows.isEmpty() || rows.get(0).getAlibaba().isEmpty() ? "null"
|
|
||||||
: rows.get(0).getAlibaba().get(0).getUrl()));
|
|
||||||
}
|
|
||||||
|
|
||||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
|
||||||
props.setLlmApiKey(apiKey);
|
|
||||||
props.setLlmRowConcurrency(2);
|
|
||||||
props.setLlmImageDownloadTimeoutSeconds(10);
|
|
||||||
|
|
||||||
SimilarAsinLlmClient client = new SimilarAsinLlmClient(props, objectMapper, null);
|
|
||||||
OssProperties ossProps = new OssProperties();
|
|
||||||
ossProps.setEndpoint("https://oss.aishufu.top");
|
|
||||||
ossProps.setPublicEndpoint("https://oss.aishufu.top");
|
|
||||||
ossProps.setBucket("nanri-ai-images");
|
|
||||||
ossProps.setAccessKeyId("appuser");
|
|
||||||
ossProps.setAccessKeySecret("AppUser@2024SecureKey");
|
|
||||||
OssStorageService oss = new OssStorageService(ossProps);
|
|
||||||
PuzzleImageMerger merger = new PuzzleImageMerger(props);
|
|
||||||
|
|
||||||
// 生产真实类目数据(导出自 biz_product_category),spy 类目服务按 parentId 过滤。
|
|
||||||
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> categories = loadCategories(objectMapper);
|
|
||||||
ProductCategoryService categoryService = Mockito.spy(new ProductCategoryService(
|
|
||||||
Mockito.mock(com.nanri.aiimage.modules.productcategory.mapper.ProductCategoryMapper.class)));
|
|
||||||
Mockito.doAnswer(invocation -> {
|
|
||||||
Long parentId = invocation.getArgument(0);
|
|
||||||
List<com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo> items = categories.stream()
|
|
||||||
.filter(c -> parentId == null ? c.getParentId() == null : parentId.equals(c.getParentId()))
|
|
||||||
.map(c -> {
|
|
||||||
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo item =
|
|
||||||
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo();
|
|
||||||
item.setId(c.getId());
|
|
||||||
item.setParentId(c.getParentId());
|
|
||||||
item.setName(c.getName());
|
|
||||||
item.setCategoryKey(c.getCategoryKey());
|
|
||||||
return item;
|
|
||||||
})
|
|
||||||
.toList();
|
|
||||||
com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo vo =
|
|
||||||
new com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo();
|
|
||||||
vo.setItems(items);
|
|
||||||
vo.setTree(List.of());
|
|
||||||
vo.setTotal((long) items.size());
|
|
||||||
vo.setPage(1L);
|
|
||||||
vo.setPageSize((long) Math.max(1, items.size()));
|
|
||||||
vo.setHasMore(false);
|
|
||||||
return vo;
|
|
||||||
}).when(categoryService).children(Mockito.any(), Mockito.anyLong(), Mockito.anyLong());
|
|
||||||
|
|
||||||
SimilarAsinLlmService service = new SimilarAsinLlmService(
|
|
||||||
client, props, ossProps, categoryService, merger, oss);
|
|
||||||
|
|
||||||
long start = System.currentTimeMillis();
|
|
||||||
List<SimilarAsinResultRowDto> result = service.inspectRows(rows, null, apiKey, imgSwitch, categorySwitch);
|
|
||||||
long elapsed = System.currentTimeMillis() - start;
|
|
||||||
System.out.println("[verify] done rows=" + result.size() + " elapsedMs=" + elapsed);
|
|
||||||
for (SimilarAsinResultRowDto row : result) {
|
|
||||||
System.out.println(String.format(
|
|
||||||
"asin=%s | status=%s | isConform=%s | category=%s | reason=%s | isStock=%s | similarity=%s | mainUrl=%s | puzzle1=%s | puzzle2=%s",
|
|
||||||
row.getAsin(), row.getStatus(), row.getIsConform(), row.getCategory(),
|
|
||||||
row.getReason(), row.getIsStock(), row.getSimilarity(),
|
|
||||||
shorten(row.getMainUrl()), shorten(row.getPuzzleImg1()), shorten(row.getPuzzleImg2())));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> loadCategories(ObjectMapper objectMapper) throws Exception {
|
|
||||||
com.fasterxml.jackson.databind.JsonNode root = objectMapper.readTree(
|
|
||||||
SimilarAsinLlmLocalVerify.class.getResourceAsStream("/categories.json"));
|
|
||||||
List<com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity> list = new ArrayList<>();
|
|
||||||
for (com.fasterxml.jackson.databind.JsonNode node : root) {
|
|
||||||
com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity entity =
|
|
||||||
new com.nanri.aiimage.modules.productcategory.model.entity.ProductCategoryEntity();
|
|
||||||
entity.setId(node.get("id").asLong());
|
|
||||||
if (!node.get("parent_id").isNull()) {
|
|
||||||
entity.setParentId(node.get("parent_id").asLong());
|
|
||||||
}
|
|
||||||
entity.setName(node.get("name").asText());
|
|
||||||
entity.setCategoryKey(node.get("category_key").asText());
|
|
||||||
entity.setSortOrder(node.get("sort_order").isNull() ? null : node.get("sort_order").asInt());
|
|
||||||
entity.setDescription(node.get("description").isNull() ? null : node.get("description").asText());
|
|
||||||
entity.setIsBuiltin(node.get("is_builtin") != null && node.get("is_builtin").asBoolean());
|
|
||||||
list.add(entity);
|
|
||||||
}
|
|
||||||
return list;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SimilarAsinResultRowDto fromJson(ObjectMapper objectMapper, JsonNode node) throws Exception {
|
|
||||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
|
||||||
row.setAsin(text(node, "asin"));
|
|
||||||
row.setTitle(text(node, "title"));
|
|
||||||
row.setSku(text(node, "sku"));
|
|
||||||
row.setCountry(text(node, "country"));
|
|
||||||
row.setUrl(text(node, "url"));
|
|
||||||
JsonNode alibaba = node.get("alibaba");
|
|
||||||
if (alibaba != null && alibaba.isArray()) {
|
|
||||||
List<SimilarAsinResultRowDto.AlibabaItem> items = new ArrayList<>();
|
|
||||||
for (JsonNode item : alibaba) {
|
|
||||||
items.add(objectMapper.treeToValue(item, SimilarAsinResultRowDto.AlibabaItem.class));
|
|
||||||
}
|
|
||||||
row.setAlibaba(items);
|
|
||||||
}
|
|
||||||
return row;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String text(JsonNode node, String field) {
|
|
||||||
JsonNode value = node.get(field);
|
|
||||||
return value == null || value.isNull() ? null : value.asText();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String shorten(String value) {
|
|
||||||
if (value == null) {
|
|
||||||
return "null";
|
|
||||||
}
|
|
||||||
return value.length() <= 70 ? value : value.substring(0, 70) + "...";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-279
@@ -1,279 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.nanri.aiimage.config.OssProperties;
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|
||||||
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryItemVo;
|
|
||||||
import com.nanri.aiimage.modules.productcategory.model.vo.ProductCategoryListVo;
|
|
||||||
import com.nanri.aiimage.modules.productcategory.service.ProductCategoryService;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.PuzzleImageMerger;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import javax.imageio.ImageIO;
|
|
||||||
import java.awt.image.BufferedImage;
|
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
import java.net.http.HttpClient;
|
|
||||||
import java.net.http.HttpRequest;
|
|
||||||
import java.net.http.HttpResponse;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyList;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyLong;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import org.mockito.Mockito;
|
|
||||||
|
|
||||||
class SimilarAsinLlmServiceTest {
|
|
||||||
|
|
||||||
private static byte[] jpegBytes() {
|
|
||||||
try {
|
|
||||||
BufferedImage image = new BufferedImage(10, 10, BufferedImage.TYPE_INT_RGB);
|
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
||||||
ImageIO.write(image, "jpg", baos);
|
|
||||||
return baos.toByteArray();
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new IllegalStateException(ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinProperties properties() {
|
|
||||||
SimilarAsinProperties props = new SimilarAsinProperties();
|
|
||||||
props.setLlmApiKey("test-key");
|
|
||||||
return props;
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinLlmService service(SimilarAsinLlmClient llmClient) {
|
|
||||||
OssStorageService ossStorage = mock(OssStorageService.class);
|
|
||||||
when(ossStorage.getPublicUrl(anyString())).thenAnswer(invocation -> "https://oss.aishufu.top/nanri-ai-images/" + invocation.getArgument(0));
|
|
||||||
PuzzleImageMerger merger = mock(PuzzleImageMerger.class);
|
|
||||||
when(merger.merge(anyList(), Mockito.<SimilarAsinResultRowDto>any())).thenReturn(jpegBytes());
|
|
||||||
ProductCategoryService categoryService = mock(ProductCategoryService.class);
|
|
||||||
when(categoryService.children(any(), anyLong(), anyLong()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
Long parentId = invocation.getArgument(0);
|
|
||||||
if (parentId == null) {
|
|
||||||
return categoryPage("类目A", 1L);
|
|
||||||
}
|
|
||||||
if (parentId == 1L) {
|
|
||||||
return categoryPage("类目B", 2L);
|
|
||||||
}
|
|
||||||
if (parentId == 2L) {
|
|
||||||
return categoryPage("类目C", 3L);
|
|
||||||
}
|
|
||||||
return emptyCategoryPage();
|
|
||||||
});
|
|
||||||
SimilarAsinLlmService svc = new SimilarAsinLlmService(
|
|
||||||
llmClient,
|
|
||||||
properties(),
|
|
||||||
mock(OssProperties.class),
|
|
||||||
categoryService,
|
|
||||||
merger,
|
|
||||||
ossStorage);
|
|
||||||
svc.setDownloadHttpClientForTest(mockHttpClient());
|
|
||||||
return svc;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ProductCategoryListVo categoryPage(String name, long id) {
|
|
||||||
ProductCategoryItemVo item = new ProductCategoryItemVo();
|
|
||||||
item.setId(id);
|
|
||||||
item.setName(name);
|
|
||||||
ProductCategoryListVo vo = new ProductCategoryListVo();
|
|
||||||
vo.setItems(List.of(item));
|
|
||||||
vo.setTotal(1L);
|
|
||||||
vo.setPage(1L);
|
|
||||||
vo.setPageSize(1L);
|
|
||||||
vo.setHasMore(false);
|
|
||||||
return vo;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ProductCategoryListVo emptyCategoryPage() {
|
|
||||||
ProductCategoryListVo vo = new ProductCategoryListVo();
|
|
||||||
vo.setItems(List.of());
|
|
||||||
vo.setTotal(0L);
|
|
||||||
vo.setPage(1L);
|
|
||||||
vo.setPageSize(1L);
|
|
||||||
vo.setHasMore(false);
|
|
||||||
return vo;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static HttpClient mockHttpClient() {
|
|
||||||
HttpClient client = mock(HttpClient.class);
|
|
||||||
try {
|
|
||||||
when(client.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class)))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
byte[] body = jpegBytes();
|
|
||||||
HttpRequest request = invocation.getArgument(0);
|
|
||||||
return new TestHttpResponse(200, body, request);
|
|
||||||
});
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new IllegalStateException(ex);
|
|
||||||
}
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static final class TestHttpResponse implements HttpResponse<byte[]> {
|
|
||||||
private final int statusCode;
|
|
||||||
private final byte[] body;
|
|
||||||
private final HttpRequest request;
|
|
||||||
|
|
||||||
TestHttpResponse(int statusCode, byte[] body, HttpRequest request) {
|
|
||||||
this.statusCode = statusCode;
|
|
||||||
this.body = body;
|
|
||||||
this.request = request;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int statusCode() {
|
|
||||||
return statusCode;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public HttpRequest request() {
|
|
||||||
return request;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public java.util.Optional<HttpResponse<byte[]>> previousResponse() {
|
|
||||||
return java.util.Optional.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public java.net.http.HttpHeaders headers() {
|
|
||||||
return java.net.http.HttpHeaders.of(Map.of(), (a, b) -> true);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public byte[] body() {
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public java.net.URI uri() {
|
|
||||||
return java.net.URI.create("http://test");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public java.net.http.HttpClient.Version version() {
|
|
||||||
return HttpClient.Version.HTTP_1_1;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public java.util.Optional<javax.net.ssl.SSLSession> sslSession() {
|
|
||||||
return java.util.Optional.empty();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private SimilarAsinLlmClient llmClient(String apiKey, Map<String, String> responses) {
|
|
||||||
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
|
|
||||||
when(client.resolveApiKey(anyString())).thenReturn(apiKey == null ? "" : apiKey);
|
|
||||||
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
|
|
||||||
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
String response = responses.get("chat");
|
|
||||||
if (response == null) {
|
|
||||||
throw new IllegalStateException("unexpected chat call");
|
|
||||||
}
|
|
||||||
return response;
|
|
||||||
});
|
|
||||||
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
String response = responses.get("images");
|
|
||||||
if (response == null) {
|
|
||||||
throw new IllegalStateException("unexpected images call");
|
|
||||||
}
|
|
||||||
return response;
|
|
||||||
});
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static final com.fasterxml.jackson.databind.ObjectMapper OBJECT_MAPPER =
|
|
||||||
new com.fasterxml.jackson.databind.ObjectMapper();
|
|
||||||
|
|
||||||
/** 类目匹配(按序号返回不同类目名)与合规检查(后续)返回不同 JSON。 */
|
|
||||||
private SimilarAsinLlmClient llmClientStaged(String apiKey, List<String> categoryJsons,
|
|
||||||
String conformJson, String imagesJson) {
|
|
||||||
SimilarAsinLlmClient client = mock(SimilarAsinLlmClient.class);
|
|
||||||
int[] categoryIndex = {0};
|
|
||||||
when(client.resolveApiKey(anyString())).thenReturn(apiKey);
|
|
||||||
when(client.hasApiKey(anyString())).thenReturn(apiKey != null && !apiKey.isBlank());
|
|
||||||
when(client.invokeChat(anyString(), anyString(), anyString(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
if (categoryIndex[0] < categoryJsons.size()) {
|
|
||||||
return categoryJsons.get(categoryIndex[0]++);
|
|
||||||
}
|
|
||||||
return conformJson;
|
|
||||||
});
|
|
||||||
when(client.invokeChatWithImages(anyString(), anyString(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> imagesJson);
|
|
||||||
when(client.parseJsonContent(anyString()))
|
|
||||||
.thenAnswer(invocation -> parseJson(invocation.getArgument(0)));
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static com.fasterxml.jackson.databind.JsonNode parseJson(String content) {
|
|
||||||
try {
|
|
||||||
return OBJECT_MAPPER.readTree(content);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new IllegalStateException(ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void noApiKeyKeepsRawRows() {
|
|
||||||
SimilarAsinLlmService service = service(llmClient("", Map.of()));
|
|
||||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
|
||||||
row.setAsin("B0TEST");
|
|
||||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "", true, true);
|
|
||||||
assertEquals(1, result.size());
|
|
||||||
assertEquals("B0TEST", result.get(0).getAsin());
|
|
||||||
assertNull(result.get(0).getStatus());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void categorySwitchOffOnlyPreparesImagesAndMarksNotExistsWhenNoMainUrl() {
|
|
||||||
SimilarAsinLlmService service = service(llmClient("k", Map.of()));
|
|
||||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
|
||||||
row.setAsin("B0TEST");
|
|
||||||
row.setTitle("Test product");
|
|
||||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", false, false);
|
|
||||||
assertEquals(1, result.size());
|
|
||||||
assertEquals("不存在", result.get(0).getStatus());
|
|
||||||
assertNull(result.get(0).getIsConform());
|
|
||||||
assertNull(result.get(0).getPuzzleImg1());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void imageCompareStopsOnStockAndFillsFields() {
|
|
||||||
// 前 2 次 chat:一级/二级类目匹配返回名称(Java 按名回查候选取真实 ID);第 3 次:合规检查。
|
|
||||||
SimilarAsinLlmClient client = llmClientStaged("k", List.of("{\"name\":\"类目A\"}", "{\"name\":\"类目B\"}"),
|
|
||||||
"{\"asin\":\"B0TEST\",\"is_conform\":\"符合\",\"reason\":\"无\",\"category\":\"类目A->类目B->类目C\"}",
|
|
||||||
"{\"asin\":\"B0TEST\",\"is_stock\":\"有货\",\"similarity\":\"95%\",\"status\":\"成功\",\"is_conform\":\"符合\",\"category\":\"类目A->类目B->类目C\"}");
|
|
||||||
SimilarAsinLlmService service = service(client);
|
|
||||||
|
|
||||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
|
||||||
row.setAsin("B0TEST");
|
|
||||||
row.setTitle("Test product");
|
|
||||||
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
|
|
||||||
SimilarAsinResultRowDto.AlibabaItem item = new SimilarAsinResultRowDto.AlibabaItem();
|
|
||||||
item.setUrl("https://cbu01.alicdn.com/img/1.jpg");
|
|
||||||
row.setAlibaba(List.of(item));
|
|
||||||
|
|
||||||
// 一级/二级都匹配(按名回查 ID),三级候选可用,合规符合 → 图片对比,有货即停。
|
|
||||||
List<SimilarAsinResultRowDto> result = service.inspectRows(List.of(row), "", "k", true, true);
|
|
||||||
assertEquals(1, result.size());
|
|
||||||
SimilarAsinResultRowDto out = result.get(0);
|
|
||||||
assertEquals("成功", out.getStatus());
|
|
||||||
assertEquals("有货", out.getIsStock());
|
|
||||||
assertEquals("95%", out.getSimilarity());
|
|
||||||
assertEquals("符合", out.getIsConform());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-204
@@ -1,204 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.mockito.ArgumentCaptor;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.Mockito.doAnswer;
|
|
||||||
import static org.mockito.Mockito.times;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 9:chunk 查询从单行分页改为批量 keyset 分页,保持低内存读取。
|
|
||||||
* loadChunksKeyset 按 id 递增分批拉取(每批 pageSize),最后按 chunkIndex 升序合并,
|
|
||||||
* 避免超大任务一次 selectList 全量载入 chunk 元数据。
|
|
||||||
* mock 分页由 wrapper 中 gt("id", lastId) 的 keyset 值驱动,保证重复调用可复现(幂等)。
|
|
||||||
*/
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
|
||||||
class SimilarAsinTaskServiceChunkKeysetTest {
|
|
||||||
|
|
||||||
private static final String MODULE = "similar-asin";
|
|
||||||
|
|
||||||
@Mock private TaskChunkMapper taskChunkMapper;
|
|
||||||
|
|
||||||
private static TaskChunkEntity chunk(long id, int chunkIndex) {
|
|
||||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
|
||||||
chunk.setId(id);
|
|
||||||
chunk.setTaskId(7004L);
|
|
||||||
chunk.setModuleType(MODULE);
|
|
||||||
chunk.setChunkIndex(chunkIndex);
|
|
||||||
return chunk;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<TaskChunkEntity> chunks(long... idsAndIndexes) {
|
|
||||||
List<TaskChunkEntity> result = new ArrayList<>();
|
|
||||||
for (int i = 0; i < idsAndIndexes.length; i += 2) {
|
|
||||||
result.add(chunk(idsAndIndexes[i], (int) idsAndIndexes[i + 1]));
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 从 wrapper 的 SQL 片段解析 keyset:匹配 "id > #{ew.paramNameValuePairs.键}" 后从参数表取值。 */
|
|
||||||
private static long keysetOf(QueryWrapper<TaskChunkEntity> wrapper) {
|
|
||||||
java.util.regex.Matcher m = java.util.regex.Pattern
|
|
||||||
.compile("id\\s*>\\s*#\\{ew\\.paramNameValuePairs\\.(\\w+)\\}", java.util.regex.Pattern.CASE_INSENSITIVE)
|
|
||||||
.matcher(wrapper.getSqlSegment());
|
|
||||||
if (m.find()) {
|
|
||||||
Object value = wrapper.getParamNameValuePairs().get(m.group(1));
|
|
||||||
if (value instanceof Number number) {
|
|
||||||
return number.longValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0L;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 按 keyset 驱动分页:每次 selectList 返回 id > keyset 的下一批,天然支持重复调用。 */
|
|
||||||
private void stubKeysetPages(List<TaskChunkEntity> all, int pageSize) {
|
|
||||||
int batch = pageSize > 0 ? pageSize : 500;
|
|
||||||
doAnswer(invocation -> {
|
|
||||||
long lastId = keysetOf(invocation.getArgument(0));
|
|
||||||
return all.stream().filter(c -> c.getId() > lastId).limit(batch).toList();
|
|
||||||
}).when(taskChunkMapper).selectList(any());
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<Long> keysetIdsFromRounds(int rounds) {
|
|
||||||
ArgumentCaptor<QueryWrapper<TaskChunkEntity>> captor =
|
|
||||||
ArgumentCaptor.forClass(QueryWrapper.class);
|
|
||||||
verify(taskChunkMapper, times(rounds)).selectList(captor.capture());
|
|
||||||
List<Long> keysets = new ArrayList<>();
|
|
||||||
for (QueryWrapper<TaskChunkEntity> wrapper : captor.getAllValues()) {
|
|
||||||
keysets.add(keysetOf(wrapper));
|
|
||||||
}
|
|
||||||
return keysets;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_normal_default_path() {
|
|
||||||
// 正常输入:chunk 数小于 pageSize,一轮拉完,结果全且按 chunkIndex 有序
|
|
||||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
|
|
||||||
stubKeysetPages(all, 500);
|
|
||||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
|
||||||
assertEquals(3, result.size());
|
|
||||||
assertEquals(List.of(1, 2, 3), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
|
||||||
assertEquals(List.of(0L), keysetIdsFromRounds(1), "首轮 keyset 为 0");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_normal_multiple_items() {
|
|
||||||
// 超过 pageSize:多轮拉取,keyset 逐轮推进,全部合并且按 chunkIndex 升序、无重复
|
|
||||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7);
|
|
||||||
stubKeysetPages(all, 3);
|
|
||||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
|
||||||
assertEquals(7, result.size());
|
|
||||||
assertEquals(List.of(1, 2, 3, 4, 5, 6, 7), result.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
|
||||||
long distinctIds = result.stream().map(TaskChunkEntity::getId).distinct().count();
|
|
||||||
assertEquals(7, distinctIds, "keyset 分页不能产生重复 chunk");
|
|
||||||
assertEquals(List.of(0L, 3L, 6L), keysetIdsFromRounds(3), "keyset 逐轮推进,不足一批即止");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_normal_repeated_operation_is_idempotent() {
|
|
||||||
// 重复执行同一输入:每轮都从 keyset=0 开始,结果一致
|
|
||||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3);
|
|
||||||
stubKeysetPages(all, 500);
|
|
||||||
List<TaskChunkEntity> first = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
|
||||||
List<TaskChunkEntity> second = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
|
||||||
assertEquals(first.size(), second.size());
|
|
||||||
for (int i = 0; i < first.size(); i++) {
|
|
||||||
assertEquals(first.get(i).getId(), second.get(i).getId());
|
|
||||||
}
|
|
||||||
assertEquals(List.of(0L, 0L), keysetIdsFromRounds(2), "重复执行每轮都从 keyset=0 开始");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_boundary_empty_input() {
|
|
||||||
// 空集合:返回空列表,不创建无效资源,且只查一轮
|
|
||||||
stubKeysetPages(List.of(), 500);
|
|
||||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500);
|
|
||||||
assertNotNull(result);
|
|
||||||
assertEquals(0, result.size());
|
|
||||||
verify(taskChunkMapper, times(1)).selectList(any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_boundary_single_item() {
|
|
||||||
// 单 chunk:一轮返回后 keyset 推进即拉空,不依赖批量路径
|
|
||||||
List<TaskChunkEntity> all = chunks(42, 9);
|
|
||||||
stubKeysetPages(all, 1);
|
|
||||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 1);
|
|
||||||
assertEquals(1, result.size());
|
|
||||||
assertEquals(9, result.get(0).getChunkIndex());
|
|
||||||
assertEquals(42L, result.get(0).getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_boundary_limit_and_overflow() {
|
|
||||||
// chunk 数恰好等于 pageSize 的倍数:最后一轮仍返回非空才继续,全部取回
|
|
||||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6);
|
|
||||||
stubKeysetPages(all, 3);
|
|
||||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
|
||||||
assertEquals(6, result.size());
|
|
||||||
// pageSize 为 0/负数:回退默认 500,不抛异常
|
|
||||||
stubKeysetPages(all, 0);
|
|
||||||
List<TaskChunkEntity> fallback = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 0);
|
|
||||||
assertEquals(6, fallback.size());
|
|
||||||
stubKeysetPages(all, -5);
|
|
||||||
List<TaskChunkEntity> negative = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, -5);
|
|
||||||
assertEquals(6, negative.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_invalid_input_rejected() {
|
|
||||||
// taskId 为 null:安全返回空列表,不发起查询
|
|
||||||
List<TaskChunkEntity> result = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, null, MODULE, 500);
|
|
||||||
assertNotNull(result);
|
|
||||||
assertEquals(0, result.size());
|
|
||||||
verify(taskChunkMapper, times(0)).selectList(any());
|
|
||||||
// mapper 查询抛异常:转项目约定异常,消息可识别
|
|
||||||
when(taskChunkMapper.selectList(any())).thenThrow(new IllegalStateException("db down"));
|
|
||||||
BusinessException ex = assertThrows(BusinessException.class,
|
|
||||||
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 500));
|
|
||||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
|
||||||
"异常消息必须可识别,实际: " + ex.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_009_chunk_dependency_failure_releases_resources() {
|
|
||||||
// 第二轮查询失败:抛异常不返回半截结果;恢复后重试可完整返回
|
|
||||||
List<TaskChunkEntity> all = chunks(1, 1, 2, 2, 3, 3, 4, 4, 5, 5);
|
|
||||||
doAnswer(invocation -> {
|
|
||||||
long lastId = keysetOf(invocation.getArgument(0));
|
|
||||||
if (lastId == 0L) {
|
|
||||||
return all.subList(0, 3);
|
|
||||||
}
|
|
||||||
if (lastId == 3L) {
|
|
||||||
throw new IllegalStateException("db down mid-page");
|
|
||||||
}
|
|
||||||
return all.subList(3, 5);
|
|
||||||
}).when(taskChunkMapper).selectList(any());
|
|
||||||
BusinessException ex = assertThrows(BusinessException.class,
|
|
||||||
() -> SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3));
|
|
||||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("chunk"),
|
|
||||||
"异常消息必须可识别,实际: " + ex.getMessage());
|
|
||||||
// 恢复后重试成功:5 个 chunk 全部取回
|
|
||||||
stubKeysetPages(all, 3);
|
|
||||||
List<TaskChunkEntity> recovered = SimilarAsinTaskService.loadChunksKeyset(taskChunkMapper, 7004L, MODULE, 3);
|
|
||||||
assertEquals(5, recovered.size());
|
|
||||||
assertEquals(List.of(1, 2, 3, 4, 5), recovered.stream().map(TaskChunkEntity::getChunkIndex).toList());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-399
@@ -1,399 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
|
||||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
|
||||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
|
||||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
|
||||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
|
||||||
import org.junit.jupiter.api.AfterEach;
|
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.mockito.InjectMocks;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.Spy;
|
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
|
||||||
|
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
|
||||||
import static org.mockito.Mockito.doAnswer;
|
|
||||||
import static org.mockito.Mockito.lenient;
|
|
||||||
import static org.mockito.Mockito.times;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 13:chunk 合并增加单次最大行数与 payload 字节上限。
|
|
||||||
* 校验点位于 mergeChunkPayload 单次合并入口:合并后总行数超过 chunkMergeMaxRows、
|
|
||||||
* 或 payload 字节超过 chunkMergePayloadMaxBytes 时,从最旧行开始降级到
|
|
||||||
* orphan 兜底(assemble 阶段 putIfAbsent 合并回结果,不丢数据);
|
|
||||||
* 单行本身超过字节上限时抛可识别异常拒绝合并。低于上限的行为与旧路径完全一致。
|
|
||||||
*/
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
|
||||||
class SimilarAsinTaskServiceChunkMergeLimitTest {
|
|
||||||
|
|
||||||
private static final AtomicLong NEXT_ID = new AtomicLong(90000);
|
|
||||||
|
|
||||||
@Mock private LocalFileStorageService localFileStorageService;
|
|
||||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
|
||||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
|
||||||
@Mock private FileTaskMapper fileTaskMapper;
|
|
||||||
@Mock private FileResultMapper fileResultMapper;
|
|
||||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
|
||||||
@Mock private TaskChunkMapper taskChunkMapper;
|
|
||||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
|
||||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
|
||||||
@Mock private SimilarAsinProperties properties;
|
|
||||||
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
|
|
||||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
|
||||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
|
||||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
|
||||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
|
||||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
|
||||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
|
||||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
|
||||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
|
||||||
|
|
||||||
@InjectMocks private SimilarAsinTaskService service;
|
|
||||||
|
|
||||||
@BeforeAll
|
|
||||||
static void initializeMybatisMetadata() {
|
|
||||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
|
||||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
|
||||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
|
||||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
|
||||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
|
||||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
|
||||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
|
||||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
|
||||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
|
||||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
|
|
||||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(16L * 1024L * 1024L);
|
|
||||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
|
||||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
|
||||||
.thenReturn("rustfs:task-parsed/similar-asin/90000/payload.json");
|
|
||||||
lenient().doAnswer(invocation -> {
|
|
||||||
FileTaskEntity task = invocation.getArgument(0);
|
|
||||||
task.setId(NEXT_ID.incrementAndGet());
|
|
||||||
return 1;
|
|
||||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
|
||||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterEach
|
|
||||||
void shutdown() {
|
|
||||||
service.shutdownAssembleExecutor();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SimilarAsinResultRowDto row(String rowToken, String asin, String title) {
|
|
||||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
|
||||||
r.setRowToken(rowToken);
|
|
||||||
r.setId(rowToken);
|
|
||||||
r.setAsin(asin);
|
|
||||||
r.setCountry("英国");
|
|
||||||
r.setTitle(title);
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
|
||||||
return new ObjectMapper().writeValueAsString(rows);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
|
||||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
|
||||||
chunk.setId(id);
|
|
||||||
chunk.setTaskId(9004L);
|
|
||||||
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
|
||||||
chunk.setScopeHash(scopeHash);
|
|
||||||
chunk.setChunkIndex(chunkIndex);
|
|
||||||
chunk.setPayloadJson(payloadJson);
|
|
||||||
return chunk;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 记录每次 storeChunkPayloadVersioned 收到的 payload 字符串。 */
|
|
||||||
private void stubChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicReference<String> storedPayload) throws Exception {
|
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
|
||||||
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
|
|
||||||
.thenReturn(payloadJson);
|
|
||||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
storedPayload.set(invocation.getArgument(4));
|
|
||||||
return "stored:" + invocation.getArgument(2);
|
|
||||||
});
|
|
||||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
|
||||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task,
|
|
||||||
String scopeHash, Integer chunkIndex,
|
|
||||||
List<SimilarAsinResultRowDto> llmRows) throws Exception {
|
|
||||||
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
|
|
||||||
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
|
|
||||||
merge.setAccessible(true);
|
|
||||||
merge.invoke(service, task, scopeHash, chunkIndex, llmRows, Map.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_normal_default_path() throws Exception {
|
|
||||||
// 正常输入:行数与 payload 字节均在上限内,合并走原路径,结果完整保留。
|
|
||||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
|
||||||
AtomicReference<String> storedPayload = new AtomicReference<>("");
|
|
||||||
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
|
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
List<SimilarAsinResultRowDto> llmRows = List.of(
|
|
||||||
row("r1", "B0A0000001", "标题1"),
|
|
||||||
row("r2", "B0A0000002", "标题2"),
|
|
||||||
row("r3", "B0A0000003", "标题3"));
|
|
||||||
|
|
||||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
|
||||||
assertNotNull(storedPayload.get());
|
|
||||||
assertTrue(storedPayload.get().contains("\"r1\"") && storedPayload.get().contains("\"r3\""),
|
|
||||||
"上限内合并必须完整保留存量行与新增行,实际: " + storedPayload.get());
|
|
||||||
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
|
||||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_normal_multiple_items() throws Exception {
|
|
||||||
// 批量场景:多 chunk 一次 merge,全部在上限内,各 chunk 分别写回、结果不丢失。
|
|
||||||
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
|
||||||
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
|
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
|
|
||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
|
||||||
.thenReturn(rowsJson(List.of(row("r1", "B0A0000001", "标题1"))));
|
|
||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
|
|
||||||
.thenReturn(rowsJson(List.of(row("r2", "B0A0000002", "标题2"))));
|
|
||||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
|
|
||||||
AtomicLong selectOneRound = new AtomicLong(0);
|
|
||||||
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
|
|
||||||
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
|
|
||||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
invokeMerge(service, task, null, null, List.of(
|
|
||||||
row("r1", "B0A0000001", "标题1-新"),
|
|
||||||
row("r2", "B0A0000002", "标题2-新")));
|
|
||||||
|
|
||||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
verify(taskChunkMapper, times(2)).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
|
||||||
// 重复执行同一输入:每次 merge 恰好写一次,不产生重复对象、重复状态。
|
|
||||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
|
||||||
AtomicLong storeCalls = new AtomicLong(0);
|
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
|
||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
|
||||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
|
||||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
storeCalls.incrementAndGet();
|
|
||||||
return "stored:" + invocation.getArgument(2);
|
|
||||||
});
|
|
||||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
|
||||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
List<SimilarAsinResultRowDto> llmRows = List.of(row("r1", "B0A0000001", "标题1"));
|
|
||||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
|
||||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
|
||||||
assertEquals(2, storeCalls.get(), "重复执行同一输入:每次 merge 恰好写回一次,无多余请求");
|
|
||||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_boundary_empty_input() throws Exception {
|
|
||||||
// 空输入:null/空列表安全跳过,不读取 chunk、不写存储、不创建资源。
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
invokeMerge(service, task, "hashA", 1, null);
|
|
||||||
invokeMerge(service, task, "hashA", 1, List.of());
|
|
||||||
verify(taskChunkMapper, times(0)).selectList(any());
|
|
||||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_boundary_single_item() throws Exception {
|
|
||||||
// 单行:不依赖批量路径,合并后结果正确。
|
|
||||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
|
||||||
AtomicReference<String> storedPayload = new AtomicReference<>("");
|
|
||||||
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
|
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
|
||||||
|
|
||||||
assertTrue(storedPayload.get().contains("\"r1\""), "单行合并也必须写回 chunk payload,实际: " + storedPayload.get());
|
|
||||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_boundary_limit_and_overflow() throws Exception {
|
|
||||||
// 超限场景三连:行数超限降级、字节超限降级、单行超字节上限拒绝。
|
|
||||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(2);
|
|
||||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
|
||||||
AtomicLong storeCalls = new AtomicLong(0);
|
|
||||||
AtomicReference<String> lastStoredPayload = new AtomicReference<>("");
|
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
|
||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
|
||||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
|
||||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
storeCalls.incrementAndGet();
|
|
||||||
lastStoredPayload.set(invocation.getArgument(4));
|
|
||||||
return "stored:" + invocation.getArgument(2);
|
|
||||||
});
|
|
||||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
|
||||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
List<SimilarAsinResultRowDto> llmRows = new ArrayList<>();
|
|
||||||
for (int i = 0; i < 5; i++) {
|
|
||||||
llmRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Phase A:行数超限(上限 2,存量 1 + 新增 5)→ 只保留上限内最新行,超限部分转 orphan。
|
|
||||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
|
||||||
assertTrue(lastStoredPayload.get().contains("\"r4\"") && lastStoredPayload.get().contains("\"r5\""),
|
|
||||||
"行数超限时保留上限内的最新行,实际: " + lastStoredPayload.get());
|
|
||||||
assertFalse(lastStoredPayload.get().contains("\"r0\""), "行数超限时最旧行被降级,实际: " + lastStoredPayload.get());
|
|
||||||
assertEquals(1, storeCalls.get());
|
|
||||||
verify(taskScopeStateMapper, times(1)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
verify(transientPayloadStorageService, times(1))
|
|
||||||
.storeParsedPayloadEntry(anyString(), any(), anyString(), anyString(), anyString(), eq(true));
|
|
||||||
|
|
||||||
// Phase B:字节超限(行数放开)→ 从最旧行降级到字节上限内,保留最新结果。
|
|
||||||
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
|
|
||||||
long oneRowBytes = rowsJson(List.of(row("r9", "B0A0000099", "样本行"))).getBytes(StandardCharsets.UTF_8).length;
|
|
||||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(oneRowBytes + 5L);
|
|
||||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
|
||||||
assertTrue(lastStoredPayload.get().contains("\"r5\""), "字节超限时保留最新行,实际: " + lastStoredPayload.get());
|
|
||||||
assertFalse(lastStoredPayload.get().contains("\"r0\""), "字节超限时最旧行被降级,实际: " + lastStoredPayload.get());
|
|
||||||
assertEquals(2, storeCalls.get());
|
|
||||||
verify(taskScopeStateMapper, times(2)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
|
|
||||||
// Phase C:单行本身超过字节上限 → 抛可识别异常拒绝合并,不写 chunk。
|
|
||||||
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(5L);
|
|
||||||
Exception ex = assertThrows(Exception.class, () -> {
|
|
||||||
try {
|
|
||||||
invokeMerge(service, task, "hashA", 1, llmRows);
|
|
||||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
|
||||||
throw e.getCause();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
assertNotNull(ex.getMessage());
|
|
||||||
assertTrue(ex.getMessage().contains("字节上限"),
|
|
||||||
"超字节上限必须抛可识别异常,实际: " + ex.getMessage());
|
|
||||||
assertEquals(2, storeCalls.get(), "拒绝合并时不写 chunk");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_invalid_input_rejected() {
|
|
||||||
// 非法输入:chunk 载荷加载失败(resolve 抛异常)时抛出可识别异常且不写 chunk。
|
|
||||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
|
||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
|
||||||
.thenThrow(new IllegalStateException("rustfs down"));
|
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
Exception ex = assertThrows(Exception.class, () -> {
|
|
||||||
try {
|
|
||||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
|
||||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
|
||||||
throw e.getCause();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
assertNotNull(ex.getMessage());
|
|
||||||
assertTrue(ex.getMessage().contains("chunk"),
|
|
||||||
"chunk 读取失败消息必须可识别,实际: " + ex.getMessage());
|
|
||||||
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_013_payload_row_count_chunk_dependency_failure_releases_resources() throws Exception {
|
|
||||||
// 依赖失败:payload 存储失败时抛带上下文的可识别异常、无残留状态;恢复后重试成功。
|
|
||||||
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
|
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
|
||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
|
||||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
|
||||||
AtomicLong storeCalls = new AtomicLong(0);
|
|
||||||
doAnswer(invocation -> {
|
|
||||||
if (storeCalls.getAndIncrement() == 0) {
|
|
||||||
throw new IllegalStateException("rustfs down");
|
|
||||||
}
|
|
||||||
return "stored:" + invocation.getArgument(2);
|
|
||||||
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
|
||||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
|
||||||
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(9004L);
|
|
||||||
Exception ex = assertThrows(Exception.class, () -> {
|
|
||||||
try {
|
|
||||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
|
||||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
|
||||||
throw e.getCause();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("相似ASIN分片载荷"),
|
|
||||||
"存储失败必须抛带上下文的可识别异常,实际: " + ex.getMessage());
|
|
||||||
assertEquals(1, storeCalls.get(), "失败时只尝试一次即抛出,不静默吞错");
|
|
||||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
|
|
||||||
// 依赖恢复后重试成功:结果正确、无残留状态。
|
|
||||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
|
||||||
assertEquals(2, storeCalls.get(), "恢复后重试成功");
|
|
||||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
|
||||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-416
@@ -1,416 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
|
||||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
|
||||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
|
||||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
|
||||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
|
||||||
import org.junit.jupiter.api.AfterEach;
|
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.mockito.InjectMocks;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.Spy;
|
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
|
||||||
|
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
|
||||||
import static org.mockito.Mockito.atLeastOnce;
|
|
||||||
import static org.mockito.Mockito.doAnswer;
|
|
||||||
import static org.mockito.Mockito.lenient;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.never;
|
|
||||||
import static org.mockito.Mockito.times;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 12:扩展 Coze 结果缓冲覆盖范围,减少频繁读写完整 chunk payload。
|
|
||||||
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1";Task 12 扩展为:
|
|
||||||
* 1) poll DONE 结果去掉 batchTotal 限制,单 batch 也走缓冲;
|
|
||||||
* 2) retry 提交同步 immediate DONE 结果也走缓冲(原立即 merge);
|
|
||||||
* 3) 统一走 bufferLlmRowsOrMerge:缓冲失败回退立即 merge,结果不丢失。
|
|
||||||
* flushLlmBufferedResults 在 finalize 前一次性合并,全任务收敛为一次 chunk 读写。
|
|
||||||
*/
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
|
||||||
class SimilarAsinTaskServiceCozeBufferScopeTest {
|
|
||||||
|
|
||||||
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
|
|
||||||
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
|
|
||||||
private static final String CREDENTIAL = "cred-1";
|
|
||||||
|
|
||||||
@Mock private LocalFileStorageService localFileStorageService;
|
|
||||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
|
||||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
|
||||||
@Mock private FileTaskMapper fileTaskMapper;
|
|
||||||
@Mock private FileResultMapper fileResultMapper;
|
|
||||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
|
||||||
@Mock private TaskChunkMapper taskChunkMapper;
|
|
||||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
|
||||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
@Mock private SimilarAsinLlmService similarAsinLlmService;
|
|
||||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
|
||||||
@Mock private SimilarAsinProperties properties;
|
|
||||||
@Mock private TaskFileJobService taskFileJobService;
|
|
||||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
|
||||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
|
||||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
|
||||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
|
||||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
|
||||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
|
||||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
|
||||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
|
||||||
|
|
||||||
@InjectMocks private SimilarAsinTaskService service;
|
|
||||||
|
|
||||||
@BeforeAll
|
|
||||||
static void initializeMybatisMetadata() {
|
|
||||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
|
||||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
|
||||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
|
||||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() throws Exception {
|
|
||||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
|
||||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
|
||||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
|
||||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
|
||||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
|
||||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
|
||||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getLlmBatchSize()).thenReturn(5);
|
|
||||||
lenient().when(properties.getLlmTextOnlyBatchSize()).thenReturn(10);
|
|
||||||
lenient().when(properties.isLlmResultBufferEnabled()).thenReturn(true);
|
|
||||||
lenient().when(properties.getDbJobTouchIntervalMillis()).thenReturn(2_000L);
|
|
||||||
lenient().when(properties.getDbTaskTouchIntervalMillis()).thenReturn(2_000L);
|
|
||||||
lenient().when(properties.getLlmFlushPendingMinutes()).thenReturn(10);
|
|
||||||
lenient().when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(
|
|
||||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
|
|
||||||
lenient().doAnswer(invocation -> {
|
|
||||||
FileTaskEntity task = invocation.getArgument(0);
|
|
||||||
task.setId(NEXT_ID.incrementAndGet());
|
|
||||||
return 1;
|
|
||||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
|
||||||
lenient().when(fileResultMapper.insert(any(FileResultEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
|
|
||||||
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
|
|
||||||
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
|
|
||||||
.thenAnswer(invocation -> "rustfs:coze-result/" + NEXT_ID.incrementAndGet());
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterEach
|
|
||||||
void shutdown() {
|
|
||||||
service.shutdownAssembleExecutor();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country, String title) {
|
|
||||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
|
||||||
r.setRowToken(rowToken);
|
|
||||||
r.setId(id);
|
|
||||||
r.setAsin(asin);
|
|
||||||
r.setCountry(country);
|
|
||||||
r.setTitle(title);
|
|
||||||
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
|
||||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
|
||||||
chunk.setId(id);
|
|
||||||
chunk.setTaskId(7104L);
|
|
||||||
chunk.setModuleType(MODULE);
|
|
||||||
chunk.setScopeHash(scopeHash);
|
|
||||||
chunk.setChunkIndex(chunkIndex);
|
|
||||||
chunk.setPayloadJson(payloadJson);
|
|
||||||
chunk.setPayloadHash("h-" + id);
|
|
||||||
return chunk;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static FileTaskEntity task() {
|
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
|
||||||
task.setId(7104L);
|
|
||||||
task.setModuleType(MODULE);
|
|
||||||
task.setStatus("RUNNING");
|
|
||||||
task.setResultJson("{\"categorySwitch\":true}");
|
|
||||||
return task;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static TaskScopeStateEntity state(FileTaskEntity task, long id, String status, int batchTotal) {
|
|
||||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
|
||||||
state.setId(id);
|
|
||||||
state.setTaskId(task.getId());
|
|
||||||
state.setModuleType(MODULE);
|
|
||||||
state.setScopeHash("scope-" + id);
|
|
||||||
state.setLlmStatus(status);
|
|
||||||
state.setParsedPayloadJson("ptr:batch-" + id);
|
|
||||||
state.setStateJson("{\"jobId\":7101,\"resultId\":7201,\"chunkScopeHash\":null,\"chunkIndex\":null,"
|
|
||||||
+ "\"batchIndex\":1,\"batchTotal\":" + batchTotal + ",\"ownerInstanceId\":\"test-instance\","
|
|
||||||
+ "\"submitRetryCount\":0,\"credentialName\":\"" + CREDENTIAL + "\",\"resultPayloadPointer\":\"ptr:buffer-" + id + "\"}");
|
|
||||||
return state;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
|
||||||
return new ObjectMapper().writeValueAsString(rows);
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinTaskService.LlmBatchContext context(int batchTotal) {
|
|
||||||
return new SimilarAsinTaskService.LlmBatchContext(
|
|
||||||
7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void stubChunkMerge(String payloadJson) throws Exception {
|
|
||||||
TaskChunkEntity chunk = chunk(1L, "scope-1", 1, "ptr:chunk-1");
|
|
||||||
// loadSubmittedChunks 只保留非空 chunk,chunk payload 必须能解析出至少一行。
|
|
||||||
// 全部 lenient:缓冲成功路径不触达 merge,仅缓冲失败/flush 合并路径消费。
|
|
||||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
|
||||||
lenient().when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
String pointer = invocation.getArgument(0);
|
|
||||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
|
||||||
return payloadJson;
|
|
||||||
}
|
|
||||||
return "[]";
|
|
||||||
});
|
|
||||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
|
||||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
|
||||||
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String chunkRowsJson() throws Exception {
|
|
||||||
// chunk-1 已含 r1 行:loadSubmittedChunks 只保留非空 chunk,且 rowKey 索引能命中缓冲行。
|
|
||||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_normal_default_path() throws Exception {
|
|
||||||
// 正常输入:DONE 结果(batchTotal=1 单 batch)经 bufferLlmRowsOrMerge 走缓冲,
|
|
||||||
// 不立即写 chunk;缓冲失败回退立即 merge 结果不丢失。
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
|
||||||
|
|
||||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
|
||||||
TaskScopeStateEntity.class,
|
|
||||||
SimilarAsinTaskService.LlmBatchContext.class,
|
|
||||||
List.class, FileTaskEntity.class, Map.class);
|
|
||||||
bufferOrMerge.setAccessible(true);
|
|
||||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
|
||||||
|
|
||||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
|
||||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
|
||||||
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
verify(taskChunkMapper, never()).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_normal_multiple_items() throws Exception {
|
|
||||||
// 批量场景:多个 DONE state(单 batch)全部缓冲;flush 后按 chunk 分组一次合并
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
stubChunkMerge(chunkRowsJson());
|
|
||||||
when(taskScopeStateMapper.selectList(any())).thenReturn(
|
|
||||||
List.of(state(task, 1L, "DONE", 1), state(task, 2L, "DONE", 1)));
|
|
||||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
|
||||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
String pointer = invocation.getArgument(0);
|
|
||||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
|
||||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
|
||||||
}
|
|
||||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
|
||||||
return chunkRowsJson();
|
|
||||||
}
|
|
||||||
return "[]";
|
|
||||||
});
|
|
||||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
|
||||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
|
||||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(1L, "scope-1", 1, "ptr:chunk-1"));
|
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1")));
|
|
||||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
|
||||||
|
|
||||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
|
|
||||||
flush.setAccessible(true);
|
|
||||||
flush.invoke(service, 7104L);
|
|
||||||
|
|
||||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
|
||||||
// pointer 清理:每个缓冲 state 都更新
|
|
||||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
|
||||||
// 重复执行同一输入:缓冲写幂等(同一 state 不产生重复 buffer/merge)
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
|
||||||
|
|
||||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
|
||||||
TaskScopeStateEntity.class,
|
|
||||||
SimilarAsinTaskService.LlmBatchContext.class,
|
|
||||||
List.class, FileTaskEntity.class, Map.class);
|
|
||||||
bufferOrMerge.setAccessible(true);
|
|
||||||
TaskScopeStateEntity state = state(task, 1L, "DONE", 2);
|
|
||||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
|
||||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
|
||||||
|
|
||||||
// 缓冲 2 次(每次重新写 pointer 是幂等语义:同一 state 覆盖写,无重复行)
|
|
||||||
verify(transientPayloadStorageService, times(2)).storeParsedPayloadEntry(
|
|
||||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
|
||||||
verify(taskChunkMapper, never()).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_boundary_empty_input() throws Exception {
|
|
||||||
// 空输入:无行时缓冲与 merge 都不发生,不创建无效资源
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
|
||||||
TaskScopeStateEntity.class,
|
|
||||||
SimilarAsinTaskService.LlmBatchContext.class,
|
|
||||||
List.class, FileTaskEntity.class, Map.class);
|
|
||||||
bufferOrMerge.setAccessible(true);
|
|
||||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of());
|
|
||||||
bufferOrMerge.invoke(service, state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of());
|
|
||||||
|
|
||||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
|
||||||
verify(taskChunkMapper, never()).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_boundary_single_item() throws Exception {
|
|
||||||
// 单 batch(batchTotal=1):原 P0-3 例外,现在也缓冲
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
|
||||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
|
||||||
TaskScopeStateEntity.class,
|
|
||||||
SimilarAsinTaskService.LlmBatchContext.class,
|
|
||||||
List.class, FileTaskEntity.class, Map.class);
|
|
||||||
bufferOrMerge.setAccessible(true);
|
|
||||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
|
||||||
|
|
||||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
|
||||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
|
||||||
verify(taskChunkMapper, never()).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_boundary_limit_and_overflow() throws Exception {
|
|
||||||
// 缓冲开关关闭:回退立即 merge,DONE 结果仍落 chunk 不丢失
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
stubChunkMerge(chunkRowsJson());
|
|
||||||
when(properties.isLlmResultBufferEnabled()).thenReturn(false);
|
|
||||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
|
||||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
|
||||||
TaskScopeStateEntity.class,
|
|
||||||
SimilarAsinTaskService.LlmBatchContext.class,
|
|
||||||
List.class, FileTaskEntity.class, Map.class);
|
|
||||||
bufferOrMerge.setAccessible(true);
|
|
||||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
|
||||||
|
|
||||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
|
||||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_invalid_input_rejected() throws Exception {
|
|
||||||
// 缓冲写失败(storeParsedPayloadEntry 抛异常):回退立即 merge,结果不丢失
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
stubChunkMerge(chunkRowsJson());
|
|
||||||
when(transientPayloadStorageService.storeParsedPayloadEntry(
|
|
||||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)))
|
|
||||||
.thenThrow(new IllegalStateException("rustfs full"));
|
|
||||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
|
||||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferLlmRowsOrMerge",
|
|
||||||
TaskScopeStateEntity.class,
|
|
||||||
SimilarAsinTaskService.LlmBatchContext.class,
|
|
||||||
List.class, FileTaskEntity.class, Map.class);
|
|
||||||
bufferOrMerge.setAccessible(true);
|
|
||||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
|
||||||
|
|
||||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_012_payload_chunk_dependency_failure_releases_resources() throws Exception {
|
|
||||||
// flush 时 chunk 写失败:抛可识别业务异常且不清 pointer(保留待重试);
|
|
||||||
// 依赖恢复后重试 flush 成功,chunk 合并一次、pointer 清理。
|
|
||||||
FileTaskEntity task = task();
|
|
||||||
stubChunkMerge(chunkRowsJson());
|
|
||||||
TaskScopeStateEntity s = state(task, 1L, "DONE", 1);
|
|
||||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(s));
|
|
||||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
|
||||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
String pointer = invocation.getArgument(0);
|
|
||||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
|
||||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
|
||||||
}
|
|
||||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
|
||||||
return chunkRowsJson();
|
|
||||||
}
|
|
||||||
return "[]";
|
|
||||||
});
|
|
||||||
java.util.concurrent.atomic.AtomicInteger storeCalls = new java.util.concurrent.atomic.AtomicInteger(0);
|
|
||||||
doAnswer(invocation -> {
|
|
||||||
if (storeCalls.incrementAndGet() == 1) {
|
|
||||||
throw new IllegalStateException("rustfs write failed");
|
|
||||||
}
|
|
||||||
return "ptr:stored-" + invocation.getArgument(3);
|
|
||||||
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
|
||||||
|
|
||||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushLlmBufferedResults", Long.class);
|
|
||||||
flush.setAccessible(true);
|
|
||||||
Exception ex = assertThrows(Exception.class, () -> {
|
|
||||||
try {
|
|
||||||
flush.invoke(service, 7104L);
|
|
||||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
|
||||||
throw e.getCause();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("刷新缓冲区"),
|
|
||||||
"flush 失败消息必须可识别, 实际: " + ex.getMessage());
|
|
||||||
// 失败分组不清 pointer:buffer 未被删除、stateJson 未更新,留待重试
|
|
||||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
|
||||||
verify(taskScopeStateMapper, never()).update(any(), any());
|
|
||||||
|
|
||||||
// 恢复后重试 flush:chunk 合并成功一次,pointer 清理
|
|
||||||
flush.invoke(service, 7104L);
|
|
||||||
assertEquals(2, storeCalls.get(), "恢复后重试应再次写 chunk");
|
|
||||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-378
@@ -1,378 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
|
||||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
|
||||||
import org.apache.poi.ss.usermodel.Row;
|
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
|
||||||
import org.junit.jupiter.api.AfterEach;
|
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.Spy;
|
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
|
||||||
import org.mockito.InjectMocks;
|
|
||||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
|
||||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
|
||||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileOutputStream;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
|
||||||
import static org.mockito.Mockito.doAnswer;
|
|
||||||
import static org.mockito.Mockito.lenient;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 6:分组数据改为索引/范围引用,避免 groups 嵌套复制完整行对象。
|
|
||||||
* 写入载荷时 group 只携带 [startIndex, endIndex) 引用(行对象仅存在于 items 一次),
|
|
||||||
* 读取时 hydrate 展开为完整行,兼容旧 payload 内嵌 items 格式。
|
|
||||||
*/
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
|
||||||
class SimilarAsinTaskServiceGroupRefTest {
|
|
||||||
|
|
||||||
private static final AtomicLong NEXT_ID = new AtomicLong(30000);
|
|
||||||
|
|
||||||
@Mock private LocalFileStorageService localFileStorageService;
|
|
||||||
@Mock private FileTaskMapper fileTaskMapper;
|
|
||||||
@Mock private FileResultMapper fileResultMapper;
|
|
||||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
|
||||||
@Mock private TaskChunkMapper taskChunkMapper;
|
|
||||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
|
||||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
|
||||||
@Mock private SimilarAsinProperties properties;
|
|
||||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
|
||||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
|
||||||
|
|
||||||
@InjectMocks private SimilarAsinTaskService service;
|
|
||||||
|
|
||||||
@BeforeAll
|
|
||||||
static void initializeMybatisMetadata() {
|
|
||||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
|
||||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
|
||||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
|
||||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
|
||||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
|
||||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
|
||||||
.thenReturn("rustfs:task-parsed/similar-asin/30000/payload.json");
|
|
||||||
lenient().doAnswer(invocation -> {
|
|
||||||
FileTaskEntity task = invocation.getArgument(0);
|
|
||||||
task.setId(NEXT_ID.incrementAndGet());
|
|
||||||
return 1;
|
|
||||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
|
||||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterEach
|
|
||||||
void shutdown() {
|
|
||||||
service.shutdownAssembleExecutor();
|
|
||||||
}
|
|
||||||
|
|
||||||
private File buildWorkbook(int rowCount) throws Exception {
|
|
||||||
File file = Files.createTempFile("similar-asin-group-ref-", ".xlsx").toFile();
|
|
||||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
|
||||||
var sheet = workbook.createSheet("Sheet1");
|
|
||||||
Row header = sheet.createRow(0);
|
|
||||||
header.createCell(0).setCellValue("id");
|
|
||||||
header.createCell(1).setCellValue("asin");
|
|
||||||
header.createCell(2).setCellValue("国家");
|
|
||||||
for (int i = 1; i <= rowCount; i++) {
|
|
||||||
Row row = sheet.createRow(i);
|
|
||||||
row.createCell(0).setCellValue(String.valueOf(i));
|
|
||||||
row.createCell(1).setCellValue(String.format("B0GRP%05d", i));
|
|
||||||
row.createCell(2).setCellValue("英国");
|
|
||||||
}
|
|
||||||
workbook.write(fos);
|
|
||||||
}
|
|
||||||
return file;
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinParseRequest request(String fileKey) {
|
|
||||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
|
||||||
request.setUserId(7L);
|
|
||||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
|
||||||
sourceFile.setFileKey(fileKey);
|
|
||||||
sourceFile.setOriginalFilename("group-ref.xlsx");
|
|
||||||
request.setFiles(List.of(sourceFile));
|
|
||||||
request.setApiKey("sk-123");
|
|
||||||
request.setImgSwitch(Boolean.FALSE);
|
|
||||||
request.setCategorySwitch(Boolean.FALSE);
|
|
||||||
return request;
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
|
||||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
|
||||||
return service.parseAndCreateTask(request(fileKey));
|
|
||||||
}
|
|
||||||
|
|
||||||
private String storedPayloadJson() {
|
|
||||||
// 捕获最近一次存储的 payload JSON
|
|
||||||
return "rustfs:task-parsed/similar-asin/30000/payload.json";
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinParsedPayloadDto readPayload(String json) throws Exception {
|
|
||||||
return objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SimilarAsinParsedRowVo row(String fileKey, int index, String groupKey) {
|
|
||||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
|
||||||
row.setSourceFileKey(fileKey);
|
|
||||||
row.setSourceFilename("group-ref.xlsx");
|
|
||||||
row.setRowIndex(index);
|
|
||||||
row.setSourceId(String.valueOf(index));
|
|
||||||
row.setDisplayId(String.valueOf(index));
|
|
||||||
row.setRowToken(fileKey + "::row::" + index);
|
|
||||||
row.setGroupKey(groupKey);
|
|
||||||
row.setAsin(String.format("B0GRP%05d", index));
|
|
||||||
row.setCountry("英国");
|
|
||||||
row.setValues(new java.util.LinkedHashMap<>());
|
|
||||||
return row;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_normal_default_path() throws Exception {
|
|
||||||
// 正常多行文件:groups 写入为索引引用,行对象只出现在 items 一次
|
|
||||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
|
||||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
|
||||||
.thenAnswer(invocation -> {
|
|
||||||
String json = invocation.getArgument(3);
|
|
||||||
return "rustfs:task-parsed/similar-asin/30000/payload.json::" + json;
|
|
||||||
});
|
|
||||||
File workbook = buildWorkbook(150);
|
|
||||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-default.xlsx");
|
|
||||||
assertEquals(150, vo.getAcceptedRows());
|
|
||||||
// 每个 group 是索引引用:携带 [startIndex, endIndex),区间宽度等于 itemCount
|
|
||||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
|
||||||
assertNotNull(group.getStartIndex());
|
|
||||||
assertNotNull(group.getEndIndex());
|
|
||||||
assertTrue(group.getStartIndex() < group.getEndIndex());
|
|
||||||
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItemCount());
|
|
||||||
}
|
|
||||||
// 响应 groups 按预览上限裁剪(默认 100),引用区间覆盖全部行、不重叠
|
|
||||||
int coverage = 0;
|
|
||||||
int prevEnd = -1;
|
|
||||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
|
||||||
assertTrue(group.getStartIndex() >= prevEnd, "组区间不能重叠且必须顺序递增");
|
|
||||||
coverage += group.getEndIndex() - group.getStartIndex();
|
|
||||||
prevEnd = group.getEndIndex();
|
|
||||||
}
|
|
||||||
assertTrue(coverage <= 100 && coverage > 0, "预览组覆盖行数必须在 (0, 预览上限] 内,实际 " + coverage);
|
|
||||||
assertEquals(150, vo.getAcceptedRows());
|
|
||||||
// 响应组内嵌预览行(前端兼容):每个组 items 与引用区间宽度一致
|
|
||||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
|
||||||
assertNotNull(group.getItems());
|
|
||||||
assertEquals(group.getEndIndex() - group.getStartIndex(), group.getItems().size(),
|
|
||||||
"响应组内嵌预览行数量必须与引用区间宽度一致");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_normal_multiple_items() throws Exception {
|
|
||||||
// 多组批量:每组行数不同,引用与 items 严格对应且顺序稳定
|
|
||||||
String json = groupRefJson(3, new int[][]{{0, 3}, {3, 8}, {8, 10}});
|
|
||||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
|
||||||
assertEquals(10, payload.getItems().size());
|
|
||||||
assertEquals(3, payload.getGroups().size());
|
|
||||||
for (int g = 0; g < payload.getGroups().size(); g++) {
|
|
||||||
SimilarAsinParsedGroupVo group = payload.getGroups().get(g);
|
|
||||||
int start = group.getStartIndex();
|
|
||||||
int end = group.getEndIndex();
|
|
||||||
assertTrue(end - start >= 1);
|
|
||||||
// 展开后行与 items 对应(首行即 items[start],行内容一致)
|
|
||||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(start, end));
|
|
||||||
assertEquals(end - start, expanded.size());
|
|
||||||
assertEquals("t" + (start + 1), expanded.get(0).getRowToken(), "展开首行必须是 items[start]");
|
|
||||||
assertEquals("B0GRP" + String.format("%05d", start + 1), expanded.get(0).getAsin());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_normal_repeated_operation_is_idempotent() throws Exception {
|
|
||||||
// 重复展开同一 payload:结果一致,且不修改 items
|
|
||||||
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 5}});
|
|
||||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
|
||||||
List<SimilarAsinParsedRowVo> first = hydrateForTest(payload);
|
|
||||||
List<SimilarAsinParsedRowVo> second = hydrateForTest(payload);
|
|
||||||
assertEquals(first.size(), second.size());
|
|
||||||
for (int i = 0; i < first.size(); i++) {
|
|
||||||
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
|
|
||||||
}
|
|
||||||
assertEquals(5, payload.getItems().size(), "展开不能修改 payload 内部状态");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_boundary_empty_input() throws Exception {
|
|
||||||
// 空 groups:引用列表为空,不创建无效引用
|
|
||||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
|
||||||
payload.setItems(List.of());
|
|
||||||
payload.setGroups(List.of());
|
|
||||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(payload);
|
|
||||||
assertNotNull(restored);
|
|
||||||
assertEquals(0, restored.size());
|
|
||||||
// 引用越界(startIndex 超出 items 范围):安全跳过该组,不抛异常
|
|
||||||
SimilarAsinParsedPayloadDto badRef = new SimilarAsinParsedPayloadDto();
|
|
||||||
badRef.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
|
|
||||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
|
||||||
group.setGroupKey("f.xlsx::1");
|
|
||||||
group.setStartIndex(5);
|
|
||||||
group.setEndIndex(7);
|
|
||||||
badRef.setGroups(List.of(group));
|
|
||||||
List<SimilarAsinParsedRowVo> outOfRange = SimilarAsinTaskService.resolveAllRows(badRef);
|
|
||||||
assertEquals(0, outOfRange.size(), "越界引用必须安全跳过");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_boundary_single_item() throws Exception {
|
|
||||||
// 单行单组:区间为 [0,1),单行不依赖批量路径
|
|
||||||
String json = groupRefJson(1, new int[][]{{0, 1}});
|
|
||||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
|
||||||
assertEquals(1, payload.getGroups().size());
|
|
||||||
SimilarAsinParsedGroupVo group = payload.getGroups().get(0);
|
|
||||||
assertEquals(0, group.getStartIndex());
|
|
||||||
assertEquals(1, group.getEndIndex());
|
|
||||||
assertEquals(1, group.getItemCount());
|
|
||||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(0, 1));
|
|
||||||
assertEquals(1, expanded.size());
|
|
||||||
assertEquals("B0GRP00001", expanded.get(0).getAsin());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_boundary_limit_and_overflow() throws Exception {
|
|
||||||
// 组引用到达 items 末尾:endIndex == items.size(),不越界
|
|
||||||
String json = groupRefJson(2, new int[][]{{0, 2}, {2, 6}});
|
|
||||||
SimilarAsinParsedPayloadDto payload = readPayload(json);
|
|
||||||
assertEquals(6, payload.getItems().size());
|
|
||||||
SimilarAsinParsedGroupVo last = payload.getGroups().get(1);
|
|
||||||
assertEquals(6, last.getEndIndex());
|
|
||||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>(payload.getItems().subList(last.getStartIndex(), last.getEndIndex()));
|
|
||||||
assertEquals(4, expanded.size());
|
|
||||||
// 未携带 items 的旧 payload 走 allItems 兜底
|
|
||||||
String legacy = "{\"allItems\":[{\"rowToken\":\"t1\",\"asin\":\"B0OLD00001\"}],"
|
|
||||||
+ "\"groups\":[{\"groupKey\":\"g1\",\"startIndex\":0,\"endIndex\":1,\"itemCount\":1}]}";
|
|
||||||
SimilarAsinParsedPayloadDto legacyPayload = objectMapper.readValue(legacy, SimilarAsinParsedPayloadDto.class);
|
|
||||||
List<SimilarAsinParsedRowVo> restored = SimilarAsinTaskService.resolveAllRows(legacyPayload);
|
|
||||||
assertEquals(1, restored.size());
|
|
||||||
assertEquals("B0OLD00001", restored.get(0).getAsin());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_invalid_input_rejected() throws Exception {
|
|
||||||
// 非法区间:endIndex <= startIndex,安全跳过
|
|
||||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
|
||||||
payload.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1"), row("f.xlsx", 2, "f.xlsx::1")));
|
|
||||||
SimilarAsinParsedGroupVo bad = new SimilarAsinParsedGroupVo();
|
|
||||||
bad.setGroupKey("f.xlsx::1");
|
|
||||||
bad.setStartIndex(1);
|
|
||||||
bad.setEndIndex(1);
|
|
||||||
payload.setGroups(List.of(bad));
|
|
||||||
assertEquals(0, SimilarAsinTaskService.resolveAllRows(payload).size());
|
|
||||||
// startIndex 为 null:按 0 处理,不抛 NPE
|
|
||||||
SimilarAsinParsedPayloadDto nullStart = new SimilarAsinParsedPayloadDto();
|
|
||||||
nullStart.setItems(List.of(row("f.xlsx", 1, "f.xlsx::1")));
|
|
||||||
SimilarAsinParsedGroupVo g = new SimilarAsinParsedGroupVo();
|
|
||||||
g.setGroupKey("f.xlsx::1");
|
|
||||||
g.setStartIndex(null);
|
|
||||||
g.setEndIndex(1);
|
|
||||||
nullStart.setGroups(List.of(g));
|
|
||||||
assertEquals(1, SimilarAsinTaskService.resolveAllRows(nullStart).size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_006_group_dependency_failure_releases_resources() throws Exception {
|
|
||||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,groups 引用与行一致
|
|
||||||
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
|
||||||
File workbook = buildWorkbook(80);
|
|
||||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/gr-fail.xlsx")).thenReturn(workbook);
|
|
||||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
|
||||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
|
||||||
.thenThrow(new IllegalStateException("rustfs down"))
|
|
||||||
.thenReturn("rustfs:task-parsed/similar-asin/30001/payload.json");
|
|
||||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/gr-fail.xlsx"));
|
|
||||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/gr-recovered.xlsx");
|
|
||||||
assertEquals(80, vo.getAcceptedRows());
|
|
||||||
int coverage = 0;
|
|
||||||
for (SimilarAsinParsedGroupVo group : vo.getGroups()) {
|
|
||||||
assertTrue(group.getStartIndex() < group.getEndIndex());
|
|
||||||
coverage += group.getEndIndex() - group.getStartIndex();
|
|
||||||
}
|
|
||||||
assertEquals(80, coverage);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- helpers ----
|
|
||||||
|
|
||||||
private static String groupRefJson(int groupCount, int[][] ranges) {
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("{\"items\":[");
|
|
||||||
// 计算总行数
|
|
||||||
int maxEnd = 0;
|
|
||||||
for (int[] r : ranges) {
|
|
||||||
maxEnd = Math.max(maxEnd, r[1]);
|
|
||||||
}
|
|
||||||
for (int i = 1; i <= maxEnd; i++) {
|
|
||||||
if (i > 1) {
|
|
||||||
sb.append(",");
|
|
||||||
}
|
|
||||||
sb.append("{\"rowToken\":\"t").append(i).append("\",\"asin\":\"B0GRP")
|
|
||||||
.append(String.format("%05d", i)).append("\",\"sourceFileKey\":\"f.xlsx\",\"rowIndex\":")
|
|
||||||
.append(i).append("}");
|
|
||||||
}
|
|
||||||
sb.append("],\"groups\":[");
|
|
||||||
for (int g = 0; g < groupCount; g++) {
|
|
||||||
if (g > 0) {
|
|
||||||
sb.append(",");
|
|
||||||
}
|
|
||||||
sb.append("{\"groupKey\":\"g").append(g + 1).append("\",\"startIndex\":")
|
|
||||||
.append(ranges[g][0]).append(",\"endIndex\":").append(ranges[g][1])
|
|
||||||
.append(",\"itemCount\":").append(ranges[g][1] - ranges[g][0]).append("}");
|
|
||||||
}
|
|
||||||
sb.append("]}");
|
|
||||||
return sb.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<SimilarAsinParsedRowVo> hydrateForTest(SimilarAsinParsedPayloadDto payload) {
|
|
||||||
// 调用 service 的引用展开实现(与 hydrateParsedPayloadRows 语义一致)
|
|
||||||
SimilarAsinParsedPayloadDto copy = new SimilarAsinParsedPayloadDto();
|
|
||||||
copy.setItems(payload.getItems());
|
|
||||||
copy.setAllItems(payload.getAllItems());
|
|
||||||
copy.setGroups(payload.getGroups());
|
|
||||||
return SimilarAsinTaskService.expandGroupRefs(copy);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-275
@@ -1,275 +0,0 @@
|
|||||||
package com.nanri.aiimage.modules.similarasin.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
|
||||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
|
||||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
|
||||||
import org.apache.poi.ss.usermodel.Row;
|
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
|
||||||
import org.junit.jupiter.api.AfterEach;
|
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
import org.mockito.Spy;
|
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
|
||||||
import org.mockito.InjectMocks;
|
|
||||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
|
||||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
|
||||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileOutputStream;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
|
||||||
import static org.mockito.Mockito.doAnswer;
|
|
||||||
import static org.mockito.Mockito.lenient;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Task 7:限制单文件大小、最大行数和最大字段长度,防止解析任务无界增长。
|
|
||||||
* 超限输入在解析入口被拒绝或截断;mock 依赖 + 真实 xlsx 验证边界行为。
|
|
||||||
*/
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
|
||||||
class SimilarAsinTaskServiceParseLimitsTest {
|
|
||||||
|
|
||||||
private static final AtomicLong NEXT_ID = new AtomicLong(40000);
|
|
||||||
|
|
||||||
@Mock private LocalFileStorageService localFileStorageService;
|
|
||||||
@Mock private FileTaskMapper fileTaskMapper;
|
|
||||||
@Mock private FileResultMapper fileResultMapper;
|
|
||||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
|
||||||
@Mock private TaskChunkMapper taskChunkMapper;
|
|
||||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
|
||||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
|
||||||
@Mock private SimilarAsinProperties properties;
|
|
||||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
|
||||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
|
||||||
|
|
||||||
@InjectMocks private SimilarAsinTaskService service;
|
|
||||||
|
|
||||||
@BeforeAll
|
|
||||||
static void initializeMybatisMetadata() {
|
|
||||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
|
||||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
|
||||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
|
||||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
|
||||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
|
||||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
|
||||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
|
||||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
|
||||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
|
||||||
.thenReturn("rustfs:task-parsed/similar-asin/40000/payload.json");
|
|
||||||
lenient().doAnswer(invocation -> {
|
|
||||||
FileTaskEntity task = invocation.getArgument(0);
|
|
||||||
task.setId(NEXT_ID.incrementAndGet());
|
|
||||||
return 1;
|
|
||||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
|
||||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
|
||||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterEach
|
|
||||||
void shutdown() {
|
|
||||||
service.shutdownAssembleExecutor();
|
|
||||||
}
|
|
||||||
|
|
||||||
private File buildWorkbook(int rowCount) throws Exception {
|
|
||||||
return buildWorkbookWithAsin(rowCount, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
private File buildWorkbookWithAsin(int rowCount, String asinValue) throws Exception {
|
|
||||||
File file = Files.createTempFile("similar-asin-parse-limits-", ".xlsx").toFile();
|
|
||||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
|
||||||
var sheet = workbook.createSheet("Sheet1");
|
|
||||||
Row header = sheet.createRow(0);
|
|
||||||
header.createCell(0).setCellValue("id");
|
|
||||||
header.createCell(1).setCellValue("asin");
|
|
||||||
header.createCell(2).setCellValue("国家");
|
|
||||||
for (int i = 1; i <= rowCount; i++) {
|
|
||||||
Row row = sheet.createRow(i);
|
|
||||||
row.createCell(0).setCellValue(String.valueOf(i));
|
|
||||||
row.createCell(1).setCellValue(asinValue != null ? asinValue : String.format("B0LIM%05d", i));
|
|
||||||
row.createCell(2).setCellValue("英国");
|
|
||||||
}
|
|
||||||
workbook.write(fos);
|
|
||||||
}
|
|
||||||
return file;
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinParseRequest request(String fileKey) {
|
|
||||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
|
||||||
request.setUserId(7L);
|
|
||||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
|
||||||
sourceFile.setFileKey(fileKey);
|
|
||||||
sourceFile.setOriginalFilename("limits.xlsx");
|
|
||||||
request.setFiles(List.of(sourceFile));
|
|
||||||
request.setApiKey("sk-123");
|
|
||||||
request.setImgSwitch(Boolean.FALSE);
|
|
||||||
request.setCategorySwitch(Boolean.FALSE);
|
|
||||||
return request;
|
|
||||||
}
|
|
||||||
|
|
||||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
|
||||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
|
||||||
return service.parseAndCreateTask(request(fileKey));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_normal_default_path() throws Exception {
|
|
||||||
// 默认配置(50MB/50000 行/2000 字符):正常文件解析成功,行数不丢失
|
|
||||||
File workbook = buildWorkbook(120);
|
|
||||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-default.xlsx");
|
|
||||||
assertEquals(120, vo.getAcceptedRows());
|
|
||||||
assertEquals(120, vo.getTotalRows());
|
|
||||||
assertEquals(100, vo.getItems().size());
|
|
||||||
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
|
|
||||||
// 源文件大小在限制内
|
|
||||||
assertTrue(workbook.length() <= 50L * 1024L * 1024L);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_normal_multiple_items() throws Exception {
|
|
||||||
// 多文件批量:每个文件都在限制内,汇总不丢行
|
|
||||||
File workbookA = buildWorkbook(30);
|
|
||||||
File workbookB = buildWorkbook(40);
|
|
||||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-a.xlsx")).thenReturn(workbookA);
|
|
||||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-b.xlsx")).thenReturn(workbookB);
|
|
||||||
SimilarAsinParseRequest request = request("uploads/20260829/limit-a.xlsx");
|
|
||||||
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
|
|
||||||
sourceB.setFileKey("uploads/20260829/limit-b.xlsx");
|
|
||||||
sourceB.setOriginalFilename("limits-b.xlsx");
|
|
||||||
request.setFiles(List.of(request.getFiles().get(0), sourceB));
|
|
||||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
|
||||||
assertEquals(70, vo.getAcceptedRows());
|
|
||||||
assertEquals(70, vo.getTotalRows());
|
|
||||||
assertNotNull(vo.getTaskId());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_normal_repeated_operation_is_idempotent() throws Exception {
|
|
||||||
// 重复解析同一文件:结果一致,不产生重复状态
|
|
||||||
File workbook = buildWorkbook(60);
|
|
||||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/limit-idem.xlsx");
|
|
||||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/limit-idem.xlsx");
|
|
||||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
|
||||||
assertEquals(first.getItems().size(), second.getItems().size());
|
|
||||||
for (int i = 0; i < first.getItems().size(); i++) {
|
|
||||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_boundary_empty_input() throws Exception {
|
|
||||||
// 空文件(无有效数据行):抛业务异常,不创建任务
|
|
||||||
File workbook = buildWorkbook(0);
|
|
||||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-empty.xlsx")).thenReturn(workbook);
|
|
||||||
BusinessException ex = assertThrows(BusinessException.class,
|
|
||||||
() -> parse(workbook, "uploads/20260829/limit-empty.xlsx"));
|
|
||||||
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_boundary_single_item() throws Exception {
|
|
||||||
// 单行小文件:不依赖批量路径,结果正确
|
|
||||||
File workbook = buildWorkbook(1);
|
|
||||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-single.xlsx");
|
|
||||||
assertEquals(1, vo.getAcceptedRows());
|
|
||||||
assertEquals(1, vo.getItems().size());
|
|
||||||
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
|
|
||||||
assertEquals(1, vo.getGroupCount());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_boundary_limit_and_overflow() throws Exception {
|
|
||||||
// 行数恰好等于上限:允许
|
|
||||||
when(properties.getMaxParseRows()).thenReturn(8);
|
|
||||||
File workbook = buildWorkbook(8);
|
|
||||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-exact.xlsx");
|
|
||||||
assertEquals(8, vo.getAcceptedRows());
|
|
||||||
// 行数超过上限:拒绝,且不创建任务
|
|
||||||
when(properties.getMaxParseRows()).thenReturn(3);
|
|
||||||
File workbookOver = buildWorkbook(4);
|
|
||||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-over.xlsx")).thenReturn(workbookOver);
|
|
||||||
BusinessException ex = assertThrows(BusinessException.class,
|
|
||||||
() -> parse(workbookOver, "uploads/20260829/limit-over.xlsx"));
|
|
||||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("行数"),
|
|
||||||
"超行数异常消息必须可识别,实际: " + ex.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_invalid_input_rejected() throws Exception {
|
|
||||||
// 文件大小超限:拒绝,异常消息可识别
|
|
||||||
when(properties.getMaxSourceFileBytes()).thenReturn(64L);
|
|
||||||
File workbook = buildWorkbook(5);
|
|
||||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-bigfile.xlsx")).thenReturn(workbook);
|
|
||||||
assertTrue(workbook.length() > 64L, "测试文件必须超过 64 字节限制,实际 " + workbook.length());
|
|
||||||
BusinessException ex = assertThrows(BusinessException.class,
|
|
||||||
() -> parse(workbook, "uploads/20260829/limit-bigfile.xlsx"));
|
|
||||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("大小限制"),
|
|
||||||
"超文件大小异常消息必须可识别,实际: " + ex.getMessage());
|
|
||||||
// 字段长度超限:截断而非拒绝,字段仍非空
|
|
||||||
when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
|
||||||
when(properties.getMaxFieldLength()).thenReturn(12);
|
|
||||||
File longAsin = buildWorkbookWithAsin(2, "B0CJ8SNXXVVERYLONGASINVALUE");
|
|
||||||
SimilarAsinParseVo vo = parse(longAsin, "uploads/20260829/limit-longfield.xlsx");
|
|
||||||
assertEquals(2, vo.getAcceptedRows());
|
|
||||||
for (var item : vo.getItems()) {
|
|
||||||
assertTrue(item.getAsin().length() <= 12, "超长字段必须截断到配置上限");
|
|
||||||
assertTrue(!item.getAsin().isBlank());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test_task_007_file_size_row_count_dependency_failure_releases_resources() throws Exception {
|
|
||||||
// RustFS 存储失败:解析抛异常;恢复后重试成功,无残留状态
|
|
||||||
File workbook = buildWorkbook(40);
|
|
||||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-fail.xlsx")).thenReturn(workbook);
|
|
||||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
|
||||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
|
||||||
.thenThrow(new IllegalStateException("rustfs down"))
|
|
||||||
.thenReturn("rustfs:task-parsed/similar-asin/40001/payload.json");
|
|
||||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/limit-fail.xlsx"));
|
|
||||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-recovered.xlsx");
|
|
||||||
assertEquals(40, vo.getAcceptedRows());
|
|
||||||
assertEquals(40, vo.getItems().size());
|
|
||||||
// 行数/文件大小/字段长度默认值均处于有效区间
|
|
||||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
|
||||||
assertNotNull(defaults.getMaxParseRows());
|
|
||||||
assertNotNull(defaults.getMaxSourceFileBytes());
|
|
||||||
assertNotNull(defaults.getMaxFieldLength());
|
|
||||||
assertTrue(defaults.getMaxParseRows() >= 1000, "默认最大行数至少 1000");
|
|
||||||
assertTrue(defaults.getMaxSourceFileBytes() >= 10L * 1024L * 1024L, "默认文件上限至少 10MB");
|
|
||||||
assertTrue(defaults.getMaxFieldLength() >= 500, "默认字段上限至少 500 字符");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user