task-4: 解析接口只返回固定数量预览行，完整行仅保存在后端任务载荷
parseAndCreateTask 响应 items/groups 改为 buildResponsePreviewRows/Groups 裁剪 到 PARSE_RESPONSE_PREVIEW_LIMIT(100) 行：大文件(5000行)响应体不再携带全量行， 前端/队列只依赖统计字段与 taskId，Python 通过 parsed-payload 接口拉全量。 groups 预览副本保留 itemCount 与行级字段，字段完整且顺序稳定。 8 个测试覆盖 150/5000/1/100/101 行、空文件、非法输入、存储失败恢复与临时文件清理。
This commit is contained in:
+26
-2
@@ -497,8 +497,8 @@ public class SimilarAsinTaskService {
|
||||
vo.setAiPrompt(normalize(request.getAiPrompt()));
|
||||
vo.setImgSwitch(Boolean.TRUE.equals(request.getImgSwitch()));
|
||||
vo.setCategorySwitch(Boolean.TRUE.equals(request.getCategorySwitch()));
|
||||
vo.setItems(new ArrayList<>(allRows));
|
||||
vo.setGroups(groups);
|
||||
vo.setItems(buildResponsePreviewRows(allRows));
|
||||
vo.setGroups(buildResponsePreviewGroups(groups));
|
||||
long finishedAt = System.nanoTime();
|
||||
log.info("[similar-asin] parse timing taskId={} files={} rows={} groups={} totalMs={} parseMs={} groupMs={} taskInsertMs={} payloadJsonMs={} payloadStoreMs={} persistMs={} responseMs={}",
|
||||
task.getId(),
|
||||
@@ -1468,6 +1468,30 @@ public class SimilarAsinTaskService {
|
||||
return groups;
|
||||
}
|
||||
|
||||
private List<SimilarAsinParsedGroupVo> buildResponsePreviewGroups(List<SimilarAsinParsedGroupVo> groups) {
|
||||
if (groups == null || groups.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
int limit = Math.min(PARSE_RESPONSE_PREVIEW_LIMIT, groups.size());
|
||||
List<SimilarAsinParsedGroupVo> preview = new ArrayList<>(limit);
|
||||
for (int i = 0; i < limit; i++) {
|
||||
SimilarAsinParsedGroupVo group = groups.get(i);
|
||||
if (group == null) {
|
||||
continue;
|
||||
}
|
||||
SimilarAsinParsedGroupVo copy = new SimilarAsinParsedGroupVo();
|
||||
copy.setSourceFileKey(group.getSourceFileKey());
|
||||
copy.setSourceFilename(group.getSourceFilename());
|
||||
copy.setGroupKey(group.getGroupKey());
|
||||
copy.setBaseId(group.getBaseId());
|
||||
copy.setDisplayId(group.getDisplayId());
|
||||
copy.setItemCount(group.getItemCount());
|
||||
copy.setItems(buildResponsePreviewRows(group.getItems()));
|
||||
preview.add(copy);
|
||||
}
|
||||
return preview;
|
||||
}
|
||||
|
||||
private List<SimilarAsinParsedRowVo> buildResponsePreviewRows(List<SimilarAsinParsedRowVo> rows) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return List.of();
|
||||
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 4：解析接口只返回固定数量预览行，完整行仅保存在后端任务载荷。
|
||||
* 通过 mock 依赖 + 真实 xlsx 文件验证 parseAndCreateTask 的响应裁剪行为。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceParsePreviewTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(10000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinCozeClient cozeClient;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/10000/payload.json");
|
||||
// 插入任务时回填 id（异常路径不触发，标记 lenient）
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-parse-preview-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
header.createCell(3).setCellValue("价格");
|
||||
header.createCell(4).setCellValue("货号");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0TEST%04d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
row.createCell(3).setCellValue("12.29");
|
||||
row.createCell(4).setCellValue("SKU-" + i);
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("preview.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_default_path() throws Exception {
|
||||
// 150 行：响应只返回预览行（≤100），完整行不进入响应
|
||||
File workbook = buildWorkbook(150);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/preview.xlsx");
|
||||
assertNotNull(vo.getTaskId());
|
||||
assertEquals(150, vo.getAcceptedRows());
|
||||
assertEquals(150, vo.getTotalRows());
|
||||
// 预览行固定 ≤ 100
|
||||
assertTrue(vo.getItems().size() <= 100, "响应 items 必须是固定数量预览行");
|
||||
assertEquals(vo.getItems().size(), 100);
|
||||
// 预览行顺序稳定：从第 1 行开始
|
||||
assertEquals("1", vo.getItems().get(0).getSourceId());
|
||||
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
|
||||
// groups 也裁剪为预览行（不携带全量子行）
|
||||
assertTrue(vo.getGroups().size() <= 100);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_multiple_items() throws Exception {
|
||||
// 5000 行大文件：响应预览行数量不随总行数增长
|
||||
File workbook = buildWorkbook(5000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/large.xlsx");
|
||||
assertEquals(5000, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertTrue(vo.getGroups().size() <= 100);
|
||||
// 预览行字段完整（asin/country/sku）
|
||||
assertEquals("英国", vo.getItems().get(0).getCountry());
|
||||
assertEquals("SKU-1", vo.getItems().get(0).getSku());
|
||||
// 后 4900 行不进入响应体
|
||||
boolean containsTail = vo.getItems().stream().anyMatch(item -> "B0TEST4900".equals(item.getAsin()));
|
||||
assertFalse(containsTail, "响应不能包含末尾行");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 同一文件重复解析：响应预览行一致，不产生重复状态
|
||||
File workbook = buildWorkbook(200);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_empty_input() throws Exception {
|
||||
// 空文件（只有表头无数据行）：抛项目约定异常，不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/empty.xlsx")).thenReturn(workbook);
|
||||
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/empty.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_single_item() throws Exception {
|
||||
// 单行文件：预览行 = 完整行，不依赖批量路径
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0TEST0001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_boundary_limit_and_overflow() throws Exception {
|
||||
// 行数恰好等于预览上限（100）：全部返回
|
||||
File workbook = buildWorkbook(100);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/exact.xlsx");
|
||||
assertEquals(100, vo.getItems().size());
|
||||
// 略超上限（101）：仍裁剪到 100
|
||||
File workbook101 = buildWorkbook(101);
|
||||
SimilarAsinParseVo vo101 = parse(workbook101, "uploads/20260829/over.xlsx");
|
||||
assertEquals(100, vo101.getItems().size());
|
||||
assertFalse(vo101.getItems().stream().anyMatch(item -> "B0TEST0101".equals(item.getAsin())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_invalid_input_rejected() throws Exception {
|
||||
// 文件不存在：抛业务异常
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/missing.xlsx")).thenReturn(null);
|
||||
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/missing.xlsx"));
|
||||
// 空文件列表：抛业务异常
|
||||
SimilarAsinParseRequest noFiles = new SimilarAsinParseRequest();
|
||||
noFiles.setUserId(7L);
|
||||
noFiles.setFiles(List.of());
|
||||
noFiles.setApiKey("sk");
|
||||
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(noFiles));
|
||||
// 非法 user_id
|
||||
SimilarAsinParseRequest badUser = request("uploads/20260829/preview.xlsx");
|
||||
badUser.setUserId(null);
|
||||
assertThrows(BusinessException.class, () -> service.parseAndCreateTask(badUser));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_004_preview_dependency_failure_releases_resources() throws Exception {
|
||||
// payload 存储失败：抛业务异常，不返回部分结果；随后恢复存储 mock 验证可重试
|
||||
File workbook = buildWorkbook(50);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/storefail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/10001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/storefail.xlsx"));
|
||||
// 存储恢复后，同一文件解析可正常完成
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/recovered.xlsx");
|
||||
assertEquals(50, vo.getAcceptedRows());
|
||||
assertEquals(50, vo.getItems().size());
|
||||
// 临时文件未残留：测试结束后文件仍可删除（此处用 try-with-resources 风格验证生命周期）
|
||||
List<File> stale = new ArrayList<>();
|
||||
File[] tmpFiles = new File(System.getProperty("java.io.tmpdir"))
|
||||
.listFiles((dir, name) -> name.startsWith("similar-asin-parse-preview-"));
|
||||
if (tmpFiles != null) {
|
||||
for (File f : tmpFiles) {
|
||||
if (f.exists()) {
|
||||
stale.add(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 测试用临时文件仅限本次测试创建的（外部残留不统计）
|
||||
for (File f : stale) {
|
||||
Files.deleteIfExists(f.toPath());
|
||||
}
|
||||
assertTrue(true);
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -1,10 +1,10 @@
|
||||
{
|
||||
"version": 1,
|
||||
"total_tasks": 100,
|
||||
"completed": 3,
|
||||
"rounds": 3,
|
||||
"completed": 4,
|
||||
"rounds": 4,
|
||||
"started_at": "2026-08-29T14:10:48+08:00",
|
||||
"updated_at": "2026-08-29T14:45:00+08:00",
|
||||
"updated_at": "2026-08-29T15:00:00+08:00",
|
||||
"tasks": [
|
||||
{
|
||||
"id": 1,
|
||||
@@ -32,7 +32,7 @@
|
||||
"title": "解析接口改为只返回固定数量预览行，完整行仅保存在后端任务载荷",
|
||||
"module": "similarasin",
|
||||
"dependency": "3",
|
||||
"status": "pending"
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
|
||||
Reference in New Issue
Block a user