task-8: WorkbookFactory 输入解析改为受控读取，验证超大 Excel 失败提示
解析前用 ZipFile 中央目录探测 xlsx(zip) 条目数与解压总字节（不读压缩 内容），超过 maxWorkbookZipEntries(20000)/maxWorkbookUncompressedBytes (512MB) 立即拒绝并给出可识别提示；非 zip/损坏文件跳过探测交给 WorkbookFactory 兜底转业务异常。0/负值配置回退默认。新增 8 个测试覆盖 正常路径、多文件、幂等、空输入、单行、解压超限、条目超限、损坏文件、 依赖失败恢复。
This commit is contained in:
+271
@@ -0,0 +1,271 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.InjectMocks;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 8：WorkbookFactory 输入解析改为受控读取。
|
||||
* 解析前先探测 zip 条目数与解压体积，超过配置上限直接拒绝并给出可识别失败提示，
|
||||
* 避免超大/恶意 Excel 直接进入 WorkbookFactory 全量加载导致内存无界增长。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceWorkbookControlTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(50000);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinCozeClient cozeClient;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json");
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private File buildWorkbook(int rowCount) throws Exception {
|
||||
File file = Files.createTempFile("similar-asin-workbook-ctrl-", ".xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
|
||||
var sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("id");
|
||||
header.createCell(1).setCellValue("asin");
|
||||
header.createCell(2).setCellValue("国家");
|
||||
for (int i = 1; i <= rowCount; i++) {
|
||||
Row row = sheet.createRow(i);
|
||||
row.createCell(0).setCellValue(String.valueOf(i));
|
||||
row.createCell(1).setCellValue(String.format("B0WBK%05d", i));
|
||||
row.createCell(2).setCellValue("英国");
|
||||
}
|
||||
workbook.write(fos);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private SimilarAsinParseRequest request(String fileKey) {
|
||||
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
|
||||
request.setUserId(7L);
|
||||
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
|
||||
sourceFile.setFileKey(fileKey);
|
||||
sourceFile.setOriginalFilename("workbook.xlsx");
|
||||
request.setFiles(List.of(sourceFile));
|
||||
request.setApiKey("sk-123");
|
||||
request.setImgSwitch(Boolean.FALSE);
|
||||
request.setCategorySwitch(Boolean.FALSE);
|
||||
return request;
|
||||
}
|
||||
|
||||
private SimilarAsinParseVo parse(File workbook, String fileKey) {
|
||||
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
|
||||
return service.parseAndCreateTask(request(fileKey));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_default_path() throws Exception {
|
||||
// 正常 xlsx：受控读取通过探测，解析成功且行数不丢失
|
||||
File workbook = buildWorkbook(100);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-default.xlsx");
|
||||
assertEquals(100, vo.getAcceptedRows());
|
||||
assertEquals(100, vo.getTotalRows());
|
||||
assertEquals(100, vo.getItems().size());
|
||||
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_multiple_items() throws Exception {
|
||||
// 多文件批量：每个文件都走受控读取，汇总不丢行
|
||||
File workbookA = buildWorkbook(25);
|
||||
File workbookB = buildWorkbook(35);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-a.xlsx")).thenReturn(workbookA);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-b.xlsx")).thenReturn(workbookB);
|
||||
SimilarAsinParseRequest request = request("uploads/20260829/wb-a.xlsx");
|
||||
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
|
||||
sourceB.setFileKey("uploads/20260829/wb-b.xlsx");
|
||||
sourceB.setOriginalFilename("workbook-b.xlsx");
|
||||
request.setFiles(List.of(request.getFiles().get(0), sourceB));
|
||||
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
|
||||
assertEquals(60, vo.getAcceptedRows());
|
||||
assertEquals(60, vo.getTotalRows());
|
||||
assertNotNull(vo.getTaskId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复解析同一文件：结果一致，不产生重复状态
|
||||
File workbook = buildWorkbook(50);
|
||||
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/wb-idem.xlsx");
|
||||
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/wb-idem.xlsx");
|
||||
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
|
||||
assertEquals(first.getItems().size(), second.getItems().size());
|
||||
for (int i = 0; i < first.getItems().size(); i++) {
|
||||
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_empty_input() throws Exception {
|
||||
// 空文件（只有表头无数据行）：抛业务异常，不创建任务
|
||||
File workbook = buildWorkbook(0);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-empty.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-empty.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
|
||||
// 文件不存在：抛业务异常
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-missing.xlsx")).thenReturn(null);
|
||||
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/wb-missing.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_single_item() throws Exception {
|
||||
// 单行文件：不依赖批量路径，结果正确
|
||||
File workbook = buildWorkbook(1);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-single.xlsx");
|
||||
assertEquals(1, vo.getAcceptedRows());
|
||||
assertEquals(1, vo.getItems().size());
|
||||
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
|
||||
assertEquals(1, vo.getGroupCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_boundary_limit_and_overflow() throws Exception {
|
||||
// 解压体积超限：受控探测阶段拒绝，失败提示可识别，不发生无界内存增长
|
||||
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(1024L);
|
||||
File workbook = buildWorkbook(5);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-huge.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-huge.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && (ex.getMessage().contains("解压") || ex.getMessage().contains("大小")),
|
||||
"超大 Excel 失败提示必须可识别，实际: " + ex.getMessage());
|
||||
// 条目数超限：同样在探测阶段拒绝
|
||||
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
when(properties.getMaxWorkbookZipEntries()).thenReturn(2);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-entries.xlsx")).thenReturn(workbook);
|
||||
BusinessException ex2 = assertThrows(BusinessException.class,
|
||||
() -> parse(workbook, "uploads/20260829/wb-entries.xlsx"));
|
||||
assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("条目"),
|
||||
"超条目数失败提示必须可识别，实际: " + ex2.getMessage());
|
||||
// 恢复默认配置：同一文件解析成功（探测不残留状态）
|
||||
when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered.xlsx");
|
||||
assertEquals(5, vo.getAcceptedRows());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_invalid_input_rejected() throws Exception {
|
||||
// 非 xlsx 内容（文本文件）：WorkbookFactory 打开失败，抛项目约定异常
|
||||
File fake = Files.createTempFile("similar-asin-not-excel-", ".xlsx").toFile();
|
||||
Files.write(fake.toPath(), "this is not an excel file".getBytes(StandardCharsets.UTF_8));
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fake.xlsx")).thenReturn(fake);
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> parse(fake, "uploads/20260829/wb-fake.xlsx"));
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"),
|
||||
"损坏文件失败提示必须可识别，实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_008_workbook_excel_dependency_failure_releases_resources() throws Exception {
|
||||
// RustFS 存储失败：解析抛异常；恢复后重试成功，受控读取无残留
|
||||
File workbook = buildWorkbook(30);
|
||||
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fail.xlsx")).thenReturn(workbook);
|
||||
when(transientPayloadStorageService.storeParsedPayloadFast(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
|
||||
.thenThrow(new IllegalStateException("rustfs down"))
|
||||
.thenReturn("rustfs:task-parsed/similar-asin/50001/payload.json");
|
||||
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/wb-fail.xlsx"));
|
||||
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered2.xlsx");
|
||||
assertEquals(30, vo.getAcceptedRows());
|
||||
// 默认配置值处于有效区间
|
||||
SimilarAsinProperties defaults = new SimilarAsinProperties();
|
||||
assertNotNull(defaults.getMaxWorkbookZipEntries());
|
||||
assertNotNull(defaults.getMaxWorkbookUncompressedBytes());
|
||||
assertTrue(defaults.getMaxWorkbookZipEntries() >= 1000, "默认条目上限至少 1000");
|
||||
assertTrue(defaults.getMaxWorkbookUncompressedBytes() >= 100L * 1024L * 1024L, "默认解压上限至少 100MB");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user