From 3b051bbafe0d1a466ea873dab60a163f407e5eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 29 Aug 2026 15:11:53 +0800 Subject: [PATCH] =?UTF-8?q?task-8:=20WorkbookFactory=20=E8=BE=93=E5=85=A5?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E6=94=B9=E4=B8=BA=E5=8F=97=E6=8E=A7=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=EF=BC=8C=E9=AA=8C=E8=AF=81=E8=B6=85=E5=A4=A7=20Excel?= =?UTF-8?q?=20=E5=A4=B1=E8=B4=A5=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解析前用 ZipFile 中央目录探测 xlsx(zip) 条目数与解压总字节（不读压缩 内容），超过 maxWorkbookZipEntries(20000)/maxWorkbookUncompressedBytes (512MB) 立即拒绝并给出可识别提示；非 zip/损坏文件跳过探测交给 WorkbookFactory 兜底转业务异常。0/负值配置回退默认。新增 8 个测试覆盖 正常路径、多文件、幂等、空输入、单行、解压超限、条目超限、损坏文件、 依赖失败恢复。 --- .../aiimage/config/SimilarAsinProperties.java | 11 + .../service/SimilarAsinTaskService.java | 64 ++++- ...larAsinTaskServiceWorkbookControlTest.java | 271 ++++++++++++++++++ 3 files changed, 343 insertions(+), 3 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceWorkbookControlTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java index 36f840d4..fb636031 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java @@ -154,6 +154,17 @@ public class SimilarAsinProperties { */ private int maxFieldLength = 2000; + /** + * Task 8：xlsx(zip) 最大条目数。受控读取在 WorkBookFactory 打开前探测， + * 超过则拒绝，防止 zip bomb / 超大工作簿拖垮内存。 + */ + private int maxWorkbookZipEntries = 20000; + + /** + * Task 8：xlsx(zip) 解压后总字节数上限。同样在打开前探测，超过则拒绝。 + */ + private long maxWorkbookUncompressedBytes = 512L * 1024L * 1024L; + /** * P0-4：抢 Coze 提交锁失败后下次重试间隔（毫秒）。 * 原硬编码 500ms，会在指数退避算法中作为基础值（500/1000/2000/4000ms 上限 4000）。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 79699659..02f82b26 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -180,7 +180,7 @@ public class SimilarAsinTaskService { } /** - * Task 7：单字段最大长度（字符）。0/负值回退默认 2000。 + * Task 8：单字段最大长度（字符）。0/负值回退默认 2000。 */ private int resolveMaxFieldLength() { Integer configured = properties.getMaxFieldLength(); @@ -189,6 +189,56 @@ public class SimilarAsinTaskService { } return configured; } + + /** + * Task 8：xlsx(zip) 最大条目数。0/负值回退默认 20000。 + */ + private int resolveMaxWorkbookZipEntries() { + Integer configured = properties.getMaxWorkbookZipEntries(); + if (configured == null || configured <= 0) { + return 20000; + } + return configured; + } + + /** + * Task 8：xlsx(zip) 解压后总字节数上限。0/负值回退默认 512MB。 + */ + private long resolveMaxWorkbookUncompressedBytes() { + Long configured = properties.getMaxWorkbookUncompressedBytes(); + if (configured == null || configured <= 0) { + return 512L * 1024L * 1024L; + } + return configured; + } + + /** + * Task 8：WorkbookFactory 打开前的受控读取。xlsx 本质是 zip， + * 先用 ZipFile 中央目录探测条目数与解压总字节（不读取压缩内容）， + * 超限立即拒绝，避免 zip bomb / 超大工作簿直接进入全量加载。 + * 非 zip 文件（或损坏文件）由调用方 catch 转业务异常。 + */ + private void probeWorkbookZipBounds(File input, String sourceName) throws Exception { + int maxEntries = resolveMaxWorkbookZipEntries(); + long maxBytes = resolveMaxWorkbookUncompressedBytes(); + try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(input)) { + if (zipFile.size() > maxEntries) { + throw new BusinessException("Excel 条目数超过上限: " + zipFile.size() + " entries > " + maxEntries + " entries"); + } + long total = 0; + var entries = zipFile.entries(); + while (entries.hasMoreElements()) { + var entry = entries.nextElement(); + long size = entry.getSize(); + if (size >= 0) { + total += size; + if (total > maxBytes) { + throw new BusinessException("Excel 解压体积超过上限: " + total + " bytes > " + maxBytes + " bytes"); + } + } + } + } + } /** * P0-2 最小风险变体：poll 调度阶段并发预取 Coze HTTP 结果时， * 控制对单个 Coze 后端的并发度。8 与 cozeTaskExecutor 的 12 并发上限对齐留 4 余量， @@ -5268,8 +5318,16 @@ public class SimilarAsinTaskService { private ParsedWorkbook parseWorkbook(File input, SimilarAsinSourceFileDto source) { DataFormatter formatter = new DataFormatter(); - try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { - Sheet sheet = workbook.getSheetAt(0); + try { + // Task 8：受控读取——先探测 zip 条目数与解压体积，超限拒绝 + probeWorkbookZipBounds(input, source.getOriginalFilename()); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + // 非 zip 或损坏文件：留给 WorkbookFactory 尝试后由下方 catch 转业务异常 + log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage()); + } + try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0); Row header = sheet.getRow(0); if (header == null) { throw new BusinessException("Excel 表头为空"); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceWorkbookControlTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceWorkbookControlTest.java new file mode 100644 index 00000000..16081315 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceWorkbookControlTest.java @@ -0,0 +1,271 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient; +import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.InjectMocks; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.apache.ibatis.builder.MapperBuilderAssistant; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +/** + * Task 8：WorkbookFactory 输入解析改为受控读取。 + * 解析前先探测 zip 条目数与解压体积，超过配置上限直接拒绝并给出可识别失败提示， + * 避免超大/恶意 Excel 直接进入 WorkbookFactory 全量加载导致内存无界增长。 + */ +@ExtendWith(MockitoExtension.class) +class SimilarAsinTaskServiceWorkbookControlTest { + + private static final AtomicLong NEXT_ID = new AtomicLong(50000); + + @Mock private LocalFileStorageService localFileStorageService; + @Mock private FileTaskMapper fileTaskMapper; + @Mock private FileResultMapper fileResultMapper; + @Mock private TaskScopeStateMapper taskScopeStateMapper; + @Mock private TaskChunkMapper taskChunkMapper; + @Mock private SimilarAsinFilterConditionMapper filterConditionMapper; + @Spy private ObjectMapper objectMapper = new ObjectMapper(); + @Mock private SimilarAsinCozeClient cozeClient; + @Mock private SimilarAsinTaskCacheService taskCacheService; + @Mock private SimilarAsinProperties properties; + @Mock private TransientPayloadStorageService transientPayloadStorageService; + @Mock private SimilarAsinImageEmbedder imageEmbedder; + + @InjectMocks private SimilarAsinTaskService service; + + @BeforeAll + static void initializeMybatisMetadata() { + MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), ""); + TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class); + } + + @BeforeEach + void setUp() { + lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true); + lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L); + lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90); + lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100); + lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L); + lenient().when(properties.getMaxParseRows()).thenReturn(50000); + lenient().when(properties.getMaxFieldLength()).thenReturn(2000); + lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000); + lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L); + lenient().when(transientPayloadStorageService.storeParsedPayloadFast( + eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false))) + .thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json"); + lenient().doAnswer(invocation -> { + FileTaskEntity task = invocation.getArgument(0); + task.setId(NEXT_ID.incrementAndGet()); + return 1; + }).when(fileTaskMapper).insert(any(FileTaskEntity.class)); + lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1); + lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1); + lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of()); + lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of()); + } + + @AfterEach + void shutdown() { + service.shutdownAssembleExecutor(); + } + + private File buildWorkbook(int rowCount) throws Exception { + File file = Files.createTempFile("similar-asin-workbook-ctrl-", ".xlsx").toFile(); + try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) { + var sheet = workbook.createSheet("Sheet1"); + Row header = sheet.createRow(0); + header.createCell(0).setCellValue("id"); + header.createCell(1).setCellValue("asin"); + header.createCell(2).setCellValue("国家"); + for (int i = 1; i <= rowCount; i++) { + Row row = sheet.createRow(i); + row.createCell(0).setCellValue(String.valueOf(i)); + row.createCell(1).setCellValue(String.format("B0WBK%05d", i)); + row.createCell(2).setCellValue("英国"); + } + workbook.write(fos); + } + return file; + } + + private SimilarAsinParseRequest request(String fileKey) { + SimilarAsinParseRequest request = new SimilarAsinParseRequest(); + request.setUserId(7L); + SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto(); + sourceFile.setFileKey(fileKey); + sourceFile.setOriginalFilename("workbook.xlsx"); + request.setFiles(List.of(sourceFile)); + request.setApiKey("sk-123"); + request.setImgSwitch(Boolean.FALSE); + request.setCategorySwitch(Boolean.FALSE); + return request; + } + + private SimilarAsinParseVo parse(File workbook, String fileKey) { + when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook); + return service.parseAndCreateTask(request(fileKey)); + } + + @Test + void test_task_008_workbook_excel_normal_default_path() throws Exception { + // 正常 xlsx：受控读取通过探测，解析成功且行数不丢失 + File workbook = buildWorkbook(100); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-default.xlsx"); + assertEquals(100, vo.getAcceptedRows()); + assertEquals(100, vo.getTotalRows()); + assertEquals(100, vo.getItems().size()); + assertEquals("B0WBK00001", vo.getItems().get(0).getAsin()); + } + + @Test + void test_task_008_workbook_excel_normal_multiple_items() throws Exception { + // 多文件批量：每个文件都走受控读取，汇总不丢行 + File workbookA = buildWorkbook(25); + File workbookB = buildWorkbook(35); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-a.xlsx")).thenReturn(workbookA); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-b.xlsx")).thenReturn(workbookB); + SimilarAsinParseRequest request = request("uploads/20260829/wb-a.xlsx"); + SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto(); + sourceB.setFileKey("uploads/20260829/wb-b.xlsx"); + sourceB.setOriginalFilename("workbook-b.xlsx"); + request.setFiles(List.of(request.getFiles().get(0), sourceB)); + SimilarAsinParseVo vo = service.parseAndCreateTask(request); + assertEquals(60, vo.getAcceptedRows()); + assertEquals(60, vo.getTotalRows()); + assertNotNull(vo.getTaskId()); + } + + @Test + void test_task_008_workbook_excel_normal_repeated_operation_is_idempotent() throws Exception { + // 重复解析同一文件：结果一致，不产生重复状态 + File workbook = buildWorkbook(50); + SimilarAsinParseVo first = parse(workbook, "uploads/20260829/wb-idem.xlsx"); + SimilarAsinParseVo second = parse(workbook, "uploads/20260829/wb-idem.xlsx"); + assertEquals(first.getAcceptedRows(), second.getAcceptedRows()); + assertEquals(first.getItems().size(), second.getItems().size()); + for (int i = 0; i < first.getItems().size(); i++) { + assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin()); + } + } + + @Test + void test_task_008_workbook_excel_boundary_empty_input() throws Exception { + // 空文件（只有表头无数据行）：抛业务异常，不创建任务 + File workbook = buildWorkbook(0); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-empty.xlsx")).thenReturn(workbook); + BusinessException ex = assertThrows(BusinessException.class, + () -> parse(workbook, "uploads/20260829/wb-empty.xlsx")); + assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank()); + // 文件不存在：抛业务异常 + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-missing.xlsx")).thenReturn(null); + assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/wb-missing.xlsx")); + } + + @Test + void test_task_008_workbook_excel_boundary_single_item() throws Exception { + // 单行文件：不依赖批量路径，结果正确 + File workbook = buildWorkbook(1); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-single.xlsx"); + assertEquals(1, vo.getAcceptedRows()); + assertEquals(1, vo.getItems().size()); + assertEquals("B0WBK00001", vo.getItems().get(0).getAsin()); + assertEquals(1, vo.getGroupCount()); + } + + @Test + void test_task_008_workbook_excel_boundary_limit_and_overflow() throws Exception { + // 解压体积超限：受控探测阶段拒绝，失败提示可识别，不发生无界内存增长 + when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(1024L); + File workbook = buildWorkbook(5); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-huge.xlsx")).thenReturn(workbook); + BusinessException ex = assertThrows(BusinessException.class, + () -> parse(workbook, "uploads/20260829/wb-huge.xlsx")); + assertTrue(ex.getMessage() != null && (ex.getMessage().contains("解压") || ex.getMessage().contains("大小")), + "超大 Excel 失败提示必须可识别，实际: " + ex.getMessage()); + // 条目数超限：同样在探测阶段拒绝 + when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L); + when(properties.getMaxWorkbookZipEntries()).thenReturn(2); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-entries.xlsx")).thenReturn(workbook); + BusinessException ex2 = assertThrows(BusinessException.class, + () -> parse(workbook, "uploads/20260829/wb-entries.xlsx")); + assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("条目"), + "超条目数失败提示必须可识别，实际: " + ex2.getMessage()); + // 恢复默认配置：同一文件解析成功（探测不残留状态） + when(properties.getMaxWorkbookZipEntries()).thenReturn(20000); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered.xlsx"); + assertEquals(5, vo.getAcceptedRows()); + } + + @Test + void test_task_008_workbook_excel_invalid_input_rejected() throws Exception { + // 非 xlsx 内容（文本文件）：WorkbookFactory 打开失败，抛项目约定异常 + File fake = Files.createTempFile("similar-asin-not-excel-", ".xlsx").toFile(); + Files.write(fake.toPath(), "this is not an excel file".getBytes(StandardCharsets.UTF_8)); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fake.xlsx")).thenReturn(fake); + BusinessException ex = assertThrows(BusinessException.class, + () -> parse(fake, "uploads/20260829/wb-fake.xlsx")); + assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"), + "损坏文件失败提示必须可识别，实际: " + ex.getMessage()); + } + + @Test + void test_task_008_workbook_excel_dependency_failure_releases_resources() throws Exception { + // RustFS 存储失败：解析抛异常；恢复后重试成功，受控读取无残留 + File workbook = buildWorkbook(30); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fail.xlsx")).thenReturn(workbook); + when(transientPayloadStorageService.storeParsedPayloadFast( + eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false))) + .thenThrow(new IllegalStateException("rustfs down")) + .thenReturn("rustfs:task-parsed/similar-asin/50001/payload.json"); + assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/wb-fail.xlsx")); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered2.xlsx"); + assertEquals(30, vo.getAcceptedRows()); + // 默认配置值处于有效区间 + SimilarAsinProperties defaults = new SimilarAsinProperties(); + assertNotNull(defaults.getMaxWorkbookZipEntries()); + assertNotNull(defaults.getMaxWorkbookUncompressedBytes()); + assertTrue(defaults.getMaxWorkbookZipEntries() >= 1000, "默认条目上限至少 1000"); + assertTrue(defaults.getMaxWorkbookUncompressedBytes() >= 100L * 1024L * 1024L, "默认解压上限至少 100MB"); + } +}