From 68b7d9a4a4dcee2c31d905456fbc89d1dbdc1403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 29 Aug 2026 15:08:14 +0800 Subject: [PATCH] =?UTF-8?q?task-7:=20=E9=99=90=E5=88=B6=E5=8D=95=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A4=A7=E5=B0=8F=E3=80=81=E6=9C=80=E5=A4=A7=E8=A1=8C?= =?UTF-8?q?=E6=95=B0=E5=92=8C=E6=9C=80=E5=A4=A7=E5=AD=97=E6=AE=B5=E9=95=BF?= =?UTF-8?q?=E5=BA=A6=EF=BC=8C=E9=98=B2=E6=AD=A2=E8=A7=A3=E6=9E=90=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=97=A0=E7=95=8C=E5=A2=9E=E9=95=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 maxSourceFileBytes(50MB)/maxParseRows(50000)/maxFieldLength(2000) 三个配置项：文件超限在解析入口拒绝（异常消息可识别）、行数超限拒绝、 超长单元格字段截断。0/负值配置回退默认。新增 8 个测试覆盖正常路径、 多文件、幂等、空文件、单行、边界（恰好=上限通过、超限拒绝）、 非法输入（文件超限/字段截断）、依赖失败恢复。 --- .../aiimage/config/SimilarAsinProperties.java | 15 + .../service/SimilarAsinTaskService.java | 52 +++- ...SimilarAsinTaskServiceParseLimitsTest.java | 277 ++++++++++++++++++ 3 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceParseLimitsTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java index b336e597..36f840d4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java @@ -139,6 +139,21 @@ public class SimilarAsinProperties { */ private int parseResponsePreviewLimit = 100; + /** + * Task 7：单个源文件大小上限（字节）。超过则拒绝解析，防止无界文件增长。 + */ + private long maxSourceFileBytes = 50L * 1024L * 1024L; + + /** + * Task 7：单次解析最大有效行数。超过则拒绝解析，防止任务无界增长。 + */ + private int maxParseRows = 50000; + + /** + * Task 7：单字段最大长度（字符）。超过的字段值截断到该上限，防止内存无界增长。 + */ + private int maxFieldLength = 2000; + /** * P0-4：抢 Coze 提交锁失败后下次重试间隔（毫秒）。 * 原硬编码 500ms，会在指数退避算法中作为基础值（500/1000/2000/4000ms 上限 4000）。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 0c35a9fe..79699659 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -156,6 +156,39 @@ public class SimilarAsinTaskService { } return Math.min(configured, PARSE_RESPONSE_PREVIEW_LIMIT_MAX); } + + /** + * Task 7：单文件大小上限。0/负值回退默认 50MB，防止误配导致解析无界增长。 + */ + private long resolveMaxSourceFileBytes() { + Long configured = properties.getMaxSourceFileBytes(); + if (configured == null || configured <= 0) { + return 50L * 1024L * 1024L; + } + return configured; + } + + /** + * Task 7：单次解析最大有效行数。0/负值回退默认 50000，防止任务无界增长。 + */ + private int resolveMaxParseRows() { + Integer configured = properties.getMaxParseRows(); + if (configured == null || configured <= 0) { + return 50000; + } + return configured; + } + + /** + * Task 7：单字段最大长度（字符）。0/负值回退默认 2000。 + */ + private int resolveMaxFieldLength() { + Integer configured = properties.getMaxFieldLength(); + if (configured == null || configured <= 0) { + return 2000; + } + return configured; + } /** * P0-2 最小风险变体：poll 调度阶段并发预取 Coze HTTP 结果时， * 控制对单个 Coze 后端的并发度。8 与 cozeTaskExecutor 的 12 并发上限对齐留 4 余量， @@ -421,6 +454,11 @@ public class SimilarAsinTaskService { if (input == null || !input.exists()) { throw new BusinessException("源文件不存在"); } + long maxBytes = resolveMaxSourceFileBytes(); + if (input.length() > maxBytes) { + throw new BusinessException("源文件超过大小限制: " + source.getOriginalFilename() + + " (" + input.length() + " bytes > " + maxBytes + " bytes)"); + } ParsedWorkbook parsed = parseWorkbook(input, source); totalRows += parsed.totalRows(); @@ -432,6 +470,10 @@ public class SimilarAsinTaskService { if (allRows.isEmpty()) { throw new BusinessException("未解析到有效 ASIN 数据"); } + int maxParseRows = resolveMaxParseRows(); + if (allRows.size() > maxParseRows) { + throw new BusinessException("解析行数超过上限: " + allRows.size() + " rows > " + maxParseRows + " rows"); + } boolean requestedCategorySwitch = Boolean.TRUE.equals(request.getCategorySwitch()); request.setCategorySwitch(requestedCategorySwitch || categoryRetryRequired); if (!requestedCategorySwitch && categoryRetryRequired) { @@ -5478,7 +5520,15 @@ public class SimilarAsinTaskService { return ""; } String value = normalize(formatter.formatCellValue(row.getCell(col))); - return isSpreadsheetErrorValue(value) ? "" : value; + if (isSpreadsheetErrorValue(value)) { + return ""; + } + // Task 7：单字段长度上限，防止超长单元格导致内存无界增长 + int maxLen = resolveMaxFieldLength(); + if (value.length() > maxLen) { + return value.substring(0, maxLen); + } + return value; } private static boolean isSpreadsheetErrorValue(String value) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceParseLimitsTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceParseLimitsTest.java new file mode 100644 index 00000000..309f2a71 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceParseLimitsTest.java @@ -0,0 +1,277 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient; +import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.InjectMocks; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.apache.ibatis.builder.MapperBuilderAssistant; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.file.Files; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +/** + * Task 7：限制单文件大小、最大行数和最大字段长度，防止解析任务无界增长。 + * 超限输入在解析入口被拒绝或截断；mock 依赖 + 真实 xlsx 验证边界行为。 + */ +@ExtendWith(MockitoExtension.class) +class SimilarAsinTaskServiceParseLimitsTest { + + private static final AtomicLong NEXT_ID = new AtomicLong(40000); + + @Mock private LocalFileStorageService localFileStorageService; + @Mock private FileTaskMapper fileTaskMapper; + @Mock private FileResultMapper fileResultMapper; + @Mock private TaskScopeStateMapper taskScopeStateMapper; + @Mock private TaskChunkMapper taskChunkMapper; + @Mock private SimilarAsinFilterConditionMapper filterConditionMapper; + @Spy private ObjectMapper objectMapper = new ObjectMapper(); + @Mock private SimilarAsinCozeClient cozeClient; + @Mock private SimilarAsinTaskCacheService taskCacheService; + @Mock private SimilarAsinProperties properties; + @Mock private TransientPayloadStorageService transientPayloadStorageService; + @Mock private SimilarAsinImageEmbedder imageEmbedder; + + @InjectMocks private SimilarAsinTaskService service; + + @BeforeAll + static void initializeMybatisMetadata() { + MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), ""); + TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class); + } + + @BeforeEach + void setUp() { + lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true); + lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L); + lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90); + lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100); + lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L); + lenient().when(properties.getMaxParseRows()).thenReturn(50000); + lenient().when(properties.getMaxFieldLength()).thenReturn(2000); + lenient().when(transientPayloadStorageService.storeParsedPayloadFast( + eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false))) + .thenReturn("rustfs:task-parsed/similar-asin/40000/payload.json"); + lenient().doAnswer(invocation -> { + FileTaskEntity task = invocation.getArgument(0); + task.setId(NEXT_ID.incrementAndGet()); + return 1; + }).when(fileTaskMapper).insert(any(FileTaskEntity.class)); + lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1); + lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1); + lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of()); + lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of()); + } + + @AfterEach + void shutdown() { + service.shutdownAssembleExecutor(); + } + + private File buildWorkbook(int rowCount) throws Exception { + return buildWorkbookWithAsin(rowCount, null); + } + + private File buildWorkbookWithAsin(int rowCount, String asinValue) throws Exception { + File file = Files.createTempFile("similar-asin-parse-limits-", ".xlsx").toFile(); + try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) { + var sheet = workbook.createSheet("Sheet1"); + Row header = sheet.createRow(0); + header.createCell(0).setCellValue("id"); + header.createCell(1).setCellValue("asin"); + header.createCell(2).setCellValue("国家"); + for (int i = 1; i <= rowCount; i++) { + Row row = sheet.createRow(i); + row.createCell(0).setCellValue(String.valueOf(i)); + row.createCell(1).setCellValue(asinValue != null ? asinValue : String.format("B0LIM%05d", i)); + row.createCell(2).setCellValue("英国"); + } + workbook.write(fos); + } + return file; + } + + private SimilarAsinParseRequest request(String fileKey) { + SimilarAsinParseRequest request = new SimilarAsinParseRequest(); + request.setUserId(7L); + SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto(); + sourceFile.setFileKey(fileKey); + sourceFile.setOriginalFilename("limits.xlsx"); + request.setFiles(List.of(sourceFile)); + request.setApiKey("sk-123"); + request.setImgSwitch(Boolean.FALSE); + request.setCategorySwitch(Boolean.FALSE); + return request; + } + + private SimilarAsinParseVo parse(File workbook, String fileKey) { + when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook); + return service.parseAndCreateTask(request(fileKey)); + } + + @Test + void test_task_007_file_size_row_count_normal_default_path() throws Exception { + // 默认配置（50MB/50000 行/2000 字符）：正常文件解析成功，行数不丢失 + File workbook = buildWorkbook(120); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-default.xlsx"); + assertEquals(120, vo.getAcceptedRows()); + assertEquals(120, vo.getTotalRows()); + assertEquals(100, vo.getItems().size()); + assertEquals("B0LIM00001", vo.getItems().get(0).getAsin()); + // 源文件大小在限制内 + assertTrue(workbook.length() <= 50L * 1024L * 1024L); + } + + @Test + void test_task_007_file_size_row_count_normal_multiple_items() throws Exception { + // 多文件批量：每个文件都在限制内，汇总不丢行 + File workbookA = buildWorkbook(30); + File workbookB = buildWorkbook(40); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-a.xlsx")).thenReturn(workbookA); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-b.xlsx")).thenReturn(workbookB); + SimilarAsinParseRequest request = request("uploads/20260829/limit-a.xlsx"); + SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto(); + sourceB.setFileKey("uploads/20260829/limit-b.xlsx"); + sourceB.setOriginalFilename("limits-b.xlsx"); + request.setFiles(List.of(request.getFiles().get(0), sourceB)); + SimilarAsinParseVo vo = service.parseAndCreateTask(request); + assertEquals(70, vo.getAcceptedRows()); + assertEquals(70, vo.getTotalRows()); + assertNotNull(vo.getTaskId()); + } + + @Test + void test_task_007_file_size_row_count_normal_repeated_operation_is_idempotent() throws Exception { + // 重复解析同一文件：结果一致，不产生重复状态 + File workbook = buildWorkbook(60); + SimilarAsinParseVo first = parse(workbook, "uploads/20260829/limit-idem.xlsx"); + SimilarAsinParseVo second = parse(workbook, "uploads/20260829/limit-idem.xlsx"); + assertEquals(first.getAcceptedRows(), second.getAcceptedRows()); + assertEquals(first.getItems().size(), second.getItems().size()); + for (int i = 0; i < first.getItems().size(); i++) { + assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin()); + } + } + + @Test + void test_task_007_file_size_row_count_boundary_empty_input() throws Exception { + // 空文件（无有效数据行）：抛业务异常，不创建任务 + File workbook = buildWorkbook(0); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-empty.xlsx")).thenReturn(workbook); + BusinessException ex = assertThrows(BusinessException.class, + () -> parse(workbook, "uploads/20260829/limit-empty.xlsx")); + assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank()); + } + + @Test + void test_task_007_file_size_row_count_boundary_single_item() throws Exception { + // 单行小文件：不依赖批量路径，结果正确 + File workbook = buildWorkbook(1); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-single.xlsx"); + assertEquals(1, vo.getAcceptedRows()); + assertEquals(1, vo.getItems().size()); + assertEquals("B0LIM00001", vo.getItems().get(0).getAsin()); + assertEquals(1, vo.getGroupCount()); + } + + @Test + void test_task_007_file_size_row_count_boundary_limit_and_overflow() throws Exception { + // 行数恰好等于上限：允许 + when(properties.getMaxParseRows()).thenReturn(8); + File workbook = buildWorkbook(8); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-exact.xlsx"); + assertEquals(8, vo.getAcceptedRows()); + // 行数超过上限：拒绝，且不创建任务 + when(properties.getMaxParseRows()).thenReturn(3); + File workbookOver = buildWorkbook(4); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-over.xlsx")).thenReturn(workbookOver); + BusinessException ex = assertThrows(BusinessException.class, + () -> parse(workbookOver, "uploads/20260829/limit-over.xlsx")); + assertTrue(ex.getMessage() != null && ex.getMessage().contains("行数"), + "超行数异常消息必须可识别，实际: " + ex.getMessage()); + } + + @Test + void test_task_007_file_size_row_count_invalid_input_rejected() throws Exception { + // 文件大小超限：拒绝，异常消息可识别 + when(properties.getMaxSourceFileBytes()).thenReturn(64L); + File workbook = buildWorkbook(5); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-bigfile.xlsx")).thenReturn(workbook); + assertTrue(workbook.length() > 64L, "测试文件必须超过 64 字节限制，实际 " + workbook.length()); + BusinessException ex = assertThrows(BusinessException.class, + () -> parse(workbook, "uploads/20260829/limit-bigfile.xlsx")); + assertTrue(ex.getMessage() != null && ex.getMessage().contains("大小限制"), + "超文件大小异常消息必须可识别，实际: " + ex.getMessage()); + // 字段长度超限：截断而非拒绝，字段仍非空 + when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L); + when(properties.getMaxFieldLength()).thenReturn(12); + File longAsin = buildWorkbookWithAsin(2, "B0CJ8SNXXVVERYLONGASINVALUE"); + SimilarAsinParseVo vo = parse(longAsin, "uploads/20260829/limit-longfield.xlsx"); + assertEquals(2, vo.getAcceptedRows()); + for (var item : vo.getItems()) { + assertTrue(item.getAsin().length() <= 12, "超长字段必须截断到配置上限"); + assertTrue(!item.getAsin().isBlank()); + } + } + + @Test + void test_task_007_file_size_row_count_dependency_failure_releases_resources() throws Exception { + // RustFS 存储失败：解析抛异常；恢复后重试成功，无残留状态 + File workbook = buildWorkbook(40); + when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-fail.xlsx")).thenReturn(workbook); + when(transientPayloadStorageService.storeParsedPayloadFast( + eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false))) + .thenThrow(new IllegalStateException("rustfs down")) + .thenReturn("rustfs:task-parsed/similar-asin/40001/payload.json"); + assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/limit-fail.xlsx")); + SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-recovered.xlsx"); + assertEquals(40, vo.getAcceptedRows()); + assertEquals(40, vo.getItems().size()); + // 行数/文件大小/字段长度默认值均处于有效区间 + SimilarAsinProperties defaults = new SimilarAsinProperties(); + assertNotNull(defaults.getMaxParseRows()); + assertNotNull(defaults.getMaxSourceFileBytes()); + assertNotNull(defaults.getMaxFieldLength()); + assertTrue(defaults.getMaxParseRows() >= 1000, "默认最大行数至少 1000"); + assertTrue(defaults.getMaxSourceFileBytes() >= 10L * 1024L * 1024L, "默认文件上限至少 10MB"); + assertTrue(defaults.getMaxFieldLength() >= 500, "默认字段上限至少 500 字符"); + } +}