super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-7: 限制单文件大小、最大行数和最大字段长度，防止解析任务无界增长

Browse Source 
新增 maxSourceFileBytes(50MB)/maxParseRows(50000)/maxFieldLength(2000)
三个配置项：文件超限在解析入口拒绝（异常消息可识别）、行数超限拒绝、
超长单元格字段截断。0/负值配置回退默认。新增 8 个测试覆盖正常路径、
多文件、幂等、空文件、单行、边界（恰好=上限通过、超限拒绝）、
非法输入（文件超限/字段截断）、依赖失败恢复。
This commit is contained in:
黄自达
2026-08-29 15:08:14 +08:00
parent db6e8f0ce8
commit 68b7d9a4a4
3 changed files with 343 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java
+15
View File
@@ -139,6 +139,21 @@ public class SimilarAsinProperties {
*/
private int parseResponsePreviewLimit = 100;
/**
* Task 7：单个源文件大小上限（字节）。超过则拒绝解析，防止无界文件增长。
*/
private long maxSourceFileBytes = 50L * 1024L * 1024L;
/**
* Task 7：单次解析最大有效行数。超过则拒绝解析，防止任务无界增长。
*/
private int maxParseRows = 50000;
/**
* Task 7：单字段最大长度（字符）。超过的字段值截断到该上限，防止内存无界增长。
*/
private int maxFieldLength = 2000;
/**
* P0-4：抢 Coze 提交锁失败后下次重试间隔（毫秒）。
* 原硬编码 500ms，会在指数退避算法中作为基础值（500/1000/2000/4000ms 上限 4000）。
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+51 -1
View File
@@ -156,6 +156,39 @@ public class SimilarAsinTaskService {
}
return Math.min(configured, PARSE_RESPONSE_PREVIEW_LIMIT_MAX);
}
/**
* Task 7：单文件大小上限。0/负值回退默认 50MB，防止误配导致解析无界增长。
*/
private long resolveMaxSourceFileBytes() {
Long configured = properties.getMaxSourceFileBytes();
if (configured == null || configured <= 0) {
return 50L * 1024L * 1024L;
}
return configured;
}
/**
* Task 7：单次解析最大有效行数。0/负值回退默认 50000，防止任务无界增长。
*/
private int resolveMaxParseRows() {
Integer configured = properties.getMaxParseRows();
if (configured == null || configured <= 0) {
return 50000;
}
return configured;
}
/**
* Task 7：单字段最大长度（字符）。0/负值回退默认 2000。
*/
private int resolveMaxFieldLength() {
Integer configured = properties.getMaxFieldLength();
if (configured == null || configured <= 0) {
return 2000;
}
return configured;
}
/**
* P0-2 最小风险变体：poll 调度阶段并发预取 Coze HTTP 结果时，
* 控制对单个 Coze 后端的并发度。8 与 cozeTaskExecutor 的 12 并发上限对齐留 4 余量，
@@ -421,6 +454,11 @@ public class SimilarAsinTaskService {
if (input == null || !input.exists()) {
throw new BusinessException("源文件不存在");
}
long maxBytes = resolveMaxSourceFileBytes();
if (input.length() > maxBytes) {
throw new BusinessException("源文件超过大小限制: " + source.getOriginalFilename()
+ " (" + input.length() + " bytes > " + maxBytes + " bytes)");
}
ParsedWorkbook parsed = parseWorkbook(input, source);
totalRows += parsed.totalRows();
@@ -432,6 +470,10 @@ public class SimilarAsinTaskService {
if (allRows.isEmpty()) {
throw new BusinessException("未解析到有效 ASIN 数据");
}
int maxParseRows = resolveMaxParseRows();
if (allRows.size() > maxParseRows) {
throw new BusinessException("解析行数超过上限: " + allRows.size() + " rows > " + maxParseRows + " rows");
}
boolean requestedCategorySwitch = Boolean.TRUE.equals(request.getCategorySwitch());
request.setCategorySwitch(requestedCategorySwitch || categoryRetryRequired);
if (!requestedCategorySwitch && categoryRetryRequired) {
@@ -5478,7 +5520,15 @@ public class SimilarAsinTaskService {
return "";
}
String value = normalize(formatter.formatCellValue(row.getCell(col)));
return isSpreadsheetErrorValue(value) ? "" : value;
if (isSpreadsheetErrorValue(value)) {
return "";
}
// Task 7：单字段长度上限，防止超长单元格导致内存无界增长
int maxLen = resolveMaxFieldLength();
if (value.length() > maxLen) {
return value.substring(0, maxLen);
}
return value;
}
private static boolean isSpreadsheetErrorValue(String value) {
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceParseLimitsTest.java
+277
View File
@@ -0,0 +1,277 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 7：限制单文件大小、最大行数和最大字段长度，防止解析任务无界增长。
* 超限输入在解析入口被拒绝或截断；mock 依赖 + 真实 xlsx 验证边界行为。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceParseLimitsTest {
private static final AtomicLong NEXT_ID = new AtomicLong(40000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/40000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
return buildWorkbookWithAsin(rowCount, null);
}
private File buildWorkbookWithAsin(int rowCount, String asinValue) throws Exception {
File file = Files.createTempFile("similar-asin-parse-limits-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(asinValue != null ? asinValue : String.format("B0LIM%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("limits.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_007_file_size_row_count_normal_default_path() throws Exception {
// 默认配置（50MB/50000 行/2000 字符）：正常文件解析成功，行数不丢失
File workbook = buildWorkbook(120);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-default.xlsx");
assertEquals(120, vo.getAcceptedRows());
assertEquals(120, vo.getTotalRows());
assertEquals(100, vo.getItems().size());
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
// 源文件大小在限制内
assertTrue(workbook.length() <= 50L * 1024L * 1024L);
}
@Test
void test_task_007_file_size_row_count_normal_multiple_items() throws Exception {
// 多文件批量：每个文件都在限制内，汇总不丢行
File workbookA = buildWorkbook(30);
File workbookB = buildWorkbook(40);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-a.xlsx")).thenReturn(workbookA);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-b.xlsx")).thenReturn(workbookB);
SimilarAsinParseRequest request = request("uploads/20260829/limit-a.xlsx");
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
sourceB.setFileKey("uploads/20260829/limit-b.xlsx");
sourceB.setOriginalFilename("limits-b.xlsx");
request.setFiles(List.of(request.getFiles().get(0), sourceB));
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
assertEquals(70, vo.getAcceptedRows());
assertEquals(70, vo.getTotalRows());
assertNotNull(vo.getTaskId());
}
@Test
void test_task_007_file_size_row_count_normal_repeated_operation_is_idempotent() throws Exception {
// 重复解析同一文件：结果一致，不产生重复状态
File workbook = buildWorkbook(60);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/limit-idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/limit-idem.xlsx");
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_007_file_size_row_count_boundary_empty_input() throws Exception {
// 空文件（无有效数据行）：抛业务异常，不创建任务
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-empty.xlsx")).thenReturn(workbook);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/limit-empty.xlsx"));
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
}
@Test
void test_task_007_file_size_row_count_boundary_single_item() throws Exception {
// 单行小文件：不依赖批量路径，结果正确
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-single.xlsx");
assertEquals(1, vo.getAcceptedRows());
assertEquals(1, vo.getItems().size());
assertEquals("B0LIM00001", vo.getItems().get(0).getAsin());
assertEquals(1, vo.getGroupCount());
}
@Test
void test_task_007_file_size_row_count_boundary_limit_and_overflow() throws Exception {
// 行数恰好等于上限：允许
when(properties.getMaxParseRows()).thenReturn(8);
File workbook = buildWorkbook(8);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-exact.xlsx");
assertEquals(8, vo.getAcceptedRows());
// 行数超过上限：拒绝，且不创建任务
when(properties.getMaxParseRows()).thenReturn(3);
File workbookOver = buildWorkbook(4);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-over.xlsx")).thenReturn(workbookOver);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbookOver, "uploads/20260829/limit-over.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("行数"),
"超行数异常消息必须可识别，实际: " + ex.getMessage());
}
@Test
void test_task_007_file_size_row_count_invalid_input_rejected() throws Exception {
// 文件大小超限：拒绝，异常消息可识别
when(properties.getMaxSourceFileBytes()).thenReturn(64L);
File workbook = buildWorkbook(5);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-bigfile.xlsx")).thenReturn(workbook);
assertTrue(workbook.length() > 64L, "测试文件必须超过 64 字节限制，实际 " + workbook.length());
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/limit-bigfile.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("大小限制"),
"超文件大小异常消息必须可识别，实际: " + ex.getMessage());
// 字段长度超限：截断而非拒绝，字段仍非空
when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
when(properties.getMaxFieldLength()).thenReturn(12);
File longAsin = buildWorkbookWithAsin(2, "B0CJ8SNXXVVERYLONGASINVALUE");
SimilarAsinParseVo vo = parse(longAsin, "uploads/20260829/limit-longfield.xlsx");
assertEquals(2, vo.getAcceptedRows());
for (var item : vo.getItems()) {
assertTrue(item.getAsin().length() <= 12, "超长字段必须截断到配置上限");
assertTrue(!item.getAsin().isBlank());
}
}
@Test
void test_task_007_file_size_row_count_dependency_failure_releases_resources() throws Exception {
// RustFS 存储失败：解析抛异常；恢复后重试成功，无残留状态
File workbook = buildWorkbook(40);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/limit-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/40001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/limit-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/limit-recovered.xlsx");
assertEquals(40, vo.getAcceptedRows());
assertEquals(40, vo.getItems().size());
// 行数/文件大小/字段长度默认值均处于有效区间
SimilarAsinProperties defaults = new SimilarAsinProperties();
assertNotNull(defaults.getMaxParseRows());
assertNotNull(defaults.getMaxSourceFileBytes());
assertNotNull(defaults.getMaxFieldLength());
assertTrue(defaults.getMaxParseRows() >= 1000, "默认最大行数至少 1000");
assertTrue(defaults.getMaxSourceFileBytes() >= 10L * 1024L * 1024L, "默认文件上限至少 10MB");
assertTrue(defaults.getMaxFieldLength() >= 500, "默认字段上限至少 500 字符");
}
}