task-8: WorkbookFactory 输入解析改为受控读取,验证超大 Excel 失败提示

解析前用 ZipFile 中央目录探测 xlsx(zip) 条目数与解压总字节(不读压缩
内容),超过 maxWorkbookZipEntries(20000)/maxWorkbookUncompressedBytes
(512MB) 立即拒绝并给出可识别提示;非 zip/损坏文件跳过探测交给
WorkbookFactory 兜底转业务异常。0/负值配置回退默认。新增 8 个测试覆盖
正常路径、多文件、幂等、空输入、单行、解压超限、条目超限、损坏文件、
依赖失败恢复。
This commit is contained in:
2026-08-29 15:11:53 +08:00
parent 68b7d9a4a4
commit 3b051bbafe
3 changed files with 343 additions and 3 deletions
@@ -154,6 +154,17 @@ public class SimilarAsinProperties {
*/
private int maxFieldLength = 2000;
/**
* Task 8xlsx(zip) 最大条目数。受控读取在 WorkBookFactory 打开前探测,
* 超过则拒绝,防止 zip bomb / 超大工作簿拖垮内存。
*/
private int maxWorkbookZipEntries = 20000;
/**
* Task 8xlsx(zip) 解压后总字节数上限。同样在打开前探测,超过则拒绝。
*/
private long maxWorkbookUncompressedBytes = 512L * 1024L * 1024L;
/**
* P0-4:抢 Coze 提交锁失败后下次重试间隔(毫秒)。
* 原硬编码 500ms,会在指数退避算法中作为基础值(500/1000/2000/4000ms 上限 4000)。
@@ -180,7 +180,7 @@ public class SimilarAsinTaskService {
}
/**
* Task 7单字段最大长度字符0/负值回退默认 2000
* Task 8单字段最大长度字符0/负值回退默认 2000
*/
private int resolveMaxFieldLength() {
Integer configured = properties.getMaxFieldLength();
@@ -189,6 +189,56 @@ public class SimilarAsinTaskService {
}
return configured;
}
/**
* Task 8xlsx(zip) 最大条目数0/负值回退默认 20000
*/
private int resolveMaxWorkbookZipEntries() {
Integer configured = properties.getMaxWorkbookZipEntries();
if (configured == null || configured <= 0) {
return 20000;
}
return configured;
}
/**
* Task 8xlsx(zip) 解压后总字节数上限0/负值回退默认 512MB
*/
private long resolveMaxWorkbookUncompressedBytes() {
Long configured = properties.getMaxWorkbookUncompressedBytes();
if (configured == null || configured <= 0) {
return 512L * 1024L * 1024L;
}
return configured;
}
/**
* Task 8WorkbookFactory 打开前的受控读取xlsx 本质是 zip
* 先用 ZipFile 中央目录探测条目数与解压总字节不读取压缩内容
* 超限立即拒绝避免 zip bomb / 超大工作簿直接进入全量加载
* zip 文件或损坏文件由调用方 catch 转业务异常
*/
private void probeWorkbookZipBounds(File input, String sourceName) throws Exception {
int maxEntries = resolveMaxWorkbookZipEntries();
long maxBytes = resolveMaxWorkbookUncompressedBytes();
try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(input)) {
if (zipFile.size() > maxEntries) {
throw new BusinessException("Excel 条目数超过上限: " + zipFile.size() + " entries > " + maxEntries + " entries");
}
long total = 0;
var entries = zipFile.entries();
while (entries.hasMoreElements()) {
var entry = entries.nextElement();
long size = entry.getSize();
if (size >= 0) {
total += size;
if (total > maxBytes) {
throw new BusinessException("Excel 解压体积超过上限: " + total + " bytes > " + maxBytes + " bytes");
}
}
}
}
}
/**
* P0-2 最小风险变体poll 调度阶段并发预取 Coze HTTP 结果时
* 控制对单个 Coze 后端的并发度8 cozeTaskExecutor 12 并发上限对齐留 4 余量
@@ -5268,8 +5318,16 @@ public class SimilarAsinTaskService {
private ParsedWorkbook parseWorkbook(File input, SimilarAsinSourceFileDto source) {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
try {
// Task 8受控读取先探测 zip 条目数与解压体积超限拒绝
probeWorkbookZipBounds(input, source.getOriginalFilename());
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
// zip 或损坏文件留给 WorkbookFactory 尝试后由下方 catch 转业务异常
log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage());
}
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
@@ -0,0 +1,271 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 8WorkbookFactory 输入解析改为受控读取。
* 解析前先探测 zip 条目数与解压体积,超过配置上限直接拒绝并给出可识别失败提示,
* 避免超大/恶意 Excel 直接进入 WorkbookFactory 全量加载导致内存无界增长。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceWorkbookControlTest {
private static final AtomicLong NEXT_ID = new AtomicLong(50000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/50000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
File file = Files.createTempFile("similar-asin-workbook-ctrl-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(String.format("B0WBK%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("workbook.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_008_workbook_excel_normal_default_path() throws Exception {
// 正常 xlsx:受控读取通过探测,解析成功且行数不丢失
File workbook = buildWorkbook(100);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-default.xlsx");
assertEquals(100, vo.getAcceptedRows());
assertEquals(100, vo.getTotalRows());
assertEquals(100, vo.getItems().size());
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
}
@Test
void test_task_008_workbook_excel_normal_multiple_items() throws Exception {
// 多文件批量:每个文件都走受控读取,汇总不丢行
File workbookA = buildWorkbook(25);
File workbookB = buildWorkbook(35);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-a.xlsx")).thenReturn(workbookA);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-b.xlsx")).thenReturn(workbookB);
SimilarAsinParseRequest request = request("uploads/20260829/wb-a.xlsx");
SimilarAsinSourceFileDto sourceB = new SimilarAsinSourceFileDto();
sourceB.setFileKey("uploads/20260829/wb-b.xlsx");
sourceB.setOriginalFilename("workbook-b.xlsx");
request.setFiles(List.of(request.getFiles().get(0), sourceB));
SimilarAsinParseVo vo = service.parseAndCreateTask(request);
assertEquals(60, vo.getAcceptedRows());
assertEquals(60, vo.getTotalRows());
assertNotNull(vo.getTaskId());
}
@Test
void test_task_008_workbook_excel_normal_repeated_operation_is_idempotent() throws Exception {
// 重复解析同一文件:结果一致,不产生重复状态
File workbook = buildWorkbook(50);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/wb-idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/wb-idem.xlsx");
assertEquals(first.getAcceptedRows(), second.getAcceptedRows());
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_008_workbook_excel_boundary_empty_input() throws Exception {
// 空文件(只有表头无数据行):抛业务异常,不创建任务
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-empty.xlsx")).thenReturn(workbook);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/wb-empty.xlsx"));
assertTrue(ex.getMessage() != null && !ex.getMessage().isBlank());
// 文件不存在:抛业务异常
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-missing.xlsx")).thenReturn(null);
assertThrows(BusinessException.class, () -> parse(null, "uploads/20260829/wb-missing.xlsx"));
}
@Test
void test_task_008_workbook_excel_boundary_single_item() throws Exception {
// 单行文件:不依赖批量路径,结果正确
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-single.xlsx");
assertEquals(1, vo.getAcceptedRows());
assertEquals(1, vo.getItems().size());
assertEquals("B0WBK00001", vo.getItems().get(0).getAsin());
assertEquals(1, vo.getGroupCount());
}
@Test
void test_task_008_workbook_excel_boundary_limit_and_overflow() throws Exception {
// 解压体积超限:受控探测阶段拒绝,失败提示可识别,不发生无界内存增长
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(1024L);
File workbook = buildWorkbook(5);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-huge.xlsx")).thenReturn(workbook);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/wb-huge.xlsx"));
assertTrue(ex.getMessage() != null && (ex.getMessage().contains("解压") || ex.getMessage().contains("大小")),
"超大 Excel 失败提示必须可识别,实际: " + ex.getMessage());
// 条目数超限:同样在探测阶段拒绝
when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
when(properties.getMaxWorkbookZipEntries()).thenReturn(2);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-entries.xlsx")).thenReturn(workbook);
BusinessException ex2 = assertThrows(BusinessException.class,
() -> parse(workbook, "uploads/20260829/wb-entries.xlsx"));
assertTrue(ex2.getMessage() != null && ex2.getMessage().contains("条目"),
"超条目数失败提示必须可识别,实际: " + ex2.getMessage());
// 恢复默认配置:同一文件解析成功(探测不残留状态)
when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered.xlsx");
assertEquals(5, vo.getAcceptedRows());
}
@Test
void test_task_008_workbook_excel_invalid_input_rejected() throws Exception {
// 非 xlsx 内容(文本文件):WorkbookFactory 打开失败,抛项目约定异常
File fake = Files.createTempFile("similar-asin-not-excel-", ".xlsx").toFile();
Files.write(fake.toPath(), "this is not an excel file".getBytes(StandardCharsets.UTF_8));
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fake.xlsx")).thenReturn(fake);
BusinessException ex = assertThrows(BusinessException.class,
() -> parse(fake, "uploads/20260829/wb-fake.xlsx"));
assertTrue(ex.getMessage() != null && ex.getMessage().contains("解析 Excel 失败"),
"损坏文件失败提示必须可识别,实际: " + ex.getMessage());
}
@Test
void test_task_008_workbook_excel_dependency_failure_releases_resources() throws Exception {
// RustFS 存储失败:解析抛异常;恢复后重试成功,受控读取无残留
File workbook = buildWorkbook(30);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/wb-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/50001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/wb-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/wb-recovered2.xlsx");
assertEquals(30, vo.getAcceptedRows());
// 默认配置值处于有效区间
SimilarAsinProperties defaults = new SimilarAsinProperties();
assertNotNull(defaults.getMaxWorkbookZipEntries());
assertNotNull(defaults.getMaxWorkbookUncompressedBytes());
assertTrue(defaults.getMaxWorkbookZipEntries() >= 1000, "默认条目上限至少 1000");
assertTrue(defaults.getMaxWorkbookUncompressedBytes() >= 100L * 1024L * 1024L, "默认解压上限至少 100MB");
}
}