task-42: 限制采集解析的文件大小、最大行数和单 chunk 行数

新增 CollectDataParseLimits 组件并在 CollectDataService 解析/回传路径接入：源文件大小（默认 50MB）、累计解析行数（默认 50000）、单 chunk 回传行数（默认 5000）超限即拒绝，配置经 aiimage.collect-data.* 环境变量可调、0/负值回退默认。8 个用例覆盖正常/批量/幂等/空/单元素/边界超限/ 非法配置/失败后可恢复路径，mvn 全量测试通过。
2026-08-30 13:04:02 +08:00
parent 3f8326afc4
commit fec89b6b7b
4 changed files with 276 additions and 0 deletions
@@ -33,6 +33,7 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
 import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
+import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits;
 import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
 import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
 import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
@@ -138,6 +139,46 @@ public class CollectDataService {
    @Value("${aiimage.collect-data.stale-timeout-minutes:30}")
    private long staleTimeoutMinutes;

+    @Value("${aiimage.collect-data.max-source-file-bytes:0}")
+    private Long maxSourceFileBytes;
+
+    @Value("${aiimage.collect-data.max-parse-rows:0}")
+    private Integer maxParseRows;
+
+    @Value("${aiimage.collect-data.max-chunk-rows:0}")
+    private Integer maxChunkRows;
+
+    /** 采集源文件大小上限。0/负值回退默认 50MB，防止解析无界增长。 */
+    private long resolveMaxSourceFileBytes() {
+        Long configured = maxSourceFileBytes;
+        if (configured == null || configured <= 0) {
+            return 50L * 1024L * 1024L;
+        }
+        return configured;
+    }
+
+    /** 采集单次解析最大有效行数。0/负值回退默认 50000，防止任务无界增长。 */
+    private int resolveMaxParseRows() {
+        Integer configured = maxParseRows;
+        if (configured == null || configured <= 0) {
+            return 50000;
+        }
+        return configured;
+    }
+
+    /** 采集单 chunk 回传最大行数。0/负值回退默认 5000，防止单次回传无界增长。 */
+    private int resolveMaxChunkRows() {
+        Integer configured = maxChunkRows;
+        if (configured == null || configured <= 0) {
+            return 5000;
+        }
+        return configured;
+    }
+
+    private CollectDataParseLimits buildParseLimits() {
+        return new CollectDataParseLimits(resolveMaxSourceFileBytes(), resolveMaxParseRows(), resolveMaxChunkRows());
+    }
+
    public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) {
        long startedAt = System.currentTimeMillis();
        if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
@@ -160,15 +201,18 @@ public class CollectDataService {
        List<ParsedRow> parsedRows = new ArrayList<>();
        int totalRows = 0;
        int droppedRows = 0;
+        CollectDataParseLimits limits = buildParseLimits();
        for (CollectDataSourceFileDto source : sources) {
            File input = localFileStorageService.findLocalSourceFile(source.getFileKey());
            if (input == null || !input.exists()) {
                throw new BusinessException("源文件不存在");
            }
+            limits.validateSourceFile(input);
            ParsedWorkbook parsed = parseWorkbook(input, source);
            totalRows += parsed.totalRows();
            droppedRows += parsed.droppedRows();
            parsedRows.addAll(parsed.rows());
+            limits.validateTotalRowCount(parsedRows.size());
        }
        if (parsedRows.isEmpty()) {
            throw new BusinessException("未解析到有效数据行");
@@ -575,6 +619,7 @@ public class CollectDataService {
            }

            List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
+            buildParseLimits().validateChunkRowCount(rows.size());
            CollectDataStats stats = loadStats(task);
            stats.receivedRows += rows.size();
            stats.currentChunkRows = rows.size();
@@ -0,0 +1,58 @@
+package com.nanri.aiimage.modules.collectdata.util;
+
+import com.nanri.aiimage.common.exception.BusinessException;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+
+/**
+ * 采集解析资源上限：文件大小、累计行数与单 chunk 行数。
+ * 超过任一上限抛 BusinessException，防止解析任务无界增长；
+ * 校验只读不残留状态，重复执行结果一致。
+ */
+@Slf4j
+public class CollectDataParseLimits {
+
+    private final long maxFileBytes;
+    private final int maxTotalRows;
+    private final int maxChunkRows;
+
+    public CollectDataParseLimits(long maxFileBytes, int maxTotalRows, int maxChunkRows) {
+        if (maxFileBytes <= 0) {
+            throw new IllegalArgumentException("maxFileBytes 必须为正数，实际 " + maxFileBytes);
+        }
+        if (maxTotalRows <= 0) {
+            throw new IllegalArgumentException("maxTotalRows 必须为正数，实际 " + maxTotalRows);
+        }
+        if (maxChunkRows <= 0) {
+            throw new IllegalArgumentException("maxChunkRows 必须为正数，实际 " + maxChunkRows);
+        }
+        this.maxFileBytes = maxFileBytes;
+        this.maxTotalRows = maxTotalRows;
+        this.maxChunkRows = maxChunkRows;
+    }
+
+    public void validateSourceFile(File file) {
+        if (file == null || !file.exists() || !file.isFile()) {
+            throw new BusinessException("源文件不存在");
+        }
+        long size = file.length();
+        if (size > maxFileBytes) {
+            log.warn("[collect-data] source file exceeds size limit file={} size={} max={}",
+                    file.getName(), size, maxFileBytes);
+            throw new BusinessException("源文件大小超过上限 " + maxFileBytes + " 字节");
+        }
+    }
+
+    public void validateTotalRowCount(int totalRows) {
+        if (totalRows > maxTotalRows) {
+            throw new BusinessException("累计行数超过上限 " + maxTotalRows);
+        }
+    }
+
+    public void validateChunkRowCount(int chunkRows) {
+        if (chunkRows > maxChunkRows) {
+            throw new BusinessException("单 chunk 行数超过上限 " + maxChunkRows);
+        }
+    }
+}