From fec89b6b7b5ac4f0e6babc50502990686b8d2edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sun, 30 Aug 2026 13:04:02 +0800 Subject: [PATCH] =?UTF-8?q?task-42:=20=E9=99=90=E5=88=B6=E9=87=87=E9=9B=86?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E7=9A=84=E6=96=87=E4=BB=B6=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E3=80=81=E6=9C=80=E5=A4=A7=E8=A1=8C=E6=95=B0=E5=92=8C=E5=8D=95?= =?UTF-8?q?=20chunk=20=E8=A1=8C=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 CollectDataParseLimits 组件并在 CollectDataService 解析/回传路径 接入：源文件大小（默认 50MB）、累计解析行数（默认 50000）、单 chunk 回传行数（默认 5000）超限即拒绝，配置经 aiimage.collect-data.* 环境变量 可调、0/负值回退默认。8 个用例覆盖正常/批量/幂等/空/单元素/边界超限/ 非法配置/失败后可恢复路径，mvn 全量测试通过。 --- .../service/CollectDataService.java | 45 +++++ .../util/CollectDataParseLimits.java | 58 ++++++ .../src/main/resources/application.yml | 6 + .../util/CollectDataParseLimitsTest.java | 167 ++++++++++++++++++ 4 files changed, 276 insertions(+) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimits.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimitsTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index 9310d3ce..521788d8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -33,6 +33,7 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo; +import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits; import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper; import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity; @@ -138,6 +139,46 @@ public class CollectDataService { @Value("${aiimage.collect-data.stale-timeout-minutes:30}") private long staleTimeoutMinutes; + @Value("${aiimage.collect-data.max-source-file-bytes:0}") + private Long maxSourceFileBytes; + + @Value("${aiimage.collect-data.max-parse-rows:0}") + private Integer maxParseRows; + + @Value("${aiimage.collect-data.max-chunk-rows:0}") + private Integer maxChunkRows; + + /** 采集源文件大小上限。0/负值回退默认 50MB，防止解析无界增长。 */ + private long resolveMaxSourceFileBytes() { + Long configured = maxSourceFileBytes; + if (configured == null || configured <= 0) { + return 50L * 1024L * 1024L; + } + return configured; + } + + /** 采集单次解析最大有效行数。0/负值回退默认 50000，防止任务无界增长。 */ + private int resolveMaxParseRows() { + Integer configured = maxParseRows; + if (configured == null || configured <= 0) { + return 50000; + } + return configured; + } + + /** 采集单 chunk 回传最大行数。0/负值回退默认 5000，防止单次回传无界增长。 */ + private int resolveMaxChunkRows() { + Integer configured = maxChunkRows; + if (configured == null || configured <= 0) { + return 5000; + } + return configured; + } + + private CollectDataParseLimits buildParseLimits() { + return new CollectDataParseLimits(resolveMaxSourceFileBytes(), resolveMaxParseRows(), resolveMaxChunkRows()); + } + public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) { long startedAt = System.currentTimeMillis(); if (request == null || request.getUserId() == null || request.getUserId() <= 0) { @@ -160,15 +201,18 @@ public class CollectDataService { List parsedRows = new ArrayList<>(); int totalRows = 0; int droppedRows = 0; + CollectDataParseLimits limits = buildParseLimits(); for (CollectDataSourceFileDto source : sources) { File input = localFileStorageService.findLocalSourceFile(source.getFileKey()); if (input == null || !input.exists()) { throw new BusinessException("源文件不存在"); } + limits.validateSourceFile(input); ParsedWorkbook parsed = parseWorkbook(input, source); totalRows += parsed.totalRows(); droppedRows += parsed.droppedRows(); parsedRows.addAll(parsed.rows()); + limits.validateTotalRowCount(parsedRows.size()); } if (parsedRows.isEmpty()) { throw new BusinessException("未解析到有效数据行"); @@ -575,6 +619,7 @@ public class CollectDataService { } List rows = normalizeSubmitRows(request.getItems()); + buildParseLimits().validateChunkRowCount(rows.size()); CollectDataStats stats = loadStats(task); stats.receivedRows += rows.size(); stats.currentChunkRows = rows.size(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimits.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimits.java new file mode 100644 index 00000000..288a9af4 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimits.java @@ -0,0 +1,58 @@ +package com.nanri.aiimage.modules.collectdata.util; + +import com.nanri.aiimage.common.exception.BusinessException; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; + +/** + * 采集解析资源上限：文件大小、累计行数与单 chunk 行数。 + * 超过任一上限抛 BusinessException，防止解析任务无界增长； + * 校验只读不残留状态，重复执行结果一致。 + */ +@Slf4j +public class CollectDataParseLimits { + + private final long maxFileBytes; + private final int maxTotalRows; + private final int maxChunkRows; + + public CollectDataParseLimits(long maxFileBytes, int maxTotalRows, int maxChunkRows) { + if (maxFileBytes <= 0) { + throw new IllegalArgumentException("maxFileBytes 必须为正数，实际 " + maxFileBytes); + } + if (maxTotalRows <= 0) { + throw new IllegalArgumentException("maxTotalRows 必须为正数，实际 " + maxTotalRows); + } + if (maxChunkRows <= 0) { + throw new IllegalArgumentException("maxChunkRows 必须为正数，实际 " + maxChunkRows); + } + this.maxFileBytes = maxFileBytes; + this.maxTotalRows = maxTotalRows; + this.maxChunkRows = maxChunkRows; + } + + public void validateSourceFile(File file) { + if (file == null || !file.exists() || !file.isFile()) { + throw new BusinessException("源文件不存在"); + } + long size = file.length(); + if (size > maxFileBytes) { + log.warn("[collect-data] source file exceeds size limit file={} size={} max={}", + file.getName(), size, maxFileBytes); + throw new BusinessException("源文件大小超过上限 " + maxFileBytes + " 字节"); + } + } + + public void validateTotalRowCount(int totalRows) { + if (totalRows > maxTotalRows) { + throw new BusinessException("累计行数超过上限 " + maxTotalRows); + } + } + + public void validateChunkRowCount(int chunkRows) { + if (chunkRows > maxChunkRows) { + throw new BusinessException("单 chunk 行数超过上限 " + maxChunkRows); + } + } +} diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 68efcba3..4a9ad642 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -255,6 +255,12 @@ aiimage: coze-include-legacy-api-key: ${AIIMAGE_SIMILAR_ASIN_COZE_INCLUDE_LEGACY_API_KEY:true} coze-use-legacy-item-field-order: ${AIIMAGE_SIMILAR_ASIN_COZE_USE_LEGACY_ITEM_ORDER:false} coze-result-buffer-enabled: ${AIIMAGE_SIMILAR_ASIN_COZE_RESULT_BUFFER_ENABLED:true} + collect-data: + stale-timeout-minutes: ${AIIMAGE_COLLECT_DATA_STALE_TIMEOUT_MINUTES:30} + stale-check-cron: ${AIIMAGE_COLLECT_DATA_STALE_CHECK_CRON:*/30 * * * * *} + max-source-file-bytes: ${AIIMAGE_COLLECT_DATA_MAX_SOURCE_FILE_BYTES:0} + max-parse-rows: ${AIIMAGE_COLLECT_DATA_MAX_PARSE_ROWS:0} + max-chunk-rows: ${AIIMAGE_COLLECT_DATA_MAX_CHUNK_ROWS:0} image-video: coze-base-url: ${AIIMAGE_IMAGE_VIDEO_COZE_BASE_URL:https://api.coze.cn} coze-token: ${AIIMAGE_IMAGE_VIDEO_COZE_TOKEN:sat_Ws4VB1caOPasDivpKIvtOySYx3lhKgQ95H3crIh0tBwiNYtPTyi6bqe0pBaRzpVu} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimitsTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimitsTest.java new file mode 100644 index 00000000..a64ecd8c --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataParseLimitsTest.java @@ -0,0 +1,167 @@ +package com.nanri.aiimage.modules.collectdata.util; + +import com.nanri.aiimage.common.exception.BusinessException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Task 42：限制采集解析的文件大小、最大行数和单 chunk 行数。 + * CollectDataParseLimits 校验真实文件大小与累计行数/chunk 行数上限； + * 空输入与单元素通过，恰好等于上限通过、超过 1 被拒绝，非法配置被拒绝， + * 超限拒绝不残留状态。 + */ +class CollectDataParseLimitsTest { + + @TempDir + Path tempDir; + + private static final long MAX_FILE_BYTES = 1024L; + private static final int MAX_TOTAL_ROWS = 1000; + private static final int MAX_CHUNK_ROWS = 100; + + private CollectDataParseLimits limits() { + return new CollectDataParseLimits(MAX_FILE_BYTES, MAX_TOTAL_ROWS, MAX_CHUNK_ROWS); + } + + @Test + void test_task_042_file_size_row_count_chunk_normal_default_path() { + // 正常输入：文件在大小上限内、累计行数与单 chunk 行数均未超限，全部校验通过。 + CollectDataParseLimits limits = limits(); + File file = writeFile("ok.xlsx", 512); + + assertDoesNotThrow(() -> limits.validateSourceFile(file), "正常大小文件通过"); + assertDoesNotThrow(() -> limits.validateTotalRowCount(500), "累计 500 行通过"); + assertDoesNotThrow(() -> limits.validateChunkRowCount(50), "单 chunk 50 行通过"); + } + + @Test + void test_task_042_file_size_row_count_chunk_normal_multiple_items() { + // 批量场景：多个源文件累计行数按总和校验，未超限通过、超限被拒。 + CollectDataParseLimits limits = limits(); + + limits.validateTotalRowCount(400); + limits.validateTotalRowCount(600); + assertDoesNotThrow(() -> limits.validateTotalRowCount(900), "两次累计 900 行未超限"); + + BusinessException ex = assertThrows(BusinessException.class, + () -> limits.validateTotalRowCount(1001), "累计超限被拒绝"); + assertTrue(ex.getMessage().contains("行数"), "异常消息可识别"); + } + + @Test + void test_task_042_file_size_row_count_chunk_normal_repeated_operation_is_idempotent() { + // 幂等：同一输入重复校验结果一致，校验不产生残留状态。 + CollectDataParseLimits limits = limits(); + File file = writeFile("repeat.xlsx", 128); + + assertDoesNotThrow(() -> limits.validateSourceFile(file)); + assertDoesNotThrow(() -> limits.validateSourceFile(file)); + assertDoesNotThrow(() -> limits.validateChunkRowCount(10)); + assertDoesNotThrow(() -> limits.validateChunkRowCount(10)); + + BusinessException ex = assertThrows(BusinessException.class, + () -> limits.validateChunkRowCount(101), "重复校验后超限仍被拒绝"); + assertTrue(ex.getMessage().contains("chunk"), "chunk 超限消息可识别"); + } + + @Test + void test_task_042_file_size_row_count_chunk_boundary_empty_input() { + // 空输入：0 字节文件与 0 行均通过，不创建无效资源。 + CollectDataParseLimits limits = limits(); + File empty = writeFile("empty.xlsx", 0); + + assertDoesNotThrow(() -> limits.validateSourceFile(empty), "0 字节文件通过"); + assertDoesNotThrow(() -> limits.validateTotalRowCount(0), "0 行通过"); + assertDoesNotThrow(() -> limits.validateChunkRowCount(0), "0 chunk 行通过"); + } + + @Test + void test_task_042_file_size_row_count_chunk_boundary_single_item() { + // 单元素：单文件、单行、单 chunk 行不依赖批量路径，全部通过。 + CollectDataParseLimits limits = limits(); + File file = writeFile("single.xlsx", 64); + + assertDoesNotThrow(() -> limits.validateSourceFile(file), "单文件通过"); + assertDoesNotThrow(() -> limits.validateTotalRowCount(1), "单行通过"); + assertDoesNotThrow(() -> limits.validateChunkRowCount(1), "单 chunk 行通过"); + } + + @Test + void test_task_042_file_size_row_count_chunk_boundary_limit_and_overflow() { + // 上限/超限：恰好等于上限通过，超过 1 个字节/行被拒绝。 + CollectDataParseLimits limits = limits(); + File exact = writeFile("exact.xlsx", MAX_FILE_BYTES); + + assertDoesNotThrow(() -> limits.validateSourceFile(exact), "恰好等于文件上限通过"); + assertDoesNotThrow(() -> limits.validateTotalRowCount(MAX_TOTAL_ROWS), "恰好等于行数上限通过"); + assertDoesNotThrow(() -> limits.validateChunkRowCount(MAX_CHUNK_ROWS), "恰好等于 chunk 上限通过"); + + File overflow = writeFile("overflow.xlsx", MAX_FILE_BYTES + 1); + BusinessException fileEx = assertThrows(BusinessException.class, + () -> limits.validateSourceFile(overflow), "文件超限被拒绝"); + assertTrue(fileEx.getMessage().contains("大小"), "文件超限消息可识别"); + assertThrows(BusinessException.class, + () -> limits.validateTotalRowCount(MAX_TOTAL_ROWS + 1), "行数超限被拒绝"); + assertThrows(BusinessException.class, + () -> limits.validateChunkRowCount(MAX_CHUNK_ROWS + 1), "chunk 超限被拒绝"); + } + + @Test + void test_task_042_file_size_row_count_chunk_invalid_input_rejected() { + // 非法输入：null/不存在文件、零与负配置值抛可识别异常。 + CollectDataParseLimits limits = limits(); + + assertThrows(BusinessException.class, + () -> limits.validateSourceFile(null), "null 文件被拒绝"); + assertThrows(BusinessException.class, + () -> limits.validateSourceFile(new File(tempDir.toFile(), "missing.xlsx")), "不存在文件被拒绝"); + assertThrows(IllegalArgumentException.class, + () -> new CollectDataParseLimits(0, MAX_TOTAL_ROWS, MAX_CHUNK_ROWS), "0 文件上限被拒绝"); + assertThrows(IllegalArgumentException.class, + () -> new CollectDataParseLimits(MAX_FILE_BYTES, -1, MAX_CHUNK_ROWS), "负行数上限被拒绝"); + assertThrows(IllegalArgumentException.class, + () -> new CollectDataParseLimits(MAX_FILE_BYTES, MAX_TOTAL_ROWS, 0), "0 chunk 上限被拒绝"); + } + + @Test + void test_task_042_file_size_row_count_chunk_dependency_failure_releases_resources() { + // 依赖失败：超限文件被拒绝后不残留状态，后续正常输入仍通过； + // 临时文件可被正常删除（无句柄泄漏）。 + CollectDataParseLimits limits = limits(); + File big = writeFile("big.xlsx", MAX_FILE_BYTES * 2); + + assertThrows(BusinessException.class, () -> limits.validateSourceFile(big), "超限文件被拒绝"); + + File normal = writeFile("after.xlsx", 32); + assertDoesNotThrow(() -> limits.validateSourceFile(normal), "拒绝后正常文件仍通过"); + assertDoesNotThrow(() -> limits.validateChunkRowCount(5), "拒绝后 chunk 校验不受影响"); + + assertTrue(big.delete(), "被拒绝的超限临时文件可删除（无句柄泄漏）"); + assertFalse(big.exists(), "临时文件已清理"); + } + + private File writeFile(String name, long bytes) { + try { + Path path = tempDir.resolve(name); + byte[] data = new byte[(int) bytes]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) ('a' + i % 26); + } + Files.write(path, data); + return path.toFile(); + } catch (Exception ex) { + throw new IllegalStateException("测试临时文件写入失败", ex); + } + } +}