super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-5: 预览行数量改为配置驱动并做边界校验

Browse Source 
解析响应预览上限从硬编码常量改为 SimilarAsinProperties.parseResponsePreviewLimit，
0/负值回退默认 100，超过 1000 clamp 到 1000 防止误配导致响应体无界膨胀。
新增 8 个测试覆盖默认路径、自定义值、幂等、空文件、单行、超限 clamp、非法配置回退、依赖失败恢复。
This commit is contained in:
黄自达
2026-08-29 14:48:01 +08:00
parent 026aa4e0c3
commit 16bec1c0fa
3 changed files with 277 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java
+7
View File
@@ -132,6 +132,13 @@ public class SimilarAsinProperties {
*/
private long cozeSubmitLockWaitMillis = 10000L;
/**
* 解析接口返回的预览行/预览组数量上限。
* 响应体只携带预览行（默认 100），全量行仅写入后端任务载荷。
* 有效范围 [1, 1000]；0/负值回退默认，超上限 clamp 到 1000。
*/
private int parseResponsePreviewLimit = 100;
/**
* P0-4：抢 Coze 提交锁失败后下次重试间隔（毫秒）。
* 原硬编码 500ms，会在指数退避算法中作为基础值（500/1000/2000/4000ms 上限 4000）。
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+15 -2
View File
@@ -143,6 +143,19 @@ public class SimilarAsinTaskService {
// SimilarAsinProperties.cozeSubmitLockWaitMillis / cozeSubmitLockRetryDelayMillis，
// 由 acquireCozeSubmitLock 在方法内读取，并支持指数退避。
private static final int PARSE_RESPONSE_PREVIEW_LIMIT = 100;
/** 预览行/预览组配置上限：超过该值 clamp，避免响应体无界增长。 */
private static final int PARSE_RESPONSE_PREVIEW_LIMIT_MAX = 1000;
/**
* 解析接口响应预览上限：优先读取配置 parseResponsePreviewLimit。
* 0/负值回退默认 100；超过 1000 clamp 到 1000，防止误配导致响应体膨胀。
*/
private int resolvePreviewLimit() {
Integer configured = properties.getParseResponsePreviewLimit();
if (configured == null || configured <= 0) {
return PARSE_RESPONSE_PREVIEW_LIMIT;
}
return Math.min(configured, PARSE_RESPONSE_PREVIEW_LIMIT_MAX);
}
/**
* P0-2 最小风险变体：poll 调度阶段并发预取 Coze HTTP 结果时，
* 控制对单个 Coze 后端的并发度。8 与 cozeTaskExecutor 的 12 并发上限对齐留 4 余量，
@@ -1472,7 +1485,7 @@ public class SimilarAsinTaskService {
if (groups == null || groups.isEmpty()) {
return List.of();
}
int limit = Math.min(PARSE_RESPONSE_PREVIEW_LIMIT, groups.size());
int limit = Math.min(resolvePreviewLimit(), groups.size());
List<SimilarAsinParsedGroupVo> preview = new ArrayList<>(limit);
for (int i = 0; i < limit; i++) {
SimilarAsinParsedGroupVo group = groups.get(i);
@@ -1496,7 +1509,7 @@ public class SimilarAsinTaskService {
if (rows == null || rows.isEmpty()) {
return List.of();
}
int limit = Math.min(PARSE_RESPONSE_PREVIEW_LIMIT, rows.size());
int limit = Math.min(resolvePreviewLimit(), rows.size());
List<SimilarAsinParsedRowVo> preview = new ArrayList<>(limit);
for (int i = 0; i < limit; i++) {
preview.add(copyPreviewRow(rows.get(i)));
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServicePreviewConfigTest.java
+255
View File
@@ -0,0 +1,255 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.InjectMocks;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.springframework.test.util.ReflectionTestUtils;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Task 5：预览行数量增加配置边界、空文件和超限输入校验。
* 预览上限从硬编码常量改为配置驱动，并对配置值做 clamp 边界保护。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServicePreviewConfigTest {
private static final AtomicLong NEXT_ID = new AtomicLong(20000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/20000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private File buildWorkbook(int rowCount) throws Exception {
File file = Files.createTempFile("similar-asin-preview-config-", ".xlsx").toFile();
try (XSSFWorkbook workbook = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(file)) {
var sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("id");
header.createCell(1).setCellValue("asin");
header.createCell(2).setCellValue("国家");
for (int i = 1; i <= rowCount; i++) {
Row row = sheet.createRow(i);
row.createCell(0).setCellValue(String.valueOf(i));
row.createCell(1).setCellValue(String.format("B0CFG%05d", i));
row.createCell(2).setCellValue("英国");
}
workbook.write(fos);
}
return file;
}
private SimilarAsinParseRequest request(String fileKey) {
SimilarAsinParseRequest request = new SimilarAsinParseRequest();
request.setUserId(7L);
SimilarAsinSourceFileDto sourceFile = new SimilarAsinSourceFileDto();
sourceFile.setFileKey(fileKey);
sourceFile.setOriginalFilename("config.xlsx");
request.setFiles(List.of(sourceFile));
request.setApiKey("sk-123");
request.setImgSwitch(Boolean.FALSE);
request.setCategorySwitch(Boolean.FALSE);
return request;
}
private SimilarAsinParseVo parse(File workbook, String fileKey) {
when(localFileStorageService.findLocalSourceFile(fileKey)).thenReturn(workbook);
return service.parseAndCreateTask(request(fileKey));
}
@Test
void test_task_005_preview_row_count_normal_default_path() throws Exception {
// 默认配置 100：150 行文件返回 100 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(150);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-default.xlsx");
assertEquals(100, vo.getItems().size());
assertEquals(150, vo.getAcceptedRows());
// 预览行从第 1 行开始
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
}
@Test
void test_task_005_preview_row_count_normal_multiple_items() throws Exception {
// 配置 50：500 行文件返回 50 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(50);
File workbook = buildWorkbook(500);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-50.xlsx");
assertEquals(50, vo.getItems().size());
assertEquals(500, vo.getAcceptedRows());
// groups 同样按配置裁剪
assertTrue(vo.getGroups().size() <= 50);
// 配置 200：300 行文件返回 200 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(200);
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-200.xlsx");
assertEquals(200, vo2.getItems().size());
}
@Test
void test_task_005_preview_row_count_normal_repeated_operation_is_idempotent() throws Exception {
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(250);
SimilarAsinParseVo first = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
SimilarAsinParseVo second = parse(workbook, "uploads/20260829/cfg-idem.xlsx");
assertEquals(first.getItems().size(), second.getItems().size());
for (int i = 0; i < first.getItems().size(); i++) {
assertEquals(first.getItems().get(i).getAsin(), second.getItems().get(i).getAsin());
}
}
@Test
void test_task_005_preview_row_count_boundary_empty_input() throws Exception {
// 空文件（无有效数据行）：抛业务异常
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(0);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-empty.xlsx")).thenReturn(workbook);
assertThrows(BusinessException.class, () -> parse(workbook, "uploads/20260829/cfg-empty.xlsx"));
}
@Test
void test_task_005_preview_row_count_boundary_single_item() throws Exception {
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(1);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-single.xlsx");
assertEquals(1, vo.getItems().size());
assertEquals("B0CFG00001", vo.getItems().get(0).getAsin());
}
@Test
void test_task_005_preview_row_count_boundary_limit_and_overflow() throws Exception {
// 配置值超过最大上限（1000）：clamp 到上限，不发生无界内存增长
when(properties.getParseResponsePreviewLimit()).thenReturn(50000);
File workbook = buildWorkbook(2000);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-over.xlsx");
assertEquals(1000, vo.getItems().size(), "超限配置必须 clamp 到最大允许值");
assertEquals(2000, vo.getAcceptedRows());
// 配置值恰好等于上限：返回 1000 预览行
when(properties.getParseResponsePreviewLimit()).thenReturn(1000);
SimilarAsinParseVo vo2 = parse(buildWorkbook(1000), "uploads/20260829/cfg-exact.xlsx");
assertEquals(1000, vo2.getItems().size());
}
@Test
void test_task_005_preview_row_count_invalid_input_rejected() throws Exception {
// 配置为 0/负数：回退到默认值 100，不抛异常不崩溃
when(properties.getParseResponsePreviewLimit()).thenReturn(0, -5);
File workbook = buildWorkbook(300);
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-zero.xlsx");
assertEquals(100, vo.getItems().size());
SimilarAsinParseVo vo2 = parse(buildWorkbook(300), "uploads/20260829/cfg-neg.xlsx");
assertEquals(100, vo2.getItems().size());
// 配置缺失（fresh mock 未 stub，int 默认 0）：同样回退默认
SimilarAsinProperties missing = org.mockito.Mockito.mock(SimilarAsinProperties.class);
lenient().when(missing.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(missing.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(missing.getResultFileTimeoutMinutes()).thenReturn(90);
ReflectionTestUtils.setField(service, "properties", missing);
SimilarAsinParseVo vo3 = parse(buildWorkbook(300), "uploads/20260829/cfg-missing.xlsx");
assertEquals(100, vo3.getItems().size());
}
@Test
void test_task_005_preview_row_count_dependency_failure_releases_resources() throws Exception {
// 存储失败抛异常；恢复后解析正常
when(properties.getParseResponsePreviewLimit()).thenReturn(100);
File workbook = buildWorkbook(120);
when(localFileStorageService.findLocalSourceFile("uploads/20260829/cfg-fail.xlsx")).thenReturn(workbook);
when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenThrow(new IllegalStateException("rustfs down"))
.thenReturn("rustfs:task-parsed/similar-asin/20001/payload.json");
assertThrows(IllegalStateException.class, () -> parse(workbook, "uploads/20260829/cfg-fail.xlsx"));
SimilarAsinParseVo vo = parse(workbook, "uploads/20260829/cfg-recovered.xlsx");
assertEquals(120, vo.getAcceptedRows());
assertEquals(100, vo.getItems().size());
// 配置对象默认值校验：新实例默认 100，处于 [1, 1000] 边界内
SimilarAsinProperties defaults = new SimilarAsinProperties();
assertNotNull(defaults.getParseResponsePreviewLimit());
int previewLimit = defaults.getParseResponsePreviewLimit();
assertTrue(previewLimit >= 1 && previewLimit <= 1000,
"默认预览上限必须在 [1, 1000] 内，实际 " + previewLimit);
}
}