diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java index b8a9d411..aa025498 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java @@ -83,7 +83,7 @@ public class ShopDataCrawlExcelAssemblyService { imageEmbedder.prefetch(prefetchBudget().boundedUrls(imageUrls(rowsByCountry)), imageCache); } - public void writeWorkbook(File outputXlsx, List items) { + public int writeWorkbook(File outputXlsx, List items) { try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream(); XSSFWorkbook workbook = new XSSFWorkbook(input); FileOutputStream output = new FileOutputStream(outputXlsx)) { @@ -96,6 +96,7 @@ public class ShopDataCrawlExcelAssemblyService { writeSheet(workbook, workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)), imageCache, pictureIndexes); } workbook.write(output); + return rowsByCountry.values().stream().mapToInt(List::size).sum(); } catch (BusinessException ex) { throw ex; } catch (Exception ex) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTemplateLargeWorkbookTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTemplateLargeWorkbookTest.java new file mode 100644 index 00000000..102414ed --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTemplateLargeWorkbookTest.java @@ -0,0 +1,323 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import org.apache.poi.openxml4j.util.ZipSecureFile; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Task 26：模板 workbook 大行数下的样式、图片和工作表兼容测试。 + * writeWorkbook（XSSFWorkbook 模板路径）返回实际写入的数据行数，与 streaming 路径一致； + * 大行数（1000/5000）下验证：模板 styleRow 样式传递到写入行、同 URL 图片只嵌入一次、 + * 5 个工作表顺序与名称保持模板语义、失败行兜底 URL 文本不中断整表。 + */ +class ShopDataCrawlTemplateLargeWorkbookTest { + @TempDir Path tempDir; + + private static final String URL_TEMPLATE = "https://thumb.example/img"; + + @BeforeAll + static void relaxZipSecurity() { + // 大行数下 XSSFWorkbook 写出含大量图片的 xlsx 内部条目数超过 POI 5.2.5 默认防护阈值。 + ZipSecureFile.setMaxFileCount(2_000_000L); + ZipSecureFile.setMinInflateRatio(0.0); + } + + private static SimilarAsinImageEmbedder okEmbedder() { + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer( + invocation -> new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2)); + return imageEmbedder; + } + + @Test + void test_task_026_row_count_image_workbook_normal_default_path() throws Exception { + // 正常输入：2000 行 5 国模板写入，返回行数正确； + // 模板表头样式保留（与模板文件一致）；5 个工作表顺序/名称保持模板语义； + // 唯一图片各嵌入一次。 + SimilarAsinImageEmbedder imageEmbedder = okEmbedder(); + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024); + List items = items(2000, 5); + + File output = tempDir.resolve("template-2000.xlsx").toFile(); + int written = service.writeWorkbook(output, items); + + assertEquals(2000, written, "writeWorkbook 返回实际写入行数"); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS, sheetNames(workbook)); + for (int i = 0; i < 5; i++) { + assertEquals(400, workbook.getSheetAt(i).getLastRowNum(), "每国 400 行"); + } + int templateHeaderStyle = templateHeaderStyleIndex(); + assertEquals(templateHeaderStyle, + workbook.getSheet("英国").getRow(0).getCell(0).getCellStyle().getIndex(), + "表头样式保留"); + assertEquals("2026-07-25", + workbook.getSheet("英国").getRow(1).getCell(0).getStringCellValue(), "数据行文本正确"); + assertEquals(2000, workbook.getAllPictures().size(), "2000 个唯一 URL 各嵌入一张图"); + assertEquals(80f, workbook.getSheet("英国").getRow(1).getHeightInPoints(), "图片行行高自适应"); + } + } + + @Test + void test_task_026_row_count_image_workbook_normal_multiple_items() throws Exception { + // 批量场景：两个任务项合并 2000 行，返回行数合计正确，顺序稳定。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List all = new ArrayList<>(items(1200, 5)); + all.addAll(items(800, 5)); + + File output = tempDir.resolve("template-multi.xlsx").toFile(); + int written = service.writeWorkbook(output, all); + assertEquals(2000, written); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + int total = 0; + for (int i = 0; i < 5; i++) { + total += workbook.getSheetAt(i).getLastRowNum(); + } + assertEquals(2000, total, "批量合并行数不丢失"); + assertEquals(1200, workbook.getAllPictures().size(), "重复 URL 按 pictureIndex 去重复用"); + } + } + + @Test + void test_task_026_row_count_image_workbook_normal_repeated_operation_is_idempotent() throws Exception { + // 重复执行：同一输入两次模板写入，行数/图片数/样式一致，不产生重复记录。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List items = items(1000, 5); + File firstOut = tempDir.resolve("template-idem-1.xlsx").toFile(); + File secondOut = tempDir.resolve("template-idem-2.xlsx").toFile(); + + int first = service.writeWorkbook(firstOut, items); + int second = service.writeWorkbook(secondOut, items); + assertEquals(first, second, "重复写入行数一致"); + try (XSSFWorkbook wb1 = new XSSFWorkbook(new FileInputStream(firstOut)); + XSSFWorkbook wb2 = new XSSFWorkbook(new FileInputStream(secondOut))) { + assertEquals(wb1.getAllPictures().size(), wb2.getAllPictures().size(), "图片数一致"); + assertEquals(wb1.getSheet("英国").getLastRowNum(), wb2.getSheet("英国").getLastRowNum()); + assertEquals(wb1.getSheet("英国").getRow(0).getCell(0).getCellStyle().getIndex(), + wb2.getSheet("英国").getRow(0).getCell(0).getCellStyle().getIndex(), "表头样式一致"); + assertEquals("B000000000", wb1.getSheet("英国").getRow(1).getCell(1).getStringCellValue(), + "首行数据一致"); + } + } + + @Test + void test_task_026_row_count_image_workbook_boundary_empty_input() throws Exception { + // 空输入：0 行模板写入返回 0，表头与模板样式保留，无图片。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + File output = tempDir.resolve("template-empty.xlsx").toFile(); + int written = service.writeWorkbook(output, List.of()); + + assertEquals(0, written); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + for (int i = 0; i < 5; i++) { + assertEquals(0, workbook.getSheetAt(i).getLastRowNum(), "只有表头行"); + assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(0), + workbook.getSheetAt(i).getRow(0).getCell(0).getStringCellValue(), "表头保留"); + } + assertEquals(0, workbook.getAllPictures().size()); + } + } + + @Test + void test_task_026_row_count_image_workbook_boundary_single_item() throws Exception { + // 单元素：1 行模板写入返回 1，样式/图片完整，不依赖批量路径。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List items = items(1, 1); + File output = tempDir.resolve("template-single.xlsx").toFile(); + int written = service.writeWorkbook(output, items); + + assertEquals(1, written); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(1, workbook.getSheet("英国").getLastRowNum()); + assertEquals("B000000000", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue()); + assertEquals(1, workbook.getAllPictures().size()); + assertEquals(templateHeaderStyleIndex(), + workbook.getSheet("英国").getRow(0).getCell(0).getCellStyle().getIndex(), + "表头样式保留"); + } + } + + @Test + void test_task_026_row_count_image_workbook_boundary_limit_and_overflow() throws Exception { + // 上限/超限：5000 行（MAX_ROWS 规模）模板写入在预算内完成，行数完整； + // 失败图片行兜底 URL 文本，成功图片各嵌入一次，不发生无界图片堆积。 + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer(invocation -> { + String url = invocation.getArgument(0); + if (url.contains("fail")) { + return null; + } + return new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2); + }); + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024); + List items = items(5000, 5); + for (int i = 0; i < 5000; i += 2) { + setImage(items, i, "https://thumb.example/fail" + i + ".jpg"); + } + + File output = tempDir.resolve("template-5000.xlsx").toFile(); + long start = System.currentTimeMillis(); + int written = service.writeWorkbook(output, items); + long elapsed = System.currentTimeMillis() - start; + + assertEquals(5000, written); + assertTrue(elapsed < 60_000, "5000 行模板写入须在预算内完成，实际=" + elapsed + "ms"); + assertEquals(5000, service.countRows(items), "countRows 与写入行数一致"); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + int total = 0; + for (int i = 0; i < 5; i++) { + total += workbook.getSheetAt(i).getLastRowNum(); + } + assertEquals(5000, total, "读回行数完整"); + assertEquals(2500, workbook.getAllPictures().size(), "2500 行成功图片嵌入"); + } + } + + @Test + void test_task_026_row_count_image_workbook_invalid_input_rejected() throws Exception { + // 非法参数：null 输出路径/失败项（success=false）被跳过/无结果项 → 明确行为。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List items = items(10, 1); + + assertThrows(BusinessException.class, () -> service.writeWorkbook(null, items)); + + ShopDataCrawlResultItemVo failed = new ShopDataCrawlResultItemVo(); + failed.setSuccess(false); + failed.setError("抓取失败"); + List withFailed = new ArrayList<>(items); + withFailed.add(failed); + File output = tempDir.resolve("template-invalid.xlsx").toFile(); + assertEquals(10, service.writeWorkbook(output, withFailed), "失败项被跳过"); + + File empty = tempDir.resolve("template-none.xlsx").toFile(); + assertEquals(0, service.writeWorkbook(empty, null), "null items 安全返回 0"); + } + + @Test + void test_task_026_row_count_image_workbook_dependency_failure_releases_resources() throws Exception { + // 依赖失败：图片下载抛异常 → 单行兜底 URL 文本，整表行数完整、无图片； + // 目标目录不可写抛 BusinessException；重试成功不残留临时状态。 + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(any())).thenThrow( + new RuntimeException("image service down")); + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024); + + List items = items(100, 1); + File output = tempDir.resolve("template-fail.xlsx").toFile(); + int written = service.writeWorkbook(output, items); + assertEquals(100, written, "图片失败不阻塞行写入"); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(100, workbook.getSheet("英国").getLastRowNum()); + assertEquals(0, workbook.getAllPictures().size(), "无图片嵌入"); + assertEquals(URL_TEMPLATE + "-0.jpg", + workbook.getSheet("英国").getRow(1).getCell(2).getStringCellValue(), "兜底 URL 文本"); + } + + File locked = tempDir.resolve("locked").toFile(); + assertTrue(locked.mkdir(), "目录占位模拟不可写目标"); + assertThrows(BusinessException.class, () -> service.writeWorkbook(locked, items)); + + File retried = tempDir.resolve("template-retry.xlsx").toFile(); + assertEquals(100, service.writeWorkbook(retried, items), "失败后重试成功"); + } + + private static int templateHeaderStyleIndex() throws Exception { + try (org.apache.poi.ss.usermodel.Workbook template = + new XSSFWorkbook(new org.springframework.core.io.ClassPathResource( + "templates/shop-data-crawl/文档格式.xlsx").getInputStream())) { + return template.getSheetAt(0).getRow(0).getCell(0).getCellStyle().getIndex(); + } + } + + private static void setImage(List items, int index, String url) { + int i = 0; + for (ShopDataCrawlResultItemVo item : items) { + for (ShopDataCrawlCountryResultDto country : item.getCountryResults()) { + for (ShopDataCrawlRowDto row : country.getItems()) { + if (i == index) { + row.setCommodityImage(url); + return; + } + i++; + } + } + } + } + + private static List items(int rowCount, int countryCount) { + List items = new ArrayList<>(); + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setSuccess(true); + List countryResults = new ArrayList<>(); + List countries = ShopDataCrawlExcelAssemblyService.COUNTRIES.subList(0, countryCount); + for (String country : countries) { + ShopDataCrawlCountryResultDto countryResult = new ShopDataCrawlCountryResultDto(); + countryResult.setCountry(country); + countryResult.setItems(new ArrayList<>()); + countryResults.add(countryResult); + } + for (int i = 0; i < rowCount; i++) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate("2026-07-25"); + row.setAsin("B0" + String.format("%08d", i)); + row.setBrand("Brand"); + row.setCommodityImage(URL_TEMPLATE + "-" + i + ".jpg"); + row.setInventorySales("11"); + row.setSalesRank("22"); + row.setPageViews("33"); + row.setUnitsSold("44"); + row.setPrice("12.50"); + row.setRecommendedOffer("12.00"); + countryResults.get(i % countryCount).getItems().add(row); + } + item.setCountryResults(countryResults); + items.add(item); + return items; + } + + private static List sheetNames(XSSFWorkbook workbook) { + List names = new ArrayList<>(); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + names.add(workbook.getSheetAt(i).getSheetName()); + } + return names; + } + + private static byte[] jpegBytes() throws Exception { + BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ImageIO.write(image, "jpg", output); + return output.toByteArray(); + } +}