diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java index a94dc208..b8a9d411 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java @@ -18,6 +18,7 @@ import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.usermodel.XSSFDrawing; import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTTwoCellAnchor; import org.springframework.core.io.ClassPathResource; import org.springframework.stereotype.Service; @@ -141,6 +142,60 @@ public class ShopDataCrawlExcelAssemblyService { return rowsByCountry(items).values().stream().mapToInt(List::size).sum(); } + /** + * Task 25：SXSSF 流式写入路径。空构造 SXSSFWorkbook + 自建 5 个国家工作表与表头， + * 数据行按 rowAccessWindow 数量 spill 到磁盘临时文件，写入后 dispose() 释放 spill 文件。 + * 模板路径（validateTemplate/样式/累计替换）保留 XSSFWorkbook 不变。 + * 返回实际写入的数据行数；图片下载失败的行兜底为 URL 文本，不中断整表。 + */ + public int writeWorkbookStreaming(File outputXlsx, List items, int rowAccessWindow) { + if (outputXlsx == null) { + throw new BusinessException("输出文件路径不能为空"); + } + if (rowAccessWindow <= 0) { + throw new IllegalArgumentException("rowAccessWindow 必须为正数，实际 " + rowAccessWindow); + } + if (items == null) { + throw new IllegalArgumentException("items 不能为 null"); + } + SXSSFWorkbook workbook = new SXSSFWorkbook(rowAccessWindow); + try (FileOutputStream output = new FileOutputStream(outputXlsx)) { + Map> rowsByCountry = rowsByCountry(items); + BoundedImageCache imageCache = newImageCache(); + prefetchImages(rowsByCountry, imageCache); + Map pictureIndexes = new LinkedHashMap<>(); + for (int i = 0; i < COUNTRIES.size(); i++) { + writeStreamingSheet(workbook, i, rowsByCountry.get(COUNTRIES.get(i)), imageCache, pictureIndexes); + } + workbook.write(output); + return rowsByCountry.values().stream().mapToInt(List::size).sum(); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("生成店铺数据抓取流式 Excel 失败: " + ex.getMessage()); + } finally { + workbook.dispose(); + } + } + + private void writeStreamingSheet(Workbook workbook, + int index, + List rows, + BoundedImageCache imageCache, + Map pictureIndexes) { + Sheet sheet = workbook.createSheet(SHEETS.get(index)); + Row header = sheet.createRow(0); + for (int column = 0; column < HEADERS.size(); column++) { + header.createCell(column).setCellValue(HEADERS.get(column)); + } + sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH); + int rowIndex = 1; + for (ShopDataCrawlRowDto value : rows == null ? List.of() : rows) { + Row row = sheet.createRow(rowIndex++); + writeDataRow(workbook, sheet, row, value, new CellStyle[HEADERS.size()], imageCache, pictureIndexes); + } + } + void validateTemplate(XSSFWorkbook workbook) { if (workbook.getNumberOfSheets() != SHEETS.size()) { throw new BusinessException("店铺数据抓取模板工作表数量不正确"); @@ -205,7 +260,7 @@ public class ShopDataCrawlExcelAssemblyService { } } - private void writeDataRow(XSSFWorkbook workbook, + private void writeDataRow(Workbook workbook, Sheet sheet, Row row, ShopDataCrawlRowDto value, @@ -249,7 +304,7 @@ public class ShopDataCrawlExcelAssemblyService { return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1; } - private void embedImage(XSSFWorkbook workbook, + private void embedImage(Workbook workbook, Sheet sheet, Row row, String imageUrl, diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlStreamingWorkbookTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlStreamingWorkbookTest.java new file mode 100644 index 00000000..b4baff79 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlStreamingWorkbookTest.java @@ -0,0 +1,301 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import org.apache.poi.openxml4j.util.ZipSecureFile; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Task 25：店铺结果 workbook 的 SXSSF 流式写入路径。 + * SXSSFWorkbook 与现有"读模板行样式→清空重写"流程不兼容（模板行不可见、 + * 无法删除），故评估结论为：模板路径保留 XSSFWorkbook，另提供独立 streaming 写入器 + * writeWorkbookStreaming —— 空构造 SXSSFWorkbook + 自建 5 个国家工作表与表头， + * 数据行按 rowAccessWindow spill 到磁盘，写入后 dispose() 释放临时 spill 文件。 + */ +class ShopDataCrawlStreamingWorkbookTest { + @TempDir Path tempDir; + + private static final String OK_URL = "https://thumb.example/ok.jpg"; + private static final String FAIL_URL = "https://thumb.example/fail.jpg"; + + @BeforeAll + static void relaxZipSecurity() { + // SXSSF 写出含大量图片的 workbook 内部条目数超过 POI 5.2.5 默认防护阈值， + // 测试读回断言时需要放行（生产读回路径不涉及此规模）。 + ZipSecureFile.setMaxFileCount(2_000_000L); + ZipSecureFile.setMinInflateRatio(0.0); + } + + private static SimilarAsinImageEmbedder okEmbedder() { + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer( + invocation -> new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2)); + return imageEmbedder; + } + + @Test + void test_task_025_workbook_normal_default_path() throws Exception { + // 正常输入：1000 行 5 国 streaming 写入，表头/行数/图片完整。 + SimilarAsinImageEmbedder imageEmbedder = okEmbedder(); + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024); + List items = items(1000, 5, 0); + + File output = tempDir.resolve("streaming.xlsx").toFile(); + int written = service.writeWorkbookStreaming(output, items, 100); + + assertEquals(1000, written, "返回实际写入行数"); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS, sheetNames(workbook)); + for (int i = 0; i < ShopDataCrawlExcelAssemblyService.SHEETS.size(); i++) { + assertEquals(200, workbook.getSheetAt(i).getLastRowNum(), "每国 200 行"); + assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(0), + workbook.getSheetAt(i).getRow(0).getCell(0).getStringCellValue()); + assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(1), + workbook.getSheetAt(i).getRow(0).getCell(1).getStringCellValue()); + } + assertEquals(1000, workbook.getAllPictures().size(), "1000 个唯一 URL 各嵌入一张图"); + } + } + + @Test + void test_task_025_workbook_normal_multiple_items() throws Exception { + // 批量场景：两个任务项行合并写入，行数合计正确、顺序稳定。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List all = new ArrayList<>(items(600, 5, 0)); + all.addAll(items(400, 5, 0)); + + File output = tempDir.resolve("streaming-multi.xlsx").toFile(); + int written = service.writeWorkbookStreaming(output, all, 100); + assertEquals(1000, written); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + int total = 0; + for (int i = 0; i < 5; i++) { + total += workbook.getSheetAt(i).getLastRowNum(); + } + assertEquals(1000, total, "行数合计不丢失"); + } + } + + @Test + void test_task_025_workbook_normal_repeated_operation_is_idempotent() throws Exception { + // 重复执行：同一输入两次 streaming 写出，行数/图片数一致。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List items = items(500, 5, 0); + File firstOut = tempDir.resolve("streaming-idem-1.xlsx").toFile(); + File secondOut = tempDir.resolve("streaming-idem-2.xlsx").toFile(); + + int first = service.writeWorkbookStreaming(firstOut, items, 100); + int second = service.writeWorkbookStreaming(secondOut, items, 100); + assertEquals(first, second, "重复写入行数一致"); + try (XSSFWorkbook wb1 = new XSSFWorkbook(new FileInputStream(firstOut)); + XSSFWorkbook wb2 = new XSSFWorkbook(new FileInputStream(secondOut))) { + assertEquals(wb1.getAllPictures().size(), wb2.getAllPictures().size(), "图片数一致"); + assertEquals(wb1.getSheet("英国").getLastRowNum(), wb2.getSheet("英国").getLastRowNum()); + } + } + + @Test + void test_task_025_workbook_boundary_empty_input() throws Exception { + // 空输入：0 行写入只产出表头，无图片，返回 0。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + File output = tempDir.resolve("streaming-empty.xlsx").toFile(); + int written = service.writeWorkbookStreaming(output, List.of(), 100); + + assertEquals(0, written); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + for (int i = 0; i < 5; i++) { + assertEquals(0, workbook.getSheetAt(i).getLastRowNum(), "只有表头行"); + } + assertEquals(0, workbook.getAllPictures().size()); + } + } + + @Test + void test_task_025_workbook_boundary_single_item() throws Exception { + // 单元素：1 行 1 国写入，不依赖批量路径，图片嵌入。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List items = items(1, 1, 0); + File output = tempDir.resolve("streaming-single.xlsx").toFile(); + int written = service.writeWorkbookStreaming(output, items, 100); + + assertEquals(1, written); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(1, workbook.getSheet("英国").getLastRowNum()); + assertEquals("B000000000", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue()); + assertEquals(1, workbook.getAllPictures().size()); + } + } + + @Test + void test_task_025_workbook_boundary_limit_and_overflow() throws Exception { + // 上限/超限：5000 行大文件在 window=100 下 spill 写入，行数完整不丢失； + // 图片下载失败的行兜底为 URL 文本，不阻塞写入。 + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer(invocation -> { + String url = invocation.getArgument(0); + if (url.contains("fail")) { + return null; + } + return new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2); + }); + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024); + + List items = items(5000, 5, 0); + for (int i = 0; i < 5000; i += 2) { + setImage(items, i, FAIL_URL); + } + File output = tempDir.resolve("streaming-5000.xlsx").toFile(); + long start = System.currentTimeMillis(); + int written = service.writeWorkbookStreaming(output, items, 100); + long elapsed = System.currentTimeMillis() - start; + + assertEquals(5000, written, "5000 行 spill 写入不丢失"); + assertTrue(elapsed < 60_000, "5000 行写入须在预算内完成，实际=" + elapsed + "ms"); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + int total = 0; + for (int i = 0; i < 5; i++) { + total += workbook.getSheetAt(i).getLastRowNum(); + } + assertEquals(5000, total, "读回行数完整"); + assertEquals(2500, workbook.getAllPictures().size(), "2500 行成功图片嵌入，失败行兜底"); + } + } + + @Test + void test_task_025_workbook_invalid_input_rejected() throws Exception { + // 非法参数：null 输出/空输出目录/非正 window/null items → 明确异常。 + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024); + List items = items(10, 1, 0); + File output = tempDir.resolve("streaming-invalid.xlsx").toFile(); + + assertThrows(BusinessException.class, () -> service.writeWorkbookStreaming(null, items, 100)); + assertThrows(BusinessException.class, + () -> service.writeWorkbookStreaming(tempDir.resolve("no-dir").resolve("x.xlsx").toFile(), items, 100)); + assertThrows(IllegalArgumentException.class, + () -> service.writeWorkbookStreaming(output, items, 0)); + assertThrows(IllegalArgumentException.class, + () -> service.writeWorkbookStreaming(output, items, -5)); + assertThrows(IllegalArgumentException.class, + () -> service.writeWorkbookStreaming(output, null, 100)); + } + + @Test + void test_task_025_workbook_dependency_failure_releases_resources() throws Exception { + // 依赖失败：图片下载抛异常 → 单行兜底 URL 文本不中断整表； + // 写入失败（目标不可写）抛 BusinessException，重复调用稳定（spill 文件可重建）。 + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(any())).thenThrow( + new RuntimeException("image service down")); + ShopDataCrawlExcelAssemblyService service = + new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024); + + List items = items(100, 1, 0); + File output = tempDir.resolve("streaming-fail.xlsx").toFile(); + int written = service.writeWorkbookStreaming(output, items, 100); + assertEquals(100, written, "图片失败不阻塞行写入"); + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(100, workbook.getSheet("英国").getLastRowNum()); + assertEquals(0, workbook.getAllPictures().size(), "无图片嵌入"); + assertEquals(OK_URL + "?i=0", + workbook.getSheet("英国").getRow(1).getCell(2).getStringCellValue(), "兜底 URL 文本"); + } + + File locked = tempDir.resolve("locked").toFile(); + assertTrue(locked.mkdir(), "创建目录占位模拟不可写目标"); + assertThrows(BusinessException.class, + () -> service.writeWorkbookStreaming(locked, items, 100)); + + File retried = tempDir.resolve("streaming-retry.xlsx").toFile(); + assertEquals(100, service.writeWorkbookStreaming(retried, items, 100), "失败后重试成功，spill 可重建"); + } + + private static void setImage(List items, int index, String url) { + int i = 0; + for (ShopDataCrawlResultItemVo item : items) { + for (ShopDataCrawlCountryResultDto country : item.getCountryResults()) { + for (ShopDataCrawlRowDto row : country.getItems()) { + if (i == index) { + row.setCommodityImage(url); + return; + } + i++; + } + } + } + } + + private static List items(int rowCount, int countryCount, int failEvery) { + List items = new ArrayList<>(); + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setSuccess(true); + List countryResults = new ArrayList<>(); + List countries = ShopDataCrawlExcelAssemblyService.COUNTRIES.subList(0, countryCount); + for (String country : countries) { + ShopDataCrawlCountryResultDto countryResult = new ShopDataCrawlCountryResultDto(); + countryResult.setCountry(country); + countryResult.setItems(new ArrayList<>()); + countryResults.add(countryResult); + } + for (int i = 0; i < rowCount; i++) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate("2026-07-25"); + row.setAsin("B0" + String.format("%08d", i)); + row.setBrand("Brand"); + row.setCommodityImage(OK_URL + "?i=" + i); + row.setInventorySales("11"); + row.setSalesRank("22"); + row.setPageViews("33"); + row.setUnitsSold("44"); + row.setPrice("12.50"); + row.setRecommendedOffer("12.00"); + countryResults.get(i % countryCount).getItems().add(row); + } + item.setCountryResults(countryResults); + items.add(item); + return items; + } + + private static List sheetNames(XSSFWorkbook workbook) { + List names = new ArrayList<>(); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + names.add(workbook.getSheetAt(i).getSheetName()); + } + return names; + } + + private static byte[] jpegBytes() throws Exception { + BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ImageIO.write(image, "jpg", output); + return output.toByteArray(); + } +}