task-25: 店铺结果 workbook 的 SXSSF 流式写入路径
评估结论：SXSSF 无法访问/删除模板行，与现有模板重写流程不兼容， 故模板路径保留 XSSFWorkbook，另提供 writeWorkbookStreaming —— 空构造 SXSSFWorkbook + 自建 5 国工作表与表头，行数据按 rowAccessWindow spill 到磁盘， 写入后 dispose() 释放 spill 文件；图片嵌入复用现有链路，失败行兜底 URL 文本。 全量测试 548 通过。
This commit is contained in:
+301
@@ -0,0 +1,301 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import org.apache.poi.openxml4j.util.ZipSecureFile;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 25：店铺结果 workbook 的 SXSSF 流式写入路径。
|
||||
* SXSSFWorkbook 与现有"读模板行样式→清空重写"流程不兼容（模板行不可见、
|
||||
* 无法删除），故评估结论为：模板路径保留 XSSFWorkbook，另提供独立 streaming 写入器
|
||||
* writeWorkbookStreaming —— 空构造 SXSSFWorkbook + 自建 5 个国家工作表与表头，
|
||||
* 数据行按 rowAccessWindow spill 到磁盘，写入后 dispose() 释放临时 spill 文件。
|
||||
*/
|
||||
class ShopDataCrawlStreamingWorkbookTest {
|
||||
@TempDir Path tempDir;
|
||||
|
||||
private static final String OK_URL = "https://thumb.example/ok.jpg";
|
||||
private static final String FAIL_URL = "https://thumb.example/fail.jpg";
|
||||
|
||||
@BeforeAll
|
||||
static void relaxZipSecurity() {
|
||||
// SXSSF 写出含大量图片的 workbook 内部条目数超过 POI 5.2.5 默认防护阈值，
|
||||
// 测试读回断言时需要放行（生产读回路径不涉及此规模）。
|
||||
ZipSecureFile.setMaxFileCount(2_000_000L);
|
||||
ZipSecureFile.setMinInflateRatio(0.0);
|
||||
}
|
||||
|
||||
private static SimilarAsinImageEmbedder okEmbedder() {
|
||||
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||
when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer(
|
||||
invocation -> new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
|
||||
return imageEmbedder;
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_normal_default_path() throws Exception {
|
||||
// 正常输入：1000 行 5 国 streaming 写入，表头/行数/图片完整。
|
||||
SimilarAsinImageEmbedder imageEmbedder = okEmbedder();
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024);
|
||||
List<ShopDataCrawlResultItemVo> items = items(1000, 5, 0);
|
||||
|
||||
File output = tempDir.resolve("streaming.xlsx").toFile();
|
||||
int written = service.writeWorkbookStreaming(output, items, 100);
|
||||
|
||||
assertEquals(1000, written, "返回实际写入行数");
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS, sheetNames(workbook));
|
||||
for (int i = 0; i < ShopDataCrawlExcelAssemblyService.SHEETS.size(); i++) {
|
||||
assertEquals(200, workbook.getSheetAt(i).getLastRowNum(), "每国 200 行");
|
||||
assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(0),
|
||||
workbook.getSheetAt(i).getRow(0).getCell(0).getStringCellValue());
|
||||
assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(1),
|
||||
workbook.getSheetAt(i).getRow(0).getCell(1).getStringCellValue());
|
||||
}
|
||||
assertEquals(1000, workbook.getAllPictures().size(), "1000 个唯一 URL 各嵌入一张图");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_normal_multiple_items() throws Exception {
|
||||
// 批量场景：两个任务项行合并写入，行数合计正确、顺序稳定。
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024);
|
||||
List<ShopDataCrawlResultItemVo> all = new ArrayList<>(items(600, 5, 0));
|
||||
all.addAll(items(400, 5, 0));
|
||||
|
||||
File output = tempDir.resolve("streaming-multi.xlsx").toFile();
|
||||
int written = service.writeWorkbookStreaming(output, all, 100);
|
||||
assertEquals(1000, written);
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
int total = 0;
|
||||
for (int i = 0; i < 5; i++) {
|
||||
total += workbook.getSheetAt(i).getLastRowNum();
|
||||
}
|
||||
assertEquals(1000, total, "行数合计不丢失");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行：同一输入两次 streaming 写出，行数/图片数一致。
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024);
|
||||
List<ShopDataCrawlResultItemVo> items = items(500, 5, 0);
|
||||
File firstOut = tempDir.resolve("streaming-idem-1.xlsx").toFile();
|
||||
File secondOut = tempDir.resolve("streaming-idem-2.xlsx").toFile();
|
||||
|
||||
int first = service.writeWorkbookStreaming(firstOut, items, 100);
|
||||
int second = service.writeWorkbookStreaming(secondOut, items, 100);
|
||||
assertEquals(first, second, "重复写入行数一致");
|
||||
try (XSSFWorkbook wb1 = new XSSFWorkbook(new FileInputStream(firstOut));
|
||||
XSSFWorkbook wb2 = new XSSFWorkbook(new FileInputStream(secondOut))) {
|
||||
assertEquals(wb1.getAllPictures().size(), wb2.getAllPictures().size(), "图片数一致");
|
||||
assertEquals(wb1.getSheet("英国").getLastRowNum(), wb2.getSheet("英国").getLastRowNum());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_boundary_empty_input() throws Exception {
|
||||
// 空输入：0 行写入只产出表头，无图片，返回 0。
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024);
|
||||
File output = tempDir.resolve("streaming-empty.xlsx").toFile();
|
||||
int written = service.writeWorkbookStreaming(output, List.of(), 100);
|
||||
|
||||
assertEquals(0, written);
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
assertEquals(0, workbook.getSheetAt(i).getLastRowNum(), "只有表头行");
|
||||
}
|
||||
assertEquals(0, workbook.getAllPictures().size());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_boundary_single_item() throws Exception {
|
||||
// 单元素：1 行 1 国写入，不依赖批量路径，图片嵌入。
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024);
|
||||
List<ShopDataCrawlResultItemVo> items = items(1, 1, 0);
|
||||
File output = tempDir.resolve("streaming-single.xlsx").toFile();
|
||||
int written = service.writeWorkbookStreaming(output, items, 100);
|
||||
|
||||
assertEquals(1, written);
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
assertEquals(1, workbook.getSheet("英国").getLastRowNum());
|
||||
assertEquals("B000000000", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue());
|
||||
assertEquals(1, workbook.getAllPictures().size());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_boundary_limit_and_overflow() throws Exception {
|
||||
// 上限/超限：5000 行大文件在 window=100 下 spill 写入，行数完整不丢失；
|
||||
// 图片下载失败的行兜底为 URL 文本，不阻塞写入。
|
||||
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||
when(imageEmbedder.fetchAndResizeForCache(any())).thenAnswer(invocation -> {
|
||||
String url = invocation.getArgument(0);
|
||||
if (url.contains("fail")) {
|
||||
return null;
|
||||
}
|
||||
return new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2);
|
||||
});
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024);
|
||||
|
||||
List<ShopDataCrawlResultItemVo> items = items(5000, 5, 0);
|
||||
for (int i = 0; i < 5000; i += 2) {
|
||||
setImage(items, i, FAIL_URL);
|
||||
}
|
||||
File output = tempDir.resolve("streaming-5000.xlsx").toFile();
|
||||
long start = System.currentTimeMillis();
|
||||
int written = service.writeWorkbookStreaming(output, items, 100);
|
||||
long elapsed = System.currentTimeMillis() - start;
|
||||
|
||||
assertEquals(5000, written, "5000 行 spill 写入不丢失");
|
||||
assertTrue(elapsed < 60_000, "5000 行写入须在预算内完成，实际=" + elapsed + "ms");
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
int total = 0;
|
||||
for (int i = 0; i < 5; i++) {
|
||||
total += workbook.getSheetAt(i).getLastRowNum();
|
||||
}
|
||||
assertEquals(5000, total, "读回行数完整");
|
||||
assertEquals(2500, workbook.getAllPictures().size(), "2500 行成功图片嵌入，失败行兜底");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_invalid_input_rejected() throws Exception {
|
||||
// 非法参数：null 输出/空输出目录/非正 window/null items → 明确异常。
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(okEmbedder(), 64L * 1024 * 1024);
|
||||
List<ShopDataCrawlResultItemVo> items = items(10, 1, 0);
|
||||
File output = tempDir.resolve("streaming-invalid.xlsx").toFile();
|
||||
|
||||
assertThrows(BusinessException.class, () -> service.writeWorkbookStreaming(null, items, 100));
|
||||
assertThrows(BusinessException.class,
|
||||
() -> service.writeWorkbookStreaming(tempDir.resolve("no-dir").resolve("x.xlsx").toFile(), items, 100));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> service.writeWorkbookStreaming(output, items, 0));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> service.writeWorkbookStreaming(output, items, -5));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> service.writeWorkbookStreaming(output, null, 100));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_025_workbook_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败：图片下载抛异常 → 单行兜底 URL 文本不中断整表；
|
||||
// 写入失败（目标不可写）抛 BusinessException，重复调用稳定（spill 文件可重建）。
|
||||
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||
when(imageEmbedder.fetchAndResizeForCache(any())).thenThrow(
|
||||
new RuntimeException("image service down"));
|
||||
ShopDataCrawlExcelAssemblyService service =
|
||||
new ShopDataCrawlExcelAssemblyService(imageEmbedder, 64L * 1024 * 1024);
|
||||
|
||||
List<ShopDataCrawlResultItemVo> items = items(100, 1, 0);
|
||||
File output = tempDir.resolve("streaming-fail.xlsx").toFile();
|
||||
int written = service.writeWorkbookStreaming(output, items, 100);
|
||||
assertEquals(100, written, "图片失败不阻塞行写入");
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
assertEquals(100, workbook.getSheet("英国").getLastRowNum());
|
||||
assertEquals(0, workbook.getAllPictures().size(), "无图片嵌入");
|
||||
assertEquals(OK_URL + "?i=0",
|
||||
workbook.getSheet("英国").getRow(1).getCell(2).getStringCellValue(), "兜底 URL 文本");
|
||||
}
|
||||
|
||||
File locked = tempDir.resolve("locked").toFile();
|
||||
assertTrue(locked.mkdir(), "创建目录占位模拟不可写目标");
|
||||
assertThrows(BusinessException.class,
|
||||
() -> service.writeWorkbookStreaming(locked, items, 100));
|
||||
|
||||
File retried = tempDir.resolve("streaming-retry.xlsx").toFile();
|
||||
assertEquals(100, service.writeWorkbookStreaming(retried, items, 100), "失败后重试成功，spill 可重建");
|
||||
}
|
||||
|
||||
private static void setImage(List<ShopDataCrawlResultItemVo> items, int index, String url) {
|
||||
int i = 0;
|
||||
for (ShopDataCrawlResultItemVo item : items) {
|
||||
for (ShopDataCrawlCountryResultDto country : item.getCountryResults()) {
|
||||
for (ShopDataCrawlRowDto row : country.getItems()) {
|
||||
if (i == index) {
|
||||
row.setCommodityImage(url);
|
||||
return;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<ShopDataCrawlResultItemVo> items(int rowCount, int countryCount, int failEvery) {
|
||||
List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
|
||||
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
|
||||
item.setSuccess(true);
|
||||
List<ShopDataCrawlCountryResultDto> countryResults = new ArrayList<>();
|
||||
List<String> countries = ShopDataCrawlExcelAssemblyService.COUNTRIES.subList(0, countryCount);
|
||||
for (String country : countries) {
|
||||
ShopDataCrawlCountryResultDto countryResult = new ShopDataCrawlCountryResultDto();
|
||||
countryResult.setCountry(country);
|
||||
countryResult.setItems(new ArrayList<>());
|
||||
countryResults.add(countryResult);
|
||||
}
|
||||
for (int i = 0; i < rowCount; i++) {
|
||||
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||
row.setDate("2026-07-25");
|
||||
row.setAsin("B0" + String.format("%08d", i));
|
||||
row.setBrand("Brand");
|
||||
row.setCommodityImage(OK_URL + "?i=" + i);
|
||||
row.setInventorySales("11");
|
||||
row.setSalesRank("22");
|
||||
row.setPageViews("33");
|
||||
row.setUnitsSold("44");
|
||||
row.setPrice("12.50");
|
||||
row.setRecommendedOffer("12.00");
|
||||
countryResults.get(i % countryCount).getItems().add(row);
|
||||
}
|
||||
item.setCountryResults(countryResults);
|
||||
items.add(item);
|
||||
return items;
|
||||
}
|
||||
|
||||
private static List<String> sheetNames(XSSFWorkbook workbook) {
|
||||
List<String> names = new ArrayList<>();
|
||||
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
|
||||
names.add(workbook.getSheetAt(i).getSheetName());
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
private static byte[] jpegBytes() throws Exception {
|
||||
BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB);
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", output);
|
||||
return output.toByteArray();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user