task-56: 采集结果 Excel 改为临时文件+原子落位，流式写入或落位失败时清理临时文件，目标路径不残留半成品

2026-08-30 17:05:56 +08:00
parent 15269c6ed9
commit adcfa28af3
2 changed files with 291 additions and 2 deletions
@@ -11,6 +11,9 @@ import org.springframework.stereotype.Service;

 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -51,14 +54,25 @@ public class CollectDataExcelAssemblyService {
                                       List<CollectDataResultRowVo> items,
                                       List<CollectDataSummaryRowDto> summaries,
                                       Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
+        if (outputXlsx == null) {
+            throw new BusinessException("生成采集数据 Excel 失败: 输出文件路径为空");
+        }
        SXSSFWorkbook workbook = new SXSSFWorkbook(200);
        workbook.setCompressTempFiles(true);
-        try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
+        // 先写目标同目录的临时文件，成功后原子落位：写入或落位失败时目标路径
+        // 不残留半成品，临时文件由 catch/finally 清理（SXSSF 滚动窗口文件由 dispose 释放）。
+        File tmpFile = new File(outputXlsx.getAbsolutePath() + ".tmp");
+        try {
            writeDetailSheet(workbook, items);
            writeSummarySheet(workbook, summaries, rawItemsSupplier);
-            workbook.write(outputStream);
+            try (FileOutputStream outputStream = new FileOutputStream(tmpFile)) {
+                workbook.write(outputStream);
+            }
+            Files.move(tmpFile.toPath(), outputXlsx.toPath(),
+                    StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
        } catch (Exception ex) {
            log.warn("[collect-data] write workbook failed: {}", ex.getMessage());
+            deleteQuietly(tmpFile);
            throw new BusinessException("生成采集数据 Excel 失败: " + ex.getMessage());
        } finally {
            try {
@@ -66,6 +80,15 @@ public class CollectDataExcelAssemblyService {
            } catch (Exception ignored) {
            }
            workbook.dispose();
+            deleteQuietly(tmpFile);
+        }
+    }
+
+    private void deleteQuietly(File file) {
+        try {
+            Files.deleteIfExists(file.toPath());
+        } catch (IOException ex) {
+            log.warn("[collect-data] delete workbook temp file failed: {}", ex.getMessage());
        }
    }

@@ -0,0 +1,266 @@
+package com.nanri.aiimage.modules.collectdata.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.nanri.aiimage.common.exception.BusinessException;
+import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
+import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.ss.usermodel.WorkbookFactory;
+import org.apache.poi.util.TempFile;
+import org.apache.poi.util.TempFileCreationStrategy;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Task 56：采集结果文件流式写入失败后的临时文件清理。
+ * 实现改为「临时文件 + 原子落位」：先写目标同目录的 <name>.tmp，写入或落位
+ * 失败时删除临时文件（目标路径不残留半成品、不被污染），成功后 move 到目标；
+ * SXSSF 滚动窗口临时文件由 dispose 清理，失败路径同样释放。
+ */
+class CollectDataExcelCleanupTest {
+
+    @TempDir
+    Path tempDir;
+
+    private final CollectDataExcelAssemblyService assembly = new CollectDataExcelAssemblyService();
+
+    private final List<File> trackedTempFiles = new ArrayList<>();
+
+    @AfterEach
+
+    private void trackSxssfTempFiles() {
+        TempFile.setTempFileCreationStrategy(new TempFileCreationStrategy() {
+            @Override
+            public File createTempFile(String prefix, String suffix) throws IOException {
+                File file = File.createTempFile(prefix, suffix);
+                trackedTempFiles.add(file);
+                return file;
+            }
+
+            @Override
+            public File createTempDirectory(String prefix) throws IOException {
+                File dir = Files.createTempDirectory(prefix).toFile();
+                trackedTempFiles.add(dir);
+                return dir;
+            }
+        });
+    }
+
+    private void assertSxssfTempFilesCleaned() {
+        assertThat(trackedTempFiles).as("SXSSF 滚动窗口临时文件均已删除")
+                .allSatisfy(f -> assertThat(f).doesNotExist());
+        // POI 5.2.5 无法读取当前策略，改用系统临时目录中 SXSSF 固定前缀的残留探测。
+        try (Stream<Path> paths = Files.list(Path.of(System.getProperty("java.io.tmpdir")))) {
+            assertThat(paths.filter(p -> p.getFileName().toString().startsWith("poi-sxssf")))
+                    .as("系统临时目录无 SXSSF 滚动窗口残留")
+                    .isEmpty();
+        } catch (IOException ex) {
+            throw new IllegalStateException(ex);
+        }
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_normal_default_path() throws Exception {
+        // 正常路径：写入成功后目标文件内容正确，同目录无 .tmp 残留，
+        // SXSSF 滚动窗口临时文件全部清理。
+        trackSxssfTempFiles();
+        File output = tempDir.resolve("out.xlsx").toFile();
+        List<CollectDataSummaryRowDto> summaries = List.of(summary("kw", 2, 1, 0, 0, 3, 1));
+
+        assembly.writeWorkbookSegmented(output,
+                List.of(row("Nike", "B000000001", "kw")),
+                summaries, () -> List.of());
+
+        assertThat(output).exists();
+        assertThat(tempDir.resolve("out.xlsx.tmp")).doesNotExist() .as("成功后临时文件已落位");
+        try (Workbook wb = WorkbookFactory.create(output)) {
+            assertThat(wb.getSheet("采集数据结果").getLastRowNum()).isEqualTo(1);
+            assertThat(wb.getSheet("结果文件").getLastRowNum()).isEqualTo(1);
+            assertThat(wb.getSheet("结果文件").getRow(1).getCell(1).getNumericCellValue()).isEqualTo(2);
+        }
+        assertSxssfTempFilesCleaned();
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_normal_multiple_items() throws Exception {
+        // 批量场景：2000 行触发 SXSSF 滚动窗口刷临时文件，结果不丢失、顺序稳定，
+        // 成功后 .tmp 与 SXSSF 临时文件均无残留。
+        trackSxssfTempFiles();
+        File output = tempDir.resolve("multi.xlsx").toFile();
+        List<CollectDataResultRowVo> items = new ArrayList<>();
+        for (int i = 0; i < 2000; i++) {
+            items.add(row("brand" + (i % 10), "B" + String.format("%09d", i + 1), "kw" + (i % 50)));
+        }
+
+        assembly.writeWorkbookSegmented(output, items, List.of(), () -> items);
+
+        assertThat(output).exists();
+        assertThat(tempDir.resolve("multi.xlsx.tmp")).doesNotExist();
+        try (Workbook wb = WorkbookFactory.create(output)) {
+            Sheet detail = wb.getSheet("采集数据结果");
+            assertThat(detail.getLastRowNum()).isEqualTo(2000) .as("2000 行不丢失");
+            assertThat(detail.getRow(1).getCell(1).getStringCellValue()).isEqualTo("B000000001");
+            assertThat(detail.getRow(1000).getCell(1).getStringCellValue()).isEqualTo("B000001000") .as("顺序稳定");
+            assertThat(detail.getRow(2000).getCell(1).getStringCellValue()).isEqualTo("B000002000");
+            assertThat(wb.getSheet("结果文件").getLastRowNum()).isEqualTo(50) .as("50 关键词聚合");
+        }
+        assertSxssfTempFilesCleaned();
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_normal_repeated_operation_is_idempotent() throws Exception {
+        // 幂等：同一目标文件重复生成两次，均成功且内容一致（第二次覆盖落位），
+        // 无 .tmp 与 SXSSF 临时文件残留。
+        trackSxssfTempFiles();
+        File output = tempDir.resolve("idem.xlsx").toFile();
+        List<CollectDataResultRowVo> items = List.of(
+                row("Nike", "B000000001", "kw"),
+                row("Zara", "B000000002", "kw"));
+
+        assembly.writeWorkbookSegmented(output, items, List.of(), () -> items);
+        assembly.writeWorkbookSegmented(output, items, List.of(), () -> items);
+
+        assertThat(output).exists();
+        assertThat(tempDir.resolve("idem.xlsx.tmp")).doesNotExist();
+        try (Workbook wb = WorkbookFactory.create(output)) {
+            assertThat(wb.getSheet("采集数据结果").getLastRowNum()).isEqualTo(2);
+            assertThat(wb.getSheet("采集数据结果").getRow(2).getCell(0).getStringCellValue()).isEqualTo("Zara");
+            assertThat(wb.getSheet("结果文件").getLastRowNum()).isEqualTo(1);
+        }
+        assertSxssfTempFilesCleaned();
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_boundary_empty_input() throws Exception {
+        // 空输入：无明细无聚合 → 仅表头成功生成，不创建无效资源，无临时文件残留。
+        trackSxssfTempFiles();
+        File output = tempDir.resolve("empty.xlsx").toFile();
+
+        assembly.writeWorkbookSegmented(output, List.of(), List.of(), () -> List.of());
+
+        assertThat(output).exists();
+        assertThat(tempDir.resolve("empty.xlsx.tmp")).doesNotExist();
+        try (Workbook wb = WorkbookFactory.create(output)) {
+            assertThat(wb.getSheet("采集数据结果").getLastRowNum()).isEqualTo(0) .as("仅表头");
+            assertThat(wb.getSheet("结果文件").getLastRowNum()).isEqualTo(0);
+        }
+        assertSxssfTempFilesCleaned();
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_boundary_single_item() throws Exception {
+        // 单元素：单行单关键词不依赖批量路径，结果正确且无临时文件残留。
+        trackSxssfTempFiles();
+        File output = tempDir.resolve("single.xlsx").toFile();
+
+        assembly.writeWorkbookSegmented(output,
+                List.of(row("solo", "B000000001", "kw")), List.of(),
+                () -> List.of(row("solo", "B000000001", "kw", "FBA", 2)));
+
+        assertThat(output).exists();
+        assertThat(tempDir.resolve("single.xlsx.tmp")).doesNotExist();
+        try (Workbook wb = WorkbookFactory.create(output)) {
+            assertThat(wb.getSheet("采集数据结果").getLastRowNum()).isEqualTo(1);
+            assertThat(wb.getSheet("结果文件").getLastRowNum()).isEqualTo(1);
+            assertThat(wb.getSheet("结果文件").getRow(1).getCell(1).getNumericCellValue()).isEqualTo(1) .as("FBA=1");
+        }
+        assertSxssfTempFilesCleaned();
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_boundary_limit_and_overflow() throws Exception {
+        // 上限/超限：5000 行触发滚动刷盘后落位到非法目标（目录）被拒绝，
+        // 临时文件全部清理，目标路径不受影响，无无界残留。
+        trackSxssfTempFiles();
+        File targetDir = tempDir.resolve("out-dir").toFile();
+        assertThat(targetDir.mkdir()).isTrue();
+        List<CollectDataResultRowVo> items = new ArrayList<>();
+        for (int i = 0; i < 5000; i++) {
+            items.add(row("brand" + (i % 10), "B" + String.format("%09d", i + 1), "kw" + (i % 50)));
+        }
+
+        assertThatThrownBy(() -> assembly.writeWorkbookSegmented(targetDir, items, List.of(), () -> items))
+                .isInstanceOf(BusinessException.class)
+                .hasMessageContaining("生成采集数据 Excel 失败");
+
+        assertThat(targetDir).isDirectory() .as("目标路径不受影响");
+        assertThat(tempDir.resolve("out-dir.tmp")).doesNotExist() .as("失败后临时文件已清理");
+        assertSxssfTempFilesCleaned();
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_invalid_input_rejected() throws Exception {
+        // 非法参数：null 输出文件 → 项目约定异常 + 可识别错误消息，不产生临时文件。
+        assertThatThrownBy(() -> assembly.writeWorkbookSegmented(null, List.of(), List.of(), () -> List.of()))
+                .isInstanceOf(BusinessException.class)
+                .hasMessageContaining("输出文件路径为空");
+    }
+
+    @Test
+    void test_task_056_collect_cleanup_dependency_failure_releases_resources() throws Exception {
+        // 依赖失败：supplier 抛错 → 异常可识别、目标文件不被半成品污染、
+        // .tmp 与 SXSSF 临时文件全部清理；恢复后同一目标重新生成成功。
+        trackSxssfTempFiles();
+        File output = tempDir.resolve("dep.xlsx").toFile();
+
+        assertThatThrownBy(() -> assembly.writeWorkbookSegmented(output, List.of(), List.of(),
+                () -> {
+                    throw new IllegalStateException("raw load down");
+                }))
+                .isInstanceOf(BusinessException.class)
+                .hasMessageContaining("生成采集数据 Excel 失败");
+
+        assertThat(output).doesNotExist() .as("目标文件未被半成品污染");
+        assertThat(tempDir.resolve("dep.xlsx.tmp")).doesNotExist() .as("临时文件已清理");
+        assertSxssfTempFilesCleaned();
+
+        assembly.writeWorkbookSegmented(output,
+                List.of(row("Nike", "B000000001", "kw")), List.of(),
+                () -> List.of(row("Nike", "B000000001", "kw", "FBA", 1)));
+        assertThat(output).exists() .as("恢复后重新生成成功");
+        assertThat(tempDir.resolve("dep.xlsx.tmp")).doesNotExist();
+        try (Workbook wb = WorkbookFactory.create(output)) {
+            assertThat(wb.getSheet("采集数据结果").getLastRowNum()).isEqualTo(1);
+        }
+    }
+
+    private static CollectDataResultRowVo row(String brand, String asin, String keyword) {
+        return row(brand, asin, keyword, "", 0);
+    }
+
+    private static CollectDataResultRowVo row(String brand, String asin, String keyword,
+                                              String deliveryMethod, int page) {
+        CollectDataResultRowVo row = new CollectDataResultRowVo();
+        row.setBrand(brand);
+        row.setAsin(asin);
+        row.setKeyword(keyword);
+        row.setDeliveryMethod(deliveryMethod);
+        row.setPage(page);
+        return row;
+    }
+
+    private static CollectDataSummaryRowDto summary(String keyword, Integer fba, Integer fbm, Integer amz,
+                                                    Integer noneCount, Integer totalPage, Integer asinFilter) {
+        try {
+            return new ObjectMapper().readValue("{\"keyword\":\"" + keyword + "\",\"fba\":" + fba
+                    + ",\"fbm\":" + fbm + ",\"amz\":" + amz + ",\"noneCount\":" + noneCount
+                    + ",\"totalPage\":" + totalPage + ",\"asinFilter\":" + asinFilter + "}",
+                    CollectDataSummaryRowDto.class);
+        } catch (Exception ex) {
+            throw new IllegalStateException(ex);
+        }
+    }
+}