super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-53: 采集结果 Excel 生成按 rawRows/finalRows 分段生命周期，rawRows 惰性加载避免同时驻留内存

Browse Source
This commit is contained in:
黄自达
2026-08-30 16:29:48 +08:00
parent 9ebb1a4012
commit ae95b294ec
3 changed files with 285 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java
+18 -2
View File
@@ -15,6 +15,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Supplier;
@Service
@Slf4j
@@ -38,11 +39,23 @@ public class CollectDataExcelAssemblyService {
List<CollectDataResultRowVo> items,
List<CollectDataSummaryRowDto> summaries,
List<CollectDataResultRowVo> rawItems) {
writeWorkbookSegmented(outputXlsx, items, summaries, () -> rawItems);
}
/**
* 分段生命周期版：rawItems 惰性加载，仅在 summaries 为空（fallback 自聚合
* 窗口）时才调用 supplier 一次性取全量原始行，用完即弃；summaries 非空时
* supplier 不被调用，rawRows 与 finalRows 不同时长期驻留内存。
*/
public void writeWorkbookSegmented(File outputXlsx,
List<CollectDataResultRowVo> items,
List<CollectDataSummaryRowDto> summaries,
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
writeDetailSheet(workbook, items);
writeSummarySheet(workbook, summaries, rawItems);
writeSummarySheet(workbook, summaries, rawItemsSupplier);
workbook.write(outputStream);
} catch (Exception ex) {
log.warn("[collect-data] write workbook failed: {}", ex.getMessage());
@@ -88,7 +101,7 @@ public class CollectDataExcelAssemblyService {
*/
private void writeSummarySheet(SXSSFWorkbook workbook,
List<CollectDataSummaryRowDto> summaries,
List<CollectDataResultRowVo> rawItems) {
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
Sheet sheet = workbook.createSheet(SHEET_SUMMARY_NAME);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < SHEET_SUMMARY_HEADER.length; i++) {
@@ -121,6 +134,9 @@ public class CollectDataExcelAssemblyService {
}
// Fallback 分支：基于 rawItems 自聚合（Python 端未接入时使用）。
// 仅在需要时才触发 supplier 一次性加载全量原始行，聚合完即弃，
// 与 finalRows（items）不同时长期驻留内存。
List<CollectDataResultRowVo> rawItems = rawItemsSupplier.get();
Map<String, KeywordSummary> grouped = new LinkedHashMap<>();
if (rawItems != null) {
for (CollectDataResultRowVo item : rawItems) {
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
+3 -3
View File
@@ -858,13 +858,13 @@ public class CollectDataService {
CollectDataStats stats = loadStats(task);
List<CollectDataResultRowVo> rows = loadFinalRows(task.getId());
// Sheet「结果文件」按需求基于 Python 回传的全量数据聚合，不经后端 ASIN/品牌过滤丢弃，
// 因此从 biz_task_chunk 反序列化全部原始行。
List<CollectDataResultRowVo> rawRows = loadRawRows(task.getId());
// 因此从 biz_task_chunk 反序列化全部原始行；rawRows 惰性加载，summaries 非空时
// 不加载（rawRows 与 finalRows 不同时长期驻留内存）。
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "collect-data-result", String.valueOf(task.getId())));
String filename = buildResultFilename(task, result);
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, rows, stats.summaries, rawRows);
excelAssemblyService.writeWorkbookSegmented(xlsx, rows, stats.summaries, () -> loadRawRows(task.getId()));
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
result.setResultFilename(filename);
result.setResultFileUrl(objectKey);