task-52: 生成结果文件按 chunk 一次读取

CollectDataResultDetailReader 从 biz_task_result_item 解析行：引用格式按 pointer 缓存整 chunk 明细（同一 chunk 对象只 resolve 一次），再按 offset 取行；旧格式逐行兜底。CollectDataService.loadFinalRows 接入 reader，移除内联逐行读取循环与 detailCache。
2026-08-30 16:00:32 +08:00
parent 7f46b9f768
commit 48284be214
3 changed files with 300 additions and 39 deletions
@@ -37,6 +37,7 @@ import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
+import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader;
 import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
 import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
 import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
@@ -149,6 +150,9 @@ public class CollectDataService {
    /** 结果明细批量 upsert 器：按唯一键 uk_task_scope_item 批量写入，替代逐行 select/insert/update。 */
    private final CollectDataResultItemBatchWriter resultItemBatchWriter;

+    /** 结果明细 chunk 级读取器：生成结果文件时按 chunk 一次读取，替代逐行对象读取。 */
+    private final CollectDataResultDetailReader resultDetailReader;
+
    @Value("${aiimage.collect-data.stale-timeout-minutes:30}")
    private long staleTimeoutMinutes;

@@ -891,47 +895,16 @@ public class CollectDataService {
                .eq(TaskResultItemEntity::getTaskId, taskId)
                .eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
                .orderByAsc(TaskResultItemEntity::getId));
-        List<CollectDataResultRowVo> out = new ArrayList<>();
-        if (rows == null) {
-            return out;
+        if (rows == null || rows.isEmpty()) {
+            return new ArrayList<>();
        }
-        TypeReference<List<CollectDataResultRowVo>> listType = new TypeReference<>() {
-        };
-        // chunk 级引用：同一 chunk 对象只解析一次，按 offset 取行。
-        Map<String, List<CollectDataResultRowVo>> detailCache = new HashMap<>();
-        for (TaskResultItemEntity row : rows) {
-            try {
-                CollectDataResultDetailCodec.ChunkRef ref = resultDetailCodec.parseRef(row.getPayloadJson());
-                if (ref != null) {
-                    List<CollectDataResultRowVo> details = detailCache.get(ref.pointer());
-                    if (details == null) {
-                        String detailJson = transientPayloadStorageService.resolvePayload(
-                                ref.pointer(), "read collect data result detail failed");
-                        if (detailJson != null && !detailJson.isBlank()) {
-                            details = objectMapper.readValue(detailJson, listType);
-                        }
-                        detailCache.put(ref.pointer(), details);
-                    }
-                    if (details != null && ref.offset() >= 0 && ref.offset() < details.size()) {
-                        CollectDataResultRowVo value = details.get(ref.offset());
-                        if (value != null) {
-                            out.add(value);
-                        }
-                    }
-                    continue;
-                }
-                // 旧格式：payload_json 直接是行 JSON。
-                String payloadJson = transientPayloadStorageService.resolvePayload(
-                        row.getPayloadJson(), "read collect data result item failed");
-                CollectDataResultRowVo value = objectMapper.readValue(payloadJson, CollectDataResultRowVo.class);
-                if (value != null) {
-                    out.add(value);
-                }
-            } catch (Exception ex) {
-                throw new BusinessException("读取采集结果明细失败");
-            }
+        try {
+            // 按 chunk 一次读取：同一 chunk 对象只 resolve 一次，按 offset 取行，
+            // 替代逐行对象读取（旧格式逐行兜底）。
+            return resultDetailReader.readRows(rows);
+        } catch (Exception ex) {
+            throw new BusinessException("读取采集结果明细失败", ex);
        }
-        return out;
    }

    /**
@@ -0,0 +1,86 @@
+package com.nanri.aiimage.modules.collectdata.util;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
+import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
+import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 结果明细 chunk 级读取器：生成结果文件时按 chunk 一次读取整对象，
+ * 同一 chunk 的明细只 resolve/解析一次，再按 offset 取行，替代逐行
+ * 对象读取。旧格式（payload_json 直接是行 JSON）逐行兜底；空输入返回
+ * 空列表，越界/损坏行安全跳过，读取失败抛可识别异常。
+ */
+@Component
+public class CollectDataResultDetailReader {
+
+    private static final String ERROR_DETAIL = "read collect data result detail failed";
+    private static final String ERROR_ITEM = "read collect data result item failed";
+
+    private final CollectDataResultDetailCodec resultDetailCodec;
+    private final ObjectMapper objectMapper;
+    private final TransientPayloadStorageService transientPayloadStorageService;
+    private final TypeReference<List<CollectDataResultRowVo>> listType = new TypeReference<>() {
+    };
+
+    public CollectDataResultDetailReader(CollectDataResultDetailCodec resultDetailCodec,
+                                         ObjectMapper objectMapper,
+                                         TransientPayloadStorageService transientPayloadStorageService) {
+        this.resultDetailCodec = resultDetailCodec;
+        this.objectMapper = objectMapper;
+        this.transientPayloadStorageService = transientPayloadStorageService;
+    }
+
+    public List<CollectDataResultRowVo> readRows(List<TaskResultItemEntity> items) {
+        List<CollectDataResultRowVo> out = new ArrayList<>();
+        if (items == null || items.isEmpty()) {
+            return out;
+        }
+        // chunk 级引用：同一 chunk 对象只 resolve + 解析一次，按 offset 取行。
+        Map<String, List<CollectDataResultRowVo>> detailCache = new HashMap<>();
+        for (TaskResultItemEntity item : items) {
+            if (item == null || item.getPayloadJson() == null || item.getPayloadJson().isBlank()) {
+                continue;
+            }
+            try {
+                CollectDataResultDetailCodec.ChunkRef ref = resultDetailCodec.parseRef(item.getPayloadJson());
+                if (ref != null) {
+                    List<CollectDataResultRowVo> details = detailCache.get(ref.pointer());
+                    if (details == null) {
+                        String detailJson = transientPayloadStorageService.resolvePayload(ref.pointer(), ERROR_DETAIL);
+                        details = (detailJson == null || detailJson.isBlank())
+                                ? null
+                                : objectMapper.readValue(detailJson, listType);
+                        detailCache.put(ref.pointer(), details);
+                    }
+                    if (details != null && ref.offset() >= 0 && ref.offset() < details.size()) {
+                        CollectDataResultRowVo value = details.get(ref.offset());
+                        if (value != null) {
+                            out.add(value);
+                        }
+                    }
+                    continue;
+                }
+                // 旧格式：payload_json 直接是行 JSON；内容缺失时安全跳过。
+                String payloadJson = transientPayloadStorageService.resolvePayload(item.getPayloadJson(), ERROR_ITEM);
+                if (payloadJson == null || payloadJson.isBlank()) {
+                    continue;
+                }
+                CollectDataResultRowVo value = objectMapper.readValue(payloadJson, CollectDataResultRowVo.class);
+                if (value != null) {
+                    out.add(value);
+                }
+            } catch (Exception ex) {
+                throw new IllegalStateException("读取采集结果明细失败", ex);
+            }
+        }
+        return out;
+    }
+}