task-31: 将任务快照改为轻量进度字段，避免每次写入完整结果 JSON

2026-08-29 20:04:22 +08:00
parent 76a51ac290
commit cb4e10c87f
2 changed files with 616 additions and 4 deletions
@@ -494,8 +494,7 @@ public class ShopDataCrawlTaskService {
            }
        }

-        persistSnapshotJson(task, snapshots);
-        fileTaskMapper.updateById(task);
+        persistProgressOrSnapshot(task, snapshots, resultRows.stream().allMatch(this::isResultFinished));
        taskCacheService.saveTaskCache(task);
        tryFinalizeTask(taskId, false);
        }
@@ -599,13 +598,14 @@ public class ShopDataCrawlTaskService {
        List<FileResultEntity> latestRows = listTaskRows(taskId);
        updateTaskStatusFromRows(task, latestRows);
        if (latestRows.stream().allMatch(this::isResultFinished)) {
-            persistSnapshotJson(task, snapshots);
+            if (changed || !snapshotJsonHasCountryRows(task)) {
+                persistSnapshotJson(task, snapshots);
+            }
            fileTaskMapper.updateById(task);
            finalizeTaskWorkbook(task, latestRows, snapshots);
            return true;
        }

-        persistSnapshotJson(task, snapshots);
        fileTaskMapper.updateById(task);
        taskCacheService.saveTaskCache(task);
        return changed;
@@ -2332,6 +2332,20 @@ public class ShopDataCrawlTaskService {
        }
    }

+    private boolean snapshotJsonHasCountryRows(FileTaskEntity task) {
+        if (task == null) {
+            return false;
+        }
+        for (ShopDataCrawlResultItemVo snapshot : parseTaskSnapshots(task.getResultJson())) {
+            if (snapshot != null && snapshot.getCountryResults() != null
+                    && snapshot.getCountryResults().stream().anyMatch(country ->
+                    country != null && country.getItems() != null && !country.getItems().isEmpty())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
    private void persistSnapshotJson(FileTaskEntity task, List<ShopDataCrawlResultItemVo> snapshots) {
        try {
            task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
@@ -2341,6 +2355,17 @@ public class ShopDataCrawlTaskService {
        }
    }

+    /**
+     * RUNNING 期间只写轻量进度字段（successFileCount/failedFileCount/status/updatedAt），
+     * 避免每次分片接收都序列化并落库完整结果 JSON；仅终态（全部结果行完成）才写完整快照。
+     */
+    private void persistProgressOrSnapshot(FileTaskEntity task, List<ShopDataCrawlResultItemVo> snapshots, boolean terminal) {
+        if (terminal) {
+            persistSnapshotJson(task, snapshots);
+        }
+        fileTaskMapper.updateById(task);
+    }
+
    private void syncSnapshotTables(FileTaskEntity task, List<ShopDataCrawlResultItemVo> snapshots) {
        List<ShopDataCrawlResultItemVo> safe = snapshots == null ? List.of() : snapshots;
        taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {