task-28: 以 scope 计数器替代每个 chunk 的 COUNT(*) 完整统计

分片接收不再对 chunk 表执行 selectCount 全量统计：新插入分片按 scope 计数器 +1 并钳制到 chunk_total，重复提交（唯一键冲突）保持不变；分片行与 scope 计数器作为一致性单元，状态写入失败时回滚本次插入的分片行与 payload，重试走全新插入，避免计数器重复计数或永久欠计。任务级分布式锁串行化同一任务的接收，读改写在锁内完成。
2026-08-29 19:25:59 +08:00
parent 7f49ef596e
commit f1db31f813
2 changed files with 645 additions and 8 deletions
@@ -1280,8 +1280,12 @@ public class ShopDataCrawlTaskService {
        chunk.setPayloadHash(payloadHash);
        chunk.setCreatedAt(now);
        chunk.setUpdatedAt(now);
+        int receivedChunkCount;
+        boolean insertedThisCall = false;
        try {
            taskChunkMapper.insert(chunk);
+            insertedThisCall = true;
+            receivedChunkCount = nextScopeChunkCount(scope, chunkTotal);
        } catch (DuplicateKeyException ex) {
            transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
            TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
@@ -1289,13 +1293,27 @@ public class ShopDataCrawlTaskService {
                throw new BusinessException("店铺数据抓取结果分片并发写入失败，请重试");
            }
            validateExistingChunk(winner, chunkTotal, payloadHash);
+            receivedChunkCount = scopeChunkCount(scope);
        } catch (RuntimeException ex) {
            transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
            throw ex;
        }

-        int receivedChunkCount = countResultChunks(taskId, scopeHash);
-        persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
+        try {
+            persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
+        } catch (RuntimeException ex) {
+            // 分片行与 scope 计数器是一致性单元：状态写入失败则回滚本次插入的分片行与 payload，
+            // 客户端重试会走全新插入路径，计数器不会因重试重复计数或永久欠计。
+            if (insertedThisCall) {
+                taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
+                        .eq(TaskChunkEntity::getTaskId, taskId)
+                        .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
+                        .eq(TaskChunkEntity::getScopeHash, scopeHash)
+                        .eq(TaskChunkEntity::getChunkIndex, chunkIndex));
+                transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
+            }
+            throw ex;
+        }
        log.info("[shop-data-crawl] result chunk received taskId={} shop={} chunk={}/{} received={}",
                taskId, shopKey, chunkIndex, chunkTotal, receivedChunkCount);
        return new ResultChunkReceipt(scopeHash, chunkTotal, receivedChunkCount >= chunkTotal);
@@ -1343,12 +1361,23 @@ public class ShopDataCrawlTaskService {
        }
    }

-    private int countResultChunks(Long taskId, String scopeHash) {
-        Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
-                .eq(TaskChunkEntity::getTaskId, taskId)
-                .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
-                .eq(TaskChunkEntity::getScopeHash, scopeHash));
-        return count == null ? 0 : count.intValue();
+    /**
+     * 以 scope 计数器替代 chunk 表全量 COUNT(*)：新分片插入成功后 +1 并钳制到 chunk_total。
+     * 任务级分布式锁串行化同一任务的接收，读改写在单锁内完成，无需再统计 chunk 行数。
+     */
+    private int nextScopeChunkCount(TaskScopeStateEntity scope, int chunkTotal) {
+        int previous = scope == null ? 0 : Math.max(0, scope.getReceivedChunkCount() == null ? 0 : scope.getReceivedChunkCount());
+        return Math.min(chunkTotal, previous + 1);
+    }
+
+    /** 重复提交（唯一键冲突，分片已计过数）时计数器保持不变，钳制到 chunk_total。 */
+    private int scopeChunkCount(TaskScopeStateEntity scope) {
+        if (scope == null || scope.getReceivedChunkCount() == null) {
+            return 0;
+        }
+        int count = Math.max(0, scope.getReceivedChunkCount());
+        Integer chunkTotal = scope.getChunkTotal();
+        return chunkTotal != null && chunkTotal > 0 ? Math.min(chunkTotal, count) : count;
    }

    private void persistResultScope(Long taskId,