task-27: chunk 接收改为原子插入/幂等 upsert，去掉先查后插

persistResultChunk 去除提交前的 findResultChunk 预查，直接 store payload + insert，唯一索引 uk_task_scope_chunk 兜底幂等：重复提交命中 DuplicateKeyException 后重查 winner 校验并清理本次重存 payload；空分片（无可处理数据）在落库前拒绝，不创建 RustFS payload 与分片行。
2026-08-29 19:11:10 +08:00
parent 0092092dbd
commit 23eddde770
3 changed files with 644 additions and 8 deletions
@@ -1257,15 +1257,11 @@ public class ShopDataCrawlTaskService {
        validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);

        List<ShopDataCrawlCountryResultDto> countryResults = copyCountryResults(incoming.getCountryResults());
+        if (!hasProcessableChunkData(incoming.getCountryResults())) {
+            throw new BusinessException("店铺数据抓取结果分片内容为空，拒绝接收");
+        }
        String payloadJson = writeJson(countryResults, "序列化店铺数据抓取结果分片失败");
        String payloadHash = DigestUtil.sha256Hex(payloadJson);
-        TaskChunkEntity existing = findResultChunk(taskId, scopeHash, chunkIndex);
-        if (existing != null) {
-            validateExistingChunk(existing, chunkTotal, payloadHash);
-            int receivedChunkCount = countResultChunks(taskId, scopeHash);
-            persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
-            return new ResultChunkReceipt(scopeHash, chunkTotal, receivedChunkCount >= chunkTotal);
-        }

        ensureRustfsPayloadStorageEnabled();
        String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
@@ -2349,6 +2345,30 @@ public class ShopDataCrawlTaskService {
        return copy;
    }

+    /**
+     * 分片是否含可处理数据：与 copyCountryResults 的拷贝语义一致 ——
+     * 任一国家含至少一个非空行（国家名非空白且行内容非空）即视为有数据。
+     */
+    private boolean hasProcessableChunkData(List<ShopDataCrawlCountryResultDto> results) {
+        if (results == null || results.isEmpty()) {
+            return false;
+        }
+        for (ShopDataCrawlCountryResultDto source : results) {
+            if (source == null || blank(source.getCountry())) {
+                continue;
+            }
+            if (source.getItems() == null) {
+                continue;
+            }
+            for (ShopDataCrawlRowDto sourceRow : source.getItems()) {
+                if (sourceRow != null && !rowEmpty(sourceRow)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
    private ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) {
        ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
        row.setDate(trim(source.getDate()));