完善任务存储、权限及货源查询流程

2026-08-14 22:49:15 +08:00
parent 5b1ccad40e
commit 7a7f1dfa21
21 changed files with 2584 additions and 566 deletions
@@ -250,16 +250,28 @@ public class PublishTaskService {
            if (lock == null) {
                throw new BusinessException("task lock is busy");
            }
-            List<String> storedPayloads = new ArrayList<>();
-            boolean[] cleanupAfterCommit = {false};
+            List<String> uploadedPayloads = new ArrayList<>();
+            PreparedResultSubmission prepared;
            try {
-                transactionTemplate.executeWithoutResult(
-                        status -> cleanupAfterCommit[0] = submitResultLocked(taskId, request, storedPayloads));
+                prepared = prepareResultSubmission(taskId, request, uploadedPayloads);
            } catch (RuntimeException ex) {
-                deleteRolledBackPayloads(storedPayloads);
+                deleteUncommittedPayloads(uploadedPayloads);
                throw ex;
            }
-            if (cleanupAfterCommit[0]) {
+            SubmitResultCommit[] committedHolder = {null};
+            try {
+                transactionTemplate.executeWithoutResult(
+                        status -> committedHolder[0] = submitResultLocked(taskId, prepared));
+            } catch (RuntimeException ex) {
+                deleteUncommittedPayloads(uploadedPayloads);
+                throw ex;
+            }
+            SubmitResultCommit committed = committedHolder[0];
+            Set<String> committedPayloads = committed == null ? Set.of() : committed.committedPayloads();
+            deleteUncommittedPayloads(uploadedPayloads.stream()
+                    .filter(payload -> !committedPayloads.contains(payload))
+                    .toList());
+            if (committed != null && committed.cleanupAfterCommit()) {
                try {
                    deleteTransientResultChunks(taskId);
                } catch (Exception ex) {
@@ -550,20 +562,43 @@ public class PublishTaskService {
                .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
    }

-    private void deleteRolledBackPayloads(List<String> storedPayloads) {
+    private void deleteUncommittedPayloads(List<String> storedPayloads) {
        if (storedPayloads == null || storedPayloads.isEmpty()) {
            return;
        }
        for (String storedPayload : storedPayloads) {
            try {
+                if (isResultPayloadReferenced(storedPayload)) {
+                    log.info("[publish] skip uncommitted payload cleanup because it is referenced pointer={}",
+                            transientPayloadStorageService.extractPointer(storedPayload));
+                    continue;
+                }
                transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
            } catch (Exception ex) {
-                log.warn("[publish] rolled-back RustFS payload cleanup failed pointer={} msg={}",
+                log.warn("[publish] uncommitted RustFS payload cleanup failed pointer={} msg={}",
                        transientPayloadStorageService.extractPointer(storedPayload), safeMessage(ex));
            }
        }
    }

+    private boolean isResultPayloadReferenced(String storedPayload) {
+        String pointer = transientPayloadStorageService.extractPointer(storedPayload);
+        if (pointer == null) {
+            return false;
+        }
+        Set<String> candidates = new LinkedHashSet<>();
+        candidates.add(storedPayload);
+        candidates.add(pointer);
+        try {
+            candidates.add(objectMapper.writeValueAsString(pointer));
+        } catch (Exception ignored) {
+            // Raw and extracted pointer values still cover the current publish format.
+        }
+        Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
+                .in(TaskChunkEntity::getPayloadJson, candidates));
+        return count != null && count > 0L;
+    }
+
    private PreparedFile prepareFile(PublishSourceFileDto source) {
        PreparedFile prepared = new PreparedFile(source);
        prepared.shopName = FileUtil.mainName(source.getOriginalFilename()).trim();
@@ -709,21 +744,19 @@ public class PublishTaskService {
        return new PersistedTask(task, result, savedFiles);
    }

-    private boolean submitResultLocked(Long taskId,
-                                       PublishSubmitResultRequest request,
-                                       List<String> storedPayloads) {
+    private PreparedResultSubmission prepareResultSubmission(Long taskId,
+                                                             PublishSubmitResultRequest request,
+                                                             List<String> uploadedPayloads) {
        FileTaskEntity task = requireTask(taskId, request.getUserId());
        if (STATUS_SUCCESS.equals(task.getStatus())) {
-            return false;
+            return new PreparedResultSubmission(request.getUserId(), List.of());
        }
        if (STATUS_FAILED.equals(task.getStatus())) {
            throw new BusinessException("任务已失败，拒绝继续回传");
        }
-        if (STATUS_PENDING.equals(task.getStatus())) {
-            task.setStatus(STATUS_RUNNING);
-        }

        Set<Long> submittedFileIds = new LinkedHashSet<>();
+        List<PreparedResultFile> preparedFiles = new ArrayList<>();
        for (PublishResultFileDto incoming : request.getFiles()) {
            if (incoming == null) {
                throw new BusinessException("files 不能包含空对象");
@@ -733,14 +766,52 @@ public class PublishTaskService {
                throw new BusinessException("同一文件不能在一次请求中重复提交");
            }
            if (STATUS_SUCCESS.equals(file.getStatus()) || STATUS_FAILED.equals(file.getStatus())) {
+                preparedFiles.add(new PreparedResultFile(file.getId(), null, null));
                continue;
            }
            if (incoming.getError() != null && !incoming.getError().isBlank()) {
+                preparedFiles.add(new PreparedResultFile(file.getId(), incoming.getError().trim(), null));
+                continue;
+            }
+            preparedFiles.add(new PreparedResultFile(
+                    file.getId(), null, prepareResultChunk(taskId, file, incoming, uploadedPayloads)));
+        }
+        return new PreparedResultSubmission(request.getUserId(), List.copyOf(preparedFiles));
+    }
+
+    private SubmitResultCommit submitResultLocked(Long taskId, PreparedResultSubmission prepared) {
+        FileTaskEntity task = requireTask(taskId, prepared.userId());
+        if (STATUS_SUCCESS.equals(task.getStatus())) {
+            return new SubmitResultCommit(false, Set.of());
+        }
+        if (STATUS_FAILED.equals(task.getStatus())) {
+            throw new BusinessException("任务已失败，拒绝继续回传");
+        }
+        if (STATUS_PENDING.equals(task.getStatus())) {
+            task.setStatus(STATUS_RUNNING);
+        }
+
+        Set<Long> submittedFileIds = new LinkedHashSet<>();
+        Set<String> committedPayloads = new LinkedHashSet<>();
+        for (PreparedResultFile preparedFile : prepared.files()) {
+            PublishFileEntity file = requireFile(taskId, preparedFile.fileId());
+            if (!submittedFileIds.add(file.getId())) {
+                throw new BusinessException("同一文件不能在一次请求中重复提交");
+            }
+            if (STATUS_SUCCESS.equals(file.getStatus()) || STATUS_FAILED.equals(file.getStatus())) {
+                continue;
+            }
+            if (preparedFile.error() != null) {
                file.setStatus(STATUS_FAILED);
                file.setProcessedRows(0);
-                file.setErrorMessage(incoming.getError().trim());
+                file.setErrorMessage(preparedFile.error());
            } else {
-                ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads);
+                PreparedResultChunk preparedChunk = preparedFile.chunk();
+                if (preparedChunk == null) {
+                    throw new BusinessException("上架结果分片准备状态无效，请重试");
+                }
+                ResultChunkReceipt receipt = persistPreparedResultChunk(
+                        taskId, file, preparedChunk, committedPayloads);
                if (!receipt.completed()) {
                    updateReceivedProgress(taskId, file, receipt.receivedRowCount());
                    file.setStatus(STATUS_RUNNING);
@@ -750,7 +821,10 @@ public class PublishTaskService {
                    publishFileMapper.updateById(file);
                    continue;
                }
-                List<PublishRowDto> rows = loadCompleteResultRows(taskId, receipt);
+                List<PublishRowDto> rows = preparedChunk.completeRows();
+                if (rows == null) {
+                    throw new BusinessException("上架结果分片状态已变化，请重试");
+                }
                validateCompleteResultRows(taskId, file.getId(), rows);
                replaceRows(taskId, file.getId(), rows);
                file.setStatus(STATUS_SUCCESS);
@@ -775,11 +849,11 @@ public class PublishTaskService {
        if (terminalCount < files.size()) {
            task.setStatus(STATUS_RUNNING);
            fileTaskMapper.updateById(task);
-            return false;
+            return new SubmitResultCommit(false, Set.copyOf(committedPayloads));
        }
        if (successCount <= 0) {
            markTaskAndResultFailed(task, result, "全部文件处理失败");
-            return true;
+            return new SubmitResultCommit(true, Set.copyOf(committedPayloads));
        }

        task.setStatus(STATUS_RUNNING);
@@ -787,7 +861,7 @@ public class PublishTaskService {
        task.setFinishedAt(null);
        fileTaskMapper.updateById(task);
        taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId));
-        return false;
+        return new SubmitResultCommit(false, Set.copyOf(committedPayloads));
    }

    private List<PublishTaskDetailVo> loadTaskDetails(List<FileTaskEntity> tasks) {
@@ -1011,10 +1085,10 @@ public class PublishTaskService {
        return file;
    }

-    private ResultChunkReceipt persistResultChunk(Long taskId,
-                                                   PublishFileEntity file,
-                                                   PublishResultFileDto incoming,
-                                                   List<String> storedPayloads) {
+    private PreparedResultChunk prepareResultChunk(Long taskId,
+                                                    PublishFileEntity file,
+                                                    PublishResultFileDto incoming,
+                                                    List<String> uploadedPayloads) {
        int chunkIndex = incoming.getChunkIndex() == null ? 1 : incoming.getChunkIndex();
        int chunkTotal = incoming.getChunkTotal() == null ? 1 : incoming.getChunkTotal();
        validateChunkMetadata(chunkIndex, chunkTotal);
@@ -1028,21 +1102,83 @@ public class PublishTaskService {
        String payloadJson = writeJson(rows, "序列化上架结果分片失败");
        String payloadHash = DigestUtil.sha256Hex(payloadJson);
        TaskChunkEntity existing = findResultChunk(taskId, scopeHash, chunkIndex);
+        String storedPayload = null;
        if (existing != null) {
            validateExistingChunk(existing, chunkTotal, payloadHash);
-            int receivedChunkCount = countResultChunks(taskId, scopeHash);
-            int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, 0, false);
-            persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
-                    receivedChunkCount, receivedRowCount);
-            return new ResultChunkReceipt(scopeHash, chunkTotal,
-                    receivedChunkCount >= chunkTotal, receivedRowCount);
+        } else {
+            ensureRustfsPayloadStorageEnabled();
+            storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
+                    MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
+            requireRustfsPayload(storedPayload, "上架结果分片必须写入 RustFS");
+            uploadedPayloads.add(storedPayload);
        }

-        ensureRustfsPayloadStorageEnabled();
-        String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
-                MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
-        requireRustfsPayload(storedPayload, "上架结果分片必须写入 RustFS");
+        List<TaskChunkEntity> persistedChunks = listResultChunks(taskId, scopeHash);
+        List<TaskChunkEntity> expectedChunks = persistedChunks;
+        if (existing == null) {
+            expectedChunks = new ArrayList<>(expectedChunks);
+            expectedChunks.add(newResultChunk(
+                    taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, storedPayload, payloadHash));
+            expectedChunks.sort((left, right) -> Integer.compare(
+                    Objects.requireNonNullElse(left.getChunkIndex(), 0),
+                    Objects.requireNonNullElse(right.getChunkIndex(), 0)));
+        }
+        List<ResultChunkManifestEntry> manifest = expectedChunks.stream()
+                .map(this::toResultChunkManifestEntry)
+                .toList();
+        List<PublishRowDto> completeRows = expectedChunks.size() >= chunkTotal
+                ? readCompleteResultRows(expectedChunks, chunkTotal)
+                : null;
+        int receivedRowCount = prepareReceivedRowCount(
+                scope, persistedChunks, existing == null ? rows.size() : 0, completeRows);
+        return new PreparedResultChunk(scopeKey, scopeHash, chunkIndex, chunkTotal,
+                payloadHash, storedPayload, receivedRowCount, manifest, completeRows);
+    }

+    private ResultChunkReceipt persistPreparedResultChunk(Long taskId,
+                                                          PublishFileEntity file,
+                                                          PreparedResultChunk prepared,
+                                                          Set<String> committedPayloads) {
+        TaskScopeStateEntity scope = findResultScope(taskId, prepared.scopeHash());
+        validateChunkTotal(scope == null ? null : scope.getChunkTotal(), prepared.chunkTotal());
+        TaskChunkEntity existing = findResultChunk(taskId, prepared.scopeHash(), prepared.chunkIndex());
+        if (existing != null) {
+            validateExistingChunk(existing, prepared.chunkTotal(), prepared.payloadHash());
+        } else if (prepared.storedPayload() != null) {
+            TaskChunkEntity chunk = newResultChunk(taskId, prepared.scopeKey(), prepared.scopeHash(),
+                    prepared.chunkIndex(), prepared.chunkTotal(), prepared.storedPayload(), prepared.payloadHash());
+            try {
+                taskChunkMapper.insert(chunk);
+                committedPayloads.add(prepared.storedPayload());
+            } catch (DuplicateKeyException ex) {
+                TaskChunkEntity winner = findResultChunk(taskId, prepared.scopeHash(), prepared.chunkIndex());
+                if (winner == null) {
+                    throw new BusinessException("上架结果分片并发写入失败，请重试");
+                }
+                validateExistingChunk(winner, prepared.chunkTotal(), prepared.payloadHash());
+            }
+        }
+
+        List<TaskChunkEntity> actualChunks = listResultChunks(taskId, prepared.scopeHash());
+        validatePreparedResultManifest(actualChunks, prepared.manifest());
+        int receivedChunkCount = actualChunks.size();
+        int receivedRowCount = prepared.receivedRowCount();
+        persistResultScope(taskId, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkTotal(),
+                receivedChunkCount, receivedRowCount);
+        log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} receivedChunks={} receivedRows={}",
+                taskId, file.getId(), prepared.chunkIndex(), prepared.chunkTotal(),
+                receivedChunkCount, receivedRowCount);
+        return new ResultChunkReceipt(
+                receivedChunkCount >= prepared.chunkTotal(), receivedRowCount);
+    }
+
+    private TaskChunkEntity newResultChunk(Long taskId,
+                                           String scopeKey,
+                                           String scopeHash,
+                                           int chunkIndex,
+                                           int chunkTotal,
+                                           String storedPayload,
+                                           String payloadHash) {
        TaskChunkEntity chunk = new TaskChunkEntity();
        chunk.setTaskId(taskId);
        chunk.setModuleType(MODULE_TYPE);
@@ -1054,31 +1190,37 @@ public class PublishTaskService {
        chunk.setPayloadHash(payloadHash);
        chunk.setCreatedAt(LocalDateTime.now());
        chunk.setUpdatedAt(LocalDateTime.now());
-        boolean inserted = false;
-        try {
-            taskChunkMapper.insert(chunk);
-            storedPayloads.add(storedPayload);
-            inserted = true;
-        } catch (DuplicateKeyException ex) {
-            transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
-            TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
-            if (winner == null) {
-                throw new BusinessException("上架结果分片并发写入失败，请重试");
-            }
-            validateExistingChunk(winner, chunkTotal, payloadHash);
-        } catch (RuntimeException ex) {
-            transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
-            throw ex;
-        }
+        return chunk;
+    }

-        int receivedChunkCount = countResultChunks(taskId, scopeHash);
-        int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, rows.size(), inserted);
-        persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
-                receivedChunkCount, receivedRowCount);
-        log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} receivedChunks={} receivedRows={}",
-                taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount, receivedRowCount);
-        return new ResultChunkReceipt(scopeHash, chunkTotal,
-                receivedChunkCount >= chunkTotal, receivedRowCount);
+    private List<TaskChunkEntity> listResultChunks(Long taskId, String scopeHash) {
+        List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
+                .eq(TaskChunkEntity::getTaskId, taskId)
+                .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
+                .eq(TaskChunkEntity::getScopeHash, scopeHash)
+                .orderByAsc(TaskChunkEntity::getChunkIndex));
+        return chunks == null ? List.of() : chunks;
+    }
+
+    private ResultChunkManifestEntry toResultChunkManifestEntry(TaskChunkEntity chunk) {
+        return new ResultChunkManifestEntry(
+                chunk.getChunkIndex(), chunk.getChunkTotal(), chunk.getPayloadHash());
+    }
+
+    private void validatePreparedResultManifest(List<TaskChunkEntity> actualChunks,
+                                                List<ResultChunkManifestEntry> expectedManifest) {
+        if (actualChunks.size() != expectedManifest.size()) {
+            throw new BusinessException("上架结果分片状态已变化，请重试");
+        }
+        for (int i = 0; i < actualChunks.size(); i++) {
+            ResultChunkManifestEntry actual = toResultChunkManifestEntry(actualChunks.get(i));
+            ResultChunkManifestEntry expected = expectedManifest.get(i);
+            if (!Objects.equals(actual.chunkIndex(), expected.chunkIndex())
+                    || !Objects.equals(actual.chunkTotal(), expected.chunkTotal())
+                    || !Objects.equals(actual.payloadHash(), expected.payloadHash())) {
+                throw new BusinessException("上架结果分片状态已变化，请重试");
+            }
+        }
    }

    private void validateChunkMetadata(int chunkIndex, int chunkTotal) {
@@ -1123,31 +1265,20 @@ public class PublishTaskService {
        }
    }

-    private int countResultChunks(Long taskId, String scopeHash) {
-        Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
-                .eq(TaskChunkEntity::getTaskId, taskId)
-                .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
-                .eq(TaskChunkEntity::getScopeHash, scopeHash));
-        return count == null ? 0 : count.intValue();
-    }
-
-    /**
-     * Resolve the number of result rows received for a file without re-reading
-     * every payload on every callback.  New callbacks keep the count in the
-     * existing scope state JSON; scopes created by older versions are repaired
-     * once by counting their stored chunks.
-     */
-    private int resolveReceivedRowCount(Long taskId,
-                                        String scopeHash,
-                                        TaskScopeStateEntity scope,
-                                        int currentChunkRows,
-                                        boolean inserted) {
+    private int prepareReceivedRowCount(TaskScopeStateEntity scope,
+                                        List<TaskChunkEntity> expectedChunks,
+                                        int newChunkRows,
+                                        List<PublishRowDto> completeRows) {
        Integer persisted = readReceivedRowCount(scope);
        if (persisted != null) {
-            long next = (long) persisted + (inserted ? Math.max(0, currentChunkRows) : 0);
+            long next = (long) persisted + Math.max(0, newChunkRows);
            return next > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, next);
        }
-        return countReceivedResultRows(taskId, scopeHash);
+        if (completeRows != null) {
+            return completeRows.size();
+        }
+        long preparedRows = (long) countPreparedResultRows(expectedChunks) + Math.max(0, newChunkRows);
+        return preparedRows > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) preparedRows;
    }

    private Integer readReceivedRowCount(TaskScopeStateEntity scope) {
@@ -1168,13 +1299,8 @@ public class PublishTaskService {
        }
    }

-    private int countReceivedResultRows(Long taskId, String scopeHash) {
-        List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
-                .eq(TaskChunkEntity::getTaskId, taskId)
-                .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
-                .eq(TaskChunkEntity::getScopeHash, scopeHash)
-                .orderByAsc(TaskChunkEntity::getChunkIndex));
-        if (chunks == null || chunks.isEmpty()) {
+    private int countPreparedResultRows(List<TaskChunkEntity> chunks) {
+        if (chunks.isEmpty()) {
            return 0;
        }
        TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
@@ -1268,13 +1394,8 @@ public class PublishTaskService {
        file.setProcessedRows(progress);
    }

-    private List<PublishRowDto> loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) {
-        List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
-                .eq(TaskChunkEntity::getTaskId, taskId)
-                .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
-                .eq(TaskChunkEntity::getScopeHash, receipt.scopeHash())
-                .orderByAsc(TaskChunkEntity::getChunkIndex));
-        if (chunks == null || chunks.size() != receipt.chunkTotal()) {
+    private List<PublishRowDto> readCompleteResultRows(List<TaskChunkEntity> chunks, int chunkTotal) {
+        if (chunks.size() != chunkTotal) {
            throw new BusinessException("上架结果分片尚未完整，暂不能合并");
        }

@@ -1287,7 +1408,7 @@ public class PublishTaskService {
            if (!Objects.equals(chunk.getChunkIndex(), expectedIndex)) {
                throw new BusinessException("上架结果缺少第 " + expectedIndex + " 个分片");
            }
-            validateChunkTotal(chunk.getChunkTotal(), receipt.chunkTotal());
+            validateChunkTotal(chunk.getChunkTotal(), chunkTotal);
            List<PublishRowDto> chunkRows = readResultChunkRows(chunk, listType);
            for (PublishRowDto row : chunkRows) {
                rows.add(copyRequiredRow(row));
@@ -1737,9 +1858,36 @@ public class PublishTaskService {
    private record TaskOptions(String publishCountry, List<String> syncCountries) {
    }

-    private record ResultChunkReceipt(String scopeHash,
-                                      int chunkTotal,
-                                      boolean completed,
+    private record PreparedResultSubmission(Long userId,
+                                            List<PreparedResultFile> files) {
+    }
+
+    private record PreparedResultFile(Long fileId,
+                                      String error,
+                                      PreparedResultChunk chunk) {
+    }
+
+    private record PreparedResultChunk(String scopeKey,
+                                       String scopeHash,
+                                       int chunkIndex,
+                                       int chunkTotal,
+                                       String payloadHash,
+                                       String storedPayload,
+                                       int receivedRowCount,
+                                       List<ResultChunkManifestEntry> manifest,
+                                       List<PublishRowDto> completeRows) {
+    }
+
+    private record ResultChunkManifestEntry(Integer chunkIndex,
+                                            Integer chunkTotal,
+                                            String payloadHash) {
+    }
+
+    private record SubmitResultCommit(boolean cleanupAfterCommit,
+                                      Set<String> committedPayloads) {
+    }
+
+    private record ResultChunkReceipt(boolean completed,
                                      int receivedRowCount) {
    }
 }