新需求更新同步更新

2026-08-06 01:11:54 +08:00
parent 9048bbb7f8
commit 28e7fce11c
112 changed files with 7739 additions and 637 deletions
@@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/publish")
-@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件；Python 按页拉取数据、发送统一任务心跳并回传当前店铺完整结果。")
+@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件；Python 按页拉取数据、通过结果分片回传进度和当前店铺完整结果，任务心跳用于保活。")
 public class PublishController {

    private final PublishTaskService publishTaskService;
@@ -38,9 +38,9 @@ public class PublishFileVo {
    private Integer percent;
    @Schema(description = "与 percent 相同，供公共进度组件使用", example = "50")
    private Integer progressPercent;
-    @Schema(description = "心跳上报的当前处理数量；未上报时使用 processedRows", example = "341")
+    @Schema(description = "结果分片累计接收的当前处理数量；兼容心跳上报", example = "341")
    private Integer progressCurrent;
-    @Schema(description = "心跳上报的总处理数量；未上报时使用 totalRows", example = "682")
+    @Schema(description = "解析得到的总处理数量；兼容心跳上报", example = "682")
    private Integer progressTotal;
    @Schema(description = "文件失败原因的进度展示副本；无错误时为空")
    private String progressMessage;
@@ -345,7 +345,11 @@ public class PublishTaskService {
                .eq(FileTaskEntity::getModuleType, MODULE_TYPE)
                .eq(FileTaskEntity::getStatus, STATUS_RUNNING)
                .set(FileTaskEntity::getUpdatedAt, now));
-        if (request == null || (request.getCurrent() == null && request.getTotal() == null)) {
+        // Generic desktop heartbeats use 0/0 when they only need to keep the task alive.
+        // Treat that as no progress update so result chunks cannot be reset to zero.
+        if (request == null
+                || ((request.getCurrent() == null || request.getCurrent() <= 0)
+                && (request.getTotal() == null || request.getTotal() <= 0))) {
            return;
        }
        PublishFileEntity active = publishFileMapper.selectOne(new LambdaQueryWrapper<PublishFileEntity>()
@@ -360,12 +364,12 @@ public class PublishTaskService {
                .eq(PublishFileEntity::getId, active.getId())
                .eq(PublishFileEntity::getStatus, STATUS_RUNNING)
                .set(PublishFileEntity::getUpdatedAt, now);
-        if (request.getTotal() != null && request.getTotal() >= 0) {
+        if (request.getTotal() != null && request.getTotal() > 0) {
            update.set(PublishFileEntity::getTotalRows, request.getTotal());
        }
-        if (request.getCurrent() != null && request.getCurrent() >= 0) {
+        if (request.getCurrent() != null && request.getCurrent() > 0) {
            int current = request.getCurrent();
-            if (request.getTotal() != null && request.getTotal() >= 0) {
+            if (request.getTotal() != null && request.getTotal() > 0) {
                current = Math.min(current, request.getTotal());
            }
            update.set(PublishFileEntity::getProcessedRows, current);
@@ -432,6 +436,7 @@ public class PublishTaskService {
                throw new BusinessException("no successful publish files");
            }

+            TaskOptions options = readTaskOptions(task);
            List<PublishWorkbookService.WorkbookInput> inputs = new ArrayList<>();
            int rowCount = 0;
            for (PublishFileEntity file : successfulFiles) {
@@ -443,7 +448,7 @@ public class PublishTaskService {
                List<PublishRowDto> rows = items.stream().map(this::toRowDto).toList();
                rowCount += rows.size();
                inputs.add(new PublishWorkbookService.WorkbookInput(
-                        file.getSourceFilename(), file.getShopName(), rows));
+                        file.getSourceFilename(), file.getShopName(), options.publishCountry(), rows));
            }

            PublishWorkbookService.PackagedResult packaged = workbookService.packageTaskResult(
@@ -737,6 +742,7 @@ public class PublishTaskService {
            } else {
                ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads);
                if (!receipt.completed()) {
+                    updateReceivedProgress(taskId, file, receipt.receivedRowCount());
                    file.setStatus(STATUS_RUNNING);
                    file.setErrorMessage(null);
                    file.setUpdatedAt(LocalDateTime.now());
@@ -1025,9 +1031,11 @@ public class PublishTaskService {
        if (existing != null) {
            validateExistingChunk(existing, chunkTotal, payloadHash);
            int receivedChunkCount = countResultChunks(taskId, scopeHash);
-            persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
+            int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, 0, false);
+            persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
+                    receivedChunkCount, receivedRowCount);
            return new ResultChunkReceipt(scopeHash, chunkTotal,
-                    receivedChunkCount >= chunkTotal);
+                    receivedChunkCount >= chunkTotal, receivedRowCount);
        }

        ensureRustfsPayloadStorageEnabled();
@@ -1046,9 +1054,11 @@ public class PublishTaskService {
        chunk.setPayloadHash(payloadHash);
        chunk.setCreatedAt(LocalDateTime.now());
        chunk.setUpdatedAt(LocalDateTime.now());
+        boolean inserted = false;
        try {
            taskChunkMapper.insert(chunk);
            storedPayloads.add(storedPayload);
+            inserted = true;
        } catch (DuplicateKeyException ex) {
            transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
            TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
@@ -1062,11 +1072,13 @@ public class PublishTaskService {
        }

        int receivedChunkCount = countResultChunks(taskId, scopeHash);
-        persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
-        log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} received={}",
-                taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount);
+        int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, rows.size(), inserted);
+        persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
+                receivedChunkCount, receivedRowCount);
+        log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} receivedChunks={} receivedRows={}",
+                taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount, receivedRowCount);
        return new ResultChunkReceipt(scopeHash, chunkTotal,
-                receivedChunkCount >= chunkTotal);
+                receivedChunkCount >= chunkTotal, receivedRowCount);
    }

    private void validateChunkMetadata(int chunkIndex, int chunkTotal) {
@@ -1119,11 +1131,70 @@ public class PublishTaskService {
        return count == null ? 0 : count.intValue();
    }

+    /**
+     * Resolve the number of result rows received for a file without re-reading
+     * every payload on every callback.  New callbacks keep the count in the
+     * existing scope state JSON; scopes created by older versions are repaired
+     * once by counting their stored chunks.
+     */
+    private int resolveReceivedRowCount(Long taskId,
+                                        String scopeHash,
+                                        TaskScopeStateEntity scope,
+                                        int currentChunkRows,
+                                        boolean inserted) {
+        Integer persisted = readReceivedRowCount(scope);
+        if (persisted != null) {
+            long next = (long) persisted + (inserted ? Math.max(0, currentChunkRows) : 0);
+            return next > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, next);
+        }
+        return countReceivedResultRows(taskId, scopeHash);
+    }
+
+    private Integer readReceivedRowCount(TaskScopeStateEntity scope) {
+        if (scope == null || scope.getStateJson() == null || scope.getStateJson().isBlank()) {
+            return null;
+        }
+        try {
+            JsonNode root = objectMapper.readTree(scope.getStateJson());
+            JsonNode receivedRows = root == null ? null : root.get("receivedRows");
+            if (receivedRows == null || !receivedRows.isIntegralNumber()) {
+                return null;
+            }
+            return Math.max(0, receivedRows.asInt(0));
+        } catch (Exception ex) {
+            log.warn("[publish] failed to read received row count from result scope taskId={} scopeHash={} msg={}",
+                    scope.getTaskId(), scope.getScopeHash(), safeMessage(ex));
+            return null;
+        }
+    }
+
+    private int countReceivedResultRows(Long taskId, String scopeHash) {
+        List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
+                .eq(TaskChunkEntity::getTaskId, taskId)
+                .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
+                .eq(TaskChunkEntity::getScopeHash, scopeHash)
+                .orderByAsc(TaskChunkEntity::getChunkIndex));
+        if (chunks == null || chunks.isEmpty()) {
+            return 0;
+        }
+        TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
+        };
+        long count = 0;
+        for (TaskChunkEntity chunk : chunks) {
+            count += readResultChunkRows(chunk, listType).size();
+            if (count >= Integer.MAX_VALUE) {
+                return Integer.MAX_VALUE;
+            }
+        }
+        return (int) count;
+    }
+
    private void persistResultScope(Long taskId,
                                    String scopeKey,
                                    String scopeHash,
                                    int chunkTotal,
-                                    int receivedChunkCount) {
+                                    int receivedChunkCount,
+                                    int receivedRowCount) {
        TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
        validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
        LocalDateTime now = LocalDateTime.now();
@@ -1141,7 +1212,7 @@ public class PublishTaskService {
        scope.setCompleted(completed ? 1 : 0);
        scope.setLastChunkAt(now);
        scope.setLastError(null);
-        scope.setStateJson(completed ? "{\"phase\":\"COMPLETE\"}" : "{\"phase\":\"RECEIVING\"}");
+        scope.setStateJson(resultScopeStateJson(completed, receivedRowCount));
        scope.setUpdatedAt(now);
        if (scope.getId() != null) {
            taskScopeStateMapper.updateById(scope);
@@ -1160,12 +1231,43 @@ public class PublishTaskService {
            winner.setCompleted(completed ? 1 : 0);
            winner.setLastChunkAt(now);
            winner.setLastError(null);
-            winner.setStateJson(scope.getStateJson());
+            Integer winnerRowCount = readReceivedRowCount(winner);
+            winner.setStateJson(resultScopeStateJson(completed,
+                    Math.max(receivedRowCount, winnerRowCount == null ? 0 : winnerRowCount)));
            winner.setUpdatedAt(now);
            taskScopeStateMapper.updateById(winner);
        }
    }

+    private String resultScopeStateJson(boolean completed, int receivedRowCount) {
+        Map<String, Object> state = new LinkedHashMap<>();
+        state.put("phase", completed ? "COMPLETE" : "RECEIVING");
+        state.put("receivedRows", Math.max(0, receivedRowCount));
+        return writeJson(state, "保存上架结果分片进度失败");
+    }
+
+    private void updateReceivedProgress(Long taskId, PublishFileEntity file, int receivedRowCount) {
+        if (file == null) {
+            return;
+        }
+        int total = safeInt(file.getTotalRows());
+        if (total <= 0) {
+            long parsedRows = Objects.requireNonNullElse(publishItemMapper.selectCount(
+                    new LambdaQueryWrapper<PublishItemEntity>()
+                            .eq(PublishItemEntity::getTaskId, taskId)
+                            .eq(PublishItemEntity::getFileId, file.getId())), 0L);
+            total = parsedRows >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0L, parsedRows);
+            if (total > 0) {
+                file.setTotalRows(total);
+            }
+        }
+        int progress = Math.max(safeInt(file.getProcessedRows()), Math.max(0, receivedRowCount));
+        if (total > 0) {
+            progress = Math.min(total, progress);
+        }
+        file.setProcessedRows(progress);
+    }
+
    private List<PublishRowDto> loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) {
        List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
                .eq(TaskChunkEntity::getTaskId, taskId)
@@ -1637,6 +1739,7 @@ public class PublishTaskService {

    private record ResultChunkReceipt(String scopeHash,
                                      int chunkTotal,
-                                      boolean completed) {
+                                      boolean completed,
+                                      int receivedRowCount) {
    }
 }
@@ -91,7 +91,7 @@ public class PublishWorkbookService {
        }
    }

-    public File writeWorkbook(File outputFile, List<PublishRowDto> rows) {
+    public File writeWorkbook(File outputFile, List<PublishRowDto> rows, String publishCountry) {
        File parent = outputFile.getParentFile();
        if (parent != null) {
            parent.mkdirs();
@@ -100,13 +100,9 @@ public class PublishWorkbookService {
        workbook.setCompressTempFiles(true);
        try (FileOutputStream output = new FileOutputStream(outputFile)) {
            CellStyle headerStyle = createHeaderStyle(workbook);
-            Map<String, List<PublishRowDto>> rowsByCountry = groupByCountry(rows);
-            Set<String> usedSheetNames = new LinkedHashSet<>();
-            for (Map.Entry<String, List<PublishRowDto>> entry : rowsByCountry.entrySet()) {
-                String sheetName = uniqueSheetName(entry.getKey(), usedSheetNames);
-                Sheet sheet = workbook.createSheet(sheetName);
-                writeSheet(sheet, entry.getValue(), headerStyle);
-            }
+            String country = countrySheetName(publishCountry);
+            Sheet sheet = workbook.createSheet(country);
+            writeSheet(sheet, rows, headerStyle, country);
            workbook.write(output);
            return outputFile;
        } catch (Exception ex) {
@@ -135,7 +131,7 @@ public class PublishWorkbookService {
                    + "_上架结果.xlsx";
            String filename = uniqueFilename(desired, usedFilenames);
            File workbook = new File(workDirectory, filename);
-            writeWorkbook(workbook, input.rows());
+            writeWorkbook(workbook, input.rows(), input.publishCountry());
            workbooks.add(workbook);
        }

@@ -202,7 +198,10 @@ public class PublishWorkbookService {
        return grouped;
    }

-    private void writeSheet(Sheet sheet, List<PublishRowDto> rows, CellStyle headerStyle) {
+    private void writeSheet(Sheet sheet,
+                            List<PublishRowDto> rows,
+                            CellStyle headerStyle,
+                            String publishCountry) {
        Row header = sheet.createRow(0);
        for (int index = 0; index < RESULT_HEADERS.size(); index++) {
            Cell cell = header.createCell(index);
@@ -210,11 +209,14 @@ public class PublishWorkbookService {
            cell.setCellStyle(headerStyle);
        }
        int rowIndex = 1;
-        for (PublishRowDto value : rows) {
+        for (PublishRowDto value : rows == null ? List.<PublishRowDto>of() : rows) {
+            if (value == null) {
+                continue;
+            }
            Row row = sheet.createRow(rowIndex++);
            setText(row, 0, value.getSourceId());
            setText(row, 1, value.getAsin());
-            setText(row, 2, value.getCountry());
+            setText(row, 2, publishCountry);
            setText(row, 3, value.getBrand());
            setPrice(row, 4, value.getPrice());
            setText(row, 5, value.getStatus());
@@ -337,7 +339,10 @@ public class PublishWorkbookService {
    public record ParsedWorkbook(List<PublishRowDto> rows) {
    }

-    public record WorkbookInput(String sourceFilename, String shopName, List<PublishRowDto> rows) {
+    public record WorkbookInput(String sourceFilename,
+                                String shopName,
+                                String publishCountry,
+                                List<PublishRowDto> rows) {
    }

    public record PackagedResult(File file, String filename, String contentType) {