diff --git a/backend-java/pom.xml b/backend-java/pom.xml index a4e1ee38..fb3ebbc7 100644 --- a/backend-java/pom.xml +++ b/backend-java/pom.xml @@ -183,7 +183,7 @@ org.apache.maven.plugins maven-surefire-plugin - -XX:+EnableDynamicAgentLoading -Xshare:off -Xmx2g -XX:MaxMetaspaceSize=512m + -XX:+EnableDynamicAgentLoading -Xshare:off -Xmx1536m -XX:MaxMetaspaceSize=512m diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckHttpConfigResolver.java b/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckHttpConfigResolver.java new file mode 100644 index 00000000..a4bc705a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckHttpConfigResolver.java @@ -0,0 +1,33 @@ +package com.nanri.aiimage.config; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +/** + * 品牌检查客户端统一配置解析(task-169)。 + * + * 从 aiimage.http-client.* 命名空间读取品牌检查客户端超时/重试,默认值与现状 + * 一致(BrandCheckProperties:connect 10s / read 60s);非法值钳制。 + */ +@Component +@RequiredArgsConstructor +public class BrandCheckHttpConfigResolver { + + private final HttpClientProperties httpClientProperties; + + public long connectTimeoutMillis() { + return httpClientProperties.effectiveConnectTimeoutMillis(); + } + + public long readTimeoutMillis() { + return httpClientProperties.effectiveReadTimeoutMillis(); + } + + public long callTimeoutMillis() { + return httpClientProperties.effectiveCallTimeoutMillis(); + } + + public int maxRetries() { + return httpClientProperties.effectiveMaxRetries(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java index e275077f..54c7bbda 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.controller; import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest; import com.nanri.aiimage.modules.dedupe.model.vo.DedupeHistoryVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo; import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo; import com.nanri.aiimage.modules.dedupe.service.DedupeRunService; import io.swagger.v3.oas.annotations.Operation; @@ -37,10 +38,13 @@ public class DedupeRunController { @PostMapping("/run") @Operation( - summary = "执行数据去重", + summary = "提交数据去重任务", description = """ 根据上传文件、保留列、ID 保留规则执行数据去重,生成清洗后的 Excel 结果文件。 + 任务异步执行:接口提交后立即返回 runId 与初始进度,前端通过 + GET /api/dedupe/run/{runId}/progress 轮询处理进度与最终结果。 + 规则与旧 Python 桌面端保持一致: - 表头会先做文本清洗; - 遇到“缩略图地址8”停止继续识别表头; @@ -53,12 +57,22 @@ public class DedupeRunController { - 输出文件命名遵循 原文件名_cleaned.xlsx,若重名自动追加序号。 """) @ApiResponses({ - @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "执行成功,返回处理统计与结果列表", content = @Content(schema = @Schema(implementation = DedupeRunVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "提交成功,返回任务标识与初始进度", content = @Content(schema = @Schema(implementation = DedupeRunProgressVo.class))), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "请求参数不合法或缺少必要字段"), - @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "执行数据去重失败") + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "提交数据去重任务失败") }) - public ApiResponse run(@Valid @RequestBody DedupeRunRequest request) { - return ApiResponse.success(dedupeRunService.run(request)); + public ApiResponse run(@Valid @RequestBody DedupeRunRequest request) { + return ApiResponse.success(dedupeRunService.submitRun(request)); + } + + @GetMapping("/run/{runId}/progress") + @Operation(summary = "查询去重任务进度", description = "轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeRunProgressVo.class))) + }) + public ApiResponse progress( + @Parameter(description = "去重任务标识", required = true) @PathVariable String runId) { + return ApiResponse.success(dedupeRunService.getProgress(runId)); } @GetMapping("/history") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeResultItemVo.java index bb1c4ddb..f94a57d8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeResultItemVo.java @@ -24,4 +24,7 @@ public class DedupeResultItemVo { @Schema(description = "下载地址") private String downloadUrl; + + @Schema(description = "结果文件大小(字节)") + private Long resultFileSize; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeRunProgressVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeRunProgressVo.java new file mode 100644 index 00000000..6c6823c9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeRunProgressVo.java @@ -0,0 +1,40 @@ +package com.nanri.aiimage.modules.dedupe.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +/** + * 去重异步任务进度。run 接口提交任务后立即返回 runId, + * 前端通过该 VO 轮询处理进度与最终结果。 + */ +@Data +@Schema(description = "数据去重异步任务进度") +public class DedupeRunProgressVo { + + @Schema(description = "异步任务标识") + private String runId; + + @Schema(description = "任务状态: running/success/failed/not_found") + private String status; + + @Schema(description = "处理文件总数") + private Integer total; + + @Schema(description = "已处理文件数") + private Integer processedCount; + + @Schema(description = "成功文件数") + private Integer successCount; + + @Schema(description = "失败文件数") + private Integer failedCount; + + @Schema(description = "处理是否已结束(成功或失败)") + private boolean finished; + + @Schema(description = "错误信息") + private String error; + + @Schema(description = "最终结果列表,任务完成后才有") + private DedupeRunVo result; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 1c7669ec..2515c3b4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -5,10 +5,12 @@ import cn.hutool.core.util.IdUtil; import cn.hutool.json.JSONUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.ExcelStreamReader; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto; import com.nanri.aiimage.modules.dedupe.model.vo.DedupeResultItemVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo; import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; @@ -17,26 +19,24 @@ import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.WorkbookUtil; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.nio.file.Files; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -48,6 +48,11 @@ public class DedupeRunService { private static final String HEADER_SPLIT_MARKER = "idASIN国家状态价格变体数量"; private static final String HEADER_STOP_COLUMN = "缩略图地址8"; private static final List EXPORT_COLUMN_PRIORITY = List.of("id", "ASIN", "国家", "价格", "品牌"); + /** 同时处理的文件数上限,避免多文件并行时打满内存与 OSS 上传带宽 */ + private static final int MAX_PARALLEL_FILES = 4; + /** 单用户同时运行中的去重任务数上限 */ + private static final int MAX_RUNNING_TASKS_PER_USER = 2; + private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L; private final FileTaskMapper fileTaskMapper; private final FileResultMapper fileResultMapper; @@ -55,106 +60,260 @@ public class DedupeRunService { private final OssStorageService ossStorageService; private final DedupeTotalDataService dedupeTotalDataService; - public DedupeRunVo run(DedupeRunRequest request) { - long runStartNs = System.nanoTime(); + /** runId -> 进度快照(任务结束后保留 1 小时供轮询) */ + private final Map runProgressMap = new ConcurrentHashMap<>(); + /** runId -> 完成时间戳,用于过期清理 */ + private final Map runCompletedAtMap = new ConcurrentHashMap<>(); + /** userId -> 运行中任务数,限制单用户并发任务 */ + private final Map runningTaskCountMap = new ConcurrentHashMap<>(); + private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES); + + /** + * 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。 + */ + public DedupeRunProgressVo submitRun(DedupeRunRequest request) { if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) { throw new BusinessException("请至少选择一种 ID 保留规则"); } + cleanupExpiredProgress(); + AtomicInteger runningCount = runningTaskCountMap.computeIfAbsent(request.getUserId(), k -> new AtomicInteger(0)); + if (runningCount.get() >= MAX_RUNNING_TASKS_PER_USER) { + throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试"); + } + runningCount.incrementAndGet(); + + String runId = IdUtil.fastSimpleUUID(); FileTaskEntity task = new FileTaskEntity(); task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr()); task.setModuleType("DEDUPE"); task.setTaskMode("IMMEDIATE"); - task.setStatus("SUCCESS"); + task.setStatus("RUNNING"); task.setSourceFileCount(request.getFiles().size()); task.setCreatedBy("user:" + request.getUserId()); task.setUserId(request.getUserId()); task.setRequestJson(JSONUtil.toJsonStr(request)); task.setCreatedAt(LocalDateTime.now()); task.setUpdatedAt(LocalDateTime.now()); - task.setFinishedAt(LocalDateTime.now()); fileTaskMapper.insert(task); + DedupeRunProgressVo progress = new DedupeRunProgressVo(); + progress.setRunId(runId); + progress.setStatus("running"); + progress.setTotal(request.getFiles().size()); + progress.setProcessedCount(0); + progress.setSuccessCount(0); + progress.setFailedCount(0); + progress.setFinished(false); + runProgressMap.put(runId, progress); + + try { + Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress)); + log.info("dedupe run submitted runId={} taskNo={} userId={} files={} thread={}", + runId, task.getTaskNo(), request.getUserId(), request.getFiles().size(), thread.threadId()); + } catch (Exception e) { + runProgressMap.remove(runId); + decrementRunningCount(request.getUserId()); + throw new BusinessException("提交去重任务失败:" + e.getMessage()); + } + + return snapshot(progress); + } + + /** + * 查询去重任务进度;runId 不存在或已过期时返回 status=not_found 的占位结果。 + */ + public DedupeRunProgressVo getProgress(String runId) { + DedupeRunProgressVo progress = runProgressMap.get(runId); + if (progress == null) { + DedupeRunProgressVo notFound = new DedupeRunProgressVo(); + notFound.setRunId(runId); + notFound.setStatus("not_found"); + notFound.setTotal(0); + notFound.setProcessedCount(0); + notFound.setSuccessCount(0); + notFound.setFailedCount(0); + notFound.setFinished(false); + notFound.setError("任务不存在或已过期"); + return notFound; + } + return snapshot(progress); + } + + private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) { boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank(); - List items = new ArrayList<>(); - int successCount = 0; - int failedCount = 0; - List archiveEntries = new ArrayList<>(); + Map archiveEntries = new ConcurrentHashMap<>(); + List outcomeItems = new ArrayList<>(); - for (DedupeSourceFileDto sourceFile : request.getFiles()) { - DedupeResultItemVo item = new DedupeResultItemVo(); - item.setSourceFilename(sourceFile.getOriginalFilename()); - try { - long fileStartNs = System.nanoTime(); - File inputFile = findLocalSourceFile(sourceFile.getFileKey()); - long findFileNs = elapsedNs(fileStartNs); - if (inputFile == null || !inputFile.exists()) { - throw new BusinessException("上传文件不存在,请重新上传"); + try { + AtomicInteger processedCount = new AtomicInteger(0); + List workers = new ArrayList<>(request.getFiles().size()); + for (DedupeSourceFileDto sourceFile : request.getFiles()) { + Thread worker = Thread.ofVirtual().start(() -> { + DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries); + synchronized (progress) { + progress.setProcessedCount(processedCount.incrementAndGet()); + if (item.isSuccess()) { + progress.setSuccessCount(progress.getSuccessCount() + 1); + } else { + progress.setFailedCount(progress.getFailedCount() + 1); + } + // 文件夹模式的结果列表只保留 ZIP 打包项,与旧实现一致 + if (!folderMode) { + outcomeItems.add(item); + } + } + }); + workers.add(worker); + } + for (Thread worker : workers) { + worker.join(); + } + } catch (Exception ex) { + log.error("dedupe run async aborted runId={} error", runId, ex); + } + + try { + if (folderMode && !archiveEntries.isEmpty()) { + DedupeResultItemVo zipItem = buildFolderZipResult(request, archiveEntries, task); + synchronized (progress) { + outcomeItems.add(0, zipItem); } + } + DedupeRunVo result = buildRunVo(request, outcomeItems, progress); + finishTaskSuccess(task, result); + synchronized (progress) { + progress.setResult(result); + progress.setStatus("success"); + progress.setFinished(true); + } + log.info("dedupe run finished runId={} userId={} files={} success={} failed={}", + runId, request.getUserId(), request.getFiles().size(), + result.getSuccessCount(), result.getFailedCount()); + } catch (Exception ex) { + log.error("dedupe run finalize failed runId={} userId={}", runId, request.getUserId(), ex); + synchronized (progress) { + progress.setStatus("failed"); + progress.setFinished(true); + progress.setError("去重任务收尾失败:" + ex.getMessage()); + } + task.setStatus("FAILED"); + task.setErrorMessage(progress.getError()); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + } finally { + runCompletedAtMap.put(runId, System.currentTimeMillis()); + decrementRunningCount(request.getUserId()); + } + } - String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); - String outputFilename = buildOutputFilename(inputName); - File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); - File outputFile = buildNamedOutputFile(outputDir, outputFilename); + private DedupeRunVo buildRunVo(DedupeRunRequest request, List items, + DedupeRunProgressVo progress) { + DedupeRunVo vo = new DedupeRunVo(); + vo.setTotal(request.getFiles().size()); + vo.setSuccessCount(progress.getSuccessCount()); + vo.setFailedCount(progress.getFailedCount()); + vo.setItems(items); + return vo; + } - long cleanStartNs = System.nanoTime(); - cleanExcelByLegacyRules( - inputFile, - outputFile, - request.getSelectedColumns(), - request.isKeepIntegerIds(), - request.isKeepUnderscoreIds(), - request.isKeepIntegerMainIdsWhenNoSubIds() - ); - long cleanNs = elapsedNs(cleanStartNs); + private void finishTaskSuccess(FileTaskEntity task, DedupeRunVo result) { + task.setStatus("SUCCESS"); + task.setSuccessFileCount(result.getSuccessCount()); + task.setFailedFileCount(result.getFailedCount()); + task.setResultJson(JSONUtil.toJsonStr(result.getItems())); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + } - if (folderMode) { - archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile)); - successCount++; - continue; - } + private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request, + boolean folderMode, FileTaskEntity task, + Map archiveEntries) { + DedupeResultItemVo item = new DedupeResultItemVo(); + item.setSourceFilename(sourceFile.getOriginalFilename()); + try { + fileParallelSemaphore.acquire(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + item.setSuccess(false); + item.setError("等待处理信号量被中断"); + return item; + } + try { + long fileStartNs = System.nanoTime(); + File inputFile = findLocalSourceFile(sourceFile.getFileKey()); + if (inputFile == null || !inputFile.exists()) { + throw new BusinessException("上传文件不存在,请重新上传"); + } - long uploadStartNs = System.nanoTime(); - OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE"); - long uploadNs = elapsedNs(uploadStartNs); - String ossObjectKey = uploadedResult.objectKey(); - String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName(); + String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); + String outputFilename = buildOutputFilename(inputName); + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); + File outputFile = buildNamedOutputFile(outputDir, outputFilename); + long cleanStartNs = System.nanoTime(); + cleanExcelByStreamRules( + inputFile, + outputFile, + request.getSelectedColumns(), + request.isKeepIntegerIds(), + request.isKeepUnderscoreIds(), + request.isKeepIntegerMainIdsWhenNoSubIds() + ); + long cleanNs = elapsedNs(cleanStartNs); + + if (folderMode) { + archiveEntries.put(sourceFile.getRelativePath() == null ? sourceFile.getFileKey() : sourceFile.getRelativePath(), + new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile)); item.setSuccess(true); - item.setOutputFilename(downloadFilename); - item.setDownloadUrl(uploadedResult.downloadUrl()); - successCount++; + item.setOutputFilename(outputFilename); + log.info("dedupe run file cleansed fileKey={} filename={} size={} cleanMs={} totalFileMs={}", + sourceFile.getFileKey(), inputName, inputFile.length(), toMs(cleanNs), toMs(elapsedNs(fileStartNs))); + return item; + } - FileResultEntity resultEntity = new FileResultEntity(); - resultEntity.setTaskId(task.getId()); - resultEntity.setModuleType("DEDUPE"); - resultEntity.setSourceFilename(inputName); - resultEntity.setResultFilename(downloadFilename); - resultEntity.setResultFileUrl(ossObjectKey); - resultEntity.setResultFileSize(outputFile.length()); - resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); - resultEntity.setSuccess(1); - resultEntity.setUserId(request.getUserId()); - resultEntity.setCreatedAt(LocalDateTime.now()); - long resultInsertStartNs = System.nanoTime(); - fileResultMapper.insert(resultEntity); - long resultInsertNs = elapsedNs(resultInsertStartNs); - log.info( - "dedupe run file done fileKey={} filename={} size={} findFileMs={} cleanMs={} uploadMs={} resultInsertMs={} totalFileMs={}", - sourceFile.getFileKey(), - inputName, - inputFile.length(), - toMs(findFileNs), - toMs(cleanNs), - toMs(uploadNs), - toMs(resultInsertNs), - toMs(elapsedNs(fileStartNs)) - ); - } catch (Exception ex) { - item.setSuccess(false); - item.setError(ex.getMessage()); - failedCount++; + long uploadStartNs = System.nanoTime(); + OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE"); + long uploadNs = elapsedNs(uploadStartNs); + String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName(); + item.setSuccess(true); + item.setOutputFilename(downloadFilename); + item.setDownloadUrl(uploadedResult.downloadUrl()); + item.setResultFileSize(outputFile.length()); + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("DEDUPE"); + resultEntity.setSourceFilename(inputName); + resultEntity.setResultFilename(downloadFilename); + resultEntity.setResultFileUrl(uploadedResult.objectKey()); + resultEntity.setResultFileSize(outputFile.length()); + resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + resultEntity.setSuccess(1); + resultEntity.setUserId(request.getUserId()); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + item.setResultId(resultEntity.getId()); + + log.info( + "dedupe run file done fileKey={} filename={} size={} cleanMs={} uploadMs={} totalFileMs={}", + sourceFile.getFileKey(), + inputName, + inputFile.length(), + toMs(cleanNs), + toMs(uploadNs), + toMs(elapsedNs(fileStartNs)) + ); + return item; + } catch (Exception ex) { + item.setSuccess(false); + item.setError(ex.getMessage()); + log.error("dedupe run file failed fileKey={} filename={} error", + sourceFile.getFileKey(), sourceFile.getOriginalFilename(), ex); + try { FileResultEntity resultEntity = new FileResultEntity(); resultEntity.setTaskId(task.getId()); resultEntity.setModuleType("DEDUPE"); @@ -164,56 +323,43 @@ public class DedupeRunService { resultEntity.setUserId(request.getUserId()); resultEntity.setCreatedAt(LocalDateTime.now()); fileResultMapper.insert(resultEntity); + } catch (Exception insertEx) { + // 失败记录写入失败不能影响进度统计,仅记录日志 + log.error("dedupe failed result record insert failed source={} error", + sourceFile.getOriginalFilename(), insertEx); } - items.add(item); + return item; + } finally { + fileParallelSemaphore.release(); } + } - if (folderMode && !archiveEntries.isEmpty()) { - File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries); - OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE"); - String ossObjectKey = uploadedResult.objectKey(); - DedupeResultItemVo item = new DedupeResultItemVo(); - item.setSourceFilename(request.getArchiveName()); - item.setOutputFilename(zipFile.getName()); - item.setSuccess(true); - item.setDownloadUrl(uploadedResult.downloadUrl()); + private DedupeResultItemVo buildFolderZipResult(DedupeRunRequest request, + Map archiveEntries, + FileTaskEntity task) { + File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), new ArrayList<>(archiveEntries.values())); + OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE"); + DedupeResultItemVo item = new DedupeResultItemVo(); + item.setSourceFilename(request.getArchiveName()); + item.setOutputFilename(zipFile.getName()); + item.setSuccess(true); + item.setDownloadUrl(uploadedResult.downloadUrl()); + item.setResultFileSize(zipFile.length()); - FileResultEntity resultEntity = new FileResultEntity(); - resultEntity.setTaskId(task.getId()); - resultEntity.setModuleType("DEDUPE"); - resultEntity.setSourceFilename(request.getArchiveName()); - resultEntity.setResultFilename(zipFile.getName()); - resultEntity.setResultFileUrl(ossObjectKey); - resultEntity.setResultFileSize(zipFile.length()); - resultEntity.setResultContentType("application/zip"); - resultEntity.setSuccess(1); - resultEntity.setUserId(request.getUserId()); - resultEntity.setCreatedAt(LocalDateTime.now()); - fileResultMapper.insert(resultEntity); - item.setResultId(resultEntity.getId()); - items.add(0, item); - } - - task.setSuccessFileCount(successCount); - task.setFailedFileCount(failedCount); - task.setResultJson(JSONUtil.toJsonStr(items)); - task.setUpdatedAt(LocalDateTime.now()); - fileTaskMapper.updateById(task); - - DedupeRunVo vo = new DedupeRunVo(); - vo.setTotal(request.getFiles().size()); - vo.setSuccessCount(successCount); - vo.setFailedCount(failedCount); - vo.setItems(items); - log.info( - "dedupe run done userId={} files={} success={} failed={} totalMs={}", - request.getUserId(), - request.getFiles().size(), - successCount, - failedCount, - toMs(elapsedNs(runStartNs)) - ); - return vo; + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("DEDUPE"); + resultEntity.setSourceFilename(request.getArchiveName()); + resultEntity.setResultFilename(zipFile.getName()); + resultEntity.setResultFileUrl(uploadedResult.objectKey()); + resultEntity.setResultFileSize(zipFile.length()); + resultEntity.setResultContentType("application/zip"); + resultEntity.setSuccess(1); + resultEntity.setUserId(request.getUserId()); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + item.setResultId(resultEntity.getId()); + return item; } public List listHistory(Long userId) { @@ -252,22 +398,81 @@ public class DedupeRunService { return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()); } - private void cleanExcelByLegacyRules(File inputFile, File outputFile, List selectedColumns, + /** + * 流式读取 Excel(EasyExcel SAX),逐行清洗后缓存候选行,再与去重总数据比对并写出结果。 + * 避免 POI 全量加载整个工作簿,大文件场景内存占用大幅下降。 + */ + private void cleanExcelByStreamRules(File inputFile, File outputFile, List selectedColumns, boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { - selectedColumns = reorderExportColumns(selectedColumns); + List orderedSelectedColumns = reorderExportColumns(selectedColumns); long readStartNs = System.nanoTime(); - DedupeReadResult readResult = readDedupeRows( - inputFile, - selectedColumns, - keepIntegerIds, - keepUnderscoreIds, - keepIntegerMainIdsWhenNoSubIds - ); + DedupeReadResult readResult = new DedupeReadResult(); + readResult.sheetName = ""; + readResult.rows = new ArrayList<>(); + readResult.scannedRows = new AtomicInteger(0); + readResult.filteredFbaRows = new AtomicInteger(0); + + PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup(); + // 表头列索引缓存:由 onHeader 填充,数据行为空时使用 + final Map[] headerIndexCache = new Map[]{Map.of()}; + + ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() { + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + readResult.sheetName = sheetName; + Map headerIndex = buildHeaderIndex(headerMap); + if (headerIndex.isEmpty()) { + throw new BusinessException("Excel 表头为空"); + } + List missing = orderedSelectedColumns.stream() + .filter(column -> !headerIndex.containsKey(column)) + .toList(); + if (!missing.isEmpty()) { + throw new BusinessException("缺少列:" + String.join("、", missing)); + } + headerIndexCache[0] = headerIndex; + } + + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, Map headerMap, Map rowMap) throws Exception { + Map headerIndex = headerIndexCache[0]; + if (headerIndex.isEmpty()) { + return; + } + readResult.scannedRows.incrementAndGet(); + + Integer shippingTypeColumnIndex = headerIndex.get("发货类型"); + if (shippingTypeColumnIndex != null) { + String shippingType = normalizeCellText(cellText(rowMap, shippingTypeColumnIndex)); + if ("FBA".equalsIgnoreCase(shippingType)) { + readResult.filteredFbaRows.incrementAndGet(); + return; + } + } + Integer idColumnIndex = headerIndex.get("id"); + if (idColumnIndex == null) { + readResult.rows.add(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null)); + return; + } + appendRowByIdRule( + normalizeCellText(cellText(rowMap, idColumnIndex)), + rowMap, + headerIndex, + orderedSelectedColumns, + keepIntegerIds, + keepUnderscoreIds, + keepIntegerMainIdsWhenNoSubIds, + pendingMainIdGroup, + readResult.rows + ); + } + }); + pendingMainIdGroup.flush(readResult.rows); long readNs = elapsedNs(readStartNs); Set candidateAsinValues = new HashSet<>(); - for (DedupeCandidateRow row : readResult.rows()) { + for (DedupeCandidateRow row : readResult.rows) { if (!row.asinValue().isBlank()) { candidateAsinValues.add(row.asinValue()); } @@ -279,7 +484,7 @@ public class DedupeRunService { long writeStartNs = System.nanoTime(); int outputRows = 0; try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) { - org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName())); + org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName)); Row outputHeaderRow = outputSheet.createRow(0); for (int i = 0; i < selectedColumns.size(); i++) { outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i)); @@ -287,7 +492,7 @@ public class DedupeRunService { Set writtenAsinValues = new HashSet<>(); int outputRowIndex = 1; - for (DedupeCandidateRow candidateRow : readResult.rows()) { + for (DedupeCandidateRow candidateRow : readResult.rows) { String asinValue = candidateRow.asinValue(); if (!asinValue.isBlank()) { if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) { @@ -314,12 +519,12 @@ public class DedupeRunService { log.info( "dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} filteredFbaRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}", inputFile.getName(), - readResult.scannedRows(), - readResult.rows().size(), + readResult.scannedRows.get(), + readResult.rows.size(), candidateAsinValues.size(), matchedAsinValues.size(), outputRows, - readResult.filteredFbaRows(), + readResult.filteredFbaRows.get(), toMs(readNs), toMs(dbNs), toMs(writeNs), @@ -327,6 +532,26 @@ public class DedupeRunService { ); } + /** 将 EasyExcel 的表头 Map(列序号->列名)转换为 列名->列序号,保留 HEADER_STOP_COLUMN 截断行为 */ + private Map buildHeaderIndex(Map headerMap) { + Map headerIndex = new LinkedHashMap<>(); + List columnIndexes = headerMap.keySet().stream().sorted().toList(); + for (Integer columnIndex : columnIndexes) { + String value = normalizeHeaderText(headerMap.get(columnIndex)); + if (value.isBlank()) { + continue; + } + if (headerIndex.containsKey(value)) { + continue; + } + headerIndex.put(value, columnIndex); + if (HEADER_STOP_COLUMN.equals(value)) { + break; + } + } + return headerIndex; + } + private List reorderExportColumns(List selectedColumns) { if (selectedColumns == null || selectedColumns.isEmpty()) { return selectedColumns; @@ -345,123 +570,23 @@ public class DedupeRunService { return ordered; } - private DedupeReadResult readDedupeRows(File inputFile, List selectedColumns, - boolean keepIntegerIds, boolean keepUnderscoreIds, - boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { - List rows = new ArrayList<>(); - PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup(); - DataFormatter formatter = new DataFormatter(); - - try (FileInputStream fis = new FileInputStream(inputFile); - Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { - Sheet sheet = workbook.getSheetAt(0); - Row headerRow = sheet.getRow(0); - if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); - } - - Map headerMap = buildHeaderMap(headerRow, formatter); - if (headerMap.isEmpty()) { - throw new BusinessException("Excel 表头为空"); - } - List missing = selectedColumns.stream() - .filter(column -> !headerMap.containsKey(column)) - .toList(); - if (!missing.isEmpty()) { - throw new BusinessException("缺少列:" + String.join("、", missing)); - } - - List selectedIndexes = new ArrayList<>(selectedColumns.size()); - for (String selectedColumn : selectedColumns) { - selectedIndexes.add(headerMap.get(selectedColumn)); - } - Integer idColumnIndex = headerMap.get("id"); - Integer asinColumnIndex = headerMap.get("ASIN"); - Integer shippingTypeColumnIndex = headerMap.get("发货类型"); - - int scannedRows = 0; - int filteredFbaRows = 0; - for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { - continue; - } - scannedRows++; - if (shippingTypeColumnIndex != null) { - String shippingType = normalizeCellText(formatter.formatCellValue(row.getCell(shippingTypeColumnIndex))); - if ("FBA".equalsIgnoreCase(shippingType)) { - filteredFbaRows++; - continue; - } - } - if (idColumnIndex == null) { - rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); - continue; - } - appendRowByIdRule( - normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))), - row, - selectedIndexes, - asinColumnIndex, - formatter, - keepIntegerIds, - keepUnderscoreIds, - keepIntegerMainIdsWhenNoSubIds, - pendingMainIdGroup, - rows - ); - } - - pendingMainIdGroup.flush(rows); - return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows, filteredFbaRows); + private DedupeCandidateRow buildCandidateRow(Map rowMap, Map headerIndex, + List selectedColumns, Integer asinColumnIndex) { + List selectedValues = new ArrayList<>(selectedColumns.size()); + for (String selectedColumn : selectedColumns) { + selectedValues.add(normalizeCellText(cellText(rowMap, headerIndex.get(selectedColumn)))); } - } - - private Map buildHeaderMap(Row headerRow, DataFormatter formatter) { - Map headerMap = new HashMap<>(); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeHeaderText(cell == null ? null : formatter.formatCellValue(cell)); - if (value.isBlank()) { - continue; - } - if (headerMap.containsKey(value)) { - continue; - } - headerMap.put(value, i); - if (HEADER_STOP_COLUMN.equals(value)) { - break; - } - } - return headerMap; - } - - private String normalizeHeaderText(String value) { - String normalized = normalizeCellText(value); - int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER); - if (markerIndex > 0) { - String prefix = normalized.substring(0, markerIndex).trim(); - if (!prefix.isBlank()) { - return prefix; - } - } - return normalized; - } - - private DedupeCandidateRow buildCandidateRow(Row row, List selectedIndexes, Integer asinColumnIndex, DataFormatter formatter) { - List selectedValues = new ArrayList<>(selectedIndexes.size()); - for (Integer selectedIndex : selectedIndexes) { - selectedValues.add(normalizeCellText(formatter.formatCellValue(row.getCell(selectedIndex)))); + if (asinColumnIndex == null) { + asinColumnIndex = headerIndex.get("ASIN"); } String asinValue = asinColumnIndex == null ? "" - : dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex))); + : dedupeTotalDataService.normalizeComparableValueOrBlank(cellText(rowMap, asinColumnIndex)); return new DedupeCandidateRow(selectedValues, asinValue); } - private void appendRowByIdRule(String idValue, Row row, - List selectedIndexes, Integer asinColumnIndex, - DataFormatter formatter, + private void appendRowByIdRule(String idValue, Map rowMap, + Map headerIndex, List selectedColumns, boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerMainIdsWhenNoSubIds, PendingMainIdGroup pendingMainIdGroup, @@ -476,21 +601,29 @@ public class DedupeRunService { if (isUnderscoreId(idValue)) { pendingMainIdGroup.discardIfSameMainId(mainId); if (keepUnderscoreIds) { - rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); + rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null)); } return; } if (isIntegerId(idValue)) { if (keepIntegerIds) { - rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); + rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null)); return; } if (keepIntegerMainIdsWhenNoSubIds) { - pendingMainIdGroup.add(mainId, buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); + pendingMainIdGroup.add(mainId, buildCandidateRow(rowMap, headerIndex, selectedColumns, null)); } } } + private String cellText(Map rowMap, Integer columnIndex) { + if (columnIndex == null) { + return ""; + } + String value = rowMap.get(columnIndex); + return value == null ? "" : value; + } + private File packageFolderDedupeResultsAsZip(String archiveName, List archiveEntries) { File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip"); @@ -606,6 +739,18 @@ public class DedupeRunService { return mainName + "_cleaned." + (extName == null || extName.isBlank() ? "xlsx" : extName); } + private String normalizeHeaderText(String value) { + String normalized = normalizeCellText(value); + int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER); + if (markerIndex > 0) { + String prefix = normalized.substring(0, markerIndex).trim(); + if (!prefix.isBlank()) { + return prefix; + } + } + return normalized; + } + private String normalizeCellText(String value) { if (value == null) { return ""; @@ -660,7 +805,36 @@ public class DedupeRunService { return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == 0x3000 || Character.isWhitespace(ch); } - private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) { + private DedupeRunProgressVo snapshot(DedupeRunProgressVo progress) { + synchronized (progress) { + DedupeRunProgressVo copy = new DedupeRunProgressVo(); + copy.setRunId(progress.getRunId()); + copy.setStatus(progress.getStatus()); + copy.setTotal(progress.getTotal()); + copy.setProcessedCount(progress.getProcessedCount()); + copy.setSuccessCount(progress.getSuccessCount()); + copy.setFailedCount(progress.getFailedCount()); + copy.setFinished(progress.isFinished()); + copy.setError(progress.getError()); + copy.setResult(progress.getResult()); + return copy; + } + } + + private void decrementRunningCount(Long userId) { + AtomicInteger runningCount = runningTaskCountMap.get(userId); + if (runningCount != null && runningCount.decrementAndGet() <= 0) { + runningTaskCountMap.remove(userId, runningCount); + } + } + + private void cleanupExpiredProgress() { + long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS; + runCompletedAtMap.forEach((runId, completedAt) -> { + if (completedAt != null && completedAt < cutoff && runCompletedAtMap.remove(runId, completedAt)) { + runProgressMap.remove(runId); + } + }); } private long elapsedNs(long startNs) { @@ -671,7 +845,14 @@ public class DedupeRunService { return nanos / 1_000_000L; } - private record DedupeReadResult(String sheetName, List rows, int scannedRows, int filteredFbaRows) { + private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) { + } + + private static final class DedupeReadResult { + private String sheetName; + private List rows; + private AtomicInteger scannedRows; + private AtomicInteger filteredFbaRows; } private record DedupeCandidateRow(List selectedValues, String asinValue) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java index f522fe67..dd6ecb8a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java @@ -378,7 +378,7 @@ public class PermissionMenuService { .filter(id -> !protectedIds.contains(id)) .toList(); } - Set operatorEffectiveIds = ensureGrantable(operator, grantIds); + ensureGrantable(operator, grantIds); LinkedHashSet finalGrantIds = new LinkedHashSet<>(grantIds); if (!protectedIds.isEmpty()) { @@ -389,12 +389,6 @@ public class PermissionMenuService { } } } - if (operatorEffectiveIds != null) { - loadDirectColumnIds(userId).stream() - .filter(id -> normalizedType == null || scopedMenuIds.contains(id)) - .filter(id -> !operatorEffectiveIds.contains(id)) - .forEach(finalGrantIds::add); - } if (normalizedType == null) { userColumnPermissionMapper.deleteByUserId(userId); @@ -541,9 +535,10 @@ public class PermissionMenuService { } } - private Set ensureGrantable(AdminUserEntity operator, List requestedIds) { + /** Throws 403 when a non-super admin requests grants outside their own effective set. */ + private void ensureGrantable(AdminUserEntity operator, List requestedIds) { if (operator == null || isSuperAdmin(operator)) { - return null; + return; } List menus = loadMenus(null); Set effective = expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(operator.getId())), menus); @@ -553,7 +548,6 @@ public class PermissionMenuService { if (!denied.isEmpty()) { throw new BusinessException(403, "普通管理员只能分配自己已有的菜单权限"); } - return effective; } private boolean isAdmin(AdminUserEntity user) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/BrandCheckHttpConfigResolverTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/BrandCheckHttpConfigResolverTest.java new file mode 100644 index 00000000..cc2668ea --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/config/BrandCheckHttpConfigResolverTest.java @@ -0,0 +1,83 @@ +package com.nanri.aiimage.config; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * task-169:品牌检查客户端配置接入契约(plan 10,同 168 模式)。 + * 从 aiimage.http-client.* 读取;默认与 BrandCheckProperties 现状一致 + * (connect 10s / read 60s);覆盖生效;非法值钳制。 + */ +class BrandCheckHttpConfigResolverTest { + + private HttpClientProperties properties; + private BrandCheckHttpConfigResolver resolver; + + @BeforeEach + void setUp() { + properties = new HttpClientProperties(); + resolver = new BrandCheckHttpConfigResolver(properties); + } + + @Test + void timeoutFromConfigApplied() { + assertEquals(10_000L, resolver.connectTimeoutMillis()); + assertEquals(60_000L, resolver.readTimeoutMillis()); + assertEquals(90_000L, resolver.callTimeoutMillis()); + } + + @Test + void defaultKeptSameAsBrandCheckCurrent() { + // 现状基线:BrandCheckProperties connect 10s / read 60s + assertEquals(10_000L, resolver.connectTimeoutMillis()); + assertEquals(60_000L, resolver.readTimeoutMillis()); + } + + @Test + void overrideTakesEffect() { + properties.setConnectTimeoutMillis(15_000); + properties.setReadTimeoutMillis(45_000); + properties.setMaxRetries(2); + + assertEquals(15_000L, resolver.connectTimeoutMillis()); + assertEquals(45_000L, resolver.readTimeoutMillis()); + assertEquals(2, resolver.maxRetries()); + } + + @Test + void callTimeoutReflected() { + properties.setCallTimeoutMillis(120_000); + assertEquals(120_000L, resolver.callTimeoutMillis()); + } + + @Test + void retryConfigApplied() { + assertEquals(3, resolver.maxRetries()); + properties.setMaxRetries(0); + assertEquals(0, resolver.maxRetries()); + } + + @Test + void invalidTimeoutsClamped() { + properties.setConnectTimeoutMillis(-100); + assertEquals(1_000L, resolver.connectTimeoutMillis()); + properties.setReadTimeoutMillis(99_999_999L); + assertEquals(3_600_000L, resolver.readTimeoutMillis()); + } + + @Test + void resolverWired() { + assertTrue(resolver.connectTimeoutMillis() > 0); + assertTrue(resolver.readTimeoutMillis() > 0); + assertTrue(resolver.maxRetries() >= 0); + } + + @Test + void behaviorSameAsCurrent() { + assertEquals(10_000L, resolver.connectTimeoutMillis()); + assertEquals(60_000L, resolver.readTimeoutMillis()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java index 25fb8715..51368e36 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java @@ -381,7 +381,9 @@ class PermissionMenuServiceTest { } @Test - void ordinaryAdminReplacementPreservesDirectGrantsOutsideEffectiveScope() { + void ordinaryAdminReplacementRemovesDirectGrantsOutsideOwnEffectiveScope() { + // 管理员权限被回收后,其名下员工超出管理员范围的直接授权必须一并移除, + // 否则回收了管理员的菜单权限,员工权限却还在(也永远删不掉)。 PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); AdminUserMapper userMapper = mock(AdminUserMapper.class); @@ -405,10 +407,10 @@ class PermissionMenuServiceTest { ArgumentCaptor inserted = ArgumentCaptor.forClass(UserColumnPermissionEntity.class); - verify(permissionMapper, times(2)).insert(inserted.capture()); + verify(permissionMapper, times(1)).insert(inserted.capture()); assertThat(inserted.getAllValues()) .extracting(UserColumnPermissionEntity::getColumnId) - .containsExactly(1L, 2L); + .containsExactly(1L); } @Test diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index 6f2f3e26..b4cacef5 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -1,6 +1,7 @@ """ 管理员 API 蓝图:用户管理、生成历史、版本管理(后台) """ +import io import json import os import re @@ -13,6 +14,8 @@ from pathlib import Path from urllib.parse import quote import requests +from openpyxl import load_workbook +from openpyxl.utils.exceptions import InvalidFileException from requests.adapters import HTTPAdapter from flask import ( Blueprint, @@ -1690,6 +1693,268 @@ def _load_shop_data_crawl_download_rows(result_ids): conn.close() +_SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES = 256 * 1024 * 1024 +_SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT = (10, 60) + + +def _shop_data_crawl_fetch_result_bytes(row, timeout=None): + """从 Java 下载接口拉取结果文件字节流(仅内存,不落盘)。""" + url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download" + headers, params = _backend_java_internal_request() + response = _get_backend_java_session().get( + url, + params=params, + headers=headers, + stream=True, + timeout=timeout or _SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT, + ) + try: + response.raise_for_status() + total = 0 + chunks = [] + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + total += len(chunk) + if total > _SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES: + raise ValueError('结果文件过大,无法分析') + chunks.append(chunk) + return b''.join(chunks) + finally: + response.close() + + +def _shop_data_crawl_cell_text(cell): + """读取单元格文本:日期/数字等统一转字符串,None 返回空串。""" + if cell is None: + return '' + value = cell.value + if value is None: + return '' + if isinstance(value, datetime): + return value.strftime('%Y-%m-%d') + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value).strip() + + +def _shop_data_crawl_parse_workbook(workbook): + """从结果 Workbook 提取 ASIN 行:字典列表,表头按 日期/ASIN/…/价格/…/品牌 识别,跳过无 ASIN 行。""" + rows = [] + for sheet in workbook.worksheets: + header_cells = list(next(sheet.iter_rows(min_row=1, max_row=1), [])) + header = [_shop_data_crawl_cell_text(cell) for cell in header_cells] + try: + asin_col = header.index('ASIN') + except ValueError: + continue + date_col = header.index('日期') if '日期' in header else None + price_col = header.index('价格') if '价格' in header else None + brand_col = header.index('品牌') if '品牌' in header else None + for sheet_row in sheet.iter_rows(min_row=2): + asin = _shop_data_crawl_cell_text(sheet_row[asin_col]) + if not asin: + continue + rows.append({ + 'asin': asin, + 'date': _shop_data_crawl_cell_text(sheet_row[date_col]) if date_col is not None else '', + 'price': _shop_data_crawl_cell_text(sheet_row[price_col]) if price_col is not None else '', + 'brand': _shop_data_crawl_cell_text(sheet_row[brand_col]) if brand_col is not None else '', + }) + return rows + + +@admin_api.route('/shop-data-crawl/duplicate-asins') +@login_required +def shop_data_crawl_duplicate_asins(): + """按当前店铺列表筛选条件,分析跨店铺重复的 ASIN 明细。 + + 读取当前页每家店铺最新结果 Excel(经 Java 下载接口拉取),按 ASIN 聚合 + 其出现的店铺、国家与「日期/价格/品牌」细节,仅返回出现在 2 家及以上店铺 + 的 ASIN(按店铺数降序、ASIN 升序),空 pagination 参数时返回全量用于导出。 + """ + _, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks') + if not denied: + _, _, denied = _ensure_shop_data_crawl_data_access() + if denied: + return denied + try: + page = max(1, int(request.args.get('page', 1))) + page_size = min(100, max(10, int(request.args.get('page_size', 20)))) + shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip() + group_name = (request.args.get('group_name') or request.args.get('group') or '').strip() + country = (request.args.get('country') or '').strip().upper() + created_from = _parse_admin_datetime_arg('created_from') + created_to = _parse_admin_datetime_arg('created_to') + + conditions = [ + "r.module_type = 'SHOP_DATA_CRAWL'", + "t.module_type = 'SHOP_DATA_CRAWL'", + "TRIM(COALESCE(r.result_file_url, '')) <> ''", + ] + params = [] + if shop_name: + conditions.append('r.source_filename LIKE %s') + params.append('%' + shop_name + '%') + if group_name: + conditions.append( + 'EXISTS (SELECT 1 FROM biz_shop_manage sm ' + 'LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id ' + 'WHERE TRIM(COALESCE(sm.shop_name, \'\')) = ' + 'TRIM(COALESCE(r.source_filename, \'\')) ' + "AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) LIKE %s)" + ) + params.append('%' + group_name + '%') + if country: + if country not in ('DE', 'UK', 'FR', 'IT', 'ES'): + raise ValueError('不支持的国家代码: ' + country) + conditions.append( + 'JSON_CONTAINS(' + 'COALESCE(df.country_codes_json, ' + 'JSON_EXTRACT(t.request_json, \'$.countryCodes\'), ' + 'JSON_EXTRACT(t.request_json, \'$.country_codes\'), \'[]\'), %s)' + ) + params.append('"' + country + '"') + if created_from: + conditions.append('t.created_at >= %s') + params.append(created_from) + if created_to: + conditions.append('t.created_at <= %s') + params.append(created_to) + where_sql = ' AND '.join(conditions) + offset = (page - 1) * page_size + + conn = get_db() + try: + with conn.cursor() as cur: + shop_key_sql = "TRIM(COALESCE(r.source_filename, ''))" + grouped_from_sql = ( + ' FROM biz_file_result r ' + 'JOIN biz_file_task t ON t.id = r.task_id ' + 'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id ' + 'LEFT JOIN users u ON u.id = r.user_id ' + 'WHERE ' + where_sql + ) + cur.execute( + 'SELECT COUNT(*) AS total FROM (' + 'SELECT ' + shop_key_sql + ' AS shop_key' + grouped_from_sql + + ' GROUP BY ' + shop_key_sql + + ') shop_groups', + tuple(params), + ) + total = int((cur.fetchone() or {}).get('total') or 0) + + cur.execute( + 'SELECT ' + shop_key_sql + ' AS shop_name, MAX(' + + _SHOP_DATA_CRAWL_LATEST_TIME_SQL + ') AS latest_created_at' + + grouped_from_sql + + ' GROUP BY ' + shop_key_sql + + ' ORDER BY latest_created_at DESC, shop_name ASC LIMIT %s OFFSET %s', + tuple(params + [page_size, offset]), + ) + group_rows = cur.fetchall() + group_names = _shop_data_crawl_group_names(cur, group_rows) + + result_rows_by_shop = {} + selected_shop_names = [row.get('shop_name') for row in group_rows] + if selected_shop_names: + placeholders = ','.join(['%s'] * len(selected_shop_names)) + cur.execute( + 'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS + + ', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql + + ' ORDER BY ' + _SHOP_DATA_CRAWL_LATEST_TIME_SQL + + ' DESC, r.id DESC) AS shop_row_number ' + ' FROM biz_file_result r ' + 'JOIN biz_file_task t ON t.id = r.task_id ' + 'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id ' + 'LEFT JOIN users u ON u.id = r.user_id ' + 'WHERE ' + where_sql + + f' AND {shop_key_sql} IN ({placeholders})' + + ') ranked WHERE ranked.shop_row_number <= 1 ' + 'ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC', + tuple(params + selected_shop_names), + ) + for row in cur.fetchall(): + shop_key = _shop_data_crawl_shop_key(row.get('shop_name')) + result_rows_by_shop.setdefault(shop_key, []).append(row) + finally: + conn.close() + + # 逐店读取结果文件并解析(每店最多一个结果文件,共 page_size 个) + shop_items = [] + for rows in result_rows_by_shop.values(): + for row in rows: + result_id = int(row.get('result_id') or 0) + if result_id <= 0: + continue + try: + raw = _shop_data_crawl_fetch_result_bytes(row) + try: + parsed = _shop_data_crawl_parse_workbook( + load_workbook(io.BytesIO(raw), read_only=True, data_only=True)) + except (InvalidFileException, KeyError, ValueError) as exc: + current_app.logger.warning( + '[shop-data-crawl] 解析结果文件失败 result_id=%s: %s', result_id, exc) + continue + shop_items.append({ + 'shop_name': row.get('shop_name') or '未命名', + 'group_name': _shop_data_crawl_group_name(group_names, row.get('shop_name')), + 'country_codes': _shop_data_crawl_country_codes_from_json(row.get('country_codes_json')) + or _shop_data_crawl_country_codes(row.get('request_json')), + 'rows': parsed, + }) + except (requests.RequestException, ValueError) as exc: + current_app.logger.warning( + '[shop-data-crawl] 拉取结果文件失败 result_id=%s: %s', result_id, exc) + + # 按 ASIN 聚合其出现的店铺/分组/国家与行细节 + asin_occurrences = {} + for shop_item in shop_items: + shop_name = shop_item['shop_name'] or '未命名' + for row in shop_item['rows']: + asin = row['asin'].strip().upper() + if not asin: + continue + asin_occurrences.setdefault(asin, []).append({ + 'asin': asin, + 'date': row['date'], + 'price': row['price'], + 'brand': row['brand'], + 'shop_name': shop_name, + 'group_name': shop_item['group_name'], + 'country_codes': shop_item['country_codes'], + }) + + occurrences_list = [] + for asin, occurrences in asin_occurrences.items(): + shop_count = len({item['shop_name'] for item in occurrences}) + if shop_count < 2: + continue + occurrences_list.append({ + 'asin': asin, + 'shop_count': shop_count, + 'occurrences': occurrences, + }) + occurrences_list.sort(key=lambda item: (-item['shop_count'], item['asin'])) + + total_details = len(occurrences_list) + paged_details = occurrences_list[offset:offset + page_size] + + return jsonify({ + 'success': True, + 'items': paged_details, + 'total': total_details, + 'page': page, + 'page_size': page_size, + 'analyzed_shop_count': len(shop_items), + 'analyzed_result_count': len(shop_items), + }) + except ValueError as exc: + return jsonify({'success': False, 'error': str(exc)}), 400 + except Exception as exc: + return _internal_error(exc) + + def _open_shop_data_crawl_download(row): url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download" headers, params = _backend_java_internal_request() diff --git a/backend/static/admin.js b/backend/static/admin.js index ba0af075..b01e10e8 100644 --- a/backend/static/admin.js +++ b/backend/static/admin.js @@ -1722,7 +1722,8 @@ if (!rawResult || typeof rawResult !== 'object') return; var result = shopDataNormalizeResult(groupBase, rawResult); var resultId = shopDataResultId(result); - if (!resultId || !result.file_ready) return; + // 失败/进行中的任务没有结果文件也保留显示(禁用下载、可删除) + if (!resultId) return; if (resultId && group.results.some(function (existing) { return shopDataResultId(existing) === resultId; })) return; group.results.push(result); if (!group.shop_name) group.shop_name = result.shop_name || ''; @@ -1747,55 +1748,58 @@ function renderShopDataStatus(item, status) { var normalized = String(status || '-').toUpperCase(); var errorTitle = item && item.error ? ' title="' + escapeHtml(item.error) + '"' : ''; - return '' + escapeHtml(imageVideoStatusLabel(normalized)) + ''; + var cls = normalized === 'SUCCESS' || normalized === 'COMPLETED' + ? 'success' + : (normalized === 'FAILED' || normalized === 'CANCELLED' ? 'failed' : 'running'); + return '' + escapeHtml(imageVideoStatusLabel(normalized)) + ''; } - function renderShopDataTaskResult(item) { + function renderShopDataRecordRow(item) { var resultId = shopDataResultId(item); var selected = resultId > 0 && selectedShopDataResultIds.has(resultId); var status = String(item.status || item.file_status || '').toUpperCase(); var terminal = ['SUCCESS', 'FAILED', 'CANCELLED'].indexOf(status) >= 0; var countryCodes = item.country_codes != null ? item.country_codes : item.countryCodes; - var countries = countryListLabel(countryCodes); - var filename = item.output_filename || '-'; + var updatedAt = item.updated_at || item.latest_created_at || item.created_at || item.finished_at || '-'; var checkbox = ' 0 ? '' : ' disabled') + '>'; - return '
' + - '
' + - '' + - renderShopDataStatus(item, status || item.file_status) + - '
' + - '
' + - '
' + escapeHtml(countries) + '
' + - '
' + escapeHtml(filename) + '
' + - '
' + - '
' + - '' + - '' + - '
' + - '
'; + return '' + + '' + checkbox + '' + + '' + escapeHtml(item.shop_name || '-') + '' + + '' + escapeHtml(item.group_name || '-') + '' + + '' + escapeHtml(countryListLabel(countryCodes)) + '' + + '' + renderShopDataStatus(item, status || item.file_status) + '' + + '' + escapeHtml(updatedAt) + '' + + '' + + '' + + '' + + '' + + ''; } - function renderShopDataTaskCard(group) { - var results = Array.isArray(group.results) ? group.results : []; - var latest = group.latest_created_at || (results[0] && (results[0].created_at || results[0].finished_at)) || '-'; - return '
' + - '
' + - '
' + - '
' + escapeHtml(group.shop_name || '-') + '
' + - '' + results.length + '/1 份当日累计文件' + - '
' + - '
' + - '
' + escapeHtml(group.group_name || '-') + '
' + - '
' + escapeHtml(latest) + '
' + - '
' + - '
' + - (results.length ? results.map(renderShopDataTaskResult).join('') : '
暂无结果
') + - '
' + - '
' + - '
'; + function renderShopDataRecordTable() { + var rows = shopDataTasks.map(renderShopDataRecordRow).join(''); + if (!shopDataTasks.length) { + return '
暂无符合条件的店铺数据任务
'; + } + return '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + rows + '' + + '
店铺分组国家状态更新时间操作
'; + } + + function renderShopDataTasks() { + var grid = document.getElementById('shopDataTaskGrid'); + grid.innerHTML = renderShopDataRecordTable(); + syncShopDataSelectionUi(); } function syncShopDataSelectionUi() { @@ -1811,9 +1815,11 @@ return selectedShopDataResultIds.has(shopDataResultId(item)); }).length; var selectAll = document.getElementById('shopDataTaskSelectAll'); - selectAll.checked = selectable.length > 0 && selectedCount === selectable.length; - selectAll.indeterminate = selectedCount > 0 && selectedCount < selectable.length; - selectAll.disabled = shopDataDownloadInProgress || selectable.length === 0; + if (selectAll) { + selectAll.checked = selectable.length > 0 && selectedCount === selectable.length; + selectAll.indeterminate = selectedCount > 0 && selectedCount < selectable.length; + selectAll.disabled = shopDataDownloadInProgress || selectable.length === 0; + } var batch = document.getElementById('btnBatchDownloadShopDataTasks'); batch.disabled = shopDataDownloadInProgress || selectedCount === 0; batch.innerHTML = imageVideoDownloadIcon() + (shopDataDownloadInProgress @@ -1821,14 +1827,6 @@ : '批量下载' + (selectedCount ? ' (' + selectedCount + ')' : '')); } - function renderShopDataTasks() { - var grid = document.getElementById('shopDataTaskGrid'); - grid.innerHTML = shopDataTaskGroups.length - ? shopDataTaskGroups.map(renderShopDataTaskCard).join('') - : '
暂无符合条件的店铺数据任务
'; - syncShopDataSelectionUi(); - } - function loadShopDataCrawlTasks(page) { shopDataTaskPage = page || 1; selectedShopDataResultIds.clear(); @@ -1847,7 +1845,7 @@ var total = payload.total != null ? Number(payload.total) : shopDataTaskGroups.length; var responsePage = payload.page || page; var responsePageSize = payload.page_size || shopDataTaskPageSize; - document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留 1 份当日累计文件'; + document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺'; renderShopDataTasks(); renderPagination('shopDataTaskPagination', total, responsePage, responsePageSize, loadShopDataCrawlTasks); }) @@ -1860,6 +1858,131 @@ }); } + // ========== 重复 ASIN 分析 ========== + var shopDataDuplicatePage = 1, shopDataDuplicatePageSize = 10; + var shopDataDuplicateItems = []; + var shopDataDuplicateTotal = 0; + var shopDataDuplicateAnalyzed = { shopCount: 0, resultCount: 0 }; + var shopDataDuplicateLoading = false; + + function buildShopDataDuplicateQuery(page) { + var params = new URLSearchParams(); + params.set('page', String(page || 1)); + params.set('page_size', String(shopDataDuplicatePageSize)); + var values = { + shop_name: document.getElementById('shopDataTaskFilterShop').value.trim(), + group_name: document.getElementById('shopDataTaskFilterGroup').value.trim(), + country: document.getElementById('shopDataTaskFilterCountry').value.trim(), + created_from: document.getElementById('shopDataTaskFilterFrom').value, + created_to: document.getElementById('shopDataTaskFilterTo').value + }; + Object.keys(values).forEach(function (key) { + if (values[key]) params.set(key, values[key]); + }); + return params.toString(); + } + + function shopDataDuplicateHeader(header) { + return '' + header.map(function (col) { + return '' + col.label + ''; + }).join('') + ''; + } + + function renderShopDataDuplicateCard(item) { + var occurrences = Array.isArray(item.occurrences) ? item.occurrences : []; + var brand = ''; + occurrences.forEach(function (occ) { if (occ.brand && !brand) brand = occ.brand; }); + var rows = occurrences.map(function (occ) { + var countries = countryListLabel(occ.country_codes); + return '' + + '' + escapeHtml(occ.shop_name || '-') + '' + + '' + escapeHtml(occ.group_name || '-') + '' + + '' + escapeHtml(countries) + '' + + '' + escapeHtml(occ.date || '-') + '' + + '' + escapeHtml(occ.price || '-') + '' + + ''; + }).join(''); + return '
' + + '
' + + '' + escapeHtml(item.asin) + '' + + '' + item.shop_count + ' 家店铺' + + '' + escapeHtml(brand || '') + '' + + '
' + + '' + + shopDataDuplicateHeader([ + { label: '店铺', width: '18%' }, + { label: '分组', width: '14%' }, + { label: '国家', width: '16%' }, + { label: '日期', width: '14%' }, + { label: '价格', width: '12%' } + ]) + + '' + rows + '' + + '
' + + '
'; + } + + function renderShopDataDuplicateList() { + var list = document.getElementById('shopDataDuplicateList'); + if (!shopDataDuplicateItems.length) { + list.innerHTML = '
暂无重复 ASIN。请在左侧筛选条件后点击"查询"生效范围,再点击"重新分析"。
'; + return; + } + list.innerHTML = shopDataDuplicateItems.map(renderShopDataDuplicateCard).join(''); + } + + function loadShopDataDuplicateAsins(page) { + if (shopDataDuplicateLoading) return; + shopDataDuplicatePage = page || 1; + shopDataDuplicateLoading = true; + var list = document.getElementById('shopDataDuplicateList'); + var progress = document.getElementById('shopDataDuplicateProgress'); + var button = document.getElementById('btnRefreshShopDataDuplicates'); + progress.textContent = '正在读取各店铺结果文件并分析,请稍候...'; + button.disabled = true; + list.innerHTML = '
分析中...
'; + fetch('/api/admin/shop-data-crawl/duplicate-asins?' + buildShopDataDuplicateQuery(shopDataDuplicatePage)) + .then(function (response) { return response.json(); }) + .then(function (res) { + if (!res.success) throw new Error(res.error || '分析失败'); + shopDataDuplicateItems = res.items || []; + shopDataDuplicateTotal = Number(res.total) || 0; + shopDataDuplicateAnalyzed.shopCount = Number(res.analyzed_shop_count) || 0; + shopDataDuplicateAnalyzed.resultCount = Number(res.analyzed_result_count) || 0; + var totalEl = document.getElementById('shopDataDuplicateTotal'); + totalEl.textContent = '共 ' + shopDataDuplicateTotal + ' 个重复 ASIN · 已分析 ' + shopDataDuplicateAnalyzed.shopCount + ' 家店铺'; + renderShopDataDuplicateList(); + renderPagination('shopDataDuplicatePagination', shopDataDuplicateTotal, shopDataDuplicatePage, shopDataDuplicatePageSize, loadShopDataDuplicateAsins); + }) + .catch(function (error) { + shopDataDuplicateItems = []; + shopDataDuplicateTotal = 0; + list.innerHTML = '
分析失败:' + escapeHtml(error.message || '') + '
'; + document.getElementById('shopDataDuplicateTotal').textContent = ''; + }) + .finally(function () { + shopDataDuplicateLoading = false; + progress.textContent = ''; + button.disabled = false; + }); + } + + function switchShopDataSubTab(view) { + var recordsView = document.getElementById('shopDataRecordsView'); + var duplicatesView = document.getElementById('shopDataDuplicatesView'); + var recordsTab = document.getElementById('shopDataSubTabRecords'); + var duplicatesTab = document.getElementById('shopDataSubTabDuplicates'); + var recordsActive = view === 'records'; + recordsView.style.display = recordsActive ? '' : 'none'; + duplicatesView.style.display = recordsActive ? 'none' : ''; + recordsTab.classList.toggle('active', recordsActive); + recordsTab.setAttribute('aria-selected', recordsActive ? 'true' : 'false'); + duplicatesTab.classList.toggle('active', !recordsActive); + duplicatesTab.setAttribute('aria-selected', !recordsActive ? 'true' : 'false'); + if (!recordsActive) { + loadShopDataDuplicateAsins(1); + } + } + function downloadShopDataTask(item) { var resultId = shopDataResultId(item); if (!item || !item.file_ready || !resultId) return; @@ -2037,12 +2160,23 @@ }); } - document.getElementById('btnFilterShopDataTasks').onclick = function () { loadShopDataCrawlTasks(1); }; + document.getElementById('btnFilterShopDataTasks').onclick = function () { + loadShopDataCrawlTasks(1); + if (document.getElementById('shopDataDuplicatesView').style.display !== 'none') { + loadShopDataDuplicateAsins(1); + } + }; document.getElementById('btnResetShopDataTasks').onclick = function () { ['shopDataTaskFilterShop', 'shopDataTaskFilterGroup', 'shopDataTaskFilterCountry', 'shopDataTaskFilterFrom', 'shopDataTaskFilterTo'] .forEach(function (id) { document.getElementById(id).value = ''; }); loadShopDataCrawlTasks(1); + if (document.getElementById('shopDataDuplicatesView').style.display !== 'none') { + loadShopDataDuplicateAsins(1); + } }; + document.getElementById('shopDataSubTabRecords').onclick = function () { switchShopDataSubTab('records'); }; + document.getElementById('shopDataSubTabDuplicates').onclick = function () { switchShopDataSubTab('duplicates'); }; + document.getElementById('btnRefreshShopDataDuplicates').onclick = function () { loadShopDataDuplicateAsins(1); }; document.getElementById('shopDataTaskSelectAll').onchange = function (event) { shopDataTasks.forEach(function (item) { var resultId = shopDataResultId(item); diff --git a/backend/tests/test_admin_shop_data_duplicate_asins.py b/backend/tests/test_admin_shop_data_duplicate_asins.py new file mode 100644 index 00000000..82d865ac --- /dev/null +++ b/backend/tests/test_admin_shop_data_duplicate_asins.py @@ -0,0 +1,178 @@ +"""重复 ASIN 分析接口单元测试:模拟数据库行与结果文件,验证跨店铺重复聚合逻辑。""" +import sys +import unittest +from datetime import datetime +from io import BytesIO +from pathlib import Path +from unittest.mock import patch + +from openpyxl import Workbook +from flask import Flask + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from blueprints import admin_api + + +def _make_workbook(rows_by_sheet): + """构造结果 Workbook:rows_by_sheet = {sheet名: [(日期, ASIN, 价格, 品牌), ...]}""" + wb = Workbook() + wb.remove(wb.active) + for sheet_name, rows in rows_by_sheet.items(): + ws = wb.create_sheet(sheet_name) + ws.append(['日期', 'ASIN', '商品图片', '库存销量', '销售排名', + '页面浏览量', '售出件数', '价格', '推荐报价', '品牌']) + for date, asin, price, brand in rows: + row = [date, asin, '', '', '', '', '', price, '', brand] + ws.append(row) + return wb + + +class ShopDataDuplicateAsinTest(unittest.TestCase): + def setUp(self): + self.app = Flask(__name__) + self.group_rows = [ + {'shop_name': 'Shop A', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 1)}, + {'shop_name': 'Shop B', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 2)}, + {'shop_name': 'Shop C', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 3)}, + ] + # 每家店一个结果文件:Shop A 与 Shop B 共享 ASIN1;Shop C 单独 ASIN3 + self.shop_files = { + 'Shop A': _make_workbook({ + '英国': [('2026-08-30', 'B0ABC111', 'GBP 9.99', 'BrandA'), + ('2026-08-30', 'B0UNIQUE1', 'GBP 5.00', 'BrandA')], + '德国': [('2026-08-30', 'B0ABC111', 'EUR 10.99', 'BrandA')], + }), + 'Shop B': _make_workbook({ + '英国': [('2026-08-31', 'B0ABC111', 'GBP 8.50', 'BrandA'), + ('2026-08-31', 'B0ABC222', 'GBP 12.00', 'BrandB')], + }), + 'Shop C': _make_workbook({ + '法国': [('2026-08-29', 'B0ABC333', 'EUR 7.50', 'BrandC')], + }), + } + + def _result_row(self, result_id, shop_name, country_codes_json=None): + return { + 'result_id': result_id, + 'task_id': result_id + 100, + 'user_id': 7, + 'shop_name': shop_name, + 'shop_id': shop_name.lower(), + 'task_no': f'task-{result_id}', + 'task_status': 'SUCCESS', + 'result_success': 1, + 'result_error': None, + 'task_error': None, + 'file_error': None, + 'result_file_url': f'object-{result_id}', + 'result_filename': f'result-{result_id}.xlsx', + 'result_file_size': 10, + 'row_count': 2, + 'request_json': '{}', + 'country_codes_json': country_codes_json, + 'created_at': '2026-08-31T01:15:00', + 'updated_at': '2026-08-31T01:15:00', + 'finished_at': '2026-08-31T01:15:00', + 'latest_file_updated_at': '2026-08-31T01:15:00', + 'file_job_id': None, + 'file_status': 'SUCCESS', + 'username': 'operator', + } + + class _FakeCursor: + def __init__(self, group_rows, result_rows): + self.group_rows = group_rows + self.result_rows = result_rows + self.kind = None + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def execute(self, sql, params=()): + if 'COUNT(*) AS total' in sql: + self.kind = 'count' + elif 'AS latest_created_at' in sql and 'GROUP BY' in sql: + self.kind = 'groups' + elif 'GROUP_CONCAT' in sql: + self.kind = 'group_names' + else: + self.kind = 'results' + + def fetchone(self): + return {'total': len(self.group_rows)} + + def fetchall(self): + if self.kind == 'groups': + return self.group_rows + if self.kind == 'group_names': + return [{'shop_name': row['shop_name'], 'group_name': 'Group-' + row['shop_name']} + for row in self.group_rows] + return self.result_rows + + class _FakeConnection: + def __init__(self, cursor): + self.cursor_value = cursor + + def cursor(self): + return self.cursor_value + + def close(self): + pass + + def _make_workbook_bytes(self, shop_name): + stream = BytesIO() + self.shop_files[shop_name].save(stream) + return stream.getvalue() + + def _run_request(self, query=''): + cursor = self._FakeCursor( + self.group_rows, + [self._result_row(i + 1, row['shop_name'], '["UK","DE"]') for i, row in enumerate(self.group_rows)], + ) + connection = self._FakeConnection(cursor) + with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-asins?' + query): + with patch.object(admin_api, 'get_db', return_value=connection), \ + patch.object(admin_api, '_ensure_backend_menu_access', return_value=(None, None, None)), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=(None, None, None)), \ + patch.object(admin_api, '_shop_data_crawl_fetch_result_bytes', + side_effect=lambda row: self._make_workbook_bytes(row['shop_name'])): + return admin_api.shop_data_crawl_duplicate_asins.__wrapped__() + + def test_detects_duplicate_asins_across_shops(self): + response = self._run_request('page=1&page_size=10') + self.assertEqual(response.status_code, 200) + body = response.get_json() + self.assertTrue(body['success']) + self.assertEqual(body['total'], 1) # 只有 B0ABC111 跨店重复 + self.assertEqual(body['analyzed_shop_count'], 3) + item = body['items'][0] + self.assertEqual(item['asin'], 'B0ABC111') + self.assertEqual(item['shop_count'], 2) + shops = {occ['shop_name'] for occ in item['occurrences']} + self.assertEqual(shops, {'Shop A', 'Shop B'}) + # 国家与日期从行/表头正确映射 + shop_a = next(occ for occ in item['occurrences'] if occ['shop_name'] == 'Shop A') + self.assertIn('UK', shop_a['country_codes']) + self.assertEqual(shop_a['date'], '2026-08-30') + + def test_pagination_when_page_out_of_range(self): + response = self._run_request('page=2&page_size=10') + body = response.get_json() + self.assertEqual(body['total'], 1) + self.assertEqual(body['items'], []) + + def test_parse_workbook_skips_unknown_sheets(self): + wb = _make_workbook({'英国': [('2026-08-30', 'B0TEST01', 'GBP 1.00', '')]}) + # 手工追加一个无标准表头的 sheet,模拟未知表 + ws = wb.create_sheet('未知表') + ws.append(['随便', '某列']) + ws.append(['2026-08-30', 'B0NOHEADER']) + rows = admin_api._shop_data_crawl_parse_workbook(wb) + self.assertEqual([row['asin'] for row in rows], ['B0TEST01']) + + +if __name__ == '__main__': + unittest.main() diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 238a2339..3325d309 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -841,6 +841,244 @@ box-shadow: 0 0 0 3px rgba(99, 102, 241, 0.12); } + /* ===== 店铺数据记录 ===== */ + .shop-data-sub-tabs { + display: inline-flex; + gap: 6px; + padding: 4px; + margin-bottom: 18px; + background: #eef0f6; + border-radius: 10px; + } + + .shop-data-sub-tab { + padding: 8px 22px; + border: 0; + border-radius: 8px; + background: transparent; + color: var(--c-text-2); + font: inherit; + font-size: 14px; + cursor: pointer; + transition: color 0.15s ease, background 0.15s ease, box-shadow 0.15s ease; + } + + .shop-data-sub-tab:hover { + color: var(--c-text); + } + + .shop-data-sub-tab.active { + background: #fff; + color: var(--c-primary); + font-weight: 700; + box-shadow: var(--shadow-card); + } + + .shop-data-record-table-scroll { + margin-top: 4px; + max-height: none; + overflow-x: auto; + } + + .shop-data-record-table-scroll table { + width: 100% !important; + table-layout: fixed; + } + + .shop-data-record-table-scroll th, + .shop-data-record-table-scroll td { + vertical-align: middle; + text-align: left; + padding: 10px 12px; + border-bottom: 1px solid var(--c-border); + } + + .shop-data-record-table-scroll td { + color: var(--c-text); + } + + .shop-data-record-table-scroll td.muted { + color: var(--c-text-2); + } + + .shop-data-record-table-scroll .shop-data-status { + display: inline-block; + min-width: 62px; + padding: 3px 10px; + border-radius: 999px; + font-size: 12px; + text-align: center; + white-space: nowrap; + } + + .shop-data-record-table-scroll .shop-data-status.success { + color: var(--c-success); + background: #e6f6ec; + } + + .shop-data-record-table-scroll .shop-data-status.failed { + color: var(--c-danger); + background: var(--c-danger-soft); + } + + .shop-data-record-table-scroll .shop-data-status.running { + color: var(--c-warning); + background: #fdf0dc; + } + + .shop-data-record-action { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 5px 10px; + border: 1px solid var(--c-border); + border-radius: var(--radius-sm); + background: #fff; + color: var(--c-text); + font: inherit; + font-size: 12.5px; + cursor: pointer; + transition: color 0.15s ease, border-color 0.15s ease, background 0.15s ease; + } + + .shop-data-record-action:hover:not(:disabled) { + color: var(--c-primary); + border-color: #b9bcf3; + background: var(--c-primary-soft); + } + + .shop-data-record-action.danger:hover:not(:disabled) { + color: var(--c-danger); + border-color: #f3b4b6; + background: var(--c-danger-soft); + } + + .shop-data-record-action:disabled { + opacity: 0.5; + cursor: not-allowed; + } + + .shop-data-record-action svg { + width: 14px; + height: 14px; + fill: none; + stroke: currentColor; + stroke-width: 2; + stroke-linecap: round; + stroke-linejoin: round; + } + + /* ===== 重复 ASIN 列表 ===== */ + .duplicate-asin-list { + display: flex; + flex-direction: column; + gap: 14px; + } + + .duplicate-asin-card { + min-width: 0; + background: #fff; + border: 1px solid var(--c-border); + border-radius: 10px; + box-shadow: var(--shadow-card); + overflow: hidden; + } + + .duplicate-asin-card-head { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 16px; + background: var(--c-primary-soft); + border-bottom: 1px solid var(--c-border); + } + + .duplicate-asin-card-head .dup-asin { + font-family: Consolas, Menlo, monospace; + font-size: 15px; + font-weight: 800; + color: var(--c-primary-strong); + letter-spacing: 0.02em; + } + + .duplicate-asin-card-head .dup-count { + padding: 2px 10px; + border-radius: 999px; + background: #fff; + color: var(--c-primary-strong); + font-size: 12px; + font-weight: 700; + white-space: nowrap; + } + + .duplicate-asin-card-head .dup-brand { + margin-left: auto; + color: var(--c-text-2); + font-size: 12.5px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .duplicate-asin-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; + } + + .duplicate-asin-table th, + .duplicate-asin-table td { + padding: 9px 16px; + text-align: left; + border-bottom: 1px solid var(--c-border); + } + + .duplicate-asin-table th { + background: #fbfcfe; + color: var(--c-text-2); + font-size: 12.5px; + font-weight: 700; + } + + .duplicate-asin-table tr:last-child td { + border-bottom: 0; + } + + .duplicate-asin-table td.dup-shop { + font-weight: 700; + } + + .duplicate-asin-table td.dup-country { + color: var(--c-text-2); + } + + .duplicate-asin-table .dup-date { + color: var(--c-text-2); + font-variant-numeric: tabular-nums; + } + + .shop-data-empty-hint { + padding: 28px 16px; + text-align: center; + color: var(--c-text-3); + font-size: 13px; + background: #fff; + border: 1px dashed var(--c-border); + border-radius: 10px; + } + + .shop-data-refresh-btn { + background: #fff; + border: 1px solid var(--c-border); + color: var(--c-text); + } + + .shop-data-refresh-btn:hover { + color: var(--c-primary); + border-color: #b9bcf3; + background: var(--c-primary-soft); + } + .thumb { width: 56px; height: 56px; @@ -1864,113 +2102,6 @@ background: var(--c-primary-soft); } - .shop-data-task-card .image-video-card-body { - min-height: 0; - display: flex; - flex-direction: column; - } - - .shop-data-task-card .image-video-card-info { - flex: none; - } - - .shop-data-group-head { - margin-bottom: 8px; - } - - .shop-data-group-meta { - display: flex; - align-items: center; - gap: 8px; - color: var(--c-text-2); - font-size: 12px; - white-space: nowrap; - } - - .shop-data-result-list { - margin-top: 8px; - border-top: 1px solid #eef1f6; - } - - .shop-data-result { - padding: 13px 0 12px; - border-bottom: 1px solid #eef1f6; - } - - .shop-data-result:last-child { - border-bottom: 0; - padding-bottom: 0; - } - - .shop-data-result.selected { - margin-left: -8px; - margin-right: -8px; - padding-left: 8px; - padding-right: 8px; - border-radius: 4px; - background: #f4f6fe; - } - - .shop-data-result-head { - display: flex; - align-items: center; - justify-content: space-between; - gap: 10px; - } - - .shop-data-result .shop-data-task-title { - font-size: 13px; - } - - .shop-data-result .image-video-card-info { - margin-top: 9px; - } - - .shop-data-result .image-video-card-actions { - margin-top: 11px; - } - - .shop-data-task-title { - min-width: 0; - display: flex; - align-items: center; - gap: 9px; - color: var(--c-text); - font-size: 15px; - font-weight: 600; - cursor: pointer; - } - - .shop-data-task-title input { - flex: 0 0 auto; - width: 17px; - height: 17px; - accent-color: #6366f1; - } - - .shop-data-task-title span { - min-width: 0; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - } - - .shop-data-result .image-video-status[title] { - cursor: help; - } - - .shop-data-delete-action { - margin-left: auto; - border-color: #f0b2b4; - color: var(--c-danger); - } - - .shop-data-delete-action:hover:not(:disabled) { - border-color: var(--c-danger); - background: var(--c-danger); - color: #fff; - } - .image-video-permission-btn { display: none; min-height: 36px; @@ -4580,7 +4711,14 @@
-

店铺数据记录筛选

+

店铺数据记录

+
+ + +
+

筛选条件

@@ -4604,30 +4742,47 @@
-
-
-
- - - - +
+
+
+
+ + + + +
+
- +
-
+ +
+ -
@@ -5528,7 +5683,7 @@ window.__initAdminMenuCollapse(); })(); - +
@@ -148,7 +148,7 @@ import { computed, onMounted, ref } from 'vue' import { ElMessage } from 'element-plus' import BrandTopBar from './BrandTopBar.vue' import { expandBrandFolderRecursive, type BrandExpandFolderItem } from '@/shared/api/brand' -import { deleteDedupeHistory, getDedupeHistory, getDedupeResultDownloadUrl, getExcelInfo, runDedupe, type DedupeResultItem, type DedupeRunVo } from '@/shared/api/java-modules' +import { deleteDedupeHistory, getDedupeHistory, getDedupeResultDownloadUrl, getDedupeRunProgress, getExcelInfo, runDedupe, type DedupeResultItem, type DedupeRunProgressVo, type DedupeRunVo } from '@/shared/api/java-modules' import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview' import { saveUrlWithProgress } from '@/shared/utils/download-progress' import { EXCEL_EXTENSIONS, checkSelectedFiles, guardBlocked } from '@/shared/dispatch-guard' @@ -163,6 +163,7 @@ const cleanKeepIntegerIds = ref(false) const cleanKeepUnderscoreIds = ref(true) const cleanKeepIntegerMainIdsWhenNoSubIds = ref(true) const cleanRunning = ref(false) +const cleanProgressProcessed = ref(0) const cleanResultItems = ref([]) const cleanSummary = ref({ total: 0, successCount: 0, failedCount: 0, items: [] }) const cleanDisplayPaths = computed(() => cleanSelectedPaths.value.slice(0, 8)) @@ -290,7 +291,7 @@ async function submitCleanRun() { try { cleanRunning.value = true - const result = await runDedupe({ + const progress = await runDedupe({ files: cleanUploadedFiles.value.map((item) => ({ fileKey: item.fileKey, originalFilename: item.originalFilename, relativePath: item.relativePath })), selectedColumns: cleanSelectedColumns.value, keepIntegerIds: cleanKeepIntegerIds.value, @@ -301,6 +302,15 @@ async function submitCleanRun() { ? cleanArchiveName.value : undefined, }) + const result = await pollDedupeRunProgress(progress.runId, (latest) => { + cleanProgressProcessed.value = latest.processedCount + cleanSummary.value = { + total: latest.total, + successCount: latest.successCount, + failedCount: latest.failedCount, + items: cleanResultItems.value, + } + }) cleanSummary.value = result cleanResultItems.value = result.items || [] await loadCleanHistory() @@ -325,6 +335,36 @@ async function submitCleanRun() { } } +// 去重任务进度轮询:2 秒一次,10 分钟超时(超时任务由后端继续执行,结果可在历史列表中查看) +const DEDUPE_POLL_INTERVAL_MS = 2000 +const DEDUPE_POLL_TIMEOUT_MS = 10 * 60 * 1000 + +async function pollDedupeRunProgress(runId: string, onProgress?: (progress: DedupeRunProgressVo) => void): Promise { + const deadline = Date.now() + DEDUPE_POLL_TIMEOUT_MS + while (true) { + const progress = await getDedupeRunProgress(runId) + if (progress.status === 'not_found') { + throw new Error('去重任务不存在或已过期') + } + if (progress.status !== 'running') { + if (progress.status === 'failed') { + throw new Error(progress.error || '去重任务执行失败') + } + if (!progress.result) { + throw new Error(progress.error || '去重任务未返回结果') + } + return progress.result + } + onProgress?.(progress) + if (Date.now() > deadline) { + throw new Error( + `去重任务仍在处理中(已处理 ${progress.processedCount}/${progress.total} 个文件),请稍后在历史列表中查看结果`, + ) + } + await new Promise((resolve) => setTimeout(resolve, DEDUPE_POLL_INTERVAL_MS)) + } +} + async function loadCleanHistory() { try { const response = await getDedupeHistory() diff --git a/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue b/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue index e0bda516..4018f2e9 100644 --- a/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue @@ -7,7 +7,7 @@
店铺输入
- 左侧负责录入店铺并加入备选区,点击“匹配店铺”只会生成匹配结果;确认后需手动点击“推送到 Python 队列”才会创建并执行任务。 + 左侧负责录入店铺并加入备选区,点击“匹配店铺”只会生成匹配结果;确认后需手动点击“推送到 Python 队列”才会创建并执行任务。任务只会跑下方勾选的国家。
+ +
@@ -363,7 +369,7 @@ + + diff --git a/frontend-vue/src/shared/country-options.ts b/frontend-vue/src/shared/country-options.ts new file mode 100644 index 00000000..cf3295c5 --- /dev/null +++ b/frontend-vue/src/shared/country-options.ts @@ -0,0 +1,43 @@ +/** + * 各功能页共用的欧洲五国选项。 + * + * code 用于接口参数与本地存储(和店铺数据抓取 / 跟价 / 商品风险等页面的 + * country_codes 保持一致),label 用于界面展示;巡店删除的模板结构直接以 + * 中文国家名作为 key,取 label 即可。 + */ +export interface CountryOption { + code: string + label: string +} + +export const EU_COUNTRY_OPTIONS: readonly CountryOption[] = [ + { code: 'DE', label: '德国' }, + { code: 'UK', label: '英国' }, + { code: 'FR', label: '法国' }, + { code: 'IT', label: '意大利' }, + { code: 'ES', label: '西班牙' }, +] + +export const EU_COUNTRY_CODES: readonly string[] = EU_COUNTRY_OPTIONS.map((row) => row.code) + +export function countryLabel(code: string) { + return EU_COUNTRY_OPTIONS.find((row) => row.code === code)?.label || code +} + +/** + * 过滤出合法国家代码并去重,用于校验本地缓存或接口返回的顺序。 + * 结果为空时回落到 fallback,避免出现「一个国家都没有」导致任务空转。 + */ +export function sanitizeCountryCodes( + raw: unknown, + fallback: readonly string[] = EU_COUNTRY_CODES, +): string[] { + if (!Array.isArray(raw)) return [...fallback] + const valid = new Set(EU_COUNTRY_CODES) + const result: string[] = [] + for (const item of raw) { + const code = String(item ?? '').trim().toUpperCase() + if (valid.has(code) && !result.includes(code)) result.push(code) + } + return result.length ? result : [...fallback] +} diff --git a/frontend-vue/tests/modules-dedupe.test.ts b/frontend-vue/tests/modules-dedupe.test.ts index 761ebf34..f9ba58f6 100644 --- a/frontend-vue/tests/modules-dedupe.test.ts +++ b/frontend-vue/tests/modules-dedupe.test.ts @@ -3,11 +3,12 @@ import assert from 'node:assert/strict' import { http } from '../src/shared/api/http.ts' import { runDedupe, + getDedupeRunProgress, getDedupeHistory, deleteDedupeHistory, getDedupeResultDownloadUrl, type DedupeRunRequest, - type DedupeRunVo, + type DedupeRunProgressVo, } from '../src/shared/api/types/modules/dedupe.ts' function setupWindow() { @@ -26,12 +27,22 @@ function mockRequest( const okResponse = (data: unknown) => Promise.resolve({ data: { success: true, message: 'ok', data } }) +const runningProgress = (runId: string): DedupeRunProgressVo => ({ + runId, + status: 'running', + total: 2, + processedCount: 1, + successCount: 1, + failedCount: 0, + finished: false, +}) + test('test_dedupe_run_url', async (t) => { setupWindow() let captured: { url?: string } = {} mockRequest(t, (config) => { captured = config - return okResponse({ total: 1, successCount: 1, failedCount: 0, items: [] }) + return okResponse(runningProgress('run-1')) }) await runDedupe({ files: [{ fileKey: 'k1' }], selectedColumns: ['a'], keepIntegerIds: false, keepUnderscoreIds: false, keepIntegerMainIdsWhenNoSubIds: true }) assert.equal(captured.url, '/newApi/api/dedupe/run') @@ -42,7 +53,7 @@ test('test_dedupe_run_method_payload', async (t) => { let captured: { method?: string; data?: unknown } = {} mockRequest(t, (config) => { captured = config - return okResponse({ total: 0, successCount: 0, failedCount: 0, items: [] }) + return okResponse(runningProgress('run-1')) }) await runDedupe({ files: [{ fileKey: 'k1' }], selectedColumns: ['a'], keepIntegerIds: true, keepUnderscoreIds: true, keepIntegerMainIdsWhenNoSubIds: false }) assert.equal(captured.method, 'POST') @@ -56,6 +67,30 @@ test('test_dedupe_run_method_payload', async (t) => { }) }) +test('test_dedupe_run_returns_progress', async (t) => { + setupWindow() + mockRequest(t, () => okResponse(runningProgress('run-1'))) + const progress = await runDedupe({ files: [{ fileKey: 'k1' }], selectedColumns: ['a'], keepIntegerIds: false, keepUnderscoreIds: false, keepIntegerMainIdsWhenNoSubIds: true }) + assert.equal(progress.runId, 'run-1', 'run 应立即返回 runId') + assert.equal(progress.status, 'running') + assert.equal(progress.finished, false) + assert.equal(progress.result, undefined, '运行中不应携带最终结果') +}) + +test('test_dedupe_progress_url', async (t) => { + setupWindow() + let captured: { url?: string; method?: string } = {} + mockRequest(t, (config) => { + captured = config + return okResponse({ runId: 'run-1', status: 'success', total: 2, processedCount: 2, successCount: 2, failedCount: 0, finished: true, result: { total: 2, successCount: 2, failedCount: 0, items: [] } }) + }) + const progress = await getDedupeRunProgress('run-1') + assert.equal(captured.url, '/newApi/api/dedupe/run/run-1/progress') + assert.equal(captured.method, 'GET') + assert.equal(progress.status, 'success') + assert.ok(progress.result, '完成后应返回结果') +}) + test('test_dedupe_history_url', async (t) => { setupWindow() let captured: { url?: string; method?: string; params?: Record } = {} @@ -90,6 +125,7 @@ test('test_dedupe_download_url', async (t) => { test('test_dedupe_signature_unchanged', async (t) => { setupWindow() assert.equal(runDedupe.length, 1, 'runDedupe 应保持单参数') + assert.equal(getDedupeRunProgress.length, 1, 'getDedupeRunProgress 应保持单参数') assert.equal(getDedupeHistory.length, 0, 'getDedupeHistory 应保持无参') assert.equal(deleteDedupeHistory.length, 1, 'deleteDedupeHistory 应保持单参数') assert.equal(getDedupeResultDownloadUrl.length, 1, 'getDedupeResultDownloadUrl 应保持单参数') @@ -114,9 +150,9 @@ test('test_dedupe_export_compat', async (t) => { assert.equal(fromJavaModules.getDedupeResultDownloadUrl, fromDedupe.getDedupeResultDownloadUrl) }) -test('test_dedupe_unwrap', async (t) => { +test('test_dedupe_unwrap_progress', async (t) => { setupWindow() - mockRequest(t, () => okResponse({ total: 2, successCount: 1, failedCount: 1, items: [{ success: true }] })) - const result = await runDedupe({ files: [{ fileKey: 'k' }], selectedColumns: [], keepIntegerIds: false, keepUnderscoreIds: false, keepIntegerMainIdsWhenNoSubIds: false }) - assert.deepEqual(result, { total: 2, successCount: 1, failedCount: 1, items: [{ success: true }] }, '应返回解包后的 data') + mockRequest(t, () => okResponse(runningProgress('run-1'))) + const progress = await runDedupe({ files: [{ fileKey: 'k' }], selectedColumns: [], keepIntegerIds: false, keepUnderscoreIds: false, keepIntegerMainIdsWhenNoSubIds: false }) + assert.deepEqual(progress, runningProgress('run-1'), '应返回解包后的 data') })