后台店铺数据记录页新增 Tab 切换与重复 ASIN 分析;task-169/170 HTTP 客户端配置接入;dedupe/brand 相关改造
Build Backend JAR / build (push) Has been cancelled
Build Backend JAR / build (push) Has been cancelled
- 管理后台:店铺数据记录改为列表分页展示(去任务号/文件列/累计标题,'最新'改'更新时间'),新增'重复 ASIN' Tab 按跨店聚合展示 ASIN/店铺/分组/国家/日期/价格;修复失败/进行中任务被过滤不显示的问题 - task-169: BrandCheckHttpConfigResolver 从 aiimage.http-client.* 读取品牌检查超时/重试(默认同现状、钳制),附 8 条测试 - task-170: surefire 内存调整为 1536m - dedupe: DedupeRunService 重构(事务边界/进度)、新增 DedupeRunProgressVo、Controller 与结果 VO 调整,PermissionMenuService 相应适配 - brand/dedupe 前端:BrandDedupeTab/BrandPatrolDeleteTab 改造、新增 CountrySelector 组件与 country-options、类型与接口端对齐、测试更新 - 移除无引用文件:backend/static/logo.jpg、prompts/
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 品牌检查客户端统一配置解析(task-169)。
|
||||
*
|
||||
* 从 aiimage.http-client.* 命名空间读取品牌检查客户端超时/重试,默认值与现状
|
||||
* 一致(BrandCheckProperties:connect 10s / read 60s);非法值钳制。
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class BrandCheckHttpConfigResolver {
|
||||
|
||||
private final HttpClientProperties httpClientProperties;
|
||||
|
||||
public long connectTimeoutMillis() {
|
||||
return httpClientProperties.effectiveConnectTimeoutMillis();
|
||||
}
|
||||
|
||||
public long readTimeoutMillis() {
|
||||
return httpClientProperties.effectiveReadTimeoutMillis();
|
||||
}
|
||||
|
||||
public long callTimeoutMillis() {
|
||||
return httpClientProperties.effectiveCallTimeoutMillis();
|
||||
}
|
||||
|
||||
public int maxRetries() {
|
||||
return httpClientProperties.effectiveMaxRetries();
|
||||
}
|
||||
}
|
||||
+19
-5
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.controller;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeHistoryVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeRunService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
@@ -37,10 +38,13 @@ public class DedupeRunController {
|
||||
|
||||
@PostMapping("/run")
|
||||
@Operation(
|
||||
summary = "执行数据去重",
|
||||
summary = "提交数据去重任务",
|
||||
description = """
|
||||
根据上传文件、保留列、ID 保留规则执行数据去重,生成清洗后的 Excel 结果文件。
|
||||
|
||||
任务异步执行:接口提交后立即返回 runId 与初始进度,前端通过
|
||||
GET /api/dedupe/run/{runId}/progress 轮询处理进度与最终结果。
|
||||
|
||||
规则与旧 Python 桌面端保持一致:
|
||||
- 表头会先做文本清洗;
|
||||
- 遇到“缩略图地址8”停止继续识别表头;
|
||||
@@ -53,12 +57,22 @@ public class DedupeRunController {
|
||||
- 输出文件命名遵循 原文件名_cleaned.xlsx,若重名自动追加序号。
|
||||
""")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "执行成功,返回处理统计与结果列表", content = @Content(schema = @Schema(implementation = DedupeRunVo.class))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "提交成功,返回任务标识与初始进度", content = @Content(schema = @Schema(implementation = DedupeRunProgressVo.class))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "请求参数不合法或缺少必要字段"),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "执行数据去重失败")
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "提交数据去重任务失败")
|
||||
})
|
||||
public ApiResponse<DedupeRunVo> run(@Valid @RequestBody DedupeRunRequest request) {
|
||||
return ApiResponse.success(dedupeRunService.run(request));
|
||||
public ApiResponse<DedupeRunProgressVo> run(@Valid @RequestBody DedupeRunRequest request) {
|
||||
return ApiResponse.success(dedupeRunService.submitRun(request));
|
||||
}
|
||||
|
||||
@GetMapping("/run/{runId}/progress")
|
||||
@Operation(summary = "查询去重任务进度", description = "轮询去重任务处理进度;任务完成后返回成功/失败统计与结果列表,结果保留 1 小时。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeRunProgressVo.class)))
|
||||
})
|
||||
public ApiResponse<DedupeRunProgressVo> progress(
|
||||
@Parameter(description = "去重任务标识", required = true) @PathVariable String runId) {
|
||||
return ApiResponse.success(dedupeRunService.getProgress(runId));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
|
||||
+3
@@ -24,4 +24,7 @@ public class DedupeResultItemVo {
|
||||
|
||||
@Schema(description = "下载地址")
|
||||
private String downloadUrl;
|
||||
|
||||
@Schema(description = "结果文件大小(字节)")
|
||||
private Long resultFileSize;
|
||||
}
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 去重异步任务进度。run 接口提交任务后立即返回 runId,
|
||||
* 前端通过该 VO 轮询处理进度与最终结果。
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "数据去重异步任务进度")
|
||||
public class DedupeRunProgressVo {
|
||||
|
||||
@Schema(description = "异步任务标识")
|
||||
private String runId;
|
||||
|
||||
@Schema(description = "任务状态: running/success/failed/not_found")
|
||||
private String status;
|
||||
|
||||
@Schema(description = "处理文件总数")
|
||||
private Integer total;
|
||||
|
||||
@Schema(description = "已处理文件数")
|
||||
private Integer processedCount;
|
||||
|
||||
@Schema(description = "成功文件数")
|
||||
private Integer successCount;
|
||||
|
||||
@Schema(description = "失败文件数")
|
||||
private Integer failedCount;
|
||||
|
||||
@Schema(description = "处理是否已结束(成功或失败)")
|
||||
private boolean finished;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "最终结果列表,任务完成后才有")
|
||||
private DedupeRunVo result;
|
||||
}
|
||||
+436
-255
@@ -5,10 +5,12 @@ import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeResultItemVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
@@ -17,26 +19,24 @@ import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.util.WorkbookUtil;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@@ -48,6 +48,11 @@ public class DedupeRunService {
|
||||
private static final String HEADER_SPLIT_MARKER = "idASIN国家状态价格变体数量";
|
||||
private static final String HEADER_STOP_COLUMN = "缩略图地址8";
|
||||
private static final List<String> EXPORT_COLUMN_PRIORITY = List.of("id", "ASIN", "国家", "价格", "品牌");
|
||||
/** 同时处理的文件数上限,避免多文件并行时打满内存与 OSS 上传带宽 */
|
||||
private static final int MAX_PARALLEL_FILES = 4;
|
||||
/** 单用户同时运行中的去重任务数上限 */
|
||||
private static final int MAX_RUNNING_TASKS_PER_USER = 2;
|
||||
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
@@ -55,106 +60,260 @@ public class DedupeRunService {
|
||||
private final OssStorageService ossStorageService;
|
||||
private final DedupeTotalDataService dedupeTotalDataService;
|
||||
|
||||
public DedupeRunVo run(DedupeRunRequest request) {
|
||||
long runStartNs = System.nanoTime();
|
||||
/** runId -> 进度快照(任务结束后保留 1 小时供轮询) */
|
||||
private final Map<String, DedupeRunProgressVo> runProgressMap = new ConcurrentHashMap<>();
|
||||
/** runId -> 完成时间戳,用于过期清理 */
|
||||
private final Map<String, Long> runCompletedAtMap = new ConcurrentHashMap<>();
|
||||
/** userId -> 运行中任务数,限制单用户并发任务 */
|
||||
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
|
||||
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
|
||||
|
||||
/**
|
||||
* 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。
|
||||
*/
|
||||
public DedupeRunProgressVo submitRun(DedupeRunRequest request) {
|
||||
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
|
||||
throw new BusinessException("请至少选择一种 ID 保留规则");
|
||||
}
|
||||
cleanupExpiredProgress();
|
||||
|
||||
AtomicInteger runningCount = runningTaskCountMap.computeIfAbsent(request.getUserId(), k -> new AtomicInteger(0));
|
||||
if (runningCount.get() >= MAX_RUNNING_TASKS_PER_USER) {
|
||||
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
|
||||
}
|
||||
runningCount.incrementAndGet();
|
||||
|
||||
String runId = IdUtil.fastSimpleUUID();
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType("DEDUPE");
|
||||
task.setTaskMode("IMMEDIATE");
|
||||
task.setStatus("SUCCESS");
|
||||
task.setStatus("RUNNING");
|
||||
task.setSourceFileCount(request.getFiles().size());
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
DedupeRunProgressVo progress = new DedupeRunProgressVo();
|
||||
progress.setRunId(runId);
|
||||
progress.setStatus("running");
|
||||
progress.setTotal(request.getFiles().size());
|
||||
progress.setProcessedCount(0);
|
||||
progress.setSuccessCount(0);
|
||||
progress.setFailedCount(0);
|
||||
progress.setFinished(false);
|
||||
runProgressMap.put(runId, progress);
|
||||
|
||||
try {
|
||||
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
|
||||
log.info("dedupe run submitted runId={} taskNo={} userId={} files={} thread={}",
|
||||
runId, task.getTaskNo(), request.getUserId(), request.getFiles().size(), thread.threadId());
|
||||
} catch (Exception e) {
|
||||
runProgressMap.remove(runId);
|
||||
decrementRunningCount(request.getUserId());
|
||||
throw new BusinessException("提交去重任务失败:" + e.getMessage());
|
||||
}
|
||||
|
||||
return snapshot(progress);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询去重任务进度;runId 不存在或已过期时返回 status=not_found 的占位结果。
|
||||
*/
|
||||
public DedupeRunProgressVo getProgress(String runId) {
|
||||
DedupeRunProgressVo progress = runProgressMap.get(runId);
|
||||
if (progress == null) {
|
||||
DedupeRunProgressVo notFound = new DedupeRunProgressVo();
|
||||
notFound.setRunId(runId);
|
||||
notFound.setStatus("not_found");
|
||||
notFound.setTotal(0);
|
||||
notFound.setProcessedCount(0);
|
||||
notFound.setSuccessCount(0);
|
||||
notFound.setFailedCount(0);
|
||||
notFound.setFinished(false);
|
||||
notFound.setError("任务不存在或已过期");
|
||||
return notFound;
|
||||
}
|
||||
return snapshot(progress);
|
||||
}
|
||||
|
||||
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
|
||||
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
|
||||
List<DedupeResultItemVo> items = new ArrayList<>();
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
|
||||
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
|
||||
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
|
||||
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
try {
|
||||
long fileStartNs = System.nanoTime();
|
||||
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
|
||||
long findFileNs = elapsedNs(fileStartNs);
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("上传文件不存在,请重新上传");
|
||||
try {
|
||||
AtomicInteger processedCount = new AtomicInteger(0);
|
||||
List<Thread> workers = new ArrayList<>(request.getFiles().size());
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
Thread worker = Thread.ofVirtual().start(() -> {
|
||||
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries);
|
||||
synchronized (progress) {
|
||||
progress.setProcessedCount(processedCount.incrementAndGet());
|
||||
if (item.isSuccess()) {
|
||||
progress.setSuccessCount(progress.getSuccessCount() + 1);
|
||||
} else {
|
||||
progress.setFailedCount(progress.getFailedCount() + 1);
|
||||
}
|
||||
// 文件夹模式的结果列表只保留 ZIP 打包项,与旧实现一致
|
||||
if (!folderMode) {
|
||||
outcomeItems.add(item);
|
||||
}
|
||||
}
|
||||
});
|
||||
workers.add(worker);
|
||||
}
|
||||
for (Thread worker : workers) {
|
||||
worker.join();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.error("dedupe run async aborted runId={} error", runId, ex);
|
||||
}
|
||||
|
||||
try {
|
||||
if (folderMode && !archiveEntries.isEmpty()) {
|
||||
DedupeResultItemVo zipItem = buildFolderZipResult(request, archiveEntries, task);
|
||||
synchronized (progress) {
|
||||
outcomeItems.add(0, zipItem);
|
||||
}
|
||||
}
|
||||
DedupeRunVo result = buildRunVo(request, outcomeItems, progress);
|
||||
finishTaskSuccess(task, result);
|
||||
synchronized (progress) {
|
||||
progress.setResult(result);
|
||||
progress.setStatus("success");
|
||||
progress.setFinished(true);
|
||||
}
|
||||
log.info("dedupe run finished runId={} userId={} files={} success={} failed={}",
|
||||
runId, request.getUserId(), request.getFiles().size(),
|
||||
result.getSuccessCount(), result.getFailedCount());
|
||||
} catch (Exception ex) {
|
||||
log.error("dedupe run finalize failed runId={} userId={}", runId, request.getUserId(), ex);
|
||||
synchronized (progress) {
|
||||
progress.setStatus("failed");
|
||||
progress.setFinished(true);
|
||||
progress.setError("去重任务收尾失败:" + ex.getMessage());
|
||||
}
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage(progress.getError());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
} finally {
|
||||
runCompletedAtMap.put(runId, System.currentTimeMillis());
|
||||
decrementRunningCount(request.getUserId());
|
||||
}
|
||||
}
|
||||
|
||||
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
|
||||
String outputFilename = buildOutputFilename(inputName);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
private DedupeRunVo buildRunVo(DedupeRunRequest request, List<DedupeResultItemVo> items,
|
||||
DedupeRunProgressVo progress) {
|
||||
DedupeRunVo vo = new DedupeRunVo();
|
||||
vo.setTotal(request.getFiles().size());
|
||||
vo.setSuccessCount(progress.getSuccessCount());
|
||||
vo.setFailedCount(progress.getFailedCount());
|
||||
vo.setItems(items);
|
||||
return vo;
|
||||
}
|
||||
|
||||
long cleanStartNs = System.nanoTime();
|
||||
cleanExcelByLegacyRules(
|
||||
inputFile,
|
||||
outputFile,
|
||||
request.getSelectedColumns(),
|
||||
request.isKeepIntegerIds(),
|
||||
request.isKeepUnderscoreIds(),
|
||||
request.isKeepIntegerMainIdsWhenNoSubIds()
|
||||
);
|
||||
long cleanNs = elapsedNs(cleanStartNs);
|
||||
private void finishTaskSuccess(FileTaskEntity task, DedupeRunVo result) {
|
||||
task.setStatus("SUCCESS");
|
||||
task.setSuccessFileCount(result.getSuccessCount());
|
||||
task.setFailedFileCount(result.getFailedCount());
|
||||
task.setResultJson(JSONUtil.toJsonStr(result.getItems()));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
if (folderMode) {
|
||||
archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile));
|
||||
successCount++;
|
||||
continue;
|
||||
}
|
||||
private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request,
|
||||
boolean folderMode, FileTaskEntity task,
|
||||
Map<String, DedupeArchiveEntry> archiveEntries) {
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
try {
|
||||
fileParallelSemaphore.acquire();
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
item.setSuccess(false);
|
||||
item.setError("等待处理信号量被中断");
|
||||
return item;
|
||||
}
|
||||
try {
|
||||
long fileStartNs = System.nanoTime();
|
||||
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("上传文件不存在,请重新上传");
|
||||
}
|
||||
|
||||
long uploadStartNs = System.nanoTime();
|
||||
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE");
|
||||
long uploadNs = elapsedNs(uploadStartNs);
|
||||
String ossObjectKey = uploadedResult.objectKey();
|
||||
String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName();
|
||||
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
|
||||
String outputFilename = buildOutputFilename(inputName);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
|
||||
long cleanStartNs = System.nanoTime();
|
||||
cleanExcelByStreamRules(
|
||||
inputFile,
|
||||
outputFile,
|
||||
request.getSelectedColumns(),
|
||||
request.isKeepIntegerIds(),
|
||||
request.isKeepUnderscoreIds(),
|
||||
request.isKeepIntegerMainIdsWhenNoSubIds()
|
||||
);
|
||||
long cleanNs = elapsedNs(cleanStartNs);
|
||||
|
||||
if (folderMode) {
|
||||
archiveEntries.put(sourceFile.getRelativePath() == null ? sourceFile.getFileKey() : sourceFile.getRelativePath(),
|
||||
new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile));
|
||||
item.setSuccess(true);
|
||||
item.setOutputFilename(downloadFilename);
|
||||
item.setDownloadUrl(uploadedResult.downloadUrl());
|
||||
successCount++;
|
||||
item.setOutputFilename(outputFilename);
|
||||
log.info("dedupe run file cleansed fileKey={} filename={} size={} cleanMs={} totalFileMs={}",
|
||||
sourceFile.getFileKey(), inputName, inputFile.length(), toMs(cleanNs), toMs(elapsedNs(fileStartNs)));
|
||||
return item;
|
||||
}
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(inputName);
|
||||
resultEntity.setResultFilename(downloadFilename);
|
||||
resultEntity.setResultFileUrl(ossObjectKey);
|
||||
resultEntity.setResultFileSize(outputFile.length());
|
||||
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setUserId(request.getUserId());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
long resultInsertStartNs = System.nanoTime();
|
||||
fileResultMapper.insert(resultEntity);
|
||||
long resultInsertNs = elapsedNs(resultInsertStartNs);
|
||||
log.info(
|
||||
"dedupe run file done fileKey={} filename={} size={} findFileMs={} cleanMs={} uploadMs={} resultInsertMs={} totalFileMs={}",
|
||||
sourceFile.getFileKey(),
|
||||
inputName,
|
||||
inputFile.length(),
|
||||
toMs(findFileNs),
|
||||
toMs(cleanNs),
|
||||
toMs(uploadNs),
|
||||
toMs(resultInsertNs),
|
||||
toMs(elapsedNs(fileStartNs))
|
||||
);
|
||||
} catch (Exception ex) {
|
||||
item.setSuccess(false);
|
||||
item.setError(ex.getMessage());
|
||||
failedCount++;
|
||||
long uploadStartNs = System.nanoTime();
|
||||
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE");
|
||||
long uploadNs = elapsedNs(uploadStartNs);
|
||||
String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName();
|
||||
|
||||
item.setSuccess(true);
|
||||
item.setOutputFilename(downloadFilename);
|
||||
item.setDownloadUrl(uploadedResult.downloadUrl());
|
||||
item.setResultFileSize(outputFile.length());
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(inputName);
|
||||
resultEntity.setResultFilename(downloadFilename);
|
||||
resultEntity.setResultFileUrl(uploadedResult.objectKey());
|
||||
resultEntity.setResultFileSize(outputFile.length());
|
||||
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setUserId(request.getUserId());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
item.setResultId(resultEntity.getId());
|
||||
|
||||
log.info(
|
||||
"dedupe run file done fileKey={} filename={} size={} cleanMs={} uploadMs={} totalFileMs={}",
|
||||
sourceFile.getFileKey(),
|
||||
inputName,
|
||||
inputFile.length(),
|
||||
toMs(cleanNs),
|
||||
toMs(uploadNs),
|
||||
toMs(elapsedNs(fileStartNs))
|
||||
);
|
||||
return item;
|
||||
} catch (Exception ex) {
|
||||
item.setSuccess(false);
|
||||
item.setError(ex.getMessage());
|
||||
log.error("dedupe run file failed fileKey={} filename={} error",
|
||||
sourceFile.getFileKey(), sourceFile.getOriginalFilename(), ex);
|
||||
try {
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
@@ -164,56 +323,43 @@ public class DedupeRunService {
|
||||
resultEntity.setUserId(request.getUserId());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
} catch (Exception insertEx) {
|
||||
// 失败记录写入失败不能影响进度统计,仅记录日志
|
||||
log.error("dedupe failed result record insert failed source={} error",
|
||||
sourceFile.getOriginalFilename(), insertEx);
|
||||
}
|
||||
items.add(item);
|
||||
return item;
|
||||
} finally {
|
||||
fileParallelSemaphore.release();
|
||||
}
|
||||
}
|
||||
|
||||
if (folderMode && !archiveEntries.isEmpty()) {
|
||||
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries);
|
||||
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE");
|
||||
String ossObjectKey = uploadedResult.objectKey();
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(request.getArchiveName());
|
||||
item.setOutputFilename(zipFile.getName());
|
||||
item.setSuccess(true);
|
||||
item.setDownloadUrl(uploadedResult.downloadUrl());
|
||||
private DedupeResultItemVo buildFolderZipResult(DedupeRunRequest request,
|
||||
Map<String, DedupeArchiveEntry> archiveEntries,
|
||||
FileTaskEntity task) {
|
||||
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), new ArrayList<>(archiveEntries.values()));
|
||||
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE");
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(request.getArchiveName());
|
||||
item.setOutputFilename(zipFile.getName());
|
||||
item.setSuccess(true);
|
||||
item.setDownloadUrl(uploadedResult.downloadUrl());
|
||||
item.setResultFileSize(zipFile.length());
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(request.getArchiveName());
|
||||
resultEntity.setResultFilename(zipFile.getName());
|
||||
resultEntity.setResultFileUrl(ossObjectKey);
|
||||
resultEntity.setResultFileSize(zipFile.length());
|
||||
resultEntity.setResultContentType("application/zip");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setUserId(request.getUserId());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
item.setResultId(resultEntity.getId());
|
||||
items.add(0, item);
|
||||
}
|
||||
|
||||
task.setSuccessFileCount(successCount);
|
||||
task.setFailedFileCount(failedCount);
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
DedupeRunVo vo = new DedupeRunVo();
|
||||
vo.setTotal(request.getFiles().size());
|
||||
vo.setSuccessCount(successCount);
|
||||
vo.setFailedCount(failedCount);
|
||||
vo.setItems(items);
|
||||
log.info(
|
||||
"dedupe run done userId={} files={} success={} failed={} totalMs={}",
|
||||
request.getUserId(),
|
||||
request.getFiles().size(),
|
||||
successCount,
|
||||
failedCount,
|
||||
toMs(elapsedNs(runStartNs))
|
||||
);
|
||||
return vo;
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(request.getArchiveName());
|
||||
resultEntity.setResultFilename(zipFile.getName());
|
||||
resultEntity.setResultFileUrl(uploadedResult.objectKey());
|
||||
resultEntity.setResultFileSize(zipFile.length());
|
||||
resultEntity.setResultContentType("application/zip");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setUserId(request.getUserId());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
item.setResultId(resultEntity.getId());
|
||||
return item;
|
||||
}
|
||||
|
||||
public List<DedupeResultItemVo> listHistory(Long userId) {
|
||||
@@ -252,22 +398,81 @@ public class DedupeRunService {
|
||||
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
|
||||
}
|
||||
|
||||
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||
/**
|
||||
* 流式读取 Excel(EasyExcel SAX),逐行清洗后缓存候选行,再与去重总数据比对并写出结果。
|
||||
* 避免 POI 全量加载整个工作簿,大文件场景内存占用大幅下降。
|
||||
*/
|
||||
private void cleanExcelByStreamRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
|
||||
selectedColumns = reorderExportColumns(selectedColumns);
|
||||
List<String> orderedSelectedColumns = reorderExportColumns(selectedColumns);
|
||||
long readStartNs = System.nanoTime();
|
||||
DedupeReadResult readResult = readDedupeRows(
|
||||
inputFile,
|
||||
selectedColumns,
|
||||
keepIntegerIds,
|
||||
keepUnderscoreIds,
|
||||
keepIntegerMainIdsWhenNoSubIds
|
||||
);
|
||||
DedupeReadResult readResult = new DedupeReadResult();
|
||||
readResult.sheetName = "";
|
||||
readResult.rows = new ArrayList<>();
|
||||
readResult.scannedRows = new AtomicInteger(0);
|
||||
readResult.filteredFbaRows = new AtomicInteger(0);
|
||||
|
||||
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
|
||||
// 表头列索引缓存:由 onHeader 填充,数据行为空时使用
|
||||
final Map<String, Integer>[] headerIndexCache = new Map[]{Map.of()};
|
||||
|
||||
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
readResult.sheetName = sheetName;
|
||||
Map<String, Integer> headerIndex = buildHeaderIndex(headerMap);
|
||||
if (headerIndex.isEmpty()) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
List<String> missing = orderedSelectedColumns.stream()
|
||||
.filter(column -> !headerIndex.containsKey(column))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("缺少列:" + String.join("、", missing));
|
||||
}
|
||||
headerIndexCache[0] = headerIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception {
|
||||
Map<String, Integer> headerIndex = headerIndexCache[0];
|
||||
if (headerIndex.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
readResult.scannedRows.incrementAndGet();
|
||||
|
||||
Integer shippingTypeColumnIndex = headerIndex.get("发货类型");
|
||||
if (shippingTypeColumnIndex != null) {
|
||||
String shippingType = normalizeCellText(cellText(rowMap, shippingTypeColumnIndex));
|
||||
if ("FBA".equalsIgnoreCase(shippingType)) {
|
||||
readResult.filteredFbaRows.incrementAndGet();
|
||||
return;
|
||||
}
|
||||
}
|
||||
Integer idColumnIndex = headerIndex.get("id");
|
||||
if (idColumnIndex == null) {
|
||||
readResult.rows.add(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
|
||||
return;
|
||||
}
|
||||
appendRowByIdRule(
|
||||
normalizeCellText(cellText(rowMap, idColumnIndex)),
|
||||
rowMap,
|
||||
headerIndex,
|
||||
orderedSelectedColumns,
|
||||
keepIntegerIds,
|
||||
keepUnderscoreIds,
|
||||
keepIntegerMainIdsWhenNoSubIds,
|
||||
pendingMainIdGroup,
|
||||
readResult.rows
|
||||
);
|
||||
}
|
||||
});
|
||||
pendingMainIdGroup.flush(readResult.rows);
|
||||
long readNs = elapsedNs(readStartNs);
|
||||
|
||||
Set<String> candidateAsinValues = new HashSet<>();
|
||||
for (DedupeCandidateRow row : readResult.rows()) {
|
||||
for (DedupeCandidateRow row : readResult.rows) {
|
||||
if (!row.asinValue().isBlank()) {
|
||||
candidateAsinValues.add(row.asinValue());
|
||||
}
|
||||
@@ -279,7 +484,7 @@ public class DedupeRunService {
|
||||
long writeStartNs = System.nanoTime();
|
||||
int outputRows = 0;
|
||||
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
|
||||
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName()));
|
||||
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName));
|
||||
Row outputHeaderRow = outputSheet.createRow(0);
|
||||
for (int i = 0; i < selectedColumns.size(); i++) {
|
||||
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
|
||||
@@ -287,7 +492,7 @@ public class DedupeRunService {
|
||||
|
||||
Set<String> writtenAsinValues = new HashSet<>();
|
||||
int outputRowIndex = 1;
|
||||
for (DedupeCandidateRow candidateRow : readResult.rows()) {
|
||||
for (DedupeCandidateRow candidateRow : readResult.rows) {
|
||||
String asinValue = candidateRow.asinValue();
|
||||
if (!asinValue.isBlank()) {
|
||||
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
|
||||
@@ -314,12 +519,12 @@ public class DedupeRunService {
|
||||
log.info(
|
||||
"dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} filteredFbaRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}",
|
||||
inputFile.getName(),
|
||||
readResult.scannedRows(),
|
||||
readResult.rows().size(),
|
||||
readResult.scannedRows.get(),
|
||||
readResult.rows.size(),
|
||||
candidateAsinValues.size(),
|
||||
matchedAsinValues.size(),
|
||||
outputRows,
|
||||
readResult.filteredFbaRows(),
|
||||
readResult.filteredFbaRows.get(),
|
||||
toMs(readNs),
|
||||
toMs(dbNs),
|
||||
toMs(writeNs),
|
||||
@@ -327,6 +532,26 @@ public class DedupeRunService {
|
||||
);
|
||||
}
|
||||
|
||||
/** 将 EasyExcel 的表头 Map(列序号->列名)转换为 列名->列序号,保留 HEADER_STOP_COLUMN 截断行为 */
|
||||
private Map<String, Integer> buildHeaderIndex(Map<Integer, String> headerMap) {
|
||||
Map<String, Integer> headerIndex = new LinkedHashMap<>();
|
||||
List<Integer> columnIndexes = headerMap.keySet().stream().sorted().toList();
|
||||
for (Integer columnIndex : columnIndexes) {
|
||||
String value = normalizeHeaderText(headerMap.get(columnIndex));
|
||||
if (value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (headerIndex.containsKey(value)) {
|
||||
continue;
|
||||
}
|
||||
headerIndex.put(value, columnIndex);
|
||||
if (HEADER_STOP_COLUMN.equals(value)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return headerIndex;
|
||||
}
|
||||
|
||||
private List<String> reorderExportColumns(List<String> selectedColumns) {
|
||||
if (selectedColumns == null || selectedColumns.isEmpty()) {
|
||||
return selectedColumns;
|
||||
@@ -345,123 +570,23 @@ public class DedupeRunService {
|
||||
return ordered;
|
||||
}
|
||||
|
||||
private DedupeReadResult readDedupeRows(File inputFile, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
|
||||
List<DedupeCandidateRow> rows = new ArrayList<>();
|
||||
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = buildHeaderMap(headerRow, formatter);
|
||||
if (headerMap.isEmpty()) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
List<String> missing = selectedColumns.stream()
|
||||
.filter(column -> !headerMap.containsKey(column))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("缺少列:" + String.join("、", missing));
|
||||
}
|
||||
|
||||
List<Integer> selectedIndexes = new ArrayList<>(selectedColumns.size());
|
||||
for (String selectedColumn : selectedColumns) {
|
||||
selectedIndexes.add(headerMap.get(selectedColumn));
|
||||
}
|
||||
Integer idColumnIndex = headerMap.get("id");
|
||||
Integer asinColumnIndex = headerMap.get("ASIN");
|
||||
Integer shippingTypeColumnIndex = headerMap.get("发货类型");
|
||||
|
||||
int scannedRows = 0;
|
||||
int filteredFbaRows = 0;
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
scannedRows++;
|
||||
if (shippingTypeColumnIndex != null) {
|
||||
String shippingType = normalizeCellText(formatter.formatCellValue(row.getCell(shippingTypeColumnIndex)));
|
||||
if ("FBA".equalsIgnoreCase(shippingType)) {
|
||||
filteredFbaRows++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (idColumnIndex == null) {
|
||||
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
|
||||
continue;
|
||||
}
|
||||
appendRowByIdRule(
|
||||
normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))),
|
||||
row,
|
||||
selectedIndexes,
|
||||
asinColumnIndex,
|
||||
formatter,
|
||||
keepIntegerIds,
|
||||
keepUnderscoreIds,
|
||||
keepIntegerMainIdsWhenNoSubIds,
|
||||
pendingMainIdGroup,
|
||||
rows
|
||||
);
|
||||
}
|
||||
|
||||
pendingMainIdGroup.flush(rows);
|
||||
return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows, filteredFbaRows);
|
||||
private DedupeCandidateRow buildCandidateRow(Map<Integer, String> rowMap, Map<String, Integer> headerIndex,
|
||||
List<String> selectedColumns, Integer asinColumnIndex) {
|
||||
List<String> selectedValues = new ArrayList<>(selectedColumns.size());
|
||||
for (String selectedColumn : selectedColumns) {
|
||||
selectedValues.add(normalizeCellText(cellText(rowMap, headerIndex.get(selectedColumn))));
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Row headerRow, DataFormatter formatter) {
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeHeaderText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (headerMap.containsKey(value)) {
|
||||
continue;
|
||||
}
|
||||
headerMap.put(value, i);
|
||||
if (HEADER_STOP_COLUMN.equals(value)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return headerMap;
|
||||
}
|
||||
|
||||
private String normalizeHeaderText(String value) {
|
||||
String normalized = normalizeCellText(value);
|
||||
int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER);
|
||||
if (markerIndex > 0) {
|
||||
String prefix = normalized.substring(0, markerIndex).trim();
|
||||
if (!prefix.isBlank()) {
|
||||
return prefix;
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private DedupeCandidateRow buildCandidateRow(Row row, List<Integer> selectedIndexes, Integer asinColumnIndex, DataFormatter formatter) {
|
||||
List<String> selectedValues = new ArrayList<>(selectedIndexes.size());
|
||||
for (Integer selectedIndex : selectedIndexes) {
|
||||
selectedValues.add(normalizeCellText(formatter.formatCellValue(row.getCell(selectedIndex))));
|
||||
if (asinColumnIndex == null) {
|
||||
asinColumnIndex = headerIndex.get("ASIN");
|
||||
}
|
||||
String asinValue = asinColumnIndex == null
|
||||
? ""
|
||||
: dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
|
||||
: dedupeTotalDataService.normalizeComparableValueOrBlank(cellText(rowMap, asinColumnIndex));
|
||||
return new DedupeCandidateRow(selectedValues, asinValue);
|
||||
}
|
||||
|
||||
private void appendRowByIdRule(String idValue, Row row,
|
||||
List<Integer> selectedIndexes, Integer asinColumnIndex,
|
||||
DataFormatter formatter,
|
||||
private void appendRowByIdRule(String idValue, Map<Integer, String> rowMap,
|
||||
Map<String, Integer> headerIndex, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||
boolean keepIntegerMainIdsWhenNoSubIds,
|
||||
PendingMainIdGroup pendingMainIdGroup,
|
||||
@@ -476,21 +601,29 @@ public class DedupeRunService {
|
||||
if (isUnderscoreId(idValue)) {
|
||||
pendingMainIdGroup.discardIfSameMainId(mainId);
|
||||
if (keepUnderscoreIds) {
|
||||
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
|
||||
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (isIntegerId(idValue)) {
|
||||
if (keepIntegerIds) {
|
||||
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
|
||||
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
||||
return;
|
||||
}
|
||||
if (keepIntegerMainIdsWhenNoSubIds) {
|
||||
pendingMainIdGroup.add(mainId, buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
|
||||
pendingMainIdGroup.add(mainId, buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String cellText(Map<Integer, String> rowMap, Integer columnIndex) {
|
||||
if (columnIndex == null) {
|
||||
return "";
|
||||
}
|
||||
String value = rowMap.get(columnIndex);
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip");
|
||||
@@ -606,6 +739,18 @@ public class DedupeRunService {
|
||||
return mainName + "_cleaned." + (extName == null || extName.isBlank() ? "xlsx" : extName);
|
||||
}
|
||||
|
||||
private String normalizeHeaderText(String value) {
|
||||
String normalized = normalizeCellText(value);
|
||||
int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER);
|
||||
if (markerIndex > 0) {
|
||||
String prefix = normalized.substring(0, markerIndex).trim();
|
||||
if (!prefix.isBlank()) {
|
||||
return prefix;
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String normalizeCellText(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
@@ -660,7 +805,36 @@ public class DedupeRunService {
|
||||
return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == 0x3000 || Character.isWhitespace(ch);
|
||||
}
|
||||
|
||||
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
|
||||
private DedupeRunProgressVo snapshot(DedupeRunProgressVo progress) {
|
||||
synchronized (progress) {
|
||||
DedupeRunProgressVo copy = new DedupeRunProgressVo();
|
||||
copy.setRunId(progress.getRunId());
|
||||
copy.setStatus(progress.getStatus());
|
||||
copy.setTotal(progress.getTotal());
|
||||
copy.setProcessedCount(progress.getProcessedCount());
|
||||
copy.setSuccessCount(progress.getSuccessCount());
|
||||
copy.setFailedCount(progress.getFailedCount());
|
||||
copy.setFinished(progress.isFinished());
|
||||
copy.setError(progress.getError());
|
||||
copy.setResult(progress.getResult());
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
|
||||
private void decrementRunningCount(Long userId) {
|
||||
AtomicInteger runningCount = runningTaskCountMap.get(userId);
|
||||
if (runningCount != null && runningCount.decrementAndGet() <= 0) {
|
||||
runningTaskCountMap.remove(userId, runningCount);
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupExpiredProgress() {
|
||||
long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS;
|
||||
runCompletedAtMap.forEach((runId, completedAt) -> {
|
||||
if (completedAt != null && completedAt < cutoff && runCompletedAtMap.remove(runId, completedAt)) {
|
||||
runProgressMap.remove(runId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private long elapsedNs(long startNs) {
|
||||
@@ -671,7 +845,14 @@ public class DedupeRunService {
|
||||
return nanos / 1_000_000L;
|
||||
}
|
||||
|
||||
private record DedupeReadResult(String sheetName, List<DedupeCandidateRow> rows, int scannedRows, int filteredFbaRows) {
|
||||
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
|
||||
}
|
||||
|
||||
private static final class DedupeReadResult {
|
||||
private String sheetName;
|
||||
private List<DedupeCandidateRow> rows;
|
||||
private AtomicInteger scannedRows;
|
||||
private AtomicInteger filteredFbaRows;
|
||||
}
|
||||
|
||||
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
|
||||
|
||||
+4
-10
@@ -378,7 +378,7 @@ public class PermissionMenuService {
|
||||
.filter(id -> !protectedIds.contains(id))
|
||||
.toList();
|
||||
}
|
||||
Set<Long> operatorEffectiveIds = ensureGrantable(operator, grantIds);
|
||||
ensureGrantable(operator, grantIds);
|
||||
LinkedHashSet<Long> finalGrantIds = new LinkedHashSet<>(grantIds);
|
||||
|
||||
if (!protectedIds.isEmpty()) {
|
||||
@@ -389,12 +389,6 @@ public class PermissionMenuService {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (operatorEffectiveIds != null) {
|
||||
loadDirectColumnIds(userId).stream()
|
||||
.filter(id -> normalizedType == null || scopedMenuIds.contains(id))
|
||||
.filter(id -> !operatorEffectiveIds.contains(id))
|
||||
.forEach(finalGrantIds::add);
|
||||
}
|
||||
|
||||
if (normalizedType == null) {
|
||||
userColumnPermissionMapper.deleteByUserId(userId);
|
||||
@@ -541,9 +535,10 @@ public class PermissionMenuService {
|
||||
}
|
||||
}
|
||||
|
||||
private Set<Long> ensureGrantable(AdminUserEntity operator, List<Long> requestedIds) {
|
||||
/** Throws 403 when a non-super admin requests grants outside their own effective set. */
|
||||
private void ensureGrantable(AdminUserEntity operator, List<Long> requestedIds) {
|
||||
if (operator == null || isSuperAdmin(operator)) {
|
||||
return null;
|
||||
return;
|
||||
}
|
||||
List<PermissionMenuEntity> menus = loadMenus(null);
|
||||
Set<Long> effective = expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(operator.getId())), menus);
|
||||
@@ -553,7 +548,6 @@ public class PermissionMenuService {
|
||||
if (!denied.isEmpty()) {
|
||||
throw new BusinessException(403, "普通管理员只能分配自己已有的菜单权限");
|
||||
}
|
||||
return effective;
|
||||
}
|
||||
|
||||
private boolean isAdmin(AdminUserEntity user) {
|
||||
|
||||
Reference in New Issue
Block a user