完成新需求
This commit is contained in:
+74
@@ -0,0 +1,74 @@
|
||||
package com.nanri.aiimage.modules.dedupe.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeHistoryVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeRunService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/dedupe")
|
||||
@Tag(name = "数据去重", description = "按照旧 Python 桌面端规则执行列保留和 ID 过滤。桌面端交互不变,只将处理逻辑迁到 Java。")
|
||||
public class DedupeRunController {
|
||||
|
||||
private final DedupeRunService dedupeRunService;
|
||||
|
||||
@PostMapping("/run")
|
||||
@Operation(
|
||||
summary = "执行数据去重",
|
||||
description = """
|
||||
根据上传文件、保留列、ID 保留规则执行数据去重,生成清洗后的 Excel 结果文件。
|
||||
|
||||
规则与旧 Python 桌面端保持一致:
|
||||
- 表头会先做文本清洗;
|
||||
- 遇到“缩略图地址8”停止继续识别表头;
|
||||
- 仅保留 selectedColumns 中指定的列;
|
||||
- keepIntegerIds=true 时保留纯数字 ID;
|
||||
- keepUnderscoreIds=true 时保留类似 1_1 的 ID;
|
||||
- 输出文件命名遵循 原文件名_cleaned.xlsx,若重名自动追加序号。
|
||||
""")
|
||||
public ApiResponse<DedupeRunVo> run(@Valid @RequestBody DedupeRunRequest request) {
|
||||
return ApiResponse.success(dedupeRunService.run(request));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
@Operation(summary = "查询去重历史", description = "查询最近的去重成功记录,用于页面右侧历史下载列表展示。")
|
||||
public ApiResponse<DedupeHistoryVo> history() {
|
||||
DedupeHistoryVo vo = new DedupeHistoryVo();
|
||||
vo.setItems(dedupeRunService.listHistory());
|
||||
return ApiResponse.success(vo);
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除去重历史", description = "删除一条去重历史记录。")
|
||||
public ApiResponse<Void> deleteHistory(@PathVariable Long resultId) {
|
||||
dedupeRunService.deleteHistory(resultId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/results/{resultId}/download")
|
||||
@Operation(summary = "下载数据去重结果", description = "根据结果记录 ID 下载清洗后的 Excel 文件。")
|
||||
public ResponseEntity<Resource> download(@PathVariable Long resultId) {
|
||||
Resource resource = dedupeRunService.getResultFile(resultId);
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment")
|
||||
.body(resource);
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重执行请求")
|
||||
public class DedupeRunRequest {
|
||||
|
||||
@Valid
|
||||
@NotEmpty(message = "请先上传待处理文件")
|
||||
@Schema(description = "已上传源文件列表")
|
||||
private List<DedupeSourceFileDto> files;
|
||||
|
||||
@NotEmpty(message = "请至少选择一列")
|
||||
@Schema(description = "保留列")
|
||||
private List<String> selectedColumns;
|
||||
|
||||
@Schema(description = "是否保留纯数字 ID")
|
||||
private boolean keepIntegerIds = true;
|
||||
|
||||
@Schema(description = "是否保留带下划线 ID")
|
||||
private boolean keepUnderscoreIds = true;
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重源文件")
|
||||
public class DedupeSourceFileDto {
|
||||
|
||||
@NotBlank(message = "fileKey 不能为空")
|
||||
@Schema(description = "上传接口返回的临时文件键")
|
||||
private String fileKey;
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重历史列表")
|
||||
public class DedupeHistoryVo {
|
||||
|
||||
@Schema(description = "历史记录")
|
||||
private List<DedupeResultItemVo> items;
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重结果项")
|
||||
public class DedupeResultItemVo {
|
||||
|
||||
@Schema(description = "结果记录ID")
|
||||
private Long resultId;
|
||||
|
||||
@Schema(description = "源文件名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "结果文件名")
|
||||
private String outputFilename;
|
||||
|
||||
@Schema(description = "是否成功")
|
||||
private boolean success;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "下载地址")
|
||||
private String downloadUrl;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重执行结果")
|
||||
public class DedupeRunVo {
|
||||
|
||||
@Schema(description = "处理文件总数")
|
||||
private Integer total;
|
||||
|
||||
@Schema(description = "成功文件数")
|
||||
private Integer successCount;
|
||||
|
||||
@Schema(description = "失败文件数")
|
||||
private Integer failedCount;
|
||||
|
||||
@Schema(description = "结果列表")
|
||||
private List<DedupeResultItemVo> items;
|
||||
}
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
package com.nanri.aiimage.modules.dedupe.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeResultItemVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DedupeRunService {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final StorageProperties storageProperties;
|
||||
private final OssStorageService ossStorageService;
|
||||
|
||||
public DedupeRunVo run(DedupeRunRequest request) {
|
||||
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds()) {
|
||||
throw new BusinessException("请至少选择一种 ID 保留规则");
|
||||
}
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType("DEDUPE");
|
||||
task.setTaskMode("IMMEDIATE");
|
||||
task.setStatus("SUCCESS");
|
||||
task.setSourceFileCount(request.getFiles().size());
|
||||
task.setCreatedBy("system");
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
List<DedupeResultItemVo> items = new ArrayList<>();
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
try {
|
||||
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("上传文件不存在,请重新上传");
|
||||
}
|
||||
|
||||
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
|
||||
String outputFilename = buildOutputFilename(inputName);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
|
||||
cleanExcelByLegacyRules(inputFile, outputFile, request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds());
|
||||
|
||||
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE");
|
||||
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
|
||||
|
||||
item.setSuccess(true);
|
||||
item.setOutputFilename(outputFile.getName());
|
||||
item.setDownloadUrl(downloadUrl);
|
||||
successCount++;
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(inputName);
|
||||
resultEntity.setResultFilename(outputFile.getName());
|
||||
resultEntity.setResultFileUrl(downloadUrl);
|
||||
resultEntity.setResultFileSize(outputFile.length());
|
||||
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
} catch (Exception ex) {
|
||||
item.setSuccess(false);
|
||||
item.setError(ex.getMessage());
|
||||
failedCount++;
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
resultEntity.setSuccess(0);
|
||||
resultEntity.setErrorMessage(ex.getMessage());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
}
|
||||
items.add(item);
|
||||
}
|
||||
|
||||
task.setSuccessFileCount(successCount);
|
||||
task.setFailedFileCount(failedCount);
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
DedupeRunVo vo = new DedupeRunVo();
|
||||
vo.setTotal(request.getFiles().size());
|
||||
vo.setSuccessCount(successCount);
|
||||
vo.setFailedCount(failedCount);
|
||||
vo.setItems(items);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public List<DedupeResultItemVo> listHistory() {
|
||||
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, "DEDUPE")
|
||||
.eq(FileResultEntity::getSuccess, 1)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit 50"))
|
||||
.stream()
|
||||
.map(entity -> {
|
||||
DedupeResultItemVo vo = new DedupeResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl());
|
||||
vo.setSuccess(true);
|
||||
return vo;
|
||||
})
|
||||
.toList();
|
||||
}
|
||||
|
||||
public void deleteHistory(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !"DEDUPE".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
}
|
||||
|
||||
public Resource getResultFile(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
File file = new File(entity.getResultFileUrl());
|
||||
if (!file.exists()) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
return new FileSystemResource(file);
|
||||
}
|
||||
|
||||
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds) throws Exception {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
|
||||
XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
|
||||
? value
|
||||
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
|
||||
if (headerMap.containsKey(value)) {
|
||||
continue;
|
||||
}
|
||||
headerMap.put(value, i);
|
||||
if ("缩略图地址8".equals(value)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
List<String> missing = selectedColumns.stream().filter(column -> !headerMap.containsKey(column)).toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("缺少列:" + String.join("、", missing));
|
||||
}
|
||||
|
||||
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
|
||||
Row outputHeaderRow = outputSheet.createRow(0);
|
||||
for (int i = 0; i < selectedColumns.size(); i++) {
|
||||
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
|
||||
}
|
||||
|
||||
Integer idColumnIndex = headerMap.get("id");
|
||||
int outputRowIndex = 1;
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
if (idColumnIndex != null) {
|
||||
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
|
||||
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Row outputRow = outputSheet.createRow(outputRowIndex++);
|
||||
for (int i = 0; i < selectedColumns.size(); i++) {
|
||||
Integer sourceIndex = headerMap.get(selectedColumns.get(i));
|
||||
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
|
||||
outputRow.createCell(i).setCellValue(value);
|
||||
}
|
||||
}
|
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
outputWorkbook.write(fos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds) {
|
||||
if (text == null || text.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
if (keepIntegerIds && text.matches("\\d+")) {
|
||||
return true;
|
||||
}
|
||||
return keepUnderscoreIds && text.matches("\\d+_\\d+");
|
||||
}
|
||||
|
||||
private File findLocalSourceFile(String fileKey) {
|
||||
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
|
||||
if (!baseDir.exists()) {
|
||||
return null;
|
||||
}
|
||||
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
|
||||
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
|
||||
}
|
||||
|
||||
private File buildNamedOutputFile(File outputDir, String filename) {
|
||||
File candidate = FileUtil.file(outputDir, filename);
|
||||
if (!candidate.exists()) {
|
||||
return candidate;
|
||||
}
|
||||
String mainName = FileUtil.mainName(filename);
|
||||
String extName = FileUtil.extName(filename);
|
||||
int index = 2;
|
||||
while (true) {
|
||||
String nextFilename = extName == null || extName.isBlank()
|
||||
? mainName + "_" + index
|
||||
: mainName + "_" + index + "." + extName;
|
||||
File nextCandidate = FileUtil.file(outputDir, nextFilename);
|
||||
if (!nextCandidate.exists()) {
|
||||
return nextCandidate;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
}
|
||||
|
||||
private String buildOutputFilename(String originalFilename) {
|
||||
String mainName = originalFilename == null || originalFilename.isBlank() ? "source" : FileUtil.mainName(originalFilename);
|
||||
String extName = originalFilename == null || originalFilename.isBlank() ? "xlsx" : FileUtil.extName(originalFilename);
|
||||
return mainName + "_cleaned." + (extName == null || extName.isBlank() ? "xlsx" : extName);
|
||||
}
|
||||
|
||||
private String normalizeCellText(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.replace("\ufeff", "")
|
||||
.replace("\u3000", " ")
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim()
|
||||
.replaceAll("\\s+", " ");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user