更新新增三个模块
This commit is contained in:
+1
@@ -45,6 +45,7 @@ public class DedupeRunController {
|
||||
- 仅保留 selectedColumns 中指定的列;
|
||||
- keepIntegerIds=true 时保留纯数字 ID;
|
||||
- keepUnderscoreIds=true 时保留类似 1_1 的 ID;
|
||||
- keepIntegerMainIdsWhenNoSubIds=true 且文件中不存在 1_1 这类子 ID 时,会自动保留纯数字主 ID;
|
||||
- 输出文件命名遵循 原文件名_cleaned.xlsx,若重名自动追加序号。
|
||||
""")
|
||||
@ApiResponses({
|
||||
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
package com.nanri.aiimage.modules.dedupe.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Content;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/admin/dedupe-total-data")
|
||||
@Tag(name = "数据去重总数据", description = "维护数据去重模块的总数据列表,支持增删改查。")
|
||||
public class DedupeTotalDataController {
|
||||
|
||||
private final DedupeTotalDataService dedupeTotalDataService;
|
||||
|
||||
@GetMapping
|
||||
@Operation(summary = "分页查询总数据", description = "分页查询数据去重总数据,支持按值模糊搜索。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataPageVo.class)))
|
||||
})
|
||||
public ApiResponse<DedupeTotalDataPageVo> page(
|
||||
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
||||
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
|
||||
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword) {
|
||||
return ApiResponse.success(dedupeTotalDataService.page(page, pageSize, keyword));
|
||||
}
|
||||
|
||||
@PostMapping
|
||||
@Operation(summary = "新增总数据", description = "新增一条数据去重总数据。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataItemVo.class))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复")
|
||||
})
|
||||
public ApiResponse<DedupeTotalDataItemVo> create(@Valid @RequestBody DedupeTotalDataCreateRequest request) {
|
||||
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request));
|
||||
}
|
||||
|
||||
@PostMapping("/import")
|
||||
@Operation(summary = "导入总数据", description = "上传 xlsx 文件,读取 ASIN 列并批量导入数据去重总数据。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "导入成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataImportVo.class))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "文件不合法或缺少 ASIN 列")
|
||||
})
|
||||
public ApiResponse<DedupeTotalDataImportVo> importExcel(
|
||||
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file) {
|
||||
return ApiResponse.success("导入成功", dedupeTotalDataService.importFromExcel(file));
|
||||
}
|
||||
|
||||
@PutMapping("/{id}")
|
||||
@Operation(summary = "更新总数据", description = "按 ID 更新一条数据去重总数据。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "更新成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataItemVo.class))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复"),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在")
|
||||
})
|
||||
public ApiResponse<DedupeTotalDataItemVo> update(
|
||||
@Parameter(description = "主键ID", required = true) @PathVariable Long id,
|
||||
@Valid @RequestBody DedupeTotalDataUpdateRequest request) {
|
||||
return ApiResponse.success("更新成功", dedupeTotalDataService.update(id, request));
|
||||
}
|
||||
|
||||
@DeleteMapping("/{id}")
|
||||
@Operation(summary = "删除总数据", description = "按 ID 删除一条数据去重总数据。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在")
|
||||
})
|
||||
public ApiResponse<Void> delete(@Parameter(description = "主键ID", required = true) @PathVariable Long id) {
|
||||
dedupeTotalDataService.delete(id);
|
||||
return ApiResponse.success("删除成功", null);
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.dedupe.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
|
||||
}
|
||||
+7
-1
@@ -21,8 +21,14 @@ public class DedupeRunRequest {
|
||||
private List<String> selectedColumns;
|
||||
|
||||
@Schema(description = "是否保留纯数字 ID")
|
||||
private boolean keepIntegerIds = true;
|
||||
private boolean keepIntegerIds = false;
|
||||
|
||||
@Schema(description = "是否保留带下划线 ID")
|
||||
private boolean keepUnderscoreIds = true;
|
||||
|
||||
@Schema(description = "当文件中不存在下划线子 ID 时,是否自动保留纯数字主 ID")
|
||||
private boolean keepIntegerMainIdsWhenNoSubIds = true;
|
||||
|
||||
@Schema(description = "文件夹上传场景下的总压缩包名称(不含扩展名)")
|
||||
private String archiveName;
|
||||
}
|
||||
|
||||
+3
@@ -14,4 +14,7 @@ public class DedupeSourceFileDto {
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
|
||||
@Schema(description = "相对上传目录路径")
|
||||
private String relativePath;
|
||||
}
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重总数据新增请求")
|
||||
public class DedupeTotalDataCreateRequest {
|
||||
|
||||
@NotBlank(message = "数据不能为空")
|
||||
@Size(max = 128, message = "数据长度不能超过128个字符")
|
||||
@Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String dataValue;
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重总数据更新请求")
|
||||
public class DedupeTotalDataUpdateRequest {
|
||||
|
||||
@NotBlank(message = "数据不能为空")
|
||||
@Size(max = 128, message = "数据长度不能超过128个字符")
|
||||
@Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String dataValue;
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_dedupe_total_data")
|
||||
public class DedupeTotalDataEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String dataValue;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重总数据导入结果")
|
||||
public class DedupeTotalDataImportVo {
|
||||
|
||||
@Schema(description = "读取总行数")
|
||||
private Integer totalRows;
|
||||
|
||||
@Schema(description = "读取到的 ASIN 数量")
|
||||
private Integer asinCount;
|
||||
|
||||
@Schema(description = "实际新增数量")
|
||||
private Integer insertedCount;
|
||||
|
||||
@Schema(description = "跳过数量")
|
||||
private Integer skippedCount;
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重总数据项")
|
||||
public class DedupeTotalDataItemVo {
|
||||
|
||||
@Schema(description = "主键ID")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "总数据值")
|
||||
private String dataValue;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重总数据分页结果")
|
||||
public class DedupeTotalDataPageVo {
|
||||
|
||||
@Schema(description = "列表")
|
||||
private List<DedupeTotalDataItemVo> items;
|
||||
|
||||
@Schema(description = "总数")
|
||||
private Long total;
|
||||
|
||||
@Schema(description = "页码")
|
||||
private Long page;
|
||||
|
||||
@Schema(description = "每页数量")
|
||||
private Long pageSize;
|
||||
}
|
||||
+136
-12
@@ -28,11 +28,14 @@ import org.springframework.stereotype.Service;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
|
||||
@@ -46,7 +49,7 @@ public class DedupeRunService {
|
||||
private final OssStorageService ossStorageService;
|
||||
|
||||
public DedupeRunVo run(DedupeRunRequest request) {
|
||||
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds()) {
|
||||
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
|
||||
throw new BusinessException("请至少选择一种 ID 保留规则");
|
||||
}
|
||||
|
||||
@@ -63,9 +66,11 @@ public class DedupeRunService {
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
|
||||
List<DedupeResultItemVo> items = new ArrayList<>();
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
|
||||
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
@@ -81,13 +86,28 @@ public class DedupeRunService {
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
|
||||
cleanExcelByLegacyRules(inputFile, outputFile, request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds());
|
||||
cleanExcelByLegacyRules(
|
||||
inputFile,
|
||||
outputFile,
|
||||
request.getSelectedColumns(),
|
||||
request.isKeepIntegerIds(),
|
||||
request.isKeepUnderscoreIds(),
|
||||
request.isKeepIntegerMainIdsWhenNoSubIds()
|
||||
);
|
||||
|
||||
if (folderMode) {
|
||||
archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile));
|
||||
successCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE");
|
||||
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
|
||||
|
||||
String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName();
|
||||
|
||||
item.setSuccess(true);
|
||||
item.setOutputFilename(outputFile.getName());
|
||||
item.setOutputFilename(downloadFilename);
|
||||
item.setDownloadUrl(downloadUrl);
|
||||
successCount++;
|
||||
|
||||
@@ -95,7 +115,7 @@ public class DedupeRunService {
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(inputName);
|
||||
resultEntity.setResultFilename(outputFile.getName());
|
||||
resultEntity.setResultFilename(downloadFilename);
|
||||
resultEntity.setResultFileUrl(downloadUrl);
|
||||
resultEntity.setResultFileSize(outputFile.length());
|
||||
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
@@ -119,6 +139,32 @@ public class DedupeRunService {
|
||||
items.add(item);
|
||||
}
|
||||
|
||||
if (folderMode && !archiveEntries.isEmpty()) {
|
||||
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries);
|
||||
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, "DEDUPE");
|
||||
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
|
||||
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(request.getArchiveName());
|
||||
item.setOutputFilename(zipFile.getName());
|
||||
item.setSuccess(true);
|
||||
item.setDownloadUrl(downloadUrl);
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(request.getArchiveName());
|
||||
resultEntity.setResultFilename(zipFile.getName());
|
||||
resultEntity.setResultFileUrl(downloadUrl);
|
||||
resultEntity.setResultFileSize(zipFile.length());
|
||||
resultEntity.setResultContentType("application/zip");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
item.setResultId(resultEntity.getId());
|
||||
items.add(0, item);
|
||||
}
|
||||
|
||||
task.setSuccessFileCount(successCount);
|
||||
task.setFailedFileCount(failedCount);
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
@@ -136,17 +182,17 @@ public class DedupeRunService {
|
||||
public List<DedupeResultItemVo> listHistory() {
|
||||
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, "DEDUPE")
|
||||
.eq(FileResultEntity::getSuccess, 1)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit 50"))
|
||||
.stream()
|
||||
.map(entity -> {
|
||||
DedupeResultItemVo vo = new DedupeResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
|
||||
vo.setSourceFilename(entity.getSourceFilename());
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl());
|
||||
vo.setSuccess(true);
|
||||
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null);
|
||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||
vo.setError(entity.getErrorMessage());
|
||||
return vo;
|
||||
})
|
||||
.toList();
|
||||
@@ -173,7 +219,8 @@ public class DedupeRunService {
|
||||
}
|
||||
|
||||
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds) throws Exception {
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
|
||||
@@ -215,6 +262,26 @@ public class DedupeRunService {
|
||||
}
|
||||
|
||||
Integer idColumnIndex = headerMap.get("id");
|
||||
Map<String, Boolean> mainIdHasSubIdMap = new HashMap<>();
|
||||
if (idColumnIndex != null) {
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
|
||||
String mainId = extractMainId(idValue);
|
||||
if (mainId.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (!mainIdHasSubIdMap.containsKey(mainId)) {
|
||||
mainIdHasSubIdMap.put(mainId, false);
|
||||
}
|
||||
if (idValue.matches("\\d+_\\d+")) {
|
||||
mainIdHasSubIdMap.put(mainId, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
int outputRowIndex = 1;
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
@@ -223,7 +290,7 @@ public class DedupeRunService {
|
||||
}
|
||||
if (idColumnIndex != null) {
|
||||
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
|
||||
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds)) {
|
||||
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, mainIdHasSubIdMap)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -242,14 +309,68 @@ public class DedupeRunService {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds) {
|
||||
private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip");
|
||||
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
|
||||
for (DedupeArchiveEntry entry : archiveEntries) {
|
||||
String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.inputName(), false);
|
||||
zos.putNextEntry(new ZipEntry(zipEntryName));
|
||||
Files.copy(entry.outputFile().toPath(), zos);
|
||||
zos.closeEntry();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("去重结果打包失败:" + ex.getMessage());
|
||||
}
|
||||
return zipFile;
|
||||
}
|
||||
|
||||
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
|
||||
String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/');
|
||||
String relativeDir = normalizedRelativePath.contains("/")
|
||||
? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/'))
|
||||
: "";
|
||||
List<String> parts = new ArrayList<>();
|
||||
if (!relativeDir.isBlank()) {
|
||||
parts.add(relativeDir);
|
||||
}
|
||||
if (includeSourceStemFolder) {
|
||||
parts.add(FileUtil.mainName(inputName));
|
||||
}
|
||||
parts.add(outputFilename);
|
||||
return String.join("/", parts);
|
||||
}
|
||||
|
||||
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||
boolean keepIntegerMainIdsWhenNoSubIds, Map<String, Boolean> mainIdHasSubIdMap) {
|
||||
if (text == null || text.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
if (keepUnderscoreIds && text.matches("\\d+_\\d+")) {
|
||||
return true;
|
||||
}
|
||||
if (keepIntegerIds && text.matches("\\d+")) {
|
||||
return true;
|
||||
}
|
||||
return keepUnderscoreIds && text.matches("\\d+_\\d+");
|
||||
if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) {
|
||||
String mainId = extractMainId(text);
|
||||
return !mainId.isEmpty() && !Boolean.TRUE.equals(mainIdHasSubIdMap.get(mainId));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private String extractMainId(String text) {
|
||||
if (text == null || text.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
if (text.matches("\\d+")) {
|
||||
return text;
|
||||
}
|
||||
if (text.matches("\\d+_\\d+")) {
|
||||
int idx = text.indexOf('_');
|
||||
return idx > 0 ? text.substring(0, idx) : "";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private File findLocalSourceFile(String fileKey) {
|
||||
@@ -300,4 +421,7 @@ public class DedupeRunService {
|
||||
.trim()
|
||||
.replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
|
||||
}
|
||||
}
|
||||
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
package com.nanri.aiimage.modules.dedupe.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DedupeTotalDataService {
|
||||
|
||||
private final DedupeTotalDataMapper dedupeTotalDataMapper;
|
||||
|
||||
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword) {
|
||||
long safePage = Math.max(page, 1);
|
||||
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
||||
String safeKeyword = keyword == null ? "" : keyword.trim();
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
|
||||
.orderByDesc(DedupeTotalDataEntity::getId);
|
||||
Long total = dedupeTotalDataMapper.selectCount(query);
|
||||
List<DedupeTotalDataItemVo> items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
|
||||
.stream()
|
||||
.map(this::toItemVo)
|
||||
.toList();
|
||||
DedupeTotalDataPageVo vo = new DedupeTotalDataPageVo();
|
||||
vo.setItems(items);
|
||||
vo.setTotal(total);
|
||||
vo.setPage(safePage);
|
||||
vo.setPageSize(safePageSize);
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request) {
|
||||
String dataValue = normalizeDataValue(request.getDataValue());
|
||||
ensureUnique(dataValue, null);
|
||||
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
|
||||
entity.setDataValue(dataValue);
|
||||
dedupeTotalDataMapper.insert(entity);
|
||||
return toItemVo(getById(entity.getId()));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public DedupeTotalDataImportVo importFromExcel(MultipartFile file) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
}
|
||||
String filename = file.getOriginalFilename() == null ? "" : file.getOriginalFilename().toLowerCase();
|
||||
if (!(filename.endsWith(".xlsx") || filename.endsWith(".xls"))) {
|
||||
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
||||
}
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (InputStream inputStream = file.getInputStream();
|
||||
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (!value.isBlank() && !headerMap.containsKey(value)) {
|
||||
headerMap.put(value, i);
|
||||
}
|
||||
}
|
||||
|
||||
Integer asinIndex = headerMap.get("ASIN");
|
||||
if (asinIndex == null) {
|
||||
throw new BusinessException("缺少 ASIN 列");
|
||||
}
|
||||
|
||||
Set<String> seenInFile = new HashSet<>();
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
int asinCount = 0;
|
||||
int insertedCount = 0;
|
||||
int skippedCount = 0;
|
||||
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex)));
|
||||
if (asin.isBlank()) {
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
asinCount++;
|
||||
String dataValue = normalizeDataValue(asin);
|
||||
if (!seenInFile.add(dataValue)) {
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
if (existsDataValue(dataValue)) {
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
|
||||
entity.setDataValue(dataValue);
|
||||
dedupeTotalDataMapper.insert(entity);
|
||||
insertedCount++;
|
||||
}
|
||||
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(totalRows);
|
||||
vo.setAsinCount(asinCount);
|
||||
vo.setInsertedCount(insertedCount);
|
||||
vo.setSkippedCount(skippedCount);
|
||||
return vo;
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("导入 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) {
|
||||
DedupeTotalDataEntity entity = getById(id);
|
||||
String dataValue = normalizeDataValue(request.getDataValue());
|
||||
ensureUnique(dataValue, id);
|
||||
entity.setDataValue(dataValue);
|
||||
dedupeTotalDataMapper.updateById(entity);
|
||||
return toItemVo(getById(id));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void delete(Long id) {
|
||||
DedupeTotalDataEntity entity = getById(id);
|
||||
dedupeTotalDataMapper.deleteById(entity.getId());
|
||||
}
|
||||
|
||||
private DedupeTotalDataEntity getById(Long id) {
|
||||
DedupeTotalDataEntity entity = dedupeTotalDataMapper.selectById(id);
|
||||
if (entity == null) {
|
||||
throw new BusinessException("数据不存在");
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
private String normalizeDataValue(String value) {
|
||||
String dataValue = value == null ? "" : value.trim();
|
||||
if (dataValue.isEmpty()) {
|
||||
throw new BusinessException("数据不能为空");
|
||||
}
|
||||
return dataValue;
|
||||
}
|
||||
|
||||
private void ensureUnique(String dataValue, Long excludeId) {
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
||||
.last("LIMIT 1");
|
||||
DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query);
|
||||
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
|
||||
throw new BusinessException("数据已存在");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean existsDataValue(String dataValue) {
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
||||
.last("LIMIT 1");
|
||||
return dedupeTotalDataMapper.selectOne(query) != null;
|
||||
}
|
||||
|
||||
private String normalizeExcelText(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.replace("\ufeff", "")
|
||||
.replace("\u3000", " ")
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim()
|
||||
.replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private DedupeTotalDataItemVo toItemVo(DedupeTotalDataEntity entity) {
|
||||
DedupeTotalDataItemVo vo = new DedupeTotalDataItemVo();
|
||||
vo.setId(entity.getId());
|
||||
vo.setDataValue(entity.getDataValue());
|
||||
vo.setCreatedAt(entity.getCreatedAt());
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user