数据去重完善

backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java

@@ -3,7 +3,13 @@ package com.nanri.aiimage.modules.dedupe.mapper;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Select;
+
+import java.util.List;
 
 @Mapper
 public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
+
+    @Select("SELECT data_value FROM biz_dedupe_total_data")
+    List<String> selectAllDataValues();
 }

backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java

@@ -47,6 +47,7 @@ public class DedupeRunService {
     private final FileResultMapper fileResultMapper;
     private final StorageProperties storageProperties;
     private final OssStorageService ossStorageService;
+    private final DedupeTotalDataService dedupeTotalDataService;
 
     public DedupeRunVo run(DedupeRunRequest request) {
         if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
@@ -71,6 +72,7 @@ public class DedupeRunService {
         int successCount = 0;
         int failedCount = 0;
         List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
+        java.util.Set<String> totalDataValues = dedupeTotalDataService.listComparableValues();
 
         for (DedupeSourceFileDto sourceFile : request.getFiles()) {
             DedupeResultItemVo item = new DedupeResultItemVo();
@@ -92,7 +94,8 @@ public class DedupeRunService {
                         request.getSelectedColumns(),
                         request.isKeepIntegerIds(),
                         request.isKeepUnderscoreIds(),
-                        request.isKeepIntegerMainIdsWhenNoSubIds()
+                        request.isKeepIntegerMainIdsWhenNoSubIds(),
+                        totalDataValues
                 );
 
                 if (folderMode) {
@@ -220,7 +223,8 @@ public class DedupeRunService {
 
     private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
                                           boolean keepIntegerIds, boolean keepUnderscoreIds,
-                                          boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
+                                          boolean keepIntegerMainIdsWhenNoSubIds,
+                                          java.util.Set<String> totalDataValues) throws Exception {
         DataFormatter formatter = new DataFormatter();
         try (FileInputStream fis = new FileInputStream(inputFile);
              Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
@@ -262,6 +266,7 @@ public class DedupeRunService {
             }
 
             Integer idColumnIndex = headerMap.get("id");
+            Integer asinColumnIndex = headerMap.get("ASIN");
             if (idColumnIndex != null) {
                 // The preliminary global scan for mainIdHasSubIdMap was removed.
                 // We now determine subset existence contextually (per group) during the main loop.
@@ -278,6 +283,12 @@ public class DedupeRunService {
                         continue;
                     }
                 }
+                if (asinColumnIndex != null && !totalDataValues.isEmpty()) {
+                    String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
+                    if (!asinValue.isBlank() && totalDataValues.contains(asinValue)) {
+                        continue;
+                    }
+                }
 
                 Row outputRow = outputSheet.createRow(outputRowIndex++);
                 for (int i = 0; i < selectedColumns.size(); i++) {

backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java

@@ -2,7 +2,6 @@ package com.nanri.aiimage.modules.dedupe.service;
 
 import cn.hutool.core.util.IdUtil;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
-import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
 import com.nanri.aiimage.common.exception.BusinessException;
 import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
 import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
@@ -62,7 +61,7 @@ public class DedupeTotalDataService {
 
     @Transactional
     public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request) {
-        String dataValue = normalizeDataValue(request.getDataValue());
+        String dataValue = normalizeComparableValue(request.getDataValue());
         ensureUnique(dataValue, null);
         DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
         entity.setDataValue(dataValue);
@@ -70,6 +69,13 @@ public class DedupeTotalDataService {
         return toItemVo(getById(entity.getId()));
     }
 
+    public Set<String> listComparableValues() {
+        return dedupeTotalDataMapper.selectAllDataValues().stream()
+                .map(this::normalizeComparableValue)
+                .filter(value -> !value.isEmpty())
+                .collect(java.util.stream.Collectors.toSet());
+    }
+
     public DedupeTotalDataImportStartVo startImport(MultipartFile file) {
         if (file == null || file.isEmpty()) {
             throw new BusinessException("请上传 xlsx 文件");
@@ -265,7 +271,7 @@ public class DedupeTotalDataService {
                     }
                     continue;
                 }
-                String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex)));
+                String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
                 if (asin.isBlank()) {
                     skippedCount++;
                     if (progress != null) {
@@ -277,7 +283,7 @@ public class DedupeTotalDataService {
                     continue;
                 }
                 asinCount++;
-                String dataValue = normalizeDataValue(asin);
+                String dataValue = asin;
                 if (!seenInFile.add(dataValue)) {
                     skippedCount++;
                     if (progress != null) {
@@ -378,7 +384,7 @@ public class DedupeTotalDataService {
                     }
                     continue;
                 }
-                String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex)));
+                String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
                 if (asin.isBlank()) {
                     skippedCount++;
                     if (progress != null) {
@@ -390,7 +396,7 @@ public class DedupeTotalDataService {
                     continue;
                 }
                 asinCount++;
-                String dataValue = normalizeDataValue(asin);
+                String dataValue = asin;
                 if (!seenInFile.add(dataValue)) {
                     skippedCount++;
                     if (progress != null) {
@@ -432,7 +438,7 @@ public class DedupeTotalDataService {
     @Transactional
     public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) {
         DedupeTotalDataEntity entity = getById(id);
-        String dataValue = normalizeDataValue(request.getDataValue());
+        String dataValue = normalizeComparableValue(request.getDataValue());
         ensureUnique(dataValue, id);
         entity.setDataValue(dataValue);
         dedupeTotalDataMapper.updateById(entity);
@@ -453,14 +459,18 @@ public class DedupeTotalDataService {
         return entity;
     }
 
-    private String normalizeDataValue(String value) {
-        String dataValue = value == null ? "" : value.trim();
+    public String normalizeComparableValue(String value) {
+        String dataValue = normalizeComparableValueOrBlank(value);
         if (dataValue.isEmpty()) {
             throw new BusinessException("数据不能为空");
         }
         return dataValue;
     }
 
+    public String normalizeComparableValueOrBlank(String value) {
+        return normalizeExcelText(value);
+    }
+
     private void ensureUnique(String dataValue, Long excludeId) {
         LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
                 .eq(DedupeTotalDataEntity::getDataValue, dataValue)