From 12318f8770838325e70f0bd6902c190224277004 Mon Sep 17 00:00:00 2001 From: super <2903208875@qq.com> Date: Sun, 22 Mar 2026 20:58:36 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=8E=BB=E9=87=8D=E5=AE=8C?= =?UTF-8?q?=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dedupe/mapper/DedupeTotalDataMapper.java | 6 ++++ .../dedupe/service/DedupeRunService.java | 15 ++++++++-- .../service/DedupeTotalDataService.java | 28 ++++++++++++------ backend/__pycache__/config.cpython-312.pyc | Bin 1510 -> 1513 bytes 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java index bc6b89d..6999757 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java @@ -3,7 +3,13 @@ package com.nanri.aiimage.modules.dedupe.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Select; + +import java.util.List; @Mapper public interface DedupeTotalDataMapper extends BaseMapper { + + @Select("SELECT data_value FROM biz_dedupe_total_data") + List selectAllDataValues(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 1355d3f..dbc004a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -47,6 +47,7 @@ public class DedupeRunService { private final FileResultMapper fileResultMapper; private final StorageProperties storageProperties; private final OssStorageService ossStorageService; + private final DedupeTotalDataService dedupeTotalDataService; public DedupeRunVo run(DedupeRunRequest request) { if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) { @@ -71,6 +72,7 @@ public class DedupeRunService { int successCount = 0; int failedCount = 0; List archiveEntries = new ArrayList<>(); + java.util.Set totalDataValues = dedupeTotalDataService.listComparableValues(); for (DedupeSourceFileDto sourceFile : request.getFiles()) { DedupeResultItemVo item = new DedupeResultItemVo(); @@ -92,7 +94,8 @@ public class DedupeRunService { request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds(), - request.isKeepIntegerMainIdsWhenNoSubIds() + request.isKeepIntegerMainIdsWhenNoSubIds(), + totalDataValues ); if (folderMode) { @@ -220,7 +223,8 @@ public class DedupeRunService { private void cleanExcelByLegacyRules(File inputFile, File outputFile, List selectedColumns, boolean keepIntegerIds, boolean keepUnderscoreIds, - boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { + boolean keepIntegerMainIdsWhenNoSubIds, + java.util.Set totalDataValues) throws Exception { DataFormatter formatter = new DataFormatter(); try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis); @@ -262,6 +266,7 @@ public class DedupeRunService { } Integer idColumnIndex = headerMap.get("id"); + Integer asinColumnIndex = headerMap.get("ASIN"); if (idColumnIndex != null) { // The preliminary global scan for mainIdHasSubIdMap was removed. // We now determine subset existence contextually (per group) during the main loop. @@ -278,6 +283,12 @@ public class DedupeRunService { continue; } } + if (asinColumnIndex != null && !totalDataValues.isEmpty()) { + String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex))); + if (!asinValue.isBlank() && totalDataValues.contains(asinValue)) { + continue; + } + } Row outputRow = outputSheet.createRow(outputRowIndex++); for (int i = 0; i < selectedColumns.size(); i++) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java index 80d2fd4..6e54b1e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -2,7 +2,6 @@ package com.nanri.aiimage.modules.dedupe.service; import cn.hutool.core.util.IdUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest; @@ -62,7 +61,7 @@ public class DedupeTotalDataService { @Transactional public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request) { - String dataValue = normalizeDataValue(request.getDataValue()); + String dataValue = normalizeComparableValue(request.getDataValue()); ensureUnique(dataValue, null); DedupeTotalDataEntity entity = new DedupeTotalDataEntity(); entity.setDataValue(dataValue); @@ -70,6 +69,13 @@ public class DedupeTotalDataService { return toItemVo(getById(entity.getId())); } + public Set listComparableValues() { + return dedupeTotalDataMapper.selectAllDataValues().stream() + .map(this::normalizeComparableValue) + .filter(value -> !value.isEmpty()) + .collect(java.util.stream.Collectors.toSet()); + } + public DedupeTotalDataImportStartVo startImport(MultipartFile file) { if (file == null || file.isEmpty()) { throw new BusinessException("请上传 xlsx 文件"); @@ -265,7 +271,7 @@ public class DedupeTotalDataService { } continue; } - String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex))); + String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex))); if (asin.isBlank()) { skippedCount++; if (progress != null) { @@ -277,7 +283,7 @@ public class DedupeTotalDataService { continue; } asinCount++; - String dataValue = normalizeDataValue(asin); + String dataValue = asin; if (!seenInFile.add(dataValue)) { skippedCount++; if (progress != null) { @@ -378,7 +384,7 @@ public class DedupeTotalDataService { } continue; } - String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex))); + String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex))); if (asin.isBlank()) { skippedCount++; if (progress != null) { @@ -390,7 +396,7 @@ public class DedupeTotalDataService { continue; } asinCount++; - String dataValue = normalizeDataValue(asin); + String dataValue = asin; if (!seenInFile.add(dataValue)) { skippedCount++; if (progress != null) { @@ -432,7 +438,7 @@ public class DedupeTotalDataService { @Transactional public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) { DedupeTotalDataEntity entity = getById(id); - String dataValue = normalizeDataValue(request.getDataValue()); + String dataValue = normalizeComparableValue(request.getDataValue()); ensureUnique(dataValue, id); entity.setDataValue(dataValue); dedupeTotalDataMapper.updateById(entity); @@ -453,14 +459,18 @@ public class DedupeTotalDataService { return entity; } - private String normalizeDataValue(String value) { - String dataValue = value == null ? "" : value.trim(); + public String normalizeComparableValue(String value) { + String dataValue = normalizeComparableValueOrBlank(value); if (dataValue.isEmpty()) { throw new BusinessException("数据不能为空"); } return dataValue; } + public String normalizeComparableValueOrBlank(String value) { + return normalizeExcelText(value); + } + private void ensureUnique(String dataValue, Long excludeId) { LambdaQueryWrapper query = new LambdaQueryWrapper() .eq(DedupeTotalDataEntity::getDataValue, dataValue) diff --git a/backend/__pycache__/config.cpython-312.pyc b/backend/__pycache__/config.cpython-312.pyc index 79187cee0b89efad4c94fcfc3b963799dbb63196..71fb25c982272d5d96139009249b9e14e234b60c 100644 GIT binary patch delta 86 zcmaFH{gRvOG%qg~0}#yKy^-q&lZa$SNlAf~zP^Q?p|P2sp{1Uox$$OU=0GNu2NF{A qxn^?B=bOoQLEY|xL-b{dnClYB7bTJ}q@-V#$hg2DdSh}2Ya9S#qa1Jm delta 83 zcmaFK{fwLIG%qg~0}#xdwvp=xlb~2eNlAf~zP_Q6xt;+K8g3S24rEffAt5!NYbMux mzL|U%)a@=fL|m4Lye^S&Q6k|&V#;NS)C&xv4<>iB#sL5o%^QdS