super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据去重完善

Browse Source
This commit is contained in:
super
2026-03-22 20:58:36 +08:00
parent ed91c2ae7d
commit 01e491e332
4 changed files with 38 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java
View File

@@ -3,7 +3,13 @@ package com.nanri.aiimage.modules.dedupe.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity; import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import java.util.List;
@Mapper @Mapper
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> { public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
@Select("SELECT data_value FROM biz_dedupe_total_data")
List<String> selectAllDataValues();
} }

15
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
View File

@@ -47,6 +47,7 @@ public class DedupeRunService {
private final FileResultMapper fileResultMapper; private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties; private final StorageProperties storageProperties;
private final OssStorageService ossStorageService; private final OssStorageService ossStorageService;
private final DedupeTotalDataService dedupeTotalDataService;
public DedupeRunVo run(DedupeRunRequest request) { public DedupeRunVo run(DedupeRunRequest request) {
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) { if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
@@ -71,6 +72,7 @@ public class DedupeRunService {
int successCount = 0; int successCount = 0;
int failedCount = 0; int failedCount = 0;
List<DedupeArchiveEntry> archiveEntries = new ArrayList<>(); List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
java.util.Set<String> totalDataValues = dedupeTotalDataService.listComparableValues();
for (DedupeSourceFileDto sourceFile : request.getFiles()) { for (DedupeSourceFileDto sourceFile : request.getFiles()) {
DedupeResultItemVo item = new DedupeResultItemVo(); DedupeResultItemVo item = new DedupeResultItemVo();
@@ -92,7 +94,8 @@ public class DedupeRunService {
request.getSelectedColumns(), request.getSelectedColumns(),
request.isKeepIntegerIds(), request.isKeepIntegerIds(),
request.isKeepUnderscoreIds(), request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds() request.isKeepIntegerMainIdsWhenNoSubIds(),
totalDataValues
); );
if (folderMode) { if (folderMode) {
@@ -220,7 +223,8 @@ public class DedupeRunService {
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns, private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { boolean keepIntegerMainIdsWhenNoSubIds,
java.util.Set<String> totalDataValues) throws Exception {
DataFormatter formatter = new DataFormatter(); DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile); try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
@@ -262,6 +266,7 @@ public class DedupeRunService {
} }
Integer idColumnIndex = headerMap.get("id"); Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
if (idColumnIndex != null) { if (idColumnIndex != null) {
// The preliminary global scan for mainIdHasSubIdMap was removed. // The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop. // We now determine subset existence contextually (per group) during the main loop.
@@ -278,6 +283,12 @@ public class DedupeRunService {
continue; continue;
} }
} }
if (asinColumnIndex != null && !totalDataValues.isEmpty()) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank() && totalDataValues.contains(asinValue)) {
continue;
}
}
Row outputRow = outputSheet.createRow(outputRowIndex++); Row outputRow = outputSheet.createRow(outputRowIndex++);
for (int i = 0; i < selectedColumns.size(); i++) { for (int i = 0; i < selectedColumns.size(); i++) {

28
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
View File

@@ -2,7 +2,6 @@ package com.nanri.aiimage.modules.dedupe.service;
import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
@@ -62,7 +61,7 @@ public class DedupeTotalDataService {
@Transactional @Transactional
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request) { public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request) {
String dataValue = normalizeDataValue(request.getDataValue()); String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, null); ensureUnique(dataValue, null);
DedupeTotalDataEntity entity = new DedupeTotalDataEntity(); DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(dataValue); entity.setDataValue(dataValue);
@@ -70,6 +69,13 @@ public class DedupeTotalDataService {
return toItemVo(getById(entity.getId())); return toItemVo(getById(entity.getId()));
} }
public Set<String> listComparableValues() {
return dedupeTotalDataMapper.selectAllDataValues().stream()
.map(this::normalizeComparableValue)
.filter(value -> !value.isEmpty())
.collect(java.util.stream.Collectors.toSet());
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file) { public DedupeTotalDataImportStartVo startImport(MultipartFile file) {
if (file == null || file.isEmpty()) { if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件"); throw new BusinessException("请上传 xlsx 文件");
@@ -265,7 +271,7 @@ public class DedupeTotalDataService {
} }
continue; continue;
} }
String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex))); String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) { if (asin.isBlank()) {
skippedCount++; skippedCount++;
if (progress != null) { if (progress != null) {
@@ -277,7 +283,7 @@ public class DedupeTotalDataService {
continue; continue;
} }
asinCount++; asinCount++;
String dataValue = normalizeDataValue(asin); String dataValue = asin;
if (!seenInFile.add(dataValue)) { if (!seenInFile.add(dataValue)) {
skippedCount++; skippedCount++;
if (progress != null) { if (progress != null) {
@@ -378,7 +384,7 @@ public class DedupeTotalDataService {
} }
continue; continue;
} }
String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex))); String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) { if (asin.isBlank()) {
skippedCount++; skippedCount++;
if (progress != null) { if (progress != null) {
@@ -390,7 +396,7 @@ public class DedupeTotalDataService {
continue; continue;
} }
asinCount++; asinCount++;
String dataValue = normalizeDataValue(asin); String dataValue = asin;
if (!seenInFile.add(dataValue)) { if (!seenInFile.add(dataValue)) {
skippedCount++; skippedCount++;
if (progress != null) { if (progress != null) {
@@ -432,7 +438,7 @@ public class DedupeTotalDataService {
@Transactional @Transactional
public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) { public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) {
DedupeTotalDataEntity entity = getById(id); DedupeTotalDataEntity entity = getById(id);
String dataValue = normalizeDataValue(request.getDataValue()); String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, id); ensureUnique(dataValue, id);
entity.setDataValue(dataValue); entity.setDataValue(dataValue);
dedupeTotalDataMapper.updateById(entity); dedupeTotalDataMapper.updateById(entity);
@@ -453,14 +459,18 @@ public class DedupeTotalDataService {
return entity; return entity;
} }
private String normalizeDataValue(String value) { public String normalizeComparableValue(String value) {
String dataValue = value == null ? "" : value.trim(); String dataValue = normalizeComparableValueOrBlank(value);
if (dataValue.isEmpty()) { if (dataValue.isEmpty()) {
throw new BusinessException("数据不能为空"); throw new BusinessException("数据不能为空");
} }
return dataValue; return dataValue;
} }
public String normalizeComparableValueOrBlank(String value) {
return normalizeExcelText(value);
}
private void ensureUnique(String dataValue, Long excludeId) { private void ensureUnique(String dataValue, Long excludeId) {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>() LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getDataValue, dataValue) .eq(DedupeTotalDataEntity::getDataValue, dataValue)

BIN
backend/__pycache__/config.cpython-312.pyc
View File

Binary file not shown.