完成品牌部分改造

This commit is contained in:
super
2026-03-24 22:22:37 +08:00
parent 7ffe5279a3
commit 2cd0fcff93
37 changed files with 2086 additions and 11 deletions
@@ -46,6 +46,8 @@ public class DedupeRunController {
- keepIntegerIds=true 时保留纯数字 ID;
- keepUnderscoreIds=true 时保留类似 1_1 的 ID;
- keepIntegerMainIdsWhenNoSubIds=true 且文件中不存在 1_1 这类子 ID 时,会自动保留纯数字主 ID;
- 整理后的数据会再按 ASIN 与去重总数据表比对,命中的行不会出现在最终输出文件;
- 上传文件内如果存在重复 ASIN,只保留第一次出现的那一行;
- 输出文件命名遵循 原文件名_cleaned.xlsx,若重名自动追加序号。
""")
@ApiResponses({
@@ -3,13 +3,29 @@ package com.nanri.aiimage.modules.dedupe.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.SelectProvider;
import java.util.List;
import java.util.StringJoiner;
@Mapper
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
@Select("SELECT data_value FROM biz_dedupe_total_data")
List<String> selectAllDataValues();
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
List<String> selectExistingDataValues(@Param("values") List<String> values);
class SqlProvider {
public String selectExistingDataValues(@Param("values") List<String> values) {
StringJoiner placeholders = new StringJoiner(", ");
for (int i = 0; i < values.size(); i++) {
placeholders.add("#{values[" + i + "]}");
}
return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")";
}
}
}
@@ -32,8 +32,10 @@ import java.nio.file.Files;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -72,7 +74,6 @@ public class DedupeRunService {
int successCount = 0;
int failedCount = 0;
List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
java.util.Set<String> totalDataValues = dedupeTotalDataService.listComparableValues();
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
DedupeResultItemVo item = new DedupeResultItemVo();
@@ -94,8 +95,7 @@ public class DedupeRunService {
request.getSelectedColumns(),
request.isKeepIntegerIds(),
request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds(),
totalDataValues
request.isKeepIntegerMainIdsWhenNoSubIds()
);
if (folderMode) {
@@ -223,8 +223,7 @@ public class DedupeRunService {
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds,
java.util.Set<String> totalDataValues) throws Exception {
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
@@ -271,7 +270,8 @@ public class DedupeRunService {
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
int outputRowIndex = 1;
List<Row> candidateRows = new ArrayList<>();
Set<String> candidateAsinValues = new HashSet<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
@@ -283,10 +283,28 @@ public class DedupeRunService {
continue;
}
}
if (asinColumnIndex != null && !totalDataValues.isEmpty()) {
candidateRows.add(row);
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank() && totalDataValues.contains(asinValue)) {
continue;
if (!asinValue.isBlank()) {
candidateAsinValues.add(asinValue);
}
}
}
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> writtenAsinValues = new HashSet<>();
int outputRowIndex = 1;
for (Row row : candidateRows) {
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
continue;
}
if (!writtenAsinValues.add(asinValue)) {
continue;
}
}
}
@@ -24,6 +24,7 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -35,6 +36,8 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class DedupeTotalDataService {
private static final int COMPARE_BATCH_SIZE = 1000;
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();
private final Map<String, DedupeTotalDataImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
@@ -76,6 +79,26 @@ public class DedupeTotalDataService {
.collect(java.util.stream.Collectors.toSet());
}
public Set<String> findExistingComparableValues(Collection<String> values) {
if (values == null || values.isEmpty()) {
return Set.of();
}
List<String> normalizedValues = values.stream()
.map(this::normalizeComparableValueOrBlank)
.filter(value -> !value.isEmpty())
.distinct()
.toList();
if (normalizedValues.isEmpty()) {
return Set.of();
}
Set<String> existingValues = new HashSet<>();
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
existingValues.addAll(dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end)));
}
return existingValues;
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");