完成品牌部分改造
This commit is contained in:
+2
@@ -46,6 +46,8 @@ public class DedupeRunController {
|
||||
- keepIntegerIds=true 时保留纯数字 ID;
|
||||
- keepUnderscoreIds=true 时保留类似 1_1 的 ID;
|
||||
- keepIntegerMainIdsWhenNoSubIds=true 且文件中不存在 1_1 这类子 ID 时,会自动保留纯数字主 ID;
|
||||
- 整理后的数据会再按 ASIN 与去重总数据表比对,命中的行不会出现在最终输出文件;
|
||||
- 上传文件内如果存在重复 ASIN,只保留第一次出现的那一行;
|
||||
- 输出文件命名遵循 原文件名_cleaned.xlsx,若重名自动追加序号。
|
||||
""")
|
||||
@ApiResponses({
|
||||
|
||||
+16
@@ -3,13 +3,29 @@ package com.nanri.aiimage.modules.dedupe.mapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.SelectProvider;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
@Mapper
|
||||
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
|
||||
|
||||
@Select("SELECT data_value FROM biz_dedupe_total_data")
|
||||
List<String> selectAllDataValues();
|
||||
|
||||
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
|
||||
List<String> selectExistingDataValues(@Param("values") List<String> values);
|
||||
|
||||
class SqlProvider {
|
||||
public String selectExistingDataValues(@Param("values") List<String> values) {
|
||||
StringJoiner placeholders = new StringJoiner(", ");
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
placeholders.add("#{values[" + i + "]}");
|
||||
}
|
||||
return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+27
-9
@@ -32,8 +32,10 @@ import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@@ -72,7 +74,6 @@ public class DedupeRunService {
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
|
||||
java.util.Set<String> totalDataValues = dedupeTotalDataService.listComparableValues();
|
||||
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
@@ -94,8 +95,7 @@ public class DedupeRunService {
|
||||
request.getSelectedColumns(),
|
||||
request.isKeepIntegerIds(),
|
||||
request.isKeepUnderscoreIds(),
|
||||
request.isKeepIntegerMainIdsWhenNoSubIds(),
|
||||
totalDataValues
|
||||
request.isKeepIntegerMainIdsWhenNoSubIds()
|
||||
);
|
||||
|
||||
if (folderMode) {
|
||||
@@ -223,8 +223,7 @@ public class DedupeRunService {
|
||||
|
||||
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||
boolean keepIntegerMainIdsWhenNoSubIds,
|
||||
java.util.Set<String> totalDataValues) throws Exception {
|
||||
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
|
||||
@@ -271,7 +270,8 @@ public class DedupeRunService {
|
||||
// The preliminary global scan for mainIdHasSubIdMap was removed.
|
||||
// We now determine subset existence contextually (per group) during the main loop.
|
||||
}
|
||||
int outputRowIndex = 1;
|
||||
List<Row> candidateRows = new ArrayList<>();
|
||||
Set<String> candidateAsinValues = new HashSet<>();
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
@@ -283,10 +283,28 @@ public class DedupeRunService {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (asinColumnIndex != null && !totalDataValues.isEmpty()) {
|
||||
candidateRows.add(row);
|
||||
if (asinColumnIndex != null) {
|
||||
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
|
||||
if (!asinValue.isBlank() && totalDataValues.contains(asinValue)) {
|
||||
continue;
|
||||
if (!asinValue.isBlank()) {
|
||||
candidateAsinValues.add(asinValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
|
||||
Set<String> writtenAsinValues = new HashSet<>();
|
||||
int outputRowIndex = 1;
|
||||
for (Row row : candidateRows) {
|
||||
if (asinColumnIndex != null) {
|
||||
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
|
||||
if (!asinValue.isBlank()) {
|
||||
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
|
||||
continue;
|
||||
}
|
||||
if (!writtenAsinValues.add(asinValue)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+23
@@ -24,6 +24,7 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
@@ -35,6 +36,8 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
@RequiredArgsConstructor
|
||||
public class DedupeTotalDataService {
|
||||
|
||||
private static final int COMPARE_BATCH_SIZE = 1000;
|
||||
|
||||
private final DedupeTotalDataMapper dedupeTotalDataMapper;
|
||||
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();
|
||||
private final Map<String, DedupeTotalDataImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
|
||||
@@ -76,6 +79,26 @@ public class DedupeTotalDataService {
|
||||
.collect(java.util.stream.Collectors.toSet());
|
||||
}
|
||||
|
||||
public Set<String> findExistingComparableValues(Collection<String> values) {
|
||||
if (values == null || values.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
List<String> normalizedValues = values.stream()
|
||||
.map(this::normalizeComparableValueOrBlank)
|
||||
.filter(value -> !value.isEmpty())
|
||||
.distinct()
|
||||
.toList();
|
||||
if (normalizedValues.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<String> existingValues = new HashSet<>();
|
||||
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
|
||||
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
|
||||
existingValues.addAll(dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end)));
|
||||
}
|
||||
return existingValues;
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportStartVo startImport(MultipartFile file) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
|
||||
Reference in New Issue
Block a user