后台查询Asin文档更新

This commit is contained in:
super
2026-04-24 15:14:14 +08:00
parent 23b9575234
commit 8ea7e949c2
34 changed files with 798 additions and 540 deletions
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
@@ -15,11 +16,13 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
@@ -39,12 +42,14 @@ import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@Service
@RequiredArgsConstructor
@Slf4j
public class DedupeRunService {
private static final String HEADER_SPLIT_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf";
private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408";
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties;
@@ -52,8 +57,9 @@ public class DedupeRunService {
private final DedupeTotalDataService dedupeTotalDataService;
public DedupeRunVo run(DedupeRunRequest request) {
long runStartNs = System.nanoTime();
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
throw new BusinessException("请至少选择一种 ID 保留规则");
throw new BusinessException("\u8bf7\u81f3\u5c11\u9009\u62e9\u4e00\u79cd ID \u4fdd\u7559\u89c4\u5219");
}
FileTaskEntity task = new FileTaskEntity();
@@ -80,9 +86,11 @@ public class DedupeRunService {
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
try {
long fileStartNs = System.nanoTime();
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
long findFileNs = elapsedNs(fileStartNs);
if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("上传文件不存在,请重新上传");
throw new BusinessException("\u4e0a\u4f20\u6587\u4ef6\u4e0d\u5b58\u5728\uff0c\u8bf7\u91cd\u65b0\u4e0a\u4f20");
}
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
@@ -90,6 +98,7 @@ public class DedupeRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
long cleanStartNs = System.nanoTime();
cleanExcelByLegacyRules(
inputFile,
outputFile,
@@ -98,6 +107,7 @@ public class DedupeRunService {
request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds()
);
long cleanNs = elapsedNs(cleanStartNs);
if (folderMode) {
archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile));
@@ -105,13 +115,15 @@ public class DedupeRunService {
continue;
}
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE");
// 只存 objectKey,不存预签名 URL
long uploadStartNs = System.nanoTime();
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE");
long uploadNs = elapsedNs(uploadStartNs);
String ossObjectKey = uploadedResult.objectKey();
String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName();
item.setSuccess(true);
item.setOutputFilename(downloadFilename);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey));
item.setDownloadUrl(uploadedResult.downloadUrl());
successCount++;
FileResultEntity resultEntity = new FileResultEntity();
@@ -119,13 +131,26 @@ public class DedupeRunService {
resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(downloadFilename);
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey
resultEntity.setResultFileUrl(ossObjectKey);
resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
resultEntity.setSuccess(1);
resultEntity.setUserId(request.getUserId());
resultEntity.setCreatedAt(LocalDateTime.now());
long resultInsertStartNs = System.nanoTime();
fileResultMapper.insert(resultEntity);
long resultInsertNs = elapsedNs(resultInsertStartNs);
log.info(
"dedupe run file done fileKey={} filename={} size={} findFileMs={} cleanMs={} uploadMs={} resultInsertMs={} totalFileMs={}",
sourceFile.getFileKey(),
inputName,
inputFile.length(),
toMs(findFileNs),
toMs(cleanNs),
toMs(uploadNs),
toMs(resultInsertNs),
toMs(elapsedNs(fileStartNs))
);
} catch (Exception ex) {
item.setSuccess(false);
item.setError(ex.getMessage());
@@ -146,20 +171,20 @@ public class DedupeRunService {
if (folderMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, "DEDUPE");
// 只存 objectKey
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE");
String ossObjectKey = uploadedResult.objectKey();
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(request.getArchiveName());
item.setOutputFilename(zipFile.getName());
item.setSuccess(true);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey));
item.setDownloadUrl(uploadedResult.downloadUrl());
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(request.getArchiveName());
resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey
resultEntity.setResultFileUrl(ossObjectKey);
resultEntity.setResultFileSize(zipFile.length());
resultEntity.setResultContentType("application/zip");
resultEntity.setSuccess(1);
@@ -181,6 +206,14 @@ public class DedupeRunService {
vo.setSuccessCount(successCount);
vo.setFailedCount(failedCount);
vo.setItems(items);
log.info(
"dedupe run done userId={} files={} success={} failed={} totalMs={}",
request.getUserId(),
request.getFiles().size(),
successCount,
failedCount,
toMs(elapsedNs(runStartNs))
);
return vo;
}
@@ -209,7 +242,7 @@ public class DedupeRunService {
public void deleteHistory(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在");
throw new BusinessException("\u8bb0\u5f55\u4e0d\u5b58\u5728");
}
fileResultMapper.deleteById(resultId);
}
@@ -217,115 +250,64 @@ public class DedupeRunService {
public Resource getResultFile(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("结果文件不存在");
throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728");
}
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("结果文件不存在");
throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728");
}
return new FileSystemResource(file);
}
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
long readStartNs = System.nanoTime();
DedupeReadResult readResult = readDedupeRows(
inputFile,
selectedColumns,
keepIntegerIds,
keepUnderscoreIds,
keepIntegerMainIdsWhenNoSubIds
);
long readNs = elapsedNs(readStartNs);
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank()) {
continue;
}
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
? value
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if ("缩略图地址8".equals(value)) {
break;
}
Set<String> candidateAsinValues = new HashSet<>();
for (DedupeCandidateRow row : readResult.rows()) {
if (!row.asinValue().isBlank()) {
candidateAsinValues.add(row.asinValue());
}
}
long dbStartNs = System.nanoTime();
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
long dbNs = elapsedNs(dbStartNs);
List<String> missing = selectedColumns.stream().filter(column -> !headerMap.containsKey(column)).toList();
if (!missing.isEmpty()) {
throw new BusinessException("缺少列:" + String.join("、", missing));
}
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
long writeStartNs = System.nanoTime();
int outputRows = 0;
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName()));
Row outputHeaderRow = outputSheet.createRow(0);
for (int i = 0; i < selectedColumns.size(); i++) {
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
}
Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
if (idColumnIndex != null) {
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
Set<String> candidateAsinValues = new HashSet<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
continue;
}
}
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
candidateAsinValues.add(asinValue);
}
}
}
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> writtenAsinValues = new HashSet<>();
int outputRowIndex = 1;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
for (DedupeCandidateRow candidateRow : readResult.rows()) {
String asinValue = candidateRow.asinValue();
if (!asinValue.isBlank()) {
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
continue;
}
}
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
continue;
}
if (!writtenAsinValues.add(asinValue)) {
continue;
}
if (!writtenAsinValues.add(asinValue)) {
continue;
}
}
Row outputRow = outputSheet.createRow(outputRowIndex++);
for (int i = 0; i < selectedColumns.size(); i++) {
Integer sourceIndex = headerMap.get(selectedColumns.get(i));
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
outputRow.createCell(i).setCellValue(value);
outputRows++;
for (int i = 0; i < candidateRow.selectedValues().size(); i++) {
outputRow.createCell(i).setCellValue(candidateRow.selectedValues().get(i));
}
}
@@ -334,6 +316,157 @@ public class DedupeRunService {
}
outputWorkbook.dispose();
}
long writeNs = elapsedNs(writeStartNs);
log.info(
"dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}",
inputFile.getName(),
readResult.scannedRows(),
readResult.rows().size(),
candidateAsinValues.size(),
matchedAsinValues.size(),
outputRows,
toMs(readNs),
toMs(dbNs),
toMs(writeNs),
toMs(readNs + dbNs + writeNs)
);
}
private DedupeReadResult readDedupeRows(File inputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
List<DedupeCandidateRow> rows = new ArrayList<>();
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a");
}
Map<String, Integer> headerMap = buildHeaderMap(headerRow, formatter);
if (headerMap.isEmpty()) {
throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a");
}
List<String> missing = selectedColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("\u7f3a\u5c11\u5217\uff1a" + String.join("\u3001", missing));
}
List<Integer> selectedIndexes = new ArrayList<>(selectedColumns.size());
for (String selectedColumn : selectedColumns) {
selectedIndexes.add(headerMap.get(selectedColumn));
}
Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
int scannedRows = 0;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
scannedRows++;
if (idColumnIndex == null) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
continue;
}
appendRowByIdRule(
normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))),
row,
selectedIndexes,
asinColumnIndex,
formatter,
keepIntegerIds,
keepUnderscoreIds,
keepIntegerMainIdsWhenNoSubIds,
pendingMainIdGroup,
rows
);
}
pendingMainIdGroup.flush(rows);
return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows);
}
}
private Map<String, Integer> buildHeaderMap(Row headerRow, DataFormatter formatter) {
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeHeaderText(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank()) {
continue;
}
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if (HEADER_STOP_COLUMN.equals(value)) {
break;
}
}
return headerMap;
}
private String normalizeHeaderText(String value) {
String normalized = normalizeCellText(value);
int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER);
if (markerIndex > 0) {
String prefix = normalized.substring(0, markerIndex).trim();
if (!prefix.isBlank()) {
return prefix;
}
}
return normalized;
}
private DedupeCandidateRow buildCandidateRow(Row row, List<Integer> selectedIndexes, Integer asinColumnIndex, DataFormatter formatter) {
List<String> selectedValues = new ArrayList<>(selectedIndexes.size());
for (Integer selectedIndex : selectedIndexes) {
selectedValues.add(normalizeCellText(formatter.formatCellValue(row.getCell(selectedIndex))));
}
String asinValue = asinColumnIndex == null
? ""
: dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
return new DedupeCandidateRow(selectedValues, asinValue);
}
private void appendRowByIdRule(String idValue, Row row,
List<Integer> selectedIndexes, Integer asinColumnIndex,
DataFormatter formatter,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds,
PendingMainIdGroup pendingMainIdGroup,
List<DedupeCandidateRow> rows) {
if (idValue == null || idValue.isBlank()) {
return;
}
String mainId = extractMainId(idValue);
if (pendingMainIdGroup.hasDifferentMainId(mainId)) {
pendingMainIdGroup.flush(rows);
}
if (isUnderscoreId(idValue)) {
pendingMainIdGroup.discardIfSameMainId(mainId);
if (keepUnderscoreIds) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
}
return;
}
if (isIntegerId(idValue)) {
if (keepIntegerIds) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
return;
}
if (keepIntegerMainIdsWhenNoSubIds) {
pendingMainIdGroup.add(mainId, buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
}
}
}
private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
@@ -347,7 +480,7 @@ public class DedupeRunService {
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("去重结果打包失败:" + ex.getMessage());
throw new BusinessException("\u53bb\u91cd\u7ed3\u679c\u6253\u5305\u5931\u8d25\uff1a" + ex.getMessage());
}
return zipFile;
}
@@ -368,67 +501,61 @@ public class DedupeRunService {
return String.join("/", parts);
}
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds, Sheet sheet, int currentRowNum, int idColumnIndex, DataFormatter formatter) {
if (text == null || text.isBlank()) {
return false;
}
if (keepUnderscoreIds && text.matches("\\d+_\\d+")) {
return true;
}
if (keepIntegerIds && text.matches("\\d+")) {
return true;
}
if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) {
String mainId = extractMainId(text);
if (mainId.isEmpty()) return false;
boolean hasSubIdInGroup = false;
for (int r = currentRowNum + 1; r <= sheet.getLastRowNum(); r++) {
Row nextRow = sheet.getRow(r);
if (nextRow == null) continue;
Cell cell = nextRow.getCell(idColumnIndex);
if (cell == null) continue;
String nextIdValue = normalizeCellText(formatter.formatCellValue(cell));
if (nextIdValue.isBlank()) continue;
String nextMainId = extractMainId(nextIdValue);
if (nextMainId.equals(mainId)) {
if (nextIdValue.matches("\\d+_\\d+")) {
hasSubIdInGroup = true;
break;
}
} else {
// Encountered a different main ID. The group for 'mainId' has ended.
break;
}
}
return !hasSubIdInGroup;
}
return false;
}
private String extractMainId(String text) {
if (text == null || text.isBlank()) {
return "";
}
if (text.matches("\\d+")) {
if (isIntegerId(text)) {
return text;
}
if (text.matches("\\d+_\\d+")) {
if (isUnderscoreId(text)) {
int idx = text.indexOf('_');
return idx > 0 ? text.substring(0, idx) : "";
}
return "";
}
private boolean isIntegerId(String text) {
if (text == null || text.isEmpty()) {
return false;
}
for (int i = 0; i < text.length(); i++) {
if (!Character.isDigit(text.charAt(i))) {
return false;
}
}
return true;
}
private boolean isUnderscoreId(String text) {
if (text == null || text.length() < 3) {
return false;
}
int underscoreIndex = text.indexOf('_');
if (underscoreIndex <= 0 || underscoreIndex == text.length() - 1 || text.indexOf('_', underscoreIndex + 1) >= 0) {
return false;
}
for (int i = 0; i < text.length(); i++) {
if (i == underscoreIndex) {
continue;
}
if (!Character.isDigit(text.charAt(i))) {
return false;
}
}
return true;
}
private File findLocalSourceFile(String fileKey) {
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
if (!baseDir.exists()) {
return null;
}
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
File[] files = baseDir.listFiles(pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
if (files == null || files.length == 0) {
return null;
}
return files[0];
}
private File buildNamedOutputFile(File outputDir, String filename) {
@@ -461,16 +588,102 @@ public class DedupeRunService {
if (value == null) {
return "";
}
return value.replace("", "")
.replace(" ", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")
.replace("\t", " ")
.trim()
.replaceAll("\\s+", " ");
int start = 0;
int end = value.length();
while (start < end && isTrimmedWhitespace(value.charAt(start))) {
start++;
}
while (end > start && isTrimmedWhitespace(value.charAt(end - 1))) {
end--;
}
if (start >= end) {
return "";
}
StringBuilder builder = null;
boolean previousWhitespace = false;
for (int i = start; i < end; i++) {
char ch = value.charAt(i);
if (ch == '\uFEFF') {
if (builder == null) {
builder = new StringBuilder(value.length());
builder.append(value, start, i);
}
continue;
}
if (isNormalizedWhitespace(ch)) {
if (builder == null) {
builder = new StringBuilder(value.length());
builder.append(value, start, i);
}
if (!previousWhitespace) {
builder.append(' ');
}
previousWhitespace = true;
continue;
}
if (builder != null) {
builder.append(ch);
}
previousWhitespace = false;
}
return builder == null ? value.substring(start, end) : builder.toString();
}
private boolean isTrimmedWhitespace(char ch) {
return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch);
}
private boolean isNormalizedWhitespace(char ch) {
return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch);
}
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
}
private long elapsedNs(long startNs) {
return System.nanoTime() - startNs;
}
private long toMs(long nanos) {
return nanos / 1_000_000L;
}
private record DedupeReadResult(String sheetName, List<DedupeCandidateRow> rows, int scannedRows) {
}
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
}
private static final class PendingMainIdGroup {
private String mainId;
private final List<DedupeCandidateRow> rows = new ArrayList<>();
private void add(String nextMainId, DedupeCandidateRow row) {
if (hasDifferentMainId(nextMainId)) {
rows.clear();
}
mainId = nextMainId;
rows.add(row);
}
private boolean hasDifferentMainId(String nextMainId) {
return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId));
}
private void discardIfSameMainId(String nextMainId) {
if (mainId != null && mainId.equals(nextMainId)) {
rows.clear();
mainId = null;
}
}
private void flush(List<DedupeCandidateRow> outputRows) {
if (!rows.isEmpty()) {
outputRows.addAll(rows);
rows.clear();
}
mainId = null;
}
}
}
@@ -38,7 +38,7 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class DedupeTotalDataService {
private static final int COMPARE_BATCH_SIZE = 1000;
private static final int COMPARE_BATCH_SIZE = 5000;
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();