diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index b5f5bb0..2d1ad09 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.IdUtil; import cn.hutool.json.JSONUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest; @@ -15,11 +16,13 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.util.WorkbookUtil; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; @@ -39,12 +42,14 @@ import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; - @Service @RequiredArgsConstructor +@Slf4j public class DedupeRunService { + private static final String HEADER_SPLIT_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf"; + private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408"; + private final FileTaskMapper fileTaskMapper; private final FileResultMapper fileResultMapper; private final StorageProperties storageProperties; @@ -52,8 +57,9 @@ public class DedupeRunService { private final DedupeTotalDataService dedupeTotalDataService; public DedupeRunVo run(DedupeRunRequest request) { + long runStartNs = System.nanoTime(); if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) { - throw new BusinessException("请至少选择一种 ID 保留规则"); + throw new BusinessException("\u8bf7\u81f3\u5c11\u9009\u62e9\u4e00\u79cd ID \u4fdd\u7559\u89c4\u5219"); } FileTaskEntity task = new FileTaskEntity(); @@ -80,9 +86,11 @@ public class DedupeRunService { DedupeResultItemVo item = new DedupeResultItemVo(); item.setSourceFilename(sourceFile.getOriginalFilename()); try { + long fileStartNs = System.nanoTime(); File inputFile = findLocalSourceFile(sourceFile.getFileKey()); + long findFileNs = elapsedNs(fileStartNs); if (inputFile == null || !inputFile.exists()) { - throw new BusinessException("上传文件不存在，请重新上传"); + throw new BusinessException("\u4e0a\u4f20\u6587\u4ef6\u4e0d\u5b58\u5728\uff0c\u8bf7\u91cd\u65b0\u4e0a\u4f20"); } String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); @@ -90,6 +98,7 @@ public class DedupeRunService { File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); File outputFile = buildNamedOutputFile(outputDir, outputFilename); + long cleanStartNs = System.nanoTime(); cleanExcelByLegacyRules( inputFile, outputFile, @@ -98,6 +107,7 @@ public class DedupeRunService { request.isKeepUnderscoreIds(), request.isKeepIntegerMainIdsWhenNoSubIds() ); + long cleanNs = elapsedNs(cleanStartNs); if (folderMode) { archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile)); @@ -105,13 +115,15 @@ public class DedupeRunService { continue; } - String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE"); - // 只存 objectKey，不存预签名 URL + long uploadStartNs = System.nanoTime(); + OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE"); + long uploadNs = elapsedNs(uploadStartNs); + String ossObjectKey = uploadedResult.objectKey(); String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName(); item.setSuccess(true); item.setOutputFilename(downloadFilename); - item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey)); + item.setDownloadUrl(uploadedResult.downloadUrl()); successCount++; FileResultEntity resultEntity = new FileResultEntity(); @@ -119,13 +131,26 @@ public class DedupeRunService { resultEntity.setModuleType("DEDUPE"); resultEntity.setSourceFilename(inputName); resultEntity.setResultFilename(downloadFilename); - resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey + resultEntity.setResultFileUrl(ossObjectKey); resultEntity.setResultFileSize(outputFile.length()); resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); resultEntity.setSuccess(1); resultEntity.setUserId(request.getUserId()); resultEntity.setCreatedAt(LocalDateTime.now()); + long resultInsertStartNs = System.nanoTime(); fileResultMapper.insert(resultEntity); + long resultInsertNs = elapsedNs(resultInsertStartNs); + log.info( + "dedupe run file done fileKey={} filename={} size={} findFileMs={} cleanMs={} uploadMs={} resultInsertMs={} totalFileMs={}", + sourceFile.getFileKey(), + inputName, + inputFile.length(), + toMs(findFileNs), + toMs(cleanNs), + toMs(uploadNs), + toMs(resultInsertNs), + toMs(elapsedNs(fileStartNs)) + ); } catch (Exception ex) { item.setSuccess(false); item.setError(ex.getMessage()); @@ -146,20 +171,20 @@ public class DedupeRunService { if (folderMode && !archiveEntries.isEmpty()) { File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries); - String ossObjectKey = ossStorageService.uploadResultFile(zipFile, "DEDUPE"); - // 只存 objectKey + OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE"); + String ossObjectKey = uploadedResult.objectKey(); DedupeResultItemVo item = new DedupeResultItemVo(); item.setSourceFilename(request.getArchiveName()); item.setOutputFilename(zipFile.getName()); item.setSuccess(true); - item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey)); + item.setDownloadUrl(uploadedResult.downloadUrl()); FileResultEntity resultEntity = new FileResultEntity(); resultEntity.setTaskId(task.getId()); resultEntity.setModuleType("DEDUPE"); resultEntity.setSourceFilename(request.getArchiveName()); resultEntity.setResultFilename(zipFile.getName()); - resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey + resultEntity.setResultFileUrl(ossObjectKey); resultEntity.setResultFileSize(zipFile.length()); resultEntity.setResultContentType("application/zip"); resultEntity.setSuccess(1); @@ -181,6 +206,14 @@ public class DedupeRunService { vo.setSuccessCount(successCount); vo.setFailedCount(failedCount); vo.setItems(items); + log.info( + "dedupe run done userId={} files={} success={} failed={} totalMs={}", + request.getUserId(), + request.getFiles().size(), + successCount, + failedCount, + toMs(elapsedNs(runStartNs)) + ); return vo; } @@ -209,7 +242,7 @@ public class DedupeRunService { public void deleteHistory(Long resultId, Long userId) { FileResultEntity entity = fileResultMapper.selectById(resultId); if (entity == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { - throw new BusinessException("记录不存在"); + throw new BusinessException("\u8bb0\u5f55\u4e0d\u5b58\u5728"); } fileResultMapper.deleteById(resultId); } @@ -217,115 +250,64 @@ public class DedupeRunService { public Resource getResultFile(Long resultId, Long userId) { FileResultEntity entity = fileResultMapper.selectById(resultId); if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { - throw new BusinessException("结果文件不存在"); + throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728"); } File file = new File(entity.getResultFileUrl()); if (!file.exists()) { - throw new BusinessException("结果文件不存在"); + throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728"); } return new FileSystemResource(file); } private void cleanExcelByLegacyRules(File inputFile, File outputFile, List selectedColumns, - boolean keepIntegerIds, boolean keepUnderscoreIds, - boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { - DataFormatter formatter = new DataFormatter(); - try (FileInputStream fis = new FileInputStream(inputFile); - Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis); - SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) { - Sheet sheet = workbook.getSheetAt(0); - Row headerRow = sheet.getRow(0); - if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); - } + boolean keepIntegerIds, boolean keepUnderscoreIds, + boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { + long readStartNs = System.nanoTime(); + DedupeReadResult readResult = readDedupeRows( + inputFile, + selectedColumns, + keepIntegerIds, + keepUnderscoreIds, + keepIntegerMainIdsWhenNoSubIds + ); + long readNs = elapsedNs(readStartNs); - Map headerMap = new HashMap<>(); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell)); - if (value.isBlank()) { - continue; - } - value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty() - ? value - : value.split("idASIN国家状态价格变体数量", 2)[0].trim(); - if (headerMap.containsKey(value)) { - continue; - } - headerMap.put(value, i); - if ("缩略图地址8".equals(value)) { - break; - } + Set candidateAsinValues = new HashSet<>(); + for (DedupeCandidateRow row : readResult.rows()) { + if (!row.asinValue().isBlank()) { + candidateAsinValues.add(row.asinValue()); } + } + long dbStartNs = System.nanoTime(); + Set matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues); + long dbNs = elapsedNs(dbStartNs); - List missing = selectedColumns.stream().filter(column -> !headerMap.containsKey(column)).toList(); - if (!missing.isEmpty()) { - throw new BusinessException("缺少列：" + String.join("、", missing)); - } - - Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName()); + long writeStartNs = System.nanoTime(); + int outputRows = 0; + try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) { + org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName())); Row outputHeaderRow = outputSheet.createRow(0); for (int i = 0; i < selectedColumns.size(); i++) { outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i)); } - Integer idColumnIndex = headerMap.get("id"); - Integer asinColumnIndex = headerMap.get("ASIN"); - if (idColumnIndex != null) { - // The preliminary global scan for mainIdHasSubIdMap was removed. - // We now determine subset existence contextually (per group) during the main loop. - } - Set candidateAsinValues = new HashSet<>(); - for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { - continue; - } - if (idColumnIndex != null) { - String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))); - if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) { - continue; - } - } - if (asinColumnIndex != null) { - String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex))); - if (!asinValue.isBlank()) { - candidateAsinValues.add(asinValue); - } - } - } - - Set matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues); Set writtenAsinValues = new HashSet<>(); int outputRowIndex = 1; - for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { - continue; - } - if (idColumnIndex != null) { - String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))); - if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) { + for (DedupeCandidateRow candidateRow : readResult.rows()) { + String asinValue = candidateRow.asinValue(); + if (!asinValue.isBlank()) { + if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) { continue; } - } - if (asinColumnIndex != null) { - String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex))); - if (!asinValue.isBlank()) { - if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) { - continue; - } - if (!writtenAsinValues.add(asinValue)) { - continue; - } + if (!writtenAsinValues.add(asinValue)) { + continue; } } Row outputRow = outputSheet.createRow(outputRowIndex++); - for (int i = 0; i < selectedColumns.size(); i++) { - Integer sourceIndex = headerMap.get(selectedColumns.get(i)); - String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); - outputRow.createCell(i).setCellValue(value); + outputRows++; + for (int i = 0; i < candidateRow.selectedValues().size(); i++) { + outputRow.createCell(i).setCellValue(candidateRow.selectedValues().get(i)); } } @@ -334,6 +316,157 @@ public class DedupeRunService { } outputWorkbook.dispose(); } + long writeNs = elapsedNs(writeStartNs); + log.info( + "dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}", + inputFile.getName(), + readResult.scannedRows(), + readResult.rows().size(), + candidateAsinValues.size(), + matchedAsinValues.size(), + outputRows, + toMs(readNs), + toMs(dbNs), + toMs(writeNs), + toMs(readNs + dbNs + writeNs) + ); + } + + private DedupeReadResult readDedupeRows(File inputFile, List selectedColumns, + boolean keepIntegerIds, boolean keepUnderscoreIds, + boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { + List rows = new ArrayList<>(); + PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup(); + DataFormatter formatter = new DataFormatter(); + + try (FileInputStream fis = new FileInputStream(inputFile); + Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a"); + } + + Map headerMap = buildHeaderMap(headerRow, formatter); + if (headerMap.isEmpty()) { + throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a"); + } + List missing = selectedColumns.stream() + .filter(column -> !headerMap.containsKey(column)) + .toList(); + if (!missing.isEmpty()) { + throw new BusinessException("\u7f3a\u5c11\u5217\uff1a" + String.join("\u3001", missing)); + } + + List selectedIndexes = new ArrayList<>(selectedColumns.size()); + for (String selectedColumn : selectedColumns) { + selectedIndexes.add(headerMap.get(selectedColumn)); + } + Integer idColumnIndex = headerMap.get("id"); + Integer asinColumnIndex = headerMap.get("ASIN"); + + int scannedRows = 0; + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + continue; + } + scannedRows++; + if (idColumnIndex == null) { + rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); + continue; + } + appendRowByIdRule( + normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))), + row, + selectedIndexes, + asinColumnIndex, + formatter, + keepIntegerIds, + keepUnderscoreIds, + keepIntegerMainIdsWhenNoSubIds, + pendingMainIdGroup, + rows + ); + } + + pendingMainIdGroup.flush(rows); + return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows); + } + } + + private Map buildHeaderMap(Row headerRow, DataFormatter formatter) { + Map headerMap = new HashMap<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeHeaderText(cell == null ? null : formatter.formatCellValue(cell)); + if (value.isBlank()) { + continue; + } + if (headerMap.containsKey(value)) { + continue; + } + headerMap.put(value, i); + if (HEADER_STOP_COLUMN.equals(value)) { + break; + } + } + return headerMap; + } + + private String normalizeHeaderText(String value) { + String normalized = normalizeCellText(value); + int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER); + if (markerIndex > 0) { + String prefix = normalized.substring(0, markerIndex).trim(); + if (!prefix.isBlank()) { + return prefix; + } + } + return normalized; + } + + private DedupeCandidateRow buildCandidateRow(Row row, List selectedIndexes, Integer asinColumnIndex, DataFormatter formatter) { + List selectedValues = new ArrayList<>(selectedIndexes.size()); + for (Integer selectedIndex : selectedIndexes) { + selectedValues.add(normalizeCellText(formatter.formatCellValue(row.getCell(selectedIndex)))); + } + String asinValue = asinColumnIndex == null + ? "" + : dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex))); + return new DedupeCandidateRow(selectedValues, asinValue); + } + + private void appendRowByIdRule(String idValue, Row row, + List selectedIndexes, Integer asinColumnIndex, + DataFormatter formatter, + boolean keepIntegerIds, boolean keepUnderscoreIds, + boolean keepIntegerMainIdsWhenNoSubIds, + PendingMainIdGroup pendingMainIdGroup, + List rows) { + if (idValue == null || idValue.isBlank()) { + return; + } + String mainId = extractMainId(idValue); + if (pendingMainIdGroup.hasDifferentMainId(mainId)) { + pendingMainIdGroup.flush(rows); + } + if (isUnderscoreId(idValue)) { + pendingMainIdGroup.discardIfSameMainId(mainId); + if (keepUnderscoreIds) { + rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); + } + return; + } + if (isIntegerId(idValue)) { + if (keepIntegerIds) { + rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); + return; + } + if (keepIntegerMainIdsWhenNoSubIds) { + pendingMainIdGroup.add(mainId, buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter)); + } + } } private File packageFolderDedupeResultsAsZip(String archiveName, List archiveEntries) { @@ -347,7 +480,7 @@ public class DedupeRunService { zos.closeEntry(); } } catch (Exception ex) { - throw new BusinessException("去重结果打包失败：" + ex.getMessage()); + throw new BusinessException("\u53bb\u91cd\u7ed3\u679c\u6253\u5305\u5931\u8d25\uff1a" + ex.getMessage()); } return zipFile; } @@ -368,67 +501,61 @@ public class DedupeRunService { return String.join("/", parts); } - private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds, - boolean keepIntegerMainIdsWhenNoSubIds, Sheet sheet, int currentRowNum, int idColumnIndex, DataFormatter formatter) { - if (text == null || text.isBlank()) { - return false; - } - if (keepUnderscoreIds && text.matches("\\d+_\\d+")) { - return true; - } - if (keepIntegerIds && text.matches("\\d+")) { - return true; - } - if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) { - String mainId = extractMainId(text); - if (mainId.isEmpty()) return false; - - boolean hasSubIdInGroup = false; - for (int r = currentRowNum + 1; r <= sheet.getLastRowNum(); r++) { - Row nextRow = sheet.getRow(r); - if (nextRow == null) continue; - Cell cell = nextRow.getCell(idColumnIndex); - if (cell == null) continue; - String nextIdValue = normalizeCellText(formatter.formatCellValue(cell)); - if (nextIdValue.isBlank()) continue; - - String nextMainId = extractMainId(nextIdValue); - if (nextMainId.equals(mainId)) { - if (nextIdValue.matches("\\d+_\\d+")) { - hasSubIdInGroup = true; - break; - } - } else { - // Encountered a different main ID. The group for 'mainId' has ended. - break; - } - } - return !hasSubIdInGroup; - } - return false; - } - private String extractMainId(String text) { if (text == null || text.isBlank()) { return ""; } - if (text.matches("\\d+")) { + if (isIntegerId(text)) { return text; } - if (text.matches("\\d+_\\d+")) { + if (isUnderscoreId(text)) { int idx = text.indexOf('_'); return idx > 0 ? text.substring(0, idx) : ""; } return ""; } + private boolean isIntegerId(String text) { + if (text == null || text.isEmpty()) { + return false; + } + for (int i = 0; i < text.length(); i++) { + if (!Character.isDigit(text.charAt(i))) { + return false; + } + } + return true; + } + + private boolean isUnderscoreId(String text) { + if (text == null || text.length() < 3) { + return false; + } + int underscoreIndex = text.indexOf('_'); + if (underscoreIndex <= 0 || underscoreIndex == text.length() - 1 || text.indexOf('_', underscoreIndex + 1) >= 0) { + return false; + } + for (int i = 0; i < text.length(); i++) { + if (i == underscoreIndex) { + continue; + } + if (!Character.isDigit(text.charAt(i))) { + return false; + } + } + return true; + } + private File findLocalSourceFile(String fileKey) { File baseDir = FileUtil.file(storageProperties.getLocalTempDir()); if (!baseDir.exists()) { return null; } - List matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); - return matchedFiles.isEmpty() ? null : matchedFiles.getFirst(); + File[] files = baseDir.listFiles(pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); + if (files == null || files.length == 0) { + return null; + } + return files[0]; } private File buildNamedOutputFile(File outputDir, String filename) { @@ -461,16 +588,102 @@ public class DedupeRunService { if (value == null) { return ""; } - return value.replace("", "") - .replace("　", " ") - .replace("\r\n", " ") - .replace("\r", " ") - .replace("\n", " ") - .replace("\t", " ") - .trim() - .replaceAll("\\s+", " "); + int start = 0; + int end = value.length(); + while (start < end && isTrimmedWhitespace(value.charAt(start))) { + start++; + } + while (end > start && isTrimmedWhitespace(value.charAt(end - 1))) { + end--; + } + if (start >= end) { + return ""; + } + + StringBuilder builder = null; + boolean previousWhitespace = false; + for (int i = start; i < end; i++) { + char ch = value.charAt(i); + if (ch == '\uFEFF') { + if (builder == null) { + builder = new StringBuilder(value.length()); + builder.append(value, start, i); + } + continue; + } + if (isNormalizedWhitespace(ch)) { + if (builder == null) { + builder = new StringBuilder(value.length()); + builder.append(value, start, i); + } + if (!previousWhitespace) { + builder.append(' '); + } + previousWhitespace = true; + continue; + } + if (builder != null) { + builder.append(ch); + } + previousWhitespace = false; + } + return builder == null ? value.substring(start, end) : builder.toString(); + } + + private boolean isTrimmedWhitespace(char ch) { + return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch); + } + + private boolean isNormalizedWhitespace(char ch) { + return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch); } private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) { } + + private long elapsedNs(long startNs) { + return System.nanoTime() - startNs; + } + + private long toMs(long nanos) { + return nanos / 1_000_000L; + } + + private record DedupeReadResult(String sheetName, List rows, int scannedRows) { + } + + private record DedupeCandidateRow(List selectedValues, String asinValue) { + } + + private static final class PendingMainIdGroup { + private String mainId; + private final List rows = new ArrayList<>(); + + private void add(String nextMainId, DedupeCandidateRow row) { + if (hasDifferentMainId(nextMainId)) { + rows.clear(); + } + mainId = nextMainId; + rows.add(row); + } + + private boolean hasDifferentMainId(String nextMainId) { + return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId)); + } + + private void discardIfSameMainId(String nextMainId) { + if (mainId != null && mainId.equals(nextMainId)) { + rows.clear(); + mainId = null; + } + } + + private void flush(List outputRows) { + if (!rows.isEmpty()) { + outputRows.addAll(rows); + rows.clear(); + } + mainId = null; + } + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java index 0516837..abcf23b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -38,7 +38,7 @@ import java.util.concurrent.ConcurrentHashMap; @RequiredArgsConstructor public class DedupeTotalDataService { - private static final int COMPARE_BATCH_SIZE = 1000; + private static final int COMPARE_BATCH_SIZE = 5000; private final DedupeTotalDataMapper dedupeTotalDataMapper; private final Map importProgressMap = new ConcurrentHashMap<>(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java index e80de33..9ebe69b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java @@ -298,6 +298,8 @@ public class DeleteBrandStaleTaskService { ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats(); long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes()); long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getShopMatchInitialTimeoutMinutes()); + long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis(); + long nowMillis = System.currentTimeMillis(); LocalDateTime now = LocalDateTime.now(); LocalDateTime threshold = now.minusMinutes(minutes); LocalDateTime initialThreshold = now.minusMinutes(initialMinutes); @@ -313,10 +315,17 @@ public class DeleteBrandStaleTaskService { log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}", runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes); for (FileTaskEntity task : runningTasks) { + long lastHeartbeatMillis = shopMatchTaskCacheService.getTaskHeartbeatMillis(task.getId()); boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId()); boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0) || (task.getFailedFileCount() != null && task.getFailedFileCount() > 0); - boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows; + boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows; + if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) { + stats.skippedTaskCount++; + log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}", + task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt()); + continue; + } if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { stats.skippedTaskCount++; log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}", diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java index 5fc49f3..d7ba8bf 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -37,6 +37,19 @@ public class OssStorageService { } } + public UploadedResult uploadResultFileWithFreshDownloadUrl(File file, String moduleType) { + String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName()); + OSS ossClient = buildClient(); + try { + ossClient.putObject(ossProperties.getBucket(), objectKey, file); + Date expiration = new Date(System.currentTimeMillis() + 3600_000L); + URL url = ossClient.generatePresignedUrl(ossProperties.getBucket(), objectKey, expiration); + return new UploadedResult(objectKey, url.toString()); + } finally { + ossClient.shutdown(); + } + } + public String uploadText(String objectKey, String content) { if (objectKey == null || objectKey.isBlank()) { throw new IllegalArgumentException("objectKey must not be blank"); @@ -160,4 +173,7 @@ public class OssStorageService { ossProperties.getAccessKeySecret() ); } + + public record UploadedResult(String objectKey, String downloadUrl) { + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java index 6e1a10f..c0859ee 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java @@ -1,6 +1,11 @@ package com.nanri.aiimage.modules.queryasin.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinTaskBatchRequest; @@ -9,11 +14,6 @@ import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinHistoryVo; import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinTaskBatchVo; import com.nanri.aiimage.modules.queryasin.service.QueryAsinResolveService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; -import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; -import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; @@ -46,14 +46,14 @@ import java.util.List; @RequestMapping("/api/query-asin") @Tag( name = "查询ASIN", - description = "查询ASIN模块：管理备选店铺、批量匹配紫鸟店铺、创建任务、接收 Python 分片结果、自动收尾生成 Excel 并提供下载。涉及查询和删除的接口需要携带 user_id。") + description = "查询 ASIN 模块：管理备选店铺、批量匹配紫鸟店铺、创建任务、接收 Python 分片结果、自动收尾生成 Excel 并提供下载。查询和删除接口需要携带 user_id。") public class QueryAsinTaskController { private final QueryAsinResolveService queryAsinResolveService; private final QueryAsinTaskService queryAsinTaskService; @GetMapping("/candidates") - @Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询ASIN模块中已保存的备选店铺。") + @Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询 ASIN 模块中已保存的备选店铺。") public ApiResponse> listCandidates( @Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1") @RequestParam("user_id") Long userId) { @@ -61,10 +61,10 @@ public class QueryAsinTaskController { } @PostMapping("/candidates") - @Operation(summary = "新增备选店铺", description = "向查询ASIN备选区新增一条店铺记录，供后续匹配和创建任务使用。") + @Operation(summary = "新增备选店铺", description = "向查询 ASIN 备选区新增一条店铺记录，供后续匹配和创建任务使用。") public ApiResponse addCandidate( @io.swagger.v3.oas.annotations.parameters.RequestBody( - description = "新增备选店铺请求，需传入用户 ID 和店铺名。", + description = "新增备选店铺请求，需要传入用户 ID 和店铺名。", required = true, content = @Content( mediaType = "application/json", @@ -93,7 +93,7 @@ public class QueryAsinTaskController { } @PostMapping("/match-shops") - @Operation(summary = "批量匹配店铺", description = "根据店铺名批量匹配紫鸟店铺索引，返回是否命中、店铺 ID、平台、公司和匹配状态。") + @Operation(summary = "批量匹配店铺", description = "根据店铺名称批量匹配紫鸟店铺索引，返回是否命中、店铺 ID、平台、公司和匹配状态。") public ApiResponse matchShops( @io.swagger.v3.oas.annotations.parameters.RequestBody( description = "批量匹配请求，传入 user_id 和待匹配的店铺名称列表。", @@ -125,7 +125,7 @@ public class QueryAsinTaskController { } @GetMapping("/history") - @Operation(summary = "查询任务记录", description = "返回当前用户在查询ASIN模块中的当前任务和历史任务记录。") + @Operation(summary = "查询任务记录", description = "返回当前用户在查询 ASIN 模块中的当前任务和历史任务记录。") public ApiResponse history( @Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1") @RequestParam("user_id") Long userId) { @@ -133,16 +133,16 @@ public class QueryAsinTaskController { } @PostMapping("/tasks/progress/batch") - @Operation(summary = "批量查询查询ASIN任务进度", description = "仅返回任务状态和店铺结果摘要，用于前端轮询降载。") + @Operation(summary = "批量查询查询 ASIN 任务进度", description = "仅返回任务状态和店铺结果摘要，用于前端轮询降载。") public ApiResponse taskProgressBatch(@Valid @RequestBody QueryAsinTaskBatchRequest request) { return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds())); } @PostMapping("/tasks") - @Operation(summary = "创建查询ASIN任务", description = "根据已匹配的店铺创建任务和占位结果记录，后续由 Python 端处理并回传结果。") + @Operation(summary = "创建查询 ASIN 任务", description = "根据已匹配的店铺创建任务和占位结果记录，并把后台维护的整张 ASIN 表数据随店铺项返回给 Python 端。") public ApiResponse createTask( @io.swagger.v3.oas.annotations.parameters.RequestBody( - description = "创建任务请求。items 中每一项代表一个待处理店铺，需携带默认模板结构。", + description = "创建任务请求。items 中每一项代表一个待处理店铺，queryAsins 是后台维护的整张 ASIN 表数据，不按店铺过滤。", required = true, content = @Content( mediaType = "application/json", @@ -161,23 +161,14 @@ public class QueryAsinTaskController { "companyName": "示例公司", "matchStatus": "MATCHED", "matchMessage": "索引已命中", - "countrySections": [ + "queryAsins": [ { - "country": "德国", - "rows": [ - { - "status": "全部", - "quantity": "", - "deleteQuantity": "", - "processStatus": "" - } - ] - } - ], - "cartRatios": [ + "country": "DE", + "asins": ["B0BRZZR3N2", "B0BRZZR3N3"] + }, { - "country": "德国", - "ratio": "" + "country": "FR", + "asins": ["B0BRZZR3N4"] } ] } @@ -190,124 +181,72 @@ public class QueryAsinTaskController { @PostMapping("/tasks/{taskId}/result") @Operation( - summary = "提交查询ASIN结果", - description = "供 Python 端回传处理结果。支持按店铺分片多次提交，服务端会合并分片、按店铺完成状态自动收尾，并在任务结束后生成 Excel。") + summary = "提交查询 ASIN 结果", + description = "供 Python 端回传查询 ASIN 结果。业务数据包含店铺名，以及德国、英国、法国、意大利、西班牙 5 个国家下的 ASIN 和处理状态；submissionId 用于日志追踪，shopDone 可不传，不传时后端默认该店铺已提交完成。") public ApiResponse submitResult( - @Parameter(description = "任务主键，必须是运行中的任务", example = "3089") + @Parameter(description = "任务主键，必须是运行中的查询 ASIN 任务", example = "3089") @PathVariable Long taskId, @io.swagger.v3.oas.annotations.parameters.RequestBody( - description = "查询ASIN结果回传请求。shops 表示本次提交的店铺结果分片列表。shopDone=true 表示该店铺已全部提交完成。", + description = "查询 ASIN 结果回传请求。shops 表示店铺结果列表；countryResults 对应导出表中的国家 ASIN 列和处理状态列；error 用于回传单店铺失败原因。", required = true, content = @Content( mediaType = "application/json", schema = @Schema(implementation = QueryAsinSubmitResultRequest.class), - examples = { - @ExampleObject( - name = "成功分片示例", - summary = "一个店铺分片回传，包含状态数据和购物车比例数据", - value = """ + examples = @ExampleObject( + name = "回传示例", + summary = "按店铺和国家返回 ASIN 与处理状态", + value = """ + { + "shops": [ + { + "shopName": "郭亚芳", + "submissionId": "query-asin:3089:郭亚芳:1711111111111", + "shopDone": true, + "countryResults": [ { - "shops": [ + "country": "DE", + "items": [ { - "shopName": "郭亚芳", - "submissionId": "query-asin:3089:郭亚芳:1711111111111", - "chunkIndex": 1, - "chunkTotal": 2, - "shopDone": false, - "countrySections": [ - { - "country": "德国", - "rows": [ - { - "status": "正常", - "quantity": "12", - "deleteQuantity": "3", - "processStatus": "处理中" - }, - { - "status": "下架", - "quantity": "2", - "deleteQuantity": "2", - "processStatus": "已完成" - } - ] - }, - { - "country": "英国", - "rows": [ - { - "status": "正常", - "quantity": "5", - "deleteQuantity": "1", - "processStatus": "处理中" - } - ] - } - ], - "cartRatios": [ - { - "country": "德国", - "ratio": "25%" - }, - { - "country": "英国", - "ratio": "18%" - } - ] + "asin": "B0BRZZR3N2", + "status": "正常" + }, + { + "asin": "B0BRZZR3N3", + "status": "未找到" } ] - } - """), - @ExampleObject( - name = "店铺完成示例", - summary = "最后一片提交完成后，将 shopDone 置为 true", - value = """ + }, { - "shops": [ + "country": "UK", + "items": [ { - "shopName": "郭亚芳", - "submissionId": "query-asin:3089:郭亚芳:1711111111111", - "chunkIndex": 2, - "chunkTotal": 2, - "shopDone": true, - "countrySections": [ - { - "country": "法国", - "rows": [ - { - "status": "正常", - "quantity": "4", - "deleteQuantity": "0", - "processStatus": "已完成" - } - ] - } - ], - "cartRatios": [ - { - "country": "法国", - "ratio": "12%" - } - ] + "asin": "B0BRZZR3N6", + "status": "正常" } ] - } - """), - @ExampleObject( - name = "失败示例", - summary = "店铺处理失败时直接回传错误信息", - value = """ + }, { - "shops": [ + "country": "FR", + "items": [ { - "shopName": "郭亚芳", - "shopDone": true, - "error": "紫鸟页面加载超时，未能完成删除数据采集" + "asin": "B0BRZZR3N4", + "status": "正常" } ] + }, + { + "country": "IT", + "items": [] + }, + { + "country": "ES", + "items": [] } - """) - })) + ] + } + ] + } + """))) @Valid @RequestBody QueryAsinSubmitResultRequest request, jakarta.servlet.http.HttpServletResponse response) { response.setCharacterEncoding(StandardCharsets.UTF_8.name()); @@ -317,7 +256,7 @@ public class QueryAsinTaskController { } @GetMapping("/results/{resultId}/download") - @Operation(summary = "下载结果文件", description = "按结果记录下载服务端已生成并上传到 OSS 的查询ASIN Excel 文件。") + @Operation(summary = "下载结果文件", description = "按结果记录下载服务端已生成并上传到 OSS 的查询 ASIN Excel 文件。") public void downloadResult( @Parameter(description = "结果记录主键", example = "4599") @PathVariable Long resultId, @@ -348,7 +287,7 @@ public class QueryAsinTaskController { } @DeleteMapping("/tasks/{taskId}") - @Operation(summary = "删除任务", description = "删除整条查询ASIN任务以及该任务下关联的所有结果记录。") + @Operation(summary = "删除任务", description = "删除整条查询 ASIN 任务以及该任务下关联的所有结果记录。") public ApiResponse deleteTask( @Parameter(description = "任务主键", example = "3089") @PathVariable Long taskId, @@ -359,7 +298,7 @@ public class QueryAsinTaskController { } @DeleteMapping("/history/{resultId}") - @Operation(summary = "删除单条历史记录", description = "删除一条查询ASIN结果记录，并同步重算其所属任务状态。") + @Operation(summary = "删除单条历史记录", description = "删除一条查询 ASIN 结果记录，并同步重算其所属任务状态。") public ApiResponse deleteHistory( @Parameter(description = "结果记录主键", example = "4599") @PathVariable Long resultId, @@ -369,6 +308,3 @@ public class QueryAsinTaskController { return ApiResponse.success(null); } } - - - diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinAsinStatusDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinAsinStatusDto.java new file mode 100644 index 0000000..ed11fc1 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinAsinStatusDto.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.queryasin.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "查询 ASIN 单条结果") +public class QueryAsinAsinStatusDto { + + @Schema(description = "ASIN", example = "B0BRZZR3N2") + private String asin; + + @Schema(description = "Python 查询后返回的状态", example = "正常") + private String status; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCartRatioDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCartRatioDto.java deleted file mode 100644 index 7b5e5a3..0000000 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCartRatioDto.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.nanri.aiimage.modules.queryasin.model.dto; - -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; - -@Data -@Schema(description = "购物车比例数据") -public class QueryAsinCartRatioDto { - - @Schema(description = "国家名称", example = "德国") - private String country; - - @Schema(description = "购物车比例", example = "25%") - private String ratio; -} - diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryAsinsDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryAsinsDto.java index 99deac9..f2cfbf2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryAsinsDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryAsinsDto.java @@ -7,12 +7,12 @@ import java.util.ArrayList; import java.util.List; @Data -@Schema(description = "查询 ASIN 国家维度 ASIN 列表") +@Schema(description = "查询 ASIN 国家维度 ASIN 清单") public class QueryAsinCountryAsinsDto { @Schema(description = "国家编码，支持 DE、UK、FR、IT、ES", example = "DE") private String country; - @Schema(description = "该国家需要查询的 ASIN 列表") + @Schema(description = "后台维护的该国家 ASIN 清单，创建任务时会整体推给 Python") private List asins = new ArrayList<>(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryMetricRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryMetricRowDto.java deleted file mode 100644 index 8be4dfa..0000000 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryMetricRowDto.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.nanri.aiimage.modules.queryasin.model.dto; - -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; - -@Data -@Schema(description = "国家状态数据中的单行记录") -public class QueryAsinCountryMetricRowDto { - - @Schema(description = "商品状态", example = "正常") - private String status; - - @Schema(description = "该状态下的商品数量", example = "12") - private String quantity; - - @JsonProperty("deleteQuantity") - @Schema(description = "已删除数量", example = "3") - private String deleteQuantity; - - @JsonProperty("processStatus") - @Schema(description = "处理结果或处理状态", example = "处理中") - private String processStatus; -} - diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryResultDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryResultDto.java new file mode 100644 index 0000000..6be5a48 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryResultDto.java @@ -0,0 +1,18 @@ +package com.nanri.aiimage.modules.queryasin.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "查询 ASIN 国家维度结果") +public class QueryAsinCountryResultDto { + + @Schema(description = "国家编码，支持 DE、UK、FR、IT、ES", example = "DE") + private String country; + + @Schema(description = "该国家下的 ASIN 查询结果列表") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountrySectionDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountrySectionDto.java deleted file mode 100644 index 903d538..0000000 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountrySectionDto.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.nanri.aiimage.modules.queryasin.model.dto; - -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -@Data -@Schema(description = "单个国家的状态数据分组") -public class QueryAsinCountrySectionDto { - - @Schema(description = "国家名称", example = "德国") - private String country; - - @Schema(description = "该国家下的多行状态数据") - private List rows = new ArrayList<>(); -} - diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinShopPayloadDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinShopPayloadDto.java index de9ad04..f590cd9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinShopPayloadDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinShopPayloadDto.java @@ -8,42 +8,26 @@ import java.util.ArrayList; import java.util.List; @Data -@Schema(description = "单个店铺的查询ASIN结果分片") +@Schema(description = "单个店铺的查询 ASIN 结果") public class QueryAsinShopPayloadDto { @JsonProperty("shopName") - @Schema(description = "店铺名称，服务端按店铺名称聚合分片", example = "郭亚芳") + @Schema(description = "店铺名称", example = "郭亚芳") private String shopName; - @Schema(description = "店铺处理失败时的错误信息；有值时该店铺会直接标记失败", example = "紫鸟页面加载超时，未能完成删除数据采集") + @Schema(description = "店铺处理失败时的错误信息；有值时该店铺会直接标记失败", example = "紫鸟页面加载超时，未能完成 ASIN 查询") private String error; - @JsonProperty("countrySections") - @Schema(description = "各国家的状态数据列表。同一国家可分多片提交，服务端会按顺序合并") - private List countrySections = new ArrayList<>(); - - @JsonProperty("cartRatios") - @Schema(description = "购物车比例数据列表，通常只在店铺首行展示") - private List cartRatios = new ArrayList<>(); - - @JsonProperty("queryAsins") - @Schema(description = "Python 回传的查询 ASIN 结果分片；如果 Python 仅透传后台 ASIN，也按该字段提交") - private List queryAsins = new ArrayList<>(); + @JsonProperty("countryResults") + @Schema(description = "Python 回传的查询 ASIN 结果。每个国家下按 ASIN + 状态返回，对应导出表中的国家列和状态列") + private List countryResults = new ArrayList<>(); @JsonProperty("shopDone") - @Schema(description = "该店铺是否已全部提交完成。最后一片应传 true", example = "true") + @Schema(description = "该店铺是否已全部提交完成。不传时按已完成处理；分片提交时，中间分片传 false，最后一片传 true", example = "true") private Boolean shopDone; @JsonProperty("submissionId") - @Schema(description = "本次店铺提交批次标识，便于问题排查与日志跟踪", example = "query-asin:3089:郭亚芳:1711111111111") + @Schema(description = "本次店铺提交批次标识，便于问题排查与日志追踪", example = "query-asin:3089:郭亚芳:1711111111111") private String submissionId; - @JsonProperty("chunkIndex") - @Schema(description = "当前分片序号，建议从 1 开始", example = "1") - private Integer chunkIndex; - - @JsonProperty("chunkTotal") - @Schema(description = "当前店铺总分片数", example = "2") - private Integer chunkTotal; } - diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinSubmitResultRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinSubmitResultRequest.java index 96d0cf1..419eb00 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinSubmitResultRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinSubmitResultRequest.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.List; @Data -@Schema(description = "查询ASIN结果回传请求") +@Schema(description = "查询 ASIN 结果回传请求") public class QueryAsinSubmitResultRequest { @Valid @@ -17,4 +17,3 @@ public class QueryAsinSubmitResultRequest { @Schema(description = "本次提交的店铺结果分片列表，支持一次提交多个店铺", requiredMode = Schema.RequiredMode.REQUIRED) private List shops = new ArrayList<>(); } - diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinTaskItemDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinTaskItemDto.java index 54a4c81..89311ac 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinTaskItemDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinTaskItemDto.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.List; @Data -@Schema(description = "查询ASIN任务中的单个店铺项") +@Schema(description = "查询 ASIN 任务中的单个店铺项") public class QueryAsinTaskItemDto { @JsonProperty("shopName") @@ -40,6 +40,6 @@ public class QueryAsinTaskItemDto { @Valid @JsonProperty("queryAsins") - @Schema(description = "后台查询 ASIN 维护表中的国家 ASIN 列表；查询ASIN任务按全表返回，不按店铺过滤") + @Schema(description = "后台查询 ASIN 维护表中的国家 ASIN 清单；查询 ASIN 任务按全表返回，不按店铺过滤") private List queryAsins = new ArrayList<>(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java index a9bb3b3..14aa5d0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java @@ -1,9 +1,8 @@ package com.nanri.aiimage.modules.queryasin.model.vo; import com.fasterxml.jackson.annotation.JsonProperty; -import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCartRatioDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; -import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountrySectionDto; +import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto; import lombok.Data; import java.time.LocalDateTime; @@ -45,13 +44,9 @@ public class QueryAsinResultItemVo { private String outputFilename; private String downloadUrl; - @JsonProperty("countrySections") - private List countrySections = new ArrayList<>(); - - @JsonProperty("cartRatios") - private List cartRatios = new ArrayList<>(); - @JsonProperty("queryAsins") private List queryAsins = new ArrayList<>(); -} + @JsonProperty("countryResults") + private List countryResults = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinExcelAssemblyService.java index 3cb680a..14a4923 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinExcelAssemblyService.java @@ -1,7 +1,9 @@ package com.nanri.aiimage.modules.queryasin.service; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinAsinStatusDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; +import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto; import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinResultItemVo; import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Row; @@ -11,6 +13,7 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.FileOutputStream; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -20,7 +23,21 @@ import java.util.Map; public class QueryAsinExcelAssemblyService { private static final List COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES"); - private static final String[] HEADER = {"店铺名", "德国", "英国", "法国", "意大利", "西班牙"}; + private static final Map COUNTRY_NAMES = Map.of( + "DE", "德国", + "UK", "英国", + "FR", "法国", + "IT", "意大利", + "ES", "西班牙" + ); + private static final String[] HEADER = { + "店铺名", + "德国", "状态", + "英国", "状态", + "法国", "状态", + "意大利", "状态", + "西班牙", "状态" + }; public void writeWorkbook(File outputXlsx, List items) { SXSSFWorkbook workbook = new SXSSFWorkbook(200); @@ -38,14 +55,17 @@ public class QueryAsinExcelAssemblyService { if (item == null || Boolean.FALSE.equals(item.getSuccess())) { continue; } - Map> asinsByCountry = asinsByCountry(item.getQueryAsins()); - int maxRows = maxRows(asinsByCountry); + Map> resultsByCountry = resultsByCountry(item); + int maxRows = maxRows(resultsByCountry); for (int i = 0; i < maxRows; i++) { Row row = sheet.createRow(rowIndex++); row.createCell(0).setCellValue(i == 0 ? safe(item.getShopName()) : ""); for (int c = 0; c < COUNTRIES.size(); c++) { - List asins = asinsByCountry.get(COUNTRIES.get(c)); - row.createCell(c + 1).setCellValue(asins != null && i < asins.size() ? safe(asins.get(i)) : ""); + List rows = resultsByCountry.get(COUNTRIES.get(c)); + QueryAsinAsinStatusDto result = rows != null && i < rows.size() ? rows.get(i) : null; + int asinColumn = 1 + c * 2; + row.createCell(asinColumn).setCellValue(result == null ? "" : safe(result.getAsin())); + row.createCell(asinColumn + 1).setCellValue(result == null ? "" : safe(result.getStatus())); } } rowIndex++; @@ -53,7 +73,7 @@ public class QueryAsinExcelAssemblyService { workbook.write(outputStream); } catch (Exception ex) { log.warn("[query-asin] write workbook failed: {}", ex.getMessage()); - throw new BusinessException("generate query-asin excel failed: " + ex.getMessage()); + throw new BusinessException("生成查询 ASIN Excel 失败: " + ex.getMessage()); } finally { try { workbook.close(); @@ -70,42 +90,106 @@ public class QueryAsinExcelAssemblyService { } for (QueryAsinResultItemVo item : items) { if (item != null && !Boolean.FALSE.equals(item.getSuccess())) { - count += maxRows(asinsByCountry(item.getQueryAsins())); + count += maxRows(resultsByCountry(item)); } } return count; } - private Map> asinsByCountry(List queryAsins) { - Map> map = new LinkedHashMap<>(); - for (String country : COUNTRIES) { - map.put(country, List.of()); - } - if (queryAsins == null) { + private Map> resultsByCountry(QueryAsinResultItemVo item) { + Map> map = emptyCountryMap(); + if (item == null) { return map; } - for (QueryAsinCountryAsinsDto item : queryAsins) { - if (item == null || item.getCountry() == null) { - continue; + if (item.getCountryResults() != null && !item.getCountryResults().isEmpty()) { + for (QueryAsinCountryResultDto countryResult : item.getCountryResults()) { + String country = normalizeCountry(countryResult == null ? null : countryResult.getCountry()); + if (!map.containsKey(country)) { + continue; + } + map.put(country, normalizeResultItems(countryResult.getItems())); } - String country = item.getCountry().trim().toUpperCase(); - if (map.containsKey(country)) { - map.put(country, item.getAsins() == null ? List.of() : item.getAsins()); + return map; + } + + // 兼容旧链路：如果 Python 仍然只透传 queryAsins，则状态列留空。 + if (item.getQueryAsins() != null) { + for (QueryAsinCountryAsinsDto countryAsins : item.getQueryAsins()) { + String country = normalizeCountry(countryAsins == null ? null : countryAsins.getCountry()); + if (!map.containsKey(country)) { + continue; + } + List rows = new ArrayList<>(); + for (String asin : countryAsins.getAsins() == null ? List.of() : countryAsins.getAsins()) { + String normalizedAsin = normalizeAsin(asin); + if (normalizedAsin.isEmpty()) { + continue; + } + QueryAsinAsinStatusDto row = new QueryAsinAsinStatusDto(); + row.setAsin(normalizedAsin); + row.setStatus(""); + rows.add(row); + } + map.put(country, rows); } } return map; } - private int maxRows(Map> asinsByCountry) { + private Map> emptyCountryMap() { + Map> map = new LinkedHashMap<>(); + for (String country : COUNTRIES) { + map.put(country, List.of()); + } + return map; + } + + private List normalizeResultItems(List items) { + List out = new ArrayList<>(); + if (items == null) { + return out; + } + for (QueryAsinAsinStatusDto source : items) { + String asin = normalizeAsin(source == null ? null : source.getAsin()); + String status = source == null ? "" : safe(source.getStatus()).trim(); + if (asin.isEmpty() && status.isEmpty()) { + continue; + } + QueryAsinAsinStatusDto item = new QueryAsinAsinStatusDto(); + item.setAsin(asin); + item.setStatus(status); + out.add(item); + } + return out; + } + + private int maxRows(Map> resultsByCountry) { int max = 1; - for (List asins : asinsByCountry.values()) { - if (asins != null && asins.size() > max) { - max = asins.size(); + for (List rows : resultsByCountry.values()) { + if (rows != null && rows.size() > max) { + max = rows.size(); } } return max; } + private String normalizeCountry(String country) { + if (country == null) { + return ""; + } + String value = country.trim().toUpperCase(); + for (Map.Entry entry : COUNTRY_NAMES.entrySet()) { + if (entry.getValue().equals(country.trim())) { + return entry.getKey(); + } + } + return value; + } + + private String normalizeAsin(String value) { + return value == null ? "" : value.trim().toUpperCase(); + } + private String safe(String value) { return value == null ? "" : value; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java index 27eee63..e9b1eb6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java @@ -8,10 +8,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; -import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCartRatioDto; +import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinAsinStatusDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; -import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryMetricRowDto; -import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountrySectionDto; +import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest; @@ -523,11 +522,8 @@ public class QueryAsinTaskService { item.setDownloadUrl(blank(entity.getResultFileUrl()) ? item.getDownloadUrl() : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); - if (item.getCountrySections() == null) { - item.setCountrySections(new ArrayList<>()); - } - if (item.getCartRatios() == null) { - item.setCartRatios(new ArrayList<>()); + if (item.getCountryResults() == null) { + item.setCountryResults(new ArrayList<>()); } if (item.getQueryAsins() == null) { item.setQueryAsins(new ArrayList<>()); @@ -568,8 +564,7 @@ public class QueryAsinTaskService { vo.setError(result.getErrorMessage()); vo.setCreatedAt(result.getCreatedAt()); vo.setFinishedAt(finishedAt); - vo.setCountrySections(new ArrayList<>()); - vo.setCartRatios(new ArrayList<>()); + vo.setCountryResults(new ArrayList<>()); vo.setQueryAsins(copyCountryAsins(item.getQueryAsins())); vo.setOutputFilename(result.getResultFilename()); vo.setDownloadUrl(null); @@ -642,15 +637,11 @@ public class QueryAsinTaskService { if (merged == null) { merged = new QueryAsinShopPayloadDto(); merged.setShopName(shopKey); - merged.setCountrySections(new ArrayList<>()); - merged.setCartRatios(new ArrayList<>()); - merged.setQueryAsins(new ArrayList<>()); + merged.setCountryResults(new ArrayList<>()); } merged.setShopName(firstNonBlank(incoming.getShopName(), merged.getShopName())); merged.setSubmissionId(firstNonBlank(incoming.getSubmissionId(), merged.getSubmissionId())); - merged.setChunkIndex(incoming.getChunkIndex()); - merged.setChunkTotal(incoming.getChunkTotal()); if (!blank(incoming.getError())) { merged.setError(incoming.getError().trim()); @@ -659,10 +650,8 @@ public class QueryAsinTaskService { return merged; } - merged.setCountrySections(mergeCountrySections(merged.getCountrySections(), incoming.getCountrySections())); - merged.setCartRatios(mergeCartRatios(merged.getCartRatios(), incoming.getCartRatios())); - merged.setQueryAsins(mergeCountryAsins(merged.getQueryAsins(), incoming.getQueryAsins())); - if (Boolean.TRUE.equals(incoming.getShopDone())) { + merged.setCountryResults(mergeCountryResults(merged.getCountryResults(), incoming.getCountryResults())); + if (incoming.getShopDone() == null || Boolean.TRUE.equals(incoming.getShopDone())) { merged.setShopDone(Boolean.TRUE); } taskCacheService.saveShopMergedPayload(taskId, shopKey, merged); @@ -674,11 +663,7 @@ public class QueryAsinTaskService { return; } snapshot.setShopName(firstNonBlank(payload.getShopName(), snapshot.getShopName())); - snapshot.setCountrySections(copyCountrySections(payload.getCountrySections())); - snapshot.setCartRatios(copyCartRatios(payload.getCartRatios())); - if (payload.getQueryAsins() != null && !payload.getQueryAsins().isEmpty()) { - snapshot.setQueryAsins(copyCountryAsins(payload.getQueryAsins())); - } + snapshot.setCountryResults(copyCountryResults(payload.getCountryResults())); if (!blank(payload.getError())) { snapshot.setError(payload.getError().trim()); } @@ -700,37 +685,38 @@ public class QueryAsinTaskService { snapshot.setError(blankToNull(error)); } - private List mergeCountrySections(List base, List incoming) { - Map map = new LinkedHashMap<>(); - for (QueryAsinCountrySectionDto section : copyCountrySections(base)) { - map.put(section.getCountry(), section); + private List mergeCountryResults(List base, List incoming) { + Map map = new LinkedHashMap<>(); + for (QueryAsinCountryResultDto item : copyCountryResults(base)) { + map.put(item.getCountry(), item); } - for (QueryAsinCountrySectionDto section : copyCountrySections(incoming)) { - QueryAsinCountrySectionDto existing = map.get(section.getCountry()); - if (existing == null) { - map.put(section.getCountry(), section); + for (QueryAsinCountryResultDto item : copyCountryResults(incoming)) { + if (blank(item.getCountry())) { continue; } - List mergedRows = existing.getRows() == null - ? new ArrayList<>() - : new ArrayList<>(existing.getRows()); - if (section.getRows() != null) { - mergedRows.addAll(section.getRows()); + QueryAsinCountryResultDto existing = map.get(item.getCountry()); + if (existing == null) { + map.put(item.getCountry(), item); + continue; } - existing.setRows(mergedRows); - } - return new ArrayList<>(map.values()); - } - - private List mergeCartRatios(List base, List incoming) { - Map map = new LinkedHashMap<>(); - for (QueryAsinCartRatioDto ratio : copyCartRatios(base)) { - map.put(ratio.getCountry(), ratio); - } - for (QueryAsinCartRatioDto ratio : copyCartRatios(incoming)) { - if (!blank(ratio.getCountry())) { - map.put(ratio.getCountry(), ratio); + List merged = new ArrayList<>(existing.getItems() == null ? List.of() : existing.getItems()); + for (QueryAsinAsinStatusDto row : item.getItems() == null ? List.of() : item.getItems()) { + String asin = normalizeAsin(row == null ? null : row.getAsin()); + String status = row == null ? "" : firstNonBlank(row.getStatus(), "").trim(); + if (asin.isEmpty() && status.isEmpty()) { + continue; + } + boolean exists = merged.stream().anyMatch(old -> + Objects.equals(normalizeAsin(old.getAsin()), asin) + && Objects.equals(firstNonBlank(old.getStatus(), "").trim(), status)); + if (!exists) { + QueryAsinAsinStatusDto normalized = new QueryAsinAsinStatusDto(); + normalized.setAsin(asin); + normalized.setStatus(status); + merged.add(normalized); + } } + existing.setItems(merged); } return new ArrayList<>(map.values()); } @@ -741,17 +727,19 @@ public class QueryAsinTaskService { map.put(item.getCountry(), item); } for (QueryAsinCountryAsinsDto item : copyCountryAsins(incoming)) { - if (blank(item.getCountry())) { + String country = normalizeCountry(item.getCountry()); + if (blank(country)) { continue; } - QueryAsinCountryAsinsDto existing = map.get(item.getCountry()); + item.setCountry(country); + QueryAsinCountryAsinsDto existing = map.get(country); if (existing == null) { - map.put(item.getCountry(), item); + map.put(country, item); continue; } List merged = new ArrayList<>(existing.getAsins() == null ? List.of() : existing.getAsins()); for (String asin : item.getAsins() == null ? List.of() : item.getAsins()) { - String normalized = asin == null ? "" : asin.trim().toUpperCase(); + String normalized = normalizeAsin(asin); if (!normalized.isEmpty() && !merged.contains(normalized)) { merged.add(normalized); } @@ -765,35 +753,18 @@ public class QueryAsinTaskService { if (payload == null) { return false; } - if (payload.getQueryAsins() != null) { - for (QueryAsinCountryAsinsDto item : payload.getQueryAsins()) { - if (item != null && item.getAsins() != null && !item.getAsins().isEmpty()) { - return true; - } - } - } - if (payload.getCountrySections() != null) { - for (QueryAsinCountrySectionDto section : payload.getCountrySections()) { - if (section == null || section.getRows() == null) { + if (payload.getCountryResults() != null) { + for (QueryAsinCountryResultDto countryResult : payload.getCountryResults()) { + if (countryResult == null || countryResult.getItems() == null) { continue; } - for (QueryAsinCountryMetricRowDto row : section.getRows()) { - if (row != null && (!blank(row.getStatus()) - || !blank(row.getQuantity()) - || !blank(row.getDeleteQuantity()) - || !blank(row.getProcessStatus()))) { + for (QueryAsinAsinStatusDto item : countryResult.getItems()) { + if (item != null && (!blank(item.getAsin()) || !blank(item.getStatus()))) { return true; } } } } - if (payload.getCartRatios() != null) { - for (QueryAsinCartRatioDto ratio : payload.getCartRatios()) { - if (ratio != null && (!blank(ratio.getCountry()) || !blank(ratio.getRatio()))) { - return true; - } - } - } return false; } @@ -872,35 +843,32 @@ public class QueryAsinTaskService { } } - private List copyCountrySections(List sections) { - List copy = new ArrayList<>(); - if (sections == null) { + private List copyCountryResults(List results) { + List copy = new ArrayList<>(); + if (results == null) { return copy; } - for (QueryAsinCountrySectionDto section : sections) { - if (section == null) { + for (QueryAsinCountryResultDto source : results) { + if (source == null || blank(source.getCountry())) { continue; } - QueryAsinCountrySectionDto item = new QueryAsinCountrySectionDto(); - item.setCountry(section.getCountry()); - item.setRows(section.getRows() == null ? new ArrayList<>() : new ArrayList<>(section.getRows())); - copy.add(item); - } - return copy; - } - - private List copyCartRatios(List ratios) { - List copy = new ArrayList<>(); - if (ratios == null) { - return copy; - } - for (QueryAsinCartRatioDto ratio : ratios) { - if (ratio == null) { - continue; + QueryAsinCountryResultDto item = new QueryAsinCountryResultDto(); + item.setCountry(normalizeCountry(source.getCountry())); + List rows = new ArrayList<>(); + if (source.getItems() != null) { + for (QueryAsinAsinStatusDto sourceRow : source.getItems()) { + String asin = normalizeAsin(sourceRow == null ? null : sourceRow.getAsin()); + String status = sourceRow == null ? "" : firstNonBlank(sourceRow.getStatus(), "").trim(); + if (asin.isEmpty() && status.isEmpty()) { + continue; + } + QueryAsinAsinStatusDto row = new QueryAsinAsinStatusDto(); + row.setAsin(asin); + row.setStatus(status); + rows.add(row); + } } - QueryAsinCartRatioDto item = new QueryAsinCartRatioDto(); - item.setCountry(ratio.getCountry()); - item.setRatio(ratio.getRatio()); + item.setItems(rows); copy.add(item); } return copy; @@ -916,11 +884,11 @@ public class QueryAsinTaskService { continue; } QueryAsinCountryAsinsDto item = new QueryAsinCountryAsinsDto(); - item.setCountry(source.getCountry().trim().toUpperCase()); + item.setCountry(normalizeCountry(source.getCountry())); List asins = new ArrayList<>(); if (source.getAsins() != null) { for (String asin : source.getAsins()) { - String normalized = asin == null ? "" : asin.trim().toUpperCase(); + String normalized = normalizeAsin(asin); if (!normalized.isEmpty() && !asins.contains(normalized)) { asins.add(normalized); } @@ -936,6 +904,25 @@ public class QueryAsinTaskService { return !blank(first) ? first : second; } + private String normalizeCountry(String country) { + if (country == null) { + return ""; + } + String value = country.trim().toUpperCase(); + return switch (value) { + case "德国" -> "DE"; + case "英国" -> "UK"; + case "法国" -> "FR"; + case "意大利" -> "IT"; + case "西班牙" -> "ES"; + default -> value; + }; + } + + private String normalizeAsin(String asin) { + return asin == null ? "" : asin.trim().toUpperCase(); + } + private String safeFileStem(String value) { String raw = value == null ? "result" : value.trim(); String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java index 4a10295..b823dc8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java @@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import com.nanri.aiimage.config.TaskPressureProperties; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; import java.io.IOException; @@ -14,6 +15,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.Map; @@ -26,11 +28,37 @@ public class ShopMatchTaskCacheService { private static final String MODULE_TYPE = "SHOP_MATCH"; private static final long PAYLOAD_TTL_HOURS = 24; + private final StringRedisTemplate stringRedisTemplate; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final ConcurrentHashMap taskEntityLocalCache = new ConcurrentHashMap<>(); + public void touchTaskHeartbeat(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + stringRedisTemplate.opsForValue().set( + buildTaskHeartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(PAYLOAD_TTL_HOURS)); + } + + public long getTaskHeartbeatMillis(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0L; + } + String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + if (raw == null || raw.isBlank()) { + return 0L; + } + try { + return Long.parseLong(raw); + } catch (NumberFormatException ignored) { + return 0L; + } + } + public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) { return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class); } @@ -40,6 +68,7 @@ public class ShopMatchTaskCacheService { return; } taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload); + touchTaskHeartbeat(taskId); } public void removeShopMergedPayload(Long taskId, String shopKey) { @@ -59,6 +88,7 @@ public class ShopMatchTaskCacheService { return; } taskEntityLocalCache.remove(taskId); + stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); try { Path taskDir = buildTaskDir(taskId); @@ -156,6 +186,10 @@ public class ShopMatchTaskCacheService { return buildTaskDir(taskId).resolve("_task-entity.json"); } + private String buildTaskHeartbeatKey(Long taskId) { + return "shop-match:task:heartbeat:" + taskId; + } + private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) { return cached != null && now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java index 1b213c9..32f33f0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java @@ -364,6 +364,9 @@ public class ShopMatchTaskService { throw new BusinessException("task 不存在"); } updateTaskAndRefreshCache(task); + if ("RUNNING".equals(task.getStatus())) { + shopMatchTaskCacheService.touchTaskHeartbeat(task.getId()); + } ProductRiskCreateTaskVo vo = new ProductRiskCreateTaskVo(); vo.setTaskId(task.getId()); @@ -411,6 +414,7 @@ public class ShopMatchTaskService { task.setUpdatedAt(now); persistTaskRequest(task, state); updateTaskAndRefreshCache(task); + shopMatchTaskCacheService.touchTaskHeartbeat(taskId); } @Transactional @@ -467,6 +471,7 @@ public class ShopMatchTaskService { throw new BusinessException(40901, "任务已结束，拒绝重复提交"); } + shopMatchTaskCacheService.touchTaskHeartbeat(taskId); List resultRows = fileResultMapper.selectList(new LambdaQueryWrapper() .eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getModuleType, MODULE_TYPE) diff --git a/backend/__pycache__/config.cpython-312.pyc b/backend/__pycache__/config.cpython-312.pyc index 0ea50cd..08477f7 100644 Binary files a/backend/__pycache__/config.cpython-312.pyc and b/backend/__pycache__/config.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc index eeba27b..d659909 100644 Binary files a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc and b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/auth.cpython-312.pyc b/backend/blueprints/__pycache__/auth.cpython-312.pyc index 00d50f0..7e1da45 100644 Binary files a/backend/blueprints/__pycache__/auth.cpython-312.pyc and b/backend/blueprints/__pycache__/auth.cpython-312.pyc differ diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index a62ddd2..8ca619d 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -6,7 +6,7 @@ import os import re import requests -from flask import Blueprint, request, jsonify, session +from flask import Blueprint, request, jsonify, session, current_app import pymysql from werkzeug.security import generate_password_hash @@ -461,7 +461,10 @@ def get_admin_current_user_menus(): @login_required def admin_logout(): session.clear() - return jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login'}) + response = jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login?logout=1'}) + response.delete_cookie(current_app.config.get('SESSION_COOKIE_NAME', 'session')) + response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0' + return response # ---------- 用户管理 ---------- diff --git a/backend/blueprints/auth.py b/backend/blueprints/auth.py index 4d1e5ff..1f64bb5 100644 --- a/backend/blueprints/auth.py +++ b/backend/blueprints/auth.py @@ -1,7 +1,7 @@ """ 认证蓝图：登录、登出、登录状态校验 """ -from flask import Blueprint, request, redirect, url_for, session, jsonify +from flask import Blueprint, request, redirect, url_for, session, jsonify, make_response, current_app from werkzeug.security import check_password_hash from utils.db import get_db @@ -13,14 +13,23 @@ auth = Blueprint('auth', __name__, url_prefix='') @auth.route('/login', methods=['GET', 'POST']) def login(): - if session.get('user_id') and is_session_user_valid(): + force_relogin = request.args.get('logout') == '1' or request.args.get('switch') == '1' + if request.method == 'GET' and force_relogin: + session.clear() + response = make_response(render_html('login.html')) + response.delete_cookie(current_app.config.get('SESSION_COOKIE_NAME', 'session')) + response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0' + return response + if request.method == 'GET' and session.get('user_id') and is_session_user_valid(): return redirect(url_for('main.admin_page')) if request.method == 'POST': + session.clear() + wants_json = request.is_json or request.headers.get('X-Requested-With') == 'XMLHttpRequest' data = request.get_json() if request.is_json else request.form username = (data.get('username') or '').strip() password = data.get('password') or '' if not username or not password: - if request.is_json: + if wants_json: return jsonify({'success': False, 'error': '请输入用户名和密码'}) return render_html('login.html', error='请输入用户名和密码') try: @@ -36,14 +45,14 @@ def login(): session.permanent = True session['user_id'] = row['id'] session['username'] = username - if request.is_json: + if wants_json: return jsonify({'success': True, 'redirect': url_for('main.admin_page')}) return redirect(url_for('main.admin_page')) except Exception as exc: - if request.is_json: + if wants_json: return jsonify({'success': False, 'error': str(exc)}) return render_html('login.html', error='登录失败，请稍后重试') - if request.is_json: + if wants_json: return jsonify({'success': False, 'error': '用户名或密码错误'}) return render_html('login.html', error='用户名或密码错误') return render_html('login.html') @@ -71,4 +80,7 @@ def api_auth_check(): @auth.route('/logout') def logout(): session.clear() - return redirect(url_for('auth.login')) + response = redirect(url_for('auth.login', logout='1')) + response.delete_cookie(current_app.config.get('SESSION_COOKIE_NAME', 'session')) + response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0' + return response diff --git a/backend/config.py b/backend/config.py index 97a3eb7..6692838 100644 --- a/backend/config.py +++ b/backend/config.py @@ -23,8 +23,8 @@ bucket_path = "nanri-image/" file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/" import os -backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') -# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') +# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') +backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6" diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 051083d..6094daf 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -4162,10 +4162,10 @@ alert(res.error || '退出失败'); return; } - window.location.href = res.redirect || '/login'; + window.location.replace(res.redirect || '/login?logout=1'); }) .catch(function () { - window.location.href = '/logout'; + window.location.replace('/logout'); }); }; diff --git a/backend/web_source/login.html b/backend/web_source/login.html index 6e2b784..b8df99d 100644 --- a/backend/web_source/login.html +++ b/backend/web_source/login.html @@ -118,6 +118,10 @@