super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

后台查询Asin文档更新

Browse Source
This commit is contained in:
super
2026-04-24 15:14:14 +08:00
parent c502afb588
commit 03e697f5d3
34 changed files with 798 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

515
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
View File

@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
@@ -15,11 +16,13 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
@@ -39,12 +42,14 @@ import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@Service
@RequiredArgsConstructor
@Slf4j
public class DedupeRunService {
private static final String HEADER_SPLIT_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf";
private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408";
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties;
@@ -52,8 +57,9 @@ public class DedupeRunService {
private final DedupeTotalDataService dedupeTotalDataService;
public DedupeRunVo run(DedupeRunRequest request) {
long runStartNs = System.nanoTime();
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
throw new BusinessException("请至少选择一种 ID 保留规则");
throw new BusinessException("\u8bf7\u81f3\u5c11\u9009\u62e9\u4e00\u79cd ID \u4fdd\u7559\u89c4\u5219");
}
FileTaskEntity task = new FileTaskEntity();
@@ -80,9 +86,11 @@ public class DedupeRunService {
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
try {
long fileStartNs = System.nanoTime();
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
long findFileNs = elapsedNs(fileStartNs);
if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("上传文件不存在，请重新上传");
throw new BusinessException("\u4e0a\u4f20\u6587\u4ef6\u4e0d\u5b58\u5728\uff0c\u8bf7\u91cd\u65b0\u4e0a\u4f20");
}
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
@@ -90,6 +98,7 @@ public class DedupeRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
long cleanStartNs = System.nanoTime();
cleanExcelByLegacyRules(
inputFile,
outputFile,
@@ -98,6 +107,7 @@ public class DedupeRunService {
request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds()
);
long cleanNs = elapsedNs(cleanStartNs);
if (folderMode) {
archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile));
@@ -105,13 +115,15 @@ public class DedupeRunService {
continue;
}
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE");
// 只存 objectKey，不存预签名 URL
long uploadStartNs = System.nanoTime();
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE");
long uploadNs = elapsedNs(uploadStartNs);
String ossObjectKey = uploadedResult.objectKey();
String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName();
item.setSuccess(true);
item.setOutputFilename(downloadFilename);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey));
item.setDownloadUrl(uploadedResult.downloadUrl());
successCount++;
FileResultEntity resultEntity = new FileResultEntity();
@@ -119,13 +131,26 @@ public class DedupeRunService {
resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(downloadFilename);
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey
resultEntity.setResultFileUrl(ossObjectKey);
resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
resultEntity.setSuccess(1);
resultEntity.setUserId(request.getUserId());
resultEntity.setCreatedAt(LocalDateTime.now());
long resultInsertStartNs = System.nanoTime();
fileResultMapper.insert(resultEntity);
long resultInsertNs = elapsedNs(resultInsertStartNs);
log.info(
"dedupe run file done fileKey={} filename={} size={} findFileMs={} cleanMs={} uploadMs={} resultInsertMs={} totalFileMs={}",
sourceFile.getFileKey(),
inputName,
inputFile.length(),
toMs(findFileNs),
toMs(cleanNs),
toMs(uploadNs),
toMs(resultInsertNs),
toMs(elapsedNs(fileStartNs))
);
} catch (Exception ex) {
item.setSuccess(false);
item.setError(ex.getMessage());
@@ -146,20 +171,20 @@ public class DedupeRunService {
if (folderMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, "DEDUPE");
// 只存 objectKey
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE");
String ossObjectKey = uploadedResult.objectKey();
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(request.getArchiveName());
item.setOutputFilename(zipFile.getName());
item.setSuccess(true);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey));
item.setDownloadUrl(uploadedResult.downloadUrl());
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(request.getArchiveName());
resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey
resultEntity.setResultFileUrl(ossObjectKey);
resultEntity.setResultFileSize(zipFile.length());
resultEntity.setResultContentType("application/zip");
resultEntity.setSuccess(1);
@@ -181,6 +206,14 @@ public class DedupeRunService {
vo.setSuccessCount(successCount);
vo.setFailedCount(failedCount);
vo.setItems(items);
log.info(
"dedupe run done userId={} files={} success={} failed={} totalMs={}",
request.getUserId(),
request.getFiles().size(),
successCount,
failedCount,
toMs(elapsedNs(runStartNs))
);
return vo;
}
@@ -209,7 +242,7 @@ public class DedupeRunService {
public void deleteHistory(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在");
throw new BusinessException("\u8bb0\u5f55\u4e0d\u5b58\u5728");
}
fileResultMapper.deleteById(resultId);
}
@@ -217,115 +250,64 @@ public class DedupeRunService {
public Resource getResultFile(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("结果文件不存在");
throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728");
}
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("结果文件不存在");
throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728");
}
return new FileSystemResource(file);
}
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
long readStartNs = System.nanoTime();
DedupeReadResult readResult = readDedupeRows(
inputFile,
selectedColumns,
keepIntegerIds,
keepUnderscoreIds,
keepIntegerMainIdsWhenNoSubIds
);
long readNs = elapsedNs(readStartNs);
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank()) {
continue;
}
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
? value
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if ("缩略图地址8".equals(value)) {
break;
}
Set<String> candidateAsinValues = new HashSet<>();
for (DedupeCandidateRow row : readResult.rows()) {
if (!row.asinValue().isBlank()) {
candidateAsinValues.add(row.asinValue());
}
}
long dbStartNs = System.nanoTime();
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
long dbNs = elapsedNs(dbStartNs);
List<String> missing = selectedColumns.stream().filter(column -> !headerMap.containsKey(column)).toList();
if (!missing.isEmpty()) {
throw new BusinessException("缺少列：" + String.join("、", missing));
}
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
long writeStartNs = System.nanoTime();
int outputRows = 0;
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName()));
Row outputHeaderRow = outputSheet.createRow(0);
for (int i = 0; i < selectedColumns.size(); i++) {
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
}
Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
if (idColumnIndex != null) {
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
Set<String> candidateAsinValues = new HashSet<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
continue;
}
}
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
candidateAsinValues.add(asinValue);
}
}
}
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> writtenAsinValues = new HashSet<>();
int outputRowIndex = 1;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
for (DedupeCandidateRow candidateRow : readResult.rows()) {
String asinValue = candidateRow.asinValue();
if (!asinValue.isBlank()) {
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
continue;
}
}
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
continue;
}
if (!writtenAsinValues.add(asinValue)) {
continue;
}
if (!writtenAsinValues.add(asinValue)) {
continue;
}
}
Row outputRow = outputSheet.createRow(outputRowIndex++);
for (int i = 0; i < selectedColumns.size(); i++) {
Integer sourceIndex = headerMap.get(selectedColumns.get(i));
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
outputRow.createCell(i).setCellValue(value);
outputRows++;
for (int i = 0; i < candidateRow.selectedValues().size(); i++) {
outputRow.createCell(i).setCellValue(candidateRow.selectedValues().get(i));
}
}
@@ -334,6 +316,157 @@ public class DedupeRunService {
}
outputWorkbook.dispose();
}
long writeNs = elapsedNs(writeStartNs);
log.info(
"dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}",
inputFile.getName(),
readResult.scannedRows(),
readResult.rows().size(),
candidateAsinValues.size(),
matchedAsinValues.size(),
outputRows,
toMs(readNs),
toMs(dbNs),
toMs(writeNs),
toMs(readNs + dbNs + writeNs)
);
}
private DedupeReadResult readDedupeRows(File inputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
List<DedupeCandidateRow> rows = new ArrayList<>();
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a");
}
Map<String, Integer> headerMap = buildHeaderMap(headerRow, formatter);
if (headerMap.isEmpty()) {
throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a");
}
List<String> missing = selectedColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("\u7f3a\u5c11\u5217\uff1a" + String.join("\u3001", missing));
}
List<Integer> selectedIndexes = new ArrayList<>(selectedColumns.size());
for (String selectedColumn : selectedColumns) {
selectedIndexes.add(headerMap.get(selectedColumn));
}
Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
int scannedRows = 0;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
scannedRows++;
if (idColumnIndex == null) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
continue;
}
appendRowByIdRule(
normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))),
row,
selectedIndexes,
asinColumnIndex,
formatter,
keepIntegerIds,
keepUnderscoreIds,
keepIntegerMainIdsWhenNoSubIds,
pendingMainIdGroup,
rows
);
}
pendingMainIdGroup.flush(rows);
return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows);
}
}
private Map<String, Integer> buildHeaderMap(Row headerRow, DataFormatter formatter) {
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeHeaderText(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank()) {
continue;
}
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if (HEADER_STOP_COLUMN.equals(value)) {
break;
}
}
return headerMap;
}
private String normalizeHeaderText(String value) {
String normalized = normalizeCellText(value);
int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER);
if (markerIndex > 0) {
String prefix = normalized.substring(0, markerIndex).trim();
if (!prefix.isBlank()) {
return prefix;
}
}
return normalized;
}
private DedupeCandidateRow buildCandidateRow(Row row, List<Integer> selectedIndexes, Integer asinColumnIndex, DataFormatter formatter) {
List<String> selectedValues = new ArrayList<>(selectedIndexes.size());
for (Integer selectedIndex : selectedIndexes) {
selectedValues.add(normalizeCellText(formatter.formatCellValue(row.getCell(selectedIndex))));
}
String asinValue = asinColumnIndex == null
? ""
: dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
return new DedupeCandidateRow(selectedValues, asinValue);
}
private void appendRowByIdRule(String idValue, Row row,
List<Integer> selectedIndexes, Integer asinColumnIndex,
DataFormatter formatter,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds,
PendingMainIdGroup pendingMainIdGroup,
List<DedupeCandidateRow> rows) {
if (idValue == null || idValue.isBlank()) {
return;
}
String mainId = extractMainId(idValue);
if (pendingMainIdGroup.hasDifferentMainId(mainId)) {
pendingMainIdGroup.flush(rows);
}
if (isUnderscoreId(idValue)) {
pendingMainIdGroup.discardIfSameMainId(mainId);
if (keepUnderscoreIds) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
}
return;
}
if (isIntegerId(idValue)) {
if (keepIntegerIds) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
return;
}
if (keepIntegerMainIdsWhenNoSubIds) {
pendingMainIdGroup.add(mainId, buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
}
}
}
private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
@@ -347,7 +480,7 @@ public class DedupeRunService {
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("去重结果打包失败：" + ex.getMessage());
throw new BusinessException("\u53bb\u91cd\u7ed3\u679c\u6253\u5305\u5931\u8d25\uff1a" + ex.getMessage());
}
return zipFile;
}
@@ -368,67 +501,61 @@ public class DedupeRunService {
return String.join("/", parts);
}
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds, Sheet sheet, int currentRowNum, int idColumnIndex, DataFormatter formatter) {
if (text == null || text.isBlank()) {
return false;
}
if (keepUnderscoreIds && text.matches("\\d+_\\d+")) {
return true;
}
if (keepIntegerIds && text.matches("\\d+")) {
return true;
}
if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) {
String mainId = extractMainId(text);
if (mainId.isEmpty()) return false;
boolean hasSubIdInGroup = false;
for (int r = currentRowNum + 1; r <= sheet.getLastRowNum(); r++) {
Row nextRow = sheet.getRow(r);
if (nextRow == null) continue;
Cell cell = nextRow.getCell(idColumnIndex);
if (cell == null) continue;
String nextIdValue = normalizeCellText(formatter.formatCellValue(cell));
if (nextIdValue.isBlank()) continue;
String nextMainId = extractMainId(nextIdValue);
if (nextMainId.equals(mainId)) {
if (nextIdValue.matches("\\d+_\\d+")) {
hasSubIdInGroup = true;
break;
}
} else {
// Encountered a different main ID. The group for 'mainId' has ended.
break;
}
}
return !hasSubIdInGroup;
}
return false;
}
private String extractMainId(String text) {
if (text == null || text.isBlank()) {
return "";
}
if (text.matches("\\d+")) {
if (isIntegerId(text)) {
return text;
}
if (text.matches("\\d+_\\d+")) {
if (isUnderscoreId(text)) {
int idx = text.indexOf('_');
return idx > 0 ? text.substring(0, idx) : "";
}
return "";
}
private boolean isIntegerId(String text) {
if (text == null || text.isEmpty()) {
return false;
}
for (int i = 0; i < text.length(); i++) {
if (!Character.isDigit(text.charAt(i))) {
return false;
}
}
return true;
}
private boolean isUnderscoreId(String text) {
if (text == null || text.length() < 3) {
return false;
}
int underscoreIndex = text.indexOf('_');
if (underscoreIndex <= 0 || underscoreIndex == text.length() - 1 || text.indexOf('_', underscoreIndex + 1) >= 0) {
return false;
}
for (int i = 0; i < text.length(); i++) {
if (i == underscoreIndex) {
continue;
}
if (!Character.isDigit(text.charAt(i))) {
return false;
}
}
return true;
}
private File findLocalSourceFile(String fileKey) {
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
if (!baseDir.exists()) {
return null;
}
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
File[] files = baseDir.listFiles(pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
if (files == null || files.length == 0) {
return null;
}
return files[0];
}
private File buildNamedOutputFile(File outputDir, String filename) {
@@ -461,16 +588,102 @@ public class DedupeRunService {
if (value == null) {
return "";
}
return value.replace("", "")
.replace("　", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")
.replace("\t", " ")
.trim()
.replaceAll("\\s+", " ");
int start = 0;
int end = value.length();
while (start < end && isTrimmedWhitespace(value.charAt(start))) {
start++;
}
while (end > start && isTrimmedWhitespace(value.charAt(end - 1))) {
end--;
}
if (start >= end) {
return "";
}
StringBuilder builder = null;
boolean previousWhitespace = false;
for (int i = start; i < end; i++) {
char ch = value.charAt(i);
if (ch == '\uFEFF') {
if (builder == null) {
builder = new StringBuilder(value.length());
builder.append(value, start, i);
}
continue;
}
if (isNormalizedWhitespace(ch)) {
if (builder == null) {
builder = new StringBuilder(value.length());
builder.append(value, start, i);
}
if (!previousWhitespace) {
builder.append(' ');
}
previousWhitespace = true;
continue;
}
if (builder != null) {
builder.append(ch);
}
previousWhitespace = false;
}
return builder == null ? value.substring(start, end) : builder.toString();
}
private boolean isTrimmedWhitespace(char ch) {
return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch);
}
private boolean isNormalizedWhitespace(char ch) {
return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch);
}
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
}
private long elapsedNs(long startNs) {
return System.nanoTime() - startNs;
}
private long toMs(long nanos) {
return nanos / 1_000_000L;
}
private record DedupeReadResult(String sheetName, List<DedupeCandidateRow> rows, int scannedRows) {
}
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
}
private static final class PendingMainIdGroup {
private String mainId;
private final List<DedupeCandidateRow> rows = new ArrayList<>();
private void add(String nextMainId, DedupeCandidateRow row) {
if (hasDifferentMainId(nextMainId)) {
rows.clear();
}
mainId = nextMainId;
rows.add(row);
}
private boolean hasDifferentMainId(String nextMainId) {
return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId));
}
private void discardIfSameMainId(String nextMainId) {
if (mainId != null && mainId.equals(nextMainId)) {
rows.clear();
mainId = null;
}
}
private void flush(List<DedupeCandidateRow> outputRows) {
if (!rows.isEmpty()) {
outputRows.addAll(rows);
rows.clear();
}
mainId = null;
}
}
}

2
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
View File

@@ -38,7 +38,7 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class DedupeTotalDataService {
private static final int COMPARE_BATCH_SIZE = 1000;
private static final int COMPARE_BATCH_SIZE = 5000;
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();

11
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
View File

@@ -298,6 +298,8 @@ public class DeleteBrandStaleTaskService {
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes());
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getShopMatchInitialTimeoutMinutes());
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
long nowMillis = System.currentTimeMillis();
LocalDateTime now = LocalDateTime.now();
LocalDateTime threshold = now.minusMinutes(minutes);
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
@@ -313,10 +315,17 @@ public class DeleteBrandStaleTaskService {
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = shopMatchTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",

16
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java
View File

@@ -37,6 +37,19 @@ public class OssStorageService {
}
}
public UploadedResult uploadResultFileWithFreshDownloadUrl(File file, String moduleType) {
String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName());
OSS ossClient = buildClient();
try {
ossClient.putObject(ossProperties.getBucket(), objectKey, file);
Date expiration = new Date(System.currentTimeMillis() + 3600_000L);
URL url = ossClient.generatePresignedUrl(ossProperties.getBucket(), objectKey, expiration);
return new UploadedResult(objectKey, url.toString());
} finally {
ossClient.shutdown();
}
}
public String uploadText(String objectKey, String content) {
if (objectKey == null || objectKey.isBlank()) {
throw new IllegalArgumentException("objectKey must not be blank");
@@ -160,4 +173,7 @@ public class OssStorageService {
ossProperties.getAccessKeySecret()
);
}
public record UploadedResult(String objectKey, String downloadUrl) {
}
}

202
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java
View File

@@ -1,6 +1,11 @@
package com.nanri.aiimage.modules.queryasin.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinTaskBatchRequest;
@@ -9,11 +14,6 @@ import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinHistoryVo;
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinTaskBatchVo;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinResolveService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
@@ -46,14 +46,14 @@ import java.util.List;
@RequestMapping("/api/query-asin")
@Tag(
name = "查询ASIN",
description = "查询ASIN模块：管理备选店铺、批量匹配紫鸟店铺、创建任务、接收 Python 分片结果、自动收尾生成 Excel 并提供下载。涉及查询和删除的接口需要携带 user_id。")
description = "查询 ASIN 模块：管理备选店铺、批量匹配紫鸟店铺、创建任务、接收 Python 分片结果、自动收尾生成 Excel 并提供下载。查询和删除接口需要携带 user_id。")
public class QueryAsinTaskController {
private final QueryAsinResolveService queryAsinResolveService;
private final QueryAsinTaskService queryAsinTaskService;
@GetMapping("/candidates")
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询ASIN模块中已保存的备选店铺。")
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询 ASIN 模块中已保存的备选店铺。")
public ApiResponse<List<ProductRiskCandidateVo>> listCandidates(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
@@ -61,10 +61,10 @@ public class QueryAsinTaskController {
}
@PostMapping("/candidates")
@Operation(summary = "新增备选店铺", description = "向查询ASIN备选区新增一条店铺记录，供后续匹配和创建任务使用。")
@Operation(summary = "新增备选店铺", description = "向查询 ASIN 备选区新增一条店铺记录，供后续匹配和创建任务使用。")
public ApiResponse<ProductRiskCandidateVo> addCandidate(
@io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "新增备选店铺请求，需传入用户 ID 和店铺名。",
description = "新增备选店铺请求，需要传入用户 ID 和店铺名。",
required = true,
content = @Content(
mediaType = "application/json",
@@ -93,7 +93,7 @@ public class QueryAsinTaskController {
}
@PostMapping("/match-shops")
@Operation(summary = "批量匹配店铺", description = "根据店铺名批量匹配紫鸟店铺索引，返回是否命中、店铺 ID、平台、公司和匹配状态。")
@Operation(summary = "批量匹配店铺", description = "根据店铺名称批量匹配紫鸟店铺索引，返回是否命中、店铺 ID、平台、公司和匹配状态。")
public ApiResponse<ProductRiskMatchShopsVo> matchShops(
@io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "批量匹配请求，传入 user_id 和待匹配的店铺名称列表。",
@@ -125,7 +125,7 @@ public class QueryAsinTaskController {
}
@GetMapping("/history")
@Operation(summary = "查询任务记录", description = "返回当前用户在查询ASIN模块中的当前任务和历史任务记录。")
@Operation(summary = "查询任务记录", description = "返回当前用户在查询 ASIN 模块中的当前任务和历史任务记录。")
public ApiResponse<QueryAsinHistoryVo> history(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
@@ -133,16 +133,16 @@ public class QueryAsinTaskController {
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询查询ASIN任务进度", description = "仅返回任务状态和店铺结果摘要，用于前端轮询降载。")
@Operation(summary = "批量查询查询 ASIN 任务进度", description = "仅返回任务状态和店铺结果摘要，用于前端轮询降载。")
public ApiResponse<QueryAsinTaskBatchVo> taskProgressBatch(@Valid @RequestBody QueryAsinTaskBatchRequest request) {
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds()));
}
@PostMapping("/tasks")
@Operation(summary = "创建查询ASIN任务", description = "根据已匹配的店铺创建任务和占位结果记录，后续由 Python 端处理并回传结果。")
@Operation(summary = "创建查询 ASIN 任务", description = "根据已匹配的店铺创建任务和占位结果记录，并把后台维护的整张 ASIN 表数据随店铺项返回给 Python 端。")
public ApiResponse<QueryAsinCreateTaskVo> createTask(
@io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "创建任务请求。items 中每一项代表一个待处理店铺，需携带默认模板结构。",
description = "创建任务请求。items 中每一项代表一个待处理店铺，queryAsins 是后台维护的整张 ASIN 表数据，不按店铺过滤。",
required = true,
content = @Content(
mediaType = "application/json",
@@ -161,23 +161,14 @@ public class QueryAsinTaskController {
"companyName": "示例公司",
"matchStatus": "MATCHED",
"matchMessage": "索引已命中",
"countrySections": [
"queryAsins": [
{
"country": "德国",
"rows": [
{
"status": "全部",
"quantity": "",
"deleteQuantity": "",
"processStatus": ""
}
]
}
],
"cartRatios": [
"country": "DE",
"asins": ["B0BRZZR3N2", "B0BRZZR3N3"]
},
{
"country": "德国",
"ratio": ""
"country": "FR",
"asins": ["B0BRZZR3N4"]
}
]
}
@@ -190,124 +181,72 @@ public class QueryAsinTaskController {
@PostMapping("/tasks/{taskId}/result")
@Operation(
summary = "提交查询ASIN结果",
description = "供 Python 端回传处理结果。支持按店铺分片多次提交，服务端会合并分片、按店铺完成状态自动收尾，并在任务结束后生成 Excel。")
summary = "提交查询 ASIN 结果",
description = "供 Python 端回传查询 ASIN 结果。业务数据包含店铺名，以及德国、英国、法国、意大利、西班牙 5 个国家下的 ASIN 和处理状态；submissionId 用于日志追踪，shopDone 可不传，不传时后端默认该店铺已提交完成。")
public ApiResponse<Void> submitResult(
@Parameter(description = "任务主键，必须是运行中的任务", example = "3089")
@Parameter(description = "任务主键，必须是运行中的查询 ASIN 任务", example = "3089")
@PathVariable Long taskId,
@io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "查询ASIN结果回传请求。shops 表示本次提交的店铺结果分片列表。shopDone=true 表示该店铺已全部提交完成。",
description = "查询 ASIN 结果回传请求。shops 表示店铺结果列表；countryResults 对应导出表中的国家 ASIN 列和处理状态列；error 用于回传单店铺失败原因。",
required = true,
content = @Content(
mediaType = "application/json",
schema = @Schema(implementation = QueryAsinSubmitResultRequest.class),
examples = {
@ExampleObject(
name = "成功分片示例",
summary = "一个店铺分片回传，包含状态数据和购物车比例数据",
value = """
examples = @ExampleObject(
name = "回传示例",
summary = "按店铺和国家返回 ASIN 与处理状态",
value = """
{
"shops": [
{
"shopName": "郭亚芳",
"submissionId": "query-asin:3089:郭亚芳:1711111111111",
"shopDone": true,
"countryResults": [
{
"shops": [
"country": "DE",
"items": [
{
"shopName": "郭亚芳",
"submissionId": "query-asin:3089:郭亚芳:1711111111111",
"chunkIndex": 1,
"chunkTotal": 2,
"shopDone": false,
"countrySections": [
{
"country": "德国",
"rows": [
{
"status": "正常",
"quantity": "12",
"deleteQuantity": "3",
"processStatus": "处理中"
},
{
"status": "下架",
"quantity": "2",
"deleteQuantity": "2",
"processStatus": "已完成"
}
]
},
{
"country": "英国",
"rows": [
{
"status": "正常",
"quantity": "5",
"deleteQuantity": "1",
"processStatus": "处理中"
}
]
}
],
"cartRatios": [
{
"country": "德国",
"ratio": "25%"
},
{
"country": "英国",
"ratio": "18%"
}
]
"asin": "B0BRZZR3N2",
"status": "正常"
},
{
"asin": "B0BRZZR3N3",
"status": "未找到"
}
]
}
"""),
@ExampleObject(
name = "店铺完成示例",
summary = "最后一片提交完成后，将 shopDone 置为 true",
value = """
},
{
"shops": [
"country": "UK",
"items": [
{
"shopName": "郭亚芳",
"submissionId": "query-asin:3089:郭亚芳:1711111111111",
"chunkIndex": 2,
"chunkTotal": 2,
"shopDone": true,
"countrySections": [
{
"country": "法国",
"rows": [
{
"status": "正常",
"quantity": "4",
"deleteQuantity": "0",
"processStatus": "已完成"
}
]
}
],
"cartRatios": [
{
"country": "法国",
"ratio": "12%"
}
]
"asin": "B0BRZZR3N6",
"status": "正常"
}
]
}
"""),
@ExampleObject(
name = "失败示例",
summary = "店铺处理失败时直接回传错误信息",
value = """
},
{
"shops": [
"country": "FR",
"items": [
{
"shopName": "郭亚芳",
"shopDone": true,
"error": "紫鸟页面加载超时，未能完成删除数据采集"
"asin": "B0BRZZR3N4",
"status": "正常"
}
]
},
{
"country": "IT",
"items": []
},
{
"country": "ES",
"items": []
}
""")
}))
]
}
]
}
""")))
@Valid @RequestBody QueryAsinSubmitResultRequest request,
jakarta.servlet.http.HttpServletResponse response) {
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
@@ -317,7 +256,7 @@ public class QueryAsinTaskController {
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载结果文件", description = "按结果记录下载服务端已生成并上传到 OSS 的查询ASIN Excel 文件。")
@Operation(summary = "下载结果文件", description = "按结果记录下载服务端已生成并上传到 OSS 的查询 ASIN Excel 文件。")
public void downloadResult(
@Parameter(description = "结果记录主键", example = "4599")
@PathVariable Long resultId,
@@ -348,7 +287,7 @@ public class QueryAsinTaskController {
}
@DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除任务", description = "删除整条查询ASIN任务以及该任务下关联的所有结果记录。")
@Operation(summary = "删除任务", description = "删除整条查询 ASIN 任务以及该任务下关联的所有结果记录。")
public ApiResponse<Void> deleteTask(
@Parameter(description = "任务主键", example = "3089")
@PathVariable Long taskId,
@@ -359,7 +298,7 @@ public class QueryAsinTaskController {
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除单条历史记录", description = "删除一条查询ASIN结果记录，并同步重算其所属任务状态。")
@Operation(summary = "删除单条历史记录", description = "删除一条查询 ASIN 结果记录，并同步重算其所属任务状态。")
public ApiResponse<Void> deleteHistory(
@Parameter(description = "结果记录主键", example = "4599")
@PathVariable Long resultId,
@@ -369,6 +308,3 @@ public class QueryAsinTaskController {
return ApiResponse.success(null);
}
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinAsinStatusDto.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "查询 ASIN 单条结果")
public class QueryAsinAsinStatusDto {
@Schema(description = "ASIN", example = "B0BRZZR3N2")
private String asin;
@Schema(description = "Python 查询后返回的状态", example = "正常")
private String status;
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCartRatioDto.java
View File

@@ -1,16 +0,0 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "购物车比例数据")
public class QueryAsinCartRatioDto {
@Schema(description = "国家名称", example = "德国")
private String country;
@Schema(description = "购物车比例", example = "25%")
private String ratio;
}

4
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryAsinsDto.java
View File

@@ -7,12 +7,12 @@ import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "查询 ASIN 国家维度 ASIN 列表")
@Schema(description = "查询 ASIN 国家维度 ASIN 清单")
public class QueryAsinCountryAsinsDto {
@Schema(description = "国家编码，支持 DE、UK、FR、IT、ES", example = "DE")
private String country;
@Schema(description = "该国家需要查询的 ASIN 列表")
@Schema(description = "后台维护的该国家 ASIN 清单，创建任务时会整体推给 Python")
private List<String> asins = new ArrayList<>();
}

25
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryMetricRowDto.java
View File

@@ -1,25 +0,0 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "国家状态数据中的单行记录")
public class QueryAsinCountryMetricRowDto {
@Schema(description = "商品状态", example = "正常")
private String status;
@Schema(description = "该状态下的商品数量", example = "12")
private String quantity;
@JsonProperty("deleteQuantity")
@Schema(description = "已删除数量", example = "3")
private String deleteQuantity;
@JsonProperty("processStatus")
@Schema(description = "处理结果或处理状态", example = "处理中")
private String processStatus;
}

18
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountryResultDto.java Normal file
View File

@@ -0,0 +1,18 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "查询 ASIN 国家维度结果")
public class QueryAsinCountryResultDto {
@Schema(description = "国家编码，支持 DE、UK、FR、IT、ES", example = "DE")
private String country;
@Schema(description = "该国家下的 ASIN 查询结果列表")
private List<QueryAsinAsinStatusDto> items = new ArrayList<>();
}

19
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinCountrySectionDto.java
View File

@@ -1,19 +0,0 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个国家的状态数据分组")
public class QueryAsinCountrySectionDto {
@Schema(description = "国家名称", example = "德国")
private String country;
@Schema(description = "该国家下的多行状态数据")
private List<QueryAsinCountryMetricRowDto> rows = new ArrayList<>();
}

32
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinShopPayloadDto.java
View File

@@ -8,42 +8,26 @@ import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个店铺的查询ASIN结果分片")
@Schema(description = "单个店铺的查询 ASIN 结果")
public class QueryAsinShopPayloadDto {
@JsonProperty("shopName")
@Schema(description = "店铺名称，服务端按店铺名称聚合分片", example = "郭亚芳")
@Schema(description = "店铺名称", example = "郭亚芳")
private String shopName;
@Schema(description = "店铺处理失败时的错误信息；有值时该店铺会直接标记失败", example = "紫鸟页面加载超时，未能完成删除数据采集")
@Schema(description = "店铺处理失败时的错误信息；有值时该店铺会直接标记失败", example = "紫鸟页面加载超时，未能完成 ASIN 查询")
private String error;
@JsonProperty("countrySections")
@Schema(description = "各国家的状态数据列表。同一国家可分多片提交，服务端会按顺序合并")
private List<QueryAsinCountrySectionDto> countrySections = new ArrayList<>();
@JsonProperty("cartRatios")
@Schema(description = "购物车比例数据列表，通常只在店铺首行展示")
private List<QueryAsinCartRatioDto> cartRatios = new ArrayList<>();
@JsonProperty("queryAsins")
@Schema(description = "Python 回传的查询 ASIN 结果分片；如果 Python 仅透传后台 ASIN，也按该字段提交")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
@JsonProperty("countryResults")
@Schema(description = "Python 回传的查询 ASIN 结果。每个国家下按 ASIN + 状态返回，对应导出表中的国家列和状态列")
private List<QueryAsinCountryResultDto> countryResults = new ArrayList<>();
@JsonProperty("shopDone")
@Schema(description = "该店铺是否已全部提交完成。最后一片应传 true", example = "true")
@Schema(description = "该店铺是否已全部提交完成。不传时按已完成处理；分片提交时，中间分片传 false，最后一片传 true", example = "true")
private Boolean shopDone;
@JsonProperty("submissionId")
@Schema(description = "本次店铺提交批次标识，便于问题排查与日志跟踪", example = "query-asin:3089:郭亚芳:1711111111111")
@Schema(description = "本次店铺提交批次标识，便于问题排查与日志追踪", example = "query-asin:3089:郭亚芳:1711111111111")
private String submissionId;
@JsonProperty("chunkIndex")
@Schema(description = "当前分片序号，建议从 1 开始", example = "1")
private Integer chunkIndex;
@JsonProperty("chunkTotal")
@Schema(description = "当前店铺总分片数", example = "2")
private Integer chunkTotal;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinSubmitResultRequest.java
View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "查询ASIN结果回传请求")
@Schema(description = "查询 ASIN 结果回传请求")
public class QueryAsinSubmitResultRequest {
@Valid
@@ -17,4 +17,3 @@ public class QueryAsinSubmitResultRequest {
@Schema(description = "本次提交的店铺结果分片列表，支持一次提交多个店铺", requiredMode = Schema.RequiredMode.REQUIRED)
private List<QueryAsinShopPayloadDto> shops = new ArrayList<>();
}

4
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/dto/QueryAsinTaskItemDto.java
View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "查询ASIN任务中的单个店铺项")
@Schema(description = "查询 ASIN 任务中的单个店铺项")
public class QueryAsinTaskItemDto {
@JsonProperty("shopName")
@@ -40,6 +40,6 @@ public class QueryAsinTaskItemDto {
@Valid
@JsonProperty("queryAsins")
@Schema(description = "后台查询 ASIN 维护表中的国家 ASIN 列表；查询ASIN任务按全表返回，不按店铺过滤")
@Schema(description = "后台查询 ASIN 维护表中的国家 ASIN 清单；查询 ASIN 任务按全表返回，不按店铺过滤")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
}

13
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java
View File

@@ -1,9 +1,8 @@
package com.nanri.aiimage.modules.queryasin.model.vo;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCartRatioDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountrySectionDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto;
import lombok.Data;
import java.time.LocalDateTime;
@@ -45,13 +44,9 @@ public class QueryAsinResultItemVo {
private String outputFilename;
private String downloadUrl;
@JsonProperty("countrySections")
private List<QueryAsinCountrySectionDto> countrySections = new ArrayList<>();
@JsonProperty("cartRatios")
private List<QueryAsinCartRatioDto> cartRatios = new ArrayList<>();
@JsonProperty("queryAsins")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
}
@JsonProperty("countryResults")
private List<QueryAsinCountryResultDto> countryResults = new ArrayList<>();
}

130
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinExcelAssemblyService.java
View File

@@ -1,7 +1,9 @@
package com.nanri.aiimage.modules.queryasin.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinAsinStatusDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto;
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinResultItemVo;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
@@ -11,6 +13,7 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -20,7 +23,21 @@ import java.util.Map;
public class QueryAsinExcelAssemblyService {
private static final List<String> COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES");
private static final String[] HEADER = {"店铺名", "德国", "英国", "法国", "意大利", "西班牙"};
private static final Map<String, String> COUNTRY_NAMES = Map.of(
"DE", "德国",
"UK", "英国",
"FR", "法国",
"IT", "意大利",
"ES", "西班牙"
);
private static final String[] HEADER = {
"店铺名",
"德国", "状态",
"英国", "状态",
"法国", "状态",
"意大利", "状态",
"西班牙", "状态"
};
public void writeWorkbook(File outputXlsx, List<QueryAsinResultItemVo> items) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
@@ -38,14 +55,17 @@ public class QueryAsinExcelAssemblyService {
if (item == null || Boolean.FALSE.equals(item.getSuccess())) {
continue;
}
Map<String, List<String>> asinsByCountry = asinsByCountry(item.getQueryAsins());
int maxRows = maxRows(asinsByCountry);
Map<String, List<QueryAsinAsinStatusDto>> resultsByCountry = resultsByCountry(item);
int maxRows = maxRows(resultsByCountry);
for (int i = 0; i < maxRows; i++) {
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(i == 0 ? safe(item.getShopName()) : "");
for (int c = 0; c < COUNTRIES.size(); c++) {
List<String> asins = asinsByCountry.get(COUNTRIES.get(c));
row.createCell(c + 1).setCellValue(asins != null && i < asins.size() ? safe(asins.get(i)) : "");
List<QueryAsinAsinStatusDto> rows = resultsByCountry.get(COUNTRIES.get(c));
QueryAsinAsinStatusDto result = rows != null && i < rows.size() ? rows.get(i) : null;
int asinColumn = 1 + c * 2;
row.createCell(asinColumn).setCellValue(result == null ? "" : safe(result.getAsin()));
row.createCell(asinColumn + 1).setCellValue(result == null ? "" : safe(result.getStatus()));
}
}
rowIndex++;
@@ -53,7 +73,7 @@ public class QueryAsinExcelAssemblyService {
workbook.write(outputStream);
} catch (Exception ex) {
log.warn("[query-asin] write workbook failed: {}", ex.getMessage());
throw new BusinessException("generate query-asin excel failed: " + ex.getMessage());
throw new BusinessException("生成查询 ASIN Excel 失败: " + ex.getMessage());
} finally {
try {
workbook.close();
@@ -70,42 +90,106 @@ public class QueryAsinExcelAssemblyService {
}
for (QueryAsinResultItemVo item : items) {
if (item != null && !Boolean.FALSE.equals(item.getSuccess())) {
count += maxRows(asinsByCountry(item.getQueryAsins()));
count += maxRows(resultsByCountry(item));
}
}
return count;
}
private Map<String, List<String>> asinsByCountry(List<QueryAsinCountryAsinsDto> queryAsins) {
Map<String, List<String>> map = new LinkedHashMap<>();
for (String country : COUNTRIES) {
map.put(country, List.of());
}
if (queryAsins == null) {
private Map<String, List<QueryAsinAsinStatusDto>> resultsByCountry(QueryAsinResultItemVo item) {
Map<String, List<QueryAsinAsinStatusDto>> map = emptyCountryMap();
if (item == null) {
return map;
}
for (QueryAsinCountryAsinsDto item : queryAsins) {
if (item == null || item.getCountry() == null) {
continue;
if (item.getCountryResults() != null && !item.getCountryResults().isEmpty()) {
for (QueryAsinCountryResultDto countryResult : item.getCountryResults()) {
String country = normalizeCountry(countryResult == null ? null : countryResult.getCountry());
if (!map.containsKey(country)) {
continue;
}
map.put(country, normalizeResultItems(countryResult.getItems()));
}
String country = item.getCountry().trim().toUpperCase();
if (map.containsKey(country)) {
map.put(country, item.getAsins() == null ? List.of() : item.getAsins());
return map;
}
// 兼容旧链路：如果 Python 仍然只透传 queryAsins，则状态列留空。
if (item.getQueryAsins() != null) {
for (QueryAsinCountryAsinsDto countryAsins : item.getQueryAsins()) {
String country = normalizeCountry(countryAsins == null ? null : countryAsins.getCountry());
if (!map.containsKey(country)) {
continue;
}
List<QueryAsinAsinStatusDto> rows = new ArrayList<>();
for (String asin : countryAsins.getAsins() == null ? List.<String>of() : countryAsins.getAsins()) {
String normalizedAsin = normalizeAsin(asin);
if (normalizedAsin.isEmpty()) {
continue;
}
QueryAsinAsinStatusDto row = new QueryAsinAsinStatusDto();
row.setAsin(normalizedAsin);
row.setStatus("");
rows.add(row);
}
map.put(country, rows);
}
}
return map;
}
private int maxRows(Map<String, List<String>> asinsByCountry) {
private Map<String, List<QueryAsinAsinStatusDto>> emptyCountryMap() {
Map<String, List<QueryAsinAsinStatusDto>> map = new LinkedHashMap<>();
for (String country : COUNTRIES) {
map.put(country, List.of());
}
return map;
}
private List<QueryAsinAsinStatusDto> normalizeResultItems(List<QueryAsinAsinStatusDto> items) {
List<QueryAsinAsinStatusDto> out = new ArrayList<>();
if (items == null) {
return out;
}
for (QueryAsinAsinStatusDto source : items) {
String asin = normalizeAsin(source == null ? null : source.getAsin());
String status = source == null ? "" : safe(source.getStatus()).trim();
if (asin.isEmpty() && status.isEmpty()) {
continue;
}
QueryAsinAsinStatusDto item = new QueryAsinAsinStatusDto();
item.setAsin(asin);
item.setStatus(status);
out.add(item);
}
return out;
}
private int maxRows(Map<String, List<QueryAsinAsinStatusDto>> resultsByCountry) {
int max = 1;
for (List<String> asins : asinsByCountry.values()) {
if (asins != null && asins.size() > max) {
max = asins.size();
for (List<QueryAsinAsinStatusDto> rows : resultsByCountry.values()) {
if (rows != null && rows.size() > max) {
max = rows.size();
}
}
return max;
}
private String normalizeCountry(String country) {
if (country == null) {
return "";
}
String value = country.trim().toUpperCase();
for (Map.Entry<String, String> entry : COUNTRY_NAMES.entrySet()) {
if (entry.getValue().equals(country.trim())) {
return entry.getKey();
}
}
return value;
}
private String normalizeAsin(String value) {
return value == null ? "" : value.trim().toUpperCase();
}
private String safe(String value) {
return value == null ? "" : value;
}

191
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java
View File

@@ -8,10 +8,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCartRatioDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinAsinStatusDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryMetricRowDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountrySectionDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest;
@@ -523,11 +522,8 @@ public class QueryAsinTaskService {
item.setDownloadUrl(blank(entity.getResultFileUrl())
? item.getDownloadUrl()
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
if (item.getCountrySections() == null) {
item.setCountrySections(new ArrayList<>());
}
if (item.getCartRatios() == null) {
item.setCartRatios(new ArrayList<>());
if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>());
}
if (item.getQueryAsins() == null) {
item.setQueryAsins(new ArrayList<>());
@@ -568,8 +564,7 @@ public class QueryAsinTaskService {
vo.setError(result.getErrorMessage());
vo.setCreatedAt(result.getCreatedAt());
vo.setFinishedAt(finishedAt);
vo.setCountrySections(new ArrayList<>());
vo.setCartRatios(new ArrayList<>());
vo.setCountryResults(new ArrayList<>());
vo.setQueryAsins(copyCountryAsins(item.getQueryAsins()));
vo.setOutputFilename(result.getResultFilename());
vo.setDownloadUrl(null);
@@ -642,15 +637,11 @@ public class QueryAsinTaskService {
if (merged == null) {
merged = new QueryAsinShopPayloadDto();
merged.setShopName(shopKey);
merged.setCountrySections(new ArrayList<>());
merged.setCartRatios(new ArrayList<>());
merged.setQueryAsins(new ArrayList<>());
merged.setCountryResults(new ArrayList<>());
}
merged.setShopName(firstNonBlank(incoming.getShopName(), merged.getShopName()));
merged.setSubmissionId(firstNonBlank(incoming.getSubmissionId(), merged.getSubmissionId()));
merged.setChunkIndex(incoming.getChunkIndex());
merged.setChunkTotal(incoming.getChunkTotal());
if (!blank(incoming.getError())) {
merged.setError(incoming.getError().trim());
@@ -659,10 +650,8 @@ public class QueryAsinTaskService {
return merged;
}
merged.setCountrySections(mergeCountrySections(merged.getCountrySections(), incoming.getCountrySections()));
merged.setCartRatios(mergeCartRatios(merged.getCartRatios(), incoming.getCartRatios()));
merged.setQueryAsins(mergeCountryAsins(merged.getQueryAsins(), incoming.getQueryAsins()));
if (Boolean.TRUE.equals(incoming.getShopDone())) {
merged.setCountryResults(mergeCountryResults(merged.getCountryResults(), incoming.getCountryResults()));
if (incoming.getShopDone() == null || Boolean.TRUE.equals(incoming.getShopDone())) {
merged.setShopDone(Boolean.TRUE);
}
taskCacheService.saveShopMergedPayload(taskId, shopKey, merged);
@@ -674,11 +663,7 @@ public class QueryAsinTaskService {
return;
}
snapshot.setShopName(firstNonBlank(payload.getShopName(), snapshot.getShopName()));
snapshot.setCountrySections(copyCountrySections(payload.getCountrySections()));
snapshot.setCartRatios(copyCartRatios(payload.getCartRatios()));
if (payload.getQueryAsins() != null && !payload.getQueryAsins().isEmpty()) {
snapshot.setQueryAsins(copyCountryAsins(payload.getQueryAsins()));
}
snapshot.setCountryResults(copyCountryResults(payload.getCountryResults()));
if (!blank(payload.getError())) {
snapshot.setError(payload.getError().trim());
}
@@ -700,37 +685,38 @@ public class QueryAsinTaskService {
snapshot.setError(blankToNull(error));
}
private List<QueryAsinCountrySectionDto> mergeCountrySections(List<QueryAsinCountrySectionDto> base, List<QueryAsinCountrySectionDto> incoming) {
Map<String, QueryAsinCountrySectionDto> map = new LinkedHashMap<>();
for (QueryAsinCountrySectionDto section : copyCountrySections(base)) {
map.put(section.getCountry(), section);
private List<QueryAsinCountryResultDto> mergeCountryResults(List<QueryAsinCountryResultDto> base, List<QueryAsinCountryResultDto> incoming) {
Map<String, QueryAsinCountryResultDto> map = new LinkedHashMap<>();
for (QueryAsinCountryResultDto item : copyCountryResults(base)) {
map.put(item.getCountry(), item);
}
for (QueryAsinCountrySectionDto section : copyCountrySections(incoming)) {
QueryAsinCountrySectionDto existing = map.get(section.getCountry());
if (existing == null) {
map.put(section.getCountry(), section);
for (QueryAsinCountryResultDto item : copyCountryResults(incoming)) {
if (blank(item.getCountry())) {
continue;
}
List<QueryAsinCountryMetricRowDto> mergedRows = existing.getRows() == null
? new ArrayList<>()
: new ArrayList<>(existing.getRows());
if (section.getRows() != null) {
mergedRows.addAll(section.getRows());
QueryAsinCountryResultDto existing = map.get(item.getCountry());
if (existing == null) {
map.put(item.getCountry(), item);
continue;
}
existing.setRows(mergedRows);
}
return new ArrayList<>(map.values());
}
private List<QueryAsinCartRatioDto> mergeCartRatios(List<QueryAsinCartRatioDto> base, List<QueryAsinCartRatioDto> incoming) {
Map<String, QueryAsinCartRatioDto> map = new LinkedHashMap<>();
for (QueryAsinCartRatioDto ratio : copyCartRatios(base)) {
map.put(ratio.getCountry(), ratio);
}
for (QueryAsinCartRatioDto ratio : copyCartRatios(incoming)) {
if (!blank(ratio.getCountry())) {
map.put(ratio.getCountry(), ratio);
List<QueryAsinAsinStatusDto> merged = new ArrayList<>(existing.getItems() == null ? List.of() : existing.getItems());
for (QueryAsinAsinStatusDto row : item.getItems() == null ? List.<QueryAsinAsinStatusDto>of() : item.getItems()) {
String asin = normalizeAsin(row == null ? null : row.getAsin());
String status = row == null ? "" : firstNonBlank(row.getStatus(), "").trim();
if (asin.isEmpty() && status.isEmpty()) {
continue;
}
boolean exists = merged.stream().anyMatch(old ->
Objects.equals(normalizeAsin(old.getAsin()), asin)
&& Objects.equals(firstNonBlank(old.getStatus(), "").trim(), status));
if (!exists) {
QueryAsinAsinStatusDto normalized = new QueryAsinAsinStatusDto();
normalized.setAsin(asin);
normalized.setStatus(status);
merged.add(normalized);
}
}
existing.setItems(merged);
}
return new ArrayList<>(map.values());
}
@@ -741,17 +727,19 @@ public class QueryAsinTaskService {
map.put(item.getCountry(), item);
}
for (QueryAsinCountryAsinsDto item : copyCountryAsins(incoming)) {
if (blank(item.getCountry())) {
String country = normalizeCountry(item.getCountry());
if (blank(country)) {
continue;
}
QueryAsinCountryAsinsDto existing = map.get(item.getCountry());
item.setCountry(country);
QueryAsinCountryAsinsDto existing = map.get(country);
if (existing == null) {
map.put(item.getCountry(), item);
map.put(country, item);
continue;
}
List<String> merged = new ArrayList<>(existing.getAsins() == null ? List.of() : existing.getAsins());
for (String asin : item.getAsins() == null ? List.<String>of() : item.getAsins()) {
String normalized = asin == null ? "" : asin.trim().toUpperCase();
String normalized = normalizeAsin(asin);
if (!normalized.isEmpty() && !merged.contains(normalized)) {
merged.add(normalized);
}
@@ -765,35 +753,18 @@ public class QueryAsinTaskService {
if (payload == null) {
return false;
}
if (payload.getQueryAsins() != null) {
for (QueryAsinCountryAsinsDto item : payload.getQueryAsins()) {
if (item != null && item.getAsins() != null && !item.getAsins().isEmpty()) {
return true;
}
}
}
if (payload.getCountrySections() != null) {
for (QueryAsinCountrySectionDto section : payload.getCountrySections()) {
if (section == null || section.getRows() == null) {
if (payload.getCountryResults() != null) {
for (QueryAsinCountryResultDto countryResult : payload.getCountryResults()) {
if (countryResult == null || countryResult.getItems() == null) {
continue;
}
for (QueryAsinCountryMetricRowDto row : section.getRows()) {
if (row != null && (!blank(row.getStatus())
|| !blank(row.getQuantity())
|| !blank(row.getDeleteQuantity())
|| !blank(row.getProcessStatus()))) {
for (QueryAsinAsinStatusDto item : countryResult.getItems()) {
if (item != null && (!blank(item.getAsin()) || !blank(item.getStatus()))) {
return true;
}
}
}
}
if (payload.getCartRatios() != null) {
for (QueryAsinCartRatioDto ratio : payload.getCartRatios()) {
if (ratio != null && (!blank(ratio.getCountry()) || !blank(ratio.getRatio()))) {
return true;
}
}
}
return false;
}
@@ -872,35 +843,32 @@ public class QueryAsinTaskService {
}
}
private List<QueryAsinCountrySectionDto> copyCountrySections(List<QueryAsinCountrySectionDto> sections) {
List<QueryAsinCountrySectionDto> copy = new ArrayList<>();
if (sections == null) {
private List<QueryAsinCountryResultDto> copyCountryResults(List<QueryAsinCountryResultDto> results) {
List<QueryAsinCountryResultDto> copy = new ArrayList<>();
if (results == null) {
return copy;
}
for (QueryAsinCountrySectionDto section : sections) {
if (section == null) {
for (QueryAsinCountryResultDto source : results) {
if (source == null || blank(source.getCountry())) {
continue;
}
QueryAsinCountrySectionDto item = new QueryAsinCountrySectionDto();
item.setCountry(section.getCountry());
item.setRows(section.getRows() == null ? new ArrayList<>() : new ArrayList<>(section.getRows()));
copy.add(item);
}
return copy;
}
private List<QueryAsinCartRatioDto> copyCartRatios(List<QueryAsinCartRatioDto> ratios) {
List<QueryAsinCartRatioDto> copy = new ArrayList<>();
if (ratios == null) {
return copy;
}
for (QueryAsinCartRatioDto ratio : ratios) {
if (ratio == null) {
continue;
QueryAsinCountryResultDto item = new QueryAsinCountryResultDto();
item.setCountry(normalizeCountry(source.getCountry()));
List<QueryAsinAsinStatusDto> rows = new ArrayList<>();
if (source.getItems() != null) {
for (QueryAsinAsinStatusDto sourceRow : source.getItems()) {
String asin = normalizeAsin(sourceRow == null ? null : sourceRow.getAsin());
String status = sourceRow == null ? "" : firstNonBlank(sourceRow.getStatus(), "").trim();
if (asin.isEmpty() && status.isEmpty()) {
continue;
}
QueryAsinAsinStatusDto row = new QueryAsinAsinStatusDto();
row.setAsin(asin);
row.setStatus(status);
rows.add(row);
}
}
QueryAsinCartRatioDto item = new QueryAsinCartRatioDto();
item.setCountry(ratio.getCountry());
item.setRatio(ratio.getRatio());
item.setItems(rows);
copy.add(item);
}
return copy;
@@ -916,11 +884,11 @@ public class QueryAsinTaskService {
continue;
}
QueryAsinCountryAsinsDto item = new QueryAsinCountryAsinsDto();
item.setCountry(source.getCountry().trim().toUpperCase());
item.setCountry(normalizeCountry(source.getCountry()));
List<String> asins = new ArrayList<>();
if (source.getAsins() != null) {
for (String asin : source.getAsins()) {
String normalized = asin == null ? "" : asin.trim().toUpperCase();
String normalized = normalizeAsin(asin);
if (!normalized.isEmpty() && !asins.contains(normalized)) {
asins.add(normalized);
}
@@ -936,6 +904,25 @@ public class QueryAsinTaskService {
return !blank(first) ? first : second;
}
private String normalizeCountry(String country) {
if (country == null) {
return "";
}
String value = country.trim().toUpperCase();
return switch (value) {
case "德国" -> "DE";
case "英国" -> "UK";
case "法国" -> "FR";
case "意大利" -> "IT";
case "西班牙" -> "ES";
default -> value;
};
}
private String normalizeAsin(String asin) {
return asin == null ? "" : asin.trim().toUpperCase();
}
private String safeFileStem(String value) {
String raw = value == null ? "result" : value.trim();
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");

34
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java
View File

@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
@@ -14,6 +15,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -26,11 +28,37 @@ public class ShopMatchTaskCacheService {
private static final String MODULE_TYPE = "SHOP_MATCH";
private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void touchTaskHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
}
@@ -40,6 +68,7 @@ public class ShopMatchTaskCacheService {
return;
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
@@ -59,6 +88,7 @@ public class ShopMatchTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
try {
Path taskDir = buildTaskDir(taskId);
@@ -156,6 +186,10 @@ public class ShopMatchTaskCacheService {
return buildTaskDir(taskId).resolve("_task-entity.json");
}
private String buildTaskHeartbeatKey(Long taskId) {
return "shop-match:task:heartbeat:" + taskId;
}
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
return cached != null
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());

5
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java
View File

@@ -364,6 +364,9 @@ public class ShopMatchTaskService {
throw new BusinessException("task 不存在");
}
updateTaskAndRefreshCache(task);
if ("RUNNING".equals(task.getStatus())) {
shopMatchTaskCacheService.touchTaskHeartbeat(task.getId());
}
ProductRiskCreateTaskVo vo = new ProductRiskCreateTaskVo();
vo.setTaskId(task.getId());
@@ -411,6 +414,7 @@ public class ShopMatchTaskService {
task.setUpdatedAt(now);
persistTaskRequest(task, state);
updateTaskAndRefreshCache(task);
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
}
@Transactional
@@ -467,6 +471,7 @@ public class ShopMatchTaskService {
throw new BusinessException(40901, "任务已结束，拒绝重复提交");
}
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
List<FileResultEntity> resultRows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)