后台查询Asin文档更新

This commit is contained in:
super
2026-04-24 15:14:14 +08:00
parent 326f78705d
commit 0caf62c3d2
34 changed files with 798 additions and 540 deletions

View File

@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service;
import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil; import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
@@ -15,11 +16,13 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
@@ -39,12 +42,14 @@ import java.util.Set;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j
public class DedupeRunService { public class DedupeRunService {
private static final String HEADER_SPLIT_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf";
private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408";
private final FileTaskMapper fileTaskMapper; private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper; private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties; private final StorageProperties storageProperties;
@@ -52,8 +57,9 @@ public class DedupeRunService {
private final DedupeTotalDataService dedupeTotalDataService; private final DedupeTotalDataService dedupeTotalDataService;
public DedupeRunVo run(DedupeRunRequest request) { public DedupeRunVo run(DedupeRunRequest request) {
long runStartNs = System.nanoTime();
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) { if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
throw new BusinessException("请至少选择一种 ID 保留规则"); throw new BusinessException("\u8bf7\u81f3\u5c11\u9009\u62e9\u4e00\u79cd ID \u4fdd\u7559\u89c4\u5219");
} }
FileTaskEntity task = new FileTaskEntity(); FileTaskEntity task = new FileTaskEntity();
@@ -80,9 +86,11 @@ public class DedupeRunService {
DedupeResultItemVo item = new DedupeResultItemVo(); DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename()); item.setSourceFilename(sourceFile.getOriginalFilename());
try { try {
long fileStartNs = System.nanoTime();
File inputFile = findLocalSourceFile(sourceFile.getFileKey()); File inputFile = findLocalSourceFile(sourceFile.getFileKey());
long findFileNs = elapsedNs(fileStartNs);
if (inputFile == null || !inputFile.exists()) { if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("上传文件不存在,请重新上传"); throw new BusinessException("\u4e0a\u4f20\u6587\u4ef6\u4e0d\u5b58\u5728\uff0c\u8bf7\u91cd\u65b0\u4e0a\u4f20");
} }
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
@@ -90,6 +98,7 @@ public class DedupeRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
File outputFile = buildNamedOutputFile(outputDir, outputFilename); File outputFile = buildNamedOutputFile(outputDir, outputFilename);
long cleanStartNs = System.nanoTime();
cleanExcelByLegacyRules( cleanExcelByLegacyRules(
inputFile, inputFile,
outputFile, outputFile,
@@ -98,6 +107,7 @@ public class DedupeRunService {
request.isKeepUnderscoreIds(), request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds() request.isKeepIntegerMainIdsWhenNoSubIds()
); );
long cleanNs = elapsedNs(cleanStartNs);
if (folderMode) { if (folderMode) {
archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile)); archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile));
@@ -105,13 +115,15 @@ public class DedupeRunService {
continue; continue;
} }
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE"); long uploadStartNs = System.nanoTime();
// 只存 objectKey不存预签名 URL OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(outputFile, "DEDUPE");
long uploadNs = elapsedNs(uploadStartNs);
String ossObjectKey = uploadedResult.objectKey();
String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName(); String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName();
item.setSuccess(true); item.setSuccess(true);
item.setOutputFilename(downloadFilename); item.setOutputFilename(downloadFilename);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey)); item.setDownloadUrl(uploadedResult.downloadUrl());
successCount++; successCount++;
FileResultEntity resultEntity = new FileResultEntity(); FileResultEntity resultEntity = new FileResultEntity();
@@ -119,13 +131,26 @@ public class DedupeRunService {
resultEntity.setModuleType("DEDUPE"); resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(inputName); resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(downloadFilename); resultEntity.setResultFilename(downloadFilename);
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey resultEntity.setResultFileUrl(ossObjectKey);
resultEntity.setResultFileSize(outputFile.length()); resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
resultEntity.setSuccess(1); resultEntity.setSuccess(1);
resultEntity.setUserId(request.getUserId()); resultEntity.setUserId(request.getUserId());
resultEntity.setCreatedAt(LocalDateTime.now()); resultEntity.setCreatedAt(LocalDateTime.now());
long resultInsertStartNs = System.nanoTime();
fileResultMapper.insert(resultEntity); fileResultMapper.insert(resultEntity);
long resultInsertNs = elapsedNs(resultInsertStartNs);
log.info(
"dedupe run file done fileKey={} filename={} size={} findFileMs={} cleanMs={} uploadMs={} resultInsertMs={} totalFileMs={}",
sourceFile.getFileKey(),
inputName,
inputFile.length(),
toMs(findFileNs),
toMs(cleanNs),
toMs(uploadNs),
toMs(resultInsertNs),
toMs(elapsedNs(fileStartNs))
);
} catch (Exception ex) { } catch (Exception ex) {
item.setSuccess(false); item.setSuccess(false);
item.setError(ex.getMessage()); item.setError(ex.getMessage());
@@ -146,20 +171,20 @@ public class DedupeRunService {
if (folderMode && !archiveEntries.isEmpty()) { if (folderMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries); File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, "DEDUPE"); OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE");
// 只存 objectKey String ossObjectKey = uploadedResult.objectKey();
DedupeResultItemVo item = new DedupeResultItemVo(); DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(request.getArchiveName()); item.setSourceFilename(request.getArchiveName());
item.setOutputFilename(zipFile.getName()); item.setOutputFilename(zipFile.getName());
item.setSuccess(true); item.setSuccess(true);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey)); item.setDownloadUrl(uploadedResult.downloadUrl());
FileResultEntity resultEntity = new FileResultEntity(); FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId()); resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("DEDUPE"); resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(request.getArchiveName()); resultEntity.setSourceFilename(request.getArchiveName());
resultEntity.setResultFilename(zipFile.getName()); resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey resultEntity.setResultFileUrl(ossObjectKey);
resultEntity.setResultFileSize(zipFile.length()); resultEntity.setResultFileSize(zipFile.length());
resultEntity.setResultContentType("application/zip"); resultEntity.setResultContentType("application/zip");
resultEntity.setSuccess(1); resultEntity.setSuccess(1);
@@ -181,6 +206,14 @@ public class DedupeRunService {
vo.setSuccessCount(successCount); vo.setSuccessCount(successCount);
vo.setFailedCount(failedCount); vo.setFailedCount(failedCount);
vo.setItems(items); vo.setItems(items);
log.info(
"dedupe run done userId={} files={} success={} failed={} totalMs={}",
request.getUserId(),
request.getFiles().size(),
successCount,
failedCount,
toMs(elapsedNs(runStartNs))
);
return vo; return vo;
} }
@@ -209,7 +242,7 @@ public class DedupeRunService {
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { if (entity == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在"); throw new BusinessException("\u8bb0\u5f55\u4e0d\u5b58\u5728");
} }
fileResultMapper.deleteById(resultId); fileResultMapper.deleteById(resultId);
} }
@@ -217,115 +250,64 @@ public class DedupeRunService {
public Resource getResultFile(Long resultId, Long userId) { public Resource getResultFile(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("结果文件不存在"); throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728");
} }
File file = new File(entity.getResultFileUrl()); File file = new File(entity.getResultFileUrl());
if (!file.exists()) { if (!file.exists()) {
throw new BusinessException("结果文件不存在"); throw new BusinessException("\u7ed3\u679c\u6587\u4ef6\u4e0d\u5b58\u5728");
} }
return new FileSystemResource(file); return new FileSystemResource(file);
} }
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns, private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
DataFormatter formatter = new DataFormatter(); long readStartNs = System.nanoTime();
try (FileInputStream fis = new FileInputStream(inputFile); DedupeReadResult readResult = readDedupeRows(
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis); inputFile,
SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) { selectedColumns,
Sheet sheet = workbook.getSheetAt(0); keepIntegerIds,
Row headerRow = sheet.getRow(0); keepUnderscoreIds,
if (headerRow == null) { keepIntegerMainIdsWhenNoSubIds
throw new BusinessException("Excel 表头为空"); );
} long readNs = elapsedNs(readStartNs);
Map<String, Integer> headerMap = new HashMap<>(); Set<String> candidateAsinValues = new HashSet<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) { for (DedupeCandidateRow row : readResult.rows()) {
Cell cell = headerRow.getCell(i); if (!row.asinValue().isBlank()) {
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell)); candidateAsinValues.add(row.asinValue());
if (value.isBlank()) {
continue;
}
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
? value
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if ("缩略图地址8".equals(value)) {
break;
}
} }
}
long dbStartNs = System.nanoTime();
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
long dbNs = elapsedNs(dbStartNs);
List<String> missing = selectedColumns.stream().filter(column -> !headerMap.containsKey(column)).toList(); long writeStartNs = System.nanoTime();
if (!missing.isEmpty()) { int outputRows = 0;
throw new BusinessException("缺少列:" + String.join("", missing)); try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
} org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName()));
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
Row outputHeaderRow = outputSheet.createRow(0); Row outputHeaderRow = outputSheet.createRow(0);
for (int i = 0; i < selectedColumns.size(); i++) { for (int i = 0; i < selectedColumns.size(); i++) {
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i)); outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
} }
Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
if (idColumnIndex != null) {
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
Set<String> candidateAsinValues = new HashSet<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
continue;
}
}
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
candidateAsinValues.add(asinValue);
}
}
}
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> writtenAsinValues = new HashSet<>(); Set<String> writtenAsinValues = new HashSet<>();
int outputRowIndex = 1; int outputRowIndex = 1;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { for (DedupeCandidateRow candidateRow : readResult.rows()) {
Row row = sheet.getRow(rowNum); String asinValue = candidateRow.asinValue();
if (row == null) { if (!asinValue.isBlank()) {
continue; if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
continue; continue;
} }
} if (!writtenAsinValues.add(asinValue)) {
if (asinColumnIndex != null) { continue;
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
continue;
}
if (!writtenAsinValues.add(asinValue)) {
continue;
}
} }
} }
Row outputRow = outputSheet.createRow(outputRowIndex++); Row outputRow = outputSheet.createRow(outputRowIndex++);
for (int i = 0; i < selectedColumns.size(); i++) { outputRows++;
Integer sourceIndex = headerMap.get(selectedColumns.get(i)); for (int i = 0; i < candidateRow.selectedValues().size(); i++) {
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); outputRow.createCell(i).setCellValue(candidateRow.selectedValues().get(i));
outputRow.createCell(i).setCellValue(value);
} }
} }
@@ -334,6 +316,157 @@ public class DedupeRunService {
} }
outputWorkbook.dispose(); outputWorkbook.dispose();
} }
long writeNs = elapsedNs(writeStartNs);
log.info(
"dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}",
inputFile.getName(),
readResult.scannedRows(),
readResult.rows().size(),
candidateAsinValues.size(),
matchedAsinValues.size(),
outputRows,
toMs(readNs),
toMs(dbNs),
toMs(writeNs),
toMs(readNs + dbNs + writeNs)
);
}
private DedupeReadResult readDedupeRows(File inputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
List<DedupeCandidateRow> rows = new ArrayList<>();
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a");
}
Map<String, Integer> headerMap = buildHeaderMap(headerRow, formatter);
if (headerMap.isEmpty()) {
throw new BusinessException("Excel \u8868\u5934\u4e3a\u7a7a");
}
List<String> missing = selectedColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("\u7f3a\u5c11\u5217\uff1a" + String.join("\u3001", missing));
}
List<Integer> selectedIndexes = new ArrayList<>(selectedColumns.size());
for (String selectedColumn : selectedColumns) {
selectedIndexes.add(headerMap.get(selectedColumn));
}
Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
int scannedRows = 0;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
scannedRows++;
if (idColumnIndex == null) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
continue;
}
appendRowByIdRule(
normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))),
row,
selectedIndexes,
asinColumnIndex,
formatter,
keepIntegerIds,
keepUnderscoreIds,
keepIntegerMainIdsWhenNoSubIds,
pendingMainIdGroup,
rows
);
}
pendingMainIdGroup.flush(rows);
return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows);
}
}
private Map<String, Integer> buildHeaderMap(Row headerRow, DataFormatter formatter) {
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeHeaderText(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank()) {
continue;
}
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if (HEADER_STOP_COLUMN.equals(value)) {
break;
}
}
return headerMap;
}
private String normalizeHeaderText(String value) {
String normalized = normalizeCellText(value);
int markerIndex = normalized.indexOf(HEADER_SPLIT_MARKER);
if (markerIndex > 0) {
String prefix = normalized.substring(0, markerIndex).trim();
if (!prefix.isBlank()) {
return prefix;
}
}
return normalized;
}
private DedupeCandidateRow buildCandidateRow(Row row, List<Integer> selectedIndexes, Integer asinColumnIndex, DataFormatter formatter) {
List<String> selectedValues = new ArrayList<>(selectedIndexes.size());
for (Integer selectedIndex : selectedIndexes) {
selectedValues.add(normalizeCellText(formatter.formatCellValue(row.getCell(selectedIndex))));
}
String asinValue = asinColumnIndex == null
? ""
: dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
return new DedupeCandidateRow(selectedValues, asinValue);
}
private void appendRowByIdRule(String idValue, Row row,
List<Integer> selectedIndexes, Integer asinColumnIndex,
DataFormatter formatter,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds,
PendingMainIdGroup pendingMainIdGroup,
List<DedupeCandidateRow> rows) {
if (idValue == null || idValue.isBlank()) {
return;
}
String mainId = extractMainId(idValue);
if (pendingMainIdGroup.hasDifferentMainId(mainId)) {
pendingMainIdGroup.flush(rows);
}
if (isUnderscoreId(idValue)) {
pendingMainIdGroup.discardIfSameMainId(mainId);
if (keepUnderscoreIds) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
}
return;
}
if (isIntegerId(idValue)) {
if (keepIntegerIds) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
return;
}
if (keepIntegerMainIdsWhenNoSubIds) {
pendingMainIdGroup.add(mainId, buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
}
}
} }
private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) { private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
@@ -347,7 +480,7 @@ public class DedupeRunService {
zos.closeEntry(); zos.closeEntry();
} }
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException("去重结果打包失败:" + ex.getMessage()); throw new BusinessException("\u53bb\u91cd\u7ed3\u679c\u6253\u5305\u5931\u8d25\uff1a" + ex.getMessage());
} }
return zipFile; return zipFile;
} }
@@ -368,67 +501,61 @@ public class DedupeRunService {
return String.join("/", parts); return String.join("/", parts);
} }
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds, Sheet sheet, int currentRowNum, int idColumnIndex, DataFormatter formatter) {
if (text == null || text.isBlank()) {
return false;
}
if (keepUnderscoreIds && text.matches("\\d+_\\d+")) {
return true;
}
if (keepIntegerIds && text.matches("\\d+")) {
return true;
}
if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) {
String mainId = extractMainId(text);
if (mainId.isEmpty()) return false;
boolean hasSubIdInGroup = false;
for (int r = currentRowNum + 1; r <= sheet.getLastRowNum(); r++) {
Row nextRow = sheet.getRow(r);
if (nextRow == null) continue;
Cell cell = nextRow.getCell(idColumnIndex);
if (cell == null) continue;
String nextIdValue = normalizeCellText(formatter.formatCellValue(cell));
if (nextIdValue.isBlank()) continue;
String nextMainId = extractMainId(nextIdValue);
if (nextMainId.equals(mainId)) {
if (nextIdValue.matches("\\d+_\\d+")) {
hasSubIdInGroup = true;
break;
}
} else {
// Encountered a different main ID. The group for 'mainId' has ended.
break;
}
}
return !hasSubIdInGroup;
}
return false;
}
private String extractMainId(String text) { private String extractMainId(String text) {
if (text == null || text.isBlank()) { if (text == null || text.isBlank()) {
return ""; return "";
} }
if (text.matches("\\d+")) { if (isIntegerId(text)) {
return text; return text;
} }
if (text.matches("\\d+_\\d+")) { if (isUnderscoreId(text)) {
int idx = text.indexOf('_'); int idx = text.indexOf('_');
return idx > 0 ? text.substring(0, idx) : ""; return idx > 0 ? text.substring(0, idx) : "";
} }
return ""; return "";
} }
private boolean isIntegerId(String text) {
if (text == null || text.isEmpty()) {
return false;
}
for (int i = 0; i < text.length(); i++) {
if (!Character.isDigit(text.charAt(i))) {
return false;
}
}
return true;
}
private boolean isUnderscoreId(String text) {
if (text == null || text.length() < 3) {
return false;
}
int underscoreIndex = text.indexOf('_');
if (underscoreIndex <= 0 || underscoreIndex == text.length() - 1 || text.indexOf('_', underscoreIndex + 1) >= 0) {
return false;
}
for (int i = 0; i < text.length(); i++) {
if (i == underscoreIndex) {
continue;
}
if (!Character.isDigit(text.charAt(i))) {
return false;
}
}
return true;
}
private File findLocalSourceFile(String fileKey) { private File findLocalSourceFile(String fileKey) {
File baseDir = FileUtil.file(storageProperties.getLocalTempDir()); File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
if (!baseDir.exists()) { if (!baseDir.exists()) {
return null; return null;
} }
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); File[] files = baseDir.listFiles(pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst(); if (files == null || files.length == 0) {
return null;
}
return files[0];
} }
private File buildNamedOutputFile(File outputDir, String filename) { private File buildNamedOutputFile(File outputDir, String filename) {
@@ -461,16 +588,102 @@ public class DedupeRunService {
if (value == null) { if (value == null) {
return ""; return "";
} }
return value.replace("", "") int start = 0;
.replace(" ", " ") int end = value.length();
.replace("\r\n", " ") while (start < end && isTrimmedWhitespace(value.charAt(start))) {
.replace("\r", " ") start++;
.replace("\n", " ") }
.replace("\t", " ") while (end > start && isTrimmedWhitespace(value.charAt(end - 1))) {
.trim() end--;
.replaceAll("\\s+", " "); }
if (start >= end) {
return "";
}
StringBuilder builder = null;
boolean previousWhitespace = false;
for (int i = start; i < end; i++) {
char ch = value.charAt(i);
if (ch == '\uFEFF') {
if (builder == null) {
builder = new StringBuilder(value.length());
builder.append(value, start, i);
}
continue;
}
if (isNormalizedWhitespace(ch)) {
if (builder == null) {
builder = new StringBuilder(value.length());
builder.append(value, start, i);
}
if (!previousWhitespace) {
builder.append(' ');
}
previousWhitespace = true;
continue;
}
if (builder != null) {
builder.append(ch);
}
previousWhitespace = false;
}
return builder == null ? value.substring(start, end) : builder.toString();
}
private boolean isTrimmedWhitespace(char ch) {
return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch);
}
private boolean isNormalizedWhitespace(char ch) {
return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\u3000' || Character.isWhitespace(ch);
} }
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) { private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
} }
private long elapsedNs(long startNs) {
return System.nanoTime() - startNs;
}
private long toMs(long nanos) {
return nanos / 1_000_000L;
}
private record DedupeReadResult(String sheetName, List<DedupeCandidateRow> rows, int scannedRows) {
}
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
}
private static final class PendingMainIdGroup {
private String mainId;
private final List<DedupeCandidateRow> rows = new ArrayList<>();
private void add(String nextMainId, DedupeCandidateRow row) {
if (hasDifferentMainId(nextMainId)) {
rows.clear();
}
mainId = nextMainId;
rows.add(row);
}
private boolean hasDifferentMainId(String nextMainId) {
return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId));
}
private void discardIfSameMainId(String nextMainId) {
if (mainId != null && mainId.equals(nextMainId)) {
rows.clear();
mainId = null;
}
}
private void flush(List<DedupeCandidateRow> outputRows) {
if (!rows.isEmpty()) {
outputRows.addAll(rows);
rows.clear();
}
mainId = null;
}
}
} }

View File

@@ -38,7 +38,7 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor @RequiredArgsConstructor
public class DedupeTotalDataService { public class DedupeTotalDataService {
private static final int COMPARE_BATCH_SIZE = 1000; private static final int COMPARE_BATCH_SIZE = 5000;
private final DedupeTotalDataMapper dedupeTotalDataMapper; private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>(); private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();

View File

@@ -298,6 +298,8 @@ public class DeleteBrandStaleTaskService {
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats(); ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes()); long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes());
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getShopMatchInitialTimeoutMinutes()); long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getShopMatchInitialTimeoutMinutes());
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
long nowMillis = System.currentTimeMillis();
LocalDateTime now = LocalDateTime.now(); LocalDateTime now = LocalDateTime.now();
LocalDateTime threshold = now.minusMinutes(minutes); LocalDateTime threshold = now.minusMinutes(minutes);
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes); LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
@@ -313,10 +315,17 @@ public class DeleteBrandStaleTaskService {
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}", log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes); runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) { for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = shopMatchTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId()); boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0) boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0); || (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows; boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}", log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",

View File

@@ -37,6 +37,19 @@ public class OssStorageService {
} }
} }
public UploadedResult uploadResultFileWithFreshDownloadUrl(File file, String moduleType) {
String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName());
OSS ossClient = buildClient();
try {
ossClient.putObject(ossProperties.getBucket(), objectKey, file);
Date expiration = new Date(System.currentTimeMillis() + 3600_000L);
URL url = ossClient.generatePresignedUrl(ossProperties.getBucket(), objectKey, expiration);
return new UploadedResult(objectKey, url.toString());
} finally {
ossClient.shutdown();
}
}
public String uploadText(String objectKey, String content) { public String uploadText(String objectKey, String content) {
if (objectKey == null || objectKey.isBlank()) { if (objectKey == null || objectKey.isBlank()) {
throw new IllegalArgumentException("objectKey must not be blank"); throw new IllegalArgumentException("objectKey must not be blank");
@@ -160,4 +173,7 @@ public class OssStorageService {
ossProperties.getAccessKeySecret() ossProperties.getAccessKeySecret()
); );
} }
public record UploadedResult(String objectKey, String downloadUrl) {
}
} }

View File

@@ -1,6 +1,11 @@
package com.nanri.aiimage.modules.queryasin.controller; package com.nanri.aiimage.modules.queryasin.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinTaskBatchRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinTaskBatchRequest;
@@ -9,11 +14,6 @@ import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinHistoryVo;
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinTaskBatchVo; import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinTaskBatchVo;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinResolveService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinResolveService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.enums.ParameterIn;
@@ -46,14 +46,14 @@ import java.util.List;
@RequestMapping("/api/query-asin") @RequestMapping("/api/query-asin")
@Tag( @Tag(
name = "查询ASIN", name = "查询ASIN",
description = "查询ASIN模块管理备选店铺、批量匹配紫鸟店铺、创建任务、接收 Python 分片结果、自动收尾生成 Excel 并提供下载。涉及查询和删除接口需要携带 user_id。") description = "查询 ASIN 模块:管理备选店铺、批量匹配紫鸟店铺、创建任务、接收 Python 分片结果、自动收尾生成 Excel 并提供下载。查询和删除接口需要携带 user_id。")
public class QueryAsinTaskController { public class QueryAsinTaskController {
private final QueryAsinResolveService queryAsinResolveService; private final QueryAsinResolveService queryAsinResolveService;
private final QueryAsinTaskService queryAsinTaskService; private final QueryAsinTaskService queryAsinTaskService;
@GetMapping("/candidates") @GetMapping("/candidates")
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询ASIN模块中已保存的备选店铺。") @Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询 ASIN 模块中已保存的备选店铺。")
public ApiResponse<List<ProductRiskCandidateVo>> listCandidates( public ApiResponse<List<ProductRiskCandidateVo>> listCandidates(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1") @Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) { @RequestParam("user_id") Long userId) {
@@ -61,10 +61,10 @@ public class QueryAsinTaskController {
} }
@PostMapping("/candidates") @PostMapping("/candidates")
@Operation(summary = "新增备选店铺", description = "向查询ASIN备选区新增一条店铺记录供后续匹配和创建任务使用。") @Operation(summary = "新增备选店铺", description = "向查询 ASIN 备选区新增一条店铺记录,供后续匹配和创建任务使用。")
public ApiResponse<ProductRiskCandidateVo> addCandidate( public ApiResponse<ProductRiskCandidateVo> addCandidate(
@io.swagger.v3.oas.annotations.parameters.RequestBody( @io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "新增备选店铺请求,需传入用户 ID 和店铺名。", description = "新增备选店铺请求,需传入用户 ID 和店铺名。",
required = true, required = true,
content = @Content( content = @Content(
mediaType = "application/json", mediaType = "application/json",
@@ -93,7 +93,7 @@ public class QueryAsinTaskController {
} }
@PostMapping("/match-shops") @PostMapping("/match-shops")
@Operation(summary = "批量匹配店铺", description = "根据店铺名批量匹配紫鸟店铺索引,返回是否命中、店铺 ID、平台、公司和匹配状态。") @Operation(summary = "批量匹配店铺", description = "根据店铺名批量匹配紫鸟店铺索引,返回是否命中、店铺 ID、平台、公司和匹配状态。")
public ApiResponse<ProductRiskMatchShopsVo> matchShops( public ApiResponse<ProductRiskMatchShopsVo> matchShops(
@io.swagger.v3.oas.annotations.parameters.RequestBody( @io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "批量匹配请求,传入 user_id 和待匹配的店铺名称列表。", description = "批量匹配请求,传入 user_id 和待匹配的店铺名称列表。",
@@ -125,7 +125,7 @@ public class QueryAsinTaskController {
} }
@GetMapping("/history") @GetMapping("/history")
@Operation(summary = "查询任务记录", description = "返回当前用户在查询ASIN模块中的当前任务和历史任务记录。") @Operation(summary = "查询任务记录", description = "返回当前用户在查询 ASIN 模块中的当前任务和历史任务记录。")
public ApiResponse<QueryAsinHistoryVo> history( public ApiResponse<QueryAsinHistoryVo> history(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1") @Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) { @RequestParam("user_id") Long userId) {
@@ -133,16 +133,16 @@ public class QueryAsinTaskController {
} }
@PostMapping("/tasks/progress/batch") @PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询查询ASIN任务进度", description = "仅返回任务状态和店铺结果摘要,用于前端轮询降载。") @Operation(summary = "批量查询查询 ASIN 任务进度", description = "仅返回任务状态和店铺结果摘要,用于前端轮询降载。")
public ApiResponse<QueryAsinTaskBatchVo> taskProgressBatch(@Valid @RequestBody QueryAsinTaskBatchRequest request) { public ApiResponse<QueryAsinTaskBatchVo> taskProgressBatch(@Valid @RequestBody QueryAsinTaskBatchRequest request) {
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds())); return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds()));
} }
@PostMapping("/tasks") @PostMapping("/tasks")
@Operation(summary = "创建查询ASIN任务", description = "根据已匹配的店铺创建任务和占位结果记录,后续由 Python 端处理并回传结果") @Operation(summary = "创建查询 ASIN 任务", description = "根据已匹配的店铺创建任务和占位结果记录,并把后台维护的整张 ASIN 表数据随店铺项返回给 Python 端。")
public ApiResponse<QueryAsinCreateTaskVo> createTask( public ApiResponse<QueryAsinCreateTaskVo> createTask(
@io.swagger.v3.oas.annotations.parameters.RequestBody( @io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "创建任务请求。items 中每一项代表一个待处理店铺,需携带默认模板结构", description = "创建任务请求。items 中每一项代表一个待处理店铺,queryAsins 是后台维护的整张 ASIN 表数据,不按店铺过滤",
required = true, required = true,
content = @Content( content = @Content(
mediaType = "application/json", mediaType = "application/json",
@@ -161,23 +161,14 @@ public class QueryAsinTaskController {
"companyName": "示例公司", "companyName": "示例公司",
"matchStatus": "MATCHED", "matchStatus": "MATCHED",
"matchMessage": "索引已命中", "matchMessage": "索引已命中",
"countrySections": [ "queryAsins": [
{ {
"country": "德国", "country": "DE",
"rows": [ "asins": ["B0BRZZR3N2", "B0BRZZR3N3"]
{ },
"status": "全部",
"quantity": "",
"deleteQuantity": "",
"processStatus": ""
}
]
}
],
"cartRatios": [
{ {
"country": "德国", "country": "FR",
"ratio": "" "asins": ["B0BRZZR3N4"]
} }
] ]
} }
@@ -190,124 +181,72 @@ public class QueryAsinTaskController {
@PostMapping("/tasks/{taskId}/result") @PostMapping("/tasks/{taskId}/result")
@Operation( @Operation(
summary = "提交查询ASIN结果", summary = "提交查询 ASIN 结果",
description = "供 Python 端回传处理结果。支持按店铺分片多次提交,服务端会合并分片、按店铺完成状态自动收尾,并在任务结束后生成 Excel") description = "供 Python 端回传查询 ASIN 结果。业务数据包含店铺名,以及德国、英国、法国、意大利、西班牙 5 个国家下的 ASIN 和处理状态submissionId 用于日志追踪shopDone 可不传,不传时后端默认该店铺已提交完成")
public ApiResponse<Void> submitResult( public ApiResponse<Void> submitResult(
@Parameter(description = "任务主键,必须是运行中的任务", example = "3089") @Parameter(description = "任务主键,必须是运行中的查询 ASIN 任务", example = "3089")
@PathVariable Long taskId, @PathVariable Long taskId,
@io.swagger.v3.oas.annotations.parameters.RequestBody( @io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "查询ASIN结果回传请求。shops 表示本次提交的店铺结果分片列表。shopDone=true 表示该店铺已全部提交完成", description = "查询 ASIN 结果回传请求。shops 表示店铺结果列表countryResults 对应导出表中的国家 ASIN 列和处理状态列error 用于回传单店铺失败原因",
required = true, required = true,
content = @Content( content = @Content(
mediaType = "application/json", mediaType = "application/json",
schema = @Schema(implementation = QueryAsinSubmitResultRequest.class), schema = @Schema(implementation = QueryAsinSubmitResultRequest.class),
examples = { examples = @ExampleObject(
@ExampleObject( name = "回传示例",
name = "成功分片示例", summary = "按店铺和国家返回 ASIN 与处理状态",
summary = "一个店铺分片回传,包含状态数据和购物车比例数据", value = """
value = """ {
"shops": [
{
"shopName": "郭亚芳",
"submissionId": "query-asin:3089:郭亚芳:1711111111111",
"shopDone": true,
"countryResults": [
{ {
"shops": [ "country": "DE",
"items": [
{ {
"shopName": "郭亚芳", "asin": "B0BRZZR3N2",
"submissionId": "query-asin:3089:郭亚芳:1711111111111", "status": "正常"
"chunkIndex": 1, },
"chunkTotal": 2, {
"shopDone": false, "asin": "B0BRZZR3N3",
"countrySections": [ "status": "未找到"
{
"country": "德国",
"rows": [
{
"status": "正常",
"quantity": "12",
"deleteQuantity": "3",
"processStatus": "处理中"
},
{
"status": "下架",
"quantity": "2",
"deleteQuantity": "2",
"processStatus": "已完成"
}
]
},
{
"country": "英国",
"rows": [
{
"status": "正常",
"quantity": "5",
"deleteQuantity": "1",
"processStatus": "处理中"
}
]
}
],
"cartRatios": [
{
"country": "德国",
"ratio": "25%"
},
{
"country": "英国",
"ratio": "18%"
}
]
} }
] ]
} },
"""),
@ExampleObject(
name = "店铺完成示例",
summary = "最后一片提交完成后,将 shopDone 置为 true",
value = """
{ {
"shops": [ "country": "UK",
"items": [
{ {
"shopName": "郭亚芳", "asin": "B0BRZZR3N6",
"submissionId": "query-asin:3089:郭亚芳:1711111111111", "status": "正常"
"chunkIndex": 2,
"chunkTotal": 2,
"shopDone": true,
"countrySections": [
{
"country": "法国",
"rows": [
{
"status": "正常",
"quantity": "4",
"deleteQuantity": "0",
"processStatus": "已完成"
}
]
}
],
"cartRatios": [
{
"country": "法国",
"ratio": "12%"
}
]
} }
] ]
} },
"""),
@ExampleObject(
name = "失败示例",
summary = "店铺处理失败时直接回传错误信息",
value = """
{ {
"shops": [ "country": "FR",
"items": [
{ {
"shopName": "郭亚芳", "asin": "B0BRZZR3N4",
"shopDone": true, "status": "正常"
"error": "紫鸟页面加载超时,未能完成删除数据采集"
} }
] ]
},
{
"country": "IT",
"items": []
},
{
"country": "ES",
"items": []
} }
""") ]
})) }
]
}
""")))
@Valid @RequestBody QueryAsinSubmitResultRequest request, @Valid @RequestBody QueryAsinSubmitResultRequest request,
jakarta.servlet.http.HttpServletResponse response) { jakarta.servlet.http.HttpServletResponse response) {
response.setCharacterEncoding(StandardCharsets.UTF_8.name()); response.setCharacterEncoding(StandardCharsets.UTF_8.name());
@@ -317,7 +256,7 @@ public class QueryAsinTaskController {
} }
@GetMapping("/results/{resultId}/download") @GetMapping("/results/{resultId}/download")
@Operation(summary = "下载结果文件", description = "按结果记录下载服务端已生成并上传到 OSS 的查询ASIN Excel 文件。") @Operation(summary = "下载结果文件", description = "按结果记录下载服务端已生成并上传到 OSS 的查询 ASIN Excel 文件。")
public void downloadResult( public void downloadResult(
@Parameter(description = "结果记录主键", example = "4599") @Parameter(description = "结果记录主键", example = "4599")
@PathVariable Long resultId, @PathVariable Long resultId,
@@ -348,7 +287,7 @@ public class QueryAsinTaskController {
} }
@DeleteMapping("/tasks/{taskId}") @DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除任务", description = "删除整条查询ASIN任务以及该任务下关联的所有结果记录。") @Operation(summary = "删除任务", description = "删除整条查询 ASIN 任务以及该任务下关联的所有结果记录。")
public ApiResponse<Void> deleteTask( public ApiResponse<Void> deleteTask(
@Parameter(description = "任务主键", example = "3089") @Parameter(description = "任务主键", example = "3089")
@PathVariable Long taskId, @PathVariable Long taskId,
@@ -359,7 +298,7 @@ public class QueryAsinTaskController {
} }
@DeleteMapping("/history/{resultId}") @DeleteMapping("/history/{resultId}")
@Operation(summary = "删除单条历史记录", description = "删除一条查询ASIN结果记录并同步重算其所属任务状态。") @Operation(summary = "删除单条历史记录", description = "删除一条查询 ASIN 结果记录,并同步重算其所属任务状态。")
public ApiResponse<Void> deleteHistory( public ApiResponse<Void> deleteHistory(
@Parameter(description = "结果记录主键", example = "4599") @Parameter(description = "结果记录主键", example = "4599")
@PathVariable Long resultId, @PathVariable Long resultId,
@@ -369,6 +308,3 @@ public class QueryAsinTaskController {
return ApiResponse.success(null); return ApiResponse.success(null);
} }
} }

View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "查询 ASIN 单条结果")
public class QueryAsinAsinStatusDto {
@Schema(description = "ASIN", example = "B0BRZZR3N2")
private String asin;
@Schema(description = "Python 查询后返回的状态", example = "正常")
private String status;
}

View File

@@ -1,16 +0,0 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "购物车比例数据")
public class QueryAsinCartRatioDto {
@Schema(description = "国家名称", example = "德国")
private String country;
@Schema(description = "购物车比例", example = "25%")
private String ratio;
}

View File

@@ -7,12 +7,12 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
@Data @Data
@Schema(description = "查询 ASIN 国家维度 ASIN 列表") @Schema(description = "查询 ASIN 国家维度 ASIN 清单")
public class QueryAsinCountryAsinsDto { public class QueryAsinCountryAsinsDto {
@Schema(description = "国家编码,支持 DE、UK、FR、IT、ES", example = "DE") @Schema(description = "国家编码,支持 DE、UK、FR、IT、ES", example = "DE")
private String country; private String country;
@Schema(description = "该国家需要查询的 ASIN 列表") @Schema(description = "后台维护的该国家 ASIN 清单,创建任务时会整体推给 Python")
private List<String> asins = new ArrayList<>(); private List<String> asins = new ArrayList<>();
} }

View File

@@ -1,25 +0,0 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "国家状态数据中的单行记录")
public class QueryAsinCountryMetricRowDto {
@Schema(description = "商品状态", example = "正常")
private String status;
@Schema(description = "该状态下的商品数量", example = "12")
private String quantity;
@JsonProperty("deleteQuantity")
@Schema(description = "已删除数量", example = "3")
private String deleteQuantity;
@JsonProperty("processStatus")
@Schema(description = "处理结果或处理状态", example = "处理中")
private String processStatus;
}

View File

@@ -0,0 +1,18 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "查询 ASIN 国家维度结果")
public class QueryAsinCountryResultDto {
@Schema(description = "国家编码,支持 DE、UK、FR、IT、ES", example = "DE")
private String country;
@Schema(description = "该国家下的 ASIN 查询结果列表")
private List<QueryAsinAsinStatusDto> items = new ArrayList<>();
}

View File

@@ -1,19 +0,0 @@
package com.nanri.aiimage.modules.queryasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个国家的状态数据分组")
public class QueryAsinCountrySectionDto {
@Schema(description = "国家名称", example = "德国")
private String country;
@Schema(description = "该国家下的多行状态数据")
private List<QueryAsinCountryMetricRowDto> rows = new ArrayList<>();
}

View File

@@ -8,42 +8,26 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
@Data @Data
@Schema(description = "单个店铺的查询ASIN结果分片") @Schema(description = "单个店铺的查询 ASIN 结果")
public class QueryAsinShopPayloadDto { public class QueryAsinShopPayloadDto {
@JsonProperty("shopName") @JsonProperty("shopName")
@Schema(description = "店铺名称,服务端按店铺名称聚合分片", example = "郭亚芳") @Schema(description = "店铺名称", example = "郭亚芳")
private String shopName; private String shopName;
@Schema(description = "店铺处理失败时的错误信息;有值时该店铺会直接标记失败", example = "紫鸟页面加载超时,未能完成删除数据采集") @Schema(description = "店铺处理失败时的错误信息;有值时该店铺会直接标记失败", example = "紫鸟页面加载超时,未能完成 ASIN 查询")
private String error; private String error;
@JsonProperty("countrySections") @JsonProperty("countryResults")
@Schema(description = "各国家的状态数据列表。同一国家可分多片提交,服务端会按顺序合并") @Schema(description = "Python 回传的查询 ASIN 结果。每个国家下按 ASIN + 状态返回,对应导出表中的国家列和状态列")
private List<QueryAsinCountrySectionDto> countrySections = new ArrayList<>(); private List<QueryAsinCountryResultDto> countryResults = new ArrayList<>();
@JsonProperty("cartRatios")
@Schema(description = "购物车比例数据列表,通常只在店铺首行展示")
private List<QueryAsinCartRatioDto> cartRatios = new ArrayList<>();
@JsonProperty("queryAsins")
@Schema(description = "Python 回传的查询 ASIN 结果分片;如果 Python 仅透传后台 ASIN也按该字段提交")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
@JsonProperty("shopDone") @JsonProperty("shopDone")
@Schema(description = "该店铺是否已全部提交完成。最后一片传 true", example = "true") @Schema(description = "该店铺是否已全部提交完成。不传时按已完成处理;分片提交时,中间分片传 false最后一片传 true", example = "true")
private Boolean shopDone; private Boolean shopDone;
@JsonProperty("submissionId") @JsonProperty("submissionId")
@Schema(description = "本次店铺提交批次标识,便于问题排查与日志", example = "query-asin:3089:郭亚芳:1711111111111") @Schema(description = "本次店铺提交批次标识,便于问题排查与日志", example = "query-asin:3089:郭亚芳:1711111111111")
private String submissionId; private String submissionId;
@JsonProperty("chunkIndex")
@Schema(description = "当前分片序号,建议从 1 开始", example = "1")
private Integer chunkIndex;
@JsonProperty("chunkTotal")
@Schema(description = "当前店铺总分片数", example = "2")
private Integer chunkTotal;
} }

View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
@Data @Data
@Schema(description = "查询ASIN结果回传请求") @Schema(description = "查询 ASIN 结果回传请求")
public class QueryAsinSubmitResultRequest { public class QueryAsinSubmitResultRequest {
@Valid @Valid
@@ -17,4 +17,3 @@ public class QueryAsinSubmitResultRequest {
@Schema(description = "本次提交的店铺结果分片列表,支持一次提交多个店铺", requiredMode = Schema.RequiredMode.REQUIRED) @Schema(description = "本次提交的店铺结果分片列表,支持一次提交多个店铺", requiredMode = Schema.RequiredMode.REQUIRED)
private List<QueryAsinShopPayloadDto> shops = new ArrayList<>(); private List<QueryAsinShopPayloadDto> shops = new ArrayList<>();
} }

View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
@Data @Data
@Schema(description = "查询ASIN任务中的单个店铺项") @Schema(description = "查询 ASIN 任务中的单个店铺项")
public class QueryAsinTaskItemDto { public class QueryAsinTaskItemDto {
@JsonProperty("shopName") @JsonProperty("shopName")
@@ -40,6 +40,6 @@ public class QueryAsinTaskItemDto {
@Valid @Valid
@JsonProperty("queryAsins") @JsonProperty("queryAsins")
@Schema(description = "后台查询 ASIN 维护表中的国家 ASIN 列表查询ASIN任务按全表返回不按店铺过滤") @Schema(description = "后台查询 ASIN 维护表中的国家 ASIN 清单;查询 ASIN 任务按全表返回,不按店铺过滤")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>(); private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
} }

View File

@@ -1,9 +1,8 @@
package com.nanri.aiimage.modules.queryasin.model.vo; package com.nanri.aiimage.modules.queryasin.model.vo;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCartRatioDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountrySectionDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto;
import lombok.Data; import lombok.Data;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -45,13 +44,9 @@ public class QueryAsinResultItemVo {
private String outputFilename; private String outputFilename;
private String downloadUrl; private String downloadUrl;
@JsonProperty("countrySections")
private List<QueryAsinCountrySectionDto> countrySections = new ArrayList<>();
@JsonProperty("cartRatios")
private List<QueryAsinCartRatioDto> cartRatios = new ArrayList<>();
@JsonProperty("queryAsins") @JsonProperty("queryAsins")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>(); private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
}
@JsonProperty("countryResults")
private List<QueryAsinCountryResultDto> countryResults = new ArrayList<>();
}

View File

@@ -1,7 +1,9 @@
package com.nanri.aiimage.modules.queryasin.service; package com.nanri.aiimage.modules.queryasin.service;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinAsinStatusDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto;
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinResultItemVo; import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinResultItemVo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Row;
@@ -11,6 +13,7 @@ import org.springframework.stereotype.Service;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -20,7 +23,21 @@ import java.util.Map;
public class QueryAsinExcelAssemblyService { public class QueryAsinExcelAssemblyService {
private static final List<String> COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES"); private static final List<String> COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES");
private static final String[] HEADER = {"店铺名", "德国", "英国", "法国", "意大利", "西班牙"}; private static final Map<String, String> COUNTRY_NAMES = Map.of(
"DE", "德国",
"UK", "英国",
"FR", "法国",
"IT", "意大利",
"ES", "西班牙"
);
private static final String[] HEADER = {
"店铺名",
"德国", "状态",
"英国", "状态",
"法国", "状态",
"意大利", "状态",
"西班牙", "状态"
};
public void writeWorkbook(File outputXlsx, List<QueryAsinResultItemVo> items) { public void writeWorkbook(File outputXlsx, List<QueryAsinResultItemVo> items) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200); SXSSFWorkbook workbook = new SXSSFWorkbook(200);
@@ -38,14 +55,17 @@ public class QueryAsinExcelAssemblyService {
if (item == null || Boolean.FALSE.equals(item.getSuccess())) { if (item == null || Boolean.FALSE.equals(item.getSuccess())) {
continue; continue;
} }
Map<String, List<String>> asinsByCountry = asinsByCountry(item.getQueryAsins()); Map<String, List<QueryAsinAsinStatusDto>> resultsByCountry = resultsByCountry(item);
int maxRows = maxRows(asinsByCountry); int maxRows = maxRows(resultsByCountry);
for (int i = 0; i < maxRows; i++) { for (int i = 0; i < maxRows; i++) {
Row row = sheet.createRow(rowIndex++); Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(i == 0 ? safe(item.getShopName()) : ""); row.createCell(0).setCellValue(i == 0 ? safe(item.getShopName()) : "");
for (int c = 0; c < COUNTRIES.size(); c++) { for (int c = 0; c < COUNTRIES.size(); c++) {
List<String> asins = asinsByCountry.get(COUNTRIES.get(c)); List<QueryAsinAsinStatusDto> rows = resultsByCountry.get(COUNTRIES.get(c));
row.createCell(c + 1).setCellValue(asins != null && i < asins.size() ? safe(asins.get(i)) : ""); QueryAsinAsinStatusDto result = rows != null && i < rows.size() ? rows.get(i) : null;
int asinColumn = 1 + c * 2;
row.createCell(asinColumn).setCellValue(result == null ? "" : safe(result.getAsin()));
row.createCell(asinColumn + 1).setCellValue(result == null ? "" : safe(result.getStatus()));
} }
} }
rowIndex++; rowIndex++;
@@ -53,7 +73,7 @@ public class QueryAsinExcelAssemblyService {
workbook.write(outputStream); workbook.write(outputStream);
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[query-asin] write workbook failed: {}", ex.getMessage()); log.warn("[query-asin] write workbook failed: {}", ex.getMessage());
throw new BusinessException("generate query-asin excel failed: " + ex.getMessage()); throw new BusinessException("生成查询 ASIN Excel 失败: " + ex.getMessage());
} finally { } finally {
try { try {
workbook.close(); workbook.close();
@@ -70,42 +90,106 @@ public class QueryAsinExcelAssemblyService {
} }
for (QueryAsinResultItemVo item : items) { for (QueryAsinResultItemVo item : items) {
if (item != null && !Boolean.FALSE.equals(item.getSuccess())) { if (item != null && !Boolean.FALSE.equals(item.getSuccess())) {
count += maxRows(asinsByCountry(item.getQueryAsins())); count += maxRows(resultsByCountry(item));
} }
} }
return count; return count;
} }
private Map<String, List<String>> asinsByCountry(List<QueryAsinCountryAsinsDto> queryAsins) { private Map<String, List<QueryAsinAsinStatusDto>> resultsByCountry(QueryAsinResultItemVo item) {
Map<String, List<String>> map = new LinkedHashMap<>(); Map<String, List<QueryAsinAsinStatusDto>> map = emptyCountryMap();
for (String country : COUNTRIES) { if (item == null) {
map.put(country, List.of());
}
if (queryAsins == null) {
return map; return map;
} }
for (QueryAsinCountryAsinsDto item : queryAsins) { if (item.getCountryResults() != null && !item.getCountryResults().isEmpty()) {
if (item == null || item.getCountry() == null) { for (QueryAsinCountryResultDto countryResult : item.getCountryResults()) {
continue; String country = normalizeCountry(countryResult == null ? null : countryResult.getCountry());
if (!map.containsKey(country)) {
continue;
}
map.put(country, normalizeResultItems(countryResult.getItems()));
} }
String country = item.getCountry().trim().toUpperCase(); return map;
if (map.containsKey(country)) { }
map.put(country, item.getAsins() == null ? List.of() : item.getAsins());
// 兼容旧链路:如果 Python 仍然只透传 queryAsins则状态列留空。
if (item.getQueryAsins() != null) {
for (QueryAsinCountryAsinsDto countryAsins : item.getQueryAsins()) {
String country = normalizeCountry(countryAsins == null ? null : countryAsins.getCountry());
if (!map.containsKey(country)) {
continue;
}
List<QueryAsinAsinStatusDto> rows = new ArrayList<>();
for (String asin : countryAsins.getAsins() == null ? List.<String>of() : countryAsins.getAsins()) {
String normalizedAsin = normalizeAsin(asin);
if (normalizedAsin.isEmpty()) {
continue;
}
QueryAsinAsinStatusDto row = new QueryAsinAsinStatusDto();
row.setAsin(normalizedAsin);
row.setStatus("");
rows.add(row);
}
map.put(country, rows);
} }
} }
return map; return map;
} }
private int maxRows(Map<String, List<String>> asinsByCountry) { private Map<String, List<QueryAsinAsinStatusDto>> emptyCountryMap() {
Map<String, List<QueryAsinAsinStatusDto>> map = new LinkedHashMap<>();
for (String country : COUNTRIES) {
map.put(country, List.of());
}
return map;
}
private List<QueryAsinAsinStatusDto> normalizeResultItems(List<QueryAsinAsinStatusDto> items) {
List<QueryAsinAsinStatusDto> out = new ArrayList<>();
if (items == null) {
return out;
}
for (QueryAsinAsinStatusDto source : items) {
String asin = normalizeAsin(source == null ? null : source.getAsin());
String status = source == null ? "" : safe(source.getStatus()).trim();
if (asin.isEmpty() && status.isEmpty()) {
continue;
}
QueryAsinAsinStatusDto item = new QueryAsinAsinStatusDto();
item.setAsin(asin);
item.setStatus(status);
out.add(item);
}
return out;
}
private int maxRows(Map<String, List<QueryAsinAsinStatusDto>> resultsByCountry) {
int max = 1; int max = 1;
for (List<String> asins : asinsByCountry.values()) { for (List<QueryAsinAsinStatusDto> rows : resultsByCountry.values()) {
if (asins != null && asins.size() > max) { if (rows != null && rows.size() > max) {
max = asins.size(); max = rows.size();
} }
} }
return max; return max;
} }
private String normalizeCountry(String country) {
if (country == null) {
return "";
}
String value = country.trim().toUpperCase();
for (Map.Entry<String, String> entry : COUNTRY_NAMES.entrySet()) {
if (entry.getValue().equals(country.trim())) {
return entry.getKey();
}
}
return value;
}
private String normalizeAsin(String value) {
return value == null ? "" : value.trim().toUpperCase();
}
private String safe(String value) { private String safe(String value) {
return value == null ? "" : value; return value == null ? "" : value;
} }

View File

@@ -8,10 +8,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCartRatioDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinAsinStatusDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryMetricRowDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryResultDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountrySectionDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto;
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest;
@@ -523,11 +522,8 @@ public class QueryAsinTaskService {
item.setDownloadUrl(blank(entity.getResultFileUrl()) item.setDownloadUrl(blank(entity.getResultFileUrl())
? item.getDownloadUrl() ? item.getDownloadUrl()
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
if (item.getCountrySections() == null) { if (item.getCountryResults() == null) {
item.setCountrySections(new ArrayList<>()); item.setCountryResults(new ArrayList<>());
}
if (item.getCartRatios() == null) {
item.setCartRatios(new ArrayList<>());
} }
if (item.getQueryAsins() == null) { if (item.getQueryAsins() == null) {
item.setQueryAsins(new ArrayList<>()); item.setQueryAsins(new ArrayList<>());
@@ -568,8 +564,7 @@ public class QueryAsinTaskService {
vo.setError(result.getErrorMessage()); vo.setError(result.getErrorMessage());
vo.setCreatedAt(result.getCreatedAt()); vo.setCreatedAt(result.getCreatedAt());
vo.setFinishedAt(finishedAt); vo.setFinishedAt(finishedAt);
vo.setCountrySections(new ArrayList<>()); vo.setCountryResults(new ArrayList<>());
vo.setCartRatios(new ArrayList<>());
vo.setQueryAsins(copyCountryAsins(item.getQueryAsins())); vo.setQueryAsins(copyCountryAsins(item.getQueryAsins()));
vo.setOutputFilename(result.getResultFilename()); vo.setOutputFilename(result.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(null);
@@ -642,15 +637,11 @@ public class QueryAsinTaskService {
if (merged == null) { if (merged == null) {
merged = new QueryAsinShopPayloadDto(); merged = new QueryAsinShopPayloadDto();
merged.setShopName(shopKey); merged.setShopName(shopKey);
merged.setCountrySections(new ArrayList<>()); merged.setCountryResults(new ArrayList<>());
merged.setCartRatios(new ArrayList<>());
merged.setQueryAsins(new ArrayList<>());
} }
merged.setShopName(firstNonBlank(incoming.getShopName(), merged.getShopName())); merged.setShopName(firstNonBlank(incoming.getShopName(), merged.getShopName()));
merged.setSubmissionId(firstNonBlank(incoming.getSubmissionId(), merged.getSubmissionId())); merged.setSubmissionId(firstNonBlank(incoming.getSubmissionId(), merged.getSubmissionId()));
merged.setChunkIndex(incoming.getChunkIndex());
merged.setChunkTotal(incoming.getChunkTotal());
if (!blank(incoming.getError())) { if (!blank(incoming.getError())) {
merged.setError(incoming.getError().trim()); merged.setError(incoming.getError().trim());
@@ -659,10 +650,8 @@ public class QueryAsinTaskService {
return merged; return merged;
} }
merged.setCountrySections(mergeCountrySections(merged.getCountrySections(), incoming.getCountrySections())); merged.setCountryResults(mergeCountryResults(merged.getCountryResults(), incoming.getCountryResults()));
merged.setCartRatios(mergeCartRatios(merged.getCartRatios(), incoming.getCartRatios())); if (incoming.getShopDone() == null || Boolean.TRUE.equals(incoming.getShopDone())) {
merged.setQueryAsins(mergeCountryAsins(merged.getQueryAsins(), incoming.getQueryAsins()));
if (Boolean.TRUE.equals(incoming.getShopDone())) {
merged.setShopDone(Boolean.TRUE); merged.setShopDone(Boolean.TRUE);
} }
taskCacheService.saveShopMergedPayload(taskId, shopKey, merged); taskCacheService.saveShopMergedPayload(taskId, shopKey, merged);
@@ -674,11 +663,7 @@ public class QueryAsinTaskService {
return; return;
} }
snapshot.setShopName(firstNonBlank(payload.getShopName(), snapshot.getShopName())); snapshot.setShopName(firstNonBlank(payload.getShopName(), snapshot.getShopName()));
snapshot.setCountrySections(copyCountrySections(payload.getCountrySections())); snapshot.setCountryResults(copyCountryResults(payload.getCountryResults()));
snapshot.setCartRatios(copyCartRatios(payload.getCartRatios()));
if (payload.getQueryAsins() != null && !payload.getQueryAsins().isEmpty()) {
snapshot.setQueryAsins(copyCountryAsins(payload.getQueryAsins()));
}
if (!blank(payload.getError())) { if (!blank(payload.getError())) {
snapshot.setError(payload.getError().trim()); snapshot.setError(payload.getError().trim());
} }
@@ -700,37 +685,38 @@ public class QueryAsinTaskService {
snapshot.setError(blankToNull(error)); snapshot.setError(blankToNull(error));
} }
private List<QueryAsinCountrySectionDto> mergeCountrySections(List<QueryAsinCountrySectionDto> base, List<QueryAsinCountrySectionDto> incoming) { private List<QueryAsinCountryResultDto> mergeCountryResults(List<QueryAsinCountryResultDto> base, List<QueryAsinCountryResultDto> incoming) {
Map<String, QueryAsinCountrySectionDto> map = new LinkedHashMap<>(); Map<String, QueryAsinCountryResultDto> map = new LinkedHashMap<>();
for (QueryAsinCountrySectionDto section : copyCountrySections(base)) { for (QueryAsinCountryResultDto item : copyCountryResults(base)) {
map.put(section.getCountry(), section); map.put(item.getCountry(), item);
} }
for (QueryAsinCountrySectionDto section : copyCountrySections(incoming)) { for (QueryAsinCountryResultDto item : copyCountryResults(incoming)) {
QueryAsinCountrySectionDto existing = map.get(section.getCountry()); if (blank(item.getCountry())) {
if (existing == null) {
map.put(section.getCountry(), section);
continue; continue;
} }
List<QueryAsinCountryMetricRowDto> mergedRows = existing.getRows() == null QueryAsinCountryResultDto existing = map.get(item.getCountry());
? new ArrayList<>() if (existing == null) {
: new ArrayList<>(existing.getRows()); map.put(item.getCountry(), item);
if (section.getRows() != null) { continue;
mergedRows.addAll(section.getRows());
} }
existing.setRows(mergedRows); List<QueryAsinAsinStatusDto> merged = new ArrayList<>(existing.getItems() == null ? List.of() : existing.getItems());
} for (QueryAsinAsinStatusDto row : item.getItems() == null ? List.<QueryAsinAsinStatusDto>of() : item.getItems()) {
return new ArrayList<>(map.values()); String asin = normalizeAsin(row == null ? null : row.getAsin());
} String status = row == null ? "" : firstNonBlank(row.getStatus(), "").trim();
if (asin.isEmpty() && status.isEmpty()) {
private List<QueryAsinCartRatioDto> mergeCartRatios(List<QueryAsinCartRatioDto> base, List<QueryAsinCartRatioDto> incoming) { continue;
Map<String, QueryAsinCartRatioDto> map = new LinkedHashMap<>(); }
for (QueryAsinCartRatioDto ratio : copyCartRatios(base)) { boolean exists = merged.stream().anyMatch(old ->
map.put(ratio.getCountry(), ratio); Objects.equals(normalizeAsin(old.getAsin()), asin)
} && Objects.equals(firstNonBlank(old.getStatus(), "").trim(), status));
for (QueryAsinCartRatioDto ratio : copyCartRatios(incoming)) { if (!exists) {
if (!blank(ratio.getCountry())) { QueryAsinAsinStatusDto normalized = new QueryAsinAsinStatusDto();
map.put(ratio.getCountry(), ratio); normalized.setAsin(asin);
normalized.setStatus(status);
merged.add(normalized);
}
} }
existing.setItems(merged);
} }
return new ArrayList<>(map.values()); return new ArrayList<>(map.values());
} }
@@ -741,17 +727,19 @@ public class QueryAsinTaskService {
map.put(item.getCountry(), item); map.put(item.getCountry(), item);
} }
for (QueryAsinCountryAsinsDto item : copyCountryAsins(incoming)) { for (QueryAsinCountryAsinsDto item : copyCountryAsins(incoming)) {
if (blank(item.getCountry())) { String country = normalizeCountry(item.getCountry());
if (blank(country)) {
continue; continue;
} }
QueryAsinCountryAsinsDto existing = map.get(item.getCountry()); item.setCountry(country);
QueryAsinCountryAsinsDto existing = map.get(country);
if (existing == null) { if (existing == null) {
map.put(item.getCountry(), item); map.put(country, item);
continue; continue;
} }
List<String> merged = new ArrayList<>(existing.getAsins() == null ? List.of() : existing.getAsins()); List<String> merged = new ArrayList<>(existing.getAsins() == null ? List.of() : existing.getAsins());
for (String asin : item.getAsins() == null ? List.<String>of() : item.getAsins()) { for (String asin : item.getAsins() == null ? List.<String>of() : item.getAsins()) {
String normalized = asin == null ? "" : asin.trim().toUpperCase(); String normalized = normalizeAsin(asin);
if (!normalized.isEmpty() && !merged.contains(normalized)) { if (!normalized.isEmpty() && !merged.contains(normalized)) {
merged.add(normalized); merged.add(normalized);
} }
@@ -765,35 +753,18 @@ public class QueryAsinTaskService {
if (payload == null) { if (payload == null) {
return false; return false;
} }
if (payload.getQueryAsins() != null) { if (payload.getCountryResults() != null) {
for (QueryAsinCountryAsinsDto item : payload.getQueryAsins()) { for (QueryAsinCountryResultDto countryResult : payload.getCountryResults()) {
if (item != null && item.getAsins() != null && !item.getAsins().isEmpty()) { if (countryResult == null || countryResult.getItems() == null) {
return true;
}
}
}
if (payload.getCountrySections() != null) {
for (QueryAsinCountrySectionDto section : payload.getCountrySections()) {
if (section == null || section.getRows() == null) {
continue; continue;
} }
for (QueryAsinCountryMetricRowDto row : section.getRows()) { for (QueryAsinAsinStatusDto item : countryResult.getItems()) {
if (row != null && (!blank(row.getStatus()) if (item != null && (!blank(item.getAsin()) || !blank(item.getStatus()))) {
|| !blank(row.getQuantity())
|| !blank(row.getDeleteQuantity())
|| !blank(row.getProcessStatus()))) {
return true; return true;
} }
} }
} }
} }
if (payload.getCartRatios() != null) {
for (QueryAsinCartRatioDto ratio : payload.getCartRatios()) {
if (ratio != null && (!blank(ratio.getCountry()) || !blank(ratio.getRatio()))) {
return true;
}
}
}
return false; return false;
} }
@@ -872,35 +843,32 @@ public class QueryAsinTaskService {
} }
} }
private List<QueryAsinCountrySectionDto> copyCountrySections(List<QueryAsinCountrySectionDto> sections) { private List<QueryAsinCountryResultDto> copyCountryResults(List<QueryAsinCountryResultDto> results) {
List<QueryAsinCountrySectionDto> copy = new ArrayList<>(); List<QueryAsinCountryResultDto> copy = new ArrayList<>();
if (sections == null) { if (results == null) {
return copy; return copy;
} }
for (QueryAsinCountrySectionDto section : sections) { for (QueryAsinCountryResultDto source : results) {
if (section == null) { if (source == null || blank(source.getCountry())) {
continue; continue;
} }
QueryAsinCountrySectionDto item = new QueryAsinCountrySectionDto(); QueryAsinCountryResultDto item = new QueryAsinCountryResultDto();
item.setCountry(section.getCountry()); item.setCountry(normalizeCountry(source.getCountry()));
item.setRows(section.getRows() == null ? new ArrayList<>() : new ArrayList<>(section.getRows())); List<QueryAsinAsinStatusDto> rows = new ArrayList<>();
copy.add(item); if (source.getItems() != null) {
} for (QueryAsinAsinStatusDto sourceRow : source.getItems()) {
return copy; String asin = normalizeAsin(sourceRow == null ? null : sourceRow.getAsin());
} String status = sourceRow == null ? "" : firstNonBlank(sourceRow.getStatus(), "").trim();
if (asin.isEmpty() && status.isEmpty()) {
private List<QueryAsinCartRatioDto> copyCartRatios(List<QueryAsinCartRatioDto> ratios) { continue;
List<QueryAsinCartRatioDto> copy = new ArrayList<>(); }
if (ratios == null) { QueryAsinAsinStatusDto row = new QueryAsinAsinStatusDto();
return copy; row.setAsin(asin);
} row.setStatus(status);
for (QueryAsinCartRatioDto ratio : ratios) { rows.add(row);
if (ratio == null) { }
continue;
} }
QueryAsinCartRatioDto item = new QueryAsinCartRatioDto(); item.setItems(rows);
item.setCountry(ratio.getCountry());
item.setRatio(ratio.getRatio());
copy.add(item); copy.add(item);
} }
return copy; return copy;
@@ -916,11 +884,11 @@ public class QueryAsinTaskService {
continue; continue;
} }
QueryAsinCountryAsinsDto item = new QueryAsinCountryAsinsDto(); QueryAsinCountryAsinsDto item = new QueryAsinCountryAsinsDto();
item.setCountry(source.getCountry().trim().toUpperCase()); item.setCountry(normalizeCountry(source.getCountry()));
List<String> asins = new ArrayList<>(); List<String> asins = new ArrayList<>();
if (source.getAsins() != null) { if (source.getAsins() != null) {
for (String asin : source.getAsins()) { for (String asin : source.getAsins()) {
String normalized = asin == null ? "" : asin.trim().toUpperCase(); String normalized = normalizeAsin(asin);
if (!normalized.isEmpty() && !asins.contains(normalized)) { if (!normalized.isEmpty() && !asins.contains(normalized)) {
asins.add(normalized); asins.add(normalized);
} }
@@ -936,6 +904,25 @@ public class QueryAsinTaskService {
return !blank(first) ? first : second; return !blank(first) ? first : second;
} }
private String normalizeCountry(String country) {
if (country == null) {
return "";
}
String value = country.trim().toUpperCase();
return switch (value) {
case "德国" -> "DE";
case "英国" -> "UK";
case "法国" -> "FR";
case "意大利" -> "IT";
case "西班牙" -> "ES";
default -> value;
};
}
private String normalizeAsin(String asin) {
return asin == null ? "" : asin.trim().toUpperCase();
}
private String safeFileStem(String value) { private String safeFileStem(String value) {
String raw = value == null ? "result" : value.trim(); String raw = value == null ? "result" : value.trim();
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_"); String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");

View File

@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.IOException; import java.io.IOException;
@@ -14,6 +15,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.time.Duration; import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
@@ -26,11 +28,37 @@ public class ShopMatchTaskCacheService {
private static final String MODULE_TYPE = "SHOP_MATCH"; private static final String MODULE_TYPE = "SHOP_MATCH";
private static final long PAYLOAD_TTL_HOURS = 24; private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties; private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>(); private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void touchTaskHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) { public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class); return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
} }
@@ -40,6 +68,7 @@ public class ShopMatchTaskCacheService {
return; return;
} }
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload); taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
} }
public void removeShopMergedPayload(Long taskId, String shopKey) { public void removeShopMergedPayload(Long taskId, String shopKey) {
@@ -59,6 +88,7 @@ public class ShopMatchTaskCacheService {
return; return;
} }
taskEntityLocalCache.remove(taskId); taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
try { try {
Path taskDir = buildTaskDir(taskId); Path taskDir = buildTaskDir(taskId);
@@ -156,6 +186,10 @@ public class ShopMatchTaskCacheService {
return buildTaskDir(taskId).resolve("_task-entity.json"); return buildTaskDir(taskId).resolve("_task-entity.json");
} }
private String buildTaskHeartbeatKey(Long taskId) {
return "shop-match:task:heartbeat:" + taskId;
}
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) { private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
return cached != null return cached != null
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis()); && now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());

View File

@@ -364,6 +364,9 @@ public class ShopMatchTaskService {
throw new BusinessException("task 不存在"); throw new BusinessException("task 不存在");
} }
updateTaskAndRefreshCache(task); updateTaskAndRefreshCache(task);
if ("RUNNING".equals(task.getStatus())) {
shopMatchTaskCacheService.touchTaskHeartbeat(task.getId());
}
ProductRiskCreateTaskVo vo = new ProductRiskCreateTaskVo(); ProductRiskCreateTaskVo vo = new ProductRiskCreateTaskVo();
vo.setTaskId(task.getId()); vo.setTaskId(task.getId());
@@ -411,6 +414,7 @@ public class ShopMatchTaskService {
task.setUpdatedAt(now); task.setUpdatedAt(now);
persistTaskRequest(task, state); persistTaskRequest(task, state);
updateTaskAndRefreshCache(task); updateTaskAndRefreshCache(task);
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
} }
@Transactional @Transactional
@@ -467,6 +471,7 @@ public class ShopMatchTaskService {
throw new BusinessException(40901, "任务已结束,拒绝重复提交"); throw new BusinessException(40901, "任务已结束,拒绝重复提交");
} }
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
List<FileResultEntity> resultRows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> resultRows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)

View File

@@ -6,7 +6,7 @@ import os
import re import re
import requests import requests
from flask import Blueprint, request, jsonify, session from flask import Blueprint, request, jsonify, session, current_app
import pymysql import pymysql
from werkzeug.security import generate_password_hash from werkzeug.security import generate_password_hash
@@ -461,7 +461,10 @@ def get_admin_current_user_menus():
@login_required @login_required
def admin_logout(): def admin_logout():
session.clear() session.clear()
return jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login'}) response = jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login?logout=1'})
response.delete_cookie(current_app.config.get('SESSION_COOKIE_NAME', 'session'))
response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0'
return response
# ---------- 用户管理 ---------- # ---------- 用户管理 ----------

View File

@@ -1,7 +1,7 @@
""" """
认证蓝图:登录、登出、登录状态校验 认证蓝图:登录、登出、登录状态校验
""" """
from flask import Blueprint, request, redirect, url_for, session, jsonify from flask import Blueprint, request, redirect, url_for, session, jsonify, make_response, current_app
from werkzeug.security import check_password_hash from werkzeug.security import check_password_hash
from utils.db import get_db from utils.db import get_db
@@ -13,14 +13,23 @@ auth = Blueprint('auth', __name__, url_prefix='')
@auth.route('/login', methods=['GET', 'POST']) @auth.route('/login', methods=['GET', 'POST'])
def login(): def login():
if session.get('user_id') and is_session_user_valid(): force_relogin = request.args.get('logout') == '1' or request.args.get('switch') == '1'
if request.method == 'GET' and force_relogin:
session.clear()
response = make_response(render_html('login.html'))
response.delete_cookie(current_app.config.get('SESSION_COOKIE_NAME', 'session'))
response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0'
return response
if request.method == 'GET' and session.get('user_id') and is_session_user_valid():
return redirect(url_for('main.admin_page')) return redirect(url_for('main.admin_page'))
if request.method == 'POST': if request.method == 'POST':
session.clear()
wants_json = request.is_json or request.headers.get('X-Requested-With') == 'XMLHttpRequest'
data = request.get_json() if request.is_json else request.form data = request.get_json() if request.is_json else request.form
username = (data.get('username') or '').strip() username = (data.get('username') or '').strip()
password = data.get('password') or '' password = data.get('password') or ''
if not username or not password: if not username or not password:
if request.is_json: if wants_json:
return jsonify({'success': False, 'error': '请输入用户名和密码'}) return jsonify({'success': False, 'error': '请输入用户名和密码'})
return render_html('login.html', error='请输入用户名和密码') return render_html('login.html', error='请输入用户名和密码')
try: try:
@@ -36,14 +45,14 @@ def login():
session.permanent = True session.permanent = True
session['user_id'] = row['id'] session['user_id'] = row['id']
session['username'] = username session['username'] = username
if request.is_json: if wants_json:
return jsonify({'success': True, 'redirect': url_for('main.admin_page')}) return jsonify({'success': True, 'redirect': url_for('main.admin_page')})
return redirect(url_for('main.admin_page')) return redirect(url_for('main.admin_page'))
except Exception as exc: except Exception as exc:
if request.is_json: if wants_json:
return jsonify({'success': False, 'error': str(exc)}) return jsonify({'success': False, 'error': str(exc)})
return render_html('login.html', error='登录失败,请稍后重试') return render_html('login.html', error='登录失败,请稍后重试')
if request.is_json: if wants_json:
return jsonify({'success': False, 'error': '用户名或密码错误'}) return jsonify({'success': False, 'error': '用户名或密码错误'})
return render_html('login.html', error='用户名或密码错误') return render_html('login.html', error='用户名或密码错误')
return render_html('login.html') return render_html('login.html')
@@ -71,4 +80,7 @@ def api_auth_check():
@auth.route('/logout') @auth.route('/logout')
def logout(): def logout():
session.clear() session.clear()
return redirect(url_for('auth.login')) response = redirect(url_for('auth.login', logout='1'))
response.delete_cookie(current_app.config.get('SESSION_COOKIE_NAME', 'session'))
response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0'
return response

View File

@@ -23,8 +23,8 @@ bucket_path = "nanri-image/"
file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/" file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/"
import os import os
backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') # backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/')
# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/')
os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId
os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6" os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"

View File

@@ -4162,10 +4162,10 @@
alert(res.error || '退出失败'); alert(res.error || '退出失败');
return; return;
} }
window.location.href = res.redirect || '/login'; window.location.replace(res.redirect || '/login?logout=1');
}) })
.catch(function () { .catch(function () {
window.location.href = '/logout'; window.location.replace('/logout');
}); });
}; };

View File

@@ -118,6 +118,10 @@
</div> </div>
<script> <script>
(function() { (function() {
var params = new URLSearchParams(window.location.search || '');
if (params.get('logout') === '1' || params.get('switch') === '1') {
return;
}
fetch('/api/auth/check', { credentials: 'same-origin' }) fetch('/api/auth/check', { credentials: 'same-origin' })
.then(function(r) { return r.json(); }) .then(function(r) { return r.json(); })
.then(function(res) { .then(function(res) {

View File

@@ -624,6 +624,7 @@ function resetQueueWorkerIfIdle() {
} }
queueWorkerRunning.value = false; queueWorkerRunning.value = false;
pushing.value = false; pushing.value = false;
autoQueueEnabled.value = false;
stopHistoryPolling(); stopHistoryPolling();
saveQueueState(); saveQueueState();
} }
@@ -818,11 +819,6 @@ async function runMatch() {
const data = await matchQueryAsinShops(names); const data = await matchQueryAsinShops(names);
matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []); matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []);
saveMatchedItems(); saveMatchedItems();
if (autoQueueEnabled.value) {
pendingQueue.value = mergeQueueItems(pendingQueue.value, matchedRunnableItems.value);
saveQueueState();
void processQueue();
}
ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`); ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`);
} catch (error) { } catch (error) {
ElMessage.error(error instanceof Error ? error.message : "匹配失败"); ElMessage.error(error instanceof Error ? error.message : "匹配失败");
@@ -856,6 +852,7 @@ function buildTaskItem(item: QueryAsinShopQueueItem): QueryAsinTaskItem {
} }
function buildQueuePayload(taskId: number, item: QueryAsinHistoryItem) { function buildQueuePayload(taskId: number, item: QueryAsinHistoryItem) {
const submissionId = createSubmissionId(taskId, item.shopName);
return { return {
type: "query-asin-run", type: "query-asin-run",
ts: Date.now(), ts: Date.now(),
@@ -863,9 +860,11 @@ function buildQueuePayload(taskId: number, item: QueryAsinHistoryItem) {
taskId, taskId,
user_id: Number(uidForStorage()) || 0, user_id: Number(uidForStorage()) || 0,
source: "frontend-vue-query-asin", source: "frontend-vue-query-asin",
submissionId,
items: [ items: [
{ {
shopName: item.shopName, shopName: item.shopName,
submissionId,
shopId: item.shopId, shopId: item.shopId,
platform: item.platform, platform: item.platform,
companyName: item.companyName, companyName: item.companyName,
@@ -973,25 +972,20 @@ async function processQueue() {
{ {
shopName: createdItem.shopName || nextItem.shopName || "", shopName: createdItem.shopName || nextItem.shopName || "",
error: pushResult?.error || `任务 ${created.taskId} 推送失败`, error: pushResult?.error || `任务 ${created.taskId} 推送失败`,
queryAsins: copyQueryAsins(createdItem.queryAsins || nextItem.queryAsins),
shopDone: true, shopDone: true,
submissionId: createSubmissionId( submissionId: createSubmissionId(
created.taskId, created.taskId,
createdItem.shopName || nextItem.shopName, createdItem.shopName || nextItem.shopName,
), ),
chunkIndex: 1,
chunkTotal: 1,
}, },
], ],
}); });
removeMatchedRowLocally(nextItem);
await refreshTaskViews(); await refreshTaskViews();
queuePushResult.value = `任务 ${created.taskId} 推送失败,已自动继续下一个任务`; queuePushResult.value = `任务 ${created.taskId} 推送失败,已自动继续下一个任务`;
clearActiveQueueTask(); clearActiveQueueTask();
continue; continue;
} }
removeMatchedRowLocally(nextItem);
queuePushResult.value = queuePushResult.value =
pendingQueue.value.length > 0 pendingQueue.value.length > 0
? `任务 ${created.taskId} 已入队,等待完成后自动继续下一个(剩余 ${pendingQueue.value.length} 条)` ? `任务 ${created.taskId} 已入队,等待完成后自动继续下一个(剩余 ${pendingQueue.value.length} 条)`

View File

@@ -1199,6 +1199,16 @@ export interface QueryAsinCountryAsins {
asins: string[]; asins: string[];
} }
export interface QueryAsinStatusItem {
asin?: string;
status?: string;
}
export interface QueryAsinCountryResult {
country: string;
items: QueryAsinStatusItem[];
}
export interface QueryAsinTaskItem { export interface QueryAsinTaskItem {
shopName?: string; shopName?: string;
matched?: boolean; matched?: boolean;
@@ -1228,6 +1238,7 @@ export interface QueryAsinHistoryItem {
outputFilename?: string; outputFilename?: string;
downloadUrl?: string; downloadUrl?: string;
queryAsins: QueryAsinCountryAsins[]; queryAsins: QueryAsinCountryAsins[];
countryResults?: QueryAsinCountryResult[];
} }
export interface QueryAsinHistoryVo { export interface QueryAsinHistoryVo {
@@ -1316,11 +1327,9 @@ export function submitQueryAsinTaskResult(
shops: Array<{ shops: Array<{
shopName: string; shopName: string;
error?: string; error?: string;
queryAsins: QueryAsinCountryAsins[]; countryResults?: QueryAsinCountryResult[];
shopDone?: boolean; shopDone?: boolean;
submissionId?: string; submissionId?: string;
chunkIndex?: number;
chunkTotal?: number;
}>; }>;
}, },
) { ) {

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -8,7 +8,7 @@
<link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js">
<link rel="modulepreload" crossorigin href="/assets/brand-UU-ckLq6.js"> <link rel="modulepreload" crossorigin href="/assets/brand-UU-ckLq6.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css">
<link rel="stylesheet" crossorigin href="/assets/price-track-RLJUgMl_.css"> <link rel="stylesheet" crossorigin href="/assets/price-track-85NbckiJ.css">
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css"> <link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
</head> </head>
<body> <body>

View File

@@ -8,7 +8,7 @@
<link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js">
<link rel="modulepreload" crossorigin href="/assets/zh-cn-DyKmB71i.js"> <link rel="modulepreload" crossorigin href="/assets/zh-cn-DyKmB71i.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css">
<link rel="stylesheet" crossorigin href="/assets/query-asin-B5e5m-vs.css"> <link rel="stylesheet" crossorigin href="/assets/query-asin-B0yMJ9Or.css">
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css"> <link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
</head> </head>
<body> <body>