super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

去重导出列固定顺序为 id/ASIN/国家/价格/品牌；QueryAsin 按店铺加载数据并修正前端回退逻辑
Build Backend JAR / build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
黄自达
2026-08-25 22:38:38 +08:00
parent c287fe01a8
commit 7b238522ed
3 changed files with 59 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
+33 -3
View File
@@ -47,6 +47,7 @@ public class DedupeRunService {
private static final String HEADER_SPLIT_MARKER = "idASIN国家状态价格变体数量";
private static final String HEADER_STOP_COLUMN = "缩略图地址8";
private static final List<String> EXPORT_COLUMN_PRIORITY = List.of("id", "ASIN", "国家", "价格", "品牌");
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
@@ -254,6 +255,7 @@ public class DedupeRunService {
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
selectedColumns = reorderExportColumns(selectedColumns);
long readStartNs = System.nanoTime();
DedupeReadResult readResult = readDedupeRows(
inputFile,
@@ -310,13 +312,14 @@ public class DedupeRunService {
}
long writeNs = elapsedNs(writeStartNs);
log.info(
"dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}",
"dedupe clean stages file={} scannedRows={} keptRows={} uniqueAsins={} matchedAsins={} outputRows={} filteredFbaRows={} readFilterMs={} dbMs={} writeMs={} totalCleanMs={}",
inputFile.getName(),
readResult.scannedRows(),
readResult.rows().size(),
candidateAsinValues.size(),
matchedAsinValues.size(),
outputRows,
readResult.filteredFbaRows(),
toMs(readNs),
toMs(dbNs),
toMs(writeNs),
@@ -324,6 +327,24 @@ public class DedupeRunService {
);
}
private List<String> reorderExportColumns(List<String> selectedColumns) {
if (selectedColumns == null || selectedColumns.isEmpty()) {
return selectedColumns;
}
List<String> ordered = new ArrayList<>(selectedColumns.size() + EXPORT_COLUMN_PRIORITY.size());
for (String priority : EXPORT_COLUMN_PRIORITY) {
if (selectedColumns.contains(priority)) {
ordered.add(priority);
}
}
for (String column : selectedColumns) {
if (!ordered.contains(column)) {
ordered.add(column);
}
}
return ordered;
}
private DedupeReadResult readDedupeRows(File inputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
@@ -356,14 +377,23 @@ public class DedupeRunService {
}
Integer idColumnIndex = headerMap.get("id");
Integer asinColumnIndex = headerMap.get("ASIN");
Integer shippingTypeColumnIndex = headerMap.get("发货类型");
int scannedRows = 0;
int filteredFbaRows = 0;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
scannedRows++;
if (shippingTypeColumnIndex != null) {
String shippingType = normalizeCellText(formatter.formatCellValue(row.getCell(shippingTypeColumnIndex)));
if ("FBA".equalsIgnoreCase(shippingType)) {
filteredFbaRows++;
continue;
}
}
if (idColumnIndex == null) {
rows.add(buildCandidateRow(row, selectedIndexes, asinColumnIndex, formatter));
continue;
@@ -383,7 +413,7 @@ public class DedupeRunService {
}
pendingMainIdGroup.flush(rows);
return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows);
return new DedupeReadResult(sheet.getSheetName(), rows, scannedRows, filteredFbaRows);
}
}
@@ -641,7 +671,7 @@ public class DedupeRunService {
return nanos / 1_000_000L;
}
private record DedupeReadResult(String sheetName, List<DedupeCandidateRow> rows, int scannedRows) {
private record DedupeReadResult(String sheetName, List<DedupeCandidateRow> rows, int scannedRows, int filteredFbaRows) {
}
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinResolveService.java
+25 -5
View File
@@ -24,6 +24,7 @@ import org.springframework.transaction.support.TransactionTemplate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -136,16 +137,33 @@ public class QueryAsinResolveService {
if (ordered.isEmpty()) {
throw new BusinessException("shop_names 无有效店铺名");
}
List<QueryAsinEntity> allAsins = queryAsinMapper.selectList(
new LambdaQueryWrapper<QueryAsinEntity>().orderByAsc(QueryAsinEntity::getId));
List<QueryAsinCountryAsinsDto> preloadedQueryAsins = toCountryAsins(allAsins);
Map<String, List<QueryAsinEntity>> rowsByShop = loadAsinsGroupByShop(new ArrayList<>(ordered));
ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo();
for (String shopName : ordered) {
vo.getItems().add(matchOneShop(shopName, preloadedQueryAsins));
vo.getItems().add(matchOneShop(shopName,
toCountryAsins(rowsByShop.getOrDefault(shopName, List.of()))));
}
return vo;
}
private Map<String, List<QueryAsinEntity>> loadAsinsGroupByShop(List<String> shopNames) {
LambdaQueryWrapper<QueryAsinEntity> wrapper = shopNames.size() == 1
? new LambdaQueryWrapper<QueryAsinEntity>()
.eq(QueryAsinEntity::getShopName, shopNames.get(0))
.orderByAsc(QueryAsinEntity::getId)
: new LambdaQueryWrapper<QueryAsinEntity>()
.in(QueryAsinEntity::getShopName, shopNames)
.orderByAsc(QueryAsinEntity::getId);
Map<String, List<QueryAsinEntity>> rowsByShop = new HashMap<>();
for (QueryAsinEntity row : queryAsinMapper.selectList(wrapper)) {
if (row.getShopName() == null) {
continue;
}
rowsByShop.computeIfAbsent(row.getShopName(), k -> new ArrayList<>()).add(row);
}
return rowsByShop;
}
public List<QueryAsinCountryAsinsDto> loadQueryAsinsForShop(Long userId, String shopName) {
validateUserId(userId);
String normalizedShopName = ziniaoShopSwitchService.normalizeShopName(shopName);
@@ -153,7 +171,9 @@ public class QueryAsinResolveService {
return List.of();
}
List<QueryAsinEntity> rows = queryAsinMapper.selectList(
new LambdaQueryWrapper<QueryAsinEntity>().orderByAsc(QueryAsinEntity::getId));
new LambdaQueryWrapper<QueryAsinEntity>()
.eq(QueryAsinEntity::getShopName, normalizedShopName)
.orderByAsc(QueryAsinEntity::getId));
return toCountryAsins(rows);
}