super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新新需求-采集

Browse Source
This commit is contained in:
super
2026-06-09 21:07:50 +08:00
parent b0462df3c1
commit c3d43aa26a
20 changed files with 2084 additions and 44185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSubmitRowDto.java
View File

@@ -29,4 +29,13 @@ public class CollectDataSubmitRowDto {
@JsonAlias({"关键词", "key_word"})
@Schema(description = "关键词", example = "phone case")
private String keyword;
@JsonProperty("delivery_method")
@JsonAlias({"配送方式", "deliveryMethod", "delivery", "fulfillment"})
@Schema(description = "配送方式：FBA / FBM / AMZ；未识别时由后端归入“无配送方式”", example = "FBA")
private String deliveryMethod;
@JsonAlias({"页数", "page", "page_no", "pageNo", "page_index", "pageIndex"})
@Schema(description = "该行数据所属页码（来自 Python 抓取端的搜索结果页码），后端按关键词取最大值作为总页数", example = "3")
private Integer page;
}

2
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataResultRowVo.java
View File

@@ -9,4 +9,6 @@ public class CollectDataResultRowVo {
private String price;
private String sellerName;
private String keyword;
private String deliveryMethod;
private Integer page;
}

124
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java
View File

@@ -10,38 +10,34 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@Service
@Slf4j
public class CollectDataExcelAssemblyService {
private static final String[] HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词"};
private static final String SHEET_DETAIL_NAME = "采集数据结果";
private static final String[] SHEET_DETAIL_HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词", "配送方式"};
public void writeWorkbook(File outputXlsx, List<CollectDataResultRowVo> items) {
private static final String SHEET_SUMMARY_NAME = "结果文件";
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数"};
/**
* 生成采集结果工作簿。
*
* @param outputXlsx 目标文件
* @param items 经过 ASIN 去重 / 无效品牌过滤 / 品牌检测后保留下来的明细行，写入「采集数据结果」sheet
* @param rawItems Python 回传的全量原始行（不经后端任何过滤），写入「结果文件」sheet 的关键词聚合统计
*/
public void writeWorkbook(File outputXlsx, List<CollectDataResultRowVo> items, List<CollectDataResultRowVo> rawItems) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
Sheet sheet = workbook.createSheet("采集数据结果");
Row headerRow = sheet.createRow(0);
for (int i = 0; i < HEADER.length; i++) {
headerRow.createCell(i).setCellValue(HEADER[i]);
sheet.setColumnWidth(i, (i == 3 ? 24 : 18) * 256);
}
int rowIndex = 1;
if (items != null) {
for (CollectDataResultRowVo item : items) {
if (item == null) {
continue;
}
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(safe(item.getBrand()));
row.createCell(1).setCellValue(safe(item.getAsin()));
row.createCell(2).setCellValue(safe(item.getPrice()));
row.createCell(3).setCellValue(safe(item.getSellerName()));
row.createCell(4).setCellValue(safe(item.getKeyword()));
}
}
writeDetailSheet(workbook, items);
writeSummarySheet(workbook, rawItems);
workbook.write(outputStream);
} catch (Exception ex) {
log.warn("[collect-data] write workbook failed: {}", ex.getMessage());
@@ -55,7 +51,91 @@ public class CollectDataExcelAssemblyService {
}
}
private void writeDetailSheet(SXSSFWorkbook workbook, List<CollectDataResultRowVo> items) {
Sheet sheet = workbook.createSheet(SHEET_DETAIL_NAME);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < SHEET_DETAIL_HEADER.length; i++) {
headerRow.createCell(i).setCellValue(SHEET_DETAIL_HEADER[i]);
sheet.setColumnWidth(i, (i == 3 ? 24 : 18) * 256);
}
int rowIndex = 1;
if (items != null) {
for (CollectDataResultRowVo item : items) {
if (item == null) {
continue;
}
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(safe(item.getBrand()));
row.createCell(1).setCellValue(safe(item.getAsin()));
row.createCell(2).setCellValue(safe(item.getPrice()));
row.createCell(3).setCellValue(safe(item.getSellerName()));
row.createCell(4).setCellValue(safe(item.getKeyword()));
row.createCell(5).setCellValue(safe(item.getDeliveryMethod()));
}
}
}
/**
* 「结果文件」sheet 基于 Python 回传的全量原始数据按关键词聚合：
* - FBA / FBM / AMZ / 无配送方式 列为各分类的命中行数；
* - 页数列取该关键词所有原始行中页码的最大值，反映 Python 实际抓取到的总页数。
*/
private void writeSummarySheet(SXSSFWorkbook workbook, List<CollectDataResultRowVo> rawItems) {
Sheet sheet = workbook.createSheet(SHEET_SUMMARY_NAME);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < SHEET_SUMMARY_HEADER.length; i++) {
headerRow.createCell(i).setCellValue(SHEET_SUMMARY_HEADER[i]);
sheet.setColumnWidth(i, (i == 0 ? 28 : 12) * 256);
}
Map<String, KeywordSummary> grouped = new LinkedHashMap<>();
if (rawItems != null) {
for (CollectDataResultRowVo item : rawItems) {
if (item == null) {
continue;
}
String keyword = safe(item.getKeyword());
KeywordSummary summary = grouped.computeIfAbsent(keyword, k -> new KeywordSummary());
String delivery = item.getDeliveryMethod() == null ? "" : item.getDeliveryMethod().trim().toUpperCase(Locale.ROOT);
switch (delivery) {
case "FBA" -> summary.fba++;
case "FBM" -> summary.fbm++;
case "AMZ" -> summary.amz++;
default -> summary.none++;
}
Integer page = item.getPage();
if (page != null && page > summary.maxPage) {
summary.maxPage = page;
}
}
}
int rowIndex = 1;
for (Map.Entry<String, KeywordSummary> entry : grouped.entrySet()) {
KeywordSummary summary = entry.getValue();
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(entry.getKey());
row.createCell(1).setCellValue(summary.fba);
row.createCell(2).setCellValue(summary.fbm);
row.createCell(3).setCellValue(summary.amz);
row.createCell(4).setCellValue(summary.none);
if (summary.maxPage > 0) {
row.createCell(5).setCellValue(summary.maxPage);
} else {
row.createCell(5).setCellValue("");
}
}
}
private String safe(String value) {
return value == null ? "" : value;
}
private static class KeywordSummary {
int fba;
int fbm;
int amz;
int none;
int maxPage;
}
}

119
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
View File

@@ -431,7 +431,16 @@ public class CollectDataService {
stats.receivedRows += rows.size();
stats.currentChunkRows = rows.size();
List<CollectDataResultRowVo> candidates = filterByExistingAsin(rows, stats);
// 「结果文件」sheet 需要 Python 回传的全量数据（不经任何后端过滤），
// 因此 chunk payload 写入未筛除空 ASIN 的全量 rows；
// 仅当 ASIN 非空时才进入去重 / 无效品牌 / 品牌检测链路并落 biz_task_result_item。
List<CollectDataResultRowVo> rowsForFiltering = new ArrayList<>(rows.size());
for (CollectDataResultRowVo row : rows) {
if (row.getAsin() != null && !row.getAsin().isBlank()) {
rowsForFiltering.add(row);
}
}
List<CollectDataResultRowVo> candidates = filterByExistingAsin(rowsForFiltering, stats);
List<CollectDataResultRowVo> accepted = filterByBrandCheck(candidates, stats);
for (CollectDataResultRowVo row : accepted) {
upsertResultItem(task.getId(), result.getId(), scopeKey, row);
@@ -483,6 +492,11 @@ public class CollectDataService {
return result;
}
/**
* 把 Python 回传的原始行映射到 VO。**保留全量数据**（包含 ASIN 为空的行），
* 因为「结果文件」sheet 需要不经过任何过滤的全量统计；ASIN 为空的行
* 仅在 chunk payload 中保留，下游过滤链由调用方按 ASIN 非空再行筛选。
*/
private List<CollectDataResultRowVo> normalizeSubmitRows(List<CollectDataSubmitRowDto> rawRows) {
List<CollectDataResultRowVo> rows = new ArrayList<>();
if (rawRows == null) {
@@ -492,16 +506,14 @@ public class CollectDataService {
if (raw == null) {
continue;
}
String asin = normalizeAsin(raw.getAsin());
if (asin.isBlank()) {
continue;
}
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setBrand(normalize(raw.getBrand()));
row.setAsin(asin);
row.setAsin(normalizeAsin(raw.getAsin()));
row.setPrice(normalize(raw.getPrice()));
row.setSellerName(normalize(raw.getSellerName()));
row.setKeyword(normalize(raw.getKeyword()));
row.setDeliveryMethod(normalizeDeliveryMethod(raw.getDeliveryMethod()));
row.setPage(raw.getPage());
rows.add(row);
}
return rows;
@@ -525,25 +537,34 @@ public class CollectDataService {
.toList());
}
}
Set<String> invalidValues = new HashSet<>();
if (!asins.isEmpty()) {
List<String> brands = rows.stream()
.map(row -> normalizeBrand(row.getBrand()))
.filter(value -> !value.isBlank())
.distinct()
.toList();
Set<String> invalidBrands = new HashSet<>();
if (!brands.isEmpty()) {
List<InvalidAsinDataEntity> invalidRows = invalidAsinDataMapper.selectList(new LambdaQueryWrapper<InvalidAsinDataEntity>()
.in(InvalidAsinDataEntity::getDataValue, asins));
.select(InvalidAsinDataEntity::getBrand)
.in(InvalidAsinDataEntity::getBrand, brands));
if (invalidRows != null) {
for (InvalidAsinDataEntity row : invalidRows) {
invalidValues.add(normalizeAsin(row.getDataValue()));
String normalized = normalizeBrand(row.getBrand());
if (!normalized.isBlank()) {
invalidBrands.add(normalized);
}
}
}
}
List<CollectDataResultRowVo> out = new ArrayList<>();
for (CollectDataResultRowVo row : rows) {
String asin = row.getAsin();
if (dedupeValues.contains(asin)) {
if (dedupeValues.contains(row.getAsin())) {
stats.dedupeFilteredCount++;
continue;
}
if (invalidValues.contains(asin)) {
String brand = normalizeBrand(row.getBrand());
if (!brand.isBlank() && invalidBrands.contains(brand)) {
stats.invalidFilteredCount++;
continue;
}
@@ -624,9 +645,13 @@ public class CollectDataService {
if (row == null || row.getAsin() == null || row.getAsin().isBlank()) {
return;
}
String brand = normalizeBrand(row.getBrand());
if (brand.isBlank()) {
return;
}
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
entity.setDataValue(row.getAsin());
entity.setBrand(row.getBrand());
entity.setBrand(brand);
try {
invalidAsinDataMapper.insert(entity);
} catch (DuplicateKeyException ignored) {
@@ -788,11 +813,14 @@ public class CollectDataService {
throw new BusinessException("结果记录不存在");
}
List<CollectDataResultRowVo> rows = loadFinalRows(task.getId());
// Sheet「结果文件」按需求基于 Python 回传的全量数据聚合，不经后端 ASIN/品牌过滤丢弃，
// 因此从 biz_task_chunk 反序列化全部原始行。
List<CollectDataResultRowVo> rawRows = loadRawRows(task.getId());
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "collect-data-result", String.valueOf(task.getId())));
String filename = buildResultFilename(task, result);
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, rows);
excelAssemblyService.writeWorkbook(xlsx, rows, rawRows);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
result.setResultFilename(filename);
result.setResultFileUrl(objectKey);
@@ -841,6 +869,45 @@ public class CollectDataService {
return out;
}
/**
* 加载 Python 回传的全量原始行，不做 ASIN / 品牌过滤。
* 数据源是 biz_task_chunk 中按 chunk_index 顺序保存的原始 payload，
* 用于「结果文件」sheet 中按关键词聚合统计配送方式与页数。
*/
private List<CollectDataResultRowVo> loadRawRows(Long taskId) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getChunkIndex)
.orderByAsc(TaskChunkEntity::getId));
List<CollectDataResultRowVo> out = new ArrayList<>();
if (chunks == null || chunks.isEmpty()) {
return out;
}
TypeReference<List<CollectDataResultRowVo>> listType = new TypeReference<>() {
};
for (TaskChunkEntity chunk : chunks) {
try {
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read collect data raw chunk failed");
if (payloadJson == null || payloadJson.isBlank()) {
continue;
}
List<CollectDataResultRowVo> values = objectMapper.readValue(payloadJson, listType);
if (values != null) {
for (CollectDataResultRowVo value : values) {
if (value != null) {
out.add(value);
}
}
}
} catch (Exception ex) {
log.warn("[collect-data] load raw chunk failed taskId={} chunkId={} err={}",
taskId, chunk.getId(), ex.getMessage());
}
}
return out;
}
private int countFinalRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
@@ -986,6 +1053,28 @@ public class CollectDataService {
return normalize(value).toLowerCase(Locale.ROOT);
}
/**
* 将 Python 回传的配送方式标准化为 FBA / FBM / AMZ；
* 空值或未识别值统一返回空字符串，由前端 / Excel 端展示为“无配送方式”。
*/
private String normalizeDeliveryMethod(String value) {
String normalized = normalize(value);
if (normalized.isBlank()) {
return "";
}
String upper = normalized.toUpperCase(Locale.ROOT);
if (upper.contains("FBA")) {
return "FBA";
}
if (upper.contains("FBM")) {
return "FBM";
}
if (upper.contains("AMZ") || upper.contains("AMAZON")) {
return "AMZ";
}
return "";
}
private String buildResultFilename(FileTaskEntity task, FileResultEntity result) {
String base = result == null || result.getSourceFilename() == null || result.getSourceFilename().isBlank()
? null

10
backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataService.java
View File

@@ -13,6 +13,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.Locale;
@Service
@RequiredArgsConstructor
@@ -47,7 +48,7 @@ public class InvalidAsinDataService {
@Transactional
public InvalidAsinDataItemVo create(InvalidAsinDataCreateRequest request) {
String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空");
String brand = normalizeOptional(request.getBrand());
String brand = normalizeRequired(request.getBrand(), "品牌不能为空").toLowerCase(Locale.ROOT);
ensureUnique(dataValue, null);
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
entity.setDataValue(dataValue);
@@ -60,7 +61,7 @@ public class InvalidAsinDataService {
public InvalidAsinDataItemVo update(Long id, InvalidAsinDataUpdateRequest request) {
InvalidAsinDataEntity entity = getById(id);
String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空");
String brand = normalizeOptional(request.getBrand());
String brand = normalizeRequired(request.getBrand(), "品牌不能为空").toLowerCase(Locale.ROOT);
ensureUnique(dataValue, id);
entity.setDataValue(dataValue);
entity.setBrand(brand);
@@ -100,11 +101,6 @@ public class InvalidAsinDataService {
return normalized;
}
private String normalizeOptional(String value) {
String normalized = normalizeText(value);
return normalized.isEmpty() ? null : normalized;
}
private String normalizeText(String value) {
if (value == null) {
return "";

11
backend-java/src/main/resources/db/V58__invalid_asin_data_brand_normalize.sql Normal file
View File

@@ -0,0 +1,11 @@
-- 不符合 ASIN 库的比对维度从 ASIN 切换到 brand：
-- 1. brand 为空的历史记录在新规则下不再产生、也不再参与比对，直接删除
-- 2. 存量 brand 统一转小写，与 CollectDataService.normalizeBrand 对齐
-- （历史数据写入前已经过 trim + 折叠空白，仅缺 lowerCase）
DELETE FROM biz_invalid_asin_data
WHERE brand IS NULL
OR TRIM(brand) = '';
UPDATE biz_invalid_asin_data
SET brand = LOWER(TRIM(brand))
WHERE brand <> LOWER(TRIM(brand));