super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-30: 为国家结果行建立稳定去重键，替换线性重复扫描

Browse Source
This commit is contained in:
黄自达
2026-08-29 19:52:08 +08:00
parent 6e3b43d591
commit 199345fd11
2 changed files with 690 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
+36 -4
View File
@@ -86,6 +86,8 @@ public class ShopDataCrawlTaskService {
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断，已保留已回传的部分数据";
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
/** 国家结果行去重键字段分隔符（控制字符，字段值 trim 后不可能包含）。 */
private static final String ROW_KEY_SEPARATOR = "";
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
@@ -1604,9 +1606,22 @@ public class ShopDataCrawlTaskService {
map.put(item.getCountry(), item);
continue;
}
List<ShopDataCrawlRowDto> merged = new ArrayList<>(existing.getItems() == null ? List.of() : existing.getItems());
List<ShopDataCrawlRowDto> merged = existing.getItems() == null ? new ArrayList<>() : new ArrayList<>(existing.getItems());
Set<String> seen = new HashSet<>();
for (ShopDataCrawlRowDto old : merged) {
String key = rowDedupKey(old);
if (key != null) {
seen.add(key);
}
}
for (ShopDataCrawlRowDto row : item.getItems() == null ? List.<ShopDataCrawlRowDto>of() : item.getItems()) {
if (row != null && merged.stream().noneMatch(old -> sameRow(old, row))) merged.add(copyRow(row));
if (row == null) {
continue;
}
String key = rowDedupKey(row);
if (key != null && seen.add(key)) {
merged.add(copyRow(row));
}
}
existing.setItems(merged);
}
@@ -2366,8 +2381,13 @@ public class ShopDataCrawlTaskService {
item.setCountry(normalizeCountry(source.getCountry()));
List<ShopDataCrawlRowDto> rows = new ArrayList<>();
if (source.getItems() != null) {
Set<String> seen = new HashSet<>();
for (ShopDataCrawlRowDto sourceRow : source.getItems()) {
if (sourceRow != null && !rowEmpty(sourceRow) && rows.stream().noneMatch(old -> sameRow(old, sourceRow))) {
if (sourceRow == null || rowEmpty(sourceRow)) {
continue;
}
String key = rowDedupKey(sourceRow);
if (key != null && seen.add(key)) {
rows.add(copyRow(sourceRow));
}
}
@@ -2455,13 +2475,25 @@ public class ShopDataCrawlTaskService {
&& Objects.equals(trim(left.getRecommendedOffer()), trim(right.getRecommendedOffer()));
}
/** 国家结果行稳定去重键：与 sameRow 的 10 字段 trim 比较语义等价，用于 O(1) 去重。 */
static String rowDedupKey(ShopDataCrawlRowDto row) {
if (row == null) {
return null;
}
return trim(row.getDate()) + ROW_KEY_SEPARATOR + trim(row.getAsin()) + ROW_KEY_SEPARATOR + trim(row.getBrand())
+ ROW_KEY_SEPARATOR + trim(row.getCommodityImage()) + ROW_KEY_SEPARATOR + trim(row.getInventorySales())
+ ROW_KEY_SEPARATOR + trim(row.getSalesRank()) + ROW_KEY_SEPARATOR + trim(row.getPageViews())
+ ROW_KEY_SEPARATOR + trim(row.getUnitsSold()) + ROW_KEY_SEPARATOR + trim(row.getPrice())
+ ROW_KEY_SEPARATOR + trim(row.getRecommendedOffer());
}
private boolean rowEmpty(ShopDataCrawlRowDto row) {
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
}
private String trim(String value) {
private static String trim(String value) {
return value == null ? "" : value.trim();
}