super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(admin): 店铺数据重复检查撞款抽屉样式优化与导入测试按钮
Build Backend JAR / build (push) Has been cancelled
Details

Browse Source 
- 撞款详情抽屉：上架时间默认降序（晚到早）、去掉列头（按时间升序）小括号、加宽至 760px、
  去掉全屏遮罩（底层仍可点击）、抽屉贴到「撞款详情」区块位置而非浏览器顶部
- 修复抽屉 fixed 定位失效：.tab-panel.active 入场动画带 transform 会成为包含块，
  打开时把 drawer mask 挂到 body 下恢复视口定位
- 新增导入（新增-测试）/ 导入（删除-测试）按钮与上传弹窗，对应后端端点
  /api/admin/shop-data-crawl/duplicate-check-import（ADD=按店名替换基线记录、DELETE=移除该店记录）
- 新增 ShopDataDuplicateCheckImportService 导入服务（复用 WorkbookParser/Aggregator，与扫描共用 Redis 锁）
- 新增 3 个单测：导入替换重聚合、删除移除、无基线报错
- _preview_mock.py 加入 .gitignore（本地预览工具不入库）
This commit is contained in:
黄自达
2026-09-04 12:25:31 +08:00
parent 8241cd704e
commit 05bda8915a
8 changed files with 766 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java
+38 -1
View File
@@ -6,6 +6,7 @@ import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckImportService;
import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckQueryService;
import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckScanService;
import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckScanService.DuplicateScanView;
@@ -18,9 +19,11 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
import java.nio.charset.StandardCharsets;
@@ -31,7 +34,7 @@ import java.util.Map;
import java.util.Set;
/**
* 店铺数据重复检查（撞款）：overview / items / detail / export 四端点，契约与 Flask 页面一致。
* 店铺数据重复检查（撞款）：overview / items / detail / export / import 五端点，契约与 Flask 页面一致。
* JWT 管理员或 Flask 内部代理（X-Internal-Token + operatorId）均可调用；统一菜单+数据权限校验。
*/
@Slf4j
@@ -51,6 +54,7 @@ public class AdminShopDataDuplicateCheckController {
private final PermissionMenuService permissionMenuService;
private final ShopDataDuplicateCheckScanService scanService;
private final ShopDataDuplicateCheckQueryService queryService;
private final ShopDataDuplicateCheckImportService importService;
@GetMapping("/duplicate-check-overview")
@Operation(summary = "撞款指标与店铺分布（force=1 触发同步重扫）")
@@ -156,6 +160,39 @@ public class AdminShopDataDuplicateCheckController {
}
}
@PostMapping("/duplicate-check-import")
@Operation(summary = "撞款导入测试：新增(mode=add)或删除(mode=delete)指定店铺记录")
public ApiResponse<Object> importExcel(
HttpServletRequest request,
@RequestParam(name = "mode", defaultValue = "add") String mode,
@RequestParam(name = "shop_name", required = false) String shopName,
@RequestParam(name = "country_codes", required = false) String countryCodes,
@RequestParam(name = "file") MultipartFile file) {
RequestOperator operator = requireDuplicateCheckAccess(request);
if (file == null || file.isEmpty()) {
throw new BusinessException(400, "请上传 Excel 文件");
}
String m = mode == null ? "add" : mode.trim().toLowerCase();
boolean deleteMode = "delete".equals(m) || "del".equals(m) || "remove".equals(m);
try {
ShopDataDuplicateCheckImportService.ImportSummary summary = importService.importExcel(
shopName, countryCodes, file.getBytes(), deleteMode);
log.info("[shop-duplicate-check-import] 端点完成 mode={} operator_id={} 解析行数={} 聚合ASIN数={}",
deleteMode ? "DELETE" : "ADD", operator.id(), summary.totalRows(), summary.asinCount());
return ApiResponse.success(Map.of(
"total_rows", summary.totalRows(),
"asin_count", summary.asinCount(),
"shop_count", summary.shopCount(),
"record_count", summary.recordCount(),
"message", (deleteMode ? "删除" : "新增") + "导入成功"));
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.error("[shop-duplicate-check-import] 导入端点异常 operator_id={}", operator.id(), ex);
throw new BusinessException(500, "导入失败：" + (ex.getMessage() == null ? "未知错误" : ex.getMessage()));
}
}
private record RequestOperator(Long id, boolean superAdmin) {
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckImportService.java
+170
View File
@@ -0,0 +1,170 @@
package com.nanri.aiimage.modules.shopduplicatecheck.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplicateScanEntity;
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateItem;
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateOccurrence;
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload;
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateShop;
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator;
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckWorkbookParser;
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow;
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* 撞款导入（新增-测试 / 删除-测试）：上传店铺结果 xlsx，对最新扫描 payload 做
* 店铺级新增/删除后重新聚合，落库成功后前端「重新分析」即读到包含导入数据的视图。
* 导入只写新增的扫描结果行，不触碰原始结果文件/扫描源，避免污染真实扫描。
*/
@Service
@Slf4j
public class ShopDataDuplicateCheckImportService {
/** 与扫描共用同一把锁，避免导入落库与定时扫描交叉写。 */
static final String IMPORT_LOCK = ShopDataDuplicateCheckScanService.SCAN_LOCK;
static final Duration IMPORT_LOCK_TTL = Duration.ofMinutes(10);
private final ShopDataDuplicateCheckScanService scanService;
private final ShopDataDuplicateScanMapper scanMapper;
private final DistributedJobLockService distributedJobLockService;
private final ObjectMapper objectMapper;
@Autowired
public ShopDataDuplicateCheckImportService(ShopDataDuplicateCheckScanService scanService,
ShopDataDuplicateScanMapper scanMapper,
DistributedJobLockService distributedJobLockService,
ObjectMapper objectMapper) {
this.scanService = scanService;
this.scanMapper = scanMapper;
this.distributedJobLockService = distributedJobLockService;
this.objectMapper = objectMapper;
}
/** 导入结果摘要：解析行数 / 聚合后 ASIN 数 / 店铺数 / 记录数。 */
public record ImportSummary(int totalRows, int asinCount, int shopCount, int recordCount) {
}
/**
* 执行一次导入：ADD=按 shop_name 加入/替换该店记录，DELETE=从聚合中移除该店全部记录。
* 无最新扫描结果时直接失败（导入无基线可重组）。
*/
public ImportSummary importExcel(String shopName, String countryCodes, byte[] bytes, boolean deleteMode) {
if (shopName == null || shopName.isBlank()) {
throw new BusinessException(400, "请填写店铺名称");
}
String name = shopName.trim();
List<String> codes = normalizeCountryCodes(countryCodes);
List<RawRow> rows;
try {
rows = DuplicateCheckWorkbookParser.parse(bytes);
} catch (Exception ex) {
log.warn("[shop-duplicate-check-import] 解析导入 Excel 失败 shop={} msg={}", name, ex.getMessage());
throw new BusinessException(400, "Excel 解析失败：" + (ex.getMessage() == null ? "格式错误" : ex.getMessage()));
}
if (rows.isEmpty()) {
throw new BusinessException(400, "Excel 中未读取到 ASIN 数据（需包含 ASIN 列）");
}
try (DistributedJobLockService.LockHandle lock = distributedJobLockService.tryLock(IMPORT_LOCK, IMPORT_LOCK_TTL)) {
if (lock == null) {
throw new BusinessException(409, "扫描/导入进行中，请稍后重试");
}
return applyToLatest(name, codes, rows, deleteMode);
}
}
private ImportSummary applyToLatest(String shopName, List<String> countryCodes,
List<RawRow> rows, boolean deleteMode) {
ShopDataDuplicateCheckScanService.DuplicateScanView view = scanService.loadLatest();
if (view == null || view.payload() == null) {
throw new BusinessException(400, "暂无扫描结果，请先点击「重新分析」");
}
DuplicateScanPayload base = view.payload();
// 以最新扫描的 payload 重建各店解析行（occurrences 是行级明细）
Map<String, List<RawRow>> rowsByShop = new LinkedHashMap<>();
Map<String, String> groupByShop = new LinkedHashMap<>();
Map<String, List<String>> codesByShop = new LinkedHashMap<>();
for (DuplicateShop shop : base.shops()) {
rowsByShop.put(shop.shopName(), new ArrayList<>());
groupByShop.put(shop.shopName(), shop.groupName());
codesByShop.put(shop.shopName(), shop.countryCodes());
}
for (DuplicateItem item : base.items()) {
for (DuplicateOccurrence occ : item.occurrences()) {
rowsByShop.computeIfAbsent(occ.shopName(), k -> new ArrayList<>())
.add(new RawRow(occ.asin(), occ.date(), occ.price(), occ.brand(), occ.country()));
}
}
List<ShopParsed> baseParsed = new ArrayList<>();
boolean touched = false;
for (Map.Entry<String, List<RawRow>> entry : rowsByShop.entrySet()) {
if (entry.getKey().equalsIgnoreCase(shopName)) {
touched = true;
if (deleteMode) {
continue;
}
// 新增-测试：该店以导入文件为准，替换基线旧记录
continue;
}
baseParsed.add(new ShopParsed(entry.getKey(), groupByShop.getOrDefault(entry.getKey(), ""),
codesByShop.getOrDefault(entry.getKey(), List.of()), entry.getValue()));
}
if (deleteMode && !touched) {
throw new BusinessException(400, "店铺「" + shopName + "」不在当前扫描基线中，无法删除");
}
// 导入行 ASIN 归一化大写，与扫描聚合口径一致
List<RawRow> normalized = new ArrayList<>(rows.size());
for (RawRow row : rows) {
normalized.add(new RawRow(
row.asin() == null ? "" : row.asin().trim().toUpperCase(Locale.ROOT),
row.date(), row.price(), row.brand(), row.country()));
}
if (!deleteMode) {
baseParsed.add(new ShopParsed(shopName, "", countryCodes, normalized));
}
DuplicateCheckAggregator.Aggregate aggregate =
DuplicateCheckAggregator.aggregate(baseParsed, "job");
ShopDataDuplicateScanEntity row = new ShopDataDuplicateScanEntity();
row.setStatus("SUCCESS");
row.setFinishedAt(LocalDateTime.now());
try {
row.setSummaryJson(objectMapper.writeValueAsString(aggregate.summary()));
row.setPayloadJson(objectMapper.writeValueAsString(aggregate.payload()));
} catch (Exception ex) {
throw new BusinessException(500, "导入结果序列化失败", ex);
}
scanMapper.insert(row);
log.info("[shop-duplicate-check-import] 导入完成 mode={} shop={} 解析行数={} 聚合ASIN数={} 落库行 id={}",
deleteMode ? "DELETE" : "ADD", shopName, normalized.size(),
aggregate.summary().asinTotal(), row.getId());
return new ImportSummary(normalized.size(), aggregate.summary().asinTotal(),
aggregate.summary().shopCount(), aggregate.summary().recordTotal());
}
private List<String> normalizeCountryCodes(String raw) {
List<String> codes = new ArrayList<>();
if (raw == null || raw.isBlank()) {
return codes;
}
for (String part : raw.split("[,，;；\\s]+")) {
String code = part.trim().toUpperCase(Locale.ROOT);
if (!code.isEmpty()) {
codes.add(code);
}
}
return codes;
}
}