diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java b/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java index 97039a44..f94725e7 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java @@ -63,6 +63,12 @@ public class AdminApiGuardFilter extends OncePerRequestFilter { if ("OPTIONS".equalsIgnoreCase(request.getMethod())) { return true; } + // 可信内部令牌（X-Internal-Token 匹配服务端配置）直接放行：让"仅凭令牌即鉴权"的 + // 内部端点（如 /api/admin/shop-manages/credential）由 controller 自校验令牌兜底， + // 避免 guard 把它们误拦为 401。放行后仍可达 controller，不会裸露敏感端点。 + if (adminAuthSupport.isTrustedInternalToken(request)) { + return true; + } for (String prefix : exemptSet()) { if (uri.startsWith(prefix)) { return true; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java index e23292c2..60c69875 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java @@ -39,7 +39,7 @@ public class AdminUserController { @RequestParam(name = "search", required = false) String search, @RequestParam(name = "created_by_id", required = false) Long createdById, @RequestParam(name = "admin_id", required = false) Long adminId) { - AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request); + AdminUserEntity currentUser = adminAuthSupport.requireAdminOrInternal(request); String kw = (username == null || username.isBlank()) ? search : username; Long filterCreatedBy = createdById != null ? createdById : adminId; AdminUserListVo vo = adminUserService.listUsers(currentUser, page, pageSize, kw, filterCreatedBy); @@ -50,7 +50,7 @@ public class AdminUserController { @Operation(summary = "创建用户") public ApiResponse createUser(HttpServletRequest request, @RequestBody AdminUserCreateRequest body) { - AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request); + AdminUserEntity currentUser = adminAuthSupport.requireAdminOrInternal(request); Long userId = adminUserService.createUser(currentUser, body); return ApiResponse.success("用户创建成功", userId); } @@ -60,7 +60,7 @@ public class AdminUserController { public ApiResponse updateUser(HttpServletRequest request, @PathVariable Long uid, @RequestBody AdminUserUpdateRequest body) { - AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request); + AdminUserEntity currentUser = adminAuthSupport.requireAdminOrInternal(request); adminUserService.updateUser(currentUser, uid, body); return ApiResponse.success("更新成功", null); } @@ -69,7 +69,7 @@ public class AdminUserController { @Operation(summary = "删除用户") public ApiResponse deleteUser(HttpServletRequest request, @PathVariable Long uid) { - AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request); + AdminUserEntity currentUser = adminAuthSupport.requireAdminOrInternal(request); adminUserService.deleteUser(currentUser, uid); return ApiResponse.success("删除成功", null); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserItemVo.java index a01d39a9..fb268a91 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserItemVo.java @@ -16,4 +16,6 @@ public class AdminUserItemVo { private String creatorUsername; @JsonProperty("created_at") private String createdAt; + @JsonProperty("pinyin_abbr") + private String pinyinAbbr; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserListVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserListVo.java index 412c1ee6..07b9f8f1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserListVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/vo/AdminUserListVo.java @@ -14,5 +14,9 @@ public class AdminUserListVo { private Integer pageSize; @JsonProperty("current_user_role") private String currentUserRole; + @JsonProperty("current_user_id") + private Long currentUserId; + @JsonProperty("current_user_username") + private String currentUserUsername; private List admins; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java index c10d3795..91a1c509 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java @@ -9,6 +9,7 @@ import com.nanri.aiimage.modules.admin.model.vo.AdminBriefVo; import com.nanri.aiimage.modules.admin.model.vo.AdminUserItemVo; import com.nanri.aiimage.modules.admin.model.vo.AdminUserListVo; import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; +import com.nanri.aiimage.modules.admin.util.PinyinAbbrUtil; import com.nanri.aiimage.modules.auth.util.WerkzeugPasswordEncoder; import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest; @@ -46,10 +47,14 @@ public class AdminUserService { if (role == null) { throw new BusinessException(403, "需要管理员权限"); } + // 用户列表被多个后台页面用作用户选择器，权限按 Flask 的宽菜单 OR 预检（任一后台菜单可见即可拉取） + ensureAdminUsersMenuAccess(currentUser, List.of( + "admin_users", "admin_history", "admin_shop_manage", "admin_skip_price_asin", + "admin_query_asin", "admin_dedupe_total_data", "admin_invalid_asin_data")); int safePage = page == null || page < 1 ? 1 : page; int safeSize = pageSize == null ? 15 : pageSize; if (safeSize < 5) safeSize = 5; - if (safeSize > 50) safeSize = 50; + if (safeSize > 999) safeSize = 999; String kw = username == null ? "" : username.trim(); Long filterCreatedBy = "super_admin".equals(role) ? createdById : null; @@ -97,6 +102,8 @@ public class AdminUserService { vo.setPage(safePage); vo.setPageSize(safeSize); vo.setCurrentUserRole(role); + vo.setCurrentUserId(currentUser.getId()); + vo.setCurrentUserUsername(currentUser.getUsername() == null ? "" : currentUser.getUsername()); vo.setAdmins(admins); return vo; } @@ -107,6 +114,7 @@ public class AdminUserService { if (role == null) { throw new BusinessException(403, "需要管理员权限"); } + ensureAdminUsersMenuAccess(currentUser, List.of("admin_users")); String username = request.getUsername() == null ? "" : request.getUsername().trim(); String password = request.getPassword() == null ? "" : request.getPassword(); String wantRole = request.getRole() == null ? "normal" : request.getRole().trim(); @@ -165,6 +173,7 @@ public class AdminUserService { if (role == null) { throw new BusinessException(403, "需要管理员权限"); } + ensureAdminUsersMenuAccess(currentUser, List.of("admin_users")); String password = request.getPassword(); String wantRole = request.getRole() == null ? null : request.getRole().trim(); @@ -233,6 +242,7 @@ public class AdminUserService { if (role == null) { throw new BusinessException(403, "需要管理员权限"); } + ensureAdminUsersMenuAccess(currentUser, List.of("admin_users")); if (currentUser.getId().equals(uid)) { throw new BusinessException("不能删除当前登录账号"); } @@ -284,6 +294,13 @@ public class AdminUserService { vo.setCreatorUsername(creatorMap.getOrDefault(entity.getCreatedById(), "")); LocalDateTime createdAt = entity.getCreatedAt(); vo.setCreatedAt(createdAt == null ? "" : createdAt.format(CREATED_AT_FORMATTER)); + vo.setPinyinAbbr(PinyinAbbrUtil.abbr(entity.getUsername())); return vo; } + + private void ensureAdminUsersMenuAccess(AdminUserEntity operator, List columnKeys) { + if (!permissionMenuService.hasAnyAdminMenu(operator, columnKeys)) { + throw new BusinessException(403, "无权访问用户管理模块"); + } + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java index bc03744d..3c008dd0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java @@ -12,6 +12,11 @@ import org.springframework.http.HttpHeaders; import org.springframework.stereotype.Component; import com.nanri.aiimage.modules.auth.config.AuthProperties; +import org.springframework.beans.factory.annotation.Value; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; @Component @RequiredArgsConstructor @@ -21,6 +26,12 @@ public class AdminAuthSupport { private final AdminUserMapper adminUserMapper; private final AuthProperties authProperties; + @Value("${aiimage.security.internal-token:}") + private String internalToken; + + @Value("${aiimage.security.internal-token-file:}") + private String internalTokenFile; + /** 解析当前请求的用户；token 缺失或无效抛 401。 */ public AdminUserEntity requireUser(HttpServletRequest request) { String token = resolveToken(request); @@ -73,6 +84,118 @@ public class AdminAuthSupport { return null; } + /** JWT 优先；无 JWT 时以可信内部代理身份（X-Internal-Token + operatorId）回退，仍要求管理员角色。 */ + public AdminUserEntity requireAdminOrInternal(HttpServletRequest request) { + try { + return requireAdmin(request); + } catch (BusinessException authFailure) { + AdminUserEntity internalOperator = resolveInternalOperator(request, true); + if (internalOperator != null) { + return internalOperator; + } + throw authFailure; + } + } + + /** JWT 优先；无 JWT 时以可信内部代理身份回退（仅要求用户存在）。 */ + public AdminUserEntity requireUserOrInternal(HttpServletRequest request) { + try { + return requireUser(request); + } catch (BusinessException authFailure) { + AdminUserEntity internalOperator = resolveInternalOperator(request, false); + if (internalOperator != null) { + return internalOperator; + } + throw authFailure; + } + } + + /** + * Flask 传统后台用 session 登录、无法签发 Java JWT，只能凭共享密钥 + * （显式配置或每个用户的本地 token 文件）标识已认证管理员。这里仍会 + * 回库校验管理员角色，superAdmin 查询标志不参与判定。 + */ + private AdminUserEntity resolveInternalOperator(HttpServletRequest request, boolean requireAdmin) { + if (!isTrustedInternalRequest(request)) { + return null; + } + String rawOperatorId = request.getParameter("operatorId"); + if (rawOperatorId == null || rawOperatorId.isBlank()) { + rawOperatorId = request.getParameter("operator_id"); + } + if (rawOperatorId == null || rawOperatorId.isBlank()) { + return null; + } + try { + Long operatorId = Long.parseLong(rawOperatorId.trim()); + AdminUserEntity operator = adminUserMapper.selectById(operatorId); + if (operator == null) { + return null; + } + if (requireAdmin && currentRole(operator) == null) { + return null; + } + return operator; + } catch (NumberFormatException ex) { + return null; + } + } + + private boolean isTrustedInternalRequest(HttpServletRequest request) { + String suppliedToken = request.getHeader("X-Internal-Token"); + String expectedToken = resolveExpectedInternalToken(); + if (expectedToken.isBlank() || suppliedToken == null || suppliedToken.isBlank()) { + return false; + } + return expectedToken.equals(suppliedToken.trim()); + } + + /** + * 仅判断请求是否携带可信内部令牌（不要求 operatorId）。供 AdminApiGuardFilter + * 在兜底鉴权前放行内部自动化调用（仅凭 X-Internal-Token 即鉴权的端点，如 + * /api/admin/shop-manages/credential 由 controller 自校验令牌）， + * 避免 guard 把合法的内部调用误拦为 401。 + */ + public boolean isTrustedInternalToken(HttpServletRequest request) { + return isTrustedInternalRequest(request); + } + + private String resolveExpectedInternalToken() { + if (internalToken != null && !internalToken.isBlank()) { + return internalToken.trim(); + } + Path path = resolveInternalTokenFile(); + if (path == null || !Files.isRegularFile(path)) { + return ""; + } + try { + return Files.readString(path, StandardCharsets.UTF_8).trim(); + } catch (Exception ignored) { + return ""; + } + } + + private Path resolveInternalTokenFile() { + String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim(); + if (!configuredPath.isEmpty()) { + if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) { + String userHome = System.getProperty("user.home", "").trim(); + if (userHome.isEmpty()) { + return null; + } + configuredPath = configuredPath.length() == 1 + ? userHome + : Path.of(userHome, configuredPath.substring(2)).toString(); + } + Path configuredTokenPath = Path.of(configuredPath); + return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null; + } + String userHome = System.getProperty("user.home", "").trim(); + return userHome.isEmpty() + ? null + : Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize(); + } + private String resolveToken(HttpServletRequest request) { String authHeader = request.getHeader(HttpHeaders.AUTHORIZATION); if (authHeader != null && authHeader.startsWith("Bearer ")) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/util/PinyinAbbrUtil.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/util/PinyinAbbrUtil.java new file mode 100644 index 00000000..de042264 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/util/PinyinAbbrUtil.java @@ -0,0 +1,72 @@ +package com.nanri.aiimage.modules.admin.util; + +/** + * 用户名拼音首字母缩写（分组管理/通讯录用），无第三方依赖。 + * 与 Python backend/blueprints/admin_api.py 的 _username_pinyin_abbr 语义保持一致： + * GB2312 区位码按拼音音序排列，用码位区间判定首字母；仅覆盖 GB2312 汉字。 + */ +public final class PinyinAbbrUtil { + + private static final char[] PYINYIN_LETTERS = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'W', 'X', 'Y', 'Z' + }; + private static final int[] PYINYIN_BOUNDS = { + 0xB0A1, 0xB0C5, 0xB2C1, 0xB4EE, 0xB6EA, 0xB7A2, 0xB8C1, 0xB9FE, + 0xBBF7, 0xBFA6, 0xC0AC, 0xC2E8, 0xC4C3, 0xC5B6, 0xC5BE, 0xC6DA, + 0xC8BB, 0xC8F6, 0xCBFA, 0xCDDA, 0xCEF4, 0xD1B9, 0xD4D1 + }; + private static final int PYINYIN_BOUNDS_END = 0xF7FF; + + private PinyinAbbrUtil() { + } + + /** 返回用户名各字符的拼音首字母串（'张伟恒' -> 'ZWH'），空输入或非 GB2312 汉字返回 '#'。 */ + public static String abbr(String username) { + if (username == null) { + return "#"; + } + String name = username.trim(); + if (name.isEmpty()) { + return "#"; + } + StringBuilder sb = new StringBuilder(name.length()); + for (int i = 0; i < name.length(); i++) { + sb.append(initial(name.charAt(i))); + } + return sb.toString(); + } + + private static char initial(char c) { + if (c >= '0' && c <= '9') { + return c; + } + if (c >= 'A' && c <= 'Z') { + return c; + } + if (c >= 'a' && c <= 'z') { + return (char) (c - 32); + } + if (c < 0x4E00 || c > 0x9FA5) { + return '#'; + } + try { + byte[] gb = String.valueOf(c).getBytes("GB2312"); + if (gb.length != 2) { + return '#'; + } + int low = ((gb[0] & 0xFF) << 8) | (gb[1] & 0xFF); + if (low < 0xB0A1 || low >= PYINYIN_BOUNDS_END) { + return '#'; + } + for (int i = PYINYIN_LETTERS.length - 1; i >= 0; i--) { + if (low >= PYINYIN_BOUNDS[i]) { + return PYINYIN_LETTERS[i]; + } + } + return '#'; + } catch (Exception e) { + return '#'; + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/imagehistory/controller/ImageHistoryController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/imagehistory/controller/ImageHistoryController.java index 027d1dd9..84eb878a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/imagehistory/controller/ImageHistoryController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/imagehistory/controller/ImageHistoryController.java @@ -1,9 +1,12 @@ package com.nanri.aiimage.modules.imagehistory.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; import com.nanri.aiimage.modules.imagehistory.model.vo.ImageHistoryListVo; import com.nanri.aiimage.modules.imagehistory.service.ImageHistoryService; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletRequest; @@ -13,6 +16,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + @RestController @RequiredArgsConstructor @RequestMapping("/api/admin") @@ -21,6 +26,7 @@ public class ImageHistoryController { private final AdminAuthSupport adminAuthSupport; private final ImageHistoryService imageHistoryService; + private final PermissionMenuService permissionMenuService; @GetMapping("/history") @Operation(summary = "分页查询所有用户生成历史") @@ -30,7 +36,10 @@ public class ImageHistoryController { @RequestParam(name = "user_id", required = false) Long userId, @RequestParam(name = "time_start", required = false) String timeStart, @RequestParam(name = "time_end", required = false) String timeEnd) { - adminAuthSupport.requireAdmin(request); + AdminUserEntity operator = adminAuthSupport.requireAdminOrInternal(request); + if (!permissionMenuService.hasAnyAdminMenu(operator, List.of("admin_history"))) { + throw new BusinessException(403, "无权访问生成记录模块"); + } ImageHistoryListVo vo = imageHistoryService.listHistory(page, pageSize, userId, timeStart, timeEnd); return ApiResponse.success(vo); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java new file mode 100644 index 00000000..82f0a7d7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java @@ -0,0 +1,265 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; +import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckQueryService; +import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckScanService; +import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckScanService.DuplicateScanView; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.Map; +import java.util.Set; + +/** + * 店铺数据重复检查（撞款）：overview / items / detail / export 四端点，契约与 Flask 页面一致。 + * JWT 管理员或 Flask 内部代理（X-Internal-Token + operatorId）均可调用；统一菜单+数据权限校验。 + */ +@Slf4j +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/admin/shop-data-crawl") +@Tag(name = "店铺数据重复检查") +public class AdminShopDataDuplicateCheckController { + + @Value("${aiimage.security.internal-token:}") + private String internalToken; + + @Value("${aiimage.security.internal-token-file:}") + private String internalTokenFile; + + private final AdminAuthSupport adminAuthSupport; + private final PermissionMenuService permissionMenuService; + private final ShopDataDuplicateCheckScanService scanService; + private final ShopDataDuplicateCheckQueryService queryService; + + @GetMapping("/duplicate-check-overview") + @Operation(summary = "撞款指标与店铺分布（force=1 触发同步重扫）") + public ApiResponse overview( + HttpServletRequest request, + @RequestParam(name = "force", required = false) String force) { + RequestOperator operator = requireDuplicateCheckAccess(request); + boolean forced = force != null && ("1".equals(force.trim().toLowerCase()) + || "true".equals(force.trim().toLowerCase()) || "yes".equals(force.trim().toLowerCase())); + if (forced) { + log.info("[shop-duplicate-check] force 手动重扫 operator_id={}", operator.id()); + scanService.scanNow(); + } + DuplicateScanView view = scanService.loadLatest(); + if (view == null) { + return ApiResponse.success(queryService.pendingOverview()); + } + Set visibleKeys = queryService.resolveVisibleShopKeys(operator.id(), operator.superAdmin()); + return ApiResponse.success(queryService.overviewData(view, visibleKeys)); + } + + @GetMapping("/duplicate-check-items") + @Operation(summary = "撞款矩阵/明细（分页）") + public ApiResponse items( + HttpServletRequest request, + @RequestParam(name = "page", required = false) String page, + @RequestParam(name = "page_size", required = false) String pageSize, + @RequestParam(name = "view", required = false) String view, + @RequestParam(name = "asin", required = false) String asin, + @RequestParam(name = "shop_name", required = false) String shopName, + @RequestParam(name = "country", required = false) String country, + @RequestParam(name = "site", required = false) String site, + @RequestParam(name = "date_from", required = false) String dateFrom, + @RequestParam(name = "date_to", required = false) String dateTo) { + RequestOperator operator = requireDuplicateCheckAccess(request); + int safePage = parsePage(page); + int safeSize = clamp(parseSize(pageSize, 20), 10, 100); + String viewMode = view == null ? "monitor" : view.trim().toLowerCase(); + ShopDataDuplicateCheckQueryService.Filters filters = + ShopDataDuplicateCheckQueryService.Filters.clean(asin, shopName, country, site, dateFrom, dateTo); + DuplicateScanView scan = scanService.loadLatest(); + if (scan == null) { + return ApiResponse.success(queryService.pendingList(safePage, safeSize)); + } + Set visibleKeys = queryService.resolveVisibleShopKeys(operator.id(), operator.superAdmin()); + return ApiResponse.success(queryService.itemsData(scan, visibleKeys, safePage, safeSize, viewMode, filters)); + } + + @GetMapping("/duplicate-check-detail") + @Operation(summary = "撞款详情卡片（分页，恒跨店重复）") + public ApiResponse detail( + HttpServletRequest request, + @RequestParam(name = "page", required = false) String page, + @RequestParam(name = "page_size", required = false) String pageSize, + @RequestParam(name = "asin", required = false) String asin, + @RequestParam(name = "shop_name", required = false) String shopName, + @RequestParam(name = "country", required = false) String country, + @RequestParam(name = "site", required = false) String site, + @RequestParam(name = "date_from", required = false) String dateFrom, + @RequestParam(name = "date_to", required = false) String dateTo) { + RequestOperator operator = requireDuplicateCheckAccess(request); + int safePage = parsePage(page); + int safeSize = clamp(parseSize(pageSize, 6), 1, 24); + ShopDataDuplicateCheckQueryService.Filters filters = + ShopDataDuplicateCheckQueryService.Filters.clean(asin, shopName, country, site, dateFrom, dateTo); + DuplicateScanView scan = scanService.loadLatest(); + if (scan == null) { + return ApiResponse.success(queryService.pendingDetail(safePage, safeSize)); + } + Set visibleKeys = queryService.resolveVisibleShopKeys(operator.id(), operator.superAdmin()); + return ApiResponse.success(queryService.detailData(scan, visibleKeys, safePage, safeSize, filters)); + } + + @GetMapping("/duplicate-check-export") + @Operation(summary = "撞款导出 CSV（UTF-8 BOM，逐行=一条上架记录）") + public void export( + HttpServletRequest request, + HttpServletResponse response, + @RequestParam(name = "view", required = false) String view, + @RequestParam(name = "asin", required = false) String asin, + @RequestParam(name = "shop_name", required = false) String shopName, + @RequestParam(name = "country", required = false) String country, + @RequestParam(name = "site", required = false) String site, + @RequestParam(name = "date_from", required = false) String dateFrom, + @RequestParam(name = "date_to", required = false) String dateTo) { + RequestOperator operator = requireDuplicateCheckAccess(request); + DuplicateScanView scan = scanService.loadLatest(); + if (scan == null) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "暂无扫描结果，请先点击「重新分析」"); + } + String viewMode = view == null ? "monitor" : view.trim().toLowerCase(); + ShopDataDuplicateCheckQueryService.Filters filters = + ShopDataDuplicateCheckQueryService.Filters.clean(asin, shopName, country, site, dateFrom, dateTo); + Set visibleKeys = queryService.resolveVisibleShopKeys(operator.id(), operator.superAdmin()); + try { + response.setContentType("text/csv; charset=utf-8"); + DownloadHeaderUtil.setAttachment(response, "shop-data-duplicate-check.csv"); + response.setHeader("Cache-Control", "no-store"); + queryService.writeExport(scan, visibleKeys, viewMode, filters, response.getOutputStream()); + } catch (Exception ex) { + log.warn("[shop-duplicate-check] 导出 CSV 失败: {}", ex.getMessage()); + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "导出失败"); + } + } + + private record RequestOperator(Long id, boolean superAdmin) { + } + + private RequestOperator requireDuplicateCheckAccess(HttpServletRequest request) { + AdminUserEntity operator; + try { + operator = adminAuthSupport.requireAdmin(request); + } catch (BusinessException authFailure) { + operator = resolveInternalOperator(request); + if (operator == null) { + throw authFailure; + } + } + permissionMenuService.requireShopDataCrawlTaskAccess(operator); + boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(operator)); + return new RequestOperator(operator.getId(), superAdmin); + } + + private AdminUserEntity resolveInternalOperator(HttpServletRequest request) { + String suppliedToken = request.getHeader("X-Internal-Token"); + if (!isTrustedInternalRequest(suppliedToken)) { + return null; + } + String rawOperatorId = request.getParameter("operatorId"); + if (rawOperatorId == null || rawOperatorId.isBlank()) { + rawOperatorId = request.getParameter("operator_id"); + } + if (rawOperatorId == null || rawOperatorId.isBlank()) { + return null; + } + try { + return permissionMenuService.requireAdminOperator(Long.parseLong(rawOperatorId.trim())); + } catch (NumberFormatException ex) { + return null; + } + } + + private boolean isTrustedInternalRequest(String suppliedToken) { + String expectedToken = resolveExpectedInternalToken(); + return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank() + && MessageDigest.isEqual( + expectedToken.getBytes(StandardCharsets.UTF_8), + suppliedToken.trim().getBytes(StandardCharsets.UTF_8)); + } + + private String resolveExpectedInternalToken() { + if (internalToken != null && !internalToken.isBlank()) { + return internalToken.trim(); + } + Path path = resolveInternalTokenFile(); + if (path == null || !Files.isRegularFile(path)) { + return ""; + } + try { + return Files.readString(path, StandardCharsets.UTF_8).trim(); + } catch (Exception ignored) { + return ""; + } + } + + private Path resolveInternalTokenFile() { + String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim(); + if (!configuredPath.isEmpty()) { + if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) { + String userHome = System.getProperty("user.home", "").trim(); + if (userHome.isEmpty()) { + return null; + } + configuredPath = configuredPath.length() == 1 + ? userHome + : Path.of(userHome, configuredPath.substring(2)).toString(); + } + Path configuredTokenPath = Path.of(configuredPath); + return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null; + } + String userHome = System.getProperty("user.home", "").trim(); + return userHome.isEmpty() + ? null + : Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize(); + } + + private static int parsePage(String page) { + if (page == null || page.isBlank()) { + return 1; + } + try { + return Math.max(1, Integer.parseInt(page.trim())); + } catch (NumberFormatException ex) { + throw new BusinessException(400, "page 参数不合法"); + } + } + + private static int parseSize(String raw, int defaultValue) { + if (raw == null || raw.isBlank()) { + return defaultValue; + } + try { + return Math.max(1, Integer.parseInt(raw.trim())); + } catch (NumberFormatException ex) { + throw new BusinessException(400, "page_size 参数不合法"); + } + } + + private static int clamp(int value, int min, int max) { + return Math.max(min, Math.min(max, value)); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDataDuplicateScanMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDataDuplicateScanMapper.java new file mode 100644 index 00000000..5757bbff --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDataDuplicateScanMapper.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanFullRowDto; +import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplicateScanEntity; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Select; + +@Mapper +public interface ShopDataDuplicateScanMapper extends BaseMapper { + + /** 轻量判活：仅取最新 SUCCESS 行 id/created_at，避免每次请求读大 payload。 */ + @Select("SELECT id, created_at AS createdAt FROM shop_data_duplicate_scan " + + "WHERE status = 'SUCCESS' ORDER BY id DESC LIMIT 1") + ScanLightRowDto selectLatestLightRow(); + + /** 全量读取最新 SUCCESS 行（含 summary/payload 反序列化）。 */ + @Select("SELECT id, summary_json AS summaryJson, payload_json AS payloadJson, " + + "created_at AS createdAt FROM shop_data_duplicate_scan " + + "WHERE status = 'SUCCESS' ORDER BY id DESC LIMIT 1") + ScanFullRowDto selectLatestFullRow(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDuplicateCheckSourceMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDuplicateCheckSourceMapper.java new file mode 100644 index 00000000..5153779c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDuplicateCheckSourceMapper.java @@ -0,0 +1,84 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.mapper; + +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ShopGroupLabelDto; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ShopSourceRowDto; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +/** + * 撞款扫描的数据源查询：等价 Python 后台 admin_api.py 中 + * _shops_with_latest_results（窗口函数取每店最新结果行）、 + * _shop_data_crawl_group_names（店铺分组名）与 _shop_data_managed_shop_names（角色可管店铺）。 + */ +@Mapper +public interface ShopDuplicateCheckSourceMapper { + + /** + * 每家店铺最新一条 biz_file_result 行（shop 最新累计结果文件，含该店全部国家/记录）。 + * 结果文件 URL 非空即参与，不要求 task/result 成功（与 Python 语义一致）。 + */ + @Select(""" + SELECT ranked.result_id AS resultId, + ranked.shop_name AS shopName, + ranked.result_file_url AS resultFileUrl, + ranked.result_file_size AS resultFileSize, + ranked.request_json AS requestJson, + ranked.country_codes_json AS countryCodesJson + FROM ( + SELECT r.id AS result_id, + r.source_filename AS shop_name, + r.result_file_url, + r.result_file_size, + t.request_json, + df.country_codes_json, + COALESCE(df.last_success_at, df.updated_at, t.finished_at, + t.updated_at, t.created_at) AS latest_file_updated_at, + ROW_NUMBER() OVER ( + PARTITION BY TRIM(COALESCE(r.source_filename, '')) + ORDER BY COALESCE(df.last_success_at, df.updated_at, + t.finished_at, t.updated_at, t.created_at) DESC, + r.id DESC) AS row_no + FROM biz_file_result r + JOIN biz_file_task t ON t.id = r.task_id + LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id + WHERE r.module_type = 'SHOP_DATA_CRAWL' + AND t.module_type = 'SHOP_DATA_CRAWL' + AND TRIM(COALESCE(r.result_file_url, '')) <> '' + ) ranked + WHERE ranked.row_no = 1 + ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC + """) + List selectLatestResultRows(); + + /** 店铺 → 分组名（g.group_name 优先，多分组用「、」连接；店铺名 trim 精确匹配）。 */ + @Select(""" + + """) + List selectGroupLabels(@Param("shopNames") List shopNames); + + /** + * 主管（admin 角色）可管店铺：自己创建或指派给自己的分组（created_by_id=me OR user_id=me） + * 下的店铺名，trim 后去重。无分组时返回空集合（什么都看不到），与 Python 语义一致。 + */ + @Select(""" + SELECT DISTINCT TRIM(sm.shop_name) AS shopName + FROM biz_shop_manage sm + WHERE sm.group_id IN ( + SELECT id FROM biz_shop_manage_group + WHERE created_by_id = #{operatorId} OR user_id = #{operatorId} + ) + """) + List selectManagedShopNames(@Param("operatorId") Long operatorId); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ScanFullRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ScanFullRowDto.java new file mode 100644 index 00000000..7d4c2875 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ScanFullRowDto.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.dto; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** 最新 SUCCESS 扫描行的完整字段。 */ +@Data +public class ScanFullRowDto { + + private Long id; + private String summaryJson; + private String payloadJson; + private LocalDateTime createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ScanLightRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ScanLightRowDto.java new file mode 100644 index 00000000..4d07c3ac --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ScanLightRowDto.java @@ -0,0 +1,13 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.dto; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** 最新 SUCCESS 扫描行的轻量字段（内存缓存判活用）。 */ +@Data +public class ScanLightRowDto { + + private Long id; + private LocalDateTime createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ShopGroupLabelDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ShopGroupLabelDto.java new file mode 100644 index 00000000..a5d2d5a9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ShopGroupLabelDto.java @@ -0,0 +1,11 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.dto; + +import lombok.Data; + +/** 店铺→分组名（多分组以「、」连接），g.group_name 优先于 sm.group_name。 */ +@Data +public class ShopGroupLabelDto { + + private String shopName; + private String groupName; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ShopSourceRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ShopSourceRowDto.java new file mode 100644 index 00000000..9756681e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/dto/ShopSourceRowDto.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.dto; + +import lombok.Data; + +/** 每店最新一条 biz_file_result 行（窗口查询产出，供扫描解析用）。 */ +@Data +public class ShopSourceRowDto { + + private Long resultId; + private String shopName; + private String resultFileUrl; + private Long resultFileSize; + private String requestJson; + private String countryCodesJson; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/entity/ShopDataDuplicateScanEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/entity/ShopDataDuplicateScanEntity.java new file mode 100644 index 00000000..70f6c895 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/entity/ShopDataDuplicateScanEntity.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** 撞款扫描结果行（Python 端建表，Java 读写同表）。 */ +@Data +@TableName("shop_data_duplicate_scan") +public class ShopDataDuplicateScanEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private String status; + private String errorText; + private String summaryJson; + private String payloadJson; + private LocalDateTime createdAt; + private LocalDateTime finishedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateItem.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateItem.java new file mode 100644 index 00000000..e125573d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateItem.java @@ -0,0 +1,13 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.payload; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** ASIN 维度明细：{asin, shop_count, record_count, occurrences[]}。 */ +public record DuplicateItem( + @JsonProperty("asin") String asin, + @JsonProperty("shop_count") int shopCount, + @JsonProperty("record_count") int recordCount, + @JsonProperty("occurrences") List occurrences) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateOccurrence.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateOccurrence.java new file mode 100644 index 00000000..e28277a0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateOccurrence.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.payload; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * 单条上架记录（occurrence）。键名/键集与 Python 后台落库 payload 完全一致： + * {asin, date, price, brand, shop_name, group_name, country_codes, country}。 + */ +public record DuplicateOccurrence( + @JsonProperty("asin") String asin, + @JsonProperty("date") String date, + @JsonProperty("price") String price, + @JsonProperty("brand") String brand, + @JsonProperty("shop_name") String shopName, + @JsonProperty("group_name") String groupName, + @JsonProperty("country_codes") List countryCodes, + @JsonProperty("country") String country) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateScanPayload.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateScanPayload.java new file mode 100644 index 00000000..8b1b7cb3 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateScanPayload.java @@ -0,0 +1,11 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.payload; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** 落库 payload：{shops:[...], items:[...]}，与 Python 现行版本逐字一致。 */ +public record DuplicateScanPayload( + @JsonProperty("shops") List shops, + @JsonProperty("items") List items) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateScanSummary.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateScanSummary.java new file mode 100644 index 00000000..d164d413 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateScanSummary.java @@ -0,0 +1,18 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.payload; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * 全量扫描指标：{shop_count, asin_total, record_total, duplicate_asin_total, + * duplicate_shop_count, site_count, asin_per_shop, source}。 + */ +public record DuplicateScanSummary( + @JsonProperty("shop_count") int shopCount, + @JsonProperty("asin_total") int asinTotal, + @JsonProperty("record_total") int recordTotal, + @JsonProperty("duplicate_asin_total") int duplicateAsinTotal, + @JsonProperty("duplicate_shop_count") int duplicateShopCount, + @JsonProperty("site_count") int siteCount, + @JsonProperty("asin_per_shop") double asinPerShop, + @JsonProperty("source") String source) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateShop.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateShop.java new file mode 100644 index 00000000..83e51471 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/model/payload/DuplicateShop.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.model.payload; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** 单店分布：{shop_name, group_name, country_codes, asin_count, record_count}。 */ +public record DuplicateShop( + @JsonProperty("shop_name") String shopName, + @JsonProperty("group_name") String groupName, + @JsonProperty("country_codes") List countryCodes, + @JsonProperty("asin_count") int asinCount, + @JsonProperty("record_count") int recordCount) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java new file mode 100644 index 00000000..b1fa2644 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java @@ -0,0 +1,417 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service; + +import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateItem; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateOccurrence; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateShop; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator; +import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckScanService.DuplicateScanView; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckCsvWriter; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckTimeNormalizer; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * 撞款读侧查询：角色裁剪 + 筛选 + 分页 + detail 排序 + overview 指标重算 + CSV 展平。 + * 内存模型/排序语义与 Python 后台逐字对齐；返回结构为可直接序列化的 Map（键 snake_case， + * 空态与 Python 响应完全一致：pending 时 summary 为 {} 空对象）。 + */ +@Service +@RequiredArgsConstructor +public class ShopDataDuplicateCheckQueryService { + + static final DateTimeFormatter SORT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final ZoneId SHANGHAI = ZoneId.of("Asia/Shanghai"); + + private final ShopDuplicateCheckSourceMapper sourceMapper; + + /** 筛选条件（参数已按 Python 清洗规则处理：asin 大写、country/site 大写、日期取前 10）。 */ + public record Filters(String asin, String shopName, String country, String site, + String dateFrom, String dateTo) { + + public static Filters clean(String asin, String shopName, String country, String site, + String dateFrom, String dateTo) { + String normAsin = trim(asin).toUpperCase(Locale.ROOT); + String normShop = trim(shopName); + String normCountry = trim(country).toUpperCase(Locale.ROOT); + String normSite = trim(site).toUpperCase(Locale.ROOT); + String from = trim(dateFrom); + String to = trim(dateTo); + from = from.length() > 10 ? from.substring(0, 10) : from; + to = to.length() > 10 ? to.substring(0, 10) : to; + return new Filters(normAsin, normShop, normCountry, normSite, from, to); + } + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static String shopKey(String shopName) { + return (shopName == null ? "" : shopName.trim()).toLowerCase(Locale.ROOT); + } + + /** 主管可见店铺 key 集；超管用 null 表示全量。 */ + public Set resolveVisibleShopKeys(Long operatorId, boolean superAdmin) { + if (superAdmin) { + return null; + } + Set keys = new LinkedHashSet<>(); + for (String name : sourceMapper.selectManagedShopNames(operatorId)) { + if (name != null && !name.isBlank()) { + keys.add(shopKey(name)); + } + } + return keys; + } + + /** 超管/null → 全量；否则保留可见店铺（shops 保持缓存序）。 */ + public List cropShops(List shops, Set visibleKeys) { + if (visibleKeys == null || shops == null) { + return shops == null ? List.of() : shops; + } + List result = new ArrayList<>(); + for (DuplicateShop shop : shops) { + if (visibleKeys.contains(shopKey(shop.shopName()))) { + result.add(shop); + } + } + return result; + } + + /** 逐 ASIN 裁剪 occurrences 并重算 shop_count/record_count；items 保持缓存序。 */ + public List cropItems(List items, Set visibleKeys) { + if (visibleKeys == null || items == null) { + return items == null ? List.of() : items; + } + List result = new ArrayList<>(items.size()); + for (DuplicateItem item : items) { + List kept = new ArrayList<>(); + Set shops = new LinkedHashSet<>(); + for (DuplicateOccurrence occurrence : item.occurrences()) { + if (visibleKeys.contains(shopKey(occurrence.shopName()))) { + kept.add(occurrence); + shops.add(occurrence.shopName()); + } + } + // 完全不可见的 ASIN 整体剔除（与 Python 主管数据范围语义一致） + if (!kept.isEmpty()) { + result.add(new DuplicateItem(item.asin(), shops.size(), kept.size(), kept)); + } + } + return result; + } + + public Map overviewData(DuplicateScanView view, Set visibleKeys) { + List shops = cropShops(view.payload().shops(), visibleKeys); + List items = cropItems(view.payload().items(), visibleKeys); + String source = view.summary() == null ? "" : safe(view.summary().source()); + DuplicateScanSummary recomputed = recomputeSummary(items, shops, source); + return mapOf("pending", false, "scanned_at", safe(view.scannedAt()), "summary", recomputed, "shops", shops); + } + + /** overview 无扫描结果时的空态结构（pending=true，summary 为空对象）。 */ + public Map pendingOverview() { + return mapOf("pending", true, "scanned_at", "", "summary", Collections.emptyMap(), "shops", Collections.emptyList()); + } + + public Map pendingList(int page, int pageSize) { + return mapOf("pending", true, "items", Collections.emptyList(), "shops", Collections.emptyList(), + "total", 0, "page", page, "page_size", pageSize, "scanned_at", ""); + } + + public Map itemsData(DuplicateScanView view, Set visibleKeys, + int page, int pageSize, String viewMode, Filters filters) { + boolean monitor = "monitor".equals(viewMode); + List cropped = cropItems(view.payload().items(), visibleKeys); + List matched = filterItems(cropped, monitor, filters); + int total = matched.size(); + int from = Math.min((page - 1) * pageSize, total); + int to = Math.min(from + pageSize, total); + List slice = matched.isEmpty() ? List.of() + : new ArrayList<>(matched.subList(from, to)); + List shops = cropShops(view.payload().shops(), visibleKeys); + return mapOf("pending", false, "items", slice, "shops", shops, + "total", total, "page", page, "page_size", pageSize, "scanned_at", safe(view.scannedAt())); + } + + public Map detailData(DuplicateScanView view, Set visibleKeys, + int page, int pageSize, Filters filters) { + List cropped = cropItems(view.payload().items(), visibleKeys); + List matched = filterItems(cropped, true, filters); + matched.sort(detailItemComparator()); + int total = matched.size(); + int from = Math.min((page - 1) * pageSize, total); + int to = Math.min(from + pageSize, total); + List> pageItems = new ArrayList<>(); + for (int i = from; i < to; i++) { + pageItems.add(detailItem(matched.get(i))); + } + return mapOf("pending", false, "items", pageItems, "total", total, + "page", page, "page_size", pageSize, "scanned_at", safe(view.scannedAt())); + } + + public Map pendingDetail(int page, int pageSize) { + return mapOf("pending", true, "items", Collections.emptyList(), "total", 0, + "page", page, "page_size", pageSize, "scanned_at", ""); + } + + public void writeExport(DuplicateScanView view, Set visibleKeys, + String viewMode, Filters filters, OutputStream out) throws IOException { + boolean monitor = "monitor".equals(viewMode); + List cropped = cropItems(view.payload().items(), visibleKeys); + List matched = filterItems(cropped, monitor, filters); + List> rows = new ArrayList<>(); + rows.add(List.of("ASIN", "店铺数", "店铺", "分组", "站点", "上架时间", "价格", "品牌")); + List flat = new ArrayList<>(); + for (DuplicateItem item : matched) { + String shopCount = String.valueOf(item.shopCount()); + for (DuplicateOccurrence occ : item.occurrences()) { + flat.add(new Object[]{item.asin(), shopCount, occ.shopName(), occ.groupName(), + effectiveSite(occ), safe(occ.date()), safe(occ.price()), safe(occ.brand())}); + } + } + flat.sort(flatExportComparator()); + for (Object[] row : flat) { + rows.add(toStringRow(row)); + } + DuplicateCheckCsvWriter.write(out, rows); + } + + public List filterItems(List items, boolean monitor, Filters f) { + List matched = new ArrayList<>(); + for (DuplicateItem item : items) { + if (monitor && item.shopCount() < 2) { + continue; + } + if (matches(item, f)) { + matched.add(item); + } + } + return matched; + } + + private boolean matches(DuplicateItem item, Filters f) { + String asin = item.asin() == null ? "" : item.asin(); + if (!f.asin().isEmpty() && !asin.contains(f.asin())) { + return false; + } + boolean hasOccFilter = !f.shopName().isEmpty() || !f.country().isEmpty() + || !f.site().isEmpty() || !f.dateFrom().isEmpty() || !f.dateTo().isEmpty(); + if (!hasOccFilter) { + return true; + } + for (DuplicateOccurrence occ : item.occurrences()) { + if (!occMatches(occ, f)) { + continue; + } + return true; + } + return false; + } + + private boolean occMatches(DuplicateOccurrence occ, Filters f) { + if (!f.shopName().isEmpty()) { + String needle = f.shopName().toLowerCase(Locale.ROOT); + String shop = safe(occ.shopName()).toLowerCase(Locale.ROOT); + String group = safe(occ.groupName()).toLowerCase(Locale.ROOT); + if (!shop.contains(needle) && !group.contains(needle)) { + return false; + } + } + if (!f.country().isEmpty()) { + boolean hit = false; + for (String code : occ.countryCodes() == null ? List.of() : occ.countryCodes()) { + if (f.country().equals(safe(code).toUpperCase(Locale.ROOT))) { + hit = true; + break; + } + } + if (!hit) { + return false; + } + } + if (!f.site().isEmpty() && !f.site().equals(safe(occ.country()).toUpperCase(Locale.ROOT))) { + return false; + } + String dateKey = DuplicateCheckTimeNormalizer.dateKey(safe(occ.date())); + if (!f.dateFrom().isEmpty() && !dateKey.isEmpty() && dateKey.compareTo(f.dateFrom()) < 0) { + return false; + } + if (!f.dateTo().isEmpty() && !dateKey.isEmpty() && dateKey.compareTo(f.dateTo()) > 0) { + return false; + } + return true; + } + + /** detail 卡片：occurrences 按归一化时间升序；brand=首个非空；first_date=最早可解析时间。 */ + private Map detailItem(DuplicateItem item) { + List sorted = new ArrayList<>(item.occurrences()); + sorted.sort(Comparator.comparing(occ -> DuplicateCheckTimeNormalizer.normalizeSortTime(safe(occ.date())))); + Set shopNames = new LinkedHashSet<>(); + String brand = ""; + String firstDate = ""; + for (DuplicateOccurrence occ : sorted) { + shopNames.add(occ.shopName() == null ? "" : occ.shopName()); + if (brand.isEmpty() && !safe(occ.brand()).isEmpty()) { + brand = occ.brand(); + } + if (firstDate.isEmpty()) { + String normalized = DuplicateCheckTimeNormalizer.normalizeSortTime(safe(occ.date())); + if (!normalized.isEmpty()) { + firstDate = normalized; + } + } + } + Map result = new LinkedHashMap<>(); + result.put("asin", item.asin()); + result.put("shop_count", item.shopCount()); + result.put("record_count", item.recordCount()); + result.put("shop_names", new ArrayList<>(shopNames)); + result.put("brand", brand); + result.put("first_date", firstDate); + result.put("occurrences", sorted); + return result; + } + + private Comparator detailItemComparator() { + return (a, b) -> { + int byShops = Integer.compare(b.shopCount(), a.shopCount()); + if (byShops != 0) { + return byShops; + } + long ra = earliestRank(a); + long rb = earliestRank(b); + if (ra != rb) { + return Long.compare(rb, ra); + } + return safe(a.asin()).compareTo(safe(b.asin())); + }; + } + + /** 最早上架归一化时间戳（Asia/Shanghai 转 epoch 秒）；无解析成功值返回 -1。 */ + private long earliestRank(DuplicateItem item) { + String earliest = ""; + for (DuplicateOccurrence occ : item.occurrences()) { + String normalized = DuplicateCheckTimeNormalizer.normalizeSortTime(safe(occ.date())); + if (!normalized.isEmpty() && (earliest.isEmpty() || normalized.compareTo(earliest) < 0)) { + earliest = normalized; + } + } + if (earliest.isEmpty()) { + return -1L; + } + try { + return LocalDateTime.parse(earliest, SORT_FORMAT).atZone(SHANGHAI).toEpochSecond(); + } catch (Exception ex) { + return -1L; + } + } + + private DuplicateScanSummary recomputeSummary(List items, List shops, String source) { + int recordTotal = 0; + int duplicateAsinTotal = 0; + Set duplicateShops = new LinkedHashSet<>(); + Set sites = new LinkedHashSet<>(); + for (DuplicateItem item : items) { + recordTotal += item.recordCount(); + if (item.shopCount() >= 2) { + duplicateAsinTotal++; + for (DuplicateOccurrence occ : item.occurrences()) { + duplicateShops.add(occ.shopName() == null ? "" : occ.shopName()); + } + } + } + for (DuplicateItem item : items) { + for (DuplicateOccurrence occ : item.occurrences()) { + if (!safe(occ.country()).isEmpty()) { + sites.add(occ.country()); + } + } + } + double asinPerShop = shops.isEmpty() ? 0.0 + : BigDecimal.valueOf((double) recordTotal / shops.size()) + .setScale(1, RoundingMode.HALF_EVEN).doubleValue(); + return new DuplicateScanSummary(shops.size(), items.size(), recordTotal, + duplicateAsinTotal, duplicateShops.size(), sites.size(), asinPerShop, + source == null ? "" : source); + } + + private String effectiveSite(DuplicateOccurrence occ) { + if (!safe(occ.country()).isEmpty()) { + return occ.country().toUpperCase(Locale.ROOT); + } + List codes = occ.countryCodes() == null ? List.of() : occ.countryCodes(); + if (codes.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (String code : codes) { + if (sb.length() > 0) { + sb.append('、'); + } + sb.append(safe(code).toUpperCase(Locale.ROOT)); + } + return sb.toString(); + } + + private Comparator flatExportComparator() { + return (a, b) -> { + int shopA = Integer.parseInt((String) a[1]); + int shopB = Integer.parseInt((String) b[1]); + if (shopA != shopB) { + return Integer.compare(shopB, shopA); + } + int byAsin = safe((String) a[0]).compareTo(safe((String) b[0])); + if (byAsin != 0) { + return byAsin; + } + int byShop = safe((String) a[2]).compareTo(safe((String) b[2])); + if (byShop != 0) { + return byShop; + } + return safe((String) a[5]).compareTo(safe((String) b[5])); + }; + } + + private List toStringRow(Object[] row) { + List list = new ArrayList<>(row.length); + for (Object value : row) { + list.add(safe(value)); + } + return list; + } + + private static String safe(Object value) { + return value == null ? "" : value.toString(); + } + + private Map mapOf(Object... entries) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < entries.length; i += 2) { + map.put((String) entries[i], entries[i + 1]); + } + return map; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckScanService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckScanService.java new file mode 100644 index 00000000..c0786b1c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckScanService.java @@ -0,0 +1,378 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper; +import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanFullRowDto; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ShopGroupLabelDto; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ShopSourceRowDto; +import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplicateScanEntity; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckWorkbookParser; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 撞款扫描：每日 00:00 定时 + force 同步全量重扫。逐店取最新结果文件、拉取并解析 xlsx、 + * 聚合成 shops/items/summary 落库 shop_data_duplicate_scan（格式与 Python 现行版本一致）。 + * 双实例通过 Redis 分布式锁防重；扫描失败落 FAILED 行并上抛，force 场景由端点转 409/500。 + */ +@Service +@Slf4j +public class ShopDataDuplicateCheckScanService { + + public static final String SCAN_LOCK = "shop-data-duplicate-check:scan"; + static final DateTimeFormatter SCANNED_AT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private static final long MAX_RESULT_BYTES = 256L * 1024 * 1024; + private static final long MEM_CACHE_MAX_BYTES = 64L * 1024 * 1024; + private static final int PARSE_THREADS = 4; + private static final java.time.Duration SCAN_LOCK_TTL = java.time.Duration.ofHours(6); + + private final ShopDataDuplicateScanMapper scanMapper; + private final ShopDuplicateCheckSourceMapper sourceMapper; + private final OssStorageService ossStorageService; + private final DistributedJobLockService distributedJobLockService; + private final ObjectMapper objectMapper; + + /** 进程内读缓存：轻量行 id 判活，行变化才全量读（对应 Python _duplicate_scan_memory）。 */ + private final AtomicLong cachedRowId = new AtomicLong(-1L); + private volatile CachedScan cachedScan; + + @Autowired + public ShopDataDuplicateCheckScanService(ShopDataDuplicateScanMapper scanMapper, + ShopDuplicateCheckSourceMapper sourceMapper, + OssStorageService ossStorageService, + DistributedJobLockService distributedJobLockService, + ObjectMapper objectMapper) { + this.scanMapper = scanMapper; + this.sourceMapper = sourceMapper; + this.ossStorageService = ossStorageService; + this.distributedJobLockService = distributedJobLockService; + this.objectMapper = objectMapper; + } + + /** 读侧视图：scanned_at 为最新 SUCCESS 行 created_at（yyyy-MM-dd HH:mm:ss）。 */ + public record DuplicateScanView(String scannedAt, DuplicateScanSummary summary, + DuplicateScanPayload payload) { + } + + private record CachedScan(long rowId, DuplicateScanView view) { + } + + /** force 端点调用：锁冲突抛 409，扫描失败落 FAILED 并抛 500。 */ + public void scanNow() { + try (DistributedJobLockService.LockHandle lock = distributedJobLockService.tryLock(SCAN_LOCK, SCAN_LOCK_TTL)) { + if (lock == null) { + throw new BusinessException(409, "扫描进行中，请稍后刷新"); + } + runScanAndSave(lock); + } + } + + /** 每日 00:00 定时（Asia/Shanghai）；另一实例持锁时直接跳过，异常不向上抛。 */ + @Scheduled(cron = "${aiimage.shop-duplicate-check.scan-cron:0 0 0 * * *}") + public void scheduledScan() { + try (DistributedJobLockService.LockHandle lock = distributedJobLockService.tryLock(SCAN_LOCK, SCAN_LOCK_TTL)) { + if (lock == null) { + log.info("[shop-duplicate-check] 定时扫描被其它实例执行，本实例跳过"); + return; + } + runScanAndSave(lock); + } catch (BusinessException ex) { + log.warn("[shop-duplicate-check] 定时扫描失败: code={} msg={}", ex.getCode(), ex.getMessage()); + } catch (Exception ex) { + log.error("[shop-duplicate-check] 定时扫描异常", ex); + } + } + + /** 最新 SUCCESS 扫描视图；无扫描结果返回 null。 */ + public DuplicateScanView loadLatest() { + ScanLightRowDto light = scanMapper.selectLatestLightRow(); + long lightId = light == null || light.getId() == null ? -1L : light.getId(); + CachedScan cached = cachedScan; + if (lightId == cachedRowId.get() && cached != null) { + return cached.view(); + } + if (light == null) { + cachedScan = null; + cachedRowId.set(-1L); + return null; + } + ScanFullRowDto full = scanMapper.selectLatestFullRow(); + if (full == null || full.getPayloadJson() == null || full.getPayloadJson().isBlank()) { + cachedScan = null; + cachedRowId.set(lightId); + return null; + } + DuplicateScanView view = buildView(full); + if (view == null) { + cachedScan = null; + cachedRowId.set(lightId); + return null; + } + long payloadBytes = full.getPayloadJson().getBytes(java.nio.charset.StandardCharsets.UTF_8).length; + if (payloadBytes <= MEM_CACHE_MAX_BYTES) { + cachedScan = new CachedScan(lightId, view); + cachedRowId.set(lightId); + } else { + // payload 过大不常驻内存，每次请求读库（等价 Python 64MB 阈值语义） + cachedScan = null; + cachedRowId.set(-1L); + } + return view; + } + + private DuplicateScanView buildView(ScanFullRowDto full) { + try { + DuplicateScanSummary summary = objectMapper.readValue(full.getSummaryJson(), DuplicateScanSummary.class); + JsonNode payloadNode = objectMapper.readTree(full.getPayloadJson()); + if (payloadNode == null || !payloadNode.isObject() || !payloadNode.has("shops") || !payloadNode.has("items")) { + // 兼容性兜底：旧版 list payload 不解析，视为无结果并告警 + log.warn("[shop-duplicate-check] 忽略无法解析的 payload 行 id={}", full.getId()); + return null; + } + DuplicateScanPayload payload = objectMapper.treeToValue(payloadNode, DuplicateScanPayload.class); + String scannedAt = full.getCreatedAt() == null ? "" + : full.getCreatedAt().format(SCANNED_AT_FORMAT); + return new DuplicateScanView(scannedAt, summary, payload); + } catch (Exception ex) { + log.warn("[shop-duplicate-check] 解析最新扫描行失败 id={} msg={}", full.getId(), ex.getMessage()); + return null; + } + } + + private void runScanAndSave(DistributedJobLockService.LockHandle lock) { + try { + List parsedShops = collectShopParsed(lock); + DuplicateCheckAggregator.Aggregate aggregate = + DuplicateCheckAggregator.aggregate(parsedShops, "job"); + ShopDataDuplicateScanEntity row = new ShopDataDuplicateScanEntity(); + row.setStatus("SUCCESS"); + row.setErrorText(null); + row.setFinishedAt(LocalDateTime.now()); + try { + row.setSummaryJson(objectMapper.writeValueAsString(aggregate.summary())); + row.setPayloadJson(objectMapper.writeValueAsString(aggregate.payload())); + } catch (Exception ex) { + throw new BusinessException("扫描结果序列化失败: " + ex.getMessage(), ex); + } + scanMapper.insert(row); + log.info("[shop-duplicate-check] 扫描完成 落库行 id={} 店铺数={} ASIN数={} 记录数={}", + row.getId(), aggregate.summary().shopCount(), + aggregate.summary().asinTotal(), aggregate.summary().recordTotal()); + } catch (BusinessException ex) { + saveFailed(ex.getMessage()); + throw ex; + } catch (Exception ex) { + log.error("[shop-duplicate-check] 扫描失败", ex); + saveFailed(ex.getMessage() == null ? ex.toString() : ex.getMessage()); + throw new BusinessException(500, "执行失败：" + fallbackMessage(ex)); + } + } + + private void saveFailed(String errorText) { + try { + ShopDataDuplicateScanEntity row = new ShopDataDuplicateScanEntity(); + row.setStatus("FAILED"); + row.setErrorText(errorText); + row.setFinishedAt(LocalDateTime.now()); + scanMapper.insert(row); + } catch (Exception ex) { + log.error("[shop-duplicate-check] 记录 FAILED 扫描行失败", ex); + } + } + + private String fallbackMessage(Exception ex) { + String message = ex.getMessage(); + return message == null || message.isBlank() ? ex.getClass().getSimpleName() : message; + } + + /** 逐店并发拉取/解析结果文件；单店失败仅告警跳过，不中断整体。 */ + private List collectShopParsed(DistributedJobLockService.LockHandle lock) { + List rows = sourceMapper.selectLatestResultRows(); + List parsedShops = new ArrayList<>(); + if (rows.isEmpty()) { + return parsedShops; + } + Map groupLabels = loadGroupLabels(rows); + ExecutorService executor = Executors.newFixedThreadPool(Math.min(PARSE_THREADS, rows.size())); + try { + List> futures = new ArrayList<>(rows.size()); + for (ShopSourceRowDto row : rows) { + futures.add(executor.submit(() -> parseShop(row, groupLabels))); + } + for (Future future : futures) { + if (lock != null) { + lock.renew(SCAN_LOCK_TTL); + } + try { + ShopParsed item = future.get(); + if (item != null) { + parsedShops.add(item); + } + } catch (ExecutionException ex) { + log.warn("[shop-duplicate-check] 并发解析店铺结果文件异常: {}", ex.getCause() == null + ? ex.getMessage() : ex.getCause().getMessage()); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + log.warn("[shop-duplicate-check] 解析线程被中断"); + break; + } + } + } finally { + executor.shutdown(); + try { + if (!executor.awaitTermination(30, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + executor.shutdownNow(); + } + } + return parsedShops; + } + + private Map loadGroupLabels(List rows) { + Set names = new LinkedHashSet<>(); + for (ShopSourceRowDto row : rows) { + String name = normalizeShopName(row.getShopName()); + if (!name.isEmpty()) { + names.add(name); + } + } + Map labels = new HashMap<>(); + if (names.isEmpty()) { + return labels; + } + for (ShopGroupLabelDto dto : sourceMapper.selectGroupLabels(new ArrayList<>(names))) { + labels.put(shopKey(dto.getShopName()), dto.getGroupName() == null ? "" : dto.getGroupName()); + } + return labels; + } + + private ShopParsed parseShop(ShopSourceRowDto row, Map groupLabels) { + long resultId = row.getResultId() == null ? 0L : row.getResultId(); + if (resultId <= 0) { + return null; + } + String shopName = normalizeShopName(row.getShopName()); + if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { + return null; + } + try { + if (row.getResultFileSize() != null && row.getResultFileSize() > MAX_RESULT_BYTES) { + log.warn("[shop-data-crawl] 结果文件过大跳过 result_id={} size={}", resultId, row.getResultFileSize()); + return null; + } + byte[] bytes; + try { + bytes = ossStorageService.readObjectBytesBounded(row.getResultFileUrl(), MAX_RESULT_BYTES); + } catch (IllegalArgumentException ex) { + log.warn("[shop-data-crawl] 结果文件过大跳过 result_id={}: {}", resultId, ex.getMessage()); + return null; + } + List parsed; + try { + parsed = DuplicateCheckWorkbookParser.parse(bytes); + } catch (Exception ex) { + log.warn("[shop-data-crawl] 解析结果文件失败 result_id={}: {}", resultId, + ex.getMessage() == null ? ex.toString() : ex.getMessage()); + return null; + } + List countryCodes = parseCountryCodesJson(row.getCountryCodesJson()); + if (countryCodes.isEmpty()) { + countryCodes = parseCountryCodesRequest(row.getRequestJson()); + } + String groupName = groupLabels.getOrDefault(shopKey(shopName), ""); + return new ShopParsed(shopName, groupName, countryCodes, parsed); + } catch (Exception ex) { + log.warn("[shop-data-crawl] 拉取结果文件失败 result_id={}: {}", resultId, + ex.getMessage() == null ? ex.toString() : ex.getMessage()); + return null; + } + } + + private String normalizeShopName(String raw) { + return raw == null ? "未命名" : (raw.trim().isEmpty() ? "未命名" : raw.trim()); + } + + private String shopKey(String name) { + return (name == null ? "" : name.trim()).toLowerCase(Locale.ROOT); + } + + private List parseCountryCodesJson(String jsonValue) { + if (jsonValue == null || jsonValue.isBlank()) { + return List.of(); + } + try { + JsonNode node = objectMapper.readTree(jsonValue); + if (node != null && node.isArray()) { + return normalizeCodes(node); + } + } catch (Exception ex) { + log.warn("[shop-data-crawl] 解析 country_codes_json 失败: {}", ex.getMessage()); + } + return List.of(); + } + + private List parseCountryCodesRequest(String requestJson) { + if (requestJson == null || requestJson.isBlank()) { + return List.of(); + } + try { + JsonNode node = objectMapper.readTree(requestJson); + if (node == null || !node.isObject()) { + return List.of(); + } + JsonNode raw = node.get("countryCodes"); + if (raw == null) { + raw = node.get("country_codes"); + } + if (raw != null && raw.isArray()) { + return normalizeCodes(raw); + } + } catch (Exception ex) { + log.warn("[shop-data-crawl] 解析 request_json 国家代码失败: {}", ex.getMessage()); + } + return List.of(); + } + + private List normalizeCodes(JsonNode array) { + List codes = new ArrayList<>(); + for (JsonNode item : array) { + String value = item.asText(); + if (value != null && !value.trim().isEmpty()) { + codes.add(value.trim().toUpperCase(Locale.ROOT)); + } + } + return codes; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckAggregator.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckAggregator.java new file mode 100644 index 00000000..a3abb69a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckAggregator.java @@ -0,0 +1,132 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateItem; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateOccurrence; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateShop; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 等价 Python _build_duplicate_scan_cache：把各店解析行聚合成 shops/items/summary。 + * items 按 (-shop_count, asin) 排序；shops 按 (-asin_count, shop_name) 排序；summary 全字段重算。 + */ +public final class DuplicateCheckAggregator { + + private static final Comparator ITEM_ORDER = + Comparator.comparingInt(DuplicateItem::shopCount).reversed() + .thenComparing(DuplicateItem::asin); + private static final Comparator SHOP_ORDER = + Comparator.comparingInt(DuplicateShop::asinCount).reversed() + .thenComparing(DuplicateShop::shopName); + + private DuplicateCheckAggregator() { + } + + /** 单店聚合元信息：分组名 + 店铺级国家代码。 */ + private record ShopMeta(String groupName, List countryCodes) { + } + + public record Aggregate(DuplicateScanPayload payload, DuplicateScanSummary summary) { + } + + public static Aggregate aggregate(List shopItems, String source) { + Map> occurrencesByAsin = new LinkedHashMap<>(); + Map shopMeta = new LinkedHashMap<>(); + + for (ShopParsed shopItem : shopItems) { + String shopName = (shopItem.shopName() == null || shopItem.shopName().isBlank()) + ? "未命名" : shopItem.shopName().trim(); + String groupName = shopItem.groupName() == null ? "" : shopItem.groupName(); + List countryCodes = shopItem.countryCodes() == null + ? List.of() : List.copyOf(shopItem.countryCodes()); + shopMeta.put(shopName, new ShopMeta(groupName, countryCodes)); + for (RawRow row : shopItem.rows()) { + String asin = (row.asin() == null ? "" : row.asin()).trim().toUpperCase(); + if (asin.isEmpty()) { + continue; + } + occurrencesByAsin.computeIfAbsent(asin, key -> new ArrayList<>()).add( + new DuplicateOccurrence( + asin, + row.date() == null ? "" : row.date(), + row.price() == null ? "" : row.price(), + row.brand() == null ? "" : row.brand(), + shopName, + groupName, + countryCodes, + row.country() == null ? "" : row.country())); + } + } + + List items = new ArrayList<>(occurrencesByAsin.size()); + int totalRecords = 0; + for (Map.Entry> entry : occurrencesByAsin.entrySet()) { + List occurrences = entry.getValue(); + Set shops = new LinkedHashSet<>(); + for (DuplicateOccurrence occurrence : occurrences) { + shops.add(occurrence.shopName()); + } + totalRecords += occurrences.size(); + items.add(new DuplicateItem(entry.getKey(), shops.size(), occurrences.size(), occurrences)); + } + items.sort(ITEM_ORDER); + + List shops = new ArrayList<>(shopMeta.size()); + for (Map.Entry entry : shopMeta.entrySet()) { + String shopName = entry.getKey(); + ShopMeta meta = entry.getValue(); + Set asinSet = new LinkedHashSet<>(); + int recordCount = 0; + for (List occurrences : occurrencesByAsin.values()) { + for (DuplicateOccurrence occurrence : occurrences) { + if (occurrence.shopName().equals(shopName)) { + asinSet.add(occurrence.asin()); + recordCount++; + } + } + } + shops.add(new DuplicateShop(shopName, meta.groupName(), meta.countryCodes(), + asinSet.size(), recordCount)); + } + shops.sort(SHOP_ORDER); + + int duplicateAsinTotal = 0; + Set duplicateShops = new LinkedHashSet<>(); + Set sites = new LinkedHashSet<>(); + for (DuplicateItem item : items) { + if (item.shopCount() >= 2) { + duplicateAsinTotal++; + for (DuplicateOccurrence occurrence : item.occurrences()) { + duplicateShops.add(occurrence.shopName()); + } + } + } + for (List occurrences : occurrencesByAsin.values()) { + for (DuplicateOccurrence occurrence : occurrences) { + if (!occurrence.country().isEmpty()) { + sites.add(occurrence.country()); + } + } + } + double asinPerShop = shops.isEmpty() ? 0.0 : roundToScale1((double) totalRecords / shops.size()); + DuplicateScanSummary summary = new DuplicateScanSummary( + shops.size(), items.size(), totalRecords, duplicateAsinTotal, + duplicateShops.size(), sites.size(), asinPerShop, + source == null ? "job" : source); + return new Aggregate(new DuplicateScanPayload(shops, items), summary); + } + + private static double roundToScale1(double value) { + return BigDecimal.valueOf(value).setScale(1, RoundingMode.HALF_EVEN).doubleValue(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckCsvWriter.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckCsvWriter.java new file mode 100644 index 00000000..5f4a6c07 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckCsvWriter.java @@ -0,0 +1,59 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** + * 导出 CSV：UTF-8 BOM + CRLF + QUOTE_MINIMAL 转义（含 , " \r \n 时双引号包裹、内部 " 翻倍）， + * 等价 Python csv.writer 默认行为。 + */ +public final class DuplicateCheckCsvWriter { + + private DuplicateCheckCsvWriter() { + } + + public static void write(OutputStream out, List> rows) throws IOException { + out.write(0xEF); + out.write(0xBB); + out.write(0xBF); + Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8); + StringBuilder line = new StringBuilder(); + for (List row : rows) { + line.setLength(0); + for (int i = 0; i < row.size(); i++) { + if (i > 0) { + line.append(','); + } + appendField(line, row.get(i)); + } + line.append("\r\n"); + writer.write(line.toString()); + } + writer.flush(); + } + + private static void appendField(StringBuilder target, String value) { + String text = value == null ? "" : value; + boolean needQuote = text.indexOf(',') >= 0 + || text.indexOf('"') >= 0 + || text.indexOf('\r') >= 0 + || text.indexOf('\n') >= 0; + if (!needQuote) { + target.append(text); + return; + } + target.append('"'); + for (int i = 0; i < text.length(); i++) { + char ch = text.charAt(i); + if (ch == '"') { + target.append('"'); + } + target.append(ch); + } + target.append('"'); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckTimeNormalizer.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckTimeNormalizer.java new file mode 100644 index 00000000..35266c9f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckTimeNormalizer.java @@ -0,0 +1,140 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 复刻 Python 后台 _shop_data_dup_sort_time / _shop_data_date_key 的逐字语义： + * 中文日期/上午下午/斜杠点格式归一化。输出固定宽度 YYYY-MM-DD HH:MM:SS 或 + * YYYY-MM-DD 字符串，直接字符串比较即时间比较；无法解析时原样返回（排最后）。 + */ +public final class DuplicateCheckTimeNormalizer { + + private static final DateTimeFormatter SORT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private static final Pattern ISO_TIME = Pattern.compile("([0-9]{1,2})\\s*[:：]\\s*([0-9]{2})(?:[:：]\\s*([0-9]{1,2}))?"); + private static final Pattern PERIOD_TIME = Pattern.compile( + "(上午|下午|晚上|凌晨)?\\s*([0-9]{1,2})\\s*[:：]\\s*([0-9]{2})(?:[:：]\\s*([0-9]{1,2}))?"); + private static final Pattern CHINESE_DATE = Pattern.compile("([0-9]{4})\\s*年\\s*([0-9]{1,2})\\s*月\\s*([0-9]{1,2})\\s*日\\s*(\\S*)?"); + private static final Pattern SLASH_DATE = Pattern.compile("([0-9]{4})[./]([0-9]{1,2})[./]([0-9]{1,2})(.*)"); + private static final Pattern SEPARATED_DATE = Pattern.compile("([0-9]{4})\\s*[年./-]\\s*([0-9]{1,2})\\s*[月./-]\\s*([0-9]{1,2})"); + + private DuplicateCheckTimeNormalizer() { + } + + /** 等价 Python _shop_data_dup_sort_time：19 位排序时间，无法解析原样返回。 */ + public static String normalizeSortTime(String raw) { + String text = raw == null ? "" : raw.trim(); + if (text.isEmpty()) { + return ""; + } + // ISO 风格：2026-08-18 04:34:00 / 2026-08-18 04:34 / 2026-08-18 + if (text.length() >= 10 && text.charAt(4) == '-' && text.charAt(7) == '-') { + String base = text.substring(0, 10); + Matcher tm = ISO_TIME.matcher(text.substring(10)); + if (tm.find()) { + int hour = Integer.parseInt(tm.group(1)); + int minute = Integer.parseInt(tm.group(2)); + int second = tm.group(3) == null ? 0 : Integer.parseInt(tm.group(3)); + LocalDateTime time = tryLocalDateTime(base, hour, minute, second); + if (time != null) { + return time.format(SORT_FORMAT); + } + return text; + } + return base + " 00:00:00"; + } + // 中文日期：2026年8月29日 上午4:34 / 下午2:15 / 晚上8:00 + Matcher m = CHINESE_DATE.matcher(text); + if (m.find()) { + int year = Integer.parseInt(m.group(1)); + int month = Integer.parseInt(m.group(2)); + int day = Integer.parseInt(m.group(3)); + String periodText = m.group(4) == null ? "" : m.group(4).trim(); + int hour = 0; + int minute = 0; + int second = 0; + Matcher tm = PERIOD_TIME.matcher(periodText); + if (tm.find()) { + hour = Integer.parseInt(tm.group(2)); + minute = Integer.parseInt(tm.group(3)); + second = tm.group(4) == null ? 0 : Integer.parseInt(tm.group(4)); + String period = tm.group(1) == null ? "" : tm.group(1); + if (("下午".equals(period) || "晚上".equals(period)) && hour < 12) { + hour += 12; + } + if (("上午".equals(period) || "凌晨".equals(period)) && hour == 12) { + hour = 0; + } + } + LocalDateTime time = tryLocalDateTime(year, month, day, hour, minute, second); + if (time != null) { + return time.format(SORT_FORMAT); + } + return text; + } + // 斜杠/点分隔日期：2026/8/18 4:34（无时段换算） + Matcher dm = SLASH_DATE.matcher(text); + if (dm.find()) { + int year = Integer.parseInt(dm.group(1)); + int month = Integer.parseInt(dm.group(2)); + int day = Integer.parseInt(dm.group(3)); + int hour = 0; + int minute = 0; + int second = 0; + Matcher tm = ISO_TIME.matcher(dm.group(4) == null ? "" : dm.group(4)); + if (tm.find()) { + hour = Integer.parseInt(tm.group(1)); + minute = Integer.parseInt(tm.group(2)); + second = tm.group(3) == null ? 0 : Integer.parseInt(tm.group(3)); + } + LocalDateTime time = tryLocalDateTime(year, month, day, hour, minute, second); + if (time != null) { + return time.format(SORT_FORMAT); + } + return text; + } + return text; + } + + /** 等价 Python _shop_data_date_key：YYYY-MM-DD 比较键，无法识别原样返回。 */ + public static String dateKey(String dateText) { + String text = dateText == null ? "" : dateText.trim(); + if (text.isEmpty()) { + return ""; + } + if (text.length() >= 10 && text.charAt(4) == '-' && text.charAt(7) == '-') { + return text.substring(0, 10); + } + Matcher m = SEPARATED_DATE.matcher(text); + if (m.find()) { + int year = Integer.parseInt(m.group(1)); + int month = Integer.parseInt(m.group(2)); + int day = Integer.parseInt(m.group(3)); + // Python 侧仅做 int + f-string 格式化、不校验月日合法性，此处保持一致 + return String.format("%04d-%02d-%02d", year, month, day); + } + return text; + } + + private static LocalDateTime tryLocalDateTime(String isoBase, int hour, int minute, int second) { + try { + int year = Integer.parseInt(isoBase.substring(0, 4)); + int month = Integer.parseInt(isoBase.substring(5, 7)); + int day = Integer.parseInt(isoBase.substring(8, 10)); + return LocalDateTime.of(year, month, day, hour, minute, second); + } catch (RuntimeException ex) { + return null; + } + } + + private static LocalDateTime tryLocalDateTime(int year, int month, int day, int hour, int minute, int second) { + try { + return LocalDateTime.of(year, month, day, hour, minute, second); + } catch (RuntimeException ex) { + return null; + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckWorkbookParser.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckWorkbookParser.java new file mode 100644 index 00000000..4adedf52 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckWorkbookParser.java @@ -0,0 +1,111 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import com.nanri.aiimage.common.util.ExcelStreamReader; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 解析店铺结果 xlsx 为 ASIN 行：第 1 行表头、必须有 ASIN 列否则整 sheet 跳过、 + * 可选列 日期/价格/品牌、sheet 名（小写精确）映射国家码。语义等价 Python + * _shop_data_crawl_parse_workbook + _shop_data_sheet_country。失败由上层捕获（不中断整体扫描）。 + */ +public final class DuplicateCheckWorkbookParser { + + private static final Map SHEET_COUNTRY = buildSheetCountryMap(); + + private DuplicateCheckWorkbookParser() { + } + + public static List parse(byte[] bytes) throws Exception { + Parser handler = new Parser(); + ExcelStreamReader.readAllSheets(new ByteArrayInputStream(bytes), handler); + return handler.rows; + } + + private static Map buildSheetCountryMap() { + Map map = new LinkedHashMap<>(); + map.put("英国", "UK"); + map.put("uk", "UK"); + map.put("u.k.", "UK"); + map.put("united kingdom", "UK"); + map.put("德国", "DE"); + map.put("de", "DE"); + map.put("germany", "DE"); + map.put("法国", "FR"); + map.put("fr", "FR"); + map.put("france", "FR"); + map.put("意大利", "IT"); + map.put("it", "IT"); + map.put("italy", "IT"); + map.put("西班牙", "ES"); + map.put("es", "ES"); + map.put("spain", "ES"); + return map; + } + + private static final class Parser implements ExcelStreamReader.SheetRowHandler { + + private final List rows = new ArrayList<>(); + private int asinCol = -1; + private int dateCol = -1; + private int priceCol = -1; + private int brandCol = -1; + private String sheetCountry = ""; + + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + asinCol = -1; + dateCol = -1; + priceCol = -1; + brandCol = -1; + sheetCountry = countryOf(sheetName); + if (headerMap == null) { + return; + } + for (Map.Entry entry : headerMap.entrySet()) { + String header = cell(entry.getValue()); + int index = entry.getKey(); + if ("ASIN".equals(header)) { + asinCol = index; + } else if ("日期".equals(header)) { + dateCol = index; + } else if ("价格".equals(header)) { + priceCol = index; + } else if ("品牌".equals(header)) { + brandCol = index; + } + } + } + + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + if (asinCol < 0) { + return; + } + String asin = cell(rowMap == null ? null : rowMap.get(asinCol)); + if (asin.isEmpty()) { + return; + } + rows.add(new RawRow( + asin, + cell(rowMap.get(dateCol)), + cell(rowMap.get(priceCol)), + cell(rowMap.get(brandCol)), + sheetCountry)); + } + } + + private static String countryOf(String sheetName) { + String key = (sheetName == null ? "" : sheetName).trim().toLowerCase(); + return SHEET_COUNTRY.getOrDefault(key, ""); + } + + private static String cell(Object value) { + return value == null ? "" : value.toString().trim(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/RawRow.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/RawRow.java new file mode 100644 index 00000000..db4280a6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/RawRow.java @@ -0,0 +1,5 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +/** 单店结果文件解析出的一行：asin 大写处理在聚合阶段完成。 */ +public record RawRow(String asin, String date, String price, String brand, String country) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/ShopParsed.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/ShopParsed.java new file mode 100644 index 00000000..111cf511 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/ShopParsed.java @@ -0,0 +1,7 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import java.util.List; + +/** 单店参与扫描的最小单元：店铺归属 + 解析行列表。 */ +public record ShopParsed(String shopName, String groupName, List countryCodes, List rows) { +} diff --git a/backend-java/src/main/resources/db/V108__shop_data_duplicate_check_scan_table.sql b/backend-java/src/main/resources/db/V108__shop_data_duplicate_check_scan_table.sql new file mode 100644 index 00000000..d4ad99ae --- /dev/null +++ b/backend-java/src/main/resources/db/V108__shop_data_duplicate_check_scan_table.sql @@ -0,0 +1,12 @@ +-- 店铺数据重复检查（撞款）扫描结果表：Java 侧等价于 Python 的 _ensure_duplicate_scan_table。 +-- 生产库该表已由 Python 侧创建，IF NOT EXISTS 保证幂等（Flyway 只追加、不改历史）。 +CREATE TABLE IF NOT EXISTS shop_data_duplicate_scan ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + status VARCHAR(16) NOT NULL, + error_text TEXT NULL, + summary_json MEDIUMTEXT NULL, + payload_json MEDIUMTEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at DATETIME NULL, + KEY idx_status_created (status, created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/AdminApiGuardFilterTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/AdminApiGuardFilterTest.java index e2006b59..c77cac1f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/config/AdminApiGuardFilterTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/config/AdminApiGuardFilterTest.java @@ -62,6 +62,44 @@ class AdminApiGuardFilterTest { assertThat(response.getContentAsString()).isEmpty(); } + @Test + void trustedInternalTokenBypassesGuardButReachesController() throws Exception { + // 仅凭 X-Internal-Token 的合法内部调用(如 shop-manages/credential 由 controller 自校验) + // 应被 guard 放行,而不是拦成 401 —— 修复客户端带令牌却被 guard 误拦的故障。 + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + when(authSupport.isTrustedInternalToken(any())).thenReturn(true); + AdminApiGuardFilter filter = newFilter(authSupport, true, ""); + MockHttpServletRequest request = new MockHttpServletRequest("GET", "/api/admin/shop-manages/credential"); + MockHttpServletResponse response = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + verify(authSupport, never()).requireUserOrInternal(any()); + assertThat(chain.getRequest()).isNotNull(); + assertThat(response.getContentAsString()).isEmpty(); + } + + @Test + void untrustedTokenStillRejected() throws Exception { + // 令牌不匹配(X-Internal-Token 错值/泄露)时 guard 仍然拦截 + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + when(authSupport.isTrustedInternalToken(any())).thenReturn(false); + when(authSupport.requireUserOrInternal(any())) + .thenThrow(new BusinessException(401, "未登录")); + AdminApiGuardFilter filter = newFilter(authSupport, true, ""); + MockHttpServletRequest request = new MockHttpServletRequest("GET", "/api/admin/shop-keys"); + MockHttpServletResponse response = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + JsonNode body = objectMapper.readTree(response.getContentAsString()); + assertThat(body.path("success").asBoolean()).isFalse(); + assertThat(body.path("code").asInt()).isEqualTo(401); + assertThat(chain.getRequest()).isNull(); + } + @Test void anonymousAdminApiRequestRejectedWith401Body() throws Exception { AdminAuthSupport authSupport = mock(AdminAuthSupport.class); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java index 89b81bd9..59494fa7 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java @@ -40,6 +40,7 @@ class AdminUserServiceTest { request.setColumnIds(List.of(11L, 12L)); when(authSupport.currentRole(operator)).thenReturn("admin"); + when(permissionService.hasAnyAdminMenu(eq(operator), any())).thenReturn(true); when(passwordEncoder.hash("secret1")).thenReturn("hashed"); when(userMapper.insert(any(AdminUserEntity.class))).thenAnswer(invocation -> { invocation.getArgument(0).setId(20L); @@ -73,6 +74,7 @@ class AdminUserServiceTest { request.setColumnIds(List.of(21L)); when(authSupport.currentRole(operator)).thenReturn("admin"); + when(permissionService.hasAnyAdminMenu(eq(operator), any())).thenReturn(true); when(userMapper.selectById(20L)).thenReturn(target); service.updateUser(operator, 20L, request); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/TestFixtures.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/TestFixtures.java new file mode 100644 index 00000000..1731fb77 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/TestFixtures.java @@ -0,0 +1,46 @@ +package com.nanri.aiimage.modules.shopduplicatecheck; + +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload; +import com.nanri.aiimage.modules.shopduplicatecheck.service.ShopDataDuplicateCheckScanService.DuplicateScanView; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow; +import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed; + +import java.util.List; + +/** + * 撞款测试共享夹具：4 家店 / 3 组 / 6 个 ASIN，与 Python test_admin_shop_data_duplicate_check.py 的 + * 分布一致。经 DuplicateCheckAggregator 聚合出真实 payload/summary（shops/items 按扫描排序规则落序）。 + */ +public final class TestFixtures { + + private TestFixtures() { + } + + public static List shopParsed() { + ShopParsed shopA = new ShopParsed("ShopA", "GroupA", List.of("UK", "DE"), List.of( + new RawRow("A0000001", "2026-08-30", "GBP 9.99", "BrandA", "UK"), + new RawRow("B0000001", "2026-08-30", "EUR 10.99", "BrandA", "DE"), + new RawRow("E0000001", "2026年8月29日 上午4:34", "GBP 12.00", "BrandE", "UK"))); + ShopParsed shopB = new ShopParsed("ShopB", "GroupA", List.of("UK"), List.of( + new RawRow("A0000001", "2026-08-30 08:30:00", "GBP 8.50", "BrandA", "UK"), + new RawRow("E0000001", "2026-08-30", "GBP 12.00", "BrandE", "UK"), + new RawRow("F0000001", "2026-08-28", "GBP 6.00", "BrandF", "UK"))); + ShopParsed shopC = new ShopParsed("ShopC", "GroupC", List.of("FR"), List.of( + new RawRow("C0000001", "2026-08-29", "EUR 7.50", "BrandC", "FR"), + new RawRow("E0000001", "2026-08-31", "EUR 12.00", "BrandE", "FR"))); + ShopParsed shopD = new ShopParsed("ShopD", "GroupD", List.of("UK"), List.of( + new RawRow("D0000001", "2026-08-28", "GBP 5.00", "BrandD", "UK"), + new RawRow("F0000001", "2026-08-30", "GBP 6.00", "BrandF", "UK"))); + return List.of(shopA, shopB, shopC, shopD); + } + + public static DuplicateScanPayload payload() { + return DuplicateCheckAggregator.aggregate(shopParsed(), "job").payload(); + } + + public static DuplicateScanView view() { + DuplicateCheckAggregator.Aggregate aggregate = DuplicateCheckAggregator.aggregate(shopParsed(), "job"); + return new DuplicateScanView("2026-09-04 03:10:00", aggregate.summary(), aggregate.payload()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryServiceTest.java new file mode 100644 index 00000000..99c7ab8f --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryServiceTest.java @@ -0,0 +1,156 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service; + +import com.nanri.aiimage.modules.shopduplicatecheck.TestFixtures; +import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateItem; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateOccurrence; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateShop; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +class ShopDataDuplicateCheckQueryServiceTest { + + private ShopDataDuplicateCheckQueryService service; + private ShopDataDuplicateCheckScanService.DuplicateScanView view; + + @BeforeEach + void setUp() { + ShopDuplicateCheckSourceMapper mapper = mock(ShopDuplicateCheckSourceMapper.class); + service = new ShopDataDuplicateCheckQueryService(mapper); + view = TestFixtures.view(); + } + + private ShopDataDuplicateCheckQueryService.Filters noFilter() { + return ShopDataDuplicateCheckQueryService.Filters.clean("", "", "", "", "", ""); + } + + @Test + void overviewSuperAdminSeesAll() { + Map data = service.overviewData(view, null); + assertThat(data.get("pending")).isEqualTo(false); + assertThat(data.get("scanned_at")).isEqualTo("2026-09-04 03:10:00"); + DuplicateScanSummary summary = (DuplicateScanSummary) data.get("summary"); + assertThat(summary.asinTotal()).isEqualTo(6); + assertThat(summary.recordTotal()).isEqualTo(10); + assertThat(summary.duplicateAsinTotal()).isEqualTo(3); + assertThat(summary.shopCount()).isEqualTo(4); + assertThat(((List) data.get("shops"))).hasSize(4); + } + + @Test + void leaderOverviewStatsRecomputedAfterCrop() { + Set visible = Set.of("shopa", "shopb"); + Map data = service.overviewData(view, visible); + DuplicateScanSummary summary = (DuplicateScanSummary) data.get("summary"); + assertThat(summary.shopCount()).isEqualTo(2); + assertThat(summary.asinTotal()).isEqualTo(4); + assertThat(summary.recordTotal()).isEqualTo(6); + assertThat(summary.duplicateAsinTotal()).isEqualTo(2); + assertThat(summary.duplicateShopCount()).isEqualTo(2); + assertThat(summary.siteCount()).isEqualTo(2); + } + + @Test + void leaderSeesOnlyOwnGroupShops() { + Set visible = Set.of("shopa", "shopb"); + Map data = service.itemsData(view, visible, 1, 20, "all", noFilter()); + @SuppressWarnings("unchecked") + List shops = (List) data.get("shops"); + assertThat(shops.stream().map(DuplicateShop::shopName).toList()) + .containsExactly("ShopA", "ShopB"); + assertThat(data.get("total")).isEqualTo(4); + @SuppressWarnings("unchecked") + List items = (List) data.get("items"); + DuplicateItem a = items.stream().filter(item -> "A0000001".equals(item.asin())).findFirst().orElseThrow(); + assertThat(a.shopCount()).isEqualTo(2); + } + + @Test + void leaderSeesNothingWithoutAnyGroup() { + Map data = service.itemsData(view, Set.of(), 1, 20, "all", noFilter()); + assertThat(data.get("shops")).isEqualTo(List.of()); + assertThat(data.get("total")).isEqualTo(0); + } + + @Test + void monitorOnlyKeepsRepeatedAsins() { + Map data = service.itemsData(view, null, 1, 20, "monitor", noFilter()); + assertThat(data.get("total")).isEqualTo(3); + @SuppressWarnings("unchecked") + List shops = (List) data.get("shops"); + assertThat(shops.stream().map(DuplicateShop::shopName).toList()) + .containsExactly("ShopA", "ShopB", "ShopC", "ShopD"); + } + + @Test + void itemFilterByAsinAndSite() { + ShopDataDuplicateCheckQueryService.Filters filters = + ShopDataDuplicateCheckQueryService.Filters.clean("A0000001", "", "", "UK", "", ""); + Map data = service.itemsData(view, null, 1, 20, "all", filters); + assertThat(data.get("total")).isEqualTo(1); + } + + @Test + void detailSortedByShopCountThenEarliestShelfDesc() { + Map data = service.detailData(view, null, 1, 24, noFilter()); + assertThat(data.get("total")).isEqualTo(3); + @SuppressWarnings("unchecked") + List> items = (List>) data.get("items"); + assertThat(items).extracting(map -> map.get("asin")).containsExactly("E0000001", "A0000001", "F0000001"); + Map e = items.get(0); + assertThat(e.get("first_date")).isEqualTo("2026-08-29 04:34:00"); + assertThat(e.get("brand")).isEqualTo("BrandE"); + assertThat(e.get("shop_names")).isEqualTo(List.of("ShopA", "ShopB", "ShopC")); + @SuppressWarnings("unchecked") + List occurrences = (List) e.get("occurrences"); + assertThat(occurrences).extracting(DuplicateOccurrence::date) + .containsExactly("2026年8月29日 上午4:34", "2026-08-30", "2026-08-31"); + } + + @Test + void detailLeaderCropRemovesSingleShopAsin() { + Set visible = Set.of("shopa", "shopb"); + Map data = service.detailData(view, visible, 1, 24, noFilter()); + @SuppressWarnings("unchecked") + List> items = (List>) data.get("items"); + // E 的 ShopC 记录被裁掉后剩 2 店；A/E 同为 2 店，按"最早上架晚者在前" A 早于 E + assertThat(items).extracting(map -> map.get("asin")).containsExactly("A0000001", "E0000001"); + } + + @Test + void pendingShapes() { + Map overview = service.pendingOverview(); + assertThat(overview.get("pending")).isEqualTo(true); + assertThat(overview.get("summary")).isEqualTo(Map.of()); + Map list = service.pendingList(1, 20); + assertThat(list.get("total")).isEqualTo(0); + assertThat(list.get("scanned_at")).isEqualTo(""); + } + + @Test + void exportCsvBomHeaderAndMonitorRows() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + service.writeExport(view, null, "monitor", noFilter(), out); + byte[] bytes = out.toByteArray(); + assertThat(bytes[0]).isEqualTo((byte) 0xEF); + assertThat(bytes[1]).isEqualTo((byte) 0xBB); + assertThat(bytes[2]).isEqualTo((byte) 0xBF); + String text = new String(bytes, StandardCharsets.UTF_8); + String[] lines = text.split("\r\n"); + assertThat(lines[0]).endsWith("ASIN,店铺数,店铺,分组,站点,上架时间,价格,品牌"); + // 数据行 = E(3 记录) + A(2) + F(2) = 7，非空行总数为 8（尾随 CRLF 会产生一个空元素） + long nonEmpty = java.util.Arrays.stream(lines).filter(line -> !line.isEmpty()).count(); + assertThat(nonEmpty).isEqualTo(8); + assertThat(lines[1]).startsWith("E0000001,3,"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckAggregatorTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckAggregatorTest.java new file mode 100644 index 00000000..8d65880d --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckAggregatorTest.java @@ -0,0 +1,53 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import com.nanri.aiimage.modules.shopduplicatecheck.TestFixtures; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateItem; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateOccurrence; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary; +import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateShop; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class DuplicateCheckAggregatorTest { + + @Test + void aggregatesFixtureExactlyLikePython() { + DuplicateCheckAggregator.Aggregate aggregate = + DuplicateCheckAggregator.aggregate(TestFixtures.shopParsed(), "job"); + DuplicateScanSummary summary = aggregate.summary(); + assertThat(summary.shopCount()).isEqualTo(4); + assertThat(summary.asinTotal()).isEqualTo(6); + assertThat(summary.recordTotal()).isEqualTo(10); + assertThat(summary.duplicateAsinTotal()).isEqualTo(3); + assertThat(summary.duplicateShopCount()).isEqualTo(4); + assertThat(summary.siteCount()).isEqualTo(3); + assertThat(summary.asinPerShop()).isEqualTo(2.5); + assertThat(summary.source()).isEqualTo("job"); + + List shops = aggregate.payload().shops(); + assertThat(shops.stream().map(DuplicateShop::shopName).toList()) + .containsExactly("ShopA", "ShopB", "ShopC", "ShopD"); + assertThat(shops.stream().map(DuplicateShop::asinCount).toList()) + .containsExactly(3, 3, 2, 2); + + List items = aggregate.payload().items(); + // 扫描排序：店铺数倒序 → ASIN 升序 + assertThat(items.stream().map(DuplicateItem::asin).toList()) + .containsExactly("E0000001", "A0000001", "F0000001", "B0000001", "C0000001", "D0000001"); + } + + @Test + void occurrenceCarriesSheetCountry() { + DuplicateCheckAggregator.Aggregate aggregate = + DuplicateCheckAggregator.aggregate(TestFixtures.shopParsed(), "job"); + DuplicateItem e = aggregate.payload().items().get(0); + assertThat(e.shopCount()).isEqualTo(3); + assertThat(e.recordCount()).isEqualTo(3); + assertThat(e.occurrences().stream() + .map(DuplicateOccurrence::country).toList()) + .containsExactlyInAnyOrder("UK", "UK", "FR"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckCsvWriterTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckCsvWriterTest.java new file mode 100644 index 00000000..f579f84c --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckCsvWriterTest.java @@ -0,0 +1,32 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class DuplicateCheckCsvWriterTest { + + @Test + void writesBomCrlfAndMinimalQuoting() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DuplicateCheckCsvWriter.write(out, List.of( + List.of("A", "B"), + List.of("with,comma", "say \"hi\"", "two\nlines"))); + byte[] bytes = out.toByteArray(); + assertThat(bytes[0]).isEqualTo((byte) 0xEF); + assertThat(bytes[1]).isEqualTo((byte) 0xBB); + assertThat(bytes[2]).isEqualTo((byte) 0xBF); + String text = new String(bytes, StandardCharsets.UTF_8); + // BOM 为首个字符，其后紧跟表头 + assertThat((int) text.charAt(0)).isEqualTo(0xFEFF); + assertThat(text.charAt(1)).isEqualTo('A'); + assertThat(text).contains("A,B\r\n"); + assertThat(text).contains("\"with,comma\""); + assertThat(text).contains("\"say \"\"hi\"\"\""); + assertThat(text).contains("\"two\nlines\""); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckTimeNormalizerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckTimeNormalizerTest.java new file mode 100644 index 00000000..44f600b1 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/support/DuplicateCheckTimeNormalizerTest.java @@ -0,0 +1,49 @@ +package com.nanri.aiimage.modules.shopduplicatecheck.service.support; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class DuplicateCheckTimeNormalizerTest { + + @Test + void normalizesPythonSortTimeCases() { + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026-08-30")) + .isEqualTo("2026-08-30 00:00:00"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026-08-30 08:30:00")) + .isEqualTo("2026-08-30 08:30:00"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026年8月29日 上午4:34")) + .isEqualTo("2026-08-29 04:34:00"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026年8月29日 下午2:15")) + .isEqualTo("2026-08-29 14:15:00"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026年8月29日 晚上8:00")) + .isEqualTo("2026-08-29 20:00:00"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026年8月29日 凌晨12:05")) + .isEqualTo("2026-08-29 00:05:00"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026/8/18 4:34")) + .isEqualTo("2026-08-18 04:34:00"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("未知日期")) + .isEqualTo("未知日期"); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("")) + .isEqualTo(""); + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime(null)) + .isEqualTo(""); + } + + @Test + void invalidCalendarDateFallsBackToOriginal() { + assertThat(DuplicateCheckTimeNormalizer.normalizeSortTime("2026-13-01 04:34")) + .isEqualTo("2026-13-01 04:34"); + } + + @Test + void dateKeyNormalizesCommonFormats() { + assertThat(DuplicateCheckTimeNormalizer.dateKey("2026-08-30T08:30")).isEqualTo("2026-08-30"); + assertThat(DuplicateCheckTimeNormalizer.dateKey("2026-08-30 08:30:00")).isEqualTo("2026-08-30"); + assertThat(DuplicateCheckTimeNormalizer.dateKey("2026年8月19日 上午4:34")).isEqualTo("2026-08-19"); + assertThat(DuplicateCheckTimeNormalizer.dateKey("2026.8.19")).isEqualTo("2026-08-19"); + assertThat(DuplicateCheckTimeNormalizer.dateKey("2026/8/19")).isEqualTo("2026-08-19"); + assertThat(DuplicateCheckTimeNormalizer.dateKey("abc")).isEqualTo("abc"); + assertThat(DuplicateCheckTimeNormalizer.dateKey("")).isEqualTo(""); + } +} diff --git a/backend/app.py b/backend/app.py index dae5a219..89f4474b 100644 --- a/backend/app.py +++ b/backend/app.py @@ -12,7 +12,7 @@ from flask_cors import CORS from utils.db import init_db from blueprints.auth import auth from blueprints.main import main -from blueprints.admin_api import admin_api, start_duplicate_scan_scheduler +from blueprints.admin_api import admin_api from blueprints.version import version_bp BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -50,8 +50,6 @@ app.register_blueprint(version_bp) def run_app(host='0.0.0.0', port=15124): init_db() - # 每日凌晨全量扫描重复 ASIN，结果缓存 MySQL，页面读取缓存不再实时拉取 Excel - start_duplicate_scan_scheduler(app) app.run(host=host, port=port, threaded=True, use_reloader=False) diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index 42a567ee..5a58effd 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -15,8 +15,6 @@ from pathlib import Path from urllib.parse import quote import requests -from openpyxl import load_workbook -from openpyxl.utils.exceptions import InvalidFileException from requests.adapters import HTTPAdapter from flask import ( Blueprint, @@ -436,7 +434,7 @@ class _PermissionProxyError(Exception): def _proxy_permission_java( - method, path, *, params=None, json_data=None, files=None, data=None, current_row=None): + method, path, *, params=None, json_data=None, files=None, data=None, current_row=None, timeout=10): """Call Java permission APIs using either forwarded JWT or trusted Flask identity.""" proxy_params = {} request_row = getattr(g, '_current_user_row', None) if has_request_context() else None @@ -458,6 +456,7 @@ def _proxy_permission_java( files=files, data=data, headers=headers, + timeout=timeout, ) return result, error_response, status @@ -1826,477 +1825,6 @@ def _load_shop_data_crawl_download_rows(result_ids): conn.close() -_SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES = 256 * 1024 * 1024 -_SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT = (10, 60) - -# 重复 ASIN 扫描缓存：结果存 MySQL（shop_data_duplicate_scan 表）， -# 定时任务每天凌晨执行一次全量扫描，页面读取最近一次结果，避免实时拉取 Excel 消耗资源。 -_DUPLICATE_SCAN_TABLE = 'shop_data_duplicate_scan' -_duplicate_scan_lock = threading.Lock() -_duplicate_scan_running = False - - -def _ensure_duplicate_scan_table(): - """按需建表：重复 ASIN 扫描结果缓存表。""" - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute( - f""" - CREATE TABLE IF NOT EXISTS {_DUPLICATE_SCAN_TABLE} ( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - status VARCHAR(16) NOT NULL, - error_text TEXT NULL, - summary_json MEDIUMTEXT NULL, - payload_json MEDIUMTEXT NULL, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - finished_at DATETIME NULL, - KEY idx_status_created (status, created_at) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 - """ - ) - conn.commit() - finally: - conn.close() - - -def _save_duplicate_scan(status, summary=None, payload=None, error_text=None): - """保存一次扫描结果记录；status: SUCCESS / FAILED。""" - import json as _json - conn = get_db() - scan_id = None - try: - with conn.cursor() as cur: - cur.execute( - f'INSERT INTO {_DUPLICATE_SCAN_TABLE} (status, error_text, summary_json, payload_json, created_at, finished_at) ' - 'VALUES (%s, %s, %s, %s, NOW(), NOW())', - ( - status, - error_text, - _json.dumps(summary or {}, ensure_ascii=False), - _json.dumps(payload or [], ensure_ascii=False), - ), - ) - scan_id = cur.lastrowid - conn.commit() - return scan_id - finally: - conn.close() - - -def _latest_duplicate_scan(): - """最近一次成功扫描记录：{'scanned_at': str, 'summary': dict, 'items': list, 'shops': list}|None。 - - 兼容两种缓存格式： - - 新版（重复检查页）：payload = {'shops': [...], 'items': [...]} - - 旧版（跨店重复明细）：payload 直接为 items 列表，shops 由明细推导。 - """ - import json as _json - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute( - f'SELECT id, summary_json, payload_json, created_at FROM {_DUPLICATE_SCAN_TABLE} ' - "WHERE status = 'SUCCESS' ORDER BY id DESC LIMIT 1" - ) - row = cur.fetchone() - finally: - conn.close() - if not row: - return None - summary_raw = row.get('summary_json') or {} - payload = row.get('payload_json') or '[]' - # summary_json / payload_json 在 MySQL 中为 JSON 字符串，读取后需反序列化 - summary = _json.loads(summary_raw) if isinstance(summary_raw, str) else (summary_raw or {}) - summary = summary if isinstance(summary, dict) else {} - payload = _json.loads(payload) if isinstance(payload, str) else (payload or []) - if isinstance(payload, dict): - shops = payload.get('shops') or [] - items = payload.get('items') or [] - else: - # 旧格式缓存：兼容旧页面，店铺概览由明细推导 - items = payload if isinstance(payload, list) else [] - shop_seen = {} - for item in items: - for occ in (item.get('occurrences') or []): - shop_name = occ.get('shop_name') or '' - shop_seen[shop_name] = shop_seen.get(shop_name, 0) + 1 - shops = [ - {'shop_name': shop_name, 'asin_count': 0, 'group_name': ''} - for shop_name in sorted(shop_seen, key=lambda name: (-shop_seen[name], name)) - ] - return { - 'scanned_at': str(row.get('created_at') or ''), - 'summary': summary, - 'items': items if isinstance(items, list) else [], - 'shops': shops if isinstance(shops, list) else [], - } - - -def _shop_data_crawl_fetch_result_bytes(row, timeout=None): - """从 Java 下载接口拉取结果文件字节流（仅内存，不落盘）。""" - # row 支持 id（下载行）或 result_id（管理列集别名） - result_id = row.get('id') or row.get('result_id') or 0 - url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(result_id)}/download" - headers, params = _backend_java_internal_request() - response = _get_backend_java_session().get( - url, - params=params, - headers=headers, - stream=True, - timeout=timeout or _SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT, - ) - try: - response.raise_for_status() - total = 0 - chunks = [] - for chunk in response.iter_content(chunk_size=1024 * 1024): - if not chunk: - continue - total += len(chunk) - if total > _SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES: - raise ValueError('结果文件过大，无法分析') - chunks.append(chunk) - raw = b''.join(chunks) - if not raw: - raise ValueError(f'结果文件为空 result_id={result_id}') - # Java 内部端点鉴权/归属失败时返回 JSON 错误体（HTTP 200），解析报错更直观 - if raw.lstrip().startswith(b'{'): - try: - err_payload = json.loads(raw.decode('utf-8', 'replace')) - message = err_payload.get('message') or err_payload.get('error') or '未知错误' - except ValueError: - message = raw[:200].decode('utf-8', 'replace') - raise ValueError(f'结果文件下载失败 result_id={result_id}: {message}') - return raw - finally: - response.close() - - -def _shop_data_crawl_cell_text(cell): - """读取单元格文本：日期/数字等统一转字符串，None 返回空串。""" - if cell is None: - return '' - value = cell.value - if value is None: - return '' - if isinstance(value, datetime): - return value.strftime('%Y-%m-%d') - if isinstance(value, float) and value.is_integer(): - return str(int(value)) - return str(value).strip() - - -def _shop_data_date_key(date_text): - """把各种日期文本归一化为 YYYY-MM-DD，供范围比较。 - - 支持：ISO（2026-08-19）、中文（2026年8月19日 上午4:34）、 - yyyy.m.d / yyyy/m/d 等常见格式；无法识别时返回原字符串。 - """ - text = (date_text or '').strip() - if not text: - return '' - if len(text) >= 10 and text[4] == '-' and text[7] == '-': - return text[:10] - year = month = day = None - m = re.search(r'(\d{4})\s*[年./-]\s*(\d{1,2})\s*[月./-]\s*(\d{1,2})', text) - if m: - year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3)) - if year and month and day: - try: - return f'{year:04d}-{month:02d}-{day:02d}' - except ValueError: - return text - return text - - -_SHOP_DATA_SHEET_COUNTRY_MAP = { - '英国': 'UK', 'uk': 'UK', 'u.k.': 'UK', 'united kingdom': 'UK', - '德国': 'DE', 'de': 'DE', 'germany': 'DE', - '法国': 'FR', 'fr': 'FR', 'france': 'FR', - '意大利': 'IT', 'it': 'IT', 'italy': 'IT', - '西班牙': 'ES', 'es': 'ES', 'spain': 'ES', -} - - -def _shop_data_sheet_country(sheet_name): - """结果文件 sheet 名 → 国家码（英国/德国/法国…）；无法识别返回空串。""" - key = str(sheet_name or '').strip().lower() - return _SHOP_DATA_SHEET_COUNTRY_MAP.get(key, '') - - -def _shop_data_crawl_parse_workbook(workbook): - """从结果 Workbook 提取 ASIN 行：字典列表，表头按 日期/ASIN/…/价格/…/品牌 识别，跳过无 ASIN 行。 - - 每行附带所在 sheet 映射的国家码 country（记录级站点，供重复检查页按站点筛选/展示）。 - """ - rows = [] - for sheet in workbook.worksheets: - header_cells = list(next(sheet.iter_rows(min_row=1, max_row=1), [])) - header = [_shop_data_crawl_cell_text(cell) for cell in header_cells] - try: - asin_col = header.index('ASIN') - except ValueError: - continue - date_col = header.index('日期') if '日期' in header else None - price_col = header.index('价格') if '价格' in header else None - brand_col = header.index('品牌') if '品牌' in header else None - sheet_country = _shop_data_sheet_country(sheet.title) - for sheet_row in sheet.iter_rows(min_row=2): - asin = _shop_data_crawl_cell_text(sheet_row[asin_col]) - if not asin: - continue - rows.append({ - 'asin': asin, - 'date': _shop_data_crawl_cell_text(sheet_row[date_col]) if date_col is not None else '', - 'price': _shop_data_crawl_cell_text(sheet_row[price_col]) if price_col is not None else '', - 'brand': _shop_data_crawl_cell_text(sheet_row[brand_col]) if brand_col is not None else '', - 'country': sheet_country, - }) - return rows - - -def _shops_with_latest_results(): - """查询每家店铺最新结果行（用于重复 ASIN 扫描）。返回 shop_items 列表。""" - conditions = [ - "r.module_type = 'SHOP_DATA_CRAWL'", - "t.module_type = 'SHOP_DATA_CRAWL'", - "TRIM(COALESCE(r.result_file_url, '')) <> ''", - ] - where_sql = ' AND '.join(conditions) - conn = get_db() - try: - with conn.cursor() as cur: - shop_key_sql = "TRIM(COALESCE(r.source_filename, ''))" - grouped_from_sql = ( - ' FROM biz_file_result r ' - 'JOIN biz_file_task t ON t.id = r.task_id ' - 'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id ' - 'LEFT JOIN users u ON u.id = r.user_id ' - 'WHERE ' + where_sql - ) - cur.execute( - 'SELECT ' + shop_key_sql + ' AS shop_name, MAX(' - + _SHOP_DATA_CRAWL_LATEST_TIME_SQL + ') AS latest_created_at' - + grouped_from_sql + - ' GROUP BY ' + shop_key_sql + - ' ORDER BY latest_created_at DESC, shop_name ASC', - ) - group_rows = cur.fetchall() - group_names = _shop_data_crawl_group_names(cur, group_rows) - result_rows_by_shop = {} - selected_shop_names = [row.get('shop_name') for row in group_rows] - if selected_shop_names: - placeholders = ','.join(['%s'] * len(selected_shop_names)) - cur.execute( - 'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS + - ', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql + - ' ORDER BY ' + _SHOP_DATA_CRAWL_LATEST_TIME_SQL - + ' DESC, r.id DESC) AS shop_row_number ' - ' FROM biz_file_result r ' - 'JOIN biz_file_task t ON t.id = r.task_id ' - 'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id ' - 'LEFT JOIN users u ON u.id = r.user_id ' - 'WHERE ' + where_sql + - f' AND {shop_key_sql} IN ({placeholders})' + - ') ranked WHERE ranked.shop_row_number <= 1 ' - 'ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC', - tuple(selected_shop_names), - ) - for row in cur.fetchall(): - shop_key = _shop_data_crawl_shop_key(row.get('shop_name')) - result_rows_by_shop.setdefault(shop_key, []).append(row) - finally: - conn.close() - - # 逐店读取结果文件并解析 - shop_items = [] - for rows in result_rows_by_shop.values(): - for row in rows: - result_id = int(row.get('result_id') or 0) - if result_id <= 0: - continue - try: - raw = _shop_data_crawl_fetch_result_bytes(row) - try: - parsed = _shop_data_crawl_parse_workbook( - load_workbook(io.BytesIO(raw), read_only=True, data_only=True)) - except (InvalidFileException, KeyError, ValueError, zipfile.BadZipFile) as exc: - current_app.logger.warning( - '[shop-data-crawl] 解析结果文件失败 result_id=%s: %s', result_id, exc) - continue - shop_items.append({ - 'shop_name': row.get('shop_name') or '未命名', - 'group_name': _shop_data_crawl_group_name(group_names, row.get('shop_name')), - 'country_codes': _shop_data_crawl_country_codes_from_json(row.get('country_codes_json')) - or _shop_data_crawl_country_codes(row.get('request_json')), - 'rows': parsed, - }) - except (requests.RequestException, ValueError) as exc: - current_app.logger.warning( - '[shop-data-crawl] 拉取结果文件失败 result_id=%s: %s', result_id, exc) - return shop_items - - -def _build_duplicate_scan_cache(shop_items, source='job'): - """由 shop_items 构建全量重复检查缓存：shops 概览 + 明细 items + 指标统计。 - - 返回 {'shops': [...], 'items': [...], 'summary': {唯一ASIN/上架记录/重复ASIN…}} - items 内 occurrences 记录带 country（结果文件 sheet 映射的站点）。 - """ - asin_occurrences = {} - shop_agg = {} - for shop_item in shop_items: - shop_name = shop_item['shop_name'] or '未命名' - group_name = shop_item['group_name'] or '' - country_codes = shop_item['country_codes'] or [] - shop_agg[shop_name] = {'group_name': group_name, 'country_codes': country_codes} - for row in shop_item['rows']: - asin = row['asin'].strip().upper() - if not asin: - continue - asin_occurrences.setdefault(asin, []).append({ - 'asin': asin, - 'date': row['date'], - 'price': row['price'], - 'brand': row['brand'], - 'shop_name': shop_name, - 'group_name': group_name, - 'country_codes': country_codes, - 'country': row.get('country') or '', - }) - items = [] - total_records = 0 - for asin, occurrences in asin_occurrences.items(): - shop_count = len({item['shop_name'] for item in occurrences}) - total_records += len(occurrences) - items.append({ - 'asin': asin, - 'shop_count': shop_count, - 'record_count': len(occurrences), - 'occurrences': occurrences, - }) - items.sort(key=lambda item: (-item['shop_count'], item['asin'])) - repeated = sum(1 for item in items if item['shop_count'] >= 2) - - shops = [] - for shop_name, agg in sorted(shop_agg.items()): - shop_asin_set = { - occ['asin'] - for occ_list in (oo for oo in asin_occurrences.values()) - for occ in occ_list - if occ['shop_name'] == shop_name - } - shops.append({ - 'shop_name': shop_name, - 'group_name': agg['group_name'], - 'country_codes': agg['country_codes'], - 'asin_count': len(shop_asin_set), - 'record_count': sum(1 for occ_list in asin_occurrences.values() for occ in occ_list if occ['shop_name'] == shop_name), - }) - shops.sort(key=lambda shop: (-shop['asin_count'], shop['shop_name'])) - - summary = { - 'shop_count': len(shops), - 'asin_total': len(items), - 'record_total': total_records, - 'duplicate_asin_total': repeated, - 'duplicate_shop_count': len({occ['shop_name'] for item in items if item['shop_count'] >= 2 for occ in item['occurrences']}), - 'site_count': len({occ['country'] for occ_list in asin_occurrences.values() for occ in occ_list if occ.get('country')}), - 'asin_per_shop': round(float(total_records) / len(shops), 1) if shops else 0.0, - 'source': source, - } - return {'shops': shops, 'items': items, 'summary': summary} - - -def _build_duplicate_scan_items(shop_items): - """由 shop_items 聚合跨店重复 ASIN 完整结果（不含筛选，返回全量列表）。 - - 保持旧接口语义（仅跨店重复），供导入模拟重分析等旧逻辑复用； - 新页面走 _build_duplicate_scan_cache 全量缓存。 - """ - cache = _build_duplicate_scan_cache(shop_items) - return cache['items'] - - -def _run_duplicate_scan_job(): - """执行一次全量重复检查扫描并落库（新格式：shops + items + summary）。返回 (ok, scanned_at, summary)。""" - global _duplicate_scan_running - if _duplicate_scan_running: - return False, '', '扫描进行中' - _duplicate_scan_running = True - try: - shop_items = _shops_with_latest_results() - cache = _build_duplicate_scan_cache(shop_items, source='job') - _save_duplicate_scan('SUCCESS', summary=cache['summary'], payload={ - 'shops': cache['shops'], - 'items': cache['items'], - }) - return True, datetime.now().strftime('%Y-%m-%d %H:%M:%S'), cache['summary'] - except Exception as exc: - current_app.logger.exception('[shop-data-crawl] 重复检查定时扫描失败: %s', exc) - try: - _save_duplicate_scan('FAILED', error_text=str(exc)) - except Exception: - current_app.logger.exception('[shop-data-crawl] 重复检查扫描失败记录落库失败') - return False, '', '执行失败：%s' % exc - finally: - _duplicate_scan_running = False - - -def start_duplicate_scan_scheduler(app): - """每天凌晨 03:10 执行一次重复 ASIN 全量扫描（后台守护线程）。""" - _ensure_duplicate_scan_table() - - def _loop(): - while True: - now = datetime.now() - # 下一个 03:10（含今天，若已过则明天） - next_run = now.replace(hour=3, minute=10, second=0, microsecond=0) - if next_run <= now: - next_run = next_run.replace(day=next_run.day) + timedelta(days=1) - delay = (next_run - now).total_seconds() - app.logger.info('[shop-data-crawl] 重复ASIN定时扫描将于 %s 执行（%d 秒后）', next_run, int(delay)) - time.sleep(delay) - try: - with app.app_context(): - ok, scanned_at, summary = _run_duplicate_scan_job() - app.logger.info('[shop-data-crawl] 重复ASIN定时扫描结束 ok=%s summary=%s', ok, summary) - except Exception as exc: - app.logger.exception('[shop-data-crawl] 重复ASIN定时扫描异常: %s', exc) - - threading.Thread(target=_loop, daemon=True, name='duplicate-scan-scheduler').start() - - -def _duplicate_check_filter_cache(cache, role, current_row): - """按角色数据范围裁剪重复检查缓存：超管返回全量，主管只返回自己组的店铺。 - - 返回 (shops, items)：shops 为可见店铺概览；items 为裁剪 occurrences 后的明细 - （裁剪后同一 ASIN 只剩 1 家店时也会保留 —— 展示完整台账，是否重复由 shop_count>=2 标记）。 - """ - visible_shops = _shop_data_managed_shop_names(role, current_row) - if visible_shops is None: - return cache.get('shops') or [], cache.get('items') or [] - shops = [] - for shop in (cache.get('shops') or []): - if _shop_data_crawl_shop_key(shop.get('shop_name')) in visible_shops: - shops.append(shop) - items = [] - for item in (cache.get('items') or []): - occurrences = [ - occ for occ in (item.get('occurrences') or []) - if _shop_data_crawl_shop_key(occ.get('shop_name')) in visible_shops - ] - if occurrences: - items.append({ - 'asin': item.get('asin'), - 'shop_count': len({occ.get('shop_name') for occ in occurrences}), - 'record_count': len(occurrences), - 'occurrences': occurrences, - }) - return shops, items - - @admin_api.route('/shop-data-crawl/duplicate-check-overview') @login_required def shop_data_crawl_duplicate_check_overview(): @@ -2307,58 +1835,17 @@ def shop_data_crawl_duplicate_check_overview(): if denied: return denied try: - force = (request.args.get('force') or '').strip() in ('1', 'true', 'yes') - if force: - if not _duplicate_scan_lock.acquire(blocking=False): - return jsonify({'success': False, 'error': '扫描进行中，请稍后刷新'}), 409 - try: - ok, scanned_at, summary = _run_duplicate_scan_job() - if not ok: - return jsonify({'success': False, 'error': summary}), 500 - cache = _latest_duplicate_scan() - finally: - _duplicate_scan_lock.release() - else: - cache = _latest_duplicate_scan() - if not cache: - return jsonify({ - 'success': True, 'pending': True, - 'scanned_at': '', 'source': '', - 'summary': {}, 'shops': [], - }) - shops, items = _duplicate_check_filter_cache(cache, role, current_row) - asin_total = len(items) - record_total = sum(item.get('record_count') or 0 for item in items) - repeated = [item for item in items if item['shop_count'] >= 2] - shop_asin_totals = {} - duplicate_shops = set() - site_codes = set() - for item in items: - occs = item.get('occurrences') or [] - for occ in occs: - shop_asin_totals[occ.get('shop_name')] = shop_asin_totals.get(occ.get('shop_name'), set()) - shop_asin_totals[occ.get('shop_name')].add(item['asin']) - if item['shop_count'] >= 2: - duplicate_shops.add(occ.get('shop_name')) - if occ.get('country'): - site_codes.add(occ['country']) - summary = { - 'shop_count': len(shops), - 'asin_total': asin_total, - 'record_total': record_total, - 'duplicate_asin_total': len(repeated), - 'duplicate_shop_count': len(duplicate_shops), - 'site_count': len(site_codes), - 'asin_per_shop': round(float(record_total) / len(shops), 1) if shops else 0.0, - 'source': (cache.get('summary') or {}).get('source', ''), - } - return jsonify({ - 'success': True, - 'pending': False, - 'scanned_at': cache['scanned_at'], - 'summary': summary, - 'shops': shops, - }) + force_raw = (request.args.get('force') or '').strip() + params = {} + if force_raw in ('1', 'true', 'yes'): + params['force'] = '1' + result, error_response, status = _proxy_permission_java( + 'GET', '/api/admin/shop-data-crawl/duplicate-check-overview', + params=params, timeout=(10, 1800)) + if error_response is not None: + return error_response, status + payload = result.get('data') or {} + return jsonify({'success': True, **payload}) except Exception as exc: return _internal_error(exc) @@ -2373,78 +1860,68 @@ def shop_data_crawl_duplicate_check_items(): if denied: return denied try: - role, current_row = get_current_admin_role() page = max(1, int(request.args.get('page', 1))) page_size = min(100, max(10, int(request.args.get('page_size', 20)))) view = (request.args.get('view') or 'monitor').strip().lower() - asin_filter = (request.args.get('asin') or '').strip().upper() - shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip() - country_filter = (request.args.get('country') or '').strip().upper() - site_filter = (request.args.get('site') or '').strip().upper() - date_from = (request.args.get('date_from') or '').strip()[:10] - date_to = (request.args.get('date_to') or '').strip()[:10] - - cache = _latest_duplicate_scan() - if not cache: - return jsonify({ - 'success': True, 'pending': True, - 'items': [], 'shops': [], - 'total': 0, 'page': page, 'page_size': page_size, - 'scanned_at': '', - }) - shops, all_items = _duplicate_check_filter_cache(cache, role, current_row) - has_filter = bool(asin_filter or shop_name_filter or country_filter or site_filter or date_from or date_to) - matched = [] - for item in all_items: - if asin_filter and asin_filter not in item['asin']: - continue - if view == 'monitor' and item['shop_count'] < 2: - continue - if not has_filter: - matched.append(item) - continue - hit = False - for occ in item['occurrences']: - if shop_name_filter and shop_name_filter.lower() not in (occ.get('shop_name') or '').lower() \ - and shop_name_filter.lower() not in (occ.get('group_name') or '').lower(): - continue - if country_filter and country_filter not in [c.upper() for c in (occ.get('country_codes') or [])]: - continue - if site_filter and site_filter not in [c.upper() for c in [occ.get('country') or '']]: - continue - row_date_key = _shop_data_date_key((occ.get('date') or '').strip()) - if date_from and row_date_key and row_date_key < date_from: - continue - if date_to and row_date_key and row_date_key > date_to: - continue - hit = True - break - if hit: - matched.append(item) - total = len(matched) - offset = (page - 1) * page_size - return jsonify({ - 'success': True, - 'pending': False, - 'items': matched[offset:offset + page_size], - 'shops': shops, - 'total': total, - 'page': page, - 'page_size': page_size, - 'scanned_at': cache['scanned_at'], - }) + params = {'page': str(page), 'pageSize': str(page_size), 'view': view} + shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip() + for key, value in ( + ('asin', (request.args.get('asin') or '').strip()), + ('shopName', shop_name), + ('country', (request.args.get('country') or '').strip()), + ('site', (request.args.get('site') or '').strip()), + ('dateFrom', (request.args.get('date_from') or '').strip()), + ('dateTo', (request.args.get('date_to') or '').strip())): + if value: + params[key] = value + result, error_response, status = _proxy_permission_java( + 'GET', '/api/admin/shop-data-crawl/duplicate-check-items', params=params) + if error_response is not None: + return error_response, status + payload = result.get('data') or {} + return jsonify({'success': True, **payload}) except ValueError as exc: return jsonify({'success': False, 'error': str(exc)}), 400 except Exception as exc: return _internal_error(exc) -def _duplicate_check_effective_site(country, country_codes): - """导出用站点：优先记录级 country，其次店铺 country_codes 汇总。""" - if country: - return country.upper() - codes = [c.upper() for c in (country_codes or []) if c] - return '、'.join(codes) if codes else '' +@admin_api.route('/shop-data-crawl/duplicate-check-detail') +@login_required +def shop_data_crawl_duplicate_check_detail(): + """撞款详情（跨店重复 ASIN 卡片区）：分页返回 shop_count>=2 的 ASIN 明细。 + + 筛选条件与矩阵接口一致（遵循当前筛选），排序：店铺数倒序 → 最早上架时间倒序 → ASIN 升序。 + """ + _, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks') + if not denied: + _, _, denied = _ensure_shop_data_crawl_data_access() + if denied: + return denied + try: + page = max(1, int(request.args.get('page', 1))) + page_size = min(24, max(1, int(request.args.get('page_size', 6)))) + params = {'page': str(page), 'pageSize': str(page_size)} + shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip() + for key, value in ( + ('asin', (request.args.get('asin') or '').strip()), + ('shopName', shop_name), + ('country', (request.args.get('country') or '').strip()), + ('site', (request.args.get('site') or '').strip()), + ('dateFrom', (request.args.get('date_from') or '').strip()), + ('dateTo', (request.args.get('date_to') or '').strip())): + if value: + params[key] = value + result, error_response, status = _proxy_permission_java( + 'GET', '/api/admin/shop-data-crawl/duplicate-check-detail', params=params) + if error_response is not None: + return error_response, status + payload = result.get('data') or {} + return jsonify({'success': True, **payload}) + except ValueError as exc: + return jsonify({'success': False, 'error': str(exc)}), 400 + except Exception as exc: + return _internal_error(exc) @admin_api.route('/shop-data-crawl/duplicate-check-export') @@ -2459,161 +1936,61 @@ def shop_data_crawl_duplicate_check_export(): try: role, current_row = get_current_admin_role() view = (request.args.get('view') or 'monitor').strip().lower() - asin_filter = (request.args.get('asin') or '').strip().upper() - shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip() - country_filter = (request.args.get('country') or '').strip().upper() - site_filter = (request.args.get('site') or '').strip().upper() - date_from = (request.args.get('date_from') or '').strip()[:10] - date_to = (request.args.get('date_to') or '').strip()[:10] - - cache = _latest_duplicate_scan() - if not cache: - return jsonify({'success': False, 'error': '暂无扫描结果，请先点击「重新分析」'}), 400 - shops, all_items = _duplicate_check_filter_cache(cache, role, current_row) - has_filter = bool(asin_filter or shop_name_filter or country_filter or site_filter or date_from or date_to) - # 展平为逐行记录（每行=一条上架记录），筛选逻辑与矩阵接口一致 - flat = [] - for item in all_items: - if asin_filter and asin_filter not in item['asin']: - continue - if view == 'monitor' and item['shop_count'] < 2: - continue - for occ in item['occurrences']: - if shop_name_filter and shop_name_filter.lower() not in (occ.get('shop_name') or '').lower() \ - and shop_name_filter.lower() not in (occ.get('group_name') or '').lower(): - continue - if country_filter and country_filter not in [c.upper() for c in (occ.get('country_codes') or [])]: - continue - if site_filter and site_filter not in [c.upper() for c in [occ.get('country') or '']]: - continue - row_date_key = _shop_data_date_key((occ.get('date') or '').strip()) - if date_from and row_date_key and row_date_key < date_from: - continue - if date_to and row_date_key and row_date_key > date_to: - continue - flat.append({ - 'asin': item['asin'], - 'shop_count': item['shop_count'], - 'shop_name': occ.get('shop_name') or '', - 'group_name': occ.get('group_name') or '', - 'country': _duplicate_check_effective_site(occ.get('country'), occ.get('country_codes')), - 'date': occ.get('date') or '', - 'price': occ.get('price') or '', - 'brand': occ.get('brand') or '', - }) - flat.sort(key=lambda r: (-r['shop_count'], r['asin'], r['shop_name'], r['date'])) - import csv - stream = io.StringIO() - writer = csv.writer(stream) - writer.writerow(['ASIN', '店铺数', '店铺', '分组', '站点', '上架时间', '价格', '品牌']) - for row in flat: - writer.writerow([row['asin'], row['shop_count'], row['shop_name'], row['group_name'], - row['country'], row['date'], row['price'], row['brand']]) - payload = ('' + stream.getvalue()).encode('utf-8') - response = Response(payload, mimetype='text/csv; charset=utf-8') - response.headers['Content-Disposition'] = 'attachment; filename="shop-data-duplicate-check.csv"' - response.headers['Cache-Control'] = 'no-store' - return response - except Exception as exc: - return _internal_error(exc) - - -@admin_api.route('/shop-data-crawl/duplicate-asins') -@login_required -def shop_data_crawl_duplicate_asins(): - """按当前筛选条件展示跨店铺重复的 ASIN 明细。 - - 默认读取最近一次成功扫描的缓存结果（定时任务每天凌晨全量扫描）， - 页面点「重新分析」携带 force=1 触发一次实时扫描（锁防并发，重复触发返回 409）。 - """ - _, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks') - if not denied: - _, _, denied = _ensure_shop_data_crawl_data_access() - if denied: - return denied - try: - page = max(1, int(request.args.get('page', 1))) - page_size = min(100, max(10, int(request.args.get('page_size', 20)))) - force = (request.args.get('force') or '').strip() in ('1', 'true', 'yes') - shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip() - country_filter = (request.args.get('country') or '').strip().upper() - asin_filter = (request.args.get('asin') or '').strip().upper() - date_from = (request.args.get('date_from') or '').strip()[:10] - date_to = (request.args.get('date_to') or '').strip()[:10] - - if force: - # 实时扫描：持锁执行，避免与定时任务/其他请求并发 - if not _duplicate_scan_lock.acquire(blocking=False): - return jsonify({'success': False, 'error': '扫描进行中，请稍后刷新'}), 409 + operator_id = (current_row or {}).get('id') + if not operator_id and has_request_context(): + operator_id = session.get('user_id') + params = {'view': view} + if operator_id: + params['operatorId'] = operator_id + shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip() + for key, value in ( + ('asin', (request.args.get('asin') or '').strip()), + ('shopName', shop_name), + ('country', (request.args.get('country') or '').strip()), + ('site', (request.args.get('site') or '').strip()), + ('dateFrom', (request.args.get('date_from') or '').strip()), + ('dateTo', (request.args.get('date_to') or '').strip())): + if value: + params[key] = value + url = f"{backend_java_base_url}/api/admin/shop-data-crawl/duplicate-check-export" + try: + resp = _get_backend_java_session().get( + url, + params=params, + headers={'X-Internal-Token': _resolve_internal_token()}, + stream=True, + timeout=(10, 1800), + ) + except requests.RequestException: + return jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502 + if resp.status_code >= 400: try: - ok, scanned_at, summary = _run_duplicate_scan_job() - if not ok: - return jsonify({'success': False, 'error': summary}), 500 - cache = _latest_duplicate_scan() + data = resp.json() + error = data.get('message') or data.get('error') or '导出失败' + except ValueError: + error = '导出失败' + resp.close() + return jsonify({'success': False, 'error': error}), resp.status_code + + headers = {} + disposition = resp.headers.get('Content-Disposition') + if disposition: + headers['Content-Disposition'] = disposition + + def generate(): + try: + for chunk in resp.iter_content(chunk_size=1024 * 1024): + if chunk: + yield chunk finally: - _duplicate_scan_lock.release() - else: - cache = _latest_duplicate_scan() + resp.close() - if not cache: - return jsonify({ - 'success': True, - 'items': [], - 'total': 0, - 'page': page, - 'page_size': page_size, - 'analyzed_shop_count': 0, - 'analyzed_result_count': 0, - 'scanned_at': '', - 'pending': True, - }) - - all_items = cache['items'] or [] - # 旧接口只展示跨店重复：由全量缓存裁剪出 shop_count>=2 的 ASIN - all_items = [item for item in all_items if (item.get('shop_count') or 0) >= 2] - has_filter = bool(asin_filter or shop_name_filter or country_filter or date_from or date_to) - matched = [] - for item in all_items: - if asin_filter and asin_filter not in item['asin']: - continue - if not has_filter: - matched.append(item) - continue - # 有筛选时：任一 occurrence 命中即保留该 ASIN（展示完整记录） - hit = False - for occ in item['occurrences']: - if shop_name_filter and shop_name_filter.lower() not in (occ.get('shop_name') or '').lower() \ - and shop_name_filter.lower() not in (occ.get('group_name') or '').lower(): - continue - if country_filter and country_filter not in [c.upper() for c in (occ.get('country_codes') or [])]: - continue - row_date_key = _shop_data_date_key((occ.get('date') or '').strip()) - if date_from and row_date_key and row_date_key < date_from: - continue - if date_to and row_date_key and row_date_key > date_to: - continue - hit = True - break - if hit: - matched.append(item) - - total_details = len(matched) - offset = (page - 1) * page_size - paged_details = matched[offset:offset + page_size] - - return jsonify({ - 'success': True, - 'items': paged_details, - 'total': total_details, - 'page': page, - 'page_size': page_size, - 'analyzed_shop_count': cache['summary'].get('shop_count', 0) if cache.get('summary') else 0, - 'analyzed_result_count': len(all_items), - 'scanned_at': cache['scanned_at'], - 'source': cache.get('summary', {}).get('source', ''), - }) - except ValueError as exc: - return jsonify({'success': False, 'error': str(exc)}), 400 + return Response( + stream_with_context(generate()), + status=resp.status_code, + headers=headers, + content_type=resp.headers.get('Content-Type', 'text/csv; charset=utf-8'), + ) except Exception as exc: return _internal_error(exc) diff --git a/backend/static/admin.js b/backend/static/admin.js index 7d92af73..51466431 100644 --- a/backend/static/admin.js +++ b/backend/static/admin.js @@ -1929,6 +1929,19 @@ var shopDataDuplicateScannedAt = ''; var shopDataDuplicateOverviewPending = false; var shopDataDuplicateExporting = false; + // 撞款详情卡片区（跨店重复 ASIN）独立分页 + var shopDataDuplicateDetailPage = 1; + var shopDataDuplicateDetailPageSize = 6; + var shopDataDuplicateDetailItems = []; + var shopDataDuplicateDetailTotal = 0; + var shopDataDuplicateDetailLoading = false; + + // 站点代码 → 中文国家名（撞款详情/抽屉展示用），未识别回退原值 + var DUP_SITE_LABELS = { DE: '德国', UK: '英国', FR: '法国', IT: '意大利', ES: '西班牙' }; + function dupSiteLabel(site) { + var key = String(site || '').trim().toUpperCase(); + return DUP_SITE_LABELS[key] || String(site || '').trim(); + } function buildShopDataDuplicateQuery(page) { var params = new URLSearchParams(); @@ -2007,14 +2020,21 @@ } var max = 0; shops.forEach(function (shop) { if (shop.asin_count > max) max = shop.asin_count; }); + // 新样式：双列网格，每格 = 店名 + 记录数 / 进度条 / N 个 ASIN（按 ASIN 数降序） barsEl.innerHTML = shops.map(function (shop) { - var width = max ? Math.max(3, Math.round(shop.asin_count / max * 100)) : 0; - return '
' + - '' + escapeHtml(shop.shop_name) + '' + - '' + - '' + shop.asin_count + '
'; + var width = max ? Math.max(2, Math.round(shop.asin_count / max * 100)) : 0; + var recordCount = Number(shop.record_count || 0); + var asinCount = Number(shop.asin_count || 0); + return '
' + + '
' + + '' + escapeHtml(shop.shop_name) + '' + + '' + recordCount.toLocaleString() + '' + + '
' + + '
' + + '
' + asinCount.toLocaleString() + ' 个 ASIN
' + + '
'; }).join(''); - noteEl.textContent = '共 ' + shops.length + ' 家店铺 · 按 ASIN 数量排序'; + noteEl.textContent = '各店铺在本批站点中的上架记录量 · 共 ' + shops.length + ' 家店铺'; } function shopDataDuplicateCellCount(item, shopName) { @@ -2070,24 +2090,25 @@ if (button) openShopDataDuplicateDrawer(button.dataset.openAsinDetail); } - // 撞款详情卡片区：同一 ASIN 在多条店铺的上架明细（仅跨店铺重复，按店铺数倒序） + // 站点徽章：中文国家（未识别回退原值） + function dupSiteBadge(site) { + return '' + + escapeHtml(dupSiteLabel(site)) + ''; + } + + // 撞款详情卡片区：同一 ASIN 在多条店铺的上架明细（独立分页，仅跨店铺重复） function renderShopDataDuplicateDetailCards() { var block = document.getElementById('dupCheckDetailBlock'); var cardsEl = document.getElementById('dupCheckDetailCards'); if (!block || !cardsEl) return; - var repeated = (shopDataDuplicateItems || []).filter(function (item) { - return Number(item.shop_count) >= 2; - }).slice().sort(function (a, b) { - return (Number(b.shop_count) - Number(a.shop_count)) || dupTimeRank(b.first_date || '') - dupTimeRank(a.first_date || ''); - }); - if (!repeated.length) { + if (!shopDataDuplicateDetailItems.length) { block.style.display = 'none'; cardsEl.innerHTML = ''; return; } block.style.display = ''; - cardsEl.innerHTML = repeated.map(function (item) { - // 按店铺分组：名称 + 次数 + 站点 + 上架时间（倒序） + cardsEl.innerHTML = shopDataDuplicateDetailItems.map(function (item) { + // 按店铺分组：名称 + 次数 + 国家 + 上架时间（倒序） var shopsMap = {}; (item.occurrences || []).forEach(function (occ) { var key = occ.shop_name || '-'; @@ -2103,7 +2124,7 @@ info.times.sort(function (a, b) { return (a < b ? 1 : (a > b ? -1 : 0)); }); info.times = Array.from(new Set(info.times)); var rowSites = Object.keys(info.sites).sort().map(function (site) { - return '' + escapeHtml(site) + ''; + return dupSiteBadge(site); }).join(''); return '' + '' + escapeHtml(info.shop) + '' + info.count + ' 次' + @@ -2119,70 +2140,130 @@ '' + '' + '' + - '' + + '' + '' + rows + '
店铺站点上架时间
店铺国家上架时间
' + ''; }).join(''); } + // 撞款详情分页加载：筛选/重置时回到第 1 页，矩阵翻页不影响详情页 + function loadShopDataDuplicateDetail(page) { + if (shopDataDuplicateDetailLoading) return; + shopDataDuplicateDetailLoading = true; + var block = document.getElementById('dupCheckDetailBlock'); + var cardsEl = document.getElementById('dupCheckDetailCards'); + var params = new URLSearchParams(); + params.set('page', String(page || 1)); + params.set('page_size', String(shopDataDuplicateDetailPageSize)); + var values = { + asin: document.getElementById('shopDataDupFilterAsin').value.trim(), + shop_name: document.getElementById('shopDataDupFilterShop').value.trim(), + country: document.getElementById('shopDataDupFilterCountry').value.trim(), + site: document.getElementById('shopDataDupFilterSite').value.trim(), + date_from: document.getElementById('shopDataDupFilterDateFrom').value, + date_to: document.getElementById('shopDataDupFilterDateTo').value + }; + Object.keys(values).forEach(function (key) { + if (values[key]) params.set(key, values[key]); + }); + cardsEl.innerHTML = '
加载中...
'; + fetch('/api/admin/shop-data-crawl/duplicate-check-detail?' + params.toString()) + .then(function (response) { return response.json(); }) + .then(function (res) { + if (!res.success) throw new Error(res.error || '加载失败'); + shopDataDuplicateDetailItems = res.items || []; + shopDataDuplicateDetailTotal = Number(res.total) || 0; + shopDataDuplicateDetailPage = res.page || 1; + renderShopDataDuplicateDetailCards(); + if (block.style.display !== 'none') { + renderPagination('dupCheckDetailPagination', shopDataDuplicateDetailTotal, + shopDataDuplicateDetailPage, shopDataDuplicateDetailPageSize, + loadShopDataDuplicateDetail); + } + }) + .catch(function (error) { + shopDataDuplicateDetailItems = []; + shopDataDuplicateDetailTotal = 0; + renderShopDataDuplicateDetailCards(); + }) + .finally(function () { + shopDataDuplicateDetailLoading = false; + }); + } + + // 抽屉徽章：基础滴状标签（按上下文附加不同配色类） + function dupDrawerBadge(text, cls) { + return '' + escapeHtml(text) + ''; + } + + // 图3 样式抽屉：ASIN + 店在售徽章 / 品牌行 / 彩色徽章组 / 按时间升序明细 + 次数列 function openShopDataDuplicateDrawer(asin) { var item = null; - (shopDataDuplicateItems || []).forEach(function (it) { if (it.asin === asin) item = it; }); + (shopDataDuplicateDetailItems || []).forEach(function (it) { if (it.asin === asin) item = it; }); + if (!item) { + (shopDataDuplicateItems || []).forEach(function (it) { if (it.asin === asin) item = it; }); + } if (!item) return; - var occurrences = item.occurrences || []; - var brand = '', dateMin = '', dateMax = '', sites = {}, prices = []; - var shopsMap = {}; - occurrences.forEach(function (occ) { - if (!brand && occ.brand) brand = occ.brand; - var d = normalizeDuplicateTime(occ.date); - if (d) { - if (!dateMin || d < dateMin) dateMin = d; - if (!dateMax || d > dateMax) dateMax = d; - } - if (occ.country) sites[occ.country] = true; - if (occ.price && prices.indexOf(occ.price) < 0) prices.push(occ.price); - var shopKey = occ.shop_name || '-'; - if (!shopsMap[shopKey]) { - shopsMap[shopKey] = { shop: occ.shop_name || '-', group: occ.group_name || '', sites: {}, times: {}, price: '', count: 0 }; - } - var info = shopsMap[shopKey]; - info.count++; - if (occ.country) info.sites[occ.country] = true; - if (d) info.times[d] = true; - if (occ.price && !info.price) info.price = occ.price; + var occurrences = (item.occurrences || []).slice(); + var brand = item.brand || ''; + // 上架时间归一化并升序排列（按时间正序展示） + occurrences.forEach(function (occ) { occ._dupTime = normalizeDuplicateTime(occ.date); }); + occurrences.sort(function (a, b) { + var ta = a._dupTime || '', tb = b._dupTime || ''; + if (ta === tb) return 0; + return ta < tb ? -1 : 1; }); - var dateRange = dateMin && dateMax ? (dateMin === dateMax ? dateMin : dateMin + ' ~ ' + dateMax) : '-'; - var priceRange = prices.length ? prices.slice(0, 4).join(' / ') + (prices.length > 4 ? ' 等' : '') : '-'; - var siteBadges = Object.keys(sites).sort().map(function (site) { - return '' + escapeHtml(site) + ''; - }).join(''); - var rows = Object.keys(shopsMap).sort().map(function (key) { - var info = shopsMap[key]; - var times = Object.keys(info.times).sort().reverse(); - var rowSites = Object.keys(info.sites).sort().map(function (site) { - return '' + escapeHtml(site) + ''; - }).join(''); + var sites = {}, firstDate = ''; + occurrences.forEach(function (occ) { + if (occ.country) sites[occ.country] = true; + if (!brand && occ.brand) brand = occ.brand; + var t = occ._dupTime; + if (t && (!firstDate || t < firstDate)) firstDate = t; + }); + // 店铺 + 国家两级聚合：次数 = 该店铺在该国家的上架记录数 + var countsMap = {}; + occurrences.forEach(function (occ) { + var shopKey = occ.shop_name || '-'; + var siteKey = (occ.country || '').trim().toUpperCase() || '-'; + var key = shopKey + '|||' + siteKey; + countsMap[key] = (countsMap[key] || 0) + 1; + }); + var rows = occurrences.map(function (occ) { + var shopKey = occ.shop_name || '-'; + var siteKey = (occ.country || '').trim().toUpperCase() || '-'; + var times = []; + if (occ._dupTime && occ._dupTime.length >= 16) times.push(occ._dupTime.slice(0, 16)); + else if (normalizeDuplicateTime(occ.date)) times.push(normalizeDuplicateTime(occ.date)); + else occ.date || ''; + var count = countsMap[shopKey + '|||' + siteKey] || 0; + var siteBadge = siteKey === '-' ? '-' : dupSiteBadge(siteKey); return '' + - '' + escapeHtml(info.shop) + '' + - '' + escapeHtml(info.group || '-') + '' + - '' + (rowSites || '-') + '' + - '' + times.map(escapeHtml).join('、') + '' + - '' + escapeHtml(info.price || '-') + '' + - '' + info.count + ''; + '' + escapeHtml(shopKey) + '' + + '' + siteBadge + '' + + '' + escapeHtml(times.join('')) + '' + + '' + count + ''; }).join(''); + // 顶部「N 店在售」红色徽章 + var shopCount = Number(item.shop_count || 0); + var recordCount = Number(item.record_count || 0); + var headerBadges = + dupDrawerBadge(shopCount + ' 店在售', 'dup-drawer-badge-danger') + + dupDrawerBadge(shopCount + ' 家店铺', 'dup-drawer-badge-blue') + + Object.keys(sites).sort().map(function (site) { + return dupDrawerBadge(dupSiteLabel(site), 'dup-site-' + site.toUpperCase()); + }).join('') + + dupDrawerBadge(recordCount + ' 次上架', 'dup-drawer-badge-green') + + (firstDate ? dupDrawerBadge('最早上架 ' + firstDate.slice(0, 16), 'dup-drawer-badge-cyan') : ''); document.getElementById('dupCheckDrawerAsin').textContent = item.asin; - document.getElementById('dupCheckDrawerSubtitle').textContent = - item.shop_count + ' 家店铺 · ' + item.record_count + ' 条上架记录'; + document.getElementById('dupCheckDrawerAsin').className = 'dup-drawer-asin'; + var subtitleEl = document.getElementById('dupCheckDrawerSubtitle'); + subtitleEl.className = 'dup-drawer-subtitle dup-drawer-inline-badges'; + subtitleEl.innerHTML = headerBadges; document.getElementById('dupCheckDrawerBody').innerHTML = - '
' + - '
品牌
' + escapeHtml(brand || '-') + '
' + - '
价格
' + escapeHtml(priceRange) + '
' + - '
日期范围
' + escapeHtml(dateRange) + '
' + - '
站点
' + (siteBadges || '-') + '
' + - '

店铺 / 站点 / 上架时间 / 次数

' + + '
品牌：' + escapeHtml(brand || '-') + '
' + '
' + - '' + - '' + rows + '
店铺分组站点上架时间价格次数
'; + '店铺国家上架时间（按时间升序）次数' + + '' + (rows || '暂无明细') + ''; document.getElementById('dupCheckDrawerMask').classList.add('show'); } @@ -2246,7 +2327,7 @@ + (shopDataDuplicateScannedAt ? ' · 扫描时间 ' + shopDataDuplicateScannedAt : (shopDataDuplicateOverviewPending ? ' · 尚无扫描结果（点击「重新分析」立即扫描）' : '')); renderShopDataDuplicateMatrix(); - renderShopDataDuplicateDetailCards(); + loadShopDataDuplicateDetail(shopDataDuplicateDetailPage || 1); renderPagination('shopDataDuplicatePagination', shopDataDuplicateTotal, shopDataDuplicatePage, shopDataDuplicatePageSize, loadShopDataDuplicateCheckItems); }) @@ -2255,7 +2336,7 @@ shopDataDuplicateTotal = 0; list.innerHTML = '
加载失败：' + escapeHtml(error.message || '') + '
'; document.getElementById('shopDataDuplicateTotal').textContent = ''; - renderShopDataDuplicateDetailCards(); + loadShopDataDuplicateDetail(1); }) .finally(function () { shopDataDuplicateLoading = false; @@ -2274,6 +2355,7 @@ monitorTab.setAttribute('aria-selected', monitorActive ? 'true' : 'false'); allTab.classList.toggle('active', !monitorActive); allTab.setAttribute('aria-selected', !monitorActive ? 'true' : 'false'); + shopDataDuplicateDetailPage = 1; loadShopDataDuplicateCheckItems(1); } @@ -2503,10 +2585,14 @@ loadShopDataCrawlTasks(1); }; document.getElementById('btnRefreshShopDataDuplicates').onclick = function () { loadShopDataDuplicateCheckOverview(true); }; - document.getElementById('btnFilterShopDataDuplicates').onclick = function () { loadShopDataDuplicateCheckItems(1); }; + document.getElementById('btnFilterShopDataDuplicates').onclick = function () { + shopDataDuplicateDetailPage = 1; + loadShopDataDuplicateCheckItems(1); + }; document.getElementById('btnResetShopDataDuplicates').onclick = function () { ['shopDataDupFilterAsin', 'shopDataDupFilterShop', 'shopDataDupFilterCountry', 'shopDataDupFilterSite', 'shopDataDupFilterDateFrom', 'shopDataDupFilterDateTo'] .forEach(function (id) { document.getElementById(id).value = ''; }); + shopDataDuplicateDetailPage = 1; loadShopDataDuplicateCheckItems(1); }; document.getElementById('dupCheckTabMonitor').onclick = function () { switchDupCheckView('monitor'); }; diff --git a/backend/tests/test_admin_shop_data_duplicate_asins.py b/backend/tests/test_admin_shop_data_duplicate_asins.py deleted file mode 100644 index c094baeb..00000000 --- a/backend/tests/test_admin_shop_data_duplicate_asins.py +++ /dev/null @@ -1,307 +0,0 @@ -"""重复 ASIN 分析接口单元测试：模拟数据库行与结果文件，验证跨店铺重复聚合逻辑。""" -import sys -import unittest -from datetime import datetime -from io import BytesIO -from pathlib import Path -from unittest.mock import patch - -from openpyxl import Workbook -from flask import Flask - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from blueprints import admin_api - - -def _make_workbook(rows_by_sheet): - """构造结果 Workbook：rows_by_sheet = {sheet名: [(日期, ASIN, 价格, 品牌), ...]}""" - wb = Workbook() - wb.remove(wb.active) - for sheet_name, rows in rows_by_sheet.items(): - ws = wb.create_sheet(sheet_name) - ws.append(['日期', 'ASIN', '商品图片', '库存销量', '销售排名', - '页面浏览量', '售出件数', '价格', '推荐报价', '品牌']) - for date, asin, price, brand in rows: - row = [date, asin, '', '', '', '', '', price, '', brand] - ws.append(row) - return wb - - -class ShopDataDuplicateAsinTest(unittest.TestCase): - def setUp(self): - self.app = Flask(__name__) - self.app.config['SECRET_KEY'] = 'test-secret' - self.group_rows = [ - {'shop_name': 'Shop A', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 1)}, - {'shop_name': 'Shop B', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 2)}, - {'shop_name': 'Shop C', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 3)}, - ] - # 每家店一个结果文件：Shop A 与 Shop B 共享 ASIN1；Shop C 单独 ASIN3 - # 国家码按店铺实际 sheet：A=UK+DE、B=仅UK、C=FR - self.shop_country = { - 'Shop A': '["UK","DE"]', - 'Shop B': '["UK"]', - 'Shop C': '["FR"]', - } - self.shop_files = { - 'Shop A': _make_workbook({ - '英国': [('2026-08-30', 'B0ABC111', 'GBP 9.99', 'BrandA'), - ('2026-08-30', 'B0UNIQUE1', 'GBP 5.00', 'BrandA')], - '德国': [('2026-08-30', 'B0ABC111', 'EUR 10.99', 'BrandA')], - }), - 'Shop B': _make_workbook({ - '英国': [('2026-08-31', 'B0ABC111', 'GBP 8.50', 'BrandA'), - ('2026-08-31', 'B0ABC222', 'GBP 12.00', 'BrandB')], - }), - 'Shop C': _make_workbook({ - '法国': [('2026-08-29', 'B0ABC333', 'EUR 7.50', 'BrandC')], - }), - } - - def _result_row(self, result_id, shop_name, country_codes_json=None): - return { - 'result_id': result_id, - 'task_id': result_id + 100, - 'user_id': 7, - 'shop_name': shop_name, - 'shop_id': shop_name.lower(), - 'task_no': f'task-{result_id}', - 'task_status': 'SUCCESS', - 'result_success': 1, - 'result_error': None, - 'task_error': None, - 'file_error': None, - 'result_file_url': f'object-{result_id}', - 'result_filename': f'result-{result_id}.xlsx', - 'result_file_size': 10, - 'row_count': 2, - 'request_json': '{}', - 'country_codes_json': country_codes_json, - 'created_at': '2026-08-31T01:15:00', - 'updated_at': '2026-08-31T01:15:00', - 'finished_at': '2026-08-31T01:15:00', - 'latest_file_updated_at': '2026-08-31T01:15:00', - 'file_job_id': None, - 'file_status': 'SUCCESS', - 'username': 'operator', - } - - class _FakeCursor: - def __init__(self, group_rows, result_rows): - self.group_rows = group_rows - self.result_rows = result_rows - self.kind = None - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def execute(self, sql, params=()): - if 'COUNT(*) AS total' in sql: - self.kind = 'count' - elif 'AS latest_created_at' in sql and 'GROUP BY' in sql: - self.kind = 'groups' - elif 'GROUP_CONCAT' in sql: - self.kind = 'group_names' - else: - self.kind = 'results' - - def fetchone(self): - return {'total': len(self.group_rows)} - - def fetchall(self): - if self.kind == 'groups': - return self.group_rows - if self.kind == 'group_names': - return [{'shop_name': row['shop_name'], 'group_name': 'Group-' + row['shop_name']} - for row in self.group_rows] - return self.result_rows - - class _FakeConnection: - def __init__(self, cursor): - self.cursor_value = cursor - - def cursor(self): - return self.cursor_value - - def close(self): - pass - - def _make_workbook_bytes(self, shop_name): - stream = BytesIO() - self.shop_files[shop_name].save(stream) - return stream.getvalue() - - def _build_cache(self): - """构造与旧扫描一致的缓存：3 家店铺、跨店重复 B0ABC111。""" - return { - 'scanned_at': '2026-09-03 03:10:00', - 'summary': {'shop_count': 3, 'total': 1}, - 'items': [{ - 'asin': 'B0ABC111', - 'shop_count': 2, - 'record_count': 3, - 'occurrences': [ - {'asin': 'B0ABC111', 'date': '2026-08-30', 'price': 'GBP 9.99', 'brand': 'BrandA', - 'shop_name': 'Shop A', 'group_name': 'Group-Shop A', 'country_codes': ['UK', 'DE']}, - {'asin': 'B0ABC111', 'date': '2026-08-30', 'price': 'EUR 10.99', 'brand': 'BrandA', - 'shop_name': 'Shop A', 'group_name': 'Group-Shop A', 'country_codes': ['DE']}, - {'asin': 'B0ABC111', 'date': '2026-08-31', 'price': 'GBP 8.50', 'brand': 'BrandA', - 'shop_name': 'Shop B', 'group_name': 'Group-Shop B', 'country_codes': ['UK']}, - ], - }], - } - - def _run_request(self, query='', cache=None): - # 界面默认读缓存；force 走实时扫描（需 mock 扫描核心） - with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-asins?' + query): - with patch('utils.auth.session', {'user_id': 1}), \ - patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_api, '_latest_duplicate_scan', return_value=cache), \ - patch.object(admin_api, '_ensure_backend_menu_access', return_value=(None, None, None)), \ - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=(None, None, None)): - return admin_api.shop_data_crawl_duplicate_asins() - - def test_detects_duplicate_asins_across_shops(self): - response = self._run_request('page=1&page_size=10', cache=self._build_cache()) - self.assertEqual(response.status_code, 200) - body = response.get_json() - self.assertTrue(body['success']) - self.assertEqual(body['total'], 1) # 只有 B0ABC111 跨店重复 - self.assertEqual(body['analyzed_shop_count'], 3) - item = body['items'][0] - self.assertEqual(item['asin'], 'B0ABC111') - self.assertEqual(item['shop_count'], 2) - self.assertEqual(item['record_count'], 3) # ShopA 英国/德国 + ShopB 英国 共 3 条 - shops = {occ['shop_name'] for occ in item['occurrences']} - self.assertEqual(shops, {'Shop A', 'Shop B'}) - # 国家与日期从行/表头正确映射 - shop_a = next(occ for occ in item['occurrences'] if occ['shop_name'] == 'Shop A') - self.assertIn('UK', shop_a['country_codes']) - self.assertEqual(shop_a['date'], '2026-08-30') - self.assertEqual(body['scanned_at'], '2026-09-03 03:10:00') - - def test_no_cache_returns_pending_empty(self): - # 无缓存时（定时任务尚未执行）：返回空 + pending 提示 - response = self._run_request('page=1&page_size=10', cache=None) - body = response.get_json() - self.assertTrue(body['success']) - self.assertEqual(body['total'], 0) - self.assertEqual(body['items'], []) - self.assertTrue(body['pending']) - - def test_cache_filters_match_live_semantics(self): - cache = self._build_cache() - # 店铺过滤 Shop B：命中 → 完整记录 - response = self._run_request('page=1&page_size=10&shop_name=Shop+B', cache=cache) - body = response.get_json() - self.assertEqual(body['total'], 1) - self.assertEqual(body['items'][0]['asin'], 'B0ABC111') - # 日期范围 08-31 命中 - response = self._run_request('page=1&page_size=10&date_from=2026-08-31&date_to=2026-08-31', cache=cache) - body = response.get_json() - self.assertEqual(body['total'], 1) - # 国家 FR 无命中 - response = self._run_request('page=1&page_size=10&country=FR', cache=cache) - body = response.get_json() - self.assertEqual(body['total'], 0) - - def test_force_runs_live_scan(self): - # force=1 触发实时扫描并保存缓存后返回 - cursor = self._FakeCursor( - self.group_rows, - [self._result_row(i + 1, row['shop_name'], self.shop_country[row['shop_name']]) - for i, row in enumerate(self.group_rows)], - ) - connection = self._FakeConnection(cursor) - with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-asins?page=1&page_size=10&force=1'): - with patch('utils.auth.session', {'user_id': 1}), \ - patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_api, 'get_db', return_value=connection), \ - patch.object(admin_api, '_ensure_backend_menu_access', return_value=(None, None, None)), \ - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=(None, None, None)), \ - patch.object(admin_api, '_save_duplicate_scan', return_value=1), \ - patch.object(admin_api, '_latest_duplicate_scan', - return_value=self._build_cache()), \ - patch.object(admin_api, '_shop_data_crawl_fetch_result_bytes', - side_effect=lambda row: self._make_workbook_bytes(row['shop_name'])): - response = admin_api.shop_data_crawl_duplicate_asins() - self.assertIsNotNone(response) - body = response.get_json() - self.assertTrue(body['success']) - self.assertEqual(body['total'], 1) - self.assertEqual(body['items'][0]['asin'], 'B0ABC111') - - def test_force_conflict_when_lock_held(self): - # 锁被占（定时任务/他请求在扫）时 force 返回 409 - cursor = self._FakeCursor(self.group_rows[:0], []) - connection = self._FakeConnection(cursor) - with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-asins?page=1&page_size=10&force=1'): - with patch('utils.auth.session', {'user_id': 1}), \ - patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_api, 'get_db', return_value=connection), \ - patch.object(admin_api, '_ensure_backend_menu_access', return_value=(None, None, None)), \ - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=(None, None, None)), \ - patch.object(admin_api, '_duplicate_scan_lock'): - admin_api._duplicate_scan_lock.acquire.return_value = False - response = admin_api.shop_data_crawl_duplicate_asins() - # 409 返回 (jsonify, status) tuple - self.assertEqual(response[1], 409) - - def test_latest_duplicate_scan_deserializes_json_columns(self): - """真实读库：summary_json / payload_json 为 JSON 字符串，需反序列化为 dict/list。""" - class _ScanCursor(self._FakeCursor): - def __init__(self): - self.kind = None - - def execute(self, sql, params=()): - self.kind = 'scan' - - def fetchone(self): - return { - 'id': 9, - 'summary_json': '{"shop_count": 3, "total": 1, "source": "import"}', - 'payload_json': '[{"asin": "B0ABC111", "shop_count": 2}]', - 'created_at': datetime(2026, 9, 3, 13, 35, 25), - } - - class _ScanConnection(self._FakeConnection): - def cursor(self): - return _ScanCursor() - - with patch.object(admin_api, 'get_db', return_value=_ScanConnection(None)): - cache = admin_api._latest_duplicate_scan() - self.assertIsNotNone(cache) - self.assertEqual(cache['summary']['shop_count'], 3) - self.assertEqual(cache['summary']['source'], 'import') - self.assertEqual(cache['items'][0]['asin'], 'B0ABC111') - self.assertEqual(cache['scanned_at'], '2026-09-03 13:35:25') - - def test_parse_workbook_skips_unknown_sheets(self): - wb = _make_workbook({'英国': [('2026-08-30', 'B0TEST01', 'GBP 1.00', '')]}) - # 手工追加一个无标准表头的 sheet，模拟未知表 - ws = wb.create_sheet('未知表') - ws.append(['随便', '某列']) - ws.append(['2026-08-30', 'B0NOHEADER']) - rows = admin_api._shop_data_crawl_parse_workbook(wb) - self.assertEqual([row['asin'] for row in rows], ['B0TEST01']) - - def test_date_key_normalizes_chinese_datetime(self): - # 中文日期归一化为 ISO 供范围比较 - cases = { - '2026年8月18日 上午4:34': '2026-08-18', - '2026年8月19日 05:48': '2026-08-19', - '2026-08-19': '2026-08-19', - '2026.8.19': '2026-08-19', - '2026/8/19': '2026-08-19', - '': '', - 'abc': 'abc', # 无法识别时原样返回，范围比较自然失败 - } - for raw, expected in cases.items(): - self.assertEqual(admin_api._shop_data_date_key(raw), expected) - - -if __name__ == '__main__': - unittest.main() diff --git a/backend/tests/test_admin_shop_data_duplicate_check.py b/backend/tests/test_admin_shop_data_duplicate_check.py index 016f40ab..4c834224 100644 --- a/backend/tests/test_admin_shop_data_duplicate_check.py +++ b/backend/tests/test_admin_shop_data_duplicate_check.py @@ -1,4 +1,11 @@ -"""店铺数据重复检查接口单元测试：全量缓存格式、矩阵分页、主管/超管数据范围过滤。""" +"""店铺数据重复检查接口（撞款 duplicate-check）转发契约测试。 + +四个端点已迁移到 Java（/api/admin/shop-data-crawl/duplicate-check-{overview,items,detail,export}）， +Flask 侧仅做：本地菜单/数据权限预检 → 带 operatorId + X-Internal-Token 转发 Java → +把 Java ApiResponse.data 原样透传并加 success 包装。本文件验证透传形状、参数名映射与错误码映射； +筛选/裁剪/排序/统计语义由 backend-java 模块的 Java 单测覆盖。 +""" +import io import sys import unittest from pathlib import Path @@ -10,215 +17,309 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from blueprints import admin_api -def _shop(name, group, country_codes=None): - return { - 'shop_name': name, - 'group_name': group, - 'country_codes': country_codes or [], - 'rows': [], - } +def _java_ok(data): + """模拟 Java ApiResponse 成功体：{'success': True, 'data': {...}, 'message': ...}。""" + return {'success': True, 'data': data, 'message': '操作成功'}, None, 200 -class DuplicateCheckApiTest(unittest.TestCase): - """直接对接口函数做单元测试：缓存通过 _latest_duplicate_scan mock 注入。""" +def _java_fail_json(code, message): + """构建转发失败态：error_response 为 Flask jsonify 对象（需在请求上下文内调用）。""" + return ({'success': False, 'message': message, 'code': code}, + admin_api.jsonify({'success': False, 'error': message}), + code) + + +class DuplicateCheckProxyTest(unittest.TestCase): def setUp(self): self.app = Flask(__name__) self.app.config['SECRET_KEY'] = 'test-secret' - # 4 家店 / 3 个组；ASIN 分布： - # A1: ShopA(UK) + ShopB(UK) —— 跨店重复 - # B1: ShopA(DE) 唯一 - # C1: ShopC(FR) 唯一 - # D1: ShopD(UK) 唯一 - self.cache = { + self.shops = [ + {'shop_name': 'ShopA', 'group_name': 'GroupA', 'country_codes': ['UK', 'DE'], 'asin_count': 3, 'record_count': 3}, + {'shop_name': 'ShopB', 'group_name': 'GroupA', 'country_codes': ['UK'], 'asin_count': 3, 'record_count': 3}, + {'shop_name': 'ShopC', 'group_name': 'GroupC', 'country_codes': ['FR'], 'asin_count': 2, 'record_count': 2}, + {'shop_name': 'ShopD', 'group_name': 'GroupD', 'country_codes': ['UK'], 'asin_count': 2, 'record_count': 2}, + ] + self.items_all = [ + {'asin': 'E0000001', 'shop_count': 3, 'record_count': 3, 'occurrences': [ + {'asin': 'E0000001', 'date': '2026年8月29日 上午4:34', 'price': 'GBP 12.00', 'brand': 'BrandE', + 'shop_name': 'ShopA', 'group_name': 'GroupA', 'country_codes': ['UK'], 'country': 'UK'}, + {'asin': 'E0000001', 'date': '2026-08-30', 'price': 'GBP 12.00', 'brand': 'BrandE', + 'shop_name': 'ShopB', 'group_name': 'GroupA', 'country_codes': ['UK'], 'country': 'UK'}, + {'asin': 'E0000001', 'date': '2026-08-31', 'price': 'EUR 12.00', 'brand': 'BrandE', + 'shop_name': 'ShopC', 'group_name': 'GroupC', 'country_codes': ['FR'], 'country': 'FR'}, + ]}, + {'asin': 'A0000001', 'shop_count': 2, 'record_count': 2, 'occurrences': [ + {'asin': 'A0000001', 'date': '2026-08-30', 'price': 'GBP 9.99', 'brand': 'BrandA', + 'shop_name': 'ShopA', 'group_name': 'GroupA', 'country_codes': ['UK', 'DE'], 'country': 'UK'}, + {'asin': 'A0000001', 'date': '2026-08-30 08:30:00', 'price': 'GBP 8.50', 'brand': 'BrandA', + 'shop_name': 'ShopB', 'group_name': 'GroupA', 'country_codes': ['UK'], 'country': 'UK'}, + ]}, + {'asin': 'F0000001', 'shop_count': 2, 'record_count': 2, 'occurrences': [ + {'asin': 'F0000001', 'date': '2026-08-28', 'price': 'GBP 6.00', 'brand': 'BrandF', + 'shop_name': 'ShopB', 'group_name': 'GroupA', 'country_codes': ['UK'], 'country': 'UK'}, + {'asin': 'F0000001', 'date': '2026-08-30', 'price': 'GBP 6.00', 'brand': 'BrandF', + 'shop_name': 'ShopD', 'group_name': 'GroupD', 'country_codes': ['UK'], 'country': 'UK'}, + ]}, + {'asin': 'B0000001', 'shop_count': 1, 'record_count': 1, 'occurrences': [ + {'asin': 'B0000001', 'date': '2026-08-30', 'price': 'EUR 10.99', 'brand': 'BrandA', + 'shop_name': 'ShopA', 'group_name': 'GroupA', 'country_codes': ['DE'], 'country': 'DE'}, + ]}, + {'asin': 'C0000001', 'shop_count': 1, 'record_count': 1, 'occurrences': [ + {'asin': 'C0000001', 'date': '2026-08-29', 'price': 'EUR 7.50', 'brand': 'BrandC', + 'shop_name': 'ShopC', 'group_name': 'GroupC', 'country_codes': ['FR'], 'country': 'FR'}, + ]}, + {'asin': 'D0000001', 'shop_count': 1, 'record_count': 1, 'occurrences': [ + {'asin': 'D0000001', 'date': '2026-08-28', 'price': 'GBP 5.00', 'brand': 'BrandD', + 'shop_name': 'ShopD', 'group_name': 'GroupD', 'country_codes': ['UK'], 'country': 'UK'}, + ]}, + ] + self.overview_all = { + 'pending': False, 'scanned_at': '2026-09-04 03:10:00', 'summary': { - 'shop_count': 4, 'asin_total': 4, 'record_total': 5, - 'duplicate_asin_total': 1, 'duplicate_shop_count': 2, - 'site_count': 3, 'asin_per_shop': 1.2, 'source': 'job', + 'shop_count': 4, 'asin_total': 6, 'record_total': 10, + 'duplicate_asin_total': 3, 'duplicate_shop_count': 4, + 'site_count': 3, 'asin_per_shop': 2.5, 'source': 'job', }, - 'shops': [ - {'shop_name': 'ShopA', 'group_name': 'GroupA', 'country_codes': ['UK', 'DE'], 'asin_count': 2, 'record_count': 3}, - {'shop_name': 'ShopB', 'group_name': 'GroupA', 'country_codes': ['UK'], 'asin_count': 1, 'record_count': 1}, - {'shop_name': 'ShopC', 'group_name': 'GroupC', 'country_codes': ['FR'], 'asin_count': 1, 'record_count': 1}, - {'shop_name': 'ShopD', 'group_name': 'GroupD', 'country_codes': ['UK'], 'asin_count': 1, 'record_count': 1}, - ], - 'items': [ - {'asin': 'A0000001', 'shop_count': 2, 'record_count': 2, 'occurrences': [ - {'asin': 'A0000001', 'date': '2026-08-30', 'price': 'GBP 9.99', 'brand': 'BrandA', - 'shop_name': 'ShopA', 'group_name': 'GroupA', 'country_codes': ['UK', 'DE'], 'country': 'UK'}, - {'asin': 'A0000001', 'date': '2026-08-31', 'price': 'GBP 8.50', 'brand': 'BrandA', - 'shop_name': 'ShopB', 'group_name': 'GroupA', 'country_codes': ['UK'], 'country': 'UK'}, - ]}, - {'asin': 'B0000001', 'shop_count': 1, 'record_count': 1, 'occurrences': [ - {'asin': 'B0000001', 'date': '2026-08-30', 'price': 'EUR 10.99', 'brand': 'BrandA', - 'shop_name': 'ShopA', 'group_name': 'GroupA', 'country_codes': ['DE'], 'country': 'DE'}, - ]}, - {'asin': 'C0000001', 'shop_count': 1, 'record_count': 1, 'occurrences': [ - {'asin': 'C0000001', 'date': '2026-08-29', 'price': 'EUR 7.50', 'brand': 'BrandC', - 'shop_name': 'ShopC', 'group_name': 'GroupC', 'country_codes': ['FR'], 'country': 'FR'}, - ]}, - {'asin': 'D0000001', 'shop_count': 1, 'record_count': 1, 'occurrences': [ - {'asin': 'D0000001', 'date': '2026-08-28', 'price': 'GBP 5.00', 'brand': 'BrandD', - 'shop_name': 'ShopD', 'group_name': 'GroupD', 'country_codes': ['UK'], 'country': 'UK'}, - ]}, - ], + 'shops': self.shops, } - def _access_patches(self, role='super_admin', current_row=None): - return [ - patch.object(admin_api, '_ensure_backend_menu_access', - return_value=(role, current_row or {'id': 1}, None)), - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', - return_value=(role, current_row or {'id': 1}, None)), - patch.object(admin_api, '_latest_duplicate_scan', return_value=self.cache), - ] + def _patched(self, data): + return patch.object(admin_api, '_proxy_backend_java', side_effect=lambda *a, **k: _java_ok(data)) - def _call(self, url, role='super_admin', current_row=None): + def _call(self, view_name, url, data, role='super_admin', current_row=None): with self.app.test_request_context(url): - with patch('utils.auth.session', {'user_id': 1}), \ - patch('utils.auth.is_session_user_valid', return_value=True): - for p in self._access_patches(role, current_row): - p.start() - try: - return admin_api.shop_data_crawl_duplicate_check_items() - finally: - for p in reversed(self._access_patches(role, current_row)): - p.stop() - - def test_overview_super_admin_sees_all(self): - with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-overview'): with patch('utils.auth.session', {'user_id': 1}), \ patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_api, '_ensure_backend_menu_access', return_value=('super_admin', {'id': 1}, None)), \ - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=('super_admin', {'id': 1}, None)), \ - patch.object(admin_api, '_latest_duplicate_scan', return_value=self.cache): - response = admin_api.shop_data_crawl_duplicate_check_overview() + patch.object(admin_api, '_ensure_backend_menu_access', + return_value=(role, current_row or {'id': 1}, None)), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', + return_value=(role, current_row or {'id': 1}, None)), \ + self._patched(data): + return getattr(admin_api, view_name)() + + def test_overview_super_admin_passthrough(self): + response = self._call('shop_data_crawl_duplicate_check_overview', + '/api/admin/shop-data-crawl/duplicate-check-overview', + self.overview_all) body = response.get_json() self.assertTrue(body['success']) self.assertFalse(body['pending']) - self.assertEqual(body['summary']['asin_total'], 4) - self.assertEqual(body['summary']['record_total'], 5) - self.assertEqual(body['summary']['duplicate_asin_total'], 1) + self.assertEqual(body['summary']['asin_total'], 6) self.assertEqual(len(body['shops']), 4) - def test_items_matix_columns_are_shops(self): - response = self._call('/api/admin/shop-data-crawl/duplicate-check-items?page=1&page_size=20&view=monitor') + def test_overview_pending_empty_summary(self): + response = self._call('shop_data_crawl_duplicate_check_overview', + '/api/admin/shop-data-crawl/duplicate-check-overview', + {'pending': True, 'scanned_at': '', 'summary': {}, 'shops': []}) body = response.get_json() self.assertTrue(body['success']) - self.assertEqual(body['total'], 1) # monitor 只保留跨店重复 - self.assertEqual(body['items'][0]['asin'], 'A0000001') + self.assertTrue(body['pending']) + self.assertEqual(body['summary'], {}) + + def test_overview_force_passed_through(self): + captured = {} + data = dict(self.overview_all) + + def side_effect(*args, **kwargs): + captured['params'] = kwargs.get('params') + captured['timeout'] = kwargs.get('timeout') + return _java_ok(data) + + with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-overview?force=1'): + with patch('utils.auth.session', {'user_id': 1}), \ + patch('utils.auth.is_session_user_valid', return_value=True), \ + patch.object(admin_api, '_ensure_backend_menu_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_proxy_backend_java', side_effect=side_effect): + response = admin_api.shop_data_crawl_duplicate_check_overview() + body = response.get_json() + self.assertTrue(body['success']) + self.assertEqual((captured['params'] or {}).get('force'), '1') + # force 同步扫描需要长超时 + self.assertEqual(captured['timeout'], (10, 1800)) + + def test_overview_scan_conflict_409(self): + def side_effect(*args, **kwargs): + return _java_fail_json(409, '扫描进行中，请稍后刷新') + + with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-overview?force=1'): + with patch('utils.auth.session', {'user_id': 1}), \ + patch('utils.auth.is_session_user_valid', return_value=True), \ + patch.object(admin_api, '_ensure_backend_menu_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_proxy_backend_java', side_effect=side_effect): + response = admin_api.shop_data_crawl_duplicate_check_overview() + self.assertEqual(response[1], 409) + self.assertFalse(response[0].get_json()['success']) + self.assertEqual(response[0].get_json()['error'], '扫描进行中，请稍后刷新') + + def test_items_matrix_monitor_total_and_columns(self): + data = {'pending': False, 'items': self.items_all[:3], 'shops': self.shops, + 'total': 3, 'page': 1, 'page_size': 20, 'scanned_at': '2026-09-04 03:10:00'} + response = self._call('shop_data_crawl_duplicate_check_items', + '/api/admin/shop-data-crawl/duplicate-check-items?page=1&page_size=20&view=monitor', + data) + body = response.get_json() + self.assertTrue(body['success']) + self.assertEqual(body['total'], 3) self.assertEqual([shop['shop_name'] for shop in body['shops']], ['ShopA', 'ShopB', 'ShopC', 'ShopD']) - def test_items_all_view_includes_unique_asins(self): - response = self._call('/api/admin/shop-data-crawl/duplicate-check-items?page=1&page_size=20&view=all') - body = response.get_json() - self.assertEqual(body['total'], 4) + def test_items_all_view_passthrough(self): + data = {'pending': False, 'items': self.items_all, 'shops': self.shops, + 'total': 6, 'page': 1, 'page_size': 20, 'scanned_at': '2026-09-04 03:10:00'} + response = self._call('shop_data_crawl_duplicate_check_items', + '/api/admin/shop-data-crawl/duplicate-check-items?page=1&page_size=20&view=all', + data) + self.assertEqual(response.get_json()['total'], 6) - def test_items_filter_by_asin_and_site(self): - response = self._call('/api/admin/shop-data-crawl/duplicate-check-items?page=1&page_size=20&asin=A0000001&site=UK') - body = response.get_json() - self.assertEqual(body['total'], 1) - self.assertEqual(body['items'][0]['asin'], 'A0000001') + def test_items_shop_name_alias_merged_and_camel_params(self): + captured = {} - def test_leader_sees_only_own_group_shops(self): - # 主管 id=653 只管理 GroupA（含 ShopA/ShopB） - with patch.object(admin_api, '_shop_data_managed_shop_names', - return_value={'shopa', 'shopb'}): - response = self._call('/api/admin/shop-data-crawl/duplicate-check-items?page=1&page_size=20&view=all', - role='admin', current_row={'id': 653}) - body = response.get_json() - self.assertEqual([shop['shop_name'] for shop in body['shops']], ['ShopA', 'ShopB']) - self.assertEqual(body['total'], 2) # A0000001（跨店）+ B0000001（唯一） - # 跨店 ASIN 在主管范围内仍是 2 家店 - item = next(i for i in body['items'] if i['asin'] == 'A0000001') - self.assertEqual(item['shop_count'], 2) + def side_effect(*args, **kwargs): + captured['params'] = kwargs.get('params') or {} + return _java_ok({'pending': False, 'items': [], 'shops': [], + 'total': 0, 'page': 1, 'page_size': 20, 'scanned_at': ''}) - def test_leader_overview_stats_recomputed_after_filter(self): - # 主管见 2 家店：唯一ASIN 2、上架记录 3、重复ASIN 1、重复店铺 2 - with patch.object(admin_api, '_shop_data_managed_shop_names', - return_value={'shopa', 'shopb'}): - with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-overview'): - with patch('utils.auth.session', {'user_id': 1}), \ - patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_api, '_ensure_backend_menu_access', return_value=('admin', {'id': 653}, None)), \ - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=('admin', {'id': 653}, None)), \ - patch.object(admin_api, '_latest_duplicate_scan', return_value=self.cache): - response = admin_api.shop_data_crawl_duplicate_check_overview() - body = response.get_json() - self.assertEqual(body['summary']['shop_count'], 2) - self.assertEqual(body['summary']['asin_total'], 2) - self.assertEqual(body['summary']['record_total'], 3) - self.assertEqual(body['summary']['duplicate_asin_total'], 1) - - def test_leader_sees_nothing_when_no_group(self): - with patch.object(admin_api, '_shop_data_managed_shop_names', return_value=set()): - response = self._call('/api/admin/shop-data-crawl/duplicate-check-items?page=1&page_size=20&view=all', - role='admin', current_row={'id': 999999}) - body = response.get_json() - self.assertEqual(body['shops'], []) - self.assertEqual(body['total'], 0) - - def test_export_csv_contains_filtered_rows(self): - """导出 CSV：行=上架记录，含 BOM，按筛选裁剪。""" - with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-export?'): + with self.app.test_request_context( + '/api/admin/shop-data-crawl/duplicate-check-items?page=2&page_size=50&view=all&shop=ShopA&asin=abc'): + # 内部代理 operatorId 从 flask session 取当前登录管理员 + admin_api.session['user_id'] = 1 with patch('utils.auth.session', {'user_id': 1}), \ patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_api, '_ensure_backend_menu_access', return_value=('super_admin', {'id': 1}, None)), \ - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=('super_admin', {'id': 1}, None)), \ - patch.object(admin_api, '_latest_duplicate_scan', return_value=self.cache): - response = admin_api.shop_data_crawl_duplicate_check_export() - self.assertEqual(response.status_code, 200) - text = response.get_data(as_text=True) - self.assertTrue(text.startswith('')) - lines = text.lstrip('').strip().splitlines() - self.assertEqual(lines[0], 'ASIN,店铺数,店铺,分组,站点,上架时间,价格,品牌') - self.assertEqual(len(lines), 3) # header + 两条上架记录（A0000001 两店各一条） + patch.object(admin_api, '_ensure_backend_menu_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_proxy_backend_java', side_effect=side_effect): + admin_api.shop_data_crawl_duplicate_check_items() + params = captured['params'] + self.assertEqual(params['page'], '2') + self.assertEqual(params['pageSize'], '50') + self.assertEqual(params['shopName'], 'ShopA') # shop 别名合并进 shop_name→shopName + self.assertEqual(params['asin'], 'abc') + self.assertIn('operatorId', params) + + def test_items_denied_403_message_preserved(self): + def denied_menu(*args, **kwargs): + return ('admin', {'id': 5}, ( + admin_api.jsonify({'success': False, 'error': '无权访问店铺数据记录模块'}), 403)) + + with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-items'): + with patch('utils.auth.session', {'user_id': 1}), \ + patch('utils.auth.is_session_user_valid', return_value=True), \ + patch.object(admin_api, '_ensure_backend_menu_access', side_effect=denied_menu), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', + return_value=('admin', {'id': 5}, None)), \ + patch.object(admin_api, '_proxy_backend_java', + side_effect=lambda *a, **k: _java_ok({})): + response = admin_api.shop_data_crawl_duplicate_check_items() + self.assertEqual(response[1], 403) + body = response[0].get_json() + self.assertEqual(body['error'], '无权访问店铺数据记录模块') + + def test_detail_sorted_passthrough(self): + detail_items = [ + {'asin': 'E0000001', 'shop_count': 3, 'record_count': 3, 'shop_names': ['ShopA', 'ShopB', 'ShopC'], + 'brand': 'BrandE', 'first_date': '2026-08-29 04:34:00', 'occurrences': self.items_all[0]['occurrences']}, + {'asin': 'A0000001', 'shop_count': 2, 'record_count': 2, 'shop_names': ['ShopA', 'ShopB'], + 'brand': 'BrandA', 'first_date': '2026-08-30 00:00:00', 'occurrences': self.items_all[1]['occurrences']}, + {'asin': 'F0000001', 'shop_count': 2, 'record_count': 2, 'shop_names': ['ShopB', 'ShopD'], + 'brand': 'BrandF', 'first_date': '2026-08-28 00:00:00', 'occurrences': self.items_all[2]['occurrences']}, + ] + data = {'pending': False, 'items': detail_items, 'total': 3, + 'page': 1, 'page_size': 6, 'scanned_at': '2026-09-04 03:10:00'} + response = self._call('shop_data_crawl_duplicate_check_detail', + '/api/admin/shop-data-crawl/duplicate-check-detail?page=1&page_size=6', + data) + body = response.get_json() + self.assertEqual(body['total'], 3) + self.assertEqual(body['items'][0]['asin'], 'E0000001') + self.assertEqual(body['items'][0]['first_date'], '2026-08-29 04:34:00') + + def test_detail_pending(self): + response = self._call('shop_data_crawl_duplicate_check_detail', + '/api/admin/shop-data-crawl/duplicate-check-detail', + {'pending': True, 'items': [], 'total': 0, 'page': 1, 'page_size': 6, 'scanned_at': ''}) + body = response.get_json() + self.assertTrue(body['pending']) + self.assertEqual(body['items'], []) + + def test_export_streams_java_csv(self): + csv_bytes = ('' + 'ASIN,店铺数,店铺,分组,站点,上架时间,价格,品牌\r\n' + 'E0000001,3,ShopA,GroupA,UK,2026年8月29日 上午4:34,GBP 12.00,BrandE\r\n').encode('utf-8') + + class FakeResp: + status_code = 200 + headers = {'Content-Disposition': 'attachment; filename="shop-data-duplicate-check.csv"', + 'Content-Type': 'text/csv; charset=utf-8'} + + def iter_content(self, chunk_size=1): + yield csv_bytes + + def close(self): + pass + + def json(self): + return {} + + class FakeSession: + def get(self, *args, **kwargs): + return FakeResp() - def test_export_monitor_view_excludes_unique_asins(self): with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-export?view=monitor'): with patch('utils.auth.session', {'user_id': 1}), \ patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_api, '_ensure_backend_menu_access', return_value=('super_admin', {'id': 1}, None)), \ - patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=('super_admin', {'id': 1}, None)), \ - patch.object(admin_api, '_latest_duplicate_scan', return_value=self.cache): + patch.object(admin_api, 'get_current_admin_role', + return_value=('super_admin', {'id': 1})), \ + patch.object(admin_api, '_ensure_backend_menu_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_get_backend_java_session', return_value=FakeSession()): response = admin_api.shop_data_crawl_duplicate_check_export() - lines = response.get_data(as_text=True).lstrip('').strip().splitlines() - self.assertEqual(len(lines), 3) - self.assertIn('A0000001', lines[1]) + body = b''.join(response.response) + self.assertTrue(body.startswith(b'\xef\xbb\xbf')) + self.assertIn('ASIN,店铺数'.encode('utf-8'), body) + self.assertIn('E0000001'.encode('utf-8'), body) - def test_internal_request_falls_back_to_system_operator(self): - """无请求上下文（定时扫描线程）时，内部请求用系统级超管作为 operatorId。""" - with patch('utils.auth.session', {'user_id': 1}): # 仅用于模拟无异常环境 - # 无请求上下文：has_request_context() 为 False，直接走 _resolve_system_operator_id - with patch.object(admin_api, '_resolve_internal_token', return_value='test-token'), \ - patch.object(admin_api, '_resolve_system_operator_id', return_value=7): - headers, params = admin_api._backend_java_internal_request() - self.assertEqual(headers.get('X-Internal-Token'), 'test-token') - self.assertEqual(params, {'operatorId': 7}) + def test_export_no_scan_400(self): + class FakeResp: + status_code = 400 + headers = {} - def test_internal_request_system_operator_missing_fails(self): - """无请求上下文且系统中没有任何管理员时，直接报错而不是传 0。""" - with patch.object(admin_api, '_resolve_internal_token', return_value='test-token'), \ - patch.object(admin_api, '_resolve_system_operator_id', return_value=None): - with self.assertRaises(ValueError): - admin_api._backend_java_internal_request() + def json(self): + return {'success': False, 'message': '暂无扫描结果，请先点击「重新分析」'} - def test_fetch_result_bytes_rejects_json_error_body(self): - """Java 内部端点返回 JSON 错误体（如鉴权失败）时，抛出业务错误而不是 BadZipFile。""" - fake_response = Mock() - fake_response.raise_for_status = Mock() - fake_response.iter_content = Mock(return_value=[ - '{"success":false,"message":"用户不存在","data":null,"code":401}'.encode('utf-8')]) - fake_response.close = Mock() - with patch.object(admin_api, '_backend_java_internal_request', - return_value=({'X-Internal-Token': 'test-token'}, {'operatorId': 7})), \ - patch.object(admin_api, '_get_backend_java_session') as fake_session: - fake_session.return_value.get = Mock(return_value=fake_response) - with self.assertRaises(ValueError) as ctx: - admin_api._shop_data_crawl_fetch_result_bytes({'result_id': 123}) - self.assertIn('用户不存在', str(ctx.exception)) + def close(self): + pass + + def iter_content(self, chunk_size=1): + return iter(()) + + class FakeSession: + def get(self, *args, **kwargs): + return FakeResp() + + with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-check-export'): + with patch('utils.auth.session', {'user_id': 1}), \ + patch('utils.auth.is_session_user_valid', return_value=True), \ + patch.object(admin_api, 'get_current_admin_role', + return_value=('super_admin', {'id': 1})), \ + patch.object(admin_api, '_ensure_backend_menu_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_ensure_shop_data_crawl_data_access', + return_value=('super_admin', {'id': 1}, None)), \ + patch.object(admin_api, '_get_backend_java_session', return_value=FakeSession()): + response = admin_api.shop_data_crawl_duplicate_check_export() + self.assertEqual(response[1], 400) + self.assertEqual(response[0].get_json()['error'], '暂无扫描结果，请先点击「重新分析」') if __name__ == '__main__': diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 619ecf02..ee92c1fa 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -1337,47 +1337,69 @@ min-height: 120px; max-height: 250px; overflow-y: auto; - display: flex; - flex-direction: column; - gap: 7px; + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + align-content: start; + gap: 8px; padding-right: 2px; } - .dup-chart-row { - display: grid; - grid-template-columns: minmax(70px, 96px) minmax(0, 1fr) 44px; - align-items: center; - gap: 9px; + /* 店铺上架分布：双列网格（店名 + 记录数 / 进度条 / N 个 ASIN） */ + .dup-dist-item { + min-width: 0; + display: flex; + flex-direction: column; + gap: 6px; + padding: 10px 12px; + background: #fbfcfe; + border: 1px solid var(--c-border); + border-radius: 8px; } - .dup-chart-name { - font-size: 12px; - color: var(--c-text-2); + .dup-dist-item-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px; + } + + .dup-dist-name { + min-width: 0; + font-size: 12.5px; + font-weight: 700; + color: var(--c-text); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } - .dup-chart-track { - height: 16px; + .dup-dist-record { + flex: 0 0 auto; + font-size: 15px; + font-weight: 800; + color: var(--c-primary-strong); + font-variant-numeric: tabular-nums; + } + + .dup-dist-track { + height: 7px; background: #eef0f6; - border-radius: 5px; + border-radius: 999px; overflow: hidden; } - .dup-chart-fill { + .dup-dist-fill { + display: block; height: 100%; min-width: 2px; background: linear-gradient(90deg, var(--c-primary), #8b8ff3); - border-radius: 5px; + border-radius: 999px; transition: width 0.3s ease; } - .dup-chart-num { + .dup-dist-asin { font-size: 12px; - font-weight: 700; - color: var(--c-text); - text-align: right; + color: var(--c-text-3); font-variant-numeric: tabular-nums; } @@ -1532,6 +1554,82 @@ border: 1px solid var(--c-border); } + /* 国家彩色徽章（撞款详情卡片 / 抽屉）：DE橙 / UK靛蓝 / FR蓝 / IT绿 / ES红 / 其他兜底 */ + .dup-site-badge { + display: inline-flex; + align-items: center; + padding: 2px 9px; + border-radius: 999px; + font-size: 11.5px; + font-weight: 700; + line-height: 1.6; + white-space: nowrap; + vertical-align: middle; + } + + .dup-site-DE { background: #fff1e3; color: #c2570a; border: 1px solid #f3cba4; } + .dup-site-UK { background: #eeedfb; color: #5451d6; border: 1px solid #c9c7f2; } + .dup-site-FR { background: #e5f0fd; color: #1d5fc2; border: 1px solid #b5d3f5; } + .dup-site-IT { background: #e3f6ec; color: #18774c; border: 1px solid #aadcc2; } + .dup-site-ES { background: #fdeaea; color: #c03a2e; border: 1px solid #f2bcba; } + .dup-site-NA, + .dup-site- { background: #f4f6fa; color: var(--c-text-3); border: 1px solid var(--c-border); } + + /* 抽屉底部徽章组：彩色滴状标签 */ + .dup-drawer-inline-badges { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; + margin-top: 6px; + } + + .dup-drawer-badge { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 3px 11px; + border-radius: 8px; + font-size: 12px; + font-weight: 700; + line-height: 1.5; + white-space: nowrap; + background: #eef2f7; + color: var(--c-text-2); + } + + .dup-drawer-badge-danger { background: #fdecec; color: #d8483a; border: 1px solid #f3bcb8; } + .dup-drawer-badge-blue { background: var(--c-primary-soft); color: var(--c-primary-strong); border: 1px solid #d3d6fa; } + .dup-drawer-badge-green { background: #e3f6ec; color: #18774c; border: 1px solid #aadcc2; } + .dup-drawer-badge-cyan { background: #e0f5f9; color: #0e7d94; border: 1px solid #a8e0ec; } + + /* 抽屉头部 ASIN 与品牌行 */ + .dup-drawer-asin { + font-family: Consolas, Menlo, monospace; + font-size: 17px; + font-weight: 800; + color: var(--c-primary-strong); + letter-spacing: 0.02em; + } + + .dup-drawer-brand-line { + padding: 12px 0; + font-size: 13px; + color: var(--c-text); + border-bottom: 1px solid var(--c-border); + overflow-wrap: anywhere; + } + + .dup-check-drawer-table td.dup-date { + color: var(--c-text-2); + font-variant-numeric: tabular-nums; + } + + .drawer-subtitle.dup-drawer-subtitle { + color: var(--c-text-2); + font-size: 12.5px; + } + .dup-check-drawer-meta { display: grid; grid-template-columns: 88px minmax(0, 1fr); @@ -1598,6 +1696,10 @@ .dup-check-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); } + + .dup-check-distribution-bars { + grid-template-columns: minmax(0, 1fr); + } } .thumb { @@ -5585,6 +5687,7 @@ 同一 ASIN 在多条店铺的上架明细
+