From 38216a18854598a57ea08e72a2f94cdac085867b Mon Sep 17 00:00:00 2001 From: supernijia Date: Sun, 9 Aug 2026 20:08:39 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=84=E7=90=86=E5=90=8E=E5=8F=B0=E6=96=B0?= =?UTF-8?q?=E9=9C=80=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/ModuleCleanupProperties.java | 4 +- .../service/CollectDataService.java | 1 + .../controller/DedupeTotalDataController.java | 166 +++- .../dto/DedupeTotalDataCreateRequest.java | 5 + .../dto/DedupeTotalDataUpdateRequest.java | 5 + .../model/entity/DedupeTotalDataEntity.java | 1 + .../model/vo/DedupeTotalDataItemVo.java | 6 + .../service/DedupeTotalDataService.java | 236 +++++- .../file/service/oss/OssStorageService.java | 12 + .../controller/InvalidAsinDataController.java | 146 +++- .../dto/InvalidAsinDataCreateRequest.java | 5 + .../dto/InvalidAsinDataUpdateRequest.java | 3 + .../model/entity/InvalidAsinDataEntity.java | 2 + .../model/vo/InvalidAsinDataItemVo.java | 9 + .../service/InvalidAsinDataService.java | 144 +++- .../mapper/ShopDataCrawlDailyFileMapper.java | 9 + .../ShopDataCrawlDailyMemberMapper.java | 9 + .../entity/ShopDataCrawlDailyFileEntity.java | 32 + .../ShopDataCrawlDailyMemberEntity.java | 20 + .../ShopDataCrawlDailyFileService.java | 218 +++++ .../ShopDataCrawlExcelAssemblyService.java | 75 ++ .../service/ShopDataCrawlTaskService.java | 794 ++++++++++++++---- .../shopkey/mapper/ShopManageGroupMapper.java | 24 + .../service/ShopManageGroupService.java | 16 + .../db/V85__shop_data_crawl_daily_file.sql | 33 + .../V86__invalid_asin_data_group_access.sql | 74 ++ .../V87__dedupe_total_data_group_access.sql | 53 ++ .../service/DedupeTotalDataServiceTest.java | 119 ++- .../InvalidAsinDataControllerTest.java | 108 +++ .../service/InvalidAsinDataServiceTest.java | 250 ++++++ ...ShopDataCrawlExcelAssemblyServiceTest.java | 49 ++ .../ShopDataCrawlTaskServiceChunkTest.java | 4 +- .../ShopDataCrawlTaskServiceOwnerTest.java | 1 + ...ShopDataCrawlTaskServiceRetentionTest.java | 393 +++++++-- backend/blueprints/admin_api.py | 201 +++-- backend/requirement.txt | 2 + backend/static/admin.js | 193 ++++- backend/tests/test_admin_dedupe_total_data.py | 161 ++++ backend/tests/test_admin_invalid_asin_data.py | 141 ++++ backend/tests/test_admin_shop_data_groups.py | 45 +- backend/web_source/admin.html | 58 +- .../brand/components/BrandCollectDataTab.vue | 22 +- 42 files changed, 3425 insertions(+), 424 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyFileMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyMemberMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyFileEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyMemberEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java create mode 100644 backend-java/src/main/resources/db/V85__shop_data_crawl_daily_file.sql create mode 100644 backend-java/src/main/resources/db/V86__invalid_asin_data_group_access.sql create mode 100644 backend-java/src/main/resources/db/V87__dedupe_total_data_group_access.sql create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataControllerTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataServiceTest.java create mode 100644 backend/tests/test_admin_dedupe_total_data.py create mode 100644 backend/tests/test_admin_invalid_asin_data.py diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java index ebd6e4ce..2d7a4076 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java @@ -12,7 +12,7 @@ public class ModuleCleanupProperties { private boolean enabled = true; private String cron = "0 0 0 * * *"; private long retentionDays = 7; - // SHOP_DATA_CRAWL is governed by per-shop latest-three retention in its - // task service and must not be removed by the age-based sweep. + // SHOP_DATA_CRAWL keeps one per-shop daily workbook in its task service + // and must not be removed by the age-based sweep. private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA")); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index 724a4cc4..4c9496b6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -805,6 +805,7 @@ public class CollectDataService { InvalidAsinDataEntity entity = new InvalidAsinDataEntity(); entity.setDataValue(row.getAsin()); entity.setBrand(brand); + entity.setRecordSource("AUTO"); try { invalidAsinDataMapper.insert(entity); } catch (DuplicateKeyException ignored) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java index e9abc63d..4db63e59 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java @@ -1,6 +1,8 @@ package com.nanri.aiimage.modules.dedupe.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest; @@ -10,6 +12,8 @@ import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo; import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo; import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo; import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; @@ -17,6 +21,8 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; +import jakarta.servlet.http.HttpServletRequest; +import org.springframework.beans.factory.annotation.Value; import lombok.RequiredArgsConstructor; import org.springframework.format.annotation.DateTimeFormat; import org.springframework.http.HttpHeaders; @@ -36,6 +42,10 @@ import org.springframework.web.multipart.MultipartFile; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; @RestController @RequiredArgsConstructor @@ -43,9 +53,20 @@ import java.time.format.DateTimeFormatter; @Tag(name = "数据去重总数据", description = "维护数据去重模块的总数据列表，支持增删改查。") public class DedupeTotalDataController { + private static final String DEDUPE_TOTAL_DATA_COLUMN_KEY = "admin_dedupe_total_data"; + private static final String DEDUPE_TOTAL_DATA_ROUTE_PATH = "dedupe-total-data"; + private static final DateTimeFormatter EXPORT_FILENAME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); private final DedupeTotalDataService dedupeTotalDataService; + private final AdminAuthSupport adminAuthSupport; + private final PermissionMenuService permissionMenuService; + + @Value("${aiimage.security.internal-token:}") + private String internalToken; + + @Value("${aiimage.security.internal-token-file:}") + private String internalTokenFile; @GetMapping @Operation(summary = "分页查询总数据", description = "分页查询数据去重总数据，支持按值模糊搜索。") @@ -61,9 +82,11 @@ public class DedupeTotalDataController { @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate, @Parameter(description = "结束日期（包含）") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) { + @Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId, + HttpServletRequest request) { + RequestOperator operator = requireDedupeTotalDataAccess(request); return ApiResponse.success(dedupeTotalDataService.page( - page, pageSize, keyword, username, startDate, endDate, operatorId)); + page, pageSize, keyword, username, startDate, endDate, groupId, operator.id())); } @GetMapping("/export") @@ -74,8 +97,10 @@ public class DedupeTotalDataController { @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate, @Parameter(description = "结束日期（包含）") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) { - byte[] bytes = dedupeTotalDataService.export(username, startDate, endDate, operatorId); + @Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId, + HttpServletRequest request) { + RequestOperator operator = requireDedupeTotalDataAccess(request); + byte[] bytes = dedupeTotalDataService.export(username, startDate, endDate, groupId, operator.id()); String filename = "dedupe-total-data-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx"; return ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename)) @@ -91,9 +116,10 @@ public class DedupeTotalDataController { @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复") }) public ApiResponse create( - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId, + HttpServletRequest httpRequest, @Valid @RequestBody DedupeTotalDataCreateRequest request) { - return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operatorId)); + RequestOperator operator = requireDedupeTotalDataAccess(httpRequest); + return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operator.id())); } @PostMapping("/import") @@ -104,16 +130,19 @@ public class DedupeTotalDataController { }) public ApiResponse importExcel( @Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) { - return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, operatorId)); + @Parameter(description = "分组 ID", required = true) @RequestParam Long groupId, + HttpServletRequest request) { + RequestOperator operator = requireDedupeTotalDataAccess(request); + return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, groupId, operator.id())); } @GetMapping("/import/{importId}") @Operation(summary = "查询导入进度", description = "根据导入任务 ID 查询当前进度。") public ApiResponse importProgress( @PathVariable String importId, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) { - return ApiResponse.success(dedupeTotalDataService.getImportProgress(importId, operatorId)); + HttpServletRequest request) { + RequestOperator operator = requireDedupeTotalDataAccess(request); + return ApiResponse.success(dedupeTotalDataService.getImportProgress(importId, operator.id())); } @PostMapping("/delete-import") @@ -124,16 +153,19 @@ public class DedupeTotalDataController { }) public ApiResponse deleteImportExcel( @Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) { - return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, operatorId)); + @Parameter(description = "分组 ID", required = true) @RequestParam Long groupId, + HttpServletRequest request) { + RequestOperator operator = requireDedupeTotalDataAccess(request); + return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, groupId, operator.id())); } @GetMapping("/delete-import/{importId}") @Operation(summary = "查询删除导入进度", description = "根据删除导入任务 ID 查询当前进度。") public ApiResponse deleteImportProgress( @PathVariable String importId, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) { - return ApiResponse.success(dedupeTotalDataService.getDeleteImportProgress(importId, operatorId)); + HttpServletRequest request) { + RequestOperator operator = requireDedupeTotalDataAccess(request); + return ApiResponse.success(dedupeTotalDataService.getDeleteImportProgress(importId, operator.id())); } @PutMapping("/{id}") @@ -145,9 +177,10 @@ public class DedupeTotalDataController { }) public ApiResponse update( @Parameter(description = "主键ID", required = true) @PathVariable Long id, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId, + HttpServletRequest httpRequest, @Valid @RequestBody DedupeTotalDataUpdateRequest request) { - return ApiResponse.success("更新成功", dedupeTotalDataService.update(id, request, operatorId)); + RequestOperator operator = requireDedupeTotalDataAccess(httpRequest); + return ApiResponse.success("更新成功", dedupeTotalDataService.update(id, request, operator.id())); } @DeleteMapping("/{id}") @@ -158,8 +191,105 @@ public class DedupeTotalDataController { }) public ApiResponse delete( @Parameter(description = "主键ID", required = true) @PathVariable Long id, - @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) { - dedupeTotalDataService.delete(id, operatorId); + HttpServletRequest request) { + RequestOperator operator = requireDedupeTotalDataAccess(request); + dedupeTotalDataService.delete(id, operator.id()); return ApiResponse.success("删除成功", null); } + + private RequestOperator requireDedupeTotalDataAccess(HttpServletRequest request) { + AdminUserEntity operator = resolveOperator(request); + if (operator == null || operator.getId() == null || operator.getId() <= 0) { + throw new BusinessException(403, "无权访问数据去重总数据"); + } + boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(operator)); + if (!superAdmin && !hasDedupeTotalDataPermission(operator)) { + throw new BusinessException(403, "无权访问数据去重总数据"); + } + return new RequestOperator(operator.getId(), superAdmin); + } + + private AdminUserEntity resolveOperator(HttpServletRequest request) { + try { + return adminAuthSupport.requireUser(request); + } catch (BusinessException authFailure) { + AdminUserEntity internalOperator = resolveInternalOperator(request); + if (internalOperator != null) { + return internalOperator; + } + throw authFailure; + } + } + + private boolean hasDedupeTotalDataPermission(AdminUserEntity operator) { + return permissionMenuService.getUserColumnPermissions(operator.getId(), "admin") + .stream() + .anyMatch(item -> DEDUPE_TOTAL_DATA_COLUMN_KEY.equals(item.getColumnKey()) + || DEDUPE_TOTAL_DATA_ROUTE_PATH.equals(item.getRoutePath())); + } + + private AdminUserEntity resolveInternalOperator(HttpServletRequest request) { + if (!isTrustedInternalRequest(request.getHeader("X-Internal-Token"))) { + return null; + } + String rawOperatorId = request.getParameter("operatorId"); + if (rawOperatorId == null || rawOperatorId.isBlank()) { + rawOperatorId = request.getParameter("operator_id"); + } + if (rawOperatorId == null || rawOperatorId.isBlank()) { + return null; + } + try { + return permissionMenuService.requireUserOperator(Long.parseLong(rawOperatorId.trim())); + } catch (NumberFormatException | BusinessException ex) { + return null; + } + } + + private boolean isTrustedInternalRequest(String suppliedToken) { + String expectedToken = resolveExpectedInternalToken(); + return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank() + && MessageDigest.isEqual( + expectedToken.getBytes(StandardCharsets.UTF_8), + suppliedToken.trim().getBytes(StandardCharsets.UTF_8)); + } + + private String resolveExpectedInternalToken() { + if (internalToken != null && !internalToken.isBlank()) { + return internalToken.trim(); + } + Path path = resolveInternalTokenFile(); + if (path == null || !Files.isRegularFile(path)) { + return ""; + } + try { + return Files.readString(path, StandardCharsets.UTF_8).trim(); + } catch (Exception ignored) { + return ""; + } + } + + private Path resolveInternalTokenFile() { + String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim(); + if (!configuredPath.isEmpty()) { + if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) { + String userHome = System.getProperty("user.home", "").trim(); + if (userHome.isEmpty()) { + return null; + } + configuredPath = configuredPath.length() == 1 + ? userHome + : Path.of(userHome, configuredPath.substring(2)).toString(); + } + Path configuredTokenPath = Path.of(configuredPath); + return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null; + } + String userHome = System.getProperty("user.home", "").trim(); + return userHome.isEmpty() + ? null + : Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize(); + } + + private record RequestOperator(Long id, boolean superAdmin) { + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java index b4eda856..fcbf6219 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.dedupe.model.dto; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; import lombok.Data; @@ -13,4 +14,8 @@ public class DedupeTotalDataCreateRequest { @Size(max = 128, message = "数据长度不能超过128个字符") @Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED) private String dataValue; + + @NotNull(message = "请选择分组") + @Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED) + private Long groupId; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java index 380ba75d..cd8622b2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.dedupe.model.dto; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; import lombok.Data; @@ -13,4 +14,8 @@ public class DedupeTotalDataUpdateRequest { @Size(max = 128, message = "数据长度不能超过128个字符") @Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED) private String dataValue; + + @NotNull(message = "请选择分组") + @Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED) + private Long groupId; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java index b25af687..fe324edb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java @@ -14,6 +14,7 @@ public class DedupeTotalDataEntity { @TableId(type = IdType.AUTO) private Long id; private String dataValue; + private Long groupId; private Long uploaderUserId; private String uploaderUsername; private LocalDateTime createdAt; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java index 6dadce78..de37f41b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java @@ -15,6 +15,12 @@ public class DedupeTotalDataItemVo { @Schema(description = "总数据值") private String dataValue; + @Schema(description = "分组 ID") + private Long groupId; + + @Schema(description = "分组名称") + private String groupName; + @Schema(description = "上传用户ID") private Long uploaderUserId; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java index cf97ef6c..12c82154 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo; import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper; +import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity; import lombok.RequiredArgsConstructor; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.DataFormatter; @@ -42,6 +43,7 @@ import java.time.format.DateTimeFormatter; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -65,6 +67,8 @@ public class DedupeTotalDataService { private final Map deleteImportProgressMap = new ConcurrentHashMap<>(); private final Map importOwnerMap = new ConcurrentHashMap<>(); private final Map deleteImportOwnerMap = new ConcurrentHashMap<>(); + private final Map importGroupMap = new ConcurrentHashMap<>(); + private final Map deleteImportGroupMap = new ConcurrentHashMap<>(); private final Map importCompletedAtMap = new ConcurrentHashMap<>(); private final Map deleteImportCompletedAtMap = new ConcurrentHashMap<>(); @@ -76,6 +80,11 @@ public class DedupeTotalDataService { public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username, LocalDate startDate, LocalDate endDate, Long operatorId) { + return page(page, pageSize, keyword, username, startDate, endDate, null, operatorId); + } + + public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username, + LocalDate startDate, LocalDate endDate, Long groupId, Long operatorId) { if (startDate != null && endDate != null && startDate.isAfter(endDate)) { throw new BusinessException("开始日期不能晚于结束日期"); } @@ -86,17 +95,21 @@ public class DedupeTotalDataService { AccessScope scope = resolveAccessScope(operatorId); LambdaQueryWrapper query = new LambdaQueryWrapper() .like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword) - .in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds()) .like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername) .ge(startDate != null, DedupeTotalDataEntity::getCreatedAt, startDate == null ? null : startDate.atStartOfDay()) .lt(endDate != null, DedupeTotalDataEntity::getCreatedAt, endDate == null ? null : endDate.plusDays(1).atStartOfDay()) .orderByDesc(DedupeTotalDataEntity::getId); + applyGroupScope(query, scope, groupId); Long total = dedupeTotalDataMapper.selectCount(query); - List items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize)) - .stream() - .map(this::toItemVo) + List rows = dedupeTotalDataMapper.selectList( + query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize)); + Map groupNames = loadGroupNames(rows); + List items = rows.stream() + .map(row -> toItemVo(row, row.getGroupId() == null + ? "" + : groupNames.getOrDefault(row.getGroupId(), ""))) .toList(); DedupeTotalDataPageVo vo = new DedupeTotalDataPageVo(); vo.setItems(items); @@ -107,24 +120,29 @@ public class DedupeTotalDataService { } public byte[] export(String username, LocalDate startDate, LocalDate endDate, Long operatorId) { + return export(username, startDate, endDate, null, operatorId); + } + + public byte[] export(String username, LocalDate startDate, LocalDate endDate, + Long groupId, Long operatorId) { if (startDate != null && endDate != null && startDate.isAfter(endDate)) { throw new BusinessException("开始日期不能晚于结束日期"); } String safeUsername = username == null ? "" : username.trim(); AccessScope scope = resolveAccessScope(operatorId); LambdaQueryWrapper query = new LambdaQueryWrapper() - .in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds()) .like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername) .ge(startDate != null, DedupeTotalDataEntity::getCreatedAt, startDate == null ? null : startDate.atStartOfDay()) .lt(endDate != null, DedupeTotalDataEntity::getCreatedAt, endDate == null ? null : endDate.plusDays(1).atStartOfDay()) .orderByDesc(DedupeTotalDataEntity::getId); + applyGroupScope(query, scope, groupId); List rows = dedupeTotalDataMapper.selectList(query); - return buildExportWorkbook(rows); + return buildExportWorkbook(rows, loadGroupNames(rows)); } - private byte[] buildExportWorkbook(List rows) { + private byte[] buildExportWorkbook(List rows, Map groupNames) { try (SXSSFWorkbook workbook = new SXSSFWorkbook(100); ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { Sheet sheet = workbook.createSheet("DedupeTotalData"); @@ -132,19 +150,24 @@ public class DedupeTotalDataService { header.createCell(0).setCellValue("ID"); header.createCell(1).setCellValue("ASIN值"); header.createCell(2).setCellValue("用户名"); - header.createCell(3).setCellValue("创建时间"); + header.createCell(3).setCellValue("分组"); + header.createCell(4).setCellValue("创建时间"); for (int index = 0; index < rows.size(); index++) { DedupeTotalDataEntity entity = rows.get(index); Row row = sheet.createRow(index + 1); row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId())); row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue()); row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername()); - row.createCell(3).setCellValue(formatExportTime(entity.getCreatedAt())); + row.createCell(3).setCellValue(entity.getGroupId() == null + ? "" + : groupNames.getOrDefault(entity.getGroupId(), "")); + row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt())); } sheet.setColumnWidth(0, 3600); sheet.setColumnWidth(1, 5200); sheet.setColumnWidth(2, 5200); - sheet.setColumnWidth(3, 5600); + sheet.setColumnWidth(3, 5200); + sheet.setColumnWidth(4, 5600); workbook.write(outputStream); workbook.dispose(); return outputStream.toByteArray(); @@ -160,14 +183,16 @@ public class DedupeTotalDataService { @Transactional public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long operatorId) { AdminUserEntity uploader = getOperator(operatorId); + ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), uploader); String dataValue = normalizeComparableValue(request.getDataValue()); ensureUnique(dataValue, null); DedupeTotalDataEntity entity = new DedupeTotalDataEntity(); entity.setDataValue(dataValue); + entity.setGroupId(group.getId()); entity.setUploaderUserId(uploader.getId()); entity.setUploaderUsername(uploader.getUsername()); dedupeTotalDataMapper.insert(entity); - return toItemVo(getById(entity.getId())); + return toItemVo(getById(entity.getId()), group.getGroupName()); } public Set listComparableValues() { @@ -204,11 +229,16 @@ public class DedupeTotalDataService { } public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long operatorId) { + return startImport(file, null, operatorId); + } + + public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long operatorId) { cleanupExpiredProgress(); if (file == null || file.isEmpty()) { throw new BusinessException("请上传 xlsx 文件"); } AdminUserEntity uploader = getOperator(operatorId); + ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader); String importId = IdUtil.fastSimpleUUID(); DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo(); progress.setStatus("pending"); @@ -219,15 +249,17 @@ public class DedupeTotalDataService { progress.setSkippedCount(0); importProgressMap.put(importId, progress); importOwnerMap.put(importId, uploader.getId()); + importGroupMap.put(importId, group.getId()); try { File tempFile = saveMultipartToTempFile(file); String filename = file.getOriginalFilename(); Thread.ofVirtual().start(() -> runImportTask( - importId, tempFile, filename, uploader.getId(), uploader.getUsername())); + importId, tempFile, filename, uploader.getId(), uploader.getUsername(), group.getId())); } catch (Exception e) { importProgressMap.remove(importId); importOwnerMap.remove(importId); + importGroupMap.remove(importId); throw new BusinessException("读取上传文件失败"); } @@ -240,19 +272,25 @@ public class DedupeTotalDataService { cleanupExpiredProgress(); DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId); Long ownerId = importOwnerMap.get(importId); - if (progress == null || ownerId == null) { + Long groupId = importGroupMap.get(importId); + if (progress == null || ownerId == null || groupId == null) { throw new BusinessException("导入任务不存在"); } - ensureUploaderAccess(ownerId, resolveAccessScope(operatorId)); + ensureImportTaskAccess(groupId, resolveAccessScope(operatorId)); return progress; } public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long operatorId) { + return startDeleteImport(file, null, operatorId); + } + + public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long operatorId) { cleanupExpiredProgress(); if (file == null || file.isEmpty()) { throw new BusinessException("请上传 xlsx 文件"); } AdminUserEntity operator = getOperator(operatorId); + ShopManageGroupEntity group = resolveWritableGroup(groupId, operator); String importId = IdUtil.fastSimpleUUID(); DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo(); progress.setStatus("pending"); @@ -263,14 +301,17 @@ public class DedupeTotalDataService { progress.setSkippedCount(0); deleteImportProgressMap.put(importId, progress); deleteImportOwnerMap.put(importId, operator.getId()); + deleteImportGroupMap.put(importId, group.getId()); try { File tempFile = saveMultipartToTempFile(file); String filename = file.getOriginalFilename(); - Thread.ofVirtual().start(() -> runDeleteImportTask(importId, tempFile, filename, operator.getId())); + Thread.ofVirtual().start(() -> runDeleteImportTask( + importId, tempFile, filename, operator.getId(), group.getId())); } catch (Exception e) { deleteImportProgressMap.remove(importId); deleteImportOwnerMap.remove(importId); + deleteImportGroupMap.remove(importId); throw new BusinessException("读取上传文件失败"); } @@ -283,14 +324,16 @@ public class DedupeTotalDataService { cleanupExpiredProgress(); DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId); Long ownerId = deleteImportOwnerMap.get(importId); - if (progress == null || ownerId == null) { + Long groupId = deleteImportGroupMap.get(importId); + if (progress == null || ownerId == null || groupId == null) { throw new BusinessException("删除任务不存在"); } - ensureUploaderAccess(ownerId, resolveAccessScope(operatorId)); + ensureImportTaskAccess(groupId, resolveAccessScope(operatorId)); return progress; } - private void runDeleteImportTask(String importId, File tempFile, String filename, Long operatorId) { + private void runDeleteImportTask(String importId, File tempFile, String filename, + Long operatorId, Long groupId) { DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId); if (progress == null) { return; @@ -298,7 +341,8 @@ public class DedupeTotalDataService { progress.setStatus("running"); try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) { AccessScope scope = resolveAccessScope(operatorId); - DedupeTotalDataImportVo result = deleteFromExcelInternal(inputStream, filename, progress, scope); + DedupeTotalDataImportVo result = deleteFromExcelInternal( + inputStream, filename, progress, scope, groupId); progress.setTotalRows(result.getTotalRows()); progress.setAsinCount(result.getAsinCount()); progress.setInsertedCount(result.getInsertedCount()); @@ -315,7 +359,7 @@ public class DedupeTotalDataService { } private void runImportTask(String importId, File tempFile, String filename, - Long uploaderUserId, String uploaderUsername) { + Long uploaderUserId, String uploaderUsername, Long groupId) { DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId); if (progress == null) { return; @@ -323,7 +367,7 @@ public class DedupeTotalDataService { progress.setStatus("running"); try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) { DedupeTotalDataImportVo result = importFromExcelInternal( - inputStream, filename, progress, uploaderUserId, uploaderUsername); + inputStream, filename, progress, uploaderUserId, uploaderUsername, groupId); progress.setTotalRows(result.getTotalRows()); progress.setAsinCount(result.getAsinCount()); progress.setInsertedCount(result.getInsertedCount()); @@ -364,13 +408,19 @@ public class DedupeTotalDataService { } public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long operatorId) { + return importFromExcel(file, null, operatorId); + } + + public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long operatorId) { if (file == null || file.isEmpty()) { throw new BusinessException("请上传 xlsx 文件"); } AdminUserEntity uploader = getOperator(operatorId); + ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader); try (InputStream inputStream = file.getInputStream()) { return importFromExcelInternal( - inputStream, file.getOriginalFilename(), null, uploader.getId(), uploader.getUsername()); + inputStream, file.getOriginalFilename(), null, uploader.getId(), uploader.getUsername(), + group.getId()); } catch (BusinessException e) { throw e; } catch (Exception e) { @@ -379,12 +429,19 @@ public class DedupeTotalDataService { } public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long operatorId) { + return deleteFromExcel(file, null, operatorId); + } + + public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long groupId, Long operatorId) { if (file == null || file.isEmpty()) { throw new BusinessException("请上传 xlsx 文件"); } + AdminUserEntity operator = getOperator(operatorId); + ShopManageGroupEntity group = resolveWritableGroup(groupId, operator); AccessScope scope = resolveAccessScope(operatorId); try (InputStream inputStream = file.getInputStream()) { - return deleteFromExcelInternal(inputStream, file.getOriginalFilename(), null, scope); + return deleteFromExcelInternal( + inputStream, file.getOriginalFilename(), null, scope, group.getId()); } catch (BusinessException e) { throw e; } catch (Exception e) { @@ -394,7 +451,8 @@ public class DedupeTotalDataService { private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename, DedupeTotalDataImportProgressVo progress, - Long uploaderUserId, String uploaderUsername) { + Long uploaderUserId, String uploaderUsername, + Long groupId) { String lowerFilename = filename == null ? "" : filename.toLowerCase(); if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) { throw new BusinessException("仅支持 .xlsx 或 .xls 文件"); @@ -490,6 +548,7 @@ public class DedupeTotalDataService { } DedupeTotalDataEntity entity = new DedupeTotalDataEntity(); entity.setDataValue(pendingDataValue); + entity.setGroupId(groupId); entity.setUploaderUserId(uploaderUserId); entity.setUploaderUsername(uploaderUsername); dedupeTotalDataMapper.insert(entity); @@ -526,7 +585,7 @@ public class DedupeTotalDataService { private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename, DedupeTotalDataImportProgressVo progress, - AccessScope scope) { + AccessScope scope, Long groupId) { String lowerFilename = filename == null ? "" : filename.toLowerCase(); if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) { throw new BusinessException("仅支持 .xlsx 或 .xls 文件"); @@ -604,7 +663,8 @@ public class DedupeTotalDataService { continue; } - int deletedThisRow = newRequiresNewTemplate().execute(status -> deleteByDataValue(dataValue, scope)); + int deletedThisRow = newRequiresNewTemplate().execute( + status -> deleteByDataValue(dataValue, scope, groupId)); if (deletedThisRow > 0) { deletedCount += deletedThisRow; } else { @@ -634,11 +694,14 @@ public class DedupeTotalDataService { @Transactional public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request, Long operatorId) { DedupeTotalDataEntity entity = getAccessibleById(id, operatorId); + AdminUserEntity operator = getOperator(operatorId); String dataValue = normalizeComparableValue(request.getDataValue()); ensureUnique(dataValue, id); entity.setDataValue(dataValue); + ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), operator); + entity.setGroupId(group.getId()); dedupeTotalDataMapper.updateById(entity); - return toItemVo(getById(id)); + return toItemVo(getById(id), group.getGroupName()); } @Transactional @@ -649,7 +712,7 @@ public class DedupeTotalDataService { private DedupeTotalDataEntity getAccessibleById(Long id, Long operatorId) { DedupeTotalDataEntity entity = getById(id); - ensureUploaderAccess(entity.getUploaderUserId(), resolveAccessScope(operatorId)); + ensureRecordAccess(entity, resolveAccessScope(operatorId)); return entity; } @@ -690,26 +753,78 @@ public class DedupeTotalDataService { return dedupeTotalDataMapper.selectOne(query) != null; } - private int deleteByDataValue(String dataValue, AccessScope scope) { - return dedupeTotalDataMapper.delete(new LambdaQueryWrapper() - .eq(DedupeTotalDataEntity::getDataValue, dataValue) - .in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds())); + private int deleteByDataValue(String dataValue, AccessScope scope, Long groupId) { + LambdaQueryWrapper query = new LambdaQueryWrapper() + .eq(DedupeTotalDataEntity::getDataValue, dataValue); + applyGroupScope(query, scope, groupId); + return dedupeTotalDataMapper.delete(query); } private AccessScope resolveAccessScope(Long operatorId) { AdminUserEntity operator = getOperator(operatorId); if (isSuperAdmin(operator)) { - return new AccessScope(true, Set.of()); + return new AccessScope(true, Set.of(), Set.of()); } + List accessibleGroupIds = shopManageGroupMapper.selectAccessibleGroupIds(operator.getId()); LinkedHashSet visibleUserIds = new LinkedHashSet<>(); visibleUserIds.add(operator.getId()); + if (accessibleGroupIds != null && !accessibleGroupIds.isEmpty()) { + List groupUserIds = shopManageGroupMapper.selectUserIdsByGroupIds(accessibleGroupIds); + if (groupUserIds != null) { + groupUserIds.stream() + .filter(id -> id != null && id > 0) + .forEach(visibleUserIds::add); + } + } List memberUserIds = shopManageGroupMapper.selectManagedMemberUserIds(operator.getId()); if (memberUserIds != null) { memberUserIds.stream() .filter(id -> id != null && id > 0) .forEach(visibleUserIds::add); } - return new AccessScope(false, Set.copyOf(visibleUserIds)); + return new AccessScope( + false, + Set.copyOf(accessibleGroupIds == null ? List.of() : accessibleGroupIds), + Set.copyOf(visibleUserIds)); + } + + private void applyGroupScope(LambdaQueryWrapper query, + AccessScope scope, Long groupId) { + if (groupId != null && groupId > 0) { + resolveAccessibleGroup(groupId, scope); + query.eq(DedupeTotalDataEntity::getGroupId, groupId); + return; + } + if (scope.allUsers()) { + return; + } + query.and(wrapper -> { + if (!scope.groupIds().isEmpty()) { + wrapper.in(DedupeTotalDataEntity::getGroupId, scope.groupIds()) + .or(); + } + wrapper.isNull(DedupeTotalDataEntity::getGroupId) + .in(DedupeTotalDataEntity::getUploaderUserId, scope.userIds()); + }); + } + + private ShopManageGroupEntity resolveWritableGroup(Long groupId, AdminUserEntity operator) { + if (groupId == null || groupId <= 0) { + throw new BusinessException("请选择分组"); + } + AccessScope scope = resolveAccessScope(operator.getId()); + return resolveAccessibleGroup(groupId, scope); + } + + private ShopManageGroupEntity resolveAccessibleGroup(Long groupId, AccessScope scope) { + ShopManageGroupEntity group = shopManageGroupMapper.selectById(groupId); + if (group == null) { + throw new BusinessException("分组不存在"); + } + if (!scope.allUsers() && !scope.groupIds().contains(groupId)) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该分组数据"); + } + return group; } private AdminUserEntity getOperator(Long operatorId) { @@ -729,27 +844,46 @@ public class DedupeTotalDataService { || (role.isEmpty() && Integer.valueOf(1).equals(user.getIsAdmin()) && user.getCreatedById() == null); } - private void ensureUploaderAccess(Long uploaderUserId, AccessScope scope) { - if (!scope.allUsers() && (uploaderUserId == null || !scope.userIds().contains(uploaderUserId))) { - throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该总数据"); + private void ensureRecordAccess(DedupeTotalDataEntity entity, AccessScope scope) { + if (scope.allUsers()) { + return; + } + if (entity.getGroupId() != null && scope.groupIds().contains(entity.getGroupId())) { + return; + } + if (entity.getGroupId() == null && entity.getUploaderUserId() != null + && scope.userIds().contains(entity.getUploaderUserId())) { + return; + } + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该总数据"); + } + + private void ensureImportTaskAccess(Long groupId, AccessScope scope) { + if (!scope.allUsers() && !scope.groupIds().contains(groupId)) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权查看该导入任务"); } } private void cleanupExpiredProgress() { long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS; - cleanupExpiredProgressEntries(importCompletedAtMap, importProgressMap, importOwnerMap, cutoff); - cleanupExpiredProgressEntries(deleteImportCompletedAtMap, deleteImportProgressMap, deleteImportOwnerMap, cutoff); + cleanupExpiredProgressEntries( + importCompletedAtMap, importProgressMap, importOwnerMap, importGroupMap, cutoff); + cleanupExpiredProgressEntries( + deleteImportCompletedAtMap, deleteImportProgressMap, deleteImportOwnerMap, + deleteImportGroupMap, cutoff); } private void cleanupExpiredProgressEntries( Map completedAtMap, Map progressMap, Map ownerMap, + Map groupMap, long cutoff) { completedAtMap.forEach((id, completedAt) -> { if (completedAt != null && completedAt < cutoff && completedAtMap.remove(id, completedAt)) { progressMap.remove(id); ownerMap.remove(id); + groupMap.remove(id); } }); } @@ -768,16 +902,36 @@ public class DedupeTotalDataService { .replaceAll("\\s+", " "); } - private DedupeTotalDataItemVo toItemVo(DedupeTotalDataEntity entity) { + private Map loadGroupNames(List rows) { + List groupIds = rows.stream() + .map(DedupeTotalDataEntity::getGroupId) + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (groupIds.isEmpty()) { + return Map.of(); + } + return shopManageGroupMapper.selectBatchIds(groupIds).stream() + .filter(group -> group.getId() != null) + .collect(java.util.stream.Collectors.toMap( + ShopManageGroupEntity::getId, + group -> group.getGroupName() == null ? "" : group.getGroupName(), + (left, right) -> left, + LinkedHashMap::new)); + } + + private DedupeTotalDataItemVo toItemVo(DedupeTotalDataEntity entity, String groupName) { DedupeTotalDataItemVo vo = new DedupeTotalDataItemVo(); vo.setId(entity.getId()); vo.setDataValue(entity.getDataValue()); + vo.setGroupId(entity.getGroupId()); + vo.setGroupName(groupName == null ? "" : groupName); vo.setUploaderUserId(entity.getUploaderUserId()); vo.setUsername(entity.getUploaderUsername()); vo.setCreatedAt(entity.getCreatedAt()); return vo; } - private record AccessScope(boolean allUsers, Set userIds) { + private record AccessScope(boolean allUsers, Set groupIds, Set userIds) { } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java index 8c87e166..2c70375f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -147,6 +147,18 @@ public class OssStorageService { } } + /** + * Reads a managed result object from either its object key or stored public URL. + * This overload keeps callers from depending on the configured bucket name. + */ + public byte[] readObjectBytes(String value) { + StorageLocation location = resolveStorageLocation(value); + if (location == null) { + throw new IllegalArgumentException("object value must not be blank"); + } + return readObjectBytes(location.bucket(), location.objectKey()); + } + public boolean objectExists(String bucket, String objectKey) { String normalizedBucket = requireStorageName(bucket, "bucket"); String normalizedObjectKey = requireStorageName(objectKey, "objectKey"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataController.java index 7f490ae8..39540261 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataController.java @@ -1,19 +1,25 @@ package com.nanri.aiimage.modules.invalidasin.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataCreateRequest; import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequest; import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo; import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo; import com.nanri.aiimage.modules.invalidasin.service.InvalidAsinDataService; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -24,13 +30,29 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; + @RestController @RequiredArgsConstructor @RequestMapping("/api/admin/invalid-asin-data") @Tag(name = "不符合ASIN数据", description = "维护不符合ASIN数据列表，支持增删改查。") public class InvalidAsinDataController { + private static final String INVALID_ASIN_DATA_COLUMN_KEY = "admin_invalid_asin_data"; + private static final String INVALID_ASIN_DATA_ROUTE_PATH = "invalid-asin-data"; + + @Value("${aiimage.security.internal-token:}") + private String internalToken; + + @Value("${aiimage.security.internal-token-file:}") + private String internalTokenFile; + private final InvalidAsinDataService invalidAsinDataService; + private final AdminAuthSupport adminAuthSupport; + private final PermissionMenuService permissionMenuService; @GetMapping @Operation(summary = "分页查询不符合ASIN数据", description = "分页查询不符合ASIN数据，支持按 ASIN/品牌 模糊搜索。") @@ -40,8 +62,12 @@ public class InvalidAsinDataController { public ApiResponse page( @Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page, @Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize, - @Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword) { - return ApiResponse.success(invalidAsinDataService.page(page, pageSize, keyword)); + @Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword, + @Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId, + HttpServletRequest request) { + RequestOperator operator = requireInvalidAsinDataAccess(request); + return ApiResponse.success(invalidAsinDataService.page( + page, pageSize, keyword, groupId, operator.id(), operator.superAdmin())); } @PostMapping @@ -50,8 +76,12 @@ public class InvalidAsinDataController { @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功", content = @Content(schema = @Schema(implementation = InvalidAsinDataItemVo.class))), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复") }) - public ApiResponse create(@Valid @RequestBody InvalidAsinDataCreateRequest request) { - return ApiResponse.success("创建成功", invalidAsinDataService.create(request)); + public ApiResponse create( + HttpServletRequest httpRequest, + @Valid @RequestBody InvalidAsinDataCreateRequest request) { + RequestOperator operator = requireInvalidAsinDataAccess(httpRequest); + return ApiResponse.success("创建成功", invalidAsinDataService.create( + request, operator.id(), operator.superAdmin())); } @PutMapping("/{id}") @@ -63,8 +93,11 @@ public class InvalidAsinDataController { }) public ApiResponse update( @Parameter(description = "主键ID", required = true) @PathVariable Long id, + HttpServletRequest httpRequest, @Valid @RequestBody InvalidAsinDataUpdateRequest request) { - return ApiResponse.success("更新成功", invalidAsinDataService.update(id, request)); + RequestOperator operator = requireInvalidAsinDataAccess(httpRequest); + return ApiResponse.success("更新成功", invalidAsinDataService.update( + id, request, operator.id(), operator.superAdmin())); } @DeleteMapping("/{id}") @@ -73,8 +106,107 @@ public class InvalidAsinDataController { @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在") }) - public ApiResponse delete(@Parameter(description = "主键ID", required = true) @PathVariable Long id) { - invalidAsinDataService.delete(id); + public ApiResponse delete( + @Parameter(description = "主键ID", required = true) @PathVariable Long id, + HttpServletRequest request) { + RequestOperator operator = requireInvalidAsinDataAccess(request); + invalidAsinDataService.delete(id, operator.id(), operator.superAdmin()); return ApiResponse.success("删除成功", null); } + + private RequestOperator requireInvalidAsinDataAccess(HttpServletRequest request) { + AdminUserEntity operator = resolveOperator(request); + if (operator == null || operator.getId() == null || operator.getId() <= 0) { + throw new BusinessException(403, "无权访问品牌数据库"); + } + boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(operator)); + if (!superAdmin && !hasInvalidAsinDataPermission(operator)) { + throw new BusinessException(403, "无权访问品牌数据库"); + } + return new RequestOperator(operator.getId(), superAdmin); + } + + private AdminUserEntity resolveOperator(HttpServletRequest request) { + try { + return adminAuthSupport.requireUser(request); + } catch (BusinessException authFailure) { + AdminUserEntity internalOperator = resolveInternalOperator(request); + if (internalOperator != null) { + return internalOperator; + } + throw authFailure; + } + } + + private boolean hasInvalidAsinDataPermission(AdminUserEntity operator) { + return permissionMenuService.getUserColumnPermissions(operator.getId(), "admin") + .stream() + .anyMatch(item -> INVALID_ASIN_DATA_COLUMN_KEY.equals(item.getColumnKey()) + || INVALID_ASIN_DATA_ROUTE_PATH.equals(item.getRoutePath())); + } + + private AdminUserEntity resolveInternalOperator(HttpServletRequest request) { + if (!isTrustedInternalRequest(request.getHeader("X-Internal-Token"))) { + return null; + } + String rawOperatorId = request.getParameter("operatorId"); + if (rawOperatorId == null || rawOperatorId.isBlank()) { + rawOperatorId = request.getParameter("operator_id"); + } + if (rawOperatorId == null || rawOperatorId.isBlank()) { + return null; + } + try { + return permissionMenuService.requireUserOperator(Long.parseLong(rawOperatorId.trim())); + } catch (NumberFormatException | BusinessException ex) { + return null; + } + } + + private boolean isTrustedInternalRequest(String suppliedToken) { + String expectedToken = resolveExpectedInternalToken(); + return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank() + && MessageDigest.isEqual( + expectedToken.getBytes(StandardCharsets.UTF_8), + suppliedToken.trim().getBytes(StandardCharsets.UTF_8)); + } + + private String resolveExpectedInternalToken() { + if (internalToken != null && !internalToken.isBlank()) { + return internalToken.trim(); + } + Path path = resolveInternalTokenFile(); + if (path == null || !Files.isRegularFile(path)) { + return ""; + } + try { + return Files.readString(path, StandardCharsets.UTF_8).trim(); + } catch (Exception ignored) { + return ""; + } + } + + private Path resolveInternalTokenFile() { + String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim(); + if (!configuredPath.isEmpty()) { + if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) { + String userHome = System.getProperty("user.home", "").trim(); + if (userHome.isEmpty()) { + return null; + } + configuredPath = configuredPath.length() == 1 + ? userHome + : Path.of(userHome, configuredPath.substring(2)).toString(); + } + Path configuredTokenPath = Path.of(configuredPath); + return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null; + } + String userHome = System.getProperty("user.home", "").trim(); + return userHome.isEmpty() + ? null + : Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize(); + } + + private record RequestOperator(Long id, boolean superAdmin) { + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataCreateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataCreateRequest.java index a9ae9b91..90eb47dd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataCreateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataCreateRequest.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.invalidasin.model.dto; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; import lombok.Data; @@ -17,4 +18,8 @@ public class InvalidAsinDataCreateRequest { @Size(max = 128, message = "品牌长度不能超过128个字符") @Schema(description = "品牌名称") private String brand; + + @NotNull(message = "请选择分组") + @Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED) + private Long groupId; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataUpdateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataUpdateRequest.java index a5bfb713..efb25d06 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataUpdateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/dto/InvalidAsinDataUpdateRequest.java @@ -17,4 +17,7 @@ public class InvalidAsinDataUpdateRequest { @Size(max = 128, message = "品牌长度不能超过128个字符") @Schema(description = "品牌名称") private String brand; + + @Schema(description = "分组 ID；自动导入数据无需填写") + private Long groupId; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/entity/InvalidAsinDataEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/entity/InvalidAsinDataEntity.java index 8a7199c9..f2f8d1e3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/entity/InvalidAsinDataEntity.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/entity/InvalidAsinDataEntity.java @@ -15,6 +15,8 @@ public class InvalidAsinDataEntity { private Long id; private String dataValue; private String brand; + private Long groupId; + private String recordSource; private LocalDateTime createdAt; private LocalDateTime updatedAt; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/vo/InvalidAsinDataItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/vo/InvalidAsinDataItemVo.java index fb2d92b6..bf001754 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/vo/InvalidAsinDataItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/model/vo/InvalidAsinDataItemVo.java @@ -18,6 +18,15 @@ public class InvalidAsinDataItemVo { @Schema(description = "品牌名称") private String brand; + @Schema(description = "分组 ID") + private Long groupId; + + @Schema(description = "分组名称") + private String groupName; + + @Schema(description = "记录来源：MANUAL/AUTO") + private String recordSource; + @Schema(description = "创建时间") private LocalDateTime createdAt; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataService.java index 52d8504c..80948ab9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataService.java @@ -8,20 +8,30 @@ import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequ import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity; import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo; import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo; +import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity; +import com.nanri.aiimage.modules.shopkey.service.ShopManageGroupService; import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.server.ResponseStatusException; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; @Service @RequiredArgsConstructor public class InvalidAsinDataService { - private final InvalidAsinDataMapper invalidAsinDataMapper; + public static final String RECORD_SOURCE_MANUAL = "MANUAL"; + public static final String RECORD_SOURCE_AUTO = "AUTO"; - public InvalidAsinDataPageVo page(long page, long pageSize, String keyword) { + private final InvalidAsinDataMapper invalidAsinDataMapper; + private final ShopManageGroupService shopManageGroupService; + + public InvalidAsinDataPageVo page(long page, long pageSize, String keyword, Long groupId, Long operatorId, boolean superAdmin) { long safePage = Math.max(page, 1); long safePageSize = Math.min(Math.max(pageSize, 1), 100); String safeKeyword = keyword == null ? "" : keyword.trim(); @@ -31,11 +41,28 @@ public class InvalidAsinDataService { .or() .like(InvalidAsinDataEntity::getBrand, safeKeyword)) .orderByDesc(InvalidAsinDataEntity::getId); + if (!superAdmin) { + Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId); + if (fixedGroupId == null) { + query.eq(InvalidAsinDataEntity::getId, -1L); + } else { + query.eq(InvalidAsinDataEntity::getRecordSource, RECORD_SOURCE_MANUAL) + .eq(InvalidAsinDataEntity::getGroupId, fixedGroupId); + } + } else if (groupId != null && groupId > 0) { + query.eq(InvalidAsinDataEntity::getGroupId, groupId); + } Long total = invalidAsinDataMapper.selectCount(query); - List items = invalidAsinDataMapper - .selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize)) - .stream() - .map(this::toItemVo) + List rows = invalidAsinDataMapper + .selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize)); + Map groupNameById = shopManageGroupService.buildGroupNameMap(rows.stream() + .map(InvalidAsinDataEntity::getGroupId) + .filter(rowGroupId -> rowGroupId != null && rowGroupId > 0) + .distinct() + .toList()); + List items = rows.stream() + .map(entity -> toItemVo(entity, + entity.getGroupId() == null ? "" : groupNameById.getOrDefault(entity.getGroupId(), ""))) .toList(); InvalidAsinDataPageVo vo = new InvalidAsinDataPageVo(); vo.setItems(items); @@ -46,35 +73,63 @@ public class InvalidAsinDataService { } @Transactional - public InvalidAsinDataItemVo create(InvalidAsinDataCreateRequest request) { + public InvalidAsinDataItemVo create(InvalidAsinDataCreateRequest request, Long operatorId, boolean superAdmin) { + ShopManageGroupEntity group = resolveManualWriteGroup(request.getGroupId(), operatorId, superAdmin); String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空"); String brand = normalizeRequired(request.getBrand(), "品牌不能为空").toLowerCase(Locale.ROOT); ensureUnique(dataValue, null); InvalidAsinDataEntity entity = new InvalidAsinDataEntity(); entity.setDataValue(dataValue); entity.setBrand(brand); + entity.setGroupId(group.getId()); + entity.setRecordSource(RECORD_SOURCE_MANUAL); invalidAsinDataMapper.insert(entity); - return toItemVo(getById(entity.getId())); + return toItemVo(getById(entity.getId()), group.getGroupName()); } @Transactional - public InvalidAsinDataItemVo update(Long id, InvalidAsinDataUpdateRequest request) { + public InvalidAsinDataItemVo update(Long id, InvalidAsinDataUpdateRequest request, Long operatorId, boolean superAdmin) { InvalidAsinDataEntity entity = getById(id); + ShopManageGroupEntity fixedGroup = ensureRecordAccess(entity, operatorId, superAdmin); + ShopManageGroupEntity writeGroup = null; + if (isManualRecord(entity)) { + Long requestedGroupId = requireGroupId(request.getGroupId()); + writeGroup = superAdmin + ? shopManageGroupService.getAccessibleById(requestedGroupId, operatorId, true) + : resolveLockedGroup(requestedGroupId, fixedGroup); + } String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空"); String brand = normalizeRequired(request.getBrand(), "品牌不能为空").toLowerCase(Locale.ROOT); ensureUnique(dataValue, id); entity.setDataValue(dataValue); entity.setBrand(brand); + String groupName = ""; + if (writeGroup != null) { + entity.setGroupId(writeGroup.getId()); + groupName = writeGroup.getGroupName(); + } invalidAsinDataMapper.updateById(entity); - return toItemVo(getById(id)); + return toItemVo(getById(id), groupName); } @Transactional - public void delete(Long id) { + public void delete(Long id, Long operatorId, boolean superAdmin) { InvalidAsinDataEntity entity = getById(id); + ensureRecordAccess(entity, operatorId, superAdmin); invalidAsinDataMapper.deleteById(entity.getId()); } + private ShopManageGroupEntity resolveManualWriteGroup(Long requestedGroupId, Long operatorId, boolean superAdmin) { + Long normalizedRequestedGroupId = requireGroupId(requestedGroupId); + if (superAdmin) { + return shopManageGroupService.getAccessibleById( + normalizedRequestedGroupId, operatorId, true); + } + Long fixedGroupId = requireFixedAccessibleGroupId(operatorId); + ensureRequestedLockedGroup(normalizedRequestedGroupId, fixedGroupId); + return shopManageGroupService.getAccessibleById(fixedGroupId, operatorId, false); + } + private InvalidAsinDataEntity getById(Long id) { InvalidAsinDataEntity entity = invalidAsinDataMapper.selectById(id); if (entity == null) { @@ -115,11 +170,76 @@ public class InvalidAsinDataService { .replaceAll("\\s+", " "); } - private InvalidAsinDataItemVo toItemVo(InvalidAsinDataEntity entity) { + private ShopManageGroupEntity ensureRecordAccess(InvalidAsinDataEntity entity, Long operatorId, boolean superAdmin) { + if (superAdmin) { + return null; + } + if (!isManualRecord(entity) || entity.getGroupId() == null || entity.getGroupId() <= 0) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该品牌数据"); + } + Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId); + if (fixedGroupId == null || !fixedGroupId.equals(entity.getGroupId())) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作非固定分组数据"); + } + try { + return shopManageGroupService.getAccessibleById(fixedGroupId, operatorId, false); + } catch (BusinessException exception) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作该品牌数据"); + } + } + + private Long requireFixedAccessibleGroupId(Long operatorId) { + Long fixedGroupId = resolveFixedAccessibleGroupId(operatorId); + if (fixedGroupId == null) { + throw new BusinessException("当前用户没有可访问的分组"); + } + return fixedGroupId; + } + + private Long resolveFixedAccessibleGroupId(Long operatorId) { + Set accessibleGroupIds = shopManageGroupService.listAccessibleGroupIds(operatorId, false); + if (accessibleGroupIds == null) { + return null; + } + return accessibleGroupIds.stream() + .filter(groupId -> groupId != null && groupId > 0) + .min(Long::compareTo) + .orElse(null); + } + + private ShopManageGroupEntity resolveLockedGroup(Long requestedGroupId, ShopManageGroupEntity fixedGroup) { + if (fixedGroup == null || fixedGroup.getId() == null) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作非固定分组数据"); + } + ensureRequestedLockedGroup(requestedGroupId, fixedGroup.getId()); + return fixedGroup; + } + + private void ensureRequestedLockedGroup(Long requestedGroupId, Long fixedGroupId) { + if (fixedGroupId == null || !fixedGroupId.equals(requestedGroupId)) { + throw new ResponseStatusException(HttpStatus.FORBIDDEN, "无权操作非固定分组数据"); + } + } + + private Long requireGroupId(Long groupId) { + if (groupId == null || groupId <= 0) { + throw new BusinessException("请选择分组"); + } + return groupId; + } + + private boolean isManualRecord(InvalidAsinDataEntity entity) { + return entity != null && RECORD_SOURCE_MANUAL.equalsIgnoreCase(normalizeText(entity.getRecordSource())); + } + + private InvalidAsinDataItemVo toItemVo(InvalidAsinDataEntity entity, String groupName) { InvalidAsinDataItemVo vo = new InvalidAsinDataItemVo(); vo.setId(entity.getId()); vo.setDataValue(entity.getDataValue()); vo.setBrand(entity.getBrand()); + vo.setGroupId(entity.getGroupId()); + vo.setGroupName(groupName == null ? "" : groupName); + vo.setRecordSource(isManualRecord(entity) ? RECORD_SOURCE_MANUAL : RECORD_SOURCE_AUTO); vo.setCreatedAt(entity.getCreatedAt()); return vo; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyFileMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyFileMapper.java new file mode 100644 index 00000000..7152bc31 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyFileMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.shopdatacrawl.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface ShopDataCrawlDailyFileMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyMemberMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyMemberMapper.java new file mode 100644 index 00000000..b5131b12 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlDailyMemberMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.shopdatacrawl.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface ShopDataCrawlDailyMemberMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyFileEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyFileEntity.java new file mode 100644 index 00000000..6ea5f184 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyFileEntity.java @@ -0,0 +1,32 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDate; +import java.time.LocalDateTime; + +@Data +@TableName("biz_shop_data_crawl_daily_file") +public class ShopDataCrawlDailyFileEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long userId; + private String shopKeyHash; + private String shopKey; + private LocalDate businessDate; + private Long latestTaskId; + private Long latestResultId; + private String resultFilename; + private String resultFileUrl; + private Long resultFileSize; + private String resultContentType; + private Integer rowCount; + private Long version; + private LocalDateTime lastSuccessAt; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyMemberEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyMemberEntity.java new file mode 100644 index 00000000..783975c3 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyMemberEntity.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_shop_data_crawl_daily_member") +public class ShopDataCrawlDailyMemberEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long dailyFileId; + private Long taskId; + private Long resultId; + private LocalDateTime createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java new file mode 100644 index 00000000..75518836 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java @@ -0,0 +1,218 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlDailyFileMapper; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlDailyMemberMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import lombok.RequiredArgsConstructor; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; + +/** + * Persistence and locking primitives for the shop-data-crawl daily workbook. + * Workbook assembly remains in ShopDataCrawlTaskService so task state changes + * and the daily pointer are committed together. + */ +@Service +@RequiredArgsConstructor +public class ShopDataCrawlDailyFileService { + + public static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai"); + private static final Duration DAILY_LOCK_TTL = Duration.ofMinutes(30); + private static final long DAILY_LOCK_WAIT_MILLIS = 15000L; + private static final String DAILY_LOCK_MODULE_PREFIX = "SHOP_DATA_CRAWL_DAILY_"; + + private final ShopDataCrawlDailyFileMapper dailyFileMapper; + private final ShopDataCrawlDailyMemberMapper dailyMemberMapper; + private final TaskDistributedLockService taskDistributedLockService; + + public LocalDate currentBusinessDate() { + return LocalDate.now(BUSINESS_ZONE); + } + + public LocalDateTime currentBusinessDateTime() { + return LocalDateTime.now(BUSINESS_ZONE); + } + + public String shopKey(FileResultEntity row) { + if (row == null) { + return null; + } + String shopId = trimToNull(row.getSourceFileUrl()); + if (shopId != null) { + return "shop-id:" + shopId; + } + String shopName = trimToNull(row.getSourceFilename()); + return shopName == null ? null : "shop-name:" + shopName; + } + + public String shopKeyHash(String shopKey) { + if (shopKey == null || shopKey.isBlank()) { + return null; + } + byte[] digest; + try { + digest = MessageDigest.getInstance("SHA-256") + .digest(shopKey.getBytes(StandardCharsets.UTF_8)); + } catch (Exception ex) { + throw new IllegalStateException("SHA-256 is unavailable", ex); + } + return HexFormat.of().formatHex(digest); + } + + public TaskDistributedLockService.LockHandle acquireLock(Long userId, String shopKey) { + if (userId == null || userId <= 0 || shopKey == null || shopKey.isBlank()) { + return null; + } + // The lock protects the shop's whole daily-file lifecycle. Including the + // date would allow yesterday and today to update the same shop together. + String identity = userId + "|" + shopKey; + String lockModule = DAILY_LOCK_MODULE_PREFIX + shopKeyHash(identity); + return taskDistributedLockService.acquire(lockModule, 1L, DAILY_LOCK_TTL, DAILY_LOCK_WAIT_MILLIS); + } + + public ShopDataCrawlDailyFileEntity findForUpdate(Long userId, String shopKeyHash, LocalDate businessDate) { + if (userId == null || shopKeyHash == null || businessDate == null) { + return null; + } + return dailyFileMapper.selectOne(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyFileEntity::getUserId, userId) + .eq(ShopDataCrawlDailyFileEntity::getShopKeyHash, shopKeyHash) + .eq(ShopDataCrawlDailyFileEntity::getBusinessDate, businessDate) + .last("FOR UPDATE")); + } + + public List findOlder(Long userId, String shopKeyHash, LocalDate businessDate) { + if (userId == null || shopKeyHash == null || businessDate == null) { + return List.of(); + } + return dailyFileMapper.selectList(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyFileEntity::getUserId, userId) + .eq(ShopDataCrawlDailyFileEntity::getShopKeyHash, shopKeyHash) + .lt(ShopDataCrawlDailyFileEntity::getBusinessDate, businessDate) + .orderByDesc(ShopDataCrawlDailyFileEntity::getBusinessDate) + .orderByDesc(ShopDataCrawlDailyFileEntity::getId) + .last("FOR UPDATE")); + } + + public List findByLatestResultId(Long resultId) { + if (resultId == null || resultId <= 0) { + return List.of(); + } + return dailyFileMapper.selectList(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyFileEntity::getLatestResultId, resultId) + .last("FOR UPDATE")); + } + + public ShopDataCrawlDailyFileEntity findById(Long dailyFileId) { + return dailyFileId == null || dailyFileId <= 0 ? null : dailyFileMapper.selectById(dailyFileId); + } + + public List findMembersByResultId(Long resultId) { + if (resultId == null || resultId <= 0) { + return List.of(); + } + return dailyMemberMapper.selectList(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyMemberEntity::getResultId, resultId)); + } + + public boolean containsResult(Long dailyFileId, Long resultId) { + if (dailyFileId == null || dailyFileId <= 0 || resultId == null || resultId <= 0) { + return false; + } + return dailyMemberMapper.selectCount(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyMemberEntity::getDailyFileId, dailyFileId) + .eq(ShopDataCrawlDailyMemberEntity::getResultId, resultId)) > 0; + } + + public boolean addMember(Long dailyFileId, Long taskId, Long resultId) { + if (dailyFileId == null || dailyFileId <= 0 || taskId == null || taskId <= 0 + || resultId == null || resultId <= 0) { + return false; + } + ShopDataCrawlDailyMemberEntity member = new ShopDataCrawlDailyMemberEntity(); + member.setDailyFileId(dailyFileId); + member.setTaskId(taskId); + member.setResultId(resultId); + member.setCreatedAt(currentBusinessDateTime()); + try { + dailyMemberMapper.insert(member); + return true; + } catch (DuplicateKeyException ignored) { + return false; + } + } + + public List listMembers(Long dailyFileId) { + if (dailyFileId == null || dailyFileId <= 0) { + return List.of(); + } + return dailyMemberMapper.selectList(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyMemberEntity::getDailyFileId, dailyFileId) + .orderByDesc(ShopDataCrawlDailyMemberEntity::getCreatedAt) + .orderByDesc(ShopDataCrawlDailyMemberEntity::getId) + .last("FOR UPDATE")); + } + + public void deleteMembers(Long dailyFileId) { + if (dailyFileId == null || dailyFileId <= 0) { + return; + } + dailyMemberMapper.delete(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyMemberEntity::getDailyFileId, dailyFileId)); + } + + public void deleteMembersForResults(Set resultIds) { + if (resultIds == null || resultIds.isEmpty()) { + return; + } + dailyMemberMapper.delete(new LambdaQueryWrapper() + .in(ShopDataCrawlDailyMemberEntity::getResultId, resultIds)); + } + + public long countObjectReferences(String objectKey) { + if (objectKey == null || objectKey.isBlank()) { + return 0L; + } + Long count = dailyFileMapper.selectCount(new LambdaQueryWrapper() + .eq(ShopDataCrawlDailyFileEntity::getResultFileUrl, objectKey)); + return count == null ? 0L : count; + } + + public void deleteDailyFile(Long dailyFileId) { + if (dailyFileId == null || dailyFileId <= 0) { + return; + } + deleteMembers(dailyFileId); + dailyFileMapper.deleteById(dailyFileId); + } + + public void insert(ShopDataCrawlDailyFileEntity entity) { + dailyFileMapper.insert(entity); + } + + public void update(ShopDataCrawlDailyFileEntity entity) { + dailyFileMapper.updateById(entity); + } + + private String trimToNull(String value) { + if (value == null) { + return null; + } + String normalized = value.trim(); + return normalized.isEmpty() ? null : normalized; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java index 5d2a45a9..ae694684 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java @@ -19,6 +19,7 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.stereotype.Service; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; import java.util.ArrayList; @@ -64,6 +65,36 @@ public class ShopDataCrawlExcelAssemblyService { } } + /** + * Appends the supplied task rows to an already assembled daily workbook. + * Existing rows, styles and drawings are deliberately left untouched. + */ + public void appendWorkbook(File baseXlsx, File outputXlsx, List items) { + if (baseXlsx == null || !baseXlsx.isFile()) { + throw new BusinessException("当天累计文件不存在"); + } + try (InputStream input = new FileInputStream(baseXlsx); + XSSFWorkbook workbook = new XSSFWorkbook(input); + FileOutputStream output = new FileOutputStream(outputXlsx)) { + validateTemplate(workbook); + Map> rowsByCountry = rowsByCountry(items); + Map imageCache = new ConcurrentHashMap<>(); + imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache); + Map pictureIndexes = new LinkedHashMap<>(); + for (int i = 0; i < COUNTRIES.size(); i++) { + List rows = rowsByCountry.get(COUNTRIES.get(i)); + if (rows != null && !rows.isEmpty()) { + appendSheet(workbook, workbook.getSheetAt(i), rows, imageCache, pictureIndexes); + } + } + workbook.write(output); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("店铺数据抓取累计 Excel 追加失败: " + ex.getMessage()); + } + } + public int countRows(List items) { return rowsByCountry(items).values().stream().mapToInt(List::size).sum(); } @@ -141,6 +172,50 @@ public class ShopDataCrawlExcelAssemblyService { } } + private void appendSheet(XSSFWorkbook workbook, + Sheet sheet, + List rows, + Map imageCache, + Map pictureIndexes) { + Row header = sheet.getRow(0); + Row styleRow = sheet.getRow(1); + boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN)); + boolean templateHasBrand = header != null && "品牌".equals(cellText(header, BRAND_COLUMN)); + CellStyle[] styles = new CellStyle[HEADERS.size()]; + for (int column = 0; column < styles.length; column++) { + int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand); + Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn); + styles[column] = cell == null ? null : cell.getCellStyle(); + } + writeHeaders(sheet, currentTemplate, templateHasBrand); + sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH); + int rowIndex = Math.max(1, sheet.getLastRowNum() + 1); + for (ShopDataCrawlRowDto value : rows) { + Row row = sheet.createRow(rowIndex++); + writeRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes); + } + } + + private void writeRow(XSSFWorkbook workbook, + Sheet sheet, + Row row, + ShopDataCrawlRowDto value, + CellStyle[] styles, + Map imageCache, + Map pictureIndexes) { + String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(), + value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()}; + for (int column = 0; column < values.length; column++) { + Cell cell = row.createCell(column); + if (styles[column] != null) cell.setCellStyle(styles[column]); + cell.setCellValue(values[column] == null ? "" : values[column]); + } + if (!blank(value.getCommodityImage())) { + row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS); + embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes); + } + } + private void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) { Row header = sheet.getRow(0); if (header == null) header = sheet.createRow(0); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java index 6a71f4a8..c509ab21 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java @@ -17,6 +17,8 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTask import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo; @@ -45,16 +47,24 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DuplicateKeyException; import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import java.io.File; +import java.nio.file.Files; import java.time.LocalDateTime; import java.time.Duration; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; @Service @RequiredArgsConstructor @@ -66,7 +76,6 @@ public class ShopDataCrawlTaskService { private static final int RESULT_PENDING = -1; private static final int RESULT_FAILED = 0; private static final int RESULT_SUCCESS = 1; - private static final int SHOP_HISTORY_RETENTION_LIMIT = 3; private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断"; private static final String PARTIAL_RESULT_MESSAGE = "Python 中断，已保留已回传的部分数据"; private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:"; @@ -88,6 +97,7 @@ public class ShopDataCrawlTaskService { private final TaskScopeStateMapper taskScopeStateMapper; private final TransientPayloadStorageService transientPayloadStorageService; private final InstanceMetadata instanceMetadata; + private final ShopDataCrawlDailyFileService dailyFileService; @Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}") private long staleTimeoutMinutes; @@ -601,18 +611,29 @@ public class ShopDataCrawlTaskService { } ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task"); try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - List resultFileUrls = listTaskRows(taskId).stream() - .map(FileResultEntity::getResultFileUrl).filter(url -> !blank(url)).distinct().toList(); - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); - taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); - taskProgressSnapshotService.delete(taskId, MODULE_TYPE); - fileTaskMapper.deleteById(taskId); - taskCacheService.deleteTaskCache(taskId); - deleteTransientResultChunks(taskId); - resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced); + List taskRows = listTaskRows(taskId); + try (DailyLockSet dailyLocks = acquireDailyLocks(task.getUserId(), taskRows)) { + Set removedResultIds = taskRows.stream() + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .collect(java.util.stream.Collectors.toSet()); + DailyDeletionResult dailyResult = prepareDailyForDeletion(removedResultIds); + registerUploadedObjectRollback(dailyResult.uploadedObjectKeys()); + List resultFileUrls = new ArrayList<>(dailyResult.obsoleteObjectKeys()); + resultFileUrls.addAll(taskRows.stream() + .map(FileResultEntity::getResultFileUrl).filter(url -> !blank(url)).distinct().toList()); + resultFileUrls = resultFileUrls.stream().filter(url -> !blank(url)).distinct().toList(); + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); + taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); + taskProgressSnapshotService.delete(taskId, MODULE_TYPE); + fileTaskMapper.deleteById(taskId); + taskCacheService.deleteTaskCache(taskId); + deleteTransientResultChunks(taskId); + resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced); + } } } @@ -632,7 +653,9 @@ public class ShopDataCrawlTaskService { if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { throw new BusinessException("记录不存在"); } - deleteResultHistoryRow(latestEntity); + try (DailyLockSet dailyLocks = acquireDailyLocks(userId, List.of(latestEntity))) { + deleteResultHistoryRow(latestEntity); + } } } @@ -661,6 +684,8 @@ public class ShopDataCrawlTaskService { Long taskId = entity.getTaskId(); Long resultId = entity.getId(); String resultFileUrl = entity.getResultFileUrl(); + DailyDeletionResult dailyResult = prepareDailyForDeletion(Set.of(resultId)); + registerUploadedObjectRollback(dailyResult.uploadedObjectKeys()); taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId); taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId); fileResultMapper.deleteById(resultId); @@ -673,7 +698,161 @@ public class ShopDataCrawlTaskService { taskId, resultId, safeMessage(ex)); } // Run this after the row delete so shared object references are counted correctly. - deleteResultObjectIfUnreferenced(resultFileUrl); + Set obsoleteObjectKeys = new HashSet<>(dailyResult.obsoleteObjectKeys()); + collectObjectKey(obsoleteObjectKeys, resultFileUrl); + obsoleteObjectKeys.forEach(this::deleteResultObjectIfUnreferenced); + } + + private DailyDeletionResult prepareDailyForDeletion(Set removedResultIds) { + if (removedResultIds == null || removedResultIds.isEmpty()) { + return new DailyDeletionResult(List.of(), List.of()); + } + Set handledDailyIds = new HashSet<>(); + Set obsoleteObjectKeys = new HashSet<>(); + List uploadedObjectKeys = new ArrayList<>(); + try { + for (Long resultId : removedResultIds) { + List latestFiles = dailyFileService.findByLatestResultId(resultId); + List affectedFiles = new ArrayList<>( + latestFiles == null ? List.of() : latestFiles); + List memberRows = dailyFileService.findMembersByResultId(resultId); + for (ShopDataCrawlDailyMemberEntity member : memberRows == null ? List.of() : memberRows) { + if (member == null) { + continue; + } + ShopDataCrawlDailyFileEntity memberFile = dailyFileService.findById(member.getDailyFileId()); + if (memberFile != null) { + affectedFiles.add(memberFile); + } + } + for (ShopDataCrawlDailyFileEntity dailyFile : affectedFiles) { + if (dailyFile == null || dailyFile.getId() == null || !handledDailyIds.add(dailyFile.getId())) { + continue; + } + ShopDataCrawlDailyFileEntity lockedDailyFile = dailyFileService.findForUpdate( + dailyFile.getUserId(), dailyFile.getShopKeyHash(), dailyFile.getBusinessDate()); + if (lockedDailyFile != null) { + dailyFile = lockedDailyFile; + } + List survivors = loadDailyMemberData(dailyFile, removedResultIds); + if (survivors.isEmpty()) { + collectObjectKey(obsoleteObjectKeys, dailyFile.getResultFileUrl()); + dailyFileService.deleteDailyFile(dailyFile.getId()); + continue; + } + rebuildDailyFileAfterDeletion(dailyFile, survivors, uploadedObjectKeys, obsoleteObjectKeys); + } + } + dailyFileService.deleteMembersForResults(removedResultIds); + return new DailyDeletionResult(new ArrayList<>(obsoleteObjectKeys), uploadedObjectKeys); + } catch (RuntimeException ex) { + registerRollbackObjectCleanup(uploadedObjectKeys); + throw ex; + } + } + + private List loadDailyMemberData(ShopDataCrawlDailyFileEntity dailyFile, + Set removedResultIds) { + List memberRows = dailyFileService.listMembers(dailyFile.getId()); + List members = new ArrayList<>( + memberRows == null ? List.of() : memberRows); + members.removeIf(Objects::isNull); + members.sort(Comparator + .comparing(ShopDataCrawlDailyMemberEntity::getCreatedAt, + Comparator.nullsLast(Comparator.naturalOrder())) + .thenComparing(ShopDataCrawlDailyMemberEntity::getId, + Comparator.nullsLast(Comparator.naturalOrder()))); + List survivors = new ArrayList<>(); + for (ShopDataCrawlDailyMemberEntity member : members) { + if (removedResultIds.contains(member.getResultId())) { + continue; + } + FileResultEntity result = fileResultMapper.selectById(member.getResultId()); + if (result == null || !Integer.valueOf(RESULT_SUCCESS).equals(result.getSuccess())) { + continue; + } + ShopDataCrawlResultItemVo snapshot = loadSnapshotForDailyMember(result); + if (snapshot == null || !Boolean.TRUE.equals(snapshot.getSuccess())) { + throw new BusinessException("无法读取累计文件中的剩余结果"); + } + survivors.add(new DailyMemberData(member, result, snapshot)); + } + return survivors; + } + + private ShopDataCrawlResultItemVo loadSnapshotForDailyMember(FileResultEntity result) { + ShopDataCrawlResultItemVo snapshot = taskResultItemService.getResultSnapshot( + result.getTaskId(), MODULE_TYPE, result.getId(), ShopDataCrawlResultItemVo.class); + if (snapshot != null) { + return snapshot; + } + FileTaskEntity task = fileTaskMapper.selectById(result.getTaskId()); + if (task == null) { + return null; + } + return indexSnapshotByResultId(buildSnapshotFromDb(task, List.of(result))).get(result.getId()); + } + + private void rebuildDailyFileAfterDeletion(ShopDataCrawlDailyFileEntity dailyFile, + List survivors, + List uploadedObjectKeys, + Set obsoleteObjectKeys) { + DailyMemberData latest = survivors.get(survivors.size() - 1); + FileTaskEntity latestTask = fileTaskMapper.selectById(latest.result().getTaskId()); + String filename = blank(dailyFile.getResultFilename()) + ? buildTaskWorkbookFilename(latestTask) + : dailyFile.getResultFilename(); + File workRoot = FileUtil.mkdir(FileUtil.file( + System.getProperty("java.io.tmpdir"), + "shop-data-crawl-result", + "daily-delete-" + UUID.randomUUID())); + File outputXlsx = FileUtil.file(workRoot, filename); + String oldObjectKey = dailyFile.getResultFileUrl(); + List snapshots = survivors.stream() + .map(DailyMemberData::snapshot) + .toList(); + try { + excelAssemblyService.writeWorkbook(outputXlsx, snapshots); + String newObjectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE); + if (blank(newObjectKey)) { + throw new BusinessException("累计文件上传结果为空"); + } + uploadedObjectKeys.add(newObjectKey); + int rowCount = excelAssemblyService.countRows(snapshots); + for (DailyMemberData survivor : survivors) { + FileResultEntity result = survivor.result(); + if (!Objects.equals(result.getId(), latest.result().getId()) && !blank(result.getResultFileUrl())) { + result.setResultFileUrl(null); + result.setResultFileSize(null); + result.setResultContentType(null); + fileResultMapper.updateById(result); + } + } + latest.result().setResultFilename(filename); + latest.result().setResultFileUrl(newObjectKey); + latest.result().setResultFileSize(outputXlsx.length()); + latest.result().setResultContentType(CONTENT_TYPE_XLSX); + latest.result().setRowCount(rowCount); + fileResultMapper.updateById(latest.result()); + + LocalDateTime now = dailyFileService.currentBusinessDateTime(); + dailyFile.setLatestTaskId(latest.result().getTaskId()); + dailyFile.setLatestResultId(latest.result().getId()); + dailyFile.setResultFilename(filename); + dailyFile.setResultFileUrl(newObjectKey); + dailyFile.setResultFileSize(latest.result().getResultFileSize()); + dailyFile.setResultContentType(CONTENT_TYPE_XLSX); + dailyFile.setRowCount(rowCount); + dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L); + dailyFile.setLastSuccessAt(now); + dailyFile.setUpdatedAt(now); + dailyFileService.update(dailyFile); + collectObjectKey(obsoleteObjectKeys, oldObjectKey); + obsoleteObjectKeys.remove(newObjectKey); + } finally { + FileUtil.del(outputXlsx); + FileUtil.del(workRoot); + } } private void reconcileTaskAfterResultRemoval(Long taskId) { @@ -700,131 +879,6 @@ public class ShopDataCrawlTaskService { taskCacheService.saveTaskCache(task); } - private void pruneCompletedHistoryQuietly(FileTaskEntity currentTask, List currentRows) { - if (currentRows == null || currentRows.isEmpty()) { - return; - } - for (FileResultEntity row : currentRows) { - if (!isRetentionCandidate(row)) { - continue; - } - Long userId = row.getUserId() != null - ? row.getUserId() - : currentTask == null ? null : currentTask.getUserId(); - String shopKey = retentionShopKey(row); - if (userId == null || shopKey == null) { - log.warn("[shop-data-crawl] skip history retention because ownership key is incomplete taskId={} resultId={}", - currentTask == null ? null : currentTask.getId(), row.getId()); - continue; - } - try { - pruneCompletedHistoryForShop(userId, shopKey); - } catch (Exception ex) { - // Retention is best effort. A cleanup failure must not fail the newly assembled workbook job. - log.warn("[shop-data-crawl] history retention failed taskId={} resultId={} msg={}", - currentTask == null ? null : currentTask.getId(), row.getId(), safeMessage(ex)); - } - } - } - - void pruneCompletedHistoryForShop(Long userId, String shopKey) { - if (userId == null || shopKey == null) { - return; - } - List candidates = fileResultMapper.selectList(new LambdaQueryWrapper() - .select(FileResultEntity::getId, - FileResultEntity::getTaskId, - FileResultEntity::getModuleType, - FileResultEntity::getSourceFilename, - FileResultEntity::getSourceFileUrl, - FileResultEntity::getResultFileUrl, - FileResultEntity::getSuccess, - FileResultEntity::getUserId, - FileResultEntity::getCreatedAt) - .eq(FileResultEntity::getModuleType, MODULE_TYPE) - .eq(FileResultEntity::getUserId, userId) - .eq(FileResultEntity::getSuccess, RESULT_SUCCESS) - .isNotNull(FileResultEntity::getResultFileUrl) - .ne(FileResultEntity::getResultFileUrl, "") - .orderByDesc(FileResultEntity::getCreatedAt) - .orderByDesc(FileResultEntity::getId)); - if (candidates == null || candidates.isEmpty()) { - return; - } - - List shopResults = new ArrayList<>(); - for (FileResultEntity candidate : candidates) { - if (!isRetentionCandidate(candidate) - || !Objects.equals(userId, candidate.getUserId())) { - continue; - } - if (!Objects.equals(shopKey, retentionShopKey(candidate))) { - continue; - } - shopResults.add(candidate); - } - - Comparator newestFirst = Comparator - .comparing(FileResultEntity::getCreatedAt, Comparator.nullsLast(Comparator.reverseOrder())) - .thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder())); - shopResults.sort(newestFirst); - for (int index = SHOP_HISTORY_RETENTION_LIMIT; index < shopResults.size(); index++) { - deleteRetentionResultQuietly(shopResults.get(index), userId, shopKey); - } - } - - private void deleteRetentionResultQuietly(FileResultEntity candidate, Long userId, String shopKey) { - if (candidate == null || candidate.getId() == null || candidate.getId() <= 0 - || candidate.getTaskId() == null || candidate.getTaskId() <= 0) { - return; - } - try { - TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(candidate.getTaskId()); - if (lockHandle == null) { - log.info("[shop-data-crawl] skip retained-history deletion because task lock is busy taskId={} resultId={}", - candidate.getTaskId(), candidate.getId()); - return; - } - try (lockHandle) { - FileResultEntity latest = fileResultMapper.selectById(candidate.getId()); - if (!isRetentionCandidate(latest) - || !Objects.equals(userId, latest.getUserId()) - || !Objects.equals(shopKey, retentionShopKey(latest))) { - return; - } - FileTaskEntity task = fileTaskMapper.selectById(candidate.getTaskId()); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) - || !isTerminalTaskStatus(task.getStatus())) { - log.info("[shop-data-crawl] skip retained-history deletion because task is not terminal taskId={} resultId={}", - candidate.getTaskId(), candidate.getId()); - return; - } - deleteResultHistoryRow(latest); - } - } catch (Exception ex) { - log.warn("[shop-data-crawl] retained-history deletion failed taskId={} resultId={} msg={}", - candidate.getTaskId(), candidate.getId(), safeMessage(ex)); - } - } - - private boolean isRetentionCandidate(FileResultEntity row) { - return row != null - && Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) - && !blank(row.getResultFileUrl()); - } - - private String retentionShopKey(FileResultEntity row) { - if (row == null) { - return null; - } - String shopId = trimToNull(row.getSourceFileUrl()); - if (shopId != null) { - return "shop-id:" + shopId; - } - String shopName = trimToNull(row.getSourceFilename()); - return shopName == null ? null : "shop-name:" + shopName; - } - private String trimToNull(String value) { if (value == null) { return null; @@ -1519,6 +1573,7 @@ public class ShopDataCrawlTaskService { } } + @Transactional public void processResultFileJob(TaskFileJobEntity job) { if (job == null || job.getTaskId() == null) { throw new BusinessException("结果文件任务参数不完整"); @@ -1539,31 +1594,434 @@ public class ShopDataCrawlTaskService { if (successItems.isEmpty()) { throw new BusinessException("没有可生成的店铺数据抓取结果"); } - File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "shop-data-crawl-result", String.valueOf(task.getId()))); - String filename = buildTaskWorkbookFilename(task); - File xlsx = FileUtil.file(workRoot, filename); + Map snapshotsByResultId = indexSnapshotByResultId(successItems); + LocalDate businessDate = dailyFileService.currentBusinessDate(); + List uploadedObjectKeys = new ArrayList<>(); + List obsoleteObjectKeys = new ArrayList<>(); try { - excelAssemblyService.writeWorkbook(xlsx, successItems); - String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); - long fileSize = xlsx.length(); - int rowCount = excelAssemblyService.countRows(successItems); for (FileResultEntity row : rows) { - if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { - row.setResultFilename(filename); - row.setResultFileUrl(objectKey); - row.setResultFileSize(fileSize); - row.setResultContentType(CONTENT_TYPE_XLSX); - row.setRowCount(rowCount); - fileResultMapper.updateById(row); + if (!Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { + continue; } + ShopDataCrawlResultItemVo snapshot = snapshotsByResultId.get(row.getId()); + if (snapshot == null) { + snapshot = successItems.size() == 1 ? successItems.get(0) : null; + } + if (snapshot == null) { + throw new BusinessException("店铺结果快照不存在，无法生成累计文件"); + } + DailyAggregationResult result = aggregateDailyResult( + task, row, snapshot, businessDate, uploadedObjectKeys); + obsoleteObjectKeys.addAll(result.obsoleteObjectKeys()); } updateTaskStatusFromRows(task, rows); persistSnapshotJson(task, buildSnapshotFromDb(task, rows)); fileTaskMapper.updateById(task); - } finally { - FileUtil.del(xlsx); + } catch (RuntimeException ex) { + registerRollbackObjectCleanup(uploadedObjectKeys); + throw ex; } - pruneCompletedHistoryQuietly(task, rows); + registerDailyObjectLifecycle(uploadedObjectKeys, obsoleteObjectKeys); + } + + private DailyAggregationResult aggregateDailyResult(FileTaskEntity task, + FileResultEntity row, + ShopDataCrawlResultItemVo snapshot, + LocalDate businessDate, + List uploadedObjectKeys) { + Long userId = row.getUserId() != null ? row.getUserId() : task.getUserId(); + String shopKey = dailyFileService.shopKey(row); + String shopKeyHash = dailyFileService.shopKeyHash(shopKey); + if (userId == null || shopKeyHash == null) { + throw new BusinessException("店铺累计文件归属信息不完整"); + } + TaskDistributedLockService.LockHandle lock = dailyFileService.acquireLock(userId, shopKey); + if (lock == null) { + throw new BusinessException("店铺当天累计文件正在处理中，请稍后重试"); + } + try { + ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(userId, shopKeyHash, businessDate); + ShopDataCrawlDailyFileEntity existingMembership = findExistingDailyMembership(row.getId()); + if (existingMembership != null) { + boolean currentFileOwnsMembership = dailyFile != null + && Objects.equals(dailyFile.getId(), existingMembership.getId()); + if (Objects.equals(existingMembership.getLatestResultId(), row.getId()) + && (dailyFile == null || currentFileOwnsMembership)) { + attachCanonicalResult(row, existingMembership); + } else if (!blank(row.getResultFileUrl())) { + row.setResultFileUrl(null); + row.setResultFileSize(null); + row.setResultContentType(null); + fileResultMapper.updateById(row); + } + return new DailyAggregationResult(List.of()); + } + if (dailyFile != null && dailyFileService.containsResult(dailyFile.getId(), row.getId())) { + if (Objects.equals(dailyFile.getLatestResultId(), row.getId())) { + attachCanonicalResult(row, dailyFile); + } else if (!blank(row.getResultFileUrl())) { + row.setResultFileUrl(null); + row.setResultFileSize(null); + row.setResultContentType(null); + fileResultMapper.updateById(row); + } + return new DailyAggregationResult(List.of()); + } + + List olderFiles = dailyFileService.findOlder(userId, shopKeyHash, businessDate); + Set obsoleteObjectKeys = new HashSet<>(); + collectObjectKey(obsoleteObjectKeys, dailyFile == null ? null : dailyFile.getResultFileUrl()); + for (ShopDataCrawlDailyFileEntity older : olderFiles) { + collectObjectKey(obsoleteObjectKeys, older.getResultFileUrl()); + } + + int addedRowCount = excelAssemblyService.countRows(List.of(snapshot)); + String filename = dailyFile != null && !blank(dailyFile.getResultFilename()) + ? dailyFile.getResultFilename() + : buildTaskWorkbookFilename(task); + File workRoot = FileUtil.mkdir(FileUtil.file( + System.getProperty("java.io.tmpdir"), + "shop-data-crawl-result", + String.valueOf(task.getId()), + "daily-" + UUID.randomUUID())); + File baseXlsx = FileUtil.file(workRoot, "base.xlsx"); + File outputXlsx = FileUtil.file(workRoot, filename); + String objectKey = null; + try { + if (dailyFile != null && !blank(dailyFile.getResultFileUrl())) { + try { + Files.write(baseXlsx.toPath(), ossStorageService.readObjectBytes(dailyFile.getResultFileUrl())); + } catch (Exception ex) { + throw new BusinessException("读取当天累计文件失败: " + safeMessage(ex)); + } + if (addedRowCount > 0) { + excelAssemblyService.appendWorkbook(baseXlsx, outputXlsx, List.of(snapshot)); + } else { + try { + Files.copy(baseXlsx.toPath(), outputXlsx.toPath()); + } catch (Exception ex) { + throw new BusinessException("复制当天累计文件失败: " + safeMessage(ex)); + } + } + } else { + excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot)); + } + if (dailyFile != null && addedRowCount == 0) { + objectKey = dailyFile.getResultFileUrl(); + } else { + objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE); + uploadedObjectKeys.add(objectKey); + } + + List shopRows = findShopResultRows(userId, row); + clearShopResultPointers(shopRows, row.getId()); + row.setResultFilename(filename); + row.setResultFileUrl(objectKey); + row.setResultFileSize(outputXlsx.length()); + row.setResultContentType(CONTENT_TYPE_XLSX); + row.setRowCount(dailyFile == null + ? addedRowCount + : Math.max(0, Objects.requireNonNullElse(dailyFile.getRowCount(), 0)) + addedRowCount); + fileResultMapper.updateById(row); + + LocalDateTime now = dailyFileService.currentBusinessDateTime(); + if (dailyFile == null) { + dailyFile = new ShopDataCrawlDailyFileEntity(); + dailyFile.setUserId(userId); + dailyFile.setShopKeyHash(shopKeyHash); + dailyFile.setShopKey(shopKey); + dailyFile.setBusinessDate(businessDate); + dailyFile.setVersion(1L); + dailyFile.setCreatedAt(now); + } else { + dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L); + } + dailyFile.setLatestTaskId(row.getTaskId()); + dailyFile.setLatestResultId(row.getId()); + dailyFile.setResultFilename(filename); + dailyFile.setResultFileUrl(objectKey); + dailyFile.setResultFileSize(row.getResultFileSize()); + dailyFile.setResultContentType(CONTENT_TYPE_XLSX); + dailyFile.setRowCount(row.getRowCount()); + dailyFile.setLastSuccessAt(now); + dailyFile.setUpdatedAt(now); + if (dailyFile.getId() == null) { + dailyFileService.insert(dailyFile); + } else { + dailyFileService.update(dailyFile); + } + if (!dailyFileService.addMember(dailyFile.getId(), row.getTaskId(), row.getId())) { + throw new BusinessException("结果已归档，请重试文件任务"); + } + + for (ShopDataCrawlDailyFileEntity older : olderFiles) { + dailyFileService.deleteDailyFile(older.getId()); + } + obsoleteObjectKeys.remove(objectKey); + return new DailyAggregationResult(new ArrayList<>(obsoleteObjectKeys)); + } finally { + FileUtil.del(baseXlsx); + FileUtil.del(outputXlsx); + FileUtil.del(workRoot); + } + } finally { + releaseDailyLockAfterTransaction(lock); + } + } + + private ShopDataCrawlDailyFileEntity findExistingDailyMembership(Long resultId) { + List members = dailyFileService.findMembersByResultId(resultId); + for (ShopDataCrawlDailyMemberEntity member : members == null ? List.of() : members) { + if (member == null) { + continue; + } + ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findById(member.getDailyFileId()); + if (dailyFile != null) { + return dailyFile; + } + } + return null; + } + + private void attachCanonicalResult(FileResultEntity row, ShopDataCrawlDailyFileEntity dailyFile) { + if (row == null || dailyFile == null) { + return; + } + row.setResultFilename(dailyFile.getResultFilename()); + row.setResultFileUrl(dailyFile.getResultFileUrl()); + row.setResultFileSize(dailyFile.getResultFileSize()); + row.setResultContentType(dailyFile.getResultContentType()); + row.setRowCount(dailyFile.getRowCount()); + fileResultMapper.updateById(row); + } + + private List findShopResultRows(Long userId, FileResultEntity sourceRow) { + if (userId == null || sourceRow == null) { + return List.of(); + } + String shopId = trimToNull(sourceRow.getSourceFileUrl()); + String shopName = trimToNull(sourceRow.getSourceFilename()); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .and(owner -> owner.eq(FileResultEntity::getUserId, userId) + .or().isNull(FileResultEntity::getUserId)); + if (shopId != null) { + wrapper.eq(FileResultEntity::getSourceFileUrl, shopId); + } else if (shopName != null) { + wrapper.eq(FileResultEntity::getSourceFilename, shopName) + .apply("TRIM(COALESCE(source_file_url, '')) = ''"); + } else { + return List.of(); + } + List candidates = fileResultMapper.selectList(wrapper); + if (candidates == null || candidates.isEmpty()) { + return List.of(); + } + Map legacyTaskOwners = loadTaskMapByIds(candidates.stream() + .filter(candidate -> candidate != null && candidate.getUserId() == null) + .map(FileResultEntity::getTaskId) + .filter(Objects::nonNull) + .distinct() + .toList()); + return candidates.stream() + .filter(Objects::nonNull) + .filter(candidate -> Objects.equals(userId, candidate.getUserId()) + || Objects.equals(userId, + legacyTaskOwners.get(candidate.getTaskId()) == null + ? null + : legacyTaskOwners.get(candidate.getTaskId()).getUserId())) + .toList(); + } + + private void clearShopResultPointers(List rows, Long keepResultId) { + if (rows == null) { + return; + } + for (FileResultEntity candidate : rows) { + if (candidate == null || Objects.equals(candidate.getId(), keepResultId) + || blank(candidate.getResultFileUrl())) { + continue; + } + candidate.setResultFileUrl(null); + candidate.setResultFileSize(null); + candidate.setResultContentType(null); + fileResultMapper.updateById(candidate); + } + } + + private void collectObjectKey(Set target, String value) { + if (target == null || blank(value)) { + return; + } + target.add(value); + } + + private void registerDailyObjectLifecycle(List uploadedObjectKeys, List obsoleteObjectKeys) { + Set uploaded = uploadedObjectKeys == null ? Set.of() : new HashSet<>(uploadedObjectKeys); + Set obsolete = obsoleteObjectKeys == null ? new HashSet<>() : new HashSet<>(obsoleteObjectKeys); + obsolete.removeAll(uploaded); + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + obsolete.forEach(this::deleteObjectQuietly); + return; + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + obsolete.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly); + } + + @Override + public void afterCompletion(int status) { + if (status != STATUS_COMMITTED) { + uploaded.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly); + } + } + }); + } + + private void registerRollbackObjectCleanup(List uploadedObjectKeys) { + if (uploadedObjectKeys == null || uploadedObjectKeys.isEmpty()) { + return; + } + Set uploaded = new HashSet<>(uploadedObjectKeys); + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + uploaded.forEach(this::deleteObjectQuietly); + return; + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + if (status != STATUS_COMMITTED) { + uploaded.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly); + } + } + }); + } + + private void registerUploadedObjectRollback(List uploadedObjectKeys) { + if (uploadedObjectKeys == null || uploadedObjectKeys.isEmpty() + || !TransactionSynchronizationManager.isSynchronizationActive()) { + return; + } + Set uploaded = new HashSet<>(uploadedObjectKeys); + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + if (status != STATUS_COMMITTED) { + uploaded.forEach(ShopDataCrawlTaskService.this::deleteObjectQuietly); + } + } + }); + } + + private void deleteObjectQuietly(String objectKey) { + if (blank(objectKey)) { + return; + } + try { + deleteResultObjectNowIfUnreferenced(objectKey); + } catch (Exception ex) { + log.warn("[shop-data-crawl] daily object cleanup failed object={} msg={}", objectKey, safeMessage(ex)); + } + } + + private DailyLockSet acquireDailyLocks(Long fallbackUserId, List rows) { + Map requests = new TreeMap<>(); + if (rows != null) { + for (FileResultEntity row : rows) { + if (row == null) { + continue; + } + Long userId = row.getUserId() != null ? row.getUserId() : fallbackUserId; + String shopKey = dailyFileService.shopKey(row); + if (userId == null || userId <= 0 || blank(shopKey)) { + continue; + } + requests.putIfAbsent(userId + "|" + shopKey, new DailyLockRequest(userId, shopKey)); + } + } + if (requests.isEmpty()) { + return new DailyLockSet(List.of()); + } + List handles = new ArrayList<>(); + try { + for (DailyLockRequest request : requests.values()) { + TaskDistributedLockService.LockHandle handle = dailyFileService.acquireLock( + request.userId(), request.shopKey()); + if (handle == null) { + throw new BusinessException("店铺累计文件正在处理中，请稍后重试"); + } + handles.add(handle); + } + return new DailyLockSet(handles); + } catch (RuntimeException ex) { + handles.forEach(TaskDistributedLockService.LockHandle::close); + throw ex; + } + } + + private void releaseDailyLockAfterTransaction(TaskDistributedLockService.LockHandle lock) { + if (lock == null) { + return; + } + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + lock.close(); + return; + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + lock.close(); + } + }); + } + + private record DailyLockRequest(Long userId, String shopKey) { + } + + private record DailyMemberData(ShopDataCrawlDailyMemberEntity member, + FileResultEntity result, + ShopDataCrawlResultItemVo snapshot) { + } + + private record DailyDeletionResult(List obsoleteObjectKeys, + List uploadedObjectKeys) { + } + + private static final class DailyLockSet implements AutoCloseable { + private final List handles; + private boolean closed; + + private DailyLockSet(List handles) { + this.handles = handles == null ? List.of() : handles; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + closeNow(); + } + }); + } else { + closeNow(); + } + } + + private void closeNow() { + for (int index = handles.size() - 1; index >= 0; index--) { + handles.get(index).close(); + } + } + } + + private record DailyAggregationResult(List obsoleteObjectKeys) { } public void cleanupResultFileJob(TaskFileJobEntity job) { @@ -1813,9 +2271,29 @@ public class ShopDataCrawlTaskService { void deleteResultObjectIfUnreferenced(String resultFileUrl) { if (blank(resultFileUrl)) return; + if (!isResultObjectUnreferenced(resultFileUrl)) return; + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + deleteObjectQuietly(resultFileUrl); + } + }); + } else { + ossStorageService.deleteObject(resultFileUrl); + } + } + + private void deleteResultObjectNowIfUnreferenced(String resultFileUrl) { + if (blank(resultFileUrl) || !isResultObjectUnreferenced(resultFileUrl)) return; + ossStorageService.deleteObject(resultFileUrl); + } + + private boolean isResultObjectUnreferenced(String resultFileUrl) { Long references = fileResultMapper.selectCount(new LambdaQueryWrapper() .eq(FileResultEntity::getResultFileUrl, resultFileUrl)); - if (references == null || references == 0L) ossStorageService.deleteObject(resultFileUrl); + long dailyReferences = dailyFileService.countObjectReferences(resultFileUrl); + return (references == null || references == 0L) && dailyReferences == 0L; } public void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/mapper/ShopManageGroupMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/mapper/ShopManageGroupMapper.java index ec88852d..bec50774 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/mapper/ShopManageGroupMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/mapper/ShopManageGroupMapper.java @@ -78,4 +78,28 @@ public interface ShopManageGroupMapper extends BaseMapper ORDER BY gm.user_id ASC """) List selectManagedMemberUserIds(@Param("operatorId") Long operatorId); + + @Select(""" + + """) + List selectUserIdsByGroupIds(@Param("groupIds") List groupIds); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java index 73a8471f..d64a2934 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java @@ -9,6 +9,8 @@ import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper; import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMemberMapper; import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper; import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper; +import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper; +import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; import com.nanri.aiimage.modules.shopkey.model.dto.ShopManageGroupCreateRequest; import com.nanri.aiimage.modules.shopkey.model.dto.ShopManageGroupUpdateRequest; import com.nanri.aiimage.modules.shopkey.model.entity.QueryAsinEntity; @@ -16,6 +18,8 @@ import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageEntity; import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity; import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupMemberEntity; import com.nanri.aiimage.modules.shopkey.model.entity.SkipPriceAsinEntity; +import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity; +import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity; import com.nanri.aiimage.modules.shopkey.model.vo.ShopManageGroupItemVo; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -39,6 +43,8 @@ public class ShopManageGroupService { private final ShopManageMapper shopManageMapper; private final QueryAsinMapper queryAsinMapper; private final SkipPriceAsinMapper skipPriceAsinMapper; + private final InvalidAsinDataMapper invalidAsinDataMapper; + private final DedupeTotalDataMapper dedupeTotalDataMapper; private final AdminUserMapper adminUserMapper; public List list() { @@ -160,6 +166,16 @@ public class ShopManageGroupService { if (queryAsinCount != null && queryAsinCount > 0) { throw new BusinessException("该分组下存在查询 ASIN，无法删除"); } + Long invalidAsinDataCount = invalidAsinDataMapper.selectCount(new LambdaQueryWrapper() + .eq(InvalidAsinDataEntity::getGroupId, entity.getId())); + if (invalidAsinDataCount != null && invalidAsinDataCount > 0) { + throw new BusinessException("该分组下存在品牌数据库数据，无法删除"); + } + Long dedupeTotalDataCount = dedupeTotalDataMapper.selectCount(new LambdaQueryWrapper() + .eq(DedupeTotalDataEntity::getGroupId, entity.getId())); + if (dedupeTotalDataCount != null && dedupeTotalDataCount > 0) { + throw new BusinessException("该分组下存在数据去重总数据，无法删除"); + } groupMemberMapper.delete(new LambdaQueryWrapper() .eq(ShopManageGroupMemberEntity::getGroupId, entity.getId())); groupMapper.deleteById(entity.getId()); diff --git a/backend-java/src/main/resources/db/V85__shop_data_crawl_daily_file.sql b/backend-java/src/main/resources/db/V85__shop_data_crawl_daily_file.sql new file mode 100644 index 00000000..63374385 --- /dev/null +++ b/backend-java/src/main/resources/db/V85__shop_data_crawl_daily_file.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS biz_shop_data_crawl_daily_file ( + id BIGINT NOT NULL AUTO_INCREMENT, + user_id BIGINT NOT NULL, + shop_key_hash CHAR(64) NOT NULL, + shop_key VARCHAR(1000) NOT NULL, + business_date DATE NOT NULL, + latest_task_id BIGINT NULL, + latest_result_id BIGINT NULL, + result_filename VARCHAR(255) NULL, + result_file_url VARCHAR(1000) NULL, + result_file_size BIGINT NULL, + result_content_type VARCHAR(128) NULL, + row_count INT NULL, + version BIGINT NOT NULL DEFAULT 0, + last_success_at DATETIME NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + UNIQUE KEY uk_shop_data_crawl_daily_file (user_id, shop_key_hash, business_date), + KEY idx_shop_data_crawl_daily_result (latest_result_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS biz_shop_data_crawl_daily_member ( + id BIGINT NOT NULL AUTO_INCREMENT, + daily_file_id BIGINT NOT NULL, + task_id BIGINT NOT NULL, + result_id BIGINT NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + UNIQUE KEY uk_shop_data_crawl_daily_member_result (result_id), + KEY idx_shop_data_crawl_daily_member_file_result (daily_file_id, result_id), + KEY idx_shop_data_crawl_daily_member_task (task_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend-java/src/main/resources/db/V86__invalid_asin_data_group_access.sql b/backend-java/src/main/resources/db/V86__invalid_asin_data_group_access.sql new file mode 100644 index 00000000..acf2d5df --- /dev/null +++ b/backend-java/src/main/resources/db/V86__invalid_asin_data_group_access.sql @@ -0,0 +1,74 @@ +SET @invalid_asin_group_id_col_exists := ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_invalid_asin_data' + AND COLUMN_NAME = 'group_id' +); + +SET @sql_add_invalid_asin_group_id := IF( + @invalid_asin_group_id_col_exists = 0, + 'ALTER TABLE biz_invalid_asin_data ADD COLUMN group_id BIGINT NULL AFTER brand', + 'SELECT 1' +); +PREPARE stmt_add_invalid_asin_group_id FROM @sql_add_invalid_asin_group_id; +EXECUTE stmt_add_invalid_asin_group_id; +DEALLOCATE PREPARE stmt_add_invalid_asin_group_id; + +SET @invalid_asin_record_source_col_exists := ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_invalid_asin_data' + AND COLUMN_NAME = 'record_source' +); + +SET @sql_add_invalid_asin_record_source := IF( + @invalid_asin_record_source_col_exists = 0, + 'ALTER TABLE biz_invalid_asin_data ADD COLUMN record_source VARCHAR(16) NOT NULL DEFAULT ''AUTO'' AFTER group_id', + 'SELECT 1' +); +PREPARE stmt_add_invalid_asin_record_source FROM @sql_add_invalid_asin_record_source; +EXECUTE stmt_add_invalid_asin_record_source; +DEALLOCATE PREPARE stmt_add_invalid_asin_record_source; + +-- Historical rows have no reliable origin marker. Treat them as automatic rows. +UPDATE biz_invalid_asin_data +SET record_source = 'AUTO' +WHERE record_source IS NULL + OR TRIM(record_source) = '' + OR UPPER(TRIM(record_source)) NOT IN ('AUTO', 'MANUAL'); + +SET @invalid_asin_source_group_index_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_invalid_asin_data' + AND INDEX_NAME = 'idx_invalid_asin_source_group_id' +); + +SET @sql_add_invalid_asin_source_group_index := IF( + @invalid_asin_source_group_index_exists = 0, + 'ALTER TABLE biz_invalid_asin_data ADD INDEX idx_invalid_asin_source_group_id (record_source, group_id, id)', + 'SELECT 1' +); +PREPARE stmt_add_invalid_asin_source_group_index FROM @sql_add_invalid_asin_source_group_index; +EXECUTE stmt_add_invalid_asin_source_group_index; +DEALLOCATE PREPARE stmt_add_invalid_asin_source_group_index; + +SET @invalid_asin_group_index_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_invalid_asin_data' + AND INDEX_NAME = 'idx_invalid_asin_group_id' +); + +SET @sql_add_invalid_asin_group_index := IF( + @invalid_asin_group_index_exists = 0, + 'ALTER TABLE biz_invalid_asin_data ADD INDEX idx_invalid_asin_group_id (group_id)', + 'SELECT 1' +); +PREPARE stmt_add_invalid_asin_group_index FROM @sql_add_invalid_asin_group_index; +EXECUTE stmt_add_invalid_asin_group_index; +DEALLOCATE PREPARE stmt_add_invalid_asin_group_index; diff --git a/backend-java/src/main/resources/db/V87__dedupe_total_data_group_access.sql b/backend-java/src/main/resources/db/V87__dedupe_total_data_group_access.sql new file mode 100644 index 00000000..d5fa2041 --- /dev/null +++ b/backend-java/src/main/resources/db/V87__dedupe_total_data_group_access.sql @@ -0,0 +1,53 @@ +SET @dedupe_group_id_col_exists := ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_dedupe_total_data' + AND COLUMN_NAME = 'group_id' +); + +SET @sql_add_dedupe_group_id := IF( + @dedupe_group_id_col_exists = 0, + 'ALTER TABLE biz_dedupe_total_data ADD COLUMN group_id BIGINT NULL AFTER data_value', + 'SELECT 1' +); +PREPARE stmt_add_dedupe_group_id FROM @sql_add_dedupe_group_id; +EXECUTE stmt_add_dedupe_group_id; +DEALLOCATE PREPARE stmt_add_dedupe_group_id; + +UPDATE biz_dedupe_total_data d +JOIN ( + SELECT matched.id, MIN(matched.group_id) AS group_id + FROM ( + SELECT d0.id, g.id AS group_id + FROM biz_dedupe_total_data d0 + INNER JOIN biz_shop_manage_group g + ON COALESCE(g.created_by_id, g.user_id) = d0.uploader_user_id + UNION + SELECT d1.id, gm.group_id + FROM biz_dedupe_total_data d1 + INNER JOIN biz_shop_manage_group_member gm + ON gm.user_id = d1.uploader_user_id + ) matched + GROUP BY matched.id + HAVING COUNT(DISTINCT matched.group_id) = 1 +) resolved ON resolved.id = d.id +SET d.group_id = resolved.group_id +WHERE d.group_id IS NULL; + +SET @dedupe_group_idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_dedupe_total_data' + AND INDEX_NAME = 'idx_dedupe_total_data_group_id' +); + +SET @sql_add_dedupe_group_idx := IF( + @dedupe_group_idx_exists = 0, + 'ALTER TABLE biz_dedupe_total_data ADD INDEX idx_dedupe_total_data_group_id (group_id, id)', + 'SELECT 1' +); +PREPARE stmt_add_dedupe_group_idx FROM @sql_add_dedupe_group_idx; +EXECUTE stmt_add_dedupe_group_idx; +DEALLOCATE PREPARE stmt_add_dedupe_group_idx; diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataServiceTest.java index d5fc9cfb..59d4b2a1 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataServiceTest.java @@ -13,6 +13,7 @@ import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo; import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper; +import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -65,6 +66,7 @@ class DedupeTotalDataServiceTest { void createRecordsUploaderIdentity() { AdminUserEntity uploader = user(23L, "normal", "member-a"); when(adminUserMapper.selectById(23L)).thenReturn(uploader); + stubWritableGroup(23L, 7L); when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null); when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))).thenAnswer(invocation -> { DedupeTotalDataEntity entity = invocation.getArgument(0); @@ -75,6 +77,7 @@ class DedupeTotalDataServiceTest { DedupeTotalDataEntity entity = new DedupeTotalDataEntity(); entity.setId(91L); entity.setDataValue("B012345678"); + entity.setGroupId(7L); entity.setUploaderUserId(23L); entity.setUploaderUsername("member-a"); return entity; @@ -82,6 +85,7 @@ class DedupeTotalDataServiceTest { DedupeTotalDataCreateRequest request = new DedupeTotalDataCreateRequest(); request.setDataValue(" b012345678 "); + request.setGroupId(7L); DedupeTotalDataItemVo item = service.create(request, 23L); ArgumentCaptor captor = ArgumentCaptor.forClass(DedupeTotalDataEntity.class); @@ -89,7 +93,10 @@ class DedupeTotalDataServiceTest { assertEquals(23L, captor.getValue().getUploaderUserId()); assertEquals("member-a", captor.getValue().getUploaderUsername()); assertEquals("B012345678", captor.getValue().getDataValue()); + assertEquals(7L, captor.getValue().getGroupId()); assertEquals("member-a", item.getUsername()); + assertEquals(7L, item.getGroupId()); + assertEquals("group-7", item.getGroupName()); } @Test @@ -117,6 +124,39 @@ class DedupeTotalDataServiceTest { verify(dedupeTotalDataMapper, never()).deleteById(any(Long.class)); } + @Test + void memberCanDeleteDataInAccessibleGroup() { + when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a")); + when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L)); + when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L)); + when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of()); + DedupeTotalDataEntity entity = data(91L, 99L); + entity.setGroupId(7L); + when(dedupeTotalDataMapper.selectById(91L)).thenReturn(entity); + + service.delete(91L, 23L); + + verify(dedupeTotalDataMapper).deleteById(91L); + } + + @Test + void memberCannotDeleteDataInAnotherGroup() { + when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a")); + when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L)); + when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L)); + when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of()); + DedupeTotalDataEntity entity = data(91L, 23L); + entity.setGroupId(8L); + when(dedupeTotalDataMapper.selectById(91L)).thenReturn(entity); + + ResponseStatusException exception = assertThrows( + ResponseStatusException.class, + () -> service.delete(91L, 23L)); + + assertEquals(403, exception.getStatusCode().value()); + verify(dedupeTotalDataMapper, never()).deleteById(any(Long.class)); + } + @Test void superAdminCanDeleteAnyUsersData() { when(adminUserMapper.selectById(1L)).thenReturn(user(1L, "super_admin", "root")); @@ -131,10 +171,11 @@ class DedupeTotalDataServiceTest { @Test void duplicateExcelValueKeepsOriginalUploader() throws Exception { when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a")); + stubWritableGroup(23L, 7L); when(dedupeTotalDataMapper.selectOne(any())).thenReturn(data(91L, 99L)); MockMultipartFile file = asinWorkbook("B012345678"); - var result = service.importFromExcel(file, 23L); + var result = service.importFromExcel(file, 7L, 23L); assertEquals(0, result.getInsertedCount()); assertEquals(1, result.getSkippedCount()); @@ -144,12 +185,13 @@ class DedupeTotalDataServiceTest { @Test void concurrentDuplicateDuringImportIsSkipped() throws Exception { when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a")); + stubWritableGroup(23L, 7L); when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null); when(transactionManager.getTransaction(any())).thenReturn(transactionStatus); when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))) .thenThrow(new DuplicateKeyException("duplicate")); - var result = service.importFromExcel(asinWorkbook("b012345678"), 23L); + var result = service.importFromExcel(asinWorkbook("b012345678"), 7L, 23L); assertEquals(0, result.getInsertedCount()); assertEquals(1, result.getSkippedCount()); @@ -172,6 +214,18 @@ class DedupeTotalDataServiceTest { verify(dedupeTotalDataMapper).selectList(any()); } + @Test + void pageSupportsUngroupedHistoricalRows() { + when(adminUserMapper.selectById(1L)).thenReturn(user(1L, "super_admin", "root")); + when(dedupeTotalDataMapper.selectCount(any())).thenReturn(1L); + when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of(data(91L, 23L))); + + DedupeTotalDataPageVo page = service.page(1, 15, "", "", null, null, 1L); + + assertEquals(1L, page.getTotal()); + assertEquals("", page.getItems().getFirst().getGroupName()); + } + @Test @SuppressWarnings({"rawtypes", "unchecked"}) void pageUsesInclusiveDateRange() { @@ -245,7 +299,8 @@ class DedupeTotalDataServiceTest { assertEquals("ASIN值", sheet.getRow(0).getCell(1).getStringCellValue()); assertEquals("B012345678", sheet.getRow(1).getCell(1).getStringCellValue()); assertEquals("member-a", sheet.getRow(1).getCell(2).getStringCellValue()); - assertEquals("2026-07-20 12:30:00", sheet.getRow(1).getCell(3).getStringCellValue()); + assertEquals("", sheet.getRow(1).getCell(3).getStringCellValue()); + assertEquals("2026-07-20 12:30:00", sheet.getRow(1).getCell(4).getStringCellValue()); } verify(shopManageGroupMapper).selectManagedMemberUserIds(10L); @@ -262,27 +317,70 @@ class DedupeTotalDataServiceTest { verify(dedupeTotalDataMapper, never()).selectList(any()); } + @Test + void importProgressAllowsAccessibleGroup() { + DedupeTotalDataImportProgressVo progress = registerImportProgress("same-group", 24L, 7L); + when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a")); + when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L)); + when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L, 24L)); + when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of()); + + assertEquals(progress, service.getImportProgress("same-group", 23L)); + } + + @Test + void importProgressRejectsVisibleOwnerFromAnotherGroup() { + registerImportProgress("other-group", 24L, 8L); + when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a")); + when(shopManageGroupMapper.selectAccessibleGroupIds(23L)).thenReturn(List.of(7L)); + when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(7L))).thenReturn(List.of(23L, 24L)); + when(shopManageGroupMapper.selectManagedMemberUserIds(23L)).thenReturn(List.of()); + + ResponseStatusException exception = assertThrows( + ResponseStatusException.class, + () -> service.getImportProgress("other-group", 23L)); + + assertEquals(403, exception.getStatusCode().value()); + } + @Test @SuppressWarnings("unchecked") void expiredCompletedProgressIsRemovedOnNextLookup() { Map progressMap = (Map) ReflectionTestUtils.getField(service, "importProgressMap"); Map ownerMap = (Map) ReflectionTestUtils.getField(service, "importOwnerMap"); + Map groupMap = (Map) ReflectionTestUtils.getField(service, "importGroupMap"); Map completedAtMap = (Map) ReflectionTestUtils.getField(service, "importCompletedAtMap"); DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo(); progress.setStatus("success"); progressMap.put("expired", progress); ownerMap.put("expired", 23L); + groupMap.put("expired", 7L); completedAtMap.put("expired", System.currentTimeMillis() - (2 * 60 * 60 * 1000L)); assertThrows(BusinessException.class, () -> service.getImportProgress("expired", 23L)); assertFalse(progressMap.containsKey("expired")); assertFalse(ownerMap.containsKey("expired")); + assertFalse(groupMap.containsKey("expired")); assertFalse(completedAtMap.containsKey("expired")); } + @SuppressWarnings("unchecked") + private DedupeTotalDataImportProgressVo registerImportProgress(String importId, Long ownerId, Long groupId) { + Map progressMap = + (Map) ReflectionTestUtils.getField(service, "importProgressMap"); + Map ownerMap = (Map) ReflectionTestUtils.getField(service, "importOwnerMap"); + Map groupMap = (Map) ReflectionTestUtils.getField(service, "importGroupMap"); + DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo(); + progress.setStatus("running"); + progressMap.put(importId, progress); + ownerMap.put(importId, ownerId); + groupMap.put(importId, groupId); + return progress; + } + private AdminUserEntity user(Long id, String role, String username) { AdminUserEntity user = new AdminUserEntity(); user.setId(id); @@ -301,6 +399,21 @@ class DedupeTotalDataServiceTest { return entity; } + private void stubWritableGroup(Long operatorId, Long groupId) { + when(shopManageGroupMapper.selectAccessibleGroupIds(operatorId)).thenReturn(List.of(groupId)); + when(shopManageGroupMapper.selectUserIdsByGroupIds(List.of(groupId))).thenReturn(List.of(operatorId)); + when(shopManageGroupMapper.selectManagedMemberUserIds(operatorId)).thenReturn(List.of()); + when(shopManageGroupMapper.selectById(groupId)).thenReturn(group(groupId)); + } + + private ShopManageGroupEntity group(Long id) { + ShopManageGroupEntity group = new ShopManageGroupEntity(); + group.setId(id); + group.setGroupName("group-" + id); + group.setCreatedById(23L); + return group; + } + private MockMultipartFile asinWorkbook(String asin) throws Exception { try (XSSFWorkbook workbook = new XSSFWorkbook(); ByteArrayOutputStream output = new ByteArrayOutputStream()) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataControllerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataControllerTest.java new file mode 100644 index 00000000..27309934 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/controller/InvalidAsinDataControllerTest.java @@ -0,0 +1,108 @@ +package com.nanri.aiimage.modules.invalidasin.controller; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; +import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo; +import com.nanri.aiimage.modules.invalidasin.service.InvalidAsinDataService; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class InvalidAsinDataControllerTest { + + @Mock + private InvalidAsinDataService invalidAsinDataService; + @Mock + private AdminAuthSupport adminAuthSupport; + @Mock + private PermissionMenuService permissionMenuService; + @InjectMocks + private InvalidAsinDataController controller; + + @BeforeEach + void setUp() { + ReflectionTestUtils.setField(controller, "internalToken", "test-internal-token"); + } + + @Test + void trustedFlaskProxyUsesTheDatabaseUserInsteadOfRequestRole() { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.addHeader("X-Internal-Token", "test-internal-token"); + request.addParameter("operatorId", "7"); + request.addParameter("superAdmin", "true"); + AdminUserEntity operator = user(7L, "normal"); + when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录")); + when(permissionMenuService.requireUserOperator(7L)).thenReturn(operator); + when(adminAuthSupport.currentRole(operator)).thenReturn(null); + when(permissionMenuService.getUserColumnPermissions(7L, "admin")) + .thenReturn(List.of(invalidAsinDataPermission())); + when(invalidAsinDataService.page(1L, 15L, "", 3L, 7L, false)) + .thenReturn(new InvalidAsinDataPageVo()); + + controller.page(1L, 15L, "", 3L, request); + + verify(invalidAsinDataService).page(1L, 15L, "", 3L, 7L, false); + } + + @Test + void directRequestCannotForgeSuperAdminWithoutTrustedAuthentication() { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.addParameter("operatorId", "1"); + request.addParameter("superAdmin", "true"); + when(adminAuthSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录")); + + assertThrows(BusinessException.class, () -> controller.page(1L, 15L, "", 3L, request)); + + verifyNoInteractions(invalidAsinDataService); + verify(permissionMenuService, never()).requireUserOperator(any()); + } + + @Test + void authenticatedSuperAdminRoleComesFromTheServerUser() { + MockHttpServletRequest request = new MockHttpServletRequest(); + request.addParameter("superAdmin", "false"); + AdminUserEntity operator = user(1L, "super_admin"); + when(adminAuthSupport.requireUser(request)).thenReturn(operator); + when(adminAuthSupport.currentRole(operator)).thenReturn("super_admin"); + when(invalidAsinDataService.page(1L, 15L, "", 3L, 1L, true)) + .thenReturn(new InvalidAsinDataPageVo()); + + controller.page(1L, 15L, "", 3L, request); + + verify(invalidAsinDataService).page(1L, 15L, "", 3L, 1L, true); + verify(permissionMenuService, never()).getUserColumnPermissions(eq(1L), eq("admin")); + } + + private AdminUserEntity user(Long id, String role) { + AdminUserEntity user = new AdminUserEntity(); + user.setId(id); + user.setRole(role); + return user; + } + + private PermissionMenuItemVo invalidAsinDataPermission() { + PermissionMenuItemVo permission = new PermissionMenuItemVo(); + permission.setColumnKey("admin_invalid_asin_data"); + permission.setRoutePath("invalid-asin-data"); + return permission; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataServiceTest.java new file mode 100644 index 00000000..67c1bdcc --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/invalidasin/service/InvalidAsinDataServiceTest.java @@ -0,0 +1,250 @@ +package com.nanri.aiimage.modules.invalidasin.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper; +import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataCreateRequest; +import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequest; +import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity; +import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo; +import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo; +import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity; +import com.nanri.aiimage.modules.shopkey.service.ShopManageGroupService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.web.server.ResponseStatusException; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class InvalidAsinDataServiceTest { + + @Mock + private InvalidAsinDataMapper invalidAsinDataMapper; + @Mock + private ShopManageGroupService shopManageGroupService; + @InjectMocks + private InvalidAsinDataService service; + + @Test + void createManualRecordUsesAccessibleGroup() { + InvalidAsinDataCreateRequest request = new InvalidAsinDataCreateRequest(); + request.setDataValue("B012345678"); + request.setBrand("Acme"); + request.setGroupId(9L); + ShopManageGroupEntity group = group(9L, "group-a"); + InvalidAsinDataEntity saved = data(91L, "MANUAL", 9L); + + when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(12L, 9L)); + when(shopManageGroupService.getAccessibleById(9L, 7L, false)).thenReturn(group); + when(invalidAsinDataMapper.selectOne(any())).thenReturn(null); + when(invalidAsinDataMapper.insert(any(InvalidAsinDataEntity.class))).thenAnswer(invocation -> { + invocation.getArgument(0, InvalidAsinDataEntity.class).setId(91L); + return 1; + }); + when(invalidAsinDataMapper.selectById(91L)).thenReturn(saved); + + InvalidAsinDataItemVo item = service.create(request, 7L, false); + + ArgumentCaptor captor = ArgumentCaptor.forClass(InvalidAsinDataEntity.class); + verify(invalidAsinDataMapper).insert(captor.capture()); + assertEquals("B012345678", captor.getValue().getDataValue()); + assertEquals("acme", captor.getValue().getBrand()); + assertEquals(9L, captor.getValue().getGroupId()); + assertEquals("MANUAL", captor.getValue().getRecordSource()); + assertEquals("group-a", item.getGroupName()); + assertEquals("MANUAL", item.getRecordSource()); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + void normalUserPageOnlyQueriesManualRowsInAccessibleGroups() { + InvalidAsinDataEntity manual = data(91L, "MANUAL", 9L); + when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L)); + when(invalidAsinDataMapper.selectCount(any())).thenReturn(1L); + when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(manual)); + when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of(9L, "group-a")); + + InvalidAsinDataPageVo page = service.page(1, 15, "", 10L, 7L, false); + + ArgumentCaptor> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class); + verify(invalidAsinDataMapper).selectCount(captor.capture()); + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + InvalidAsinDataEntity.class); + String sql = captor.getValue().getSqlSegment(); + assertTrue(sql.contains("record_source")); + assertTrue(sql.contains("group_id")); + assertTrue(captor.getValue().getParamNameValuePairs().containsValue("MANUAL")); + assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L)); + assertFalse(captor.getValue().getParamNameValuePairs().containsValue(10L)); + assertEquals("group-a", page.getItems().getFirst().getGroupName()); + } + + @Test + void superAdminPageHandlesRowsWithoutResolvableGroup() { + InvalidAsinDataEntity automatic = data(92L, "AUTO", null); + InvalidAsinDataEntity orphan = data(93L, "MANUAL", 999L); + when(invalidAsinDataMapper.selectCount(any())).thenReturn(1L); + when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of(automatic, orphan)); + when(shopManageGroupService.buildGroupNameMap(any())).thenReturn(Map.of()); + + InvalidAsinDataPageVo page = service.page(1, 15, "", null, 1L, true); + + assertEquals(2, page.getItems().size()); + assertEquals("AUTO", page.getItems().getFirst().getRecordSource()); + assertEquals("", page.getItems().getFirst().getGroupName()); + assertEquals("", page.getItems().get(1).getGroupName()); + } + + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + void superAdminPageFiltersByRequestedGroup() { + when(invalidAsinDataMapper.selectCount(any())).thenReturn(0L); + when(invalidAsinDataMapper.selectList(any())).thenReturn(List.of()); + + service.page(1, 15, "", 9L, 1L, true); + + ArgumentCaptor> captor = ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class); + verify(invalidAsinDataMapper).selectCount(captor.capture()); + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + InvalidAsinDataEntity.class); + assertTrue(captor.getValue().getSqlSegment().contains("group_id")); + assertTrue(captor.getValue().getParamNameValuePairs().containsValue(9L)); + } + + @Test + void normalUserCannotDeleteAutoRecord() { + when(invalidAsinDataMapper.selectById(91L)).thenReturn(data(91L, "AUTO", null)); + + ResponseStatusException exception = assertThrows( + ResponseStatusException.class, + () -> service.delete(91L, 7L, false)); + + assertEquals(403, exception.getStatusCode().value()); + verify(invalidAsinDataMapper, never()).deleteById(any(Long.class)); + verifyNoInteractions(shopManageGroupService); + } + + @Test + void normalUserCannotUpdateIntoAnotherAccessibleGroup() { + InvalidAsinDataEntity record = data(91L, "MANUAL", 9L); + InvalidAsinDataUpdateRequest request = new InvalidAsinDataUpdateRequest(); + request.setDataValue("B012345678"); + request.setBrand("New Brand"); + request.setGroupId(10L); + when(invalidAsinDataMapper.selectById(91L)).thenReturn(record); + when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L)); + when(shopManageGroupService.getAccessibleById(9L, 7L, false)).thenReturn(group(9L, "group-a")); + + ResponseStatusException exception = assertThrows( + ResponseStatusException.class, + () -> service.update(91L, request, 7L, false)); + + assertEquals(403, exception.getStatusCode().value()); + assertEquals(9L, record.getGroupId()); + assertEquals("acme", record.getBrand()); + verify(invalidAsinDataMapper, never()).updateById(any(InvalidAsinDataEntity.class)); + verify(shopManageGroupService, never()).getAccessibleById(10L, 7L, false); + } + + @Test + void normalUserCannotCreateInAnotherAccessibleGroup() { + InvalidAsinDataCreateRequest request = new InvalidAsinDataCreateRequest(); + request.setDataValue("B012345678"); + request.setBrand("Acme"); + request.setGroupId(10L); + when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L)); + + ResponseStatusException exception = assertThrows( + ResponseStatusException.class, + () -> service.create(request, 7L, false)); + + assertEquals(403, exception.getStatusCode().value()); + verify(invalidAsinDataMapper, never()).insert(any(InvalidAsinDataEntity.class)); + } + + @Test + void normalUserCannotDeleteRecordOutsideFixedGroup() { + InvalidAsinDataEntity record = data(91L, "MANUAL", 10L); + when(invalidAsinDataMapper.selectById(91L)).thenReturn(record); + when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of(10L, 9L)); + + ResponseStatusException exception = assertThrows( + ResponseStatusException.class, + () -> service.delete(91L, 7L, false)); + + assertEquals(403, exception.getStatusCode().value()); + verify(invalidAsinDataMapper, never()).deleteById(any(Long.class)); + verify(shopManageGroupService, never()).getAccessibleById(any(Long.class), any(Long.class), org.mockito.ArgumentMatchers.eq(false)); + } + + @Test + void normalUserCreateRequiresAtLeastOneAccessibleGroup() { + InvalidAsinDataCreateRequest request = new InvalidAsinDataCreateRequest(); + request.setDataValue("B012345678"); + request.setBrand("Acme"); + request.setGroupId(9L); + when(shopManageGroupService.listAccessibleGroupIds(7L, false)).thenReturn(Set.of()); + + assertThrows(BusinessException.class, () -> service.create(request, 7L, false)); + + verify(invalidAsinDataMapper, never()).insert(any(InvalidAsinDataEntity.class)); + verify(shopManageGroupService, never()).getAccessibleById(any(Long.class), any(Long.class), org.mockito.ArgumentMatchers.eq(false)); + } + + @Test + void superAdminCanUpdateAutoRecordWithoutGroup() { + InvalidAsinDataEntity record = data(91L, "AUTO", null); + InvalidAsinDataUpdateRequest request = new InvalidAsinDataUpdateRequest(); + request.setDataValue("B012345678"); + request.setBrand("New Brand"); + when(invalidAsinDataMapper.selectById(91L)).thenReturn(record); + when(invalidAsinDataMapper.selectOne(any())).thenReturn(null); + + InvalidAsinDataItemVo item = service.update(91L, request, 1L, true); + + assertEquals("AUTO", item.getRecordSource()); + assertNull(record.getGroupId()); + verify(invalidAsinDataMapper).updateById(record); + verifyNoInteractions(shopManageGroupService); + } + + private ShopManageGroupEntity group(Long id, String groupName) { + ShopManageGroupEntity group = new ShopManageGroupEntity(); + group.setId(id); + group.setGroupName(groupName); + return group; + } + + private InvalidAsinDataEntity data(Long id, String source, Long groupId) { + InvalidAsinDataEntity entity = new InvalidAsinDataEntity(); + entity.setId(id); + entity.setDataValue("B012345678"); + entity.setBrand("acme"); + entity.setGroupId(groupId); + entity.setRecordSource(source); + return entity; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java index 4348b96a..e2cf148c 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java @@ -72,6 +72,55 @@ class ShopDataCrawlExcelAssemblyServiceTest { } } + @Test + void appendsOnlyTheSelectedCountryAndPreservesExistingPictures() throws Exception { + ShopDataCrawlRowDto firstRow = row("2026-07-25", "B012345678"); + firstRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg"); + ShopDataCrawlRowDto secondRow = row("2026-07-26", "B099999999"); + secondRow.setCommodityImage(null); + + SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); + when(imageEmbedder.fetchAndResizeForCache(firstRow.getCommodityImage())) + .thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2)); + ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder); + File base = tempDir.resolve("base.xlsx").toFile(); + File output = tempDir.resolve("daily.xlsx").toFile(); + service.writeWorkbook(base, List.of(item("UK", firstRow))); + service.appendWorkbook(base, output, List.of(item("DE", secondRow))); + + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue()); + assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue()); + assertEquals(1, workbook.getAllPictures().size()); + assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size()); + assertEquals(0, workbook.getSheetAt(2).getLastRowNum()); + } + } + + private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) { + ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto(); + country.setCountry(countryCode); + country.setItems(List.of(row)); + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setSuccess(true); + item.setCountryResults(List.of(country)); + return item; + } + + private ShopDataCrawlRowDto row(String date, String asin) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate(date); + row.setAsin(asin); + row.setBrand("Example Brand"); + row.setInventorySales("11"); + row.setSalesRank("22"); + row.setPageViews("33"); + row.setUnitsSold("44"); + row.setPrice("12.50"); + row.setRecommendedOffer("12.00"); + return row; + } + private byte[] jpegBytes() throws Exception { BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB); ByteArrayOutputStream output = new ByteArrayOutputStream(); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java index 1a8383d9..de2614d3 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java @@ -92,6 +92,7 @@ class ShopDataCrawlTaskServiceChunkTest { @Mock private TaskScopeStateMapper taskScopeStateMapper; @Mock private TransientPayloadStorageService transientPayloadStorageService; @Mock private InstanceMetadata instanceMetadata; + @Mock private ShopDataCrawlDailyFileService dailyFileService; @Spy private ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); private ShopDataCrawlTaskService service; @@ -122,7 +123,8 @@ class ShopDataCrawlTaskServiceChunkTest { taskChunkMapper, taskScopeStateMapper, transientPayloadStorageService, - instanceMetadata); + instanceMetadata, + dailyFileService); storedChunks.clear(); storedScopes.clear(); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java index be847d4f..056c4793 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java @@ -39,6 +39,7 @@ class ShopDataCrawlTaskServiceOwnerTest { @Mock com.nanri.aiimage.modules.task.mapper.TaskChunkMapper taskChunkMapper; @Mock com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper taskScopeStateMapper; @Mock com.nanri.aiimage.modules.task.service.TransientPayloadStorageService transientPayloadStorageService; + @Mock ShopDataCrawlDailyFileService dailyFileService; @Spy private final ObjectMapper objectMapper = new ObjectMapper(); @InjectMocks ShopDataCrawlTaskService service; diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceRetentionTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceRetentionTest.java index 2593f5ac..2d8662ce 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceRetentionTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceRetentionTest.java @@ -5,12 +5,18 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; @@ -20,34 +26,55 @@ import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import org.apache.ibatis.builder.MapperBuilderAssistant; import org.apache.ibatis.session.Configuration; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import java.time.LocalDate; import java.time.LocalDateTime; import java.util.List; -import java.util.Map; +import java.util.Set; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) class ShopDataCrawlTaskServiceRetentionTest { private static final String MODULE_TYPE = "SHOP_DATA_CRAWL"; private static final Long USER_ID = 7L; + private static final Long TASK_ID = 101L; + private static final Long RESULT_ID = 201L; + private static final LocalDate BUSINESS_DATE = LocalDate.of(2026, 8, 6); + private static final LocalDateTime BUSINESS_TIME = LocalDateTime.of(2026, 8, 6, 12, 0); @BeforeAll static void initializeMybatisMetadata() { MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), ""); TableInfoHelper.initTableInfo(assistant, FileResultEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class); } @Mock private FileTaskMapper fileTaskMapper; @@ -66,106 +93,302 @@ class ShopDataCrawlTaskServiceRetentionTest { @Mock private TaskScopeStateMapper taskScopeStateMapper; @Mock private TransientPayloadStorageService transientPayloadStorageService; @Mock private InstanceMetadata instanceMetadata; - @Spy private final ObjectMapper objectMapper = new ObjectMapper(); + @Mock private ShopDataCrawlDailyFileService dailyFileService; + @Spy private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); @InjectMocks private ShopDataCrawlTaskService service; - @Test - void keepsNewestThreePerStableShopAndFallsBackToShopName() { - LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0); - FileResultEntity shopIdOldest = result(11L, 111L, USER_ID, "shop-1", "Renamed Shop", - "result/shop-id-oldest.xlsx", 1, now.minusDays(2)); - FileResultEntity shopNameOldest = result(21L, 121L, USER_ID, " ", "Fallback Shop", - "result/shop-name-oldest.xlsx", 1, now.minusDays(4)); + private FileTaskEntity task; + private FileResultEntity currentRow; + private ShopDataCrawlResultItemVo snapshot; + private TaskFileJobEntity job; - List rows = List.of( - result(13L, 113L, USER_ID, "shop-1", "Current Name", "result/13.xlsx", 1, now.minusDays(1)), - result(23L, 123L, USER_ID, null, "Fallback Shop", "result/23.xlsx", 1, now.minusDays(2)), - shopIdOldest, - result(14L, 114L, USER_ID, "shop-1", "Current Name", "result/14.xlsx", 1, now), - result(24L, 124L, USER_ID, null, "Fallback Shop", "result/24.xlsx", 1, now), - result(12L, 112L, USER_ID, "shop-1", "Old Name", "result/12.xlsx", 1, now.minusDays(2)), - shopNameOldest, - result(22L, 122L, USER_ID, null, "Fallback Shop", "result/22.xlsx", 1, now.minusDays(3)), - result(1L, 101L, USER_ID, "shop-1", "Current Name", "result/failed.xlsx", 0, now.minusDays(9)), - result(2L, 102L, USER_ID, "shop-1", "Current Name", null, 1, now.minusDays(9)), - result(3L, 103L, 99L, "shop-1", "Current Name", "result/other-user.xlsx", 1, now.minusDays(9)), - result(4L, 104L, USER_ID, "shop-2", "Current Name", "result/other-shop.xlsx", 1, now.minusDays(9))); + @BeforeEach + void setUp() { + task = task(); + currentRow = result(RESULT_ID, TASK_ID, null); + snapshot = snapshot(RESULT_ID, TASK_ID); + job = new TaskFileJobEntity(); + job.setTaskId(TASK_ID); + job.setResultId(RESULT_ID); - when(fileResultMapper.selectList(any())).thenReturn(rows); - when(fileResultMapper.selectById(11L)).thenReturn(shopIdOldest); - when(fileResultMapper.selectById(21L)).thenReturn(shopNameOldest); - when(fileTaskMapper.selectById(111L)).thenReturn(terminalTask(111L)); - when(fileTaskMapper.selectById(121L)).thenReturn(terminalTask(121L)); - when(taskDistributedLockService.acquire(MODULE_TYPE, 111L)) - .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); - when(taskDistributedLockService.acquire(MODULE_TYPE, 121L)) - .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); - when(cacheService.getTaskCacheBatch(any())).thenReturn(Map.of()); + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); + when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1); + when(fileResultMapper.updateById(any(FileResultEntity.class))).thenReturn(1); when(fileResultMapper.selectCount(any())).thenReturn(0L); - - service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-1"); - service.pruneCompletedHistoryForShop(USER_ID, "shop-name:Fallback Shop"); - - verify(fileResultMapper).deleteById(11L); - verify(fileResultMapper).deleteById(21L); - verify(taskFileJobService).deleteResultJobs(111L, MODULE_TYPE, 11L); - verify(taskFileJobService).deleteResultJobs(121L, MODULE_TYPE, 21L); - verify(taskResultItemService).deleteResultItem(111L, MODULE_TYPE, 11L); - verify(taskResultItemService).deleteResultItem(121L, MODULE_TYPE, 21L); - verify(ossStorageService).deleteObject("result/shop-id-oldest.xlsx"); - verify(ossStorageService).deleteObject("result/shop-name-oldest.xlsx"); - verify(fileResultMapper, never()).deleteById(1L); - verify(fileResultMapper, never()).deleteById(2L); - verify(fileResultMapper, never()).deleteById(3L); - verify(fileResultMapper, never()).deleteById(4L); + when(taskResultItemService.listResultSnapshots(TASK_ID, MODULE_TYPE, ShopDataCrawlResultItemVo.class)) + .thenReturn(List.of(snapshot)); + when(excelAssemblyService.countRows(any())).thenReturn(1); + when(dailyFileService.currentBusinessDate()).thenReturn(BUSINESS_DATE); + when(dailyFileService.currentBusinessDateTime()).thenReturn(BUSINESS_TIME); + when(dailyFileService.shopKey(any())).thenReturn("shop-id:shop-1"); + when(dailyFileService.shopKeyHash(anyString())).thenReturn("hash-1"); + when(dailyFileService.acquireLock(eq(USER_ID), eq("shop-id:shop-1"))) + .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); + when(dailyFileService.findMembersByResultId(RESULT_ID)).thenReturn(List.of()); + when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of()); + when(dailyFileService.countObjectReferences(anyString())).thenReturn(0L); + when(dailyFileService.addMember(anyLong(), anyLong(), anyLong())).thenReturn(true); + doAnswer(invocation -> { + ShopDataCrawlDailyFileEntity entity = invocation.getArgument(0); + entity.setId(301L); + return null; + }).when(dailyFileService).insert(any()); } @Test - void doesNotDeleteOldFileWhileOwningTaskIsStillRunning() { - LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0); - FileResultEntity oldest = result(31L, 131L, USER_ID, "shop-running", "Running Shop", - "result/running-oldest.xlsx", 1, now.minusDays(3)); - when(fileResultMapper.selectList(any())).thenReturn(List.of( - result(34L, 134L, USER_ID, "shop-running", "Running Shop", "result/34.xlsx", 1, now), - result(33L, 133L, USER_ID, "shop-running", "Running Shop", "result/33.xlsx", 1, now.minusDays(1)), - result(32L, 132L, USER_ID, "shop-running", "Running Shop", "result/32.xlsx", 1, now.minusDays(2)), - oldest)); - when(fileResultMapper.selectById(31L)).thenReturn(oldest); - when(fileTaskMapper.selectById(131L)).thenReturn(task(131L, "RUNNING")); - when(taskDistributedLockService.acquire(MODULE_TYPE, 131L)) + void firstSuccessCreatesDailyWorkbookAndMembership() { + when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(currentRow)); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null); + when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx"); + + service.processResultFileJob(job); + + verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot))); + verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); + verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID); + assertEquals("result/new.xlsx", currentRow.getResultFileUrl()); + assertEquals(1, currentRow.getRowCount()); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ShopDataCrawlDailyFileEntity.class); + verify(dailyFileService).insert(captor.capture()); + assertEquals(BUSINESS_DATE, captor.getValue().getBusinessDate()); + assertEquals(1L, captor.getValue().getVersion()); + } + + @Test + void sameDaySuccessAppendsAndReplacesPreviousPointer() { + FileResultEntity previous = result(200L, 100L, "result/old.xlsx"); + previous.setUserId(null); + FileTaskEntity previousTask = task(); + previousTask.setId(100L); + ShopDataCrawlDailyFileEntity daily = daily("result/old.xlsx", 2); + when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow)); + when(fileTaskMapper.selectBatchIds(List.of(100L))).thenReturn(List.of(previousTask)); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily); + when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false); + when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3}); + when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx"); + + TransactionSynchronizationManager.initSynchronization(); + try { + service.processResultFileJob(job); + + verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot))); + verify(dailyFileService).update(daily); + verify(ossStorageService, never()).deleteObject("result/old.xlsx"); + assertNull(previous.getResultFileUrl()); + assertEquals("result/new.xlsx", currentRow.getResultFileUrl()); + assertEquals(3, currentRow.getRowCount()); + + List synchronizations = TransactionSynchronizationManager.getSynchronizations(); + synchronizations.forEach(TransactionSynchronization::afterCommit); + verify(ossStorageService).deleteObject("result/old.xlsx"); + synchronizations.forEach(synchronization -> + synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED)); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + @Test + void repeatedResultDoesNotAppendOrUploadAgain() { + ShopDataCrawlDailyFileEntity daily = daily("result/current.xlsx", 3); + daily.setLatestResultId(RESULT_ID); + when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow)); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily); + when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(true); + + service.processResultFileJob(job); + + verify(excelAssemblyService, never()).writeWorkbook(any(), any()); + verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); + verify(ossStorageService, never()).uploadResultFile(any(), anyString()); + assertEquals("result/current.xlsx", currentRow.getResultFileUrl()); + assertEquals(3, currentRow.getRowCount()); + } + + @Test + void sameDaySuccessKeepsLegacyPointerOwnedByAnotherUser() { + FileResultEntity previous = result(200L, 100L, "result/old.xlsx"); + previous.setUserId(null); + FileTaskEntity previousTask = task(); + previousTask.setId(100L); + previousTask.setUserId(8L); + ShopDataCrawlDailyFileEntity daily = daily("result/old.xlsx", 2); + when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow)); + when(fileTaskMapper.selectBatchIds(List.of(100L))).thenReturn(List.of(previousTask)); + when(fileResultMapper.selectCount(any())).thenReturn(1L); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily); + when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false); + when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3}); + when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx"); + + service.processResultFileJob(job); + + assertEquals("result/old.xlsx", previous.getResultFileUrl()); + assertEquals("result/new.xlsx", currentRow.getResultFileUrl()); + } + + @Test + void previousDayRetryDoesNotAppendToCurrentDailyFile() { + currentRow.setResultFileUrl("result/yesterday.xlsx"); + ShopDataCrawlDailyFileEntity today = daily("result/today.xlsx", 1); + today.setId(302L); + ShopDataCrawlDailyFileEntity yesterday = daily("result/yesterday.xlsx", 3); + yesterday.setBusinessDate(BUSINESS_DATE.minusDays(1)); + yesterday.setLatestResultId(RESULT_ID); + ShopDataCrawlDailyMemberEntity member = member(301L, TASK_ID, RESULT_ID, BUSINESS_TIME.minusDays(1)); + when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow)); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(today); + when(dailyFileService.findMembersByResultId(RESULT_ID)).thenReturn(List.of(member)); + when(dailyFileService.findById(301L)).thenReturn(yesterday); + + service.processResultFileJob(job); + + verify(excelAssemblyService, never()).writeWorkbook(any(), any()); + verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); + verify(ossStorageService, never()).uploadResultFile(any(), anyString()); + assertNull(currentRow.getResultFileUrl()); + } + + @Test + void newDaySuccessDeletesOlderDailyFileAfterNewUpload() { + FileResultEntity previous = result(200L, 100L, "result/yesterday.xlsx"); + ShopDataCrawlDailyFileEntity yesterday = daily("result/yesterday.xlsx", 4); + yesterday.setId(300L); + yesterday.setBusinessDate(BUSINESS_DATE.minusDays(1)); + when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow)); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null); + when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday)); + when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx"); + + service.processResultFileJob(job); + + verify(dailyFileService).deleteDailyFile(300L); + verify(ossStorageService).deleteObject("result/yesterday.xlsx"); + assertEquals("result/today.xlsx", currentRow.getResultFileUrl()); + assertNull(previous.getResultFileUrl()); + } + + @Test + void failedNewDayUploadKeepsOlderDailyFile() { + ShopDataCrawlDailyFileEntity yesterday = daily("result/yesterday.xlsx", 4); + yesterday.setId(300L); + yesterday.setBusinessDate(BUSINESS_DATE.minusDays(1)); + when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow)); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null); + when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday)); + doThrow(new IllegalStateException("upload failed")) + .when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE)); + + assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job)); + + verify(dailyFileService, never()).deleteDailyFile(anyLong()); + verify(ossStorageService, never()).deleteObject("result/yesterday.xlsx"); + } + + @Test + void deletingOneMemberRebuildsDailyWorkbookFromRemainingResults() { + task.setStatus("SUCCESS"); + currentRow.setResultFileUrl("result/old.xlsx"); + FileResultEntity previous = result(200L, 100L, null); + ShopDataCrawlResultItemVo previousSnapshot = snapshot(200L, 100L); + ShopDataCrawlDailyFileEntity daily = daily("result/old.xlsx", 2); + daily.setLatestTaskId(TASK_ID); + daily.setLatestResultId(RESULT_ID); + ShopDataCrawlDailyMemberEntity previousMember = member( + 301L, 100L, 200L, BUSINESS_TIME.minusMinutes(10)); + ShopDataCrawlDailyMemberEntity removedMember = member( + 301L, TASK_ID, RESULT_ID, BUSINESS_TIME.minusMinutes(5)); + when(taskDistributedLockService.acquire(MODULE_TYPE, TASK_ID)) .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); + when(fileResultMapper.selectById(RESULT_ID)).thenReturn(currentRow); + when(fileResultMapper.selectById(200L)).thenReturn(previous); + when(fileResultMapper.selectList(any())).thenReturn(List.of()); + when(dailyFileService.findByLatestResultId(RESULT_ID)).thenReturn(List.of(daily)); + when(dailyFileService.findMembersByResultId(RESULT_ID)).thenReturn(List.of(removedMember)); + when(dailyFileService.findById(301L)).thenReturn(daily); + when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily); + when(dailyFileService.listMembers(301L)).thenReturn(List.of(removedMember, previousMember)); + when(taskResultItemService.getResultSnapshot( + 100L, MODULE_TYPE, 200L, ShopDataCrawlResultItemVo.class)).thenReturn(previousSnapshot); + when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/rebuilt.xlsx"); - service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-running"); + service.deleteHistory(RESULT_ID, USER_ID); - verify(fileResultMapper, never()).deleteById(31L); - verify(taskFileJobService, never()).deleteResultJobs(131L, MODULE_TYPE, 31L); - verify(ossStorageService, never()).deleteObject("result/running-oldest.xlsx"); + verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(previousSnapshot))); + verify(dailyFileService).deleteMembersForResults(Set.of(RESULT_ID)); + verify(dailyFileService).update(daily); + verify(ossStorageService).deleteObject("result/old.xlsx"); + assertEquals(200L, daily.getLatestResultId()); + assertEquals("result/rebuilt.xlsx", daily.getResultFileUrl()); + assertEquals("result/rebuilt.xlsx", previous.getResultFileUrl()); + assertEquals(1, previous.getRowCount()); } - private FileResultEntity result(Long id, Long taskId, Long userId, String shopId, String shopName, - String resultFileUrl, int success, LocalDateTime createdAt) { - FileResultEntity row = new FileResultEntity(); - row.setId(id); - row.setTaskId(taskId); - row.setModuleType(MODULE_TYPE); - row.setUserId(userId); - row.setSourceFileUrl(shopId); - row.setSourceFilename(shopName); - row.setResultFileUrl(resultFileUrl); - row.setSuccess(success); - row.setCreatedAt(createdAt); - return row; + private FileTaskEntity task() { + FileTaskEntity entity = new FileTaskEntity(); + entity.setId(TASK_ID); + entity.setModuleType(MODULE_TYPE); + entity.setUserId(USER_ID); + entity.setStatus("RUNNING"); + entity.setTaskNo("task-101"); + return entity; } - private FileTaskEntity terminalTask(Long id) { - return task(id, "SUCCESS"); + private FileResultEntity result(Long id, Long taskId, String objectKey) { + FileResultEntity entity = new FileResultEntity(); + entity.setId(id); + entity.setTaskId(taskId); + entity.setModuleType(MODULE_TYPE); + entity.setUserId(USER_ID); + entity.setSourceFilename("Demo Shop"); + entity.setSourceFileUrl("shop-1"); + entity.setSuccess(1); + entity.setResultFileUrl(objectKey); + entity.setCreatedAt(BUSINESS_TIME.minusMinutes(5)); + return entity; } - private FileTaskEntity task(Long id, String status) { - FileTaskEntity task = new FileTaskEntity(); - task.setId(id); - task.setModuleType(MODULE_TYPE); - task.setStatus(status); - return task; + private ShopDataCrawlResultItemVo snapshot(Long resultId, Long taskId) { + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setResultId(resultId); + item.setTaskId(taskId); + item.setShopName("Demo Shop"); + item.setShopId("shop-1"); + item.setSuccess(true); + item.setCountryResults(List.of()); + return item; + } + + private ShopDataCrawlDailyFileEntity daily(String objectKey, int rowCount) { + ShopDataCrawlDailyFileEntity entity = new ShopDataCrawlDailyFileEntity(); + entity.setId(301L); + entity.setUserId(USER_ID); + entity.setShopKey("shop-id:shop-1"); + entity.setShopKeyHash("hash-1"); + entity.setBusinessDate(BUSINESS_DATE); + entity.setLatestTaskId(100L); + entity.setLatestResultId(200L); + entity.setResultFilename("daily.xlsx"); + entity.setResultFileUrl(objectKey); + entity.setResultFileSize(10L); + entity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + entity.setRowCount(rowCount); + entity.setVersion(1L); + return entity; + } + + private ShopDataCrawlDailyMemberEntity member(Long dailyFileId, + Long taskId, + Long resultId, + LocalDateTime createdAt) { + ShopDataCrawlDailyMemberEntity entity = new ShopDataCrawlDailyMemberEntity(); + entity.setId(resultId); + entity.setDailyFileId(dailyFileId); + entity.setTaskId(taskId); + entity.setResultId(resultId); + entity.setCreatedAt(createdAt); + return entity; } } diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index f697f50a..e7192719 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -334,7 +334,8 @@ class _PermissionProxyError(Exception): self.status = status -def _proxy_permission_java(method, path, *, params=None, json_data=None, current_row=None): +def _proxy_permission_java( + method, path, *, params=None, json_data=None, files=None, data=None, current_row=None): """Call Java permission APIs using either forwarded JWT or trusted Flask identity.""" proxy_params = {} request_row = getattr(g, '_current_user_row', None) if has_request_context() else None @@ -353,6 +354,8 @@ def _proxy_permission_java(method, path, *, params=None, json_data=None, current path, params=proxy_params or None, json_data=json_data, + files=files, + data=data, headers=headers, ) return result, error_response, status @@ -905,7 +908,7 @@ def list_users(): return jsonify({'success': False, 'error': '需要登录'}), 403 page = max(1, int(request.args.get('page', 1))) _, _, denied = _ensure_backend_menu_access( - 'users', 'history', 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data' + 'users', 'history', 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data' ) if denied: return denied @@ -1335,7 +1338,12 @@ def _parse_admin_datetime_arg(name): raise ValueError(f'{name} 时间格式无效') from exc -_SHOP_DATA_CRAWL_ADMIN_COLUMNS = """ +_SHOP_DATA_CRAWL_LATEST_TIME_SQL = ( + 'COALESCE(df.last_success_at, df.updated_at, t.finished_at, t.updated_at, t.created_at)' +) + + +_SHOP_DATA_CRAWL_ADMIN_COLUMNS = f""" r.id AS result_id, r.task_id, r.user_id, r.source_filename AS shop_name, r.source_file_url AS shop_id, r.result_filename, r.result_file_url, r.result_file_size, r.result_content_type, r.row_count, @@ -1343,6 +1351,7 @@ _SHOP_DATA_CRAWL_ADMIN_COLUMNS = """ r.created_at AS result_created_at, t.task_no, t.status AS task_status, t.request_json, t.result_json, t.error_message AS task_error, t.created_at, t.updated_at, t.finished_at, + {_SHOP_DATA_CRAWL_LATEST_TIME_SQL} AS latest_file_updated_at, u.username, (SELECT j.id FROM biz_task_file_job j WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id @@ -1446,16 +1455,18 @@ def _shop_data_crawl_admin_item(row, group_names=None): def _shop_data_crawl_group_item(group_row, result_rows, group_names): - """Build one shop group and cap its children to the newest three results.""" + """Build one shop group and expose its current daily workbook only.""" raw_shop_name = group_row.get('shop_name') or '' display_shop_name = raw_shop_name or '未命名' group_key = _shop_data_crawl_shop_key(raw_shop_name) children = result_rows.get(group_key) if children is None: children = result_rows.get(str(raw_shop_name).strip(), []) - children = children[:3] + children = children[:1] result_items = [_shop_data_crawl_admin_item(row, group_names) for row in children] latest_created_at = group_row.get('latest_created_at') + if latest_created_at is None and children: + latest_created_at = children[0].get('latest_file_updated_at') if latest_created_at is None and result_items: latest_created_at = result_items[0].get('created_at') return { @@ -1541,6 +1552,7 @@ def list_shop_data_crawl_tasks(): grouped_from_sql = ( ' FROM biz_file_result r ' 'JOIN biz_file_task t ON t.id = r.task_id ' + 'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id ' 'LEFT JOIN users u ON u.id = r.user_id ' 'WHERE ' + where_sql ) @@ -1554,7 +1566,8 @@ def list_shop_data_crawl_tasks(): total = int((cur.fetchone() or {}).get('total') or 0) cur.execute( - 'SELECT ' + shop_key_sql + ' AS shop_name, MAX(t.created_at) AS latest_created_at' + 'SELECT ' + shop_key_sql + ' AS shop_name, MAX(' + + _SHOP_DATA_CRAWL_LATEST_TIME_SQL + ') AS latest_created_at' + grouped_from_sql + ' GROUP BY ' + shop_key_sql + ' ORDER BY latest_created_at DESC, shop_name ASC LIMIT %s OFFSET %s', @@ -1570,14 +1583,16 @@ def list_shop_data_crawl_tasks(): cur.execute( 'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS + ', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql + - ' ORDER BY t.created_at DESC, r.id DESC) AS shop_row_number ' + ' ORDER BY ' + _SHOP_DATA_CRAWL_LATEST_TIME_SQL + + ' DESC, r.id DESC) AS shop_row_number ' ' FROM biz_file_result r ' 'JOIN biz_file_task t ON t.id = r.task_id ' + 'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id ' 'LEFT JOIN users u ON u.id = r.user_id ' 'WHERE ' + where_sql + f' AND {shop_key_sql} IN ({placeholders})' + - ') ranked WHERE ranked.shop_row_number <= 3 ' - 'ORDER BY ranked.created_at DESC, ranked.result_id DESC', + ') ranked WHERE ranked.shop_row_number <= 1 ' + 'ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC', tuple(params + selected_shop_names), ) for row in cur.fetchall(): @@ -2863,6 +2878,26 @@ def delete_shop_key(item_id): # ---------- 数据去重总数据 ---------- +def _parse_positive_group_id(raw_value): + try: + group_id = int(str(raw_value or '').strip()) + except (TypeError, ValueError): + return None + return group_id if group_id > 0 else None + + +def _format_dedupe_total_data_item(item): + return { + 'id': item.get('id'), + 'data_value': item.get('dataValue') or '', + 'group_id': item.get('groupId'), + 'group_name': item.get('groupName') or '', + 'uploader_user_id': item.get('uploaderUserId'), + 'username': item.get('username') or '', + 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], + } + + @admin_api.route('/dedupe-total-data') @login_required def list_dedupe_total_data(): @@ -2873,6 +2908,7 @@ def list_dedupe_total_data(): page_size = min(100, max(1, int(request.args.get('page_size', 15)))) keyword = (request.args.get('keyword') or '').strip() username = (request.args.get('username') or '').strip() + group_id = _parse_positive_group_id(request.args.get('group_id') or request.args.get('groupId')) start_date = (request.args.get('start_date') or request.args.get('startDate') or '').strip() end_date = (request.args.get('end_date') or request.args.get('endDate') or '').strip() params = { @@ -2886,22 +2922,19 @@ def list_dedupe_total_data(): params['startDate'] = start_date if end_date: params['endDate'] = end_date - data, error_response, status = _proxy_backend_java( + if group_id is not None: + params['groupId'] = group_id + data, error_response, status = _proxy_permission_java( 'GET', '/api/admin/dedupe-total-data', params=params, + current_row=current_row, ) if error_response is not None: return error_response, status payload = data.get('data') or {} items = [ - { - 'id': item.get('id'), - 'data_value': item.get('dataValue') or '', - 'uploader_user_id': item.get('uploaderUserId'), - 'username': item.get('username') or '', - 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], - } + _format_dedupe_total_data_item(item) for item in (payload.get('items') or []) ] return jsonify({ @@ -2929,10 +2962,18 @@ def export_dedupe_total_data(): params['startDate'] = start_date if end_date: params['endDate'] = end_date + group_id = _parse_positive_group_id(request.args.get('group_id') or request.args.get('groupId')) + if group_id is not None: + params['groupId'] = group_id url = f"{backend_java_base_url}/api/admin/dedupe-total-data/export" try: - resp = _get_backend_java_session().get(url, params=params, timeout=60) + resp = _get_backend_java_session().get( + url, + params=params, + headers={'X-Internal-Token': _resolve_internal_token()}, + timeout=60, + ) except requests.RequestException: return jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502 if resp.status_code >= 400: @@ -2964,10 +3005,10 @@ def dedupe_total_data_import_progress(import_id): _, current_row, denied = _ensure_dedupe_total_data_access() if denied: return denied - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'GET', f'/api/admin/dedupe-total-data/import/{import_id}', - params={'operatorId': current_row.get('id')}, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -2995,14 +3036,18 @@ def import_dedupe_total_data(): file_storage = request.files.get('file') if not file_storage or file_storage.filename == '': return jsonify({'success': False, 'error': '请选择 Excel 文件'}) + group_id = _parse_positive_group_id(request.form.get('group_id') or request.form.get('groupId')) + if group_id is None: + return jsonify({'success': False, 'error': '请选择分组'}) files = { 'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream') } - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'POST', '/api/admin/dedupe-total-data/import', files=files, - data={'operatorId': current_row.get('id')}, + data={'groupId': group_id}, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3020,10 +3065,10 @@ def dedupe_total_data_delete_import_progress(import_id): _, current_row, denied = _ensure_dedupe_total_data_access() if denied: return denied - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'GET', f'/api/admin/dedupe-total-data/delete-import/{import_id}', - params={'operatorId': current_row.get('id')}, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3051,14 +3096,18 @@ def delete_import_dedupe_total_data(): file_storage = request.files.get('file') if not file_storage or file_storage.filename == '': return jsonify({'success': False, 'error': '请选择 Excel 文件'}) + group_id = _parse_positive_group_id(request.form.get('group_id') or request.form.get('groupId')) + if group_id is None: + return jsonify({'success': False, 'error': '请选择分组'}) files = { 'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream') } - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'POST', '/api/admin/dedupe-total-data/delete-import', files=files, - data={'operatorId': current_row.get('id')}, + data={'groupId': group_id}, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3077,12 +3126,15 @@ def create_dedupe_total_data(): if denied: return denied data = request.get_json() or {} - payload = {'dataValue': (data.get('data_value') or '').strip()} - result, error_response, status = _proxy_backend_java( + payload = { + 'dataValue': (data.get('data_value') or '').strip(), + 'groupId': _parse_positive_group_id(data.get('group_id') or data.get('groupId')), + } + result, error_response, status = _proxy_permission_java( 'POST', '/api/admin/dedupe-total-data', - params={'operatorId': current_row.get('id')}, json_data=payload, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3090,13 +3142,7 @@ def create_dedupe_total_data(): return jsonify({ 'success': True, 'msg': result.get('message') or '创建成功', - 'item': { - 'id': item.get('id'), - 'data_value': item.get('dataValue') or '', - 'uploader_user_id': item.get('uploaderUserId'), - 'username': item.get('username') or '', - 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], - }, + 'item': _format_dedupe_total_data_item(item), }) @@ -3107,12 +3153,15 @@ def update_dedupe_total_data(item_id): if denied: return denied data = request.get_json() or {} - payload = {'dataValue': (data.get('data_value') or '').strip()} - result, error_response, status = _proxy_backend_java( + payload = { + 'dataValue': (data.get('data_value') or '').strip(), + 'groupId': _parse_positive_group_id(data.get('group_id') or data.get('groupId')), + } + result, error_response, status = _proxy_permission_java( 'PUT', f'/api/admin/dedupe-total-data/{item_id}', - params={'operatorId': current_row.get('id')}, json_data=payload, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3120,13 +3169,7 @@ def update_dedupe_total_data(item_id): return jsonify({ 'success': True, 'msg': result.get('message') or '更新成功', - 'item': { - 'id': item.get('id'), - 'data_value': item.get('dataValue') or '', - 'uploader_user_id': item.get('uploaderUserId'), - 'username': item.get('username') or '', - 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], - }, + 'item': _format_dedupe_total_data_item(item), }) @@ -3136,10 +3179,10 @@ def delete_dedupe_total_data(item_id): _, current_row, denied = _ensure_dedupe_total_data_access() if denied: return denied - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'DELETE', f'/api/admin/dedupe-total-data/{item_id}', - params={'operatorId': current_row.get('id')}, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3157,23 +3200,36 @@ def _format_invalid_asin_data_item(item): 'id': item.get('id'), 'data_value': item.get('dataValue') or '', 'brand': item.get('brand') or '', + 'group_id': item.get('groupId'), + 'group_name': item.get('groupName') or '', + 'record_source': item.get('recordSource') or 'AUTO', 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], } @admin_api.route('/invalid-asin-data') -@admin_required +@login_required def list_invalid_asin_data(): - _, _, denied = _ensure_admin_menu_access('invalid-asin-data') + role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data') if denied: return denied page = max(1, int(request.args.get('page', 1))) page_size = min(100, max(1, int(request.args.get('page_size', 15)))) keyword = (request.args.get('keyword') or '').strip() - data, error_response, status = _proxy_backend_java( + group_id_raw = (request.args.get('group_id') or '').strip() + group_id = int(group_id_raw) if group_id_raw.isdigit() and int(group_id_raw) > 0 else None + params = { + 'page': page, + 'pageSize': page_size, + 'keyword': keyword, + } + if group_id is not None: + params['groupId'] = group_id + data, error_response, status = _proxy_permission_java( 'GET', '/api/admin/invalid-asin-data', - params={'page': page, 'pageSize': page_size, 'keyword': keyword}, + params=params, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3189,20 +3245,22 @@ def list_invalid_asin_data(): @admin_api.route('/invalid-asin-data', methods=['POST']) -@admin_required +@login_required def create_invalid_asin_data(): - _, _, denied = _ensure_admin_menu_access('invalid-asin-data') + role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data') if denied: return denied data = request.get_json() or {} payload = { 'dataValue': (data.get('data_value') or '').strip(), 'brand': (data.get('brand') or '').strip(), + 'groupId': data.get('group_id'), } - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'POST', '/api/admin/invalid-asin-data', json_data=payload, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3215,20 +3273,22 @@ def create_invalid_asin_data(): @admin_api.route('/invalid-asin-data/', methods=['PUT']) -@admin_required +@login_required def update_invalid_asin_data(item_id): - _, _, denied = _ensure_admin_menu_access('invalid-asin-data') + role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data') if denied: return denied data = request.get_json() or {} payload = { 'dataValue': (data.get('data_value') or '').strip(), 'brand': (data.get('brand') or '').strip(), + 'groupId': data.get('group_id'), } - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'PUT', f'/api/admin/invalid-asin-data/{item_id}', json_data=payload, + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3241,14 +3301,15 @@ def update_invalid_asin_data(item_id): @admin_api.route('/invalid-asin-data/', methods=['DELETE']) -@admin_required +@login_required def delete_invalid_asin_data(item_id): - _, _, denied = _ensure_admin_menu_access('invalid-asin-data') + role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data') if denied: return denied - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'DELETE', f'/api/admin/invalid-asin-data/{item_id}', + current_row=current_row, ) if error_response is not None: return error_response, status @@ -3515,21 +3576,29 @@ def delete_shop_manage(item_id): @login_required def list_shop_manage_groups(): role, current_row, denied = _ensure_backend_menu_access( - 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data' + 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data' ) if denied: return denied groups, error_response, status = _load_expanded_shop_manage_groups(role, current_row) if error_response is not None: return error_response, status - return jsonify({'success': True, 'items': [_format_shop_manage_group_item(item) for item in (groups or [])]}) + items = [_format_shop_manage_group_item(item) for item in (groups or [])] + locked_group_id = None + if role != 'super_admin': + locked_group_id = next((item.get('id') for item in items if item.get('id')), None) + return jsonify({ + 'success': True, + 'items': items, + 'locked_group_id': locked_group_id, + }) @admin_api.route('/shop-manage-group', methods=['POST']) @login_required def create_shop_manage_group(): role, current_row, denied = _ensure_backend_menu_access( - 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data' + 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data' ) if denied: return denied @@ -3576,7 +3645,7 @@ def create_shop_manage_group(): @login_required def update_shop_manage_group(item_id): role, current_row, denied = _ensure_backend_menu_access( - 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data' + 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data' ) if denied: return denied @@ -3823,7 +3892,7 @@ def delete_skip_price_asin_country(item_id, country): @login_required def delete_shop_manage_group(item_id): role, current_row, denied = _ensure_backend_menu_access( - 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data' + 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data' ) if denied: return denied diff --git a/backend/requirement.txt b/backend/requirement.txt index 7f798fd5..5cfef75c 100644 --- a/backend/requirement.txt +++ b/backend/requirement.txt @@ -3,6 +3,8 @@ annotated-types==0.7.0 anyio==4.12.1 Authlib==1.6.8 blinker==1.9.0 +boto3==1.43.65 +botocore==1.43.65 bottle==0.13.4 certifi==2026.1.4 cffi==2.0.0 diff --git a/backend/static/admin.js b/backend/static/admin.js index c4ec8a5a..8e15778a 100644 --- a/backend/static/admin.js +++ b/backend/static/admin.js @@ -120,7 +120,10 @@ loadDedupeTotalData(1); loadDedupeGroupSummary(); } - else if (tabName === 'invalid-asin-data') loadInvalidAsinData(1); + else if (tabName === 'invalid-asin-data') { + loadShopManageGroups(); + loadInvalidAsinData(1); + } else if (tabName === 'shop-keys') loadShopKeys(1); else if (tabName === 'shop-manage') loadShopManage(1); else if (tabName === 'skip-price-asin') loadSkipPriceAsin(1); @@ -1526,7 +1529,7 @@ }); groups.forEach(function (group) { group.results.sort(shopDataResultSort); - group.results = group.results.slice(0, 3); + group.results = group.results.slice(0, 1); if (!group.latest_created_at && group.results.length) { group.latest_created_at = group.results[0].created_at || group.results[0].finished_at || ''; } @@ -1579,7 +1582,7 @@ '
' + '
' + '
' + escapeHtml(group.shop_name || '-') + '
' + - '' + results.length + '/3 份结果' + + '' + results.length + '/1 份当日累计文件' + '
' + '
' + '
' + escapeHtml(group.group_name || '-') + '
' + @@ -1641,7 +1644,7 @@ var total = payload.total != null ? Number(payload.total) : shopDataTaskGroups.length; var responsePage = payload.page || page; var responsePageSize = payload.page_size || shopDataTaskPageSize; - document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留最新 3 份任务结果'; + document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留 1 份当日累计文件'; renderShopDataTasks(); renderPagination('shopDataTaskPagination', total, responsePage, responsePageSize, loadShopDataCrawlTasks); }) @@ -2176,9 +2179,11 @@ var q = 'page=' + (page || 1) + '&page_size=' + dedupeTotalDataPageSize; var keyword = (document.getElementById('searchDedupeTotalData').value || '').trim(); var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim(); + var groupId = (document.getElementById('dedupeTotalDataGroupFilterId').value || '').trim(); var dateRange = getDedupeTotalDataDateRange(); if (keyword) q += '&keyword=' + encodeURIComponent(keyword); if (username) q += '&username=' + encodeURIComponent(username); + if (groupId) q += '&group_id=' + encodeURIComponent(groupId); if (dateRange.startDate) q += '&start_date=' + encodeURIComponent(dateRange.startDate); if (dateRange.endDate) q += '&end_date=' + encodeURIComponent(dateRange.endDate); return q; @@ -2191,16 +2196,16 @@ .then(function (res) { var tbody = document.getElementById('dedupeTotalDataListBody'); if (!res.success) { - tbody.innerHTML = '加载失败: ' + escapeHtml(res.error || '') + ''; + tbody.innerHTML = '加载失败: ' + escapeHtml(res.error || '') + ''; return; } var items = res.items || []; if (items.length === 0) { - tbody.innerHTML = '暂无总数据'; + tbody.innerHTML = '暂无总数据'; } else { tbody.innerHTML = items.map(function (item) { - return '' + escapeHtml(item.id) + '' + escapeHtml(item.data_value || '') + '' + escapeHtml(item.username || '') + '' + escapeHtml(item.created_at || '') + '' + - ' ' + + return '' + escapeHtml(item.id) + '' + escapeHtml(item.data_value || '') + '' + escapeHtml(item.username || '') + '' + escapeHtml(item.group_name || '未分组') + '' + escapeHtml(item.created_at || '') + '' + + ' ' + '' + ''; }).join(''); @@ -2209,7 +2214,7 @@ bindDedupeTotalDataActions(); }) .catch(function () { - document.getElementById('dedupeTotalDataListBody').innerHTML = '请求失败'; + document.getElementById('dedupeTotalDataListBody').innerHTML = '请求失败'; }); } function bindDedupeTotalDataActions() { @@ -2219,7 +2224,11 @@ document.getElementById('editDedupeTotalDataValue').value = (btn.dataset.value || '').replace(/"/g, '"'); document.getElementById('msgEditDedupeTotalData').textContent = ''; document.getElementById('msgEditDedupeTotalData').className = 'msg'; - document.getElementById('editDedupeTotalDataModal').classList.add('show'); + var groupId = btn.dataset.groupId || ''; + loadShopManageGroups().then(function () { + document.getElementById('editDedupeTotalDataGroupId').value = groupId; + document.getElementById('editDedupeTotalDataModal').classList.add('show'); + }); }; }); document.querySelectorAll('[data-dedupe-total-delete]').forEach(function (btn) { @@ -2238,10 +2247,12 @@ document.getElementById('btnSearchDedupeTotalData').onclick = function () { loadDedupeTotalData(1); }; document.getElementById('btnExportDedupeTotalData').onclick = function () { var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim(); + var groupId = (document.getElementById('dedupeTotalDataGroupFilterId').value || '').trim(); var dateRange = getDedupeTotalDataDateRange(); if (!validateDedupeTotalDataDateRange()) return; var params = []; if (username) params.push('username=' + encodeURIComponent(username)); + if (groupId) params.push('group_id=' + encodeURIComponent(groupId)); if (dateRange.startDate) params.push('start_date=' + encodeURIComponent(dateRange.startDate)); if (dateRange.endDate) params.push('end_date=' + encodeURIComponent(dateRange.endDate)); fetch('/api/admin/dedupe-total-data/export' + (params.length ? ('?' + params.join('&')) : '')) @@ -2386,6 +2397,12 @@ msgEl.classList.add('err'); return; } + var groupId = (document.getElementById('dedupeTotalDataImportGroupId').value || '').trim(); + if (!groupId) { + msgEl.textContent = '请选择分组'; + msgEl.classList.add('err'); + return; + } var file = fileInput.files[0]; var lowerName = (file.name || '').toLowerCase(); if (!(lowerName.endsWith('.xlsx') || lowerName.endsWith('.xls'))) { @@ -2395,6 +2412,7 @@ } var formData = new FormData(); formData.append('file', file); + formData.append('group_id', groupId); var xhr = new XMLHttpRequest(); xhr.open('POST', '/api/admin/dedupe-total-data/import', true); xhr.upload.onprogress = function (event) { @@ -2449,8 +2467,15 @@ if (!confirm('确定按 Excel 中的 ASIN 批量删除匹配的总数据吗？')) { return; } + var groupId = (document.getElementById('dedupeTotalDataDeleteGroupId').value || '').trim(); + if (!groupId) { + msgEl.textContent = '请选择分组'; + msgEl.classList.add('err'); + return; + } var formData = new FormData(); formData.append('file', file); + formData.append('group_id', groupId); var xhr = new XMLHttpRequest(); xhr.open('POST', '/api/admin/dedupe-total-data/delete-import', true); xhr.upload.onprogress = function (event) { @@ -2486,6 +2511,7 @@ document.getElementById('btnSaveDedupeTotalData').onclick = function () { var itemId = document.getElementById('editDedupeTotalDataId').value; var value = (document.getElementById('editDedupeTotalDataValue').value || '').trim(); + var groupId = (document.getElementById('editDedupeTotalDataGroupId').value || '').trim(); var msgEl = document.getElementById('msgEditDedupeTotalData'); msgEl.textContent = ''; msgEl.className = 'msg'; @@ -2494,10 +2520,15 @@ msgEl.classList.add('err'); return; } + if (!groupId) { + msgEl.textContent = '请选择分组'; + msgEl.classList.add('err'); + return; + } fetch('/api/admin/dedupe-total-data/' + itemId, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ data_value: value }) + body: JSON.stringify({ data_value: value, group_id: Number(groupId) }) }) .then(function (r) { return r.json(); }) .then(function (res) { @@ -2519,7 +2550,9 @@ function buildInvalidAsinDataQuery(page) { var q = 'page=' + (page || 1) + '&page_size=' + invalidAsinDataPageSize; var keyword = (document.getElementById('searchInvalidAsinData').value || '').trim(); + var groupId = (document.getElementById('invalidAsinDataFilterGroupId').value || '').trim(); if (keyword) q += '&keyword=' + encodeURIComponent(keyword); + if (groupId) q += '&group_id=' + encodeURIComponent(groupId); return q; } function loadInvalidAsinData(page) { @@ -2529,18 +2562,21 @@ .then(function (res) { var tbody = document.getElementById('invalidAsinDataListBody'); if (!res.success) { - tbody.innerHTML = '加载失败: ' + (res.error || '') + ''; + tbody.innerHTML = '加载失败: ' + (res.error || '') + ''; return; } var items = res.items || []; if (items.length === 0) { - tbody.innerHTML = '暂无数据'; + tbody.innerHTML = '暂无数据'; } else { tbody.innerHTML = items.map(function (item) { var dataValueAttr = (item.data_value || '').replace(/"/g, '"'); var brandAttr = (item.brand || '').replace(/"/g, '"'); - return '' + item.id + '' + (item.data_value || '') + '' + (item.brand || '') + '' + (item.created_at || '') + '' + - ' ' + + var groupId = item.group_id == null ? '' : String(item.group_id); + var source = String(item.record_source || 'AUTO').toUpperCase(); + var sourceLabel = source === 'MANUAL' ? '\u624b\u52a8\u65b0\u589e' : '\u81ea\u52a8\u5bfc\u5165'; + return '' + escapeHtml(item.id || '') + '' + escapeHtml(item.data_value || '') + '' + escapeHtml(item.brand || '') + '' + escapeHtml(item.group_name || '') + '' + sourceLabel + '' + escapeHtml(item.created_at || '') + '' + + ' ' + '' + ''; }).join(''); @@ -2549,7 +2585,7 @@ bindInvalidAsinDataActions(); }) .catch(function () { - document.getElementById('invalidAsinDataListBody').innerHTML = '请求失败'; + document.getElementById('invalidAsinDataListBody').innerHTML = '请求失败'; }); } function bindInvalidAsinDataActions() { @@ -2558,6 +2594,20 @@ document.getElementById('editInvalidAsinDataId').value = btn.dataset.invalidAsinEdit || ''; document.getElementById('editInvalidAsinDataValue').value = (btn.dataset.value || '').replace(/"/g, '"'); document.getElementById('editInvalidAsinDataBrand').value = (btn.dataset.brand || '').replace(/"/g, '"'); + var source = String(btn.dataset.source || 'AUTO').toUpperCase(); + var isManual = source === 'MANUAL'; + document.getElementById('editInvalidAsinDataRecordSource').value = source; + document.getElementById('editInvalidAsinDataSourceLabel').value = isManual + ? '\u624b\u52a8\u65b0\u589e' + : '\u81ea\u52a8\u5bfc\u5165'; + document.getElementById('editInvalidAsinDataGroupForm').style.display = isManual ? '' : 'none'; + if (isManual) { + loadShopManageGroups().then(function () { + document.getElementById('editInvalidAsinDataGroupSelect').value = currentUserRole === 'super_admin' + ? (btn.dataset.groupId || '') + : getInvalidAsinDataLockedGroupId(); + }); + } document.getElementById('msgEditInvalidAsinData').textContent = ''; document.getElementById('msgEditInvalidAsinData').className = 'msg'; document.getElementById('editInvalidAsinDataModal').classList.add('show'); @@ -2580,6 +2630,7 @@ document.getElementById('btnAddInvalidAsinData').onclick = function () { var dataValue = (document.getElementById('invalidAsinDataValue').value || '').trim(); var brand = (document.getElementById('invalidAsinDataBrand').value || '').trim(); + var groupId = (document.getElementById('invalidAsinDataGroupSelect').value || '').trim(); var msgEl = document.getElementById('msgInvalidAsinData'); msgEl.textContent = ''; msgEl.className = 'msg'; @@ -2588,10 +2639,15 @@ msgEl.classList.add('err'); return; } + if (!groupId) { + msgEl.textContent = '请选择分组'; + msgEl.classList.add('err'); + return; + } fetch('/api/admin/invalid-asin-data', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ data_value: dataValue, brand: brand }) + body: JSON.stringify({ data_value: dataValue, brand: brand, group_id: Number(groupId) }) }) .then(function (r) { return r.json(); }) .then(function (res) { @@ -2600,6 +2656,9 @@ msgEl.classList.add('ok'); document.getElementById('invalidAsinDataValue').value = ''; document.getElementById('invalidAsinDataBrand').value = ''; + document.getElementById('invalidAsinDataGroupSelect').value = currentUserRole === 'super_admin' + ? '' + : getInvalidAsinDataLockedGroupId(); loadInvalidAsinData(1); } else { msgEl.textContent = res.error || '创建失败'; @@ -2615,6 +2674,8 @@ var itemId = document.getElementById('editInvalidAsinDataId').value; var dataValue = (document.getElementById('editInvalidAsinDataValue').value || '').trim(); var brand = (document.getElementById('editInvalidAsinDataBrand').value || '').trim(); + var source = String(document.getElementById('editInvalidAsinDataRecordSource').value || 'AUTO').toUpperCase(); + var groupId = (document.getElementById('editInvalidAsinDataGroupSelect').value || '').trim(); var msgEl = document.getElementById('msgEditInvalidAsinData'); msgEl.textContent = ''; msgEl.className = 'msg'; @@ -2623,10 +2684,19 @@ msgEl.classList.add('err'); return; } + if (source === 'MANUAL' && !groupId) { + msgEl.textContent = '请选择分组'; + msgEl.classList.add('err'); + return; + } fetch('/api/admin/invalid-asin-data/' + itemId, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ data_value: dataValue, brand: brand }) + body: JSON.stringify({ + data_value: dataValue, + brand: brand, + group_id: source === 'MANUAL' ? Number(groupId) : null + }) }) .then(function (r) { return r.json(); }) .then(function (res) { @@ -2803,6 +2873,7 @@ var shopManagePage = 1, shopManagePageSize = 15; var shopManageGroups = []; var shopManageGroupsLoadedAt = 0; + var shopManageLockedGroupId = null; var currentShopManageGroupGrantRoutes = []; function buildShopManageQuery(page) { @@ -2926,9 +2997,22 @@ }); } + function getInvalidAsinDataLockedGroupId() { + if (currentUserRole === 'super_admin') return ''; + if (shopManageLockedGroupId) return String(shopManageLockedGroupId); + return shopManageGroups.length ? String(shopManageGroups[0].id || '') : ''; + } + function refreshShopGroupSelects(selectedCreateId, selectedEditId) { var createSel = document.getElementById('shopManageGroupSelect'); var editSel = document.getElementById('editShopManageGroupSelect'); + var invalidCreateSel = document.getElementById('invalidAsinDataGroupSelect'); + var invalidEditSel = document.getElementById('editInvalidAsinDataGroupSelect'); + var invalidFilterSel = document.getElementById('invalidAsinDataFilterGroupId'); + var dedupeImportSel = document.getElementById('dedupeTotalDataImportGroupId'); + var dedupeDeleteSel = document.getElementById('dedupeTotalDataDeleteGroupId'); + var dedupeFilterSel = document.getElementById('dedupeTotalDataGroupFilterId'); + var dedupeEditSel = document.getElementById('editDedupeTotalDataGroupId'); var filterSel = document.getElementById('shopManageFilterGroupId'); var skipCreateSel = document.getElementById('skipPriceAsinGroupSelect'); var skipFilterSel = document.getElementById('skipPriceAsinFilterGroupId'); @@ -2936,6 +3020,13 @@ var queryCreateSel = document.getElementById('queryAsinGroupSelect'); var queryFilterSel = document.getElementById('queryAsinFilterGroupId'); var chooseQueryShopSel = document.getElementById('chooseQueryAsinShopGroupId'); + var selectedInvalidCreateId = invalidCreateSel ? invalidCreateSel.value : ''; + var selectedInvalidEditId = invalidEditSel ? invalidEditSel.value : ''; + var selectedInvalidFilterId = invalidFilterSel ? invalidFilterSel.value : ''; + var selectedDedupeImportId = dedupeImportSel ? dedupeImportSel.value : ''; + var selectedDedupeDeleteId = dedupeDeleteSel ? dedupeDeleteSel.value : ''; + var selectedDedupeFilterId = dedupeFilterSel ? dedupeFilterSel.value : ''; + var selectedDedupeEditId = dedupeEditSel ? dedupeEditSel.value : ''; var selectedFilterId = filterSel ? filterSel.value : ''; var selectedSkipCreateId = skipCreateSel ? skipCreateSel.value : ''; var selectedSkipFilterId = skipFilterSel ? skipFilterSel.value : ''; @@ -2952,6 +3043,37 @@ }); createSel.innerHTML = createOpts.join(''); editSel.innerHTML = createOpts.join(''); + var lockedInvalidGroupId = getInvalidAsinDataLockedGroupId(); + var lockedGroup = shopManageGroups.find(function (group) { + return String(group.id || '') === lockedInvalidGroupId; + }); + var invalidGroupOpts = createOpts; + if (currentUserRole !== 'super_admin') { + invalidGroupOpts = lockedGroup + ? [''] + : [createOpts[0]]; + } + var invalidFilterOpts = currentUserRole === 'super_admin' + ? filterOpts + : (lockedGroup + ? [''] + : [createOpts[0]]); + if (invalidCreateSel) { + invalidCreateSel.innerHTML = invalidGroupOpts.join(''); + invalidCreateSel.disabled = currentUserRole !== 'super_admin'; + } + if (invalidEditSel) { + invalidEditSel.innerHTML = invalidGroupOpts.join(''); + invalidEditSel.disabled = currentUserRole !== 'super_admin'; + } + if (invalidFilterSel) { + invalidFilterSel.innerHTML = invalidFilterOpts.join(''); + invalidFilterSel.disabled = currentUserRole !== 'super_admin'; + } + if (dedupeImportSel) dedupeImportSel.innerHTML = createOpts.join(''); + if (dedupeDeleteSel) dedupeDeleteSel.innerHTML = createOpts.join(''); + if (dedupeFilterSel) dedupeFilterSel.innerHTML = filterOpts.join(''); + if (dedupeEditSel) dedupeEditSel.innerHTML = createOpts.join(''); if (skipCreateSel) skipCreateSel.innerHTML = createOpts.join(''); if (queryCreateSel) queryCreateSel.innerHTML = createOpts.join(''); if (filterSel) filterSel.innerHTML = filterOpts.join(''); @@ -2961,6 +3083,25 @@ if (chooseQueryShopSel) chooseQueryShopSel.innerHTML = filterOpts.join(''); if (selectedCreateId != null) createSel.value = String(selectedCreateId); if (selectedEditId != null) editSel.value = String(selectedEditId); + if (invalidCreateSel) { + invalidCreateSel.value = currentUserRole === 'super_admin' + ? selectedInvalidCreateId + : lockedInvalidGroupId; + } + if (invalidEditSel) { + invalidEditSel.value = currentUserRole === 'super_admin' + ? selectedInvalidEditId + : lockedInvalidGroupId; + } + if (invalidFilterSel) { + invalidFilterSel.value = currentUserRole === 'super_admin' + ? selectedInvalidFilterId + : lockedInvalidGroupId; + } + if (dedupeImportSel && selectedDedupeImportId) dedupeImportSel.value = selectedDedupeImportId; + if (dedupeDeleteSel && selectedDedupeDeleteId) dedupeDeleteSel.value = selectedDedupeDeleteId; + if (dedupeFilterSel && selectedDedupeFilterId) dedupeFilterSel.value = selectedDedupeFilterId; + if (dedupeEditSel && selectedDedupeEditId) dedupeEditSel.value = selectedDedupeEditId; if (filterSel && selectedFilterId) filterSel.value = selectedFilterId; if (skipCreateSel && selectedSkipCreateId) skipCreateSel.value = selectedSkipCreateId; if (skipFilterSel && selectedSkipFilterId) skipFilterSel.value = selectedSkipFilterId; @@ -2980,12 +3121,14 @@ .then(function (res) { if (!res.success) throw new Error(res.error || '加载分组失败'); shopManageGroups = res.items || []; + shopManageLockedGroupId = res.locked_group_id || null; shopManageGroupsLoadedAt = Date.now(); refreshShopGroupSelects(selectedCreateId, selectedEditId); return shopManageGroups; }) .catch(function () { shopManageGroups = []; + shopManageLockedGroupId = null; shopManageGroupsLoadedAt = 0; refreshShopGroupSelects(); return []; @@ -3040,7 +3183,7 @@ function updateShopManageGroupButtonsAccess() { var canManageGroups = !!currentUserId; - ['btnManageShopGroups', 'btnManageShopGroupsFromEdit', 'btnManageSkipPriceAsinGroups', 'btnManageQueryAsinGroups', 'btnManageDedupeGroups'].forEach(function (id) { + ['btnManageShopGroups', 'btnManageShopGroupsFromEdit', 'btnManageSkipPriceAsinGroups', 'btnManageQueryAsinGroups', 'btnManageDedupeGroups', 'btnManageInvalidAsinDataGroups', 'btnManageInvalidAsinDataGroupsFromEdit'].forEach(function (id) { var btn = document.getElementById(id); if (btn) btn.style.display = canManageGroups ? '' : 'none'; }); @@ -3123,6 +3266,8 @@ renderDedupeGroupSummary(); loadShopManage(shopManagePage); loadSkipPriceAsin(skipPriceAsinPage); + if (getActiveAdminTabName() === 'dedupe-total-data') loadDedupeTotalData(dedupeTotalDataPage); + if (getActiveAdminTabName() === 'invalid-asin-data') loadInvalidAsinData(invalidAsinDataPage); }); }); }; @@ -3149,6 +3294,14 @@ setShopManageGroupGrantRoutes(['dedupe-total-data']); openShopManageGroupModal(); }; + document.getElementById('btnManageInvalidAsinDataGroups').onclick = function () { + setShopManageGroupGrantRoutes(['invalid-asin-data']); + openShopManageGroupModal(); + }; + document.getElementById('btnManageInvalidAsinDataGroupsFromEdit').onclick = function () { + setShopManageGroupGrantRoutes(['invalid-asin-data']); + openShopManageGroupModal(); + }; document.getElementById('btnSearchShopManage').onclick = function () { loadShopManage(1); }; @@ -3200,6 +3353,8 @@ renderDedupeGroupSummary(); loadShopManage(shopManagePage); loadSkipPriceAsin(skipPriceAsinPage); + if (getActiveAdminTabName() === 'dedupe-total-data') loadDedupeTotalData(dedupeTotalDataPage); + if (getActiveAdminTabName() === 'invalid-asin-data') loadInvalidAsinData(invalidAsinDataPage); }); }) .catch(function () { diff --git a/backend/tests/test_admin_dedupe_total_data.py b/backend/tests/test_admin_dedupe_total_data.py new file mode 100644 index 00000000..7a1f0059 --- /dev/null +++ b/backend/tests/test_admin_dedupe_total_data.py @@ -0,0 +1,161 @@ +import io +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +from flask import Flask + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from blueprints import admin_api + + +class _FakeExportResponse: + status_code = 200 + content = b'xlsx' + headers = { + 'Content-Disposition': 'attachment; filename=dedupe.xlsx', + 'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + } + + +class AdminDedupeTotalDataTest(unittest.TestCase): + def setUp(self): + self.app = Flask(__name__) + + def _menu_access(self): + return patch.object( + admin_api, + '_ensure_backend_menu_access', + return_value=('admin', {'id': 7}, None), + ) + + def test_list_forwards_group_and_formats_item(self): + java_response = { + 'data': { + 'items': [{ + 'id': 9, + 'dataValue': 'B012345678', + 'groupId': 3, + 'groupName': 'group-a', + 'uploaderUserId': 7, + 'username': 'operator', + 'createdAt': '2026-08-08T12:30:00', + }], + 'total': 1, + 'page': 1, + 'pageSize': 15, + }, + } + with self.app.test_request_context( + '/api/admin/dedupe-total-data?page=1&page_size=15&group_id=3'): + with self._menu_access(), patch.object( + admin_api, + '_proxy_permission_java', + return_value=(java_response, None, 200), + ) as proxy: + response = admin_api.list_dedupe_total_data.__wrapped__() + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.get_json()['items'][0]['group_name'], 'group-a') + self.assertEqual(proxy.call_args.kwargs['params'], { + 'page': 1, + 'pageSize': 15, + 'keyword': '', + 'username': '', + 'operatorId': 7, + 'groupId': 3, + }) + + def test_create_and_update_forward_group(self): + java_response = { + 'message': 'ok', + 'data': {'id': 9, 'dataValue': 'B012345678', 'groupId': 3, 'groupName': 'group-a'}, + } + with self.app.test_request_context( + '/api/admin/dedupe-total-data', method='POST', + json={'data_value': ' B012345678 ', 'group_id': 3}): + with self._menu_access(), patch.object( + admin_api, + '_proxy_permission_java', + return_value=(java_response, None, 200), + ) as proxy: + response = admin_api.create_dedupe_total_data.__wrapped__() + + self.assertEqual(response.status_code, 200) + self.assertEqual(proxy.call_args.kwargs['json_data'], { + 'dataValue': 'B012345678', + 'groupId': 3, + }) + + with self.app.test_request_context( + '/api/admin/dedupe-total-data/9', method='PUT', + json={'data_value': ' C012345678 ', 'group_id': 8}): + with self._menu_access(), patch.object( + admin_api, + '_proxy_permission_java', + return_value=(java_response, None, 200), + ) as proxy: + response = admin_api.update_dedupe_total_data.__wrapped__(9) + + self.assertEqual(response.status_code, 200) + self.assertEqual(proxy.call_args.kwargs['json_data'], { + 'dataValue': 'C012345678', + 'groupId': 8, + }) + + def test_import_forwards_multipart_group(self): + with self.app.test_request_context( + '/api/admin/dedupe-total-data/import', method='POST', + data={'file': (io.BytesIO(b'xlsx'), 'data.xlsx'), 'group_id': '3'}): + with self._menu_access(), patch.object( + admin_api, + '_proxy_permission_java', + return_value=({'message': 'started', 'data': {'importId': 'task-1'}}, None, 200), + ) as proxy: + response = admin_api.import_dedupe_total_data.__wrapped__() + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.get_json()['import_id'], 'task-1') + self.assertEqual(proxy.call_args.kwargs['data'], {'groupId': 3}) + self.assertEqual(set(proxy.call_args.kwargs['files']), {'file'}) + + def test_export_forwards_group_filter_and_internal_token(self): + class _Session: + def get(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + return _FakeExportResponse() + + session = _Session() + with self.app.test_request_context( + '/api/admin/dedupe-total-data/export?group_id=3&username=operator'): + with self._menu_access(), patch.object( + admin_api, + '_get_backend_java_session', + return_value=session, + ), patch.object(admin_api, '_resolve_internal_token', return_value='token'): + response = admin_api.export_dedupe_total_data.__wrapped__() + + self.assertEqual(response.status_code, 200) + self.assertEqual(session.kwargs['params'], { + 'operatorId': 7, + 'username': 'operator', + 'groupId': 3, + }) + self.assertEqual(session.kwargs['headers'], {'X-Internal-Token': 'token'}) + + def test_import_requires_group(self): + with self.app.test_request_context( + '/api/admin/dedupe-total-data/import', method='POST', + data={'file': (io.BytesIO(b'xlsx'), 'data.xlsx')}): + with self._menu_access(), patch.object(admin_api, '_proxy_permission_java') as proxy: + response = admin_api.import_dedupe_total_data.__wrapped__() + + self.assertEqual(response.status_code, 200) + self.assertFalse(response.get_json()['success']) + proxy.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/backend/tests/test_admin_invalid_asin_data.py b/backend/tests/test_admin_invalid_asin_data.py new file mode 100644 index 00000000..095ad70a --- /dev/null +++ b/backend/tests/test_admin_invalid_asin_data.py @@ -0,0 +1,141 @@ +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +from flask import Flask + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from blueprints import admin_api + + +class AdminInvalidAsinDataTest(unittest.TestCase): + def setUp(self): + self.app = Flask(__name__) + + def test_list_forwards_operator_and_formats_group_fields(self): + java_response = { + 'data': { + 'items': [{ + 'id': 9, + 'dataValue': 'B012345678', + 'brand': 'acme', + 'groupId': 3, + 'groupName': 'group-a', + 'recordSource': 'MANUAL', + 'createdAt': '2026-08-08T12:30:00', + }], + 'total': 1, + 'page': 1, + 'pageSize': 15, + }, + } + with self.app.test_request_context('/api/admin/invalid-asin-data?page=1&page_size=15&group_id=3'): + with patch.object( + admin_api, + '_ensure_backend_menu_access', + return_value=('admin', {'id': 7}, None), + ), patch.object( + admin_api, + '_proxy_permission_java', + return_value=(java_response, None, 200), + ) as proxy: + response = admin_api.list_invalid_asin_data.__wrapped__() + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.get_json()['items'], [{ + 'id': 9, + 'data_value': 'B012345678', + 'brand': 'acme', + 'group_id': 3, + 'group_name': 'group-a', + 'record_source': 'MANUAL', + 'created_at': '2026-08-08 12:30', + }]) + self.assertEqual( + proxy.call_args.kwargs['params'], + { + 'page': 1, + 'pageSize': 15, + 'keyword': '', + 'groupId': 3, + }, + ) + self.assertEqual(proxy.call_args.kwargs['current_row'], {'id': 7}) + + def test_create_forwards_group_with_trusted_current_user(self): + java_response = { + 'message': 'created', + 'data': { + 'id': 9, + 'dataValue': 'B012345678', + 'brand': 'acme', + 'groupId': 3, + 'groupName': 'group-a', + 'recordSource': 'MANUAL', + }, + } + with self.app.test_request_context( + '/api/admin/invalid-asin-data', + method='POST', + json={'data_value': ' B012345678 ', 'brand': 'Acme', 'group_id': 3}, + ): + with patch.object( + admin_api, + '_ensure_backend_menu_access', + return_value=('super_admin', {'id': 1}, None), + ), patch.object( + admin_api, + '_proxy_permission_java', + return_value=(java_response, None, 200), + ) as proxy: + response = admin_api.create_invalid_asin_data.__wrapped__() + + self.assertEqual(response.status_code, 200) + self.assertNotIn('params', proxy.call_args.kwargs) + self.assertEqual(proxy.call_args.kwargs['current_row'], {'id': 1}) + self.assertEqual(proxy.call_args.kwargs['json_data'], { + 'dataValue': 'B012345678', + 'brand': 'Acme', + 'groupId': 3, + }) + + def test_group_list_marks_first_group_as_locked_for_normal_account(self): + groups = [ + {'id': 3, 'groupName': 'group-a'}, + {'id': 8, 'groupName': 'group-b'}, + ] + with self.app.test_request_context('/api/admin/shop-manage-groups'): + with patch.object( + admin_api, + '_ensure_backend_menu_access', + return_value=('admin', {'id': 7}, None), + ), patch.object( + admin_api, + '_load_expanded_shop_manage_groups', + return_value=(groups, None, 200), + ): + response = admin_api.list_shop_manage_groups.__wrapped__() + + body = response.get_json() + self.assertEqual(body['locked_group_id'], 3) + self.assertEqual([item['id'] for item in body['items']], [3, 8]) + + def test_group_list_does_not_lock_super_admin(self): + with self.app.test_request_context('/api/admin/shop-manage-groups'): + with patch.object( + admin_api, + '_ensure_backend_menu_access', + return_value=('super_admin', {'id': 1}, None), + ), patch.object( + admin_api, + '_load_expanded_shop_manage_groups', + return_value=([{'id': 3, 'groupName': 'group-a'}], None, 200), + ): + response = admin_api.list_shop_manage_groups.__wrapped__() + + self.assertIsNone(response.get_json()['locked_group_id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/backend/tests/test_admin_shop_data_groups.py b/backend/tests/test_admin_shop_data_groups.py index 06b08880..0e7c861d 100644 --- a/backend/tests/test_admin_shop_data_groups.py +++ b/backend/tests/test_admin_shop_data_groups.py @@ -32,7 +32,7 @@ class _FakeCursor: self.calls.append((sql, tuple(params))) if 'COUNT(*) AS total' in sql: self.kind = 'count' - elif 'MAX(t.created_at)' in sql: + elif 'AS latest_created_at' in sql and 'GROUP BY' in sql: self.kind = 'groups' self.group_limit = int(params[-2]) self.group_offset = int(params[-1]) @@ -59,14 +59,14 @@ class _FakeCursor: limited = [] for row in rows: shop_key = row['shop_name'].strip().casefold() - if counts.get(shop_key, 0) >= 3: + if counts.get(shop_key, 0) >= 1: continue counts[shop_key] = counts.get(shop_key, 0) + 1 limited.append(row) return limited if self.current_shop is not None: rows = [row for row in rows if row['shop_name'].strip() == self.current_shop] - return rows[:3] + return rows[:1] return [] @@ -85,7 +85,7 @@ class AdminShopDataGroupTest(unittest.TestCase): def setUp(self): self.app = Flask(__name__) self.group_rows = [ - {'shop_name': 'Shop A', 'latest_created_at': datetime(2026, 8, 5, 12, 0)}, + {'shop_name': 'Shop A', 'latest_created_at': datetime(2026, 8, 7, 5, 21, 45)}, {'shop_name': '', 'latest_created_at': datetime(2026, 8, 4, 12, 0)}, ] self.result_rows = [ @@ -98,7 +98,8 @@ class AdminShopDataGroupTest(unittest.TestCase): ] @staticmethod - def _result_row(result_id, shop_name, created_at, result_file_url=None): + def _result_row( + result_id, shop_name, created_at, result_file_url=None, latest_file_updated_at=None): return { 'result_id': result_id, 'task_id': result_id + 100, @@ -119,6 +120,7 @@ class AdminShopDataGroupTest(unittest.TestCase): 'created_at': created_at, 'updated_at': created_at, 'finished_at': created_at, + 'latest_file_updated_at': latest_file_updated_at or created_at, 'file_job_id': None, 'file_status': 'SUCCESS', 'username': 'operator', @@ -133,9 +135,25 @@ class AdminShopDataGroupTest(unittest.TestCase): self.assertEqual(group['shop_name'], 'Shop A') self.assertEqual(group['group_name'], 'Group 1') - self.assertEqual([item['result_id'] for item in group['results']], [5, 4, 3]) + self.assertEqual([item['result_id'] for item in group['results']], [5]) self.assertEqual(group['results'][0]['result_file_url'], 'object-5') + def test_group_item_falls_back_to_daily_file_update_time(self): + row = self._result_row( + 21427, + 'Shop A', + '2026-08-06T15:20:43', + latest_file_updated_at=datetime(2026, 8, 7, 5, 21, 45), + ) + + group = admin_api._shop_data_crawl_group_item( + {'shop_name': 'Shop A', 'latest_created_at': None}, + {'shop a': [row]}, + {}, + ) + + self.assertEqual(group['latest_created_at'], '2026-08-07 05:21:45') + def test_list_paginates_groups_and_ignores_removed_user_status_filters(self): cursor = _FakeCursor( self.group_rows, @@ -160,10 +178,11 @@ class AdminShopDataGroupTest(unittest.TestCase): self.assertEqual(payload['total'], 2) self.assertEqual(payload['page'], 1) self.assertEqual(payload['items'][0]['shop_name'], 'Shop A') - self.assertEqual(len(payload['items'][0]['results']), 3) + self.assertEqual(payload['items'][0]['latest_created_at'], '2026-08-07 05:21:45') + self.assertEqual(len(payload['items'][0]['results']), 1) self.assertEqual( [item['result_id'] for item in payload['items'][0]['results']], - [5, 4, 3], + [5], ) self.assertEqual(payload['items'][1]['shop_name'], '未命名') self.assertEqual(len(payload['items'][1]['results']), 1) @@ -173,7 +192,15 @@ class AdminShopDataGroupTest(unittest.TestCase): self.assertNotIn('FAILED', params) self.assertTrue(any('GROUP BY TRIM(COALESCE(r.source_filename, ' in sql for sql, _ in cursor.calls)) self.assertTrue(any('ROW_NUMBER() OVER' in sql for sql, _ in cursor.calls)) - self.assertTrue(any('shop_row_number <= 3' in sql for sql, _ in cursor.calls)) + self.assertTrue(any('shop_row_number <= 1' in sql for sql, _ in cursor.calls)) + self.assertTrue(any( + 'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id' in sql + for sql, _ in cursor.calls + )) + self.assertTrue(any( + 'MAX(COALESCE(df.last_success_at, df.updated_at, t.finished_at, t.updated_at, t.created_at))' + in sql for sql, _ in cursor.calls + )) self.assertTrue(any('TRIM(COALESCE(sm.shop_name' in sql for sql, _ in cursor.calls)) self.assertTrue(any("TRIM(COALESCE(r.result_file_url, '')) <> ''" in sql for sql, _ in cursor.calls)) diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 0047d05a..ce01b3ba 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -1975,6 +1975,12 @@
+
+ + +

@@ -1992,6 +1998,12 @@ +
+ + +

@@ -2015,6 +2027,12 @@ +
+ + +
@@ -2034,6 +2052,7 @@ ID ASIN值 用户名 + 分组 创建时间 操作 @@ -2057,6 +2076,15 @@
+
+ +
+ + +
+

@@ -2068,6 +2096,12 @@ +
+ + +
@@ -2076,6 +2110,8 @@ + + @@ -2662,6 +2698,12 @@ +
+ + +

@@ -2675,6 +2717,7 @@

编辑不符合ASIN数据

+
@@ -2683,6 +2726,19 @@
+
+ + +
+
+ +
+ + +
+

@@ -3052,7 +3108,7 @@
- + diff --git a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue index c92b0e06..ce33ca70 100644 --- a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue @@ -161,7 +161,10 @@
- {{ statusText(item) }} + + {{ statusText(item) }} + （{{ taskFilterSummary(item) }}） +
@@ -190,7 +193,10 @@
- {{ statusText(item) }} + + {{ statusText(item) }} + （{{ taskFilterSummary(item) }}） +
@@ -565,6 +571,17 @@ function statusClass(item: CollectDataHistoryItem) { return 'pending' } +function taskFilterSummary(item: CollectDataHistoryItem) { + const status = normalizeTaskStatus(item) + if (status !== 'SUCCESS' && !item.success) return '' + + const dedupeFiltered = Number(item.dedupeFilteredCount) || 0 + const invalidFiltered = Number(item.invalidFilteredCount) || 0 + const brandRejected = Number(item.brandRejectedCount) || 0 + const finalRowCount = Number(item.finalRowCount ?? item.rowCount) || 0 + return `去重 ${dedupeFiltered} · 无效品牌 ${invalidFiltered} · 品牌拒绝 ${brandRejected} · 保留 ${finalRowCount}` +} + function taskProgressPercent(item: CollectDataHistoryItem) { const value = Number(item.progressPercent) if (Number.isFinite(value)) return Math.max(0, Math.min(100, Math.round(value))) @@ -794,6 +811,7 @@ onBeforeUnmount(() => { .status.failed { background: rgba(231, 76, 60, .18); color: #ff6b6b; } .status.running { background: rgba(52, 152, 219, .18); color: #3498db; } .status.pending { background: rgba(149, 165, 166, .18); color: #bdc3c7; } +.status-filter-summary { font-weight: 400; } .download { padding: 6px 10px; color: #d6ecff; background: rgba(52, 152, 219, .18); } .btn-delete { padding: 6px 10px; color: #ff8f8f; background: rgba(231, 76, 60, .12); } .task-progress { margin-top: 8px; max-width: 520px; }
ID ASIN 品牌分组来源 创建时间 操作