From 7c3c9e53a0102dcfd93e43fe7e7e510a04a80341 Mon Sep 17 00:00:00 2001 From: supernijia Date: Tue, 28 Jul 2026 13:40:19 +0800 Subject: [PATCH] =?UTF-8?q?=E8=8F=9C=E5=8D=95=E4=BF=AE=E6=94=B9=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend-java/docs/baota-start-commands.md | 7 +- .../config/ModuleCleanupProperties.java | 2 +- .../admin/controller/AdminUserController.java | 6 +- .../model/dto/AdminUserCreateRequest.java | 3 + .../model/dto/AdminUserUpdateRequest.java | 3 + .../admin/service/AdminUserService.java | 24 +- .../admin/support/AdminAuthSupport.java | 19 +- .../vo/AppearancePatentHistoryItemVo.java | 2 +- .../brand/model/dto/BrandSourceFileDto.java | 2 +- .../file/service/oss/OssStorageService.java | 99 + .../service/ImageVideoAsyncTaskService.java | 87 +- .../controller/PermissionMenuController.java | 173 +- ...ImageVideoDataPermissionUpdateRequest.java | 12 + .../dto/PermissionMenuCreateRequest.java | 5 + .../dto/PermissionMenuUpdateRequest.java | 5 + .../model/entity/PermissionMenuEntity.java | 3 + .../vo/ImageVideoDataPermissionUserVo.java | 11 + .../model/vo/PermissionMenuItemVo.java | 4 + .../permission/model/vo/UserColumnIdsVo.java | 4 + .../PermissionMenuSchemaInitializer.java | 75 +- .../service/PermissionMenuService.java | 580 +++++- .../dto/ProductRiskMatchShopsRequest.java | 2 +- .../vo/ProductRiskCountryPreferenceVo.java | 2 +- .../model/vo/ProductRiskMatchShopsVo.java | 4 +- .../publish/controller/PublishController.java | 4 +- .../model/dto/PublishResultFileDto.java | 23 +- .../publish/service/PublishTaskService.java | 395 +++- .../ShopDataCrawlTaskController.java | 308 +++ .../ShopDataCrawlCountryPrefMapper.java | 8 + .../ShopDataCrawlShopCandidateMapper.java | 10 + .../dto/ShopDataCrawlCountryResultDto.java | 20 + .../dto/ShopDataCrawlCreateTaskRequest.java | 35 + .../model/dto/ShopDataCrawlRowDto.java | 41 + .../dto/ShopDataCrawlShopPayloadDto.java | 44 + .../dto/ShopDataCrawlSubmitResultRequest.java | 19 + .../dto/ShopDataCrawlTaskBatchRequest.java | 21 + .../model/dto/ShopDataCrawlTaskItemDto.java | 44 + .../ShopDataCrawlCountryPrefEntity.java | 15 + .../ShopDataCrawlShopCandidateEntity.java | 20 + .../model/vo/ShopDataCrawlCreateTaskVo.java | 19 + .../model/vo/ShopDataCrawlHistoryVo.java | 16 + .../model/vo/ShopDataCrawlResultItemVo.java | 97 + .../model/vo/ShopDataCrawlTaskBatchVo.java | 19 + .../ShopDataCrawlExcelAssemblyService.java | 110 ++ .../service/ShopDataCrawlResolveService.java | 313 +++ .../ShopDataCrawlTaskCacheService.java | 226 +++ .../service/ShopDataCrawlTaskService.java | 1686 +++++++++++++++++ .../model/vo/SimilarAsinHistoryItemVo.java | 2 +- .../service/SimilarAsinTaskService.java | 12 +- .../util/SimilarAsinImageEmbedder.java | 25 +- .../task/service/TaskFileJobService.java | 9 +- .../task/service/TaskHeartbeatService.java | 10 + .../task/service/TaskResultFileJobWorker.java | 21 +- .../resources/application-local.example.yml | 5 +- .../src/main/resources/application.yml | 7 +- .../resources/db/V79__shop_data_crawl.sql | 29 + .../V80__permission_menu_parent_hierarchy.sql | 220 +++ .../db/V81__expand_publish_item_status.sql | 2 + .../templates/shop-data-crawl/文档格式.xlsx | Bin 0 -> 11754 bytes .../admin/service/AdminUserServiceTest.java | 99 + .../admin/support/AdminAuthSupportTest.java | 52 + .../service/oss/OssStorageServiceTest.java | 91 + .../ImageVideoAsyncTaskServiceTest.java | 151 +- .../PermissionMenuControllerTest.java | 251 +++ .../service/PermissionMenuServiceTest.java | 508 +++++ .../service/PublishTaskServiceTest.java | 533 +++++- ...ShopDataCrawlExcelAssemblyServiceTest.java | 56 + .../ShopDataCrawlResolveServiceTest.java | 39 + .../ShopDataCrawlTaskServiceChunkTest.java | 539 ++++++ .../ShopDataCrawlTaskServiceOwnerTest.java | 99 + .../service/SimilarAsinTaskServiceTest.java | 10 + .../util/SimilarAsinImageEmbedderTest.java | 30 +- .../service/TaskHeartbeatServiceTest.java | 27 + .../service/TaskResultFileJobWorkerTest.java | 61 +- backend/blueprints/admin_api.py | 1182 ++++++------ backend/static/admin.js | 556 +++--- backend/tests/test_dedupe_total_data_admin.py | 160 -- backend/tests/test_image_video_admin.py | 340 ---- backend/utils/db.py | 38 - backend/web_source/admin.html | 149 +- frontend-vue/shop-data-crawl.html | 12 + .../brand/components/BrandPublishTab.vue | 100 +- .../components/BrandShopDataCrawlTab.vue | 604 ++++++ .../brand/components/BrandSimilarAsinTab.vue | 13 +- .../pages/brand/components/BrandTopBar.vue | 11 +- .../src/pages/image-video/ImageVideoPage.vue | 33 +- frontend-vue/src/shared/api/java-modules.ts | 193 ++ frontend-vue/src/shared/api/permission.ts | 14 +- frontend-vue/src/shop-data-crawl-main.ts | 12 + frontend-vue/vite.config.ts | 1 + 90 files changed, 9388 insertions(+), 1545 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/ImageVideoDataPermissionUpdateRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/ImageVideoDataPermissionUserVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlCountryPrefMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlShopCandidateMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCountryResultDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCreateTaskRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlShopPayloadDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlSubmitResultRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskBatchRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskItemDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlCountryPrefEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlShopCandidateEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlCreateTaskVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlHistoryVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlResultItemVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlTaskBatchVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskCacheService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java create mode 100644 backend-java/src/main/resources/db/V79__shop_data_crawl.sql create mode 100644 backend-java/src/main/resources/db/V80__permission_menu_parent_hierarchy.sql create mode 100644 backend-java/src/main/resources/db/V81__expand_publish_item_status.sql create mode 100644 backend-java/src/main/resources/templates/shop-data-crawl/文档格式.xlsx create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupportTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuControllerTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveServiceTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java delete mode 100644 backend/tests/test_dedupe_total_data_admin.py delete mode 100644 backend/tests/test_image_video_admin.py create mode 100644 frontend-vue/shop-data-crawl.html create mode 100644 frontend-vue/src/pages/brand/components/BrandShopDataCrawlTab.vue create mode 100644 frontend-vue/src/shop-data-crawl-main.ts diff --git a/backend-java/docs/baota-start-commands.md b/backend-java/docs/baota-start-commands.md index 29fd6eee..8c88c105 100644 --- a/backend-java/docs/baota-start-commands.md +++ b/backend-java/docs/baota-start-commands.md @@ -8,6 +8,7 @@ Requirements: - Put JVM options before `-jar` - Pass `AIIMAGE_INSTANCE_ID` and Redis address from the startup command - Result file job MQ is expected to stay enabled in packaged `server` deployments +- Use `https://oss.aishufu.top` for the MinIO SDK and public object URLs; `https://minio.aishufu.top` is the management console only 1Panel server 121: @@ -22,6 +23,8 @@ AIIMAGE_REDIS_PASSWORD=B6COTcY094TYe545 AIIMAGE_REDIS_DATABASE=0 AIIMAGE_SERVER_PORT=18080 AIIMAGE_STORAGE_LOCAL_TEMP_DIR=/app/data/tmp +AIIMAGE_OSS_ENDPOINT=https://oss.aishufu.top +AIIMAGE_OSS_PUBLIC_ENDPOINT=https://oss.aishufu.top AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876 AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://121.196.149.225:9000 @@ -36,13 +39,13 @@ java -Xmx2048M -Xms2048M -jar /app/aiimage-backend-0.0.1-SNAPSHOT.jar Baota server 111: ```bash -/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-111 --spring.data.redis.host=47.111.163.154 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp +/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-111 --spring.data.redis.host=47.111.163.154 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp --aiimage.oss.endpoint=https://oss.aishufu.top --aiimage.oss.public-endpoint=https://oss.aishufu.top ``` Optional Baota server 121 command: ```bash -/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-121 --spring.data.redis.host=192.168.0.172 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp +/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-121 --spring.data.redis.host=192.168.0.172 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp --aiimage.oss.endpoint=https://oss.aishufu.top --aiimage.oss.public-endpoint=https://oss.aishufu.top ``` Successful startup should show a line similar to: diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java index 127b514f..3a40eda5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java @@ -12,5 +12,5 @@ public class ModuleCleanupProperties { private boolean enabled = true; private String cron = "0 0 0 * * *"; private long retentionDays = 7; - private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA")); + private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA")); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java index 5060d497..e23292c2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/controller/AdminUserController.java @@ -48,11 +48,11 @@ public class AdminUserController { @PostMapping("/user") @Operation(summary = "创建用户") - public ApiResponse createUser(HttpServletRequest request, + public ApiResponse createUser(HttpServletRequest request, @RequestBody AdminUserCreateRequest body) { AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request); - adminUserService.createUser(currentUser, body); - return ApiResponse.success("用户创建成功", null); + Long userId = adminUserService.createUser(currentUser, body); + return ApiResponse.success("用户创建成功", userId); } @PutMapping("/user/{uid}") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserCreateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserCreateRequest.java index e4ff19a5..6e49c4e0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserCreateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserCreateRequest.java @@ -2,10 +2,13 @@ package com.nanri.aiimage.modules.admin.model.dto; import lombok.Data; +import java.util.List; + @Data public class AdminUserCreateRequest { private String username; private String password; private String role; private Long createdById; + private List columnIds; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserUpdateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserUpdateRequest.java index 7db1e222..46ea1de1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserUpdateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/model/dto/AdminUserUpdateRequest.java @@ -2,8 +2,11 @@ package com.nanri.aiimage.modules.admin.model.dto; import lombok.Data; +import java.util.List; + @Data public class AdminUserUpdateRequest { private String password; private String role; + private List columnIds; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java index e775f1a3..c10d3795 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/service/AdminUserService.java @@ -11,10 +11,13 @@ import com.nanri.aiimage.modules.admin.model.vo.AdminUserListVo; import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; import com.nanri.aiimage.modules.auth.util.WerkzeugPasswordEncoder; import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; +import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest; import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; import lombok.RequiredArgsConstructor; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -35,6 +38,7 @@ public class AdminUserService { private final AdminUserMapper adminUserMapper; private final WerkzeugPasswordEncoder passwordEncoder; private final AdminAuthSupport adminAuthSupport; + private final PermissionMenuService permissionMenuService; public AdminUserListVo listUsers(AdminUserEntity currentUser, Integer page, Integer pageSize, String username, Long createdById) { @@ -97,7 +101,8 @@ public class AdminUserService { return vo; } - public void createUser(AdminUserEntity currentUser, AdminUserCreateRequest request) { + @Transactional + public Long createUser(AdminUserEntity currentUser, AdminUserCreateRequest request) { String role = adminAuthSupport.currentRole(currentUser); if (role == null) { throw new BusinessException(403, "需要管理员权限"); @@ -150,8 +155,11 @@ public class AdminUserService { } catch (DuplicateKeyException e) { throw new BusinessException("用户名已存在"); } + replaceDirectPermissions(currentUser, entity.getId(), request.getColumnIds()); + return entity.getId(); } + @Transactional public void updateUser(AdminUserEntity currentUser, Long uid, AdminUserUpdateRequest request) { String role = adminAuthSupport.currentRole(currentUser); if (role == null) { @@ -160,7 +168,9 @@ public class AdminUserService { String password = request.getPassword(); String wantRole = request.getRole() == null ? null : request.getRole().trim(); - if ((password == null || password.isEmpty()) && (wantRole == null || wantRole.isEmpty())) { + if ((password == null || password.isEmpty()) + && (wantRole == null || wantRole.isEmpty()) + && request.getColumnIds() == null) { throw new BusinessException("请提供要修改的内容"); } @@ -206,6 +216,16 @@ public class AdminUserService { if (dirty) { adminUserMapper.update(null, update); } + if (request.getColumnIds() != null) { + replaceDirectPermissions(currentUser, uid, request.getColumnIds()); + } + } + + private void replaceDirectPermissions(AdminUserEntity operator, Long userId, List columnIds) { + UserColumnPermissionUpdateRequest permissionRequest = new UserColumnPermissionUpdateRequest(); + permissionRequest.setColumnIds(columnIds == null ? List.of() : columnIds); + permissionMenuService.updateUserColumnPermissions( + operator, userId, permissionRequest, PermissionMenuService.MENU_TYPE_APP); } public void deleteUser(AdminUserEntity currentUser, Long uid) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java index e5905c25..bc03744d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupport.java @@ -1,6 +1,5 @@ package com.nanri.aiimage.modules.admin.support; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.auth.service.JwtService; import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; @@ -65,27 +64,15 @@ public class AdminAuthSupport { return "super_admin"; } if ("admin".equals(storedRole)) { - return resolveAdminRole(user); + return "admin"; } boolean isAdminFlag = user.getIsAdmin() != null && user.getIsAdmin() == 1; - if (isAdminFlag) { - return resolveAdminRole(user); + if (storedRole.isEmpty() && isAdminFlag) { + return user.getCreatedById() == null ? "super_admin" : "admin"; } return null; } - /** 数据库里 role='admin' 中 id 最小者视作超级管理员（与 Python 行为一致）。 */ - private String resolveAdminRole(AdminUserEntity user) { - AdminUserEntity superAdmin = adminUserMapper.selectOne(new LambdaQueryWrapper() - .eq(AdminUserEntity::getRole, "admin") - .orderByAsc(AdminUserEntity::getId) - .last("LIMIT 1")); - if (superAdmin != null && superAdmin.getId().equals(user.getId())) { - return "super_admin"; - } - return "admin"; - } - private String resolveToken(HttpServletRequest request) { String authHeader = request.getHeader(HttpHeaders.AUTHORIZATION); if (authHeader != null && authHeader.startsWith("Bearer ")) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java index c5f497ce..adc6395a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java @@ -14,7 +14,7 @@ public class AppearancePatentHistoryItemVo { private String sourceFilename; @Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx") private String resultFilename; - @Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/appearance_patent/xxx/17-result.xlsx") + @Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "https://oss.aishufu.top/nanri-ai-images/result/appearance_patent/xxx/17-result.xlsx") private String downloadUrl; private Long fileJobId; private String fileStatus; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java index f313423e..b946d86b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java @@ -9,7 +9,7 @@ import lombok.Data; public class BrandSourceFileDto { @NotBlank(message = "fileUrl 不能为空") - @Schema(description = "对象存储文件下载链接，Java 将先下载文件再读取内容。", example = "http://47.110.241.161:9000/nanri-ai-images/source/brand.xlsx") + @Schema(description = "对象存储文件下载链接，Java 将先下载文件再读取内容。", example = "https://oss.aishufu.top/nanri-ai-images/source/brand.xlsx") private String fileUrl; @Schema(description = "原始文件名，用于生成任务描述和结果文件名。", example = "品牌样例.xlsx") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java index b9ba7d51..04b34dcf 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -27,6 +27,7 @@ public class OssStorageService { private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO"; private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/"; + private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000"; private final OssProperties ossProperties; private final MinioClient minioClient; @@ -164,6 +165,52 @@ public class OssStorageService { return location == null ? null : getPublicUrl(location.objectKey(), location.bucket()); } + /** Rewrites managed MinIO URLs to the current public endpoint without touching external URLs. */ + public String normalizeManagedPublicUrl(String value) { + if (value == null || value.isBlank()) { + return value; + } + + URI source; + try { + source = new URI(value.trim()); + } catch (Exception ignored) { + return value; + } + if (!isHttpUrl(source) || source.getHost() == null || source.getUserInfo() != null) { + return value; + } + + String objectPath; + try { + objectPath = trimLeadingSlash(decodePath(source.getRawPath())); + } catch (Exception ignored) { + return value; + } + if (objectPath.isBlank() || hasDotPathSegment(objectPath)) { + return value; + } + + List endpoints = managedEndpoints(); + for (URI endpoint : endpoints) { + for (String bucket : configuredBuckets()) { + if (matchesVirtualHostEndpoint(source, endpoint, bucket)) { + return getPublicUrl(objectPath, bucket); + } + } + } + if (endpoints.stream().noneMatch(endpoint -> matchesEndpoint(source, endpoint))) { + return value; + } + + String bucket = bucketPrefix(objectPath); + if (bucket == null) { + return value; + } + String objectKey = trimLeadingSlash(objectPath.substring(bucket.length())); + return objectKey.isBlank() ? value : getPublicUrl(objectKey, bucket); + } + public String resolveObjectKey(String value) { StorageLocation location = resolveStorageLocation(value); return location == null ? value : location.objectKey(); @@ -271,6 +318,58 @@ public class OssStorageService { .toList(); } + private List managedEndpoints() { + return Stream.of(ossProperties.getEndpoint(), ossProperties.getPublicEndpoint(), LEGACY_MINIO_ENDPOINT) + .filter(Objects::nonNull) + .map(String::trim) + .filter(endpoint -> !endpoint.isBlank()) + .map(this::parseEndpoint) + .filter(Objects::nonNull) + .distinct() + .toList(); + } + + private URI parseEndpoint(String value) { + try { + URI endpoint = new URI(withScheme(value)); + return endpoint.getHost() == null ? null : endpoint; + } catch (Exception ignored) { + return null; + } + } + + private boolean matchesEndpoint(URI source, URI endpoint) { + return sameSchemeAndPort(source, endpoint) + && source.getHost().equalsIgnoreCase(endpoint.getHost()); + } + + private boolean matchesVirtualHostEndpoint(URI source, URI endpoint, String bucket) { + String expectedHost = bucket + "." + endpoint.getHost(); + return sameSchemeAndPort(source, endpoint) + && source.getHost().equalsIgnoreCase(expectedHost); + } + + private boolean sameSchemeAndPort(URI left, URI right) { + return left.getScheme().equalsIgnoreCase(right.getScheme()) + && effectivePort(left) == effectivePort(right); + } + + private int effectivePort(URI uri) { + if (uri.getPort() >= 0) { + return uri.getPort(); + } + return "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80; + } + + private boolean isHttpUrl(URI uri) { + return "http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme()); + } + + private boolean hasDotPathSegment(String path) { + return Stream.of(path.split("/", -1)) + .anyMatch(segment -> ".".equals(segment) || "..".equals(segment)); + } + private MinioClient buildClient() { return minioClient; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java index 817299ed..f33ae5d3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java @@ -49,6 +49,15 @@ public class ImageVideoAsyncTaskService { private static final Set COZE_PRIMARY_STATUS_FIELDS = Set.of( "execute_status", "executeStatus", "workflow_status", "workflowStatus" ); + private static final Set COZE_BUSINESS_ERROR_FIELDS = Set.of( + "error", "error_message", "errorMessage" + ); + private static final Set COZE_BUSINESS_STATUS_CODE_FIELDS = Set.of( + "status_code", "statusCode" + ); + private static final Set COZE_BUSINESS_STATUS_MESSAGE_FIELDS = Set.of( + "status_msg", "statusMessage" + ); private final ImageVideoAsyncTaskMapper taskMapper; private final ImageVideoCozeService cozeService; @@ -233,7 +242,10 @@ public class ImageVideoAsyncTaskService { if (FAILED_STATUSES.contains(cozeStatus)) { task.setCozeStatus(cozeStatus); task.setResultJson(writeJson(result)); - failTask(task, new BusinessException("Coze workflow finished with status " + cozeStatus)); + String businessError = resolveCozeBusinessError(result); + failTask(task, new BusinessException(businessError.isBlank() + ? "Coze workflow finished with status " + cozeStatus + : businessError)); } else { completeTask(task, transformResult(type, result), cozeStatus); } @@ -315,6 +327,19 @@ public class ImageVideoAsyncTaskService { } private void completeTask(ImageVideoAsyncTaskEntity task, Object result, String cozeStatus) { + String businessError = resolveCozeBusinessError(result); + if (!businessError.isBlank()) { + task.setCozeStatus(normalizeStatus(cozeStatus)); + task.setResultJson(writeJson(result)); + failTask(task, new BusinessException(businessError)); + return; + } + if (requiresVideoResult(task) && !hasFinalVideoUrl(result)) { + task.setCozeStatus(normalizeStatus(cozeStatus)); + task.setResultJson(writeJson(result)); + failTask(task, new BusinessException("Coze workflow succeeded but video_url is empty")); + return; + } task.setStatus(TaskStatus.SUCCESS.name()); task.setResultJson(writeJson(result)); archiveService.enrichCompletedTask(task, result); @@ -511,6 +536,60 @@ public class ImageVideoAsyncTaskService { return value; } + private String resolveCozeBusinessError(Object result) { + String error = findText(result, COZE_BUSINESS_ERROR_FIELDS); + if (!error.isBlank()) { + return error; + } + + String statusCode = findText(result, COZE_BUSINESS_STATUS_CODE_FIELDS); + if (statusCode.isBlank() || "0".equals(statusCode)) { + return ""; + } + String statusMessage = findText(result, COZE_BUSINESS_STATUS_MESSAGE_FIELDS); + return statusMessage.isBlank() + ? "Coze workflow returned business status " + statusCode + : statusMessage + " (status_code: " + statusCode + ")"; + } + + private boolean hasFinalVideoUrl(Object result) { + Object data = parsePossiblyJson(childValue(result, "data")); + Object scope = data == null ? parsePossiblyJson(result) : data; + if (scope == null) { + return false; + } + + Collection records = scope instanceof Collection collection ? collection : List.of(scope); + boolean hasRunOutput = false; + for (Object record : records) { + Object output = childValue(parsePossiblyJson(record), "output"); + if (output == null) { + continue; + } + hasRunOutput = true; + if (hasVideoUrlInOutput(output)) { + return true; + } + } + return !hasRunOutput && hasVideoUrlInOutput(scope); + } + + private boolean hasVideoUrlInOutput(Object output) { + Object envelope = parsePossiblyJson(output); + Object finalOutput = childValue(envelope, "Output"); + if (finalOutput != null) { + return hasRecognizedVideoUrl(parsePossiblyJson(finalOutput)); + } + if (childValue(envelope, "node_status") != null) { + return false; + } + return hasRecognizedVideoUrl(envelope); + } + + private boolean hasRecognizedVideoUrl(Object payload) { + return !ImageVideoArchiveService.findUrls(payload, Set.of("videourl")).isEmpty(); + } + private String normalizeStatus(String value) { return value == null ? "" : value.trim().toUpperCase(); } @@ -529,6 +608,12 @@ public class ImageVideoAsyncTaskService { && task.getSubmittedAt().plusHours(1).isBefore(LocalDateTime.now()); } + private boolean requiresVideoResult(ImageVideoAsyncTaskEntity task) { + String taskType = task == null ? "" : task.getTaskType(); + return TaskType.IMAGE_VIDEO_WORKFLOW.name().equals(taskType) + || TaskType.WORKFLOW_RESULT.name().equals(taskType); + } + private void ensureTaskOwnedByCurrentInstance(ImageVideoAsyncTaskEntity task, String operation) { String owner = task == null ? null : task.getOwnerInstanceId(); if (owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId())) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java index 6db3afb8..e26cc7dc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java @@ -1,17 +1,23 @@ package com.nanri.aiimage.modules.permission.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest; import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest; +import com.nanri.aiimage.modules.permission.model.dto.ImageVideoDataPermissionUpdateRequest; import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo; import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo; +import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo; import com.nanri.aiimage.modules.permission.service.PermissionMenuService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; +import jakarta.servlet.http.HttpServletRequest; import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -22,6 +28,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; @RestController @@ -30,54 +39,188 @@ import java.util.List; @Tag(name = "菜单权限管理", description = "统一维护软件端与后台管理端的菜单权限配置") public class PermissionMenuController { + @Value("${aiimage.security.internal-token:}") + private String internalToken; + + @Value("${aiimage.security.internal-token-file:}") + private String internalTokenFile; + + private final AdminAuthSupport adminAuthSupport; private final PermissionMenuService permissionMenuService; @GetMapping("/permission-menus") @Operation(summary = "查询菜单权限列表") - public ApiResponse> listMenus( + public ApiResponse> listMenus(HttpServletRequest request, @Parameter(description = "菜单类型: app/admin") @RequestParam(required = false) String menuType) { + requireAdmin(request); return ApiResponse.success(permissionMenuService.list(menuType)); } @PostMapping("/permission-menus") @Operation(summary = "新增菜单权限项") - public ApiResponse createMenu(@Valid @RequestBody PermissionMenuCreateRequest request) { - return ApiResponse.success("创建成功", permissionMenuService.create(request)); + public ApiResponse createMenu(HttpServletRequest httpRequest, + @Valid @RequestBody PermissionMenuCreateRequest request) { + return ApiResponse.success("创建成功", permissionMenuService.create( + requireAdmin(httpRequest), request)); } @PutMapping("/permission-menus/{id}") @Operation(summary = "编辑菜单权限项") - public ApiResponse updateMenu(@PathVariable Long id, + public ApiResponse updateMenu(HttpServletRequest httpRequest, + @PathVariable Long id, @Valid @RequestBody PermissionMenuUpdateRequest request) { - return ApiResponse.success("更新成功", permissionMenuService.update(id, request)); + return ApiResponse.success("更新成功", permissionMenuService.update( + requireAdmin(httpRequest), id, request)); } @DeleteMapping("/permission-menus/{id}") @Operation(summary = "删除菜单权限项") - public ApiResponse deleteMenu(@PathVariable Long id) { - permissionMenuService.delete(id); + public ApiResponse deleteMenu(HttpServletRequest httpRequest, @PathVariable Long id) { + permissionMenuService.delete(requireAdmin(httpRequest), id); return ApiResponse.success("删除成功", null); } @GetMapping("/permission-users/{userId}/columns") - @Operation(summary = "查询用户菜单权限 ID 列表") - public ApiResponse getUserColumnIds(@PathVariable Long userId, + @Operation(summary = "查询用户直接授权菜单 ID 列表", + description = "columnIds 仅表示数据库保存的直接授权，所有角色都不包含递归或虚拟权限") + public ApiResponse getUserColumnIds(HttpServletRequest httpRequest, + @PathVariable Long userId, @RequestParam(required = false) String menuType) { - return ApiResponse.success(permissionMenuService.getUserColumnIds(userId, menuType)); + return ApiResponse.success(permissionMenuService.getUserColumnIds( + requireAdmin(httpRequest), userId, menuType)); } @PutMapping("/permission-users/{userId}/columns") @Operation(summary = "更新用户菜单权限") - public ApiResponse updateUserColumnIds(@PathVariable Long userId, + public ApiResponse updateUserColumnIds(HttpServletRequest httpRequest, + @PathVariable Long userId, + @RequestParam(required = false) String menuType, @RequestBody(required = false) UserColumnPermissionUpdateRequest request) { - permissionMenuService.updateUserColumnPermissions(userId, request); + permissionMenuService.updateUserColumnPermissions( + requireAdmin(httpRequest), userId, request, menuType); return ApiResponse.success("保存成功", null); } @GetMapping("/permission-users/{userId}/column-permissions") - @Operation(summary = "查询用户菜单权限详情") - public ApiResponse> getUserColumnPermissions(@PathVariable Long userId, + @Operation(summary = "查询用户有效菜单权限详情", + description = "返回直接授权及其递归展开的全部子菜单；不会改变数据库中的直接授权") + public ApiResponse> getUserColumnPermissions(HttpServletRequest httpRequest, + @PathVariable Long userId, @RequestParam(required = false) String menuType) { - return ApiResponse.success(permissionMenuService.getUserColumnPermissions(userId, menuType)); + return ApiResponse.success(permissionMenuService.getUserColumnPermissions( + requireUser(httpRequest), userId, menuType)); + } + + @GetMapping("/image-video-task-permissions") + @Operation(summary = "查询视频任务数据权限用户") + public ApiResponse> listImageVideoDataPermissionUsers( + HttpServletRequest request) { + return ApiResponse.success(permissionMenuService.listImageVideoDataPermissionUsers(requireAdmin(request))); + } + + @PutMapping("/image-video-task-permissions") + @Operation(summary = "更新视频任务数据权限用户") + public ApiResponse updateImageVideoDataPermissionUsers( + HttpServletRequest request, + @RequestBody(required = false) ImageVideoDataPermissionUpdateRequest body) { + int grantedCount = permissionMenuService.updateImageVideoDataPermissionUsers( + requireAdmin(request), body == null ? List.of() : body.getUserIds()); + return ApiResponse.success("视频任务权限已更新", grantedCount); + } + + private AdminUserEntity requireAdmin(HttpServletRequest request) { + try { + return adminAuthSupport.requireAdmin(request); + } catch (com.nanri.aiimage.common.exception.BusinessException authFailure) { + AdminUserEntity internalOperator = resolveInternalOperator(request, true); + if (internalOperator != null) { + return internalOperator; + } + throw authFailure; + } + } + + private AdminUserEntity requireUser(HttpServletRequest request) { + try { + return adminAuthSupport.requireUser(request); + } catch (com.nanri.aiimage.common.exception.BusinessException authFailure) { + AdminUserEntity internalOperator = resolveInternalOperator(request, false); + if (internalOperator != null) { + return internalOperator; + } + throw authFailure; + } + } + + /** + * Flask's legacy session cannot mint a Java JWT. It may identify its + * already-authenticated user only with a shared secret supplied explicitly + * or through the per-user local token file. Admin routes still revalidate + * the database role, and the superAdmin query flag is deliberately ignored. + */ + private AdminUserEntity resolveInternalOperator(HttpServletRequest request, boolean requireAdmin) { + String suppliedToken = request.getHeader("X-Internal-Token"); + if (!isTrustedInternalRequest(request, suppliedToken)) { + return null; + } + String rawOperatorId = request.getParameter("operatorId"); + if (rawOperatorId == null || rawOperatorId.isBlank()) { + rawOperatorId = request.getParameter("operator_id"); + } + if (rawOperatorId == null || rawOperatorId.isBlank()) { + return null; + } + try { + Long operatorId = Long.parseLong(rawOperatorId.trim()); + return requireAdmin + ? permissionMenuService.requireAdminOperator(operatorId) + : permissionMenuService.requireUserOperator(operatorId); + } catch (NumberFormatException ex) { + return null; + } + } + + private boolean isTrustedInternalRequest(HttpServletRequest request, String suppliedToken) { + String expectedToken = resolveExpectedInternalToken(); + if (expectedToken.isBlank() || suppliedToken == null || suppliedToken.isBlank()) { + return false; + } + return expectedToken.equals(suppliedToken); + } + + private String resolveExpectedInternalToken() { + if (internalToken != null && !internalToken.isBlank()) { + return internalToken.trim(); + } + Path path = resolveInternalTokenFile(); + if (path == null || !Files.isRegularFile(path)) { + return ""; + } + try { + return Files.readString(path, StandardCharsets.UTF_8).trim(); + } catch (Exception ignored) { + return ""; + } + } + + private Path resolveInternalTokenFile() { + String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim(); + if (!configuredPath.isEmpty()) { + if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) { + String userHome = System.getProperty("user.home", "").trim(); + if (userHome.isEmpty()) { + return null; + } + configuredPath = configuredPath.length() == 1 + ? userHome + : Path.of(userHome, configuredPath.substring(2)).toString(); + } + Path configuredTokenPath = Path.of(configuredPath); + return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null; + } + String userHome = System.getProperty("user.home", "").trim(); + return userHome.isEmpty() + ? null + : Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize(); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/ImageVideoDataPermissionUpdateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/ImageVideoDataPermissionUpdateRequest.java new file mode 100644 index 00000000..10d1d35d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/ImageVideoDataPermissionUpdateRequest.java @@ -0,0 +1,12 @@ +package com.nanri.aiimage.modules.permission.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import lombok.Data; + +import java.util.List; + +@Data +public class ImageVideoDataPermissionUpdateRequest { + @JsonAlias("user_ids") + private List userIds; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuCreateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuCreateRequest.java index 4cbdfe70..fea96350 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuCreateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuCreateRequest.java @@ -1,5 +1,6 @@ package com.nanri.aiimage.modules.permission.model.dto; +import com.fasterxml.jackson.annotation.JsonAlias; import jakarta.validation.constraints.NotBlank; import lombok.Data; @@ -12,6 +13,10 @@ public class PermissionMenuCreateRequest { @NotBlank(message = "菜单标识不能为空") private String columnKey; + /** 直接父菜单；null 表示根菜单。 */ + @JsonAlias("parent_id") + private Long parentId; + @NotBlank(message = "菜单类型不能为空") private String menuType; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuUpdateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuUpdateRequest.java index e2b69278..89782930 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuUpdateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/dto/PermissionMenuUpdateRequest.java @@ -1,5 +1,6 @@ package com.nanri.aiimage.modules.permission.model.dto; +import com.fasterxml.jackson.annotation.JsonAlias; import jakarta.validation.constraints.NotBlank; import lombok.Data; @@ -12,6 +13,10 @@ public class PermissionMenuUpdateRequest { @NotBlank(message = "菜单标识不能为空") private String columnKey; + /** 直接父菜单；null 表示根菜单。 */ + @JsonAlias("parent_id") + private Long parentId; + @NotBlank(message = "菜单类型不能为空") private String menuType; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/entity/PermissionMenuEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/entity/PermissionMenuEntity.java index cbebbf82..77377046 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/entity/PermissionMenuEntity.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/entity/PermissionMenuEntity.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.permission.model.entity; import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; @@ -15,6 +16,8 @@ public class PermissionMenuEntity { private Long id; private String name; private String columnKey; + @TableField("parent_id") + private Long parentId; private String menuType; private String routePath; private Integer sortOrder; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/ImageVideoDataPermissionUserVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/ImageVideoDataPermissionUserVo.java new file mode 100644 index 00000000..46f669e2 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/ImageVideoDataPermissionUserVo.java @@ -0,0 +1,11 @@ +package com.nanri.aiimage.modules.permission.model.vo; + +import lombok.Data; + +@Data +public class ImageVideoDataPermissionUserVo { + private Long id; + private String username; + private String role; + private boolean granted; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/PermissionMenuItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/PermissionMenuItemVo.java index 46c6d495..8991471c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/PermissionMenuItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/PermissionMenuItemVo.java @@ -12,6 +12,10 @@ public class PermissionMenuItemVo { private String name; @JsonProperty("column_key") private String columnKey; + @JsonProperty("parent_id") + private Long parentId; + @JsonProperty("root_column_key") + private String rootColumnKey; @JsonProperty("menu_type") private String menuType; @JsonProperty("route_path") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/UserColumnIdsVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/UserColumnIdsVo.java index 98798269..29024648 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/UserColumnIdsVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/UserColumnIdsVo.java @@ -7,5 +7,9 @@ import java.util.List; @Data public class UserColumnIdsVo { + /** + * 仅返回数据库中保存的直接授权 ID；不包含递归子菜单， + * 也不会将超级管理员的虚拟全量权限写入响应。 + */ private List columnIds; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java index a84ac7f6..3d6ecda3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java @@ -22,8 +22,29 @@ public class PermissionMenuSchemaInitializer { private static final List DEFAULT_APP_MENUS = List.of( new DefaultAppMenu("前端工具", "brand_front_tools", "brand-front-tools", 110), new DefaultAppMenu("运营工具", "brand_operation_tools", "brand-operation-tools", 120), - new DefaultAppMenu("后勤工具", "brand_logistics_tools", "brand-logistics-tools", 130), - new DefaultAppMenu("取款", "withdraw", "withdraw", 140) + new DefaultAppMenu("后勤工具", "brand_logistics_tools", "brand-logistics-tools", 130) + ); + + private static final List DEFAULT_APP_CHILD_MENUS = List.of( + new DefaultAppChildMenu("采集数据", "collect-data", "collect-data", 111, "brand_front_tools"), + new DefaultAppChildMenu("品牌检测", "brand", "brand", 112, "brand_front_tools"), + new DefaultAppChildMenu("外观专利检测", "appearance-patent", "appearance-patent", 113, "brand_front_tools"), + new DefaultAppChildMenu("货源查询", "similar-asin", "similar-asin", 114, "brand_front_tools"), + new DefaultAppChildMenu("数据去重", "dedupe", "dedupe", 115, "brand_front_tools"), + new DefaultAppChildMenu("数据拆分", "split", "split", 116, "brand_front_tools"), + new DefaultAppChildMenu("格式转换", "convert", "convert", 117, "brand_front_tools"), + new DefaultAppChildMenu("上架", "publish", "publish", 121, "brand_operation_tools"), + new DefaultAppChildMenu("删除ASIN", "delete-brand", "delete-brand", 122, "brand_operation_tools"), + new DefaultAppChildMenu("商品风险解决", "product-risk", "product-risk", 123, "brand_operation_tools"), + new DefaultAppChildMenu("定时匹配", "shop-match", "shop-match", 124, "brand_operation_tools"), + new DefaultAppChildMenu("跟价", "pricing", "price-track", 125, "brand_operation_tools"), + new DefaultAppChildMenu("巡店删除", "patrol-delete", "patrol-delete", 126, "brand_operation_tools"), + new DefaultAppChildMenu("查询ASIN", "query-asin", "query-asin", 127, "brand_operation_tools"), + // Keep this key for compatibility with the existing Java/Flask module. + new DefaultAppChildMenu("店铺数据抓取", "shop_data_crawl", "shop-data-crawl", 128, "brand_operation_tools"), + new DefaultAppChildMenu("取款", "withdraw", "withdraw", 129, "brand_operation_tools"), + new DefaultAppChildMenu("采购", "purchase", "purchase", 131, "brand_logistics_tools"), + new DefaultAppChildMenu("ERP", "erp", "erp", 132, "brand_logistics_tools") ); private static final List DEFAULT_ADMIN_MENUS = List.of( @@ -52,11 +73,13 @@ public class PermissionMenuSchemaInitializer { id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(128) NOT NULL COMMENT '菜单标题', column_key VARCHAR(64) NOT NULL COMMENT '菜单唯一标识', + parent_id INT NULL COMMENT '直接父菜单 ID', created_at DATETIME DEFAULT CURRENT_TIMESTAMP, UNIQUE KEY uk_column_key (column_key) ) """); executeQuietly("ALTER TABLE columns ADD COLUMN menu_type VARCHAR(20) NOT NULL DEFAULT 'app' COMMENT '菜单类型: app/admin' AFTER column_key"); + executeQuietly("ALTER TABLE columns ADD COLUMN parent_id INT NULL COMMENT '直接父菜单 ID' AFTER id"); executeQuietly("ALTER TABLE columns ADD COLUMN route_path VARCHAR(255) NOT NULL DEFAULT '' COMMENT '菜单路由或页面标识' AFTER menu_type"); executeQuietly("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path"); executeQuietly("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''"); @@ -64,6 +87,8 @@ public class PermissionMenuSchemaInitializer { executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)"); ensureDefaultAdminMenus(); ensureDefaultAppMenus(); + ensureDefaultAppChildMenus(); + ensureInternalDataPermissions(); executeQuietly(""" CREATE TABLE IF NOT EXISTS user_column_permission ( user_id INT NOT NULL, @@ -86,7 +111,7 @@ public class PermissionMenuSchemaInitializer { """.formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey())); executeQuietly(""" UPDATE columns - SET sort_order = %d + SET sort_order = %d, parent_id = NULL WHERE column_key = '%s' AND (sort_order IS NULL OR sort_order = 0) """.formatted(menu.sortOrder(), menu.columnKey())); } @@ -103,12 +128,50 @@ public class PermissionMenuSchemaInitializer { """.formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey())); executeQuietly(""" UPDATE columns - SET sort_order = %d, route_path = '%s', menu_type = 'app' + SET sort_order = %d, route_path = '%s', menu_type = 'app', parent_id = NULL WHERE column_key = '%s' """.formatted(menu.sortOrder(), menu.routePath(), menu.columnKey())); } } + private void ensureDefaultAppChildMenus() { + // Older migrations used underscore keys for a few app pages. Normalize + // them once so the frontend item key and permission key stay aligned. + executeQuietly("UPDATE columns SET column_key = 'collect-data' WHERE column_key = 'collect_data'"); + executeQuietly("UPDATE columns SET column_key = 'appearance-patent' WHERE column_key = 'appearance_patent'"); + executeQuietly("UPDATE columns SET column_key = 'similar-asin' WHERE column_key = 'similar_asin'"); + + for (DefaultAppChildMenu menu : DEFAULT_APP_CHILD_MENUS) { + executeQuietly(""" + INSERT INTO columns (name, column_key, menu_type, route_path, sort_order, parent_id) + SELECT '%s', '%s', 'app', '%s', %d, parent.id + FROM columns parent + WHERE parent.column_key = '%s' + AND NOT EXISTS ( + SELECT 1 FROM columns existing WHERE existing.column_key = '%s' + ) + """.formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), + menu.parentKey(), menu.columnKey())); + executeQuietly(""" + UPDATE columns child + JOIN columns parent ON parent.column_key = '%s' + SET child.name = '%s', child.menu_type = 'app', child.route_path = '%s', + child.sort_order = %d, child.parent_id = parent.id + WHERE child.column_key = '%s' + """.formatted(menu.parentKey(), menu.name(), menu.routePath(), menu.sortOrder(), menu.columnKey())); + } + } + + private void ensureInternalDataPermissions() { + executeQuietly(""" + INSERT INTO columns (name, column_key, menu_type, route_path, sort_order) + SELECT '视频任务数据查看', 'admin_image_video_task_data', 'internal', 'image-video-task-data', 0 + WHERE NOT EXISTS ( + SELECT 1 FROM columns WHERE column_key = 'admin_image_video_task_data' + ) + """); + } + private void executeQuietly(String sql) { try { jdbcTemplate.execute(sql); @@ -122,4 +185,8 @@ public class PermissionMenuSchemaInitializer { private record DefaultAppMenu(String name, String columnKey, String routePath, int sortOrder) { } + + private record DefaultAppChildMenu(String name, String columnKey, String routePath, int sortOrder, + String parentKey) { + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java index ba5f493e..5c25facd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java @@ -12,22 +12,34 @@ import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdate import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity; import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity; +import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo; import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo; import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +/** + * Menu hierarchy and user-column permissions. + * + *

The assignment table stores direct grants only. A parent grant is + * expanded to all descendants when effective permissions are read or checked; + * descendants are never inserted as a side effect of an update.

+ */ @Service @RequiredArgsConstructor public class PermissionMenuService { @@ -40,28 +52,37 @@ public class PermissionMenuService { private final UserColumnPermissionMapper userColumnPermissionMapper; private final AdminUserMapper adminUserMapper; + /** Returns the flat menu catalog, including each item's direct parent ID. */ public List list(String menuType) { - LambdaQueryWrapper query = new LambdaQueryWrapper() - .eq(isValidMenuType(menuType), PermissionMenuEntity::getMenuType, normalizeMenuType(menuType)) - .orderByAsc(PermissionMenuEntity::getSortOrder) - .orderByAsc(PermissionMenuEntity::getId); - return permissionMenuMapper.selectList(query).stream() - .map(this::toItemVo) - .toList(); + List menus = loadMenus(menuType); + return toItemVos(menus, menus); } @Transactional public PermissionMenuItemVo create(PermissionMenuCreateRequest request) { + return create(null, request); + } + + /** Controller-facing variant; the caller may pass the authenticated admin. */ + @Transactional + public PermissionMenuItemVo create(AdminUserEntity operator, PermissionMenuCreateRequest request) { + ensureAdminOperatorIfPresent(operator); + if (request == null) { + throw new BusinessException("菜单参数不能为空"); + } String name = normalizeRequired(request.getName(), "菜单名称不能为空"); String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空"); String menuType = normalizeMenuTypeRequired(request.getMenuType()); String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空"); + Long parentId = normalizeParentId(request.getParentId()); + ensureParentValid(parentId, null, menuType); ensureUniqueColumnKey(columnKey, null); ensureUniqueRoutePath(menuType, routePath, null); PermissionMenuEntity entity = new PermissionMenuEntity(); entity.setName(name); entity.setColumnKey(columnKey); + entity.setParentId(parentId); entity.setMenuType(menuType); entity.setRoutePath(routePath); entity.setSortOrder(resolveSortOrder(request.getSortOrder(), null)); @@ -71,16 +92,30 @@ public class PermissionMenuService { @Transactional public PermissionMenuItemVo update(Long id, PermissionMenuUpdateRequest request) { + return update(null, id, request); + } + + /** Controller-facing variant; validates a parent change against the full ancestor chain. */ + @Transactional + public PermissionMenuItemVo update(AdminUserEntity operator, Long id, PermissionMenuUpdateRequest request) { + ensureAdminOperatorIfPresent(operator); PermissionMenuEntity entity = getMenuById(id); + if (request == null) { + throw new BusinessException("菜单参数不能为空"); + } String name = normalizeRequired(request.getName(), "菜单名称不能为空"); String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空"); String menuType = normalizeMenuTypeRequired(request.getMenuType()); String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空"); + Long parentId = normalizeParentId(request.getParentId()); + ensureParentValid(parentId, id, menuType); + ensureMenuTypeChangeAllowed(entity, menuType); ensureUniqueColumnKey(columnKey, id); ensureUniqueRoutePath(menuType, routePath, id); entity.setName(name); entity.setColumnKey(columnKey); + entity.setParentId(parentId); entity.setMenuType(menuType); entity.setRoutePath(routePath); entity.setSortOrder(resolveSortOrder(request.getSortOrder(), id)); @@ -90,102 +125,224 @@ public class PermissionMenuService { @Transactional public void delete(Long id) { + delete(null, id); + } + + /** + * Deletes a leaf menu only. Refusing a parent deletion prevents orphaned + * hierarchy rows and makes the direct-grant meaning deterministic. + */ + @Transactional + public void delete(AdminUserEntity operator, Long id) { + ensureAdminOperatorIfPresent(operator); PermissionMenuEntity entity = getMenuById(id); + Long childCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper() + .eq(PermissionMenuEntity::getParentId, entity.getId())); + if (childCount != null && childCount > 0) { + throw new BusinessException("请先删除该菜单下的子菜单"); + } + // Do this explicitly even when the database FK is configured, so old + // installations without the FK are cleaned up as well. userColumnPermissionMapper.delete(new LambdaUpdateWrapper() .eq(UserColumnPermissionEntity::getColumnId, entity.getId())); permissionMenuMapper.deleteById(entity.getId()); } + /** Returns only persisted direct grant IDs (never recursively expanded). */ public UserColumnIdsVo getUserColumnIds(Long userId, String menuType) { + return getUserColumnIds(null, userId, menuType); + } + + /** Controller-facing variant with target-user scope checks. */ + public UserColumnIdsVo getUserColumnIds(AdminUserEntity operator, Long userId, String menuType) { AdminUserEntity user = getUserById(userId); - List columnIds; - if (isSuperAdmin(user)) { - columnIds = list(menuType).stream().map(PermissionMenuItemVo::getId).toList(); - } else { - List assignedIds = userColumnPermissionMapper.selectList( - new LambdaQueryWrapper() - .eq(UserColumnPermissionEntity::getUserId, userId)) - .stream() - .map(UserColumnPermissionEntity::getColumnId) - .distinct() - .toList(); - if (assignedIds.isEmpty()) { - columnIds = List.of(); - } else if (isValidMenuType(menuType)) { - Set allowedIds = list(menuType).stream() - .map(PermissionMenuItemVo::getId) - .collect(Collectors.toSet()); - columnIds = assignedIds.stream().filter(allowedIds::contains).toList(); - } else { - columnIds = assignedIds; - } - } + ensureTargetAccessible(operator, user); + + List menus = loadMenus(menuType); + Map menuMap = menus.stream() + .filter(menu -> menu.getId() != null) + .collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity(), (left, right) -> left)); UserColumnIdsVo vo = new UserColumnIdsVo(); - vo.setColumnIds(columnIds); + vo.setColumnIds(sortDirectIds(loadDirectColumnIds(userId), menuMap)); return vo; } + /** + * Returns effective menu items: each directly granted menu plus every + * descendant in the same menu type. The response is flat and sorted. + */ public List getUserColumnPermissions(Long userId, String menuType) { + return getUserColumnPermissions(null, userId, menuType); + } + + /** Controller-facing variant with target-user scope checks. */ + public List getUserColumnPermissions(AdminUserEntity operator, + Long userId, + String menuType) { AdminUserEntity user = getUserById(userId); + ensureTargetAccessible(operator, user); + List menus = loadMenus(menuType); if (isSuperAdmin(user)) { - return list(menuType); + return toItemVos(menus, menus); } - List assignedIds = userColumnPermissionMapper.selectList( - new LambdaQueryWrapper() - .eq(UserColumnPermissionEntity::getUserId, userId)) - .stream() - .map(UserColumnPermissionEntity::getColumnId) - .distinct() - .toList(); - if (assignedIds.isEmpty()) { + Set directIds = new LinkedHashSet<>(loadDirectColumnIds(userId)); + if (directIds.isEmpty() || menus.isEmpty()) { return List.of(); } - List menus = permissionMenuMapper.selectBatchIds(assignedIds); - Map menuMap = menus.stream() - .collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity())); - String safeMenuType = normalizeMenuType(menuType); - return assignedIds.stream() - .map(menuMap::get) - .filter(menu -> menu != null) - .filter(menu -> safeMenuType == null || safeMenuType.equals(menu.getMenuType())) - .sorted(Comparator - .comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo)) - .thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo))) - .map(this::toItemVo) + Set effectiveIds = expandDescendantIds(directIds, menus); + List effectiveMenus = menus.stream() + .filter(menu -> menu.getId() != null && effectiveIds.contains(menu.getId())) + .toList(); + return toItemVos(effectiveMenus, menus); + } + + /** Effective authorization check used by backend callers that need a boolean decision. */ + public boolean hasEffectiveColumnPermission(Long userId, Long columnId) { + if (columnId == null || columnId <= 0) { + return false; + } + AdminUserEntity user = getUserById(userId); + PermissionMenuEntity target = permissionMenuMapper.selectById(columnId); + if (target == null) { + return false; + } + if (isSuperAdmin(user)) { + return true; + } + List menus = loadMenus(target.getMenuType()); + return expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(userId)), menus) + .contains(columnId); + } + + public List listImageVideoDataPermissionUsers(AdminUserEntity operator) { + ensureSuperAdminOperator(operator); + PermissionMenuEntity dataPermission = requireImageVideoDataPermission(); + Set grantedUserIds = userColumnPermissionMapper.selectList( + new LambdaQueryWrapper() + .eq(UserColumnPermissionEntity::getColumnId, dataPermission.getId())) + .stream() + .map(UserColumnPermissionEntity::getUserId) + .filter(id -> id != null && id > 0) + .collect(Collectors.toSet()); + return adminUserMapper.selectList(new LambdaQueryWrapper() + .orderByAsc(AdminUserEntity::getUsername) + .orderByAsc(AdminUserEntity::getId)) + .stream() + .filter(user -> !isSuperAdmin(user)) + .map(user -> toImageVideoDataPermissionUserVo(user, grantedUserIds.contains(user.getId()))) .toList(); } + @Transactional + public int updateImageVideoDataPermissionUsers(AdminUserEntity operator, List userIds) { + ensureSuperAdminOperator(operator); + PermissionMenuEntity dataPermission = requireImageVideoDataPermission(); + List requestedIds = normalizeColumnIds(userIds); + List users = adminUserMapper.selectList(new LambdaQueryWrapper()); + Map grantableUsers = users.stream() + .filter(user -> user.getId() != null && !isSuperAdmin(user)) + .collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left)); + if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) { + throw new BusinessException("包含不存在或不可授权的用户"); + } + + userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId())); + for (Long userId : requestedIds) { + UserColumnPermissionEntity grant = new UserColumnPermissionEntity(); + grant.setUserId(userId); + grant.setColumnId(dataPermission.getId()); + userColumnPermissionMapper.insert(grant); + } + return requestedIds.size(); + } + @Transactional public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) { - getUserById(userId); + updateUserColumnPermissions(null, userId, request); + } + + /** + * Replaces direct grants. A non-super admin may only grant effective menus + * that the operator already has, and only to themself or a directly-created + * normal user. + */ + @Transactional + public void updateUserColumnPermissions(AdminUserEntity operator, + Long userId, + UserColumnPermissionUpdateRequest request) { + updateUserColumnPermissions(operator, userId, request, null); + } + + /** Replaces direct grants only within menuType; a blank type keeps legacy full-replacement behavior. */ + @Transactional + public void updateUserColumnPermissions(AdminUserEntity operator, + Long userId, + UserColumnPermissionUpdateRequest request, + String menuType) { + AdminUserEntity target = getUserById(userId); + ensureTargetAccessible(operator, target); List requestedIds = request == null ? List.of() : normalizeColumnIds(request.getColumnIds()); - if (!requestedIds.isEmpty()) { - Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper() - .in(PermissionMenuEntity::getId, requestedIds)); - if (validCount == null || validCount != requestedIds.size()) { - throw new BusinessException("存在无效的菜单权限项"); + String normalizedType = menuType == null || menuType.isBlank() + ? null + : normalizeMenuTypeRequired(menuType); + Set scopedMenuIds = new LinkedHashSet<>(); + if (normalizedType == null) { + if (!requestedIds.isEmpty()) { + Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper() + .in(PermissionMenuEntity::getId, requestedIds)); + if (validCount == null || validCount != requestedIds.size()) { + throw new BusinessException("存在无效的菜单权限项"); + } + } + } else { + scopedMenuIds.addAll(loadMenus(normalizedType).stream() + .map(PermissionMenuEntity::getId) + .filter(Objects::nonNull) + .toList()); + if (!scopedMenuIds.containsAll(requestedIds)) { + throw new BusinessException("存在无效或不属于当前类型的菜单权限项"); } } - PermissionMenuEntity imageVideoDataPermission = permissionMenuMapper.selectOne( - new LambdaQueryWrapper() - .eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY) - .last("LIMIT 1")); - if (imageVideoDataPermission != null) { - requestedIds = new ArrayList<>(requestedIds); + PermissionMenuEntity imageVideoDataPermission = normalizedType == null + ? findImageVideoDataPermission() + : null; + Long protectedId = imageVideoDataPermission == null ? null : imageVideoDataPermission.getId(); + List grantIds = requestedIds; + if (normalizedType == null && protectedId != null) { + grantIds = requestedIds.stream() + .filter(id -> !protectedId.equals(id)) + .toList(); + } + Set operatorEffectiveIds = ensureGrantable(operator, grantIds); + LinkedHashSet finalGrantIds = new LinkedHashSet<>(grantIds); + + if (normalizedType == null && protectedId != null) { Long existingCount = userColumnPermissionMapper.selectCount( new LambdaQueryWrapper() .eq(UserColumnPermissionEntity::getUserId, userId) - .eq(UserColumnPermissionEntity::getColumnId, imageVideoDataPermission.getId())); - requestedIds.remove(imageVideoDataPermission.getId()); + .eq(UserColumnPermissionEntity::getColumnId, protectedId)); if (existingCount != null && existingCount > 0) { - requestedIds.add(imageVideoDataPermission.getId()); + finalGrantIds.add(protectedId); } } + if (operatorEffectiveIds != null) { + loadDirectColumnIds(userId).stream() + .filter(id -> normalizedType == null || scopedMenuIds.contains(id)) + .filter(id -> !operatorEffectiveIds.contains(id)) + .forEach(finalGrantIds::add); + } - userColumnPermissionMapper.delete(new LambdaUpdateWrapper() - .eq(UserColumnPermissionEntity::getUserId, userId)); - for (Long columnId : requestedIds) { + LambdaUpdateWrapper delete = + new LambdaUpdateWrapper() + .eq(UserColumnPermissionEntity::getUserId, userId); + if (normalizedType == null) { + userColumnPermissionMapper.delete(delete); + } else if (!scopedMenuIds.isEmpty()) { + userColumnPermissionMapper.delete(delete.in( + UserColumnPermissionEntity::getColumnId, scopedMenuIds)); + } + for (Long columnId : finalGrantIds) { UserColumnPermissionEntity entity = new UserColumnPermissionEntity(); entity.setUserId(userId); entity.setColumnId(columnId); @@ -193,6 +350,75 @@ public class PermissionMenuService { } } + private List loadMenus(String menuType) { + String normalizedType = normalizeMenuType(menuType); + LambdaQueryWrapper query = new LambdaQueryWrapper() + .eq(normalizedType != null, PermissionMenuEntity::getMenuType, normalizedType) + .orderByAsc(PermissionMenuEntity::getSortOrder) + .orderByAsc(PermissionMenuEntity::getId); + List rows = permissionMenuMapper.selectList(query); + if (rows == null || rows.isEmpty()) { + return List.of(); + } + return rows.stream() + .filter(row -> row != null && row.getId() != null) + .sorted(menuComparator()) + .toList(); + } + + private List loadDirectColumnIds(Long userId) { + List rows = userColumnPermissionMapper.selectList( + new LambdaQueryWrapper() + .eq(UserColumnPermissionEntity::getUserId, userId)); + if (rows == null || rows.isEmpty()) { + return List.of(); + } + return rows.stream() + .map(UserColumnPermissionEntity::getColumnId) + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + } + + private List sortDirectIds(List ids, Map menuMap) { + return ids.stream() + .filter(menuMap::containsKey) + .sorted(Comparator.comparing(menuMap::get, menuComparator())) + .toList(); + } + + private Set expandDescendantIds(Set directIds, List menus) { + Map> childrenByParent = new HashMap<>(); + Map menuMap = new HashMap<>(); + for (PermissionMenuEntity menu : menus) { + if (menu.getId() == null) { + continue; + } + menuMap.put(menu.getId(), menu); + if (menu.getParentId() != null) { + childrenByParent.computeIfAbsent(menu.getParentId(), ignored -> new ArrayList<>()).add(menu); + } + } + childrenByParent.values().forEach(children -> children.sort(menuComparator())); + + LinkedHashSet effective = new LinkedHashSet<>(); + ArrayDeque queue = new ArrayDeque<>(); + for (Long directId : directIds) { + if (directId != null && menuMap.containsKey(directId) && effective.add(directId)) { + queue.addLast(directId); + } + } + while (!queue.isEmpty()) { + Long parentId = queue.removeFirst(); + for (PermissionMenuEntity child : childrenByParent.getOrDefault(parentId, List.of())) { + if (effective.add(child.getId())) { + queue.addLast(child.getId()); + } + } + } + return effective; + } + private PermissionMenuEntity getMenuById(Long id) { PermissionMenuEntity entity = permissionMenuMapper.selectById(id); if (entity == null) { @@ -209,8 +435,175 @@ public class PermissionMenuService { return entity; } + /** Resolves and validates an operator supplied by a trusted internal proxy. */ + public AdminUserEntity requireAdminOperator(Long userId) { + AdminUserEntity operator = getUserById(userId); + ensureAdminOperatorIfPresent(operator); + return operator; + } + + /** Resolves any authenticated user supplied by a trusted internal proxy. */ + public AdminUserEntity requireUserOperator(Long userId) { + return getUserById(userId); + } + + private void ensureAdminOperatorIfPresent(AdminUserEntity operator) { + if (operator == null) { + return; + } + if (!isAdmin(operator)) { + throw new BusinessException(403, "需要管理员权限"); + } + } + + private void ensureTargetAccessible(AdminUserEntity operator, AdminUserEntity target) { + if (operator == null) { + return; + } + boolean self = operator.getId() != null && operator.getId().equals(target.getId()); + if (self || isSuperAdmin(operator)) { + return; + } + if (!isAdmin(operator)) { + throw new BusinessException(403, "需要管理员权限"); + } + boolean ownedNormal = isNormalUser(target) + && operator.getId() != null + && operator.getId().equals(target.getCreatedById()); + if (!ownedNormal) { + throw new BusinessException(403, "普通管理员只能管理自己创建的普通用户"); + } + } + + private Set ensureGrantable(AdminUserEntity operator, List requestedIds) { + if (operator == null || isSuperAdmin(operator)) { + return null; + } + List menus = loadMenus(null); + Set effective = expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(operator.getId())), menus); + Set denied = requestedIds.stream() + .filter(id -> !effective.contains(id)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (!denied.isEmpty()) { + throw new BusinessException(403, "普通管理员只能分配自己已有的菜单权限"); + } + return effective; + } + + private boolean isAdmin(AdminUserEntity user) { + if (user == null) { + return false; + } + String role = normalizeRole(user.getRole()); + if (!role.isEmpty()) { + return "super_admin".equals(role) || "admin".equals(role); + } + return Integer.valueOf(1).equals(user.getIsAdmin()); + } + private boolean isSuperAdmin(AdminUserEntity user) { - return user != null && "super_admin".equalsIgnoreCase(normalizeRequired(user.getRole(), "")); + if (user == null) { + return false; + } + String role = normalizeRole(user.getRole()); + if ("super_admin".equals(role)) { + return true; + } + return role.isEmpty() + && Integer.valueOf(1).equals(user.getIsAdmin()) + && user.getCreatedById() == null; + } + + private boolean isNormalUser(AdminUserEntity user) { + if (user == null) { + return false; + } + String role = normalizeRole(user.getRole()); + if (!role.isEmpty()) { + return !"admin".equals(role) && !"super_admin".equals(role); + } + return !Integer.valueOf(1).equals(user.getIsAdmin()); + } + + private PermissionMenuEntity findImageVideoDataPermission() { + return permissionMenuMapper.selectOne(new LambdaQueryWrapper() + .eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY) + .last("LIMIT 1")); + } + + private PermissionMenuEntity requireImageVideoDataPermission() { + PermissionMenuEntity permission = findImageVideoDataPermission(); + if (permission == null || permission.getId() == null) { + throw new BusinessException("视频任务数据权限尚未初始化"); + } + return permission; + } + + private void ensureSuperAdminOperator(AdminUserEntity operator) { + ensureAdminOperatorIfPresent(operator); + if (operator == null || !isSuperAdmin(operator)) { + throw new BusinessException(403, "仅超级管理员可以配置视频任务权限"); + } + } + + private ImageVideoDataPermissionUserVo toImageVideoDataPermissionUserVo(AdminUserEntity user, + boolean granted) { + ImageVideoDataPermissionUserVo vo = new ImageVideoDataPermissionUserVo(); + vo.setId(user.getId()); + vo.setUsername(user.getUsername()); + String role = normalizeRole(user.getRole()); + vo.setRole(role.isEmpty() + ? (Integer.valueOf(1).equals(user.getIsAdmin()) ? "admin" : "normal") + : role); + vo.setGranted(granted); + return vo; + } + + private void ensureParentValid(Long parentId, Long currentId, String menuType) { + if (parentId == null) { + return; + } + if (currentId != null && currentId.equals(parentId)) { + throw new BusinessException("父级菜单不能选择自己"); + } + PermissionMenuEntity parent = permissionMenuMapper.selectById(parentId); + if (parent == null) { + throw new BusinessException("父级菜单不存在"); + } + String parentType = normalizeMenuType(parent.getMenuType()); + if (!menuType.equals(parentType)) { + throw new BusinessException("父级菜单必须与当前菜单属于同一类型"); + } + + Set visited = new HashSet<>(); + PermissionMenuEntity cursor = parent; + while (cursor != null && cursor.getId() != null) { + if (!visited.add(cursor.getId())) { + throw new BusinessException("菜单层级存在循环"); + } + if (currentId != null && currentId.equals(cursor.getId())) { + throw new BusinessException("父级菜单不能选择自己的子级"); + } + Long ancestorId = cursor.getParentId(); + if (ancestorId == null) { + break; + } + cursor = permissionMenuMapper.selectById(ancestorId); + if (cursor == null) { + throw new BusinessException("父级菜单不存在"); + } + } + } + + private void ensureMenuTypeChangeAllowed(PermissionMenuEntity entity, String menuType) { + if (menuType.equals(normalizeMenuType(entity.getMenuType()))) { + return; + } + Long childCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper() + .eq(PermissionMenuEntity::getParentId, entity.getId())); + if (childCount != null && childCount > 0) { + throw new BusinessException("请先移出该菜单下的子菜单再修改菜单类型"); + } } private void ensureUniqueColumnKey(String columnKey, Long excludeId) { @@ -240,8 +633,12 @@ public class PermissionMenuService { return normalized; } - private boolean isValidMenuType(String menuType) { - return normalizeMenuType(menuType) != null; + private Long normalizeParentId(Long value) { + return value == null || value <= 0 ? null : value; + } + + private String normalizeRole(String role) { + return role == null ? "" : role.trim().toLowerCase(Locale.ROOT); } private String normalizeMenuTypeRequired(String menuType) { @@ -260,6 +657,8 @@ public class PermissionMenuService { if (MENU_TYPE_APP.equals(normalized) || MENU_TYPE_ADMIN.equals(normalized)) { return normalized; } + // Keep the internal data permission visible to legacy callers that do + // not pass a menu type, but never treat an unknown filter as a type. return null; } @@ -276,11 +675,34 @@ public class PermissionMenuService { return new ArrayList<>(uniqueIds); } + private Comparator menuComparator() { + return Comparator + .comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo)) + .thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo)); + } + + private List toItemVos(List visibleMenus, + List hierarchyMenus) { + Map menuMap = hierarchyMenus.stream() + .filter(menu -> menu.getId() != null) + .collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity(), (left, right) -> left)); + return visibleMenus.stream() + .map(menu -> toItemVo(menu, menuMap)) + .toList(); + } + private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity) { + return toItemVos(List.of(entity), loadMenus(entity.getMenuType())).getFirst(); + } + + private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity, + Map menuMap) { PermissionMenuItemVo vo = new PermissionMenuItemVo(); vo.setId(entity.getId()); vo.setName(entity.getName()); vo.setColumnKey(entity.getColumnKey()); + vo.setParentId(entity.getParentId()); + vo.setRootColumnKey(resolveRootColumnKey(entity, menuMap)); vo.setMenuType(entity.getMenuType()); vo.setRoutePath(entity.getRoutePath()); vo.setSortOrder(entity.getSortOrder()); @@ -288,6 +710,20 @@ public class PermissionMenuService { return vo; } + private String resolveRootColumnKey(PermissionMenuEntity entity, + Map menuMap) { + PermissionMenuEntity root = entity; + Set visited = new HashSet<>(); + while (root.getId() != null && visited.add(root.getId()) && root.getParentId() != null) { + PermissionMenuEntity parent = menuMap.get(root.getParentId()); + if (parent == null) { + break; + } + root = parent; + } + return root.getColumnKey(); + } + private Integer resolveSortOrder(Integer sortOrder, Long excludeId) { if (sortOrder != null) { return sortOrder; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskMatchShopsRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskMatchShopsRequest.java index cd7a4b85..621ed8db 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskMatchShopsRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskMatchShopsRequest.java @@ -20,7 +20,7 @@ public class ProductRiskMatchShopsRequest { @NotEmpty(message = "shop_names 不能为空") @JsonProperty("shop_names") @Schema( - description = "待匹配的店铺名称列表，至少 1 个；服务端按列表顺序逐项查索引，返回体 `items` 与列表顺序一一对应", + description = "待匹配的店铺名称列表，至少 1 个；服务端规范化名称、过滤空项并按首次出现顺序去重", requiredMode = Schema.RequiredMode.REQUIRED, example = "[\"店铺甲\",\"店铺乙\"]") private List shopNames; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCountryPreferenceVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCountryPreferenceVo.java index c6b0ab51..c1c741db 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCountryPreferenceVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCountryPreferenceVo.java @@ -12,6 +12,6 @@ import java.util.List; public class ProductRiskCountryPreferenceVo { @JsonProperty("country_codes") - @Schema(description = "已选国家代码，顺序即处理顺序；未持久化时服务端返回默认德国→英国→法国→意大利→西班牙（全选）") + @Schema(description = "已选国家代码，顺序即处理顺序；未保存偏好时返回当前业务模块配置的默认顺序") private List countryCodes = new ArrayList<>(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskMatchShopsVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskMatchShopsVo.java index e1cdd050..baecf492 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskMatchShopsVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskMatchShopsVo.java @@ -7,9 +7,9 @@ import java.util.ArrayList; import java.util.List; @Data -@Schema(description = "批量匹配响应体：items 与请求 shop_names 顺序一致") +@Schema(description = "批量匹配响应体：按规范化、过滤空项并去重后的有效店铺顺序返回") public class ProductRiskMatchShopsVo { - @Schema(description = "各店匹配结果列表，一项对应请求中的一个店名") + @Schema(description = "各店匹配结果列表，一项对应一个处理后的有效店铺名") private List items = new ArrayList<>(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java index f228a79f..a593d99e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java @@ -106,8 +106,8 @@ public class PublishController { @PostMapping("/tasks/{taskId}/result") @Operation( - summary = "Python 按文件回传当前店铺完整数据", - description = "请求只需 taskId 和 files，user_id 为兼容旧客户端的可选字段；后端按 taskId 反查任务所属用户，传入 user_id 时会校验归属。成功回传必须包含该文件全部原始行，可使用 rows 数组，或在 rows 为空时使用 countries 按国家分组；行数少于原始数据、包含 null 行或八列全空白对象时会被拒绝且不会覆盖已解析数据。error 非空时将该文件标记为 FAILED。全部文件进入终态后，只要至少一个文件成功就异步组装结果；全部失败则不生成结果文件。") + summary = "Python 按文件分片回传上架结果", + description = "请求只需 taskId 和 files，user_id 为兼容旧客户端的可选字段；后端始终按 taskId 处理任务数据。每个文件通过 chunk_index/chunk_total 声明从 1 开始的分片，不传时按 1/1 兼容旧客户端。正常分片先写入 RustFS，同一文件的全部分片到齐后才按序合并、校验总行数并覆盖解析数据；重复分片内容相同则幂等接受，内容不同或 chunk_total 不一致则拒绝。error 非空时直接将文件标记为 FAILED。全部文件进入终态后，只要至少一个文件成功就异步组装结果；全部失败则不生成结果文件。") public ApiResponse submitResult( @Parameter(description = "上架任务 ID", required = true, example = "9001") @PathVariable Long taskId, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/dto/PublishResultFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/dto/PublishResultFileDto.java index e7cf7400..99f0ba5f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/dto/PublishResultFileDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/dto/PublishResultFileDto.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.publish.model.dto; import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.Valid; import lombok.Data; @@ -11,29 +12,39 @@ import java.util.List; import java.util.Map; @Data -@Schema(description = "单个文件的 Python 处理结果。fileId、fileKey、sourceFilename 至少提供一个用于定位文件。成功时必须提交完整 rows 或 countries；失败时填写 error。同一次请求中，不同定位字段指向同一任务文件也视为重复提交。") +@Schema(description = "单个文件的一次 Python 分片回传。fileId/file_key/source_filename 三者可用于定位文件，rows 或 countries 提供当前分片数据，error 用于返回失败。") public class PublishResultFileDto { @JsonAlias("file_id") @Schema(description = "任务内文件 ID；同时兼容 file_id，优先使用该字段定位", example = "9101") private Long fileId; @JsonAlias("file_key") - @Schema(description = "源文件标识；同时兼容 file_key，可在 fileId 缺失时定位文件", example = "uploads/20260724/uuid/郭亚庆.xlsx") + @Schema(description = "任务内文件标识；兼容 file_key，fileId 缺失时可通过该字段定位", example = "uploads/20260724/uuid/郭亚庆.xlsx") private String fileKey; @JsonAlias("source_filename") - @Schema(description = "原始文件名；同时兼容 source_filename，可在 fileId/fileKey 缺失时定位文件", example = "郭亚庆.xlsx") + @Schema(description = "原始文件名；兼容 source_filename，fileId/fileKey 都缺失时可定位文件", example = "郭亚庆.xlsx") private String sourceFilename; - @Schema(description = "文件级失败原因。非空时文件标记为 FAILED，rows/countries 不会覆盖原始数据", example = "打开店铺失败") + @JsonProperty("chunk_index") + @JsonAlias("chunkIndex") + @Schema(description = "分片序号，从 1 开始；不传时与 chunk_total 同时未传表示 1/1，兼容旧版。", example = "1", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + private Integer chunkIndex; + + @JsonProperty("chunk_total") + @JsonAlias("chunkTotal") + @Schema(description = "该文件的分片总数；不传时按 1 处理。分片先暂存到 RustFS，全部到齐后再按序合并。", example = "36", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + private Integer chunkTotal; + + @Schema(description = "文件处理结果；FAILED 时通过 error 携带错误信息，rows/countries 可为空", example = "打开文件失败") private String error; @Valid @JsonAlias({"items", "data"}) - @Schema(description = "当前店铺的完整结果行；同时兼容 items/data。成功时行数不能少于原始 Excel 数据行数，且不能包含 null 行或八列全空白对象") + @Schema(description = "当前分片的结果行，可为一条或多条；rows 中每条均不能为空对象") private List rows = new ArrayList<>(); @Valid - @Schema(description = "按国家名称或代码分组的完整结果，可替代 rows。仅当 rows 为空时读取；缺少国家字段的行会使用当前 Map key；分组内不能包含 null 行或八列全空白对象") + @Schema(description = "当前分片按国家分组的结果；rows 为空时可使用 countries，key 为国家或店铺名") private Map> countries = new LinkedHashMap<>(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java index 5c2432e7..3a91248f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java @@ -2,8 +2,10 @@ package com.nanri.aiimage.modules.publish.service; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.IdUtil; +import cn.hutool.crypto.digest.DigestUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; @@ -31,18 +33,24 @@ import com.nanri.aiimage.modules.publish.model.vo.PublishTaskDetailVo; import com.nanri.aiimage.modules.publish.model.vo.PublishTaskVo; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DuplicateKeyException; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -84,8 +92,11 @@ public class PublishTaskService { private final PublishItemMapper publishItemMapper; private final FileTaskMapper fileTaskMapper; private final FileResultMapper fileResultMapper; + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; private final TaskFileJobService taskFileJobService; private final TaskDistributedLockService taskDistributedLockService; + private final TransientPayloadStorageService transientPayloadStorageService; private final OssStorageService ossStorageService; private final ObjectMapper objectMapper; private final TransactionTemplate transactionTemplate; @@ -239,7 +250,23 @@ public class PublishTaskService { if (lock == null) { throw new BusinessException("task lock is busy"); } - transactionTemplate.executeWithoutResult(status -> submitResultLocked(taskId, request)); + List storedPayloads = new ArrayList<>(); + boolean[] cleanupAfterCommit = {false}; + try { + transactionTemplate.executeWithoutResult( + status -> cleanupAfterCommit[0] = submitResultLocked(taskId, request, storedPayloads)); + } catch (RuntimeException ex) { + deleteRolledBackPayloads(storedPayloads); + throw ex; + } + if (cleanupAfterCommit[0]) { + try { + deleteTransientResultChunks(taskId); + } catch (Exception ex) { + log.warn("[publish] failed-task chunk cleanup failed taskId={} msg={}", + taskId, safeMessage(ex)); + } + } } } @@ -455,7 +482,7 @@ public class PublishTaskService { @Transactional public void deleteTask(Long taskId, Long userId) { - FileTaskEntity task = requireTask(taskId, userId); + FileTaskEntity task = requireTaskForDeletion(taskId, userId); List results = fileResultMapper.selectList(new LambdaQueryWrapper() .eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getModuleType, MODULE_TYPE)); @@ -470,6 +497,7 @@ public class PublishTaskService { } } taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); + deleteTransientResultChunks(taskId); publishItemMapper.delete(new LambdaQueryWrapper() .eq(PublishItemEntity::getTaskId, taskId)); publishFileMapper.delete(new LambdaQueryWrapper() @@ -486,8 +514,49 @@ public class PublishTaskService { if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { throw new BusinessException("结果不存在"); } - FileTaskEntity task = requireTask(result.getTaskId(), userId); - deleteTask(task.getId(), userId); + deleteTask(result.getTaskId(), userId); + } + + public void cleanupResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + return; + } + deleteTransientResultChunks(job.getTaskId()); + } + + private void deleteTransientResultChunks(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); + } + } + taskChunkMapper.delete(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskScopeStateMapper.delete(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + } + + private void deleteRolledBackPayloads(List storedPayloads) { + if (storedPayloads == null || storedPayloads.isEmpty()) { + return; + } + for (String storedPayload : storedPayloads) { + try { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + } catch (Exception ex) { + log.warn("[publish] rolled-back RustFS payload cleanup failed pointer={} msg={}", + transientPayloadStorageService.extractPointer(storedPayload), safeMessage(ex)); + } + } } private PreparedFile prepareFile(PublishSourceFileDto source) { @@ -538,6 +607,26 @@ public class PublishTaskService { } LocalDateTime now = LocalDateTime.now(); String error = "任务心跳超时"; + List files = listTaskFiles(taskId); + for (PublishFileEntity file : files) { + if (!STATUS_PENDING.equals(file.getStatus()) && !STATUS_RUNNING.equals(file.getStatus())) { + continue; + } + List recoveredRows = loadReceivedResultRows(taskId, file.getId()); + if (recoveredRows.isEmpty()) { + continue; + } + replaceRows(taskId, file.getId(), recoveredRows); + file.setStatus(STATUS_SUCCESS); + file.setTotalRows(recoveredRows.size()); + file.setProcessedRows(recoveredRows.size()); + file.setErrorMessage("任务心跳超时，已保留 " + recoveredRows.size() + " 行已回传数据"); + file.setUpdatedAt(now); + file.setFinishedAt(now); + publishFileMapper.updateById(file); + log.info("[publish] recovered partial result from stale task taskId={} fileId={} rows={}", + taskId, file.getId(), recoveredRows.size()); + } publishFileMapper.update(null, new LambdaUpdateWrapper() .eq(PublishFileEntity::getTaskId, taskId) .in(PublishFileEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING)) @@ -545,7 +634,7 @@ public class PublishTaskService { .set(PublishFileEntity::getErrorMessage, error) .set(PublishFileEntity::getUpdatedAt, now) .set(PublishFileEntity::getFinishedAt, now)); - List files = listTaskFiles(taskId); + files = listTaskFiles(taskId); int successCount = (int) files.stream().filter(file -> STATUS_SUCCESS.equals(file.getStatus())).count(); task.setSuccessFileCount(successCount); task.setFailedFileCount(Math.max(0, files.size() - successCount)); @@ -615,10 +704,12 @@ public class PublishTaskService { return new PersistedTask(task, result, savedFiles); } - private void submitResultLocked(Long taskId, PublishSubmitResultRequest request) { + private boolean submitResultLocked(Long taskId, + PublishSubmitResultRequest request, + List storedPayloads) { FileTaskEntity task = requireTask(taskId, request.getUserId()); if (STATUS_SUCCESS.equals(task.getStatus())) { - return; + return false; } if (STATUS_FAILED.equals(task.getStatus())) { throw new BusinessException("任务已失败，拒绝继续回传"); @@ -629,6 +720,9 @@ public class PublishTaskService { Set submittedFileIds = new LinkedHashSet<>(); for (PublishResultFileDto incoming : request.getFiles()) { + if (incoming == null) { + throw new BusinessException("files 不能包含空对象"); + } PublishFileEntity file = findCallbackFile(taskId, incoming); if (!submittedFileIds.add(file.getId())) { throw new BusinessException("同一文件不能在一次请求中重复提交"); @@ -641,7 +735,16 @@ public class PublishTaskService { file.setProcessedRows(0); file.setErrorMessage(incoming.getError().trim()); } else { - List rows = flattenRows(incoming); + ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads); + if (!receipt.completed()) { + file.setStatus(STATUS_RUNNING); + file.setErrorMessage(null); + file.setUpdatedAt(LocalDateTime.now()); + file.setFinishedAt(null); + publishFileMapper.updateById(file); + continue; + } + List rows = loadCompleteResultRows(taskId, receipt); validateCompleteResultRows(taskId, file.getId(), rows); replaceRows(taskId, file.getId(), rows); file.setStatus(STATUS_SUCCESS); @@ -666,11 +769,11 @@ public class PublishTaskService { if (terminalCount < files.size()) { task.setStatus(STATUS_RUNNING); fileTaskMapper.updateById(task); - return; + return false; } if (successCount <= 0) { markTaskAndResultFailed(task, result, "全部文件处理失败"); - return; + return true; } task.setStatus(STATUS_RUNNING); @@ -678,6 +781,7 @@ public class PublishTaskService { task.setFinishedAt(null); fileTaskMapper.updateById(task); taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId)); + return false; } private List loadTaskDetails(List tasks) { @@ -805,6 +909,20 @@ public class PublishTaskService { } private FileTaskEntity requireTask(Long taskId, Long userId) { + FileTaskEntity task = requireTaskRecord(taskId, userId); + ensureTaskOwnedByCurrentInstance(task, "access publish task"); + return task; + } + + private FileTaskEntity requireTaskForDeletion(Long taskId, Long userId) { + FileTaskEntity task = requireTaskRecord(taskId, userId); + if (!isTerminal(task.getStatus())) { + ensureTaskOwnedByCurrentInstance(task, "delete publish task"); + } + return task; + } + + private FileTaskEntity requireTaskRecord(Long taskId, Long userId) { if (taskId == null || taskId <= 0) { throw new BusinessException("taskId 不合法"); } @@ -813,7 +931,6 @@ public class PublishTaskService { || (userId != null && !userId.equals(task.getUserId()))) { throw new BusinessException("任务不存在"); } - ensureTaskOwnedByCurrentInstance(task, "access publish task"); return task; } @@ -888,6 +1005,257 @@ public class PublishTaskService { return file; } + private ResultChunkReceipt persistResultChunk(Long taskId, + PublishFileEntity file, + PublishResultFileDto incoming, + List storedPayloads) { + int chunkIndex = incoming.getChunkIndex() == null ? 1 : incoming.getChunkIndex(); + int chunkTotal = incoming.getChunkTotal() == null ? 1 : incoming.getChunkTotal(); + validateChunkMetadata(chunkIndex, chunkTotal); + + String scopeKey = "file:" + file.getId(); + String scopeHash = DigestUtil.sha256Hex(scopeKey); + TaskScopeStateEntity scope = findResultScope(taskId, scopeHash); + validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal); + + List rows = flattenRows(incoming); + String payloadJson = writeJson(rows, "序列化上架结果分片失败"); + String payloadHash = DigestUtil.sha256Hex(payloadJson); + TaskChunkEntity existing = findResultChunk(taskId, scopeHash, chunkIndex); + if (existing != null) { + validateExistingChunk(existing, chunkTotal, payloadHash); + int receivedChunkCount = countResultChunks(taskId, scopeHash); + persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount); + return new ResultChunkReceipt(scopeHash, chunkTotal, + receivedChunkCount >= chunkTotal); + } + + ensureRustfsPayloadStorageEnabled(); + String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned( + MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + requireRustfsPayload(storedPayload, "上架结果分片必须写入 RustFS"); + + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setTaskId(taskId); + chunk.setModuleType(MODULE_TYPE); + chunk.setScopeKey(scopeKey); + chunk.setScopeHash(scopeHash); + chunk.setChunkIndex(chunkIndex); + chunk.setChunkTotal(chunkTotal); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(payloadHash); + chunk.setCreatedAt(LocalDateTime.now()); + chunk.setUpdatedAt(LocalDateTime.now()); + try { + taskChunkMapper.insert(chunk); + storedPayloads.add(storedPayload); + } catch (DuplicateKeyException ex) { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex); + if (winner == null) { + throw new BusinessException("上架结果分片并发写入失败，请重试"); + } + validateExistingChunk(winner, chunkTotal, payloadHash); + } catch (RuntimeException ex) { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + throw ex; + } + + int receivedChunkCount = countResultChunks(taskId, scopeHash); + persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount); + log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} received={}", + taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount); + return new ResultChunkReceipt(scopeHash, chunkTotal, + receivedChunkCount >= chunkTotal); + } + + private void validateChunkMetadata(int chunkIndex, int chunkTotal) { + if (chunkIndex <= 0) { + throw new BusinessException("chunk_index 必须从 1 开始"); + } + if (chunkTotal <= 0) { + throw new BusinessException("chunk_total 必须大于 0"); + } + if (chunkIndex > chunkTotal) { + throw new BusinessException("chunk_index 不能大于 chunk_total"); + } + } + + private TaskScopeStateEntity findResultScope(Long taskId, String scopeHash) { + return taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + } + + private TaskChunkEntity findResultChunk(Long taskId, String scopeHash, int chunkIndex) { + return taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + } + + private void validateExistingChunk(TaskChunkEntity existing, int chunkTotal, String payloadHash) { + validateChunkTotal(existing.getChunkTotal(), chunkTotal); + if (!Objects.equals(existing.getPayloadHash(), payloadHash)) { + throw new BusinessException("同一 chunk_index 已回传不同内容，拒绝覆盖"); + } + } + + private void validateChunkTotal(Integer existingChunkTotal, int chunkTotal) { + if (existingChunkTotal != null && existingChunkTotal > 0 && existingChunkTotal != chunkTotal) { + throw new BusinessException("同一文件的 chunk_total 必须保持一致"); + } + } + + private int countResultChunks(Long taskId, String scopeHash) { + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash)); + return count == null ? 0 : count.intValue(); + } + + private void persistResultScope(Long taskId, + String scopeKey, + String scopeHash, + int chunkTotal, + int receivedChunkCount) { + TaskScopeStateEntity scope = findResultScope(taskId, scopeHash); + validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal); + LocalDateTime now = LocalDateTime.now(); + if (scope == null) { + scope = new TaskScopeStateEntity(); + scope.setTaskId(taskId); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(scopeKey); + scope.setScopeHash(scopeHash); + scope.setCreatedAt(now); + } + boolean completed = receivedChunkCount >= chunkTotal; + scope.setChunkTotal(chunkTotal); + scope.setReceivedChunkCount(receivedChunkCount); + scope.setCompleted(completed ? 1 : 0); + scope.setLastChunkAt(now); + scope.setLastError(null); + scope.setStateJson(completed ? "{\"phase\":\"COMPLETE\"}" : "{\"phase\":\"RECEIVING\"}"); + scope.setUpdatedAt(now); + if (scope.getId() != null) { + taskScopeStateMapper.updateById(scope); + return; + } + try { + taskScopeStateMapper.insert(scope); + } catch (DuplicateKeyException ex) { + TaskScopeStateEntity winner = findResultScope(taskId, scopeHash); + if (winner == null) { + throw new BusinessException("上架结果分片状态写入失败，请重试"); + } + validateChunkTotal(winner.getChunkTotal(), chunkTotal); + winner.setChunkTotal(chunkTotal); + winner.setReceivedChunkCount(receivedChunkCount); + winner.setCompleted(completed ? 1 : 0); + winner.setLastChunkAt(now); + winner.setLastError(null); + winner.setStateJson(scope.getStateJson()); + winner.setUpdatedAt(now); + taskScopeStateMapper.updateById(winner); + } + } + + private List loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) { + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, receipt.scopeHash()) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + if (chunks == null || chunks.size() != receipt.chunkTotal()) { + throw new BusinessException("上架结果分片尚未完整，暂不能合并"); + } + + List rows = new ArrayList<>(); + TypeReference> listType = new TypeReference<>() { + }; + for (int i = 0; i < chunks.size(); i++) { + TaskChunkEntity chunk = chunks.get(i); + int expectedIndex = i + 1; + if (!Objects.equals(chunk.getChunkIndex(), expectedIndex)) { + throw new BusinessException("上架结果缺少第 " + expectedIndex + " 个分片"); + } + validateChunkTotal(chunk.getChunkTotal(), receipt.chunkTotal()); + List chunkRows = readResultChunkRows(chunk, listType); + for (PublishRowDto row : chunkRows) { + rows.add(copyRequiredRow(row)); + } + } + return rows; + } + + private List loadReceivedResultRows(Long taskId, Long fileId) { + String scopeHash = DigestUtil.sha256Hex("file:" + fileId); + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + if (chunks == null || chunks.isEmpty()) { + return List.of(); + } + + List rows = new ArrayList<>(); + TypeReference> listType = new TypeReference<>() { + }; + for (TaskChunkEntity chunk : chunks) { + try { + for (PublishRowDto row : readResultChunkRows(chunk, listType)) { + rows.add(copyRequiredRow(row)); + } + } catch (Exception ex) { + log.warn("[publish] skip unreadable stale result chunk taskId={} fileId={} chunkIndex={} msg={}", + taskId, fileId, chunk.getChunkIndex(), safeMessage(ex)); + } + } + return rows; + } + + private List readResultChunkRows(TaskChunkEntity chunk, + TypeReference> listType) { + int chunkIndex = chunk.getChunkIndex() == null ? 0 : chunk.getChunkIndex(); + String pointer = transientPayloadStorageService.extractPointer(chunk.getPayloadJson()); + if (pointer == null || !pointer.startsWith("rustfs:")) { + throw new BusinessException("上架结果分片不是 RustFS 数据，拒绝合并"); + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + chunk.getPayloadJson(), "读取上架结果分片失败"); + List rows = objectMapper.readValue(payloadJson, listType); + return rows == null ? List.of() : rows; + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("读取上架结果第 " + chunkIndex + " 个分片失败: " + + safeMessage(ex)); + } + } + + private void ensureRustfsPayloadStorageEnabled() { + if (!transientPayloadStorageService.isSharedWriteEnabled()) { + throw new BusinessException("RustFS 未配置，上架结果分片暂不可接收"); + } + } + + private void requireRustfsPayload(String storedPayload, String message) { + String pointer = transientPayloadStorageService.extractPointer(storedPayload); + if (pointer != null && pointer.startsWith("rustfs:")) { + return; + } + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + throw new BusinessException(message); + } + private List flattenRows(PublishResultFileDto incoming) { if (incoming.getRows() != null && !incoming.getRows().isEmpty()) { return incoming.getRows().stream().map(this::copyRequiredRow).toList(); @@ -1266,4 +1634,9 @@ public class PublishTaskService { private record TaskOptions(String publishCountry, List syncCountries) { } + + private record ResultChunkReceipt(String scopeHash, + int chunkTotal, + boolean completed) { + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java new file mode 100644 index 00000000..6d530edc --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java @@ -0,0 +1,308 @@ +package com.nanri.aiimage.modules.shopdatacrawl.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskBatchRequest; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.server.ResponseStatusException; + +import java.io.InputStream; +import java.net.URI; +import java.util.List; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/shop-data-crawl") +@Tag( + name = "店铺数据抓取", + description = "运营工具中的店铺数据抓取接口：管理待处理店铺、匹配店铺、保存五国处理顺序、创建并轮询串行任务、接收 Python 分片结果及下载最终 Excel。任务相关操作均以 taskId 为准。") +public class ShopDataCrawlTaskController { + private final ShopDataCrawlResolveService resolveService; + private final ShopDataCrawlTaskService taskService; + + @GetMapping("/candidates") + @Operation( + summary = "查询待处理店铺", + description = "返回当前用户已添加到店铺数据抓取管理区的店铺，按新增时间倒序排列。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功")) + public ApiResponse> candidates( + @Parameter(name = "user_id", description = "当前用户 ID，店铺数据按用户隔离", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId) { + return ApiResponse.success(resolveService.listCandidates(userId)); + } + + @PostMapping("/candidates") + @Operation( + summary = "添加待处理店铺", + description = "先校验店铺已在后台管理中维护且能够命中紫鸟店铺索引，再添加到当前用户的管理区；重复添加时返回已有记录。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "添加成功")) + public ApiResponse addCandidate( + @Valid @RequestBody ProductRiskCandidateAddRequest request) { + return ApiResponse.success(resolveService.addCandidate(request)); + } + + @DeleteMapping("/candidates/{id}") + @Operation( + summary = "删除待处理店铺", + description = "从当前用户的店铺数据抓取管理区删除指定记录，不会删除后台管理中的店铺。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功")) + public ApiResponse deleteCandidate( + @Parameter(description = "待处理店铺记录主键", required = true, example = "10") + @PathVariable Long id, + @Parameter(name = "user_id", description = "当前用户 ID，用于校验记录归属", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId) { + resolveService.deleteCandidate(userId, id); + return ApiResponse.success(null); + } + + @GetMapping("/country-preference") + @Operation( + summary = "查询国家处理顺序", + description = "返回当前用户勾选的国家代码，数组顺序就是抓取顺序。尚未保存时默认返回 UK、DE、FR、ES、IT。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功")) + public ApiResponse countryPreference( + @Parameter(name = "user_id", description = "当前用户 ID", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId) { + return ApiResponse.success(resolveService.getCountryPreference(userId)); + } + + @PutMapping("/country-preference") + @Operation( + summary = "保存国家处理顺序", + description = "保存当前用户勾选并排序后的国家列表。支持 DE、FR、ES、IT、UK，至少选择 1 个且不能重复。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "保存成功")) + public ApiResponse saveCountryPreference( + @Valid @RequestBody ProductRiskCountryPreferenceSaveRequest request) { + return ApiResponse.success(resolveService.saveCountryPreference(request)); + } + + @PostMapping("/match-shops") + @Operation( + summary = "批量匹配店铺", + description = "店铺名称规范化、去重后逐一匹配。先确认店铺已在后台管理中维护，再查询紫鸟店铺索引，返回店铺 ID、平台、公司名称和匹配状态。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "匹配完成")) + public ApiResponse matchShops( + @Valid @RequestBody ProductRiskMatchShopsRequest request) { + return ApiResponse.success(resolveService.matchShops(request)); + } + + @GetMapping("/dashboard") + @Operation( + summary = "查询任务统计", + description = "返回当前用户的待处理店铺数、已结束任务数、成功任务数和失败任务数。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功")) + public ApiResponse dashboard( + @Parameter(name = "user_id", description = "当前用户 ID", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId) { + return ApiResponse.success(taskService.dashboard(userId)); + } + + @GetMapping("/history") + @Operation( + summary = "查询抓取记录", + description = "返回当前用户最近 100 条店铺抓取结果，按创建时间倒序排列，包含执行状态和异步结果文件状态。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功")) + public ApiResponse history( + @Parameter(name = "user_id", description = "当前用户 ID", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId) { + return ApiResponse.success(taskService.listHistory(userId)); + } + + @PostMapping("/tasks/progress/batch") + @Operation( + summary = "批量查询任务进度", + description = "供前端轮询任务队列。对有效正数 taskId 去重后最多查询前 50 个；不存在、无结果或非本模块的任务写入 missingTaskIds。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功")) + public ApiResponse progress( + @Valid @RequestBody ShopDataCrawlTaskBatchRequest request) { + return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds())); + } + + @PostMapping("/tasks") + @Operation( + summary = "创建店铺数据抓取任务", + description = "每次必须且只能提交一个已匹配店铺，并按 country_codes 的顺序抓取。任务创建后状态为 RUNNING，返回 taskId 和该店铺的初始结果项。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功")) + public ApiResponse createTask( + @Valid @RequestBody ShopDataCrawlCreateTaskRequest request) { + return ApiResponse.success(taskService.createTask(request)); + } + + @PostMapping("/tasks/{taskId}/result") + @Operation( + summary = "回传店铺抓取结果", + description = "供 Python 按 taskId 回传，不依赖 user_id。每次 shops 必须且只能包含任务中的一个店铺。新协议要求同时传 chunk_index/chunk_total，分片从 1 开始、允许乱序和同内容重试，数据严格暂存到 RustFS，全部到齐后按序合并；旧客户端未传分片字段时仍按 shopDone 累加。error 非空时店铺立即失败；成功结束后异步生成 Excel。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "回传成功")) + public ApiResponse submitResult( + @Parameter(description = "店铺数据抓取任务 ID；服务端据此路由到任务归属实例，任务必须尚未结束", required = true, + example = "20363") + @PathVariable Long taskId, + @io.swagger.v3.oas.annotations.parameters.RequestBody( + description = "单个店铺的一次结果分片。chunk_index 与 chunk_total 必须同时传入。结果行依次包含日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价。", + required = true, + content = @Content( + mediaType = "application/json", + schema = @Schema(implementation = ShopDataCrawlSubmitResultRequest.class), + examples = { + @ExampleObject( + name = "中间分片", + summary = "继续等待后续分片", + value = """ + { + "shops": [ + { + "shopName": "示例店铺", + "chunk_index": 1, + "chunk_total": 2, + "countryResults": [ + { + "country": "DE", + "items": [ + { + "date": "2026-07-25", + "asin": "B0EXAMPLE1", + "inventorySales": "120", + "salesRank": "35", + "pageViews": "860", + "unitsSold": "18", + "price": "29.99", + "recommendedOffer": "27.99" + } + ] + } + ], + "submissionId": "shop-data-crawl:20363:part-1" + } + ] + } + """), + @ExampleObject( + name = "最终分片", + summary = "完成该店铺并触发异步生成 Excel", + value = """ + { + "shops": [ + { + "shopName": "示例店铺", + "chunk_index": 2, + "chunk_total": 2, + "countryResults": [ + { + "country": "UK", + "items": [] + } + ], + "submissionId": "shop-data-crawl:20363:final" + } + ] + } + """), + @ExampleObject( + name = "失败结果", + summary = "错误信息非空时直接将店铺标记为失败", + value = """ + { + "shops": [ + { + "shopName": "示例店铺", + "error": "紫鸟页面加载超时", + "submissionId": "shop-data-crawl:20363:error" + } + ] + } + """) + })) + @Valid @RequestBody ShopDataCrawlSubmitResultRequest request) { + taskService.submitResult(taskId, request); + return ApiResponse.success(null); + } + + @GetMapping("/results/{resultId}/download") + @Operation( + summary = "下载抓取结果 Excel", + description = "校验 resultId 属于当前 user_id 后，从对象存储读取最终 XLSX 并以附件返回。结果文件异步生成，只有列表中的 fileReady=true 后才能下载。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse( + responseCode = "200", + description = "店铺数据抓取结果 Excel 文件", + content = @Content( + mediaType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + schema = @Schema(type = "string", format = "binary")))) + public void download( + @Parameter(description = "结果记录主键", required = true, example = "1001") + @PathVariable Long resultId, + @Parameter(name = "user_id", description = "当前用户 ID，用于校验结果归属", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId, + jakarta.servlet.http.HttpServletResponse response) { + String url = taskService.resolveResultDownloadUrl(resultId, userId); + String filename = taskService.resolveResultDownloadFilename(resultId, userId); + try { + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + DownloadHeaderUtil.setAttachment(response, filename); + try (InputStream input = URI.create(url).toURL().openStream()) { + input.transferTo(response.getOutputStream()); + } + } catch (Exception ex) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败"); + } + } + + @DeleteMapping("/tasks/{taskId}") + @Operation( + summary = "删除整条抓取任务", + description = "校验任务归属后，删除任务、结果记录、文件生成作业、进度快照和不再被引用的结果文件。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功")) + public ApiResponse deleteTask( + @Parameter(description = "店铺数据抓取任务 ID", required = true, example = "20363") + @PathVariable Long taskId, + @Parameter(name = "user_id", description = "当前用户 ID，用于校验任务归属", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId) { + taskService.deleteTask(taskId, userId); + return ApiResponse.success(null); + } + + @DeleteMapping("/history/{resultId}") + @Operation( + summary = "删除单条抓取记录", + description = "校验结果归属后，删除指定结果及其文件作业和快照，并重新计算父任务状态；任务没有其他结果时会一并删除父任务。", + responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功")) + public ApiResponse deleteHistory( + @Parameter(description = "结果记录主键", required = true, example = "1001") + @PathVariable Long resultId, + @Parameter(name = "user_id", description = "当前用户 ID，用于校验结果归属", required = true, + in = ParameterIn.QUERY, example = "1") + @RequestParam("user_id") Long userId) { + taskService.deleteHistory(resultId, userId); + return ApiResponse.success(null); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlCountryPrefMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlCountryPrefMapper.java new file mode 100644 index 00000000..7e03ed1b --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlCountryPrefMapper.java @@ -0,0 +1,8 @@ +package com.nanri.aiimage.modules.shopdatacrawl.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface ShopDataCrawlCountryPrefMapper extends BaseMapper {} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlShopCandidateMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlShopCandidateMapper.java new file mode 100644 index 00000000..439acdb6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlShopCandidateMapper.java @@ -0,0 +1,10 @@ +package com.nanri.aiimage.modules.shopdatacrawl.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface ShopDataCrawlShopCandidateMapper extends BaseMapper { +} + diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCountryResultDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCountryResultDto.java new file mode 100644 index 00000000..d6eb2b03 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCountryResultDto.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "单个国家站点的抓取结果分片") +public class ShopDataCrawlCountryResultDto { + + @Schema( + description = "国家站点代码；生成结果文件时仅 DE、FR、ES、IT、UK 会写入对应工作表", + example = "UK", + allowableValues = {"DE", "FR", "ES", "IT", "UK"}) + private String country; + + @Schema(description = "该国家站点的抓取数据行；多次分片回传时后端会合并并去重") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCreateTaskRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCreateTaskRequest.java new file mode 100644 index 00000000..393f6962 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlCreateTaskRequest.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; +import lombok.Data; +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "创建店铺数据抓取任务请求；每个任务必须且只能处理一个已匹配店铺") +public class ShopDataCrawlCreateTaskRequest { + @NotNull + @JsonProperty("user_id") + @Schema(description = "任务所属用户 ID，必须为正数", example = "10001", requiredMode = Schema.RequiredMode.REQUIRED) + private Long userId; + + @Valid + @NotEmpty + @Size(min = 1, max = 1, message = "items 必须且只能包含一个店铺") + @Schema(description = "待抓取店铺列表，必须且只能包含一个有效且已匹配的店铺", requiredMode = Schema.RequiredMode.REQUIRED) + private List items = new ArrayList<>(); + + @NotEmpty + @JsonProperty("country_codes") + @Schema( + description = "本任务处理的国家代码，必须提交 1-5 个且不得重复，仅支持 DE、FR、ES、IT、UK；列表顺序就是处理顺序。用户未保存偏好时，偏好接口默认返回 UK、DE、FR、ES、IT，但创建任务仍需显式提交本字段", + example = "[\"UK\", \"DE\", \"FR\", \"ES\", \"IT\"]", + allowableValues = {"DE", "FR", "ES", "IT", "UK"}, + requiredMode = Schema.RequiredMode.REQUIRED) + private List countryCodes = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java new file mode 100644 index 00000000..d528ccd9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java @@ -0,0 +1,41 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "店铺数据抓取结果行；生成 Excel 时按日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价的固定列顺序写入") +public class ShopDataCrawlRowDto { + @JsonAlias("日期") + @Schema(description = "日期列，按来源文本原样保留", example = "2026-07-25") + private String date; + + @JsonAlias("ASIN") + @Schema(description = "亚马逊商品 ASIN", example = "B0CJ8SNXXV") + private String asin; + + @JsonAlias({"库存销量", "inventory_sales"}) + @Schema(description = "库存销量列，按来源文本原样保留", example = "128") + private String inventorySales; + + @JsonAlias({"销售排名", "sales_rank"}) + @Schema(description = "销售排名列，按来源文本原样保留", example = "#1,245") + private String salesRank; + + @JsonAlias({"页面浏览量", "page_views"}) + @Schema(description = "页面浏览量列，按来源文本原样保留", example = "3560") + private String pageViews; + + @JsonAlias({"售出件数", "units_sold"}) + @Schema(description = "售出件数列，按来源文本原样保留", example = "42") + private String unitsSold; + + @JsonAlias("价格") + @Schema(description = "价格列，含货币格式时按来源文本原样保留", example = "GBP 19.99") + private String price; + + @JsonAlias({"推荐报价", "recommended_offer"}) + @Schema(description = "推荐报价列，含货币格式时按来源文本原样保留", example = "GBP 18.99") + private String recommendedOffer; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlShopPayloadDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlShopPayloadDto.java new file mode 100644 index 00000000..63bb446b --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlShopPayloadDto.java @@ -0,0 +1,44 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "单个店铺的一次抓取结果分片；新客户端通过 chunk_index/chunk_total 回传，旧客户端仍可使用 shopDone 增量协议") +public class ShopDataCrawlShopPayloadDto { + + @JsonProperty("shopName") + @Schema(description = "店铺名称，必须与任务中的唯一店铺一致；为空时后端使用任务店铺名称", example = "示例英国店") + private String shopName; + + @Schema(description = "店铺处理错误信息；只要是非空文本，后端就会忽略 shopDone 并立即将店铺及任务标记失败", example = "紫鸟页面加载超时，未能完成数据抓取") + private String error; + + @JsonProperty("countryResults") + @Schema(description = "本分片的国家结果；显式分片暂存到 RustFS，全部到齐后按 chunk_index 顺序合并") + private List countryResults = new ArrayList<>(); + + @JsonProperty("chunk_index") + @JsonAlias("chunkIndex") + @Schema(description = "分片序号，从 1 开始；必须与 chunk_total 同时传入。不传时沿用旧版 shopDone 增量协议。", example = "1") + private Integer chunkIndex; + + @JsonProperty("chunk_total") + @JsonAlias("chunkTotal") + @Schema(description = "该店铺本次结果的分片总数；必须与 chunk_index 同时传入，且同一任务内保持一致。", example = "10") + private Integer chunkTotal; + + @JsonProperty("shopDone") + @Schema(description = "旧版增量协议的完成标志。显式传入 chunk_index/chunk_total 时，后端只按分片是否全部到齐判断完成，不会被 shopDone 提前结束。", example = "true") + private Boolean shopDone; + + @JsonProperty("submissionId") + @Schema(description = "本次店铺提交批次标识，便于问题排查与日志追踪；不参与业务去重", example = "shop-data-crawl:3089:示例英国店:1784941200000") + private String submissionId; + +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlSubmitResultRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlSubmitResultRequest.java new file mode 100644 index 00000000..6c72b9e6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlSubmitResultRequest.java @@ -0,0 +1,19 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "店铺数据抓取结果回传请求；支持同一店铺分多次回传，但每次请求只能包含一个店铺结果") +public class ShopDataCrawlSubmitResultRequest { + + @Valid + @NotEmpty + @Schema(description = "本次提交的店铺结果列表，必须且只能包含一个与任务匹配的店铺；中间分片和最后一片均遵循此限制", requiredMode = Schema.RequiredMode.REQUIRED) + private List shops = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskBatchRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskBatchRequest.java new file mode 100644 index 00000000..0569b7bb --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskBatchRequest.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "批量查询店铺数据抓取任务进度的请求") +public class ShopDataCrawlTaskBatchRequest { + + @NotEmpty(message = "taskIds 不能为空") + @Schema( + description = "待查询的任务 ID 列表；后端按输入顺序过滤 null、零和负数，去重后最多处理前 50 个有效 ID", + example = "[12001, 12002, 12001, -1]", + requiredMode = Schema.RequiredMode.REQUIRED) + private List taskIds = new ArrayList<>(); +} + diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskItemDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskItemDto.java new file mode 100644 index 00000000..d1d1e6ba --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlTaskItemDto.java @@ -0,0 +1,44 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "店铺数据抓取任务中的单店铺项；创建任务时后端会按店铺名称重新校验匹配信息") +public class ShopDataCrawlTaskItemDto { + @JsonProperty("shopName") + @Schema(description = "店铺名称，是创建任务时重新匹配紫鸟店铺的依据，不得为空", example = "示例英国店", requiredMode = Schema.RequiredMode.REQUIRED) + private String shopName; + + @Schema(description = "是否已命中紫鸟店铺索引；创建成功后由后端写为 true", example = "true") + private boolean matched; + + @JsonProperty("shopId") + @Schema(description = "紫鸟侧店铺 ID；创建任务时以后端重新匹配到的值为准", example = "3089") + private String shopId; + + @Schema(description = "店铺所属平台；创建任务时以后端重新匹配到的值为准", example = "Amazon") + private String platform; + + @JsonProperty("companyName") + @Schema(description = "店铺所属公司名称；创建任务时以后端重新匹配到的值为准", example = "示例跨境贸易有限公司") + private String companyName; + + @JsonProperty("matchStatus") + @Schema(description = "店铺匹配状态码；创建任务时以后端重新匹配结果为准", example = "MATCHED") + private String matchStatus; + + @JsonProperty("matchMessage") + @Schema(description = "店铺匹配状态说明；创建任务时以后端重新匹配结果为准", example = "店铺索引匹配成功") + private String matchMessage; + + @JsonProperty("countryCodes") + @Schema( + description = "该店铺实际处理的国家代码及顺序；创建任务后由顶层 country_codes 覆盖", + example = "[\"UK\", \"DE\"]", + allowableValues = {"DE", "FR", "ES", "IT", "UK"}) + private List countryCodes = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlCountryPrefEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlCountryPrefEntity.java new file mode 100644 index 00000000..febd8788 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlCountryPrefEntity.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import java.time.LocalDateTime; + +@Data +@TableName("biz_shop_data_crawl_country_pref") +public class ShopDataCrawlCountryPrefEntity { + @TableId(type = IdType.INPUT) private Long userId; + private String countryCodesJson; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlShopCandidateEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlShopCandidateEntity.java new file mode 100644 index 00000000..1c6f55fb --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlShopCandidateEntity.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_shop_data_crawl_shop_candidate") +public class ShopDataCrawlShopCandidateEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long userId; + private String shopName; + private LocalDateTime createdAt; +} + diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlCreateTaskVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlCreateTaskVo.java new file mode 100644 index 00000000..d11df6a7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlCreateTaskVo.java @@ -0,0 +1,19 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "店铺数据抓取任务创建结果") +public class ShopDataCrawlCreateTaskVo { + + @Schema(description = "新创建的任务 ID，用于轮询进度和回传抓取结果", example = "12001") + private Long taskId; + + @Schema(description = "任务创建后的单店铺初始快照；由于每个任务只允许一个店铺，本列表固定包含一项") + private List items = new ArrayList<>(); +} + diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlHistoryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlHistoryVo.java new file mode 100644 index 00000000..b40a5309 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlHistoryVo.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "当前用户的店铺数据抓取历史记录") +public class ShopDataCrawlHistoryVo { + + @Schema(description = "历史记录项，按创建时间倒序返回，最多返回最近 100 条") + private List items = new ArrayList<>(); +} + diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlResultItemVo.java new file mode 100644 index 00000000..5c078a77 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlResultItemVo.java @@ -0,0 +1,97 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.vo; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "单个店铺的数据抓取任务结果快照，同时用于任务创建、进度查询和历史记录") +public class ShopDataCrawlResultItemVo { + + @Schema(description = "结果记录 ID，用于删除历史记录和下载结果文件", example = "45001") + private Long resultId; + + @Schema(description = "所属店铺数据抓取任务 ID", example = "12001") + private Long taskId; + + @JsonProperty("shopName") + @Schema(description = "店铺名称", example = "示例英国店") + private String shopName; + + @JsonProperty("shopId") + @Schema(description = "紫鸟侧店铺 ID", example = "3089") + private String shopId; + + @Schema(description = "店铺所属平台", example = "Amazon") + private String platform; + + @JsonProperty("companyName") + @Schema(description = "店铺所属公司名称", example = "示例跨境贸易有限公司") + private String companyName; + + @Schema(description = "创建任务时是否成功命中紫鸟店铺索引", example = "true") + private boolean matched; + + @JsonProperty("matchStatus") + @Schema(description = "创建任务时保存的店铺匹配状态码", example = "MATCHED") + private String matchStatus; + + @JsonProperty("matchMessage") + @Schema(description = "创建任务时保存的店铺匹配说明", example = "店铺索引匹配成功") + private String matchMessage; + + @JsonProperty("taskStatus") + @Schema( + description = "任务状态：RUNNING=抓取或结果文件生成中，SUCCESS=任务成功且文件生成完成，FAILED=抓取或任务处理失败", + example = "RUNNING", + allowableValues = {"RUNNING", "SUCCESS", "FAILED"}) + private String taskStatus; + + @Schema(description = "店铺抓取结果：null=尚未完成，true=抓取成功，false=抓取失败；true 时结果文件仍可能在异步生成", example = "true") + private Boolean success; + + @Schema(description = "抓取或任务失败原因；无错误时为空", example = "紫鸟页面加载超时") + private String error; + + @Schema(description = "结果记录创建时间，ISO 本地日期时间", example = "2026-07-25T10:00:00") + private LocalDateTime createdAt; + + @Schema(description = "任务结束时间；任务仍在运行时为空，ISO 本地日期时间", example = "2026-07-25T10:05:00") + private LocalDateTime finishedAt; + + @Schema(description = "结果 Excel 文件名；文件异步生成期间可能已返回文件名，但不能据此判断是否可下载", example = "店铺数据抓取-示例英国店-12001.xlsx") + private String outputFilename; + + @Schema(description = "兼容保留的下载地址字段，当前接口通常返回空；fileReady=true 后应通过结果下载接口获取文件") + private String downloadUrl; + + @Schema(description = "异步结果文件生成任务 ID；尚未创建文件任务时为空", example = "88001") + private Long fileJobId; + + @Schema( + description = "异步结果文件生成状态；为空表示尚未创建文件任务，PENDING=等待生成，RUNNING=生成中，SUCCESS=生成成功，FAILED=生成失败", + example = "PENDING", + allowableValues = {"PENDING", "RUNNING", "SUCCESS", "FAILED"}) + private String fileStatus; + + @Schema(description = "异步结果文件生成失败原因；文件任务未失败时为空", example = "上传结果文件失败") + private String fileError; + + @Schema(description = "结果文件是否已生成并可下载；只有 true 时才能调用结果下载接口", example = "false") + private Boolean fileReady; + + @JsonProperty("countryCodes") + @Schema( + description = "任务实际处理的国家代码，顺序与创建任务时提交的 country_codes 一致", + example = "[\"UK\", \"DE\"]", + allowableValues = {"DE", "FR", "ES", "IT", "UK"}) + private List countryCodes = new ArrayList<>(); + + @JsonProperty("countryResults") + @Schema(description = "按国家累计合并后的抓取结果；中间分片阶段可能只包含部分国家或部分数据行") + private List countryResults = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlTaskBatchVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlTaskBatchVo.java new file mode 100644 index 00000000..fa62ff35 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlTaskBatchVo.java @@ -0,0 +1,19 @@ +package com.nanri.aiimage.modules.shopdatacrawl.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "店铺数据抓取任务的批量进度查询结果") +public class ShopDataCrawlTaskBatchVo { + + @Schema(description = "查询到的店铺结果快照，按规范化后的 taskIds 顺序返回；每个任务当前只包含一个店铺结果") + private List items = new ArrayList<>(); + + @Schema(description = "未找到任务或任务没有结果记录的任务 ID，顺序与规范化后的请求顺序一致", example = "[99999]") + private List missingTaskIds = new ArrayList<>(); +} + diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java new file mode 100644 index 00000000..5fb69d3f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java @@ -0,0 +1,110 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Service +public class ShopDataCrawlExcelAssemblyService { + static final List COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT"); + static final List SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利"); + static final List HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价"); + private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx"; + + public void writeWorkbook(File outputXlsx, List items) { + try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream(); + XSSFWorkbook workbook = new XSSFWorkbook(input); + FileOutputStream output = new FileOutputStream(outputXlsx)) { + validateTemplate(workbook); + Map> rowsByCountry = rowsByCountry(items); + for (int i = 0; i < COUNTRIES.size(); i++) { + writeSheet(workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i))); + } + workbook.write(output); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("生成店铺数据抓取 Excel 失败: " + ex.getMessage()); + } + } + + public int countRows(List items) { + return rowsByCountry(items).values().stream().mapToInt(List::size).sum(); + } + + void validateTemplate(XSSFWorkbook workbook) { + if (workbook.getNumberOfSheets() != SHEETS.size()) { + throw new BusinessException("店铺数据抓取模板工作表数量不正确"); + } + for (int i = 0; i < SHEETS.size(); i++) { + Sheet sheet = workbook.getSheetAt(i); + if (!SHEETS.get(i).equals(sheet.getSheetName())) { + throw new BusinessException("店铺数据抓取模板工作表顺序不正确"); + } + Row header = sheet.getRow(0); + for (int column = 0; column < HEADERS.size(); column++) { + String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim(); + if (!HEADERS.get(column).equals(actual)) { + throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName()); + } + } + } + } + + private void writeSheet(Sheet sheet, List rows) { + Row styleRow = sheet.getRow(1); + CellStyle[] styles = new CellStyle[HEADERS.size()]; + for (int column = 0; column < styles.length; column++) { + Cell cell = styleRow == null ? null : styleRow.getCell(column); + styles[column] = cell == null ? null : cell.getCellStyle(); + } + int last = sheet.getLastRowNum(); + for (int rowIndex = 1; rowIndex <= last; rowIndex++) { + Row row = sheet.getRow(rowIndex); + if (row != null) { + sheet.removeRow(row); + } + } + int rowIndex = 1; + for (ShopDataCrawlRowDto value : rows == null ? List.of() : rows) { + Row row = sheet.createRow(rowIndex++); + String[] values = {value.getDate(), value.getAsin(), value.getInventorySales(), value.getSalesRank(), + value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer()}; + for (int column = 0; column < values.length; column++) { + Cell cell = row.createCell(column); + if (styles[column] != null) cell.setCellStyle(styles[column]); + cell.setCellValue(values[column] == null ? "" : values[column]); + } + } + } + + private Map> rowsByCountry(List items) { + Map> result = new LinkedHashMap<>(); + COUNTRIES.forEach(country -> result.put(country, new ArrayList<>())); + if (items == null) return result; + for (ShopDataCrawlResultItemVo item : items) { + if (item == null || Boolean.FALSE.equals(item.getSuccess()) || item.getCountryResults() == null) continue; + for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) { + String country = countryResult == null || countryResult.getCountry() == null ? "" : countryResult.getCountry().trim().toUpperCase(); + if (result.containsKey(country) && countryResult.getItems() != null) result.get(country).addAll(countryResult.getItems()); + } + } + return result; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java new file mode 100644 index 00000000..b1b97cab --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java @@ -0,0 +1,313 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; +import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity; +import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity; +import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.modules.shopkey.service.ShopManageService; +import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; + +@Service +@RequiredArgsConstructor +public class ShopDataCrawlResolveService { + + /** + * 默认国家偏好顺序：英国 -> 德国 -> 法国 -> 西班牙 -> 意大利，与前端和结果模板保持一致。 + */ + public static final List DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("UK", "DE", "FR", "ES", "IT"); + + private final ShopDataCrawlShopCandidateMapper candidateMapper; + private final ShopDataCrawlCountryPrefMapper countryPrefMapper; + private final ObjectMapper objectMapper; + private final ZiniaoShopSwitchService ziniaoShopSwitchService; + private final ShopManageService shopManageService; + + public List listCandidates(Long userId) { + if (userId == null || userId <= 0) { + throw new BusinessException("user_id 不合法"); + } + List rows = candidateMapper.selectList( + new LambdaQueryWrapper() + .eq(ShopDataCrawlShopCandidateEntity::getUserId, userId) + .orderByDesc(ShopDataCrawlShopCandidateEntity::getId)); + List list = new ArrayList<>(); + for (ShopDataCrawlShopCandidateEntity row : rows) { + if (row == null) { + continue; + } + ProductRiskCandidateVo vo = new ProductRiskCandidateVo(); + vo.setId(row.getId()); + vo.setShopName(row.getShopName()); + vo.setCreatedAt(row.getCreatedAt()); + list.add(vo); + } + return list; + } + + public long countCandidates(Long userId) { + if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); + Long count = candidateMapper.selectCount(new LambdaQueryWrapper() + .eq(ShopDataCrawlShopCandidateEntity::getUserId, userId)); + return count == null ? 0L : count; + } + + @Transactional + public ProductRiskCandidateVo addCandidate(ProductRiskCandidateAddRequest request) { + if (request.getUserId() == null || request.getUserId() <= 0) { + throw new BusinessException("user_id 不合法"); + } + String normalized = ziniaoShopSwitchService.normalizeShopName(request.getShopName()); + if (normalized.isBlank()) { + throw new BusinessException("店铺名不能为空"); + } + shopManageService.requireShopByName(normalized); + ZiniaoShopMatchResultVo indexHit = ziniaoShopSwitchService.findIndexedStoreByName(normalized, false); + if (indexHit == null || !indexHit.isMatched()) { + String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank() + ? indexHit.getMatchMessage() + : "店铺索引未命中，无法加入待处理列表"; + throw new BusinessException(hint); + } + if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) { + throw new BusinessException(indexHit.getMatchMessage() != null ? indexHit.getMatchMessage() : "存在多个同名店铺，请人工确认"); + } + ShopDataCrawlShopCandidateEntity existing = candidateMapper.selectOne( + new LambdaQueryWrapper() + .eq(ShopDataCrawlShopCandidateEntity::getUserId, request.getUserId()) + .eq(ShopDataCrawlShopCandidateEntity::getShopName, normalized) + .last("LIMIT 1")); + if (existing != null) { + ProductRiskCandidateVo vo = new ProductRiskCandidateVo(); + vo.setId(existing.getId()); + vo.setShopName(existing.getShopName()); + vo.setCreatedAt(existing.getCreatedAt()); + return vo; + } + ShopDataCrawlShopCandidateEntity entity = new ShopDataCrawlShopCandidateEntity(); + entity.setUserId(request.getUserId()); + entity.setShopName(normalized); + entity.setCreatedAt(LocalDateTime.now()); + candidateMapper.insert(entity); + ProductRiskCandidateVo vo = new ProductRiskCandidateVo(); + vo.setId(entity.getId()); + vo.setShopName(entity.getShopName()); + vo.setCreatedAt(entity.getCreatedAt()); + return vo; + } + + @Transactional + public void deleteCandidate(Long userId, Long id) { + if (userId == null || userId <= 0) { + throw new BusinessException("user_id 不合法"); + } + if (id == null || id <= 0) { + throw new BusinessException("id 不合法"); + } + ShopDataCrawlShopCandidateEntity row = candidateMapper.selectById(id); + if (row == null || !userId.equals(row.getUserId())) { + throw new BusinessException("记录不存在"); + } + candidateMapper.deleteById(id); + } + + public ProductRiskMatchShopsVo matchShops(ProductRiskMatchShopsRequest request) { + if (request.getUserId() == null || request.getUserId() <= 0) { + throw new BusinessException("user_id 不合法"); + } + LinkedHashSet ordered = new LinkedHashSet<>(); + for (String raw : request.getShopNames()) { + String n = ziniaoShopSwitchService.normalizeShopName(raw); + if (!n.isBlank()) { + ordered.add(n); + } + } + if (ordered.isEmpty()) { + throw new BusinessException("shop_names 没有有效店铺名"); + } + ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo(); + for (String shopName : ordered) { + vo.getItems().add(matchOneShop(shopName)); + } + return vo; + } + + public ZiniaoShopMatchResultVo requireMatchedShop(String shopName) { + String normalized = ziniaoShopSwitchService.normalizeShopName(shopName); + shopManageService.requireShopByName(normalized); + ZiniaoShopMatchResultVo matched = ziniaoShopSwitchService.findIndexedStoreByName(normalized, false); + if (matched == null || !matched.isMatched()) { + throw new BusinessException(matched != null && matched.getMatchMessage() != null + ? matched.getMatchMessage() : "店铺索引未命中"); + } + if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(matched.getMatchStatus())) { + throw new BusinessException(matched.getMatchMessage() == null ? "存在多个同名店铺，请人工确认" : matched.getMatchMessage()); + } + return matched; + } + + public ProductRiskCountryPreferenceVo getCountryPreference(Long userId) { + if (userId == null || userId <= 0) { + throw new BusinessException("user_id 不合法"); + } + ShopDataCrawlCountryPrefEntity row = countryPrefMapper.selectById(userId); + ProductRiskCountryPreferenceVo vo = new ProductRiskCountryPreferenceVo(); + if (row == null || row.getCountryCodesJson() == null || row.getCountryCodesJson().isBlank()) { + vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER); + return vo; + } + try { + List parsed = objectMapper.readValue(row.getCountryCodesJson(), new TypeReference>() { + }); + vo.getCountryCodes().addAll(sanitizeStoredCodes(parsed)); + } catch (Exception ex) { + vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER); + } + return vo; + } + + @Transactional + public ProductRiskCountryPreferenceVo saveCountryPreference(ProductRiskCountryPreferenceSaveRequest request) { + if (request.getUserId() == null || request.getUserId() <= 0) { + throw new BusinessException("user_id 不合法"); + } + List normalized = validateCountryCodesForSave(request.getCountryCodes()); + String json; + try { + json = objectMapper.writeValueAsString(normalized); + } catch (Exception ex) { + throw new BusinessException("保存偏好失败"); + } + ShopDataCrawlCountryPrefEntity row = countryPrefMapper.selectById(request.getUserId()); + if (row == null) { + row = new ShopDataCrawlCountryPrefEntity(); + row.setUserId(request.getUserId()); + row.setCountryCodesJson(json); + countryPrefMapper.insert(row); + } else { + row.setCountryCodesJson(json); + countryPrefMapper.updateById(row); + } + ProductRiskCountryPreferenceVo vo = new ProductRiskCountryPreferenceVo(); + vo.getCountryCodes().addAll(normalized); + return vo; + } + + private static List sanitizeStoredCodes(List raw) { + List parsed = parseValidCountryCodes(raw); + if (parsed.isEmpty()) { + return new ArrayList<>(DEFAULT_COUNTRY_PREFERENCE_ORDER); + } + return parsed; + } + + private static List validateCountryCodesForSave(List raw) { + if (raw == null || raw.isEmpty()) { + throw new BusinessException("country_codes 至少选择 1 个国家"); + } + LinkedHashSet seen = new LinkedHashSet<>(); + List out = new ArrayList<>(); + for (String s : raw) { + if (s == null || s.isBlank()) { + throw new BusinessException("country_codes 含空项"); + } + String u = s.trim().toUpperCase(); + try { + ProductRiskCountryCode.valueOf(u); + } catch (IllegalArgumentException ex) { + throw new BusinessException("非法国家代码: " + s); + } + if (!seen.add(u)) { + throw new BusinessException("country_codes 存在重复: " + u); + } + out.add(u); + } + if (out.size() > ProductRiskCountryCode.values().length) { + throw new BusinessException("country_codes 最多 5 项"); + } + return out; + } + + public List validateCountryCodes(List raw) { + return validateCountryCodesForSave(raw); + } + + /** + * 仅保留合法国家代码，去重并保持顺序；非法值静默忽略，用于读取库存量数据。 + */ + private static List parseValidCountryCodes(List raw) { + if (raw == null || raw.isEmpty()) { + return new ArrayList<>(); + } + LinkedHashSet seen = new LinkedHashSet<>(); + List out = new ArrayList<>(); + for (String s : raw) { + if (s == null || s.isBlank()) { + continue; + } + String u = s.trim().toUpperCase(); + try { + ProductRiskCountryCode.valueOf(u); + } catch (IllegalArgumentException ex) { + continue; + } + if (seen.add(u)) { + out.add(u); + } + } + return out; + } + + private ProductRiskShopQueueItemVo matchOneShop(String shopName) { + ProductRiskShopQueueItemVo item = new ProductRiskShopQueueItemVo(); + item.setShopName(shopName); + try { + shopManageService.requireShopByName(shopName); + ZiniaoShopMatchResultVo m = ziniaoShopSwitchService.findIndexedStoreByName(shopName, false); + if (m == null) { + item.setMatched(false); + item.setMatchStatus("PENDING"); + item.setMatchMessage("匹配结果为空"); + return item; + } + item.setMatched(m.isMatched()); + item.setShopId(m.getShopId()); + item.setPlatform(m.getPlatform()); + item.setCompanyName(m.getCompanyName()); + item.setMatchedUserId(m.getMatchedUserId()); + item.setMatchStatus(m.getMatchStatus()); + item.setMatchMessage(m.getMatchMessage()); + } catch (BusinessException ex) { + item.setMatched(false); + item.setMatchStatus("PENDING"); + item.setMatchMessage(ex.getMessage()); + } catch (Exception ex) { + item.setMatched(false); + item.setMatchStatus("PENDING"); + item.setMatchMessage(Objects.toString(ex.getMessage(), "匹配异常")); + } + return item; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskCacheService.java new file mode 100644 index 00000000..dc4f558e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskCacheService.java @@ -0,0 +1,226 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.TaskPressureProperties; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ShopDataCrawlTaskCacheService { + + private static final String MODULE_TYPE = "SHOP_DATA_CRAWL"; + private static final long PAYLOAD_TTL_HOURS = 24; + private final StringRedisTemplate stringRedisTemplate; + private final ObjectMapper objectMapper; + private final TaskPressureProperties taskPressureProperties; + private final TaskScopePayloadStorageService taskScopePayloadStorageService; + private final ConcurrentHashMap taskEntityLocalCache = new ConcurrentHashMap<>(); + + public ShopDataCrawlShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) { + return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopDataCrawlShopPayloadDto.class); + } + + public void saveShopMergedPayload(Long taskId, String shopKey, ShopDataCrawlShopPayloadDto payload) { + if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) { + return; + } + taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload); + touchTaskHeartbeat(taskId); + } + + public void removeShopMergedPayload(Long taskId, String shopKey) { + taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey); + } + + public Map getAllShopMergedPayload(Long taskId) { + return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ShopDataCrawlShopPayloadDto.class); + } + + public boolean hasAnyShopMergedPayload(Long taskId) { + return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE); + } + + public long countShopMergedPayload(Long taskId) { + return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE); + } + + public long getTaskHeartbeatMillis(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0L; + } + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[shop-data-crawl-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } + if (raw == null || raw.isBlank()) { + return 0L; + } + try { + return Long.parseLong(raw); + } catch (NumberFormatException ignored) { + return 0L; + } + } + + public Map getTaskHeartbeatMillisBatch(List taskIds) { + Map result = new LinkedHashMap<>(); + if (taskIds == null || taskIds.isEmpty()) { + return result; + } + List normalized = taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalized.isEmpty()) { + return result; + } + List keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList(); + List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[shop-data-crawl-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage()); + return result; + } + for (int i = 0; i < normalized.size(); i++) { + String raw = values != null && i < values.size() ? values.get(i) : null; + if (raw == null || raw.isBlank()) { + result.put(normalized.get(i), 0L); + continue; + } + try { + result.put(normalized.get(i), Long.parseLong(raw)); + } catch (NumberFormatException ignored) { + result.put(normalized.get(i), 0L); + } + } + return result; + } + + public void touchTaskHeartbeat(Long taskId) { + try { + stringRedisTemplate.opsForValue().set( + buildTaskHeartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(PAYLOAD_TTL_HOURS)); + } catch (Exception ex) { + log.warn("[shop-data-crawl-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } + } + + public void deleteTaskCache(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + taskEntityLocalCache.remove(taskId); + try { + stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); + stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + } catch (Exception ex) { + log.warn("[shop-data-crawl-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage()); + } + taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); + } + + public void saveTaskCache(FileTaskEntity task) { + if (task == null || task.getId() == null) { + return; + } + long now = System.currentTimeMillis(); + taskEntityLocalCache.put(task.getId(), new LocalTaskEntityCacheEntry( + now, + objectMapper.convertValue(task, FileTaskEntity.class) + )); + try { + stringRedisTemplate.opsForValue().set( + buildTaskEntityKey(task.getId()), + objectMapper.writeValueAsString(task), + Duration.ofHours(PAYLOAD_TTL_HOURS) + ); + } catch (Exception ignored) { + } + } + + public Map getTaskCacheBatch(java.util.List taskIds) { + Map result = new LinkedHashMap<>(); + if (taskIds == null || taskIds.isEmpty()) { + return result; + } + java.util.List normalized = taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalized.isEmpty()) { + return result; + } + long now = System.currentTimeMillis(); + java.util.List missingIds = new ArrayList<>(); + for (Long taskId : normalized) { + LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId); + if (isLocalCacheFresh(cached, now)) { + result.put(taskId, objectMapper.convertValue(cached.task(), FileTaskEntity.class)); + } else { + missingIds.add(taskId); + } + } + if (missingIds.isEmpty()) { + return result; + } + java.util.List keys = missingIds.stream().map(this::buildTaskEntityKey).toList(); + java.util.List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[shop-data-crawl-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage()); + return result; + } + for (int i = 0; i < missingIds.size(); i++) { + Long taskId = missingIds.get(i); + String val = values != null && i < values.size() ? values.get(i) : null; + if (val == null || val.isBlank()) { + continue; + } + try { + FileTaskEntity task = objectMapper.readValue(val, FileTaskEntity.class); + result.put(taskId, task); + taskEntityLocalCache.put(taskId, new LocalTaskEntityCacheEntry(now, task)); + } catch (Exception ignored) { + } + } + return result; + } + + private String buildTaskHeartbeatKey(Long taskId) { + return "shop-data-crawl:task:heartbeat:" + taskId; + } + + private String buildTaskEntityKey(Long taskId) { + return "shop-data-crawl:task:entity:" + taskId; + } + + private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) { + return cached != null + && now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis()); + } + + private record LocalTaskEntityCacheEntry(long cachedAtMillis, FileTaskEntity task) {} +} + diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java new file mode 100644 index 00000000..24734653 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java @@ -0,0 +1,1686 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.crypto.digest.DigestUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.TaskPressureProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TaskResultItemService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; +import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.scheduling.annotation.Scheduled; + +import java.io.File; +import java.time.LocalDateTime; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ShopDataCrawlTaskService { + + private static final String MODULE_TYPE = "SHOP_DATA_CRAWL"; + private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final int RESULT_PENDING = -1; + private static final int RESULT_FAILED = 0; + private static final int RESULT_SUCCESS = 1; + private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断"; + private static final String PARTIAL_RESULT_MESSAGE = "Python 中断，已保留已回传的部分数据"; + private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:"; + + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final ShopDataCrawlResolveService shopDataCrawlResolveService; + private final ShopDataCrawlExcelAssemblyService excelAssemblyService; + private final ShopDataCrawlTaskCacheService taskCacheService; + private final OssStorageService ossStorageService; + private final ZiniaoShopSwitchService ziniaoShopSwitchService; + private final ObjectMapper objectMapper; + private final TaskPressureProperties taskPressureProperties; + private final TaskFileJobService taskFileJobService; + private final TaskResultItemService taskResultItemService; + private final TaskProgressSnapshotService taskProgressSnapshotService; + private final TaskDistributedLockService taskDistributedLockService; + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + private final InstanceMetadata instanceMetadata; + + @Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}") + private long staleTimeoutMinutes; + + @Scheduled(cron = "${aiimage.delete-brand-progress.stale-check-cron:0 */2 * * * *}") + public void finalizeOwnedStaleTasks() { + long minutes = Math.max(1L, staleTimeoutMinutes); + long nowMillis = System.currentTimeMillis(); + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, "RUNNING") + .apply("JSON_UNQUOTE(JSON_EXTRACT(request_json, '$.ownerInstanceId')) = {0}", currentInstanceId()) + .lt(FileTaskEntity::getUpdatedAt, LocalDateTime.now().minusMinutes(minutes)) + .last("limit 200")); + Map heartbeats = taskCacheService.getTaskHeartbeatMillisBatch( + tasks.stream().map(FileTaskEntity::getId).toList()); + for (FileTaskEntity task : tasks) { + long heartbeat = heartbeats.getOrDefault(task.getId(), 0L); + if (heartbeat > 0 && nowMillis - heartbeat < Duration.ofMinutes(minutes).toMillis()) continue; + try { + ensureTaskOwnedByCurrentInstance(task, "finalize stale shop data crawl task"); + if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) continue; + if (!tryFinalizeTask(task.getId(), true)) { + task.setStatus("FAILED"); + task.setErrorMessage("长时间未收到 Python 结果回传，任务已自动失败"); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + taskCacheService.deleteTaskCache(task.getId()); + cleanupResultChunksQuietly(task.getId(), "stale task failure"); + } + } catch (TaskOwnerMismatchException ignored) { + // The owner may change between the scan and finalization. + } catch (Exception ex) { + log.warn("[shop-data-crawl] stale task finalization failed taskId={} msg={}", task.getId(), ex.getMessage()); + } + } + } + + private FileTaskEntity loadTaskForExecution(Long taskId) { + Map cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); + FileTaskEntity cached = cachedTasks.get(taskId); + if (cached != null) { + return cached; + } + FileTaskEntity dbTask = fileTaskMapper.selectById(taskId); + if (dbTask != null && MODULE_TYPE.equals(dbTask.getModuleType())) { + taskCacheService.saveTaskCache(dbTask); + } + return dbTask; + } + + private Map loadTaskMapByIds(List taskIds) { + Map result = new LinkedHashMap<>(); + if (taskIds == null || taskIds.isEmpty()) { + return result; + } + List normalizedTaskIds = taskIds.stream() + .filter(taskId -> taskId != null && taskId > 0) + .distinct() + .toList(); + if (normalizedTaskIds.isEmpty()) { + return result; + } + Map cachedTasks = taskCacheService.getTaskCacheBatch(normalizedTaskIds); + result.putAll(cachedTasks); + List missingTaskIds = normalizedTaskIds.stream() + .filter(taskId -> !cachedTasks.containsKey(taskId)) + .toList(); + if (!missingTaskIds.isEmpty()) { + for (FileTaskEntity dbTask : selectTasksByIdsInBatches(missingTaskIds)) { + if (dbTask == null || !MODULE_TYPE.equals(dbTask.getModuleType())) { + continue; + } + result.put(dbTask.getId(), dbTask); + if ("RUNNING".equals(dbTask.getStatus())) { + taskCacheService.saveTaskCache(dbTask); + } + } + } + return result; + } + + private List selectTasksByIdsInBatches(List taskIds) { + List tasks = new ArrayList<>(); + if (taskIds == null || taskIds.isEmpty()) { + return tasks; + } + int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize()); + for (int start = 0; start < taskIds.size(); start += batchSize) { + int end = Math.min(start + batchSize, taskIds.size()); + tasks.addAll(fileTaskMapper.selectBatchIds(taskIds.subList(start, end))); + } + return tasks; + } + + public ProductRiskDashboardVo dashboard(Long userId) { + validateUserId(userId); + ProductRiskDashboardVo vo = new ProductRiskDashboardVo(); + vo.setCandidateCount(shopDataCrawlResolveService.countCandidates(userId)); + vo.setProcessedTaskCount(countTasks(userId, List.of("SUCCESS", "FAILED"))); + vo.setSuccessTaskCount(countTasks(userId, List.of("SUCCESS"))); + vo.setFailedTaskCount(countTasks(userId, List.of("FAILED"))); + return vo; + } + + public ShopDataCrawlHistoryVo listHistory(Long userId) { + long startedAt = System.nanoTime(); + validateUserId(userId); + ShopDataCrawlHistoryVo vo = new ShopDataCrawlHistoryVo(); + List entities = fileResultMapper.selectList(new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getModuleType, + FileResultEntity::getSourceFilename, + FileResultEntity::getSourceFileUrl, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getUserId, + FileResultEntity::getCreatedAt) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .eq(FileResultEntity::getUserId, userId) + .orderByDesc(FileResultEntity::getCreatedAt) + .last("limit 100")); + long resultRowsLoadedAt = System.nanoTime(); + if (entities.isEmpty()) { + vo.setItems(List.of()); + log.info("[shop-data-crawl] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0", + userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt)); + return vo; + } + + Map taskMap = loadHistoryTaskMap(entities); + long tasksLoadedAt = System.nanoTime(); + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList()); + long jobsLoadedAt = System.nanoTime(); + List items = new ArrayList<>(); + for (FileResultEntity entity : entities) { + FileTaskEntity task = taskMap.get(entity.getTaskId()); + items.add(toHistoryItem(entity, task, null, jobMap.get(entity.getId()))); + } + vo.setItems(items); + long finishedAt = System.nanoTime(); + log.info("[shop-data-crawl] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}", + userId, + entities.size(), + taskMap.size(), + jobMap.size(), + elapsedMs(startedAt, finishedAt), + elapsedMs(startedAt, resultRowsLoadedAt), + elapsedMs(resultRowsLoadedAt, tasksLoadedAt), + elapsedMs(tasksLoadedAt, jobsLoadedAt), + elapsedMs(jobsLoadedAt, finishedAt)); + return vo; + } + + public ShopDataCrawlTaskBatchVo getTaskProgressBatch(List taskIds) { + ShopDataCrawlTaskBatchVo batch = new ShopDataCrawlTaskBatchVo(); + if (taskIds == null || taskIds.isEmpty()) { + return batch; + } + + List normalizedTaskIds = taskIds.stream() + .filter(taskId -> taskId != null && taskId > 0) + .distinct() + .limit(50) + .toList(); + if (normalizedTaskIds.isEmpty()) { + return batch; + } + + Map taskMap = loadTaskMapByIds(normalizedTaskIds); + List rows = fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .in(FileResultEntity::getTaskId, normalizedTaskIds) + .orderByAsc(FileResultEntity::getId)); + Map> rowsByTaskId = new LinkedHashMap<>(); + for (FileResultEntity row : rows) { + rowsByTaskId.computeIfAbsent(row.getTaskId(), ignored -> new ArrayList<>()).add(row); + } + + for (Long taskId : normalizedTaskIds) { + FileTaskEntity task = taskMap.get(taskId); + if (task == null) { + batch.getMissingTaskIds().add(taskId); + continue; + } + List taskRows = rowsByTaskId.get(taskId); + if (taskRows == null || taskRows.isEmpty()) { + batch.getMissingTaskIds().add(taskId); + continue; + } + batch.getItems().addAll(buildProgressItems(task, taskRows)); + } + return batch; + } + + public String resolveResultDownloadUrl(Long resultId, Long userId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { + throw new BusinessException("记录不存在"); + } + if (blank(entity.getResultFileUrl())) { + throw new BusinessException("暂无可下载文件"); + } + return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()); + } + + public String resolveResultDownloadFilename(Long resultId, Long userId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { + throw new BusinessException("记录不存在"); + } + return !blank(entity.getResultFilename()) + ? entity.getResultFilename() + : safeFileStem(entity.getSourceFilename()) + ".xlsx"; + } + + @Transactional + public ShopDataCrawlCreateTaskVo createTask(ShopDataCrawlCreateTaskRequest request) { + validateUserId(request.getUserId()); + if (request.getItems() == null || request.getItems().size() != 1) { + throw new BusinessException("items 必须且只能包含一个店铺"); + } + List countryCodes = shopDataCrawlResolveService.validateCountryCodes(request.getCountryCodes()); + + List uniqueItems = dedupeItems(request.getItems()); + if (uniqueItems.size() != 1) { + throw new BusinessException("items 必须且只能包含一个有效店铺"); + } + for (ShopDataCrawlTaskItemDto item : uniqueItems) { + if (item == null) throw new BusinessException("店铺数据不能为空"); + ZiniaoShopMatchResultVo matched = shopDataCrawlResolveService.requireMatchedShop(item.getShopName()); + item.setShopName(ziniaoShopSwitchService.normalizeShopName(item.getShopName())); + item.setShopId(matched.getShopId()); + item.setPlatform(matched.getPlatform()); + item.setCompanyName(matched.getCompanyName()); + item.setMatchStatus(matched.getMatchStatus()); + item.setMatchMessage(matched.getMatchMessage()); + item.setMatched(true); + item.setCountryCodes(new ArrayList<>(countryCodes)); + } + + LocalDateTime now = LocalDateTime.now(); + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType(MODULE_TYPE); + task.setTaskMode("PYTHON_QUEUE"); + task.setStatus("RUNNING"); + task.setSourceFileCount(uniqueItems.size()); + task.setSuccessFileCount(0); + task.setFailedFileCount(0); + task.setCreatedBy("user:" + request.getUserId()); + task.setUserId(request.getUserId()); + task.setCreatedAt(now); + task.setUpdatedAt(now); + fileTaskMapper.insert(task); + taskCacheService.saveTaskCache(task); + taskCacheService.touchTaskHeartbeat(task.getId()); + + List snapshots = new ArrayList<>(); + for (ShopDataCrawlTaskItemDto item : uniqueItems) { + String normalizedShopName = ziniaoShopSwitchService.normalizeShopName(item.getShopName()); + if (blank(normalizedShopName)) { + throw new BusinessException("任务数据已失效，请刷新后重试"); + } + FileResultEntity result = new FileResultEntity(); + result.setTaskId(task.getId()); + result.setModuleType(MODULE_TYPE); + result.setSourceFilename(normalizedShopName); + result.setSourceFileUrl(item.getShopId()); + result.setUserId(request.getUserId()); + result.setSuccess(RESULT_PENDING); + result.setCreatedAt(now); + fileResultMapper.insert(result); + snapshots.add(toSnapshotVo(result, item, task.getStatus(), null)); + } + persistTaskJson(task, uniqueItems, countryCodes, snapshots); + taskCacheService.saveTaskCache(task); + + ShopDataCrawlCreateTaskVo vo = new ShopDataCrawlCreateTaskVo(); + vo.setTaskId(task.getId()); + vo.setItems(snapshots); + return vo; + } + + public void submitResult(Long taskId, ShopDataCrawlSubmitResultRequest request) { + if (taskId == null || taskId <= 0) { + throw new BusinessException("taskId 不合法"); + } + if (request == null || request.getShops() == null || request.getShops().size() != 1) { + throw new BusinessException("shops 必须且只能包含一个店铺结果"); + } + + try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + ensureTaskOwnedByCurrentInstance(task, "submit shop data crawl result"); + if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { + throw new BusinessException("任务已结束，拒绝重复提交"); + } + + taskCacheService.touchTaskHeartbeat(taskId); + List resultRows = listTaskRows(taskId); + List snapshots = buildSnapshotFromDb(task, resultRows); + Map snapshotByResultId = indexSnapshotByResultId(snapshots); + if (resultRows.size() != 1) throw new BusinessException("任务店铺数据异常"); + Map payloadByShop = normalizePayloadByShop( + request.getShops(), resultRows.get(0).getSourceFilename()); + + for (FileResultEntity row : resultRows) { + String shopKey = row.getSourceFilename(); + ShopDataCrawlShopPayloadDto incoming = payloadByShop.get(shopKey); + if (incoming == null) { + continue; + } + ShopDataCrawlResultItemVo snapshot = snapshotByResultId.get(row.getId()); + + if (!blank(incoming.getError())) { + mergePayloadIntoSnapshot(snapshot, incoming); + markResultFailed(row, incoming.getError()); + applyFailureToSnapshot(snapshot, incoming.getError()); + taskCacheService.removeShopMergedPayload(taskId, shopKey); + continue; + } + + ShopDataCrawlShopPayloadDto mergedPayload; + if (hasExplicitChunkMetadata(incoming)) { + ResultChunkReceipt receipt = persistResultChunk(taskId, shopKey, incoming); + if (!receipt.completed()) { + continue; + } + mergedPayload = loadCompleteResultPayload(taskId, shopKey, receipt); + } else { + mergedPayload = mergeShopPayload(taskId, shopKey, incoming); + } + mergePayloadIntoSnapshot(snapshot, mergedPayload); + + if (isShopPayloadComplete(mergedPayload)) { + markResultSuccess(row); + applySuccessToSnapshot(snapshot); + taskCacheService.removeShopMergedPayload(taskId, shopKey); + } + } + + persistSnapshotJson(task, snapshots); + fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); + tryFinalizeTask(taskId, false); + } + } + + public boolean tryFinalizeTask(Long taskId, boolean fromCompensation) { + if (taskId == null || taskId <= 0) { + return false; + } + TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId); + if (lockHandle == null) { + log.info("[shop-data-crawl] tryFinalizeTask skipped because task lock is busy taskId={} fromCompensation={}", taskId, fromCompensation); + return false; + } + try (lockHandle) { + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + return false; + } + ensureTaskOwnedByCurrentInstance(task, "finalize shop data crawl task"); + if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { + return true; + } + + List rows = listTaskRows(taskId); + if (rows.isEmpty()) { + return false; + } + + Map cachedPayloads = taskCacheService.getAllShopMergedPayload(taskId); + List snapshots = buildSnapshotFromDb(task, rows); + Map snapshotByResultId = indexSnapshotByResultId(snapshots); + boolean changed = false; + + for (FileResultEntity row : rows) { + if (isResultFinished(row)) { + continue; + } + ShopDataCrawlResultItemVo snapshot = snapshotByResultId.get(row.getId()); + ShopDataCrawlShopPayloadDto completedChunkPayload = loadCompletedResultPayloadIfPresent( + taskId, row.getSourceFilename()); + if (completedChunkPayload != null) { + mergePayloadIntoSnapshot(snapshot, completedChunkPayload); + markResultSuccess(row); + applySuccessToSnapshot(snapshot); + changed = true; + continue; + } + if (fromCompensation) { + ShopDataCrawlShopPayloadDto partialChunkPayload = loadReceivedResultPayloadIfPresent( + taskId, row.getSourceFilename()); + if (hasResultRows(partialChunkPayload)) { + mergePayloadIntoSnapshot(snapshot, partialChunkPayload); + markResultPartialSuccess(row); + applyPartialSuccessToSnapshot(snapshot); + changed = true; + continue; + } + } + ShopDataCrawlShopPayloadDto cached = cachedPayloads.get(row.getSourceFilename()); + if (cached == null) { + if (fromCompensation) { + markResultFailed(row, INTERRUPTED_MESSAGE); + applyFailureToSnapshot(snapshot, INTERRUPTED_MESSAGE); + changed = true; + } + continue; + } + + mergePayloadIntoSnapshot(snapshot, cached); + + if (!blank(cached.getError())) { + markResultFailed(row, cached.getError()); + applyFailureToSnapshot(snapshot, cached.getError()); + taskCacheService.removeShopMergedPayload(taskId, row.getSourceFilename()); + changed = true; + continue; + } + + if (isShopPayloadComplete(cached)) { + markResultSuccess(row); + applySuccessToSnapshot(snapshot); + taskCacheService.removeShopMergedPayload(taskId, row.getSourceFilename()); + changed = true; + continue; + } + + if (fromCompensation) { + if (hasResultRows(cached)) { + markResultPartialSuccess(row); + applyPartialSuccessToSnapshot(snapshot); + } else { + markResultFailed(row, INTERRUPTED_MESSAGE); + applyFailureToSnapshot(snapshot, INTERRUPTED_MESSAGE); + } + taskCacheService.removeShopMergedPayload(taskId, row.getSourceFilename()); + changed = true; + } + } + + List latestRows = listTaskRows(taskId); + updateTaskStatusFromRows(task, latestRows); + if (latestRows.stream().allMatch(this::isResultFinished)) { + persistSnapshotJson(task, snapshots); + fileTaskMapper.updateById(task); + finalizeTaskWorkbook(task, latestRows, snapshots); + return true; + } + + persistSnapshotJson(task, snapshots); + fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); + return changed; + } + } + + @Transactional + public void deleteTask(Long taskId, Long userId) { + validateUserId(userId); + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { + throw new BusinessException("任务不存在"); + } + ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task"); + try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { + List resultFileUrls = listTaskRows(taskId).stream() + .map(FileResultEntity::getResultFileUrl).filter(url -> !blank(url)).distinct().toList(); + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); + taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); + taskProgressSnapshotService.delete(taskId, MODULE_TYPE); + fileTaskMapper.deleteById(taskId); + taskCacheService.deleteTaskCache(taskId); + deleteTransientResultChunks(taskId); + resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced); + } + } + + @Transactional + public void deleteHistory(Long resultId, Long userId) { + validateUserId(userId); + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { + throw new BusinessException("记录不存在"); + } + Long taskId = entity.getTaskId(); + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) throw new BusinessException("任务不存在"); + ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl history"); + try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { + FileResultEntity latestEntity = fileResultMapper.selectById(resultId); + if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { + throw new BusinessException("记录不存在"); + } + taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId); + taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId); + String resultFileUrl = latestEntity.getResultFileUrl(); + fileResultMapper.deleteById(resultId); + reconcileTaskAfterResultRemoval(taskId); + deleteResultObjectIfUnreferenced(resultFileUrl); + } + } + + private void reconcileTaskAfterResultRemoval(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + return; + } + List rows = listTaskRows(taskId); + if (rows.isEmpty()) { + taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); + taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); + taskProgressSnapshotService.delete(taskId, MODULE_TYPE); + fileTaskMapper.deleteById(taskId); + taskCacheService.deleteTaskCache(taskId); + deleteTransientResultChunks(taskId); + return; + } + updateTaskStatusFromRows(task, rows); + persistSnapshotJson(task, buildSnapshotFromDb(task, rows)); + fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); + } + + private long countTasks(Long userId, List statuses) { + Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getUserId, userId) + .in(FileTaskEntity::getStatus, statuses)); + return count == null ? 0L : count; + } + + private List listTaskRows(Long taskId) { + return fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .orderByAsc(FileResultEntity::getId)); + } + + Map normalizePayloadByShop( + List shops, String expectedShopName) { + Map payloadByShop = new LinkedHashMap<>(); + for (ShopDataCrawlShopPayloadDto item : shops) { + if (item == null) throw new BusinessException("shops 包含空结果"); + String shopKey = ziniaoShopSwitchService.normalizeShopName(item.getShopName()); + if (blank(shopKey)) shopKey = expectedShopName; + if (!Objects.equals(expectedShopName, shopKey)) throw new BusinessException("回传店铺与任务不匹配"); + item.setShopName(shopKey); + payloadByShop.put(shopKey, item); + } + return payloadByShop; + } + + boolean isShopPayloadComplete(ShopDataCrawlShopPayloadDto payload) { + return payload != null && Boolean.TRUE.equals(payload.getShopDone()); + } + + private boolean hasExplicitChunkMetadata(ShopDataCrawlShopPayloadDto payload) { + boolean hasChunkIndex = payload != null && payload.getChunkIndex() != null; + boolean hasChunkTotal = payload != null && payload.getChunkTotal() != null; + if (hasChunkIndex != hasChunkTotal) { + throw new BusinessException("chunk_index 与 chunk_total 必须同时传入"); + } + return hasChunkIndex; + } + + private Map loadTaskMap(List entities) { + List taskIds = entities.stream() + .map(FileResultEntity::getTaskId) + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + Map taskMap = new LinkedHashMap<>(); + if (taskIds.isEmpty()) { + return taskMap; + } + for (FileTaskEntity task : selectTasksByIdsInBatches(taskIds)) { + if (task != null && MODULE_TYPE.equals(task.getModuleType())) { + taskMap.put(task.getId(), task); + } + } + return taskMap; + } + + private Map loadHistoryTaskMap(List entities) { + List taskIds = entities.stream() + .map(FileResultEntity::getTaskId) + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + Map taskMap = new LinkedHashMap<>(); + if (taskIds.isEmpty()) { + return taskMap; + } + int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize()); + for (int start = 0; start < taskIds.size(); start += batchSize) { + int end = Math.min(start + batchSize, taskIds.size()); + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getId, FileTaskEntity::getStatus, FileTaskEntity::getFinishedAt) + .in(FileTaskEntity::getId, taskIds.subList(start, end))); + for (FileTaskEntity task : tasks) { + if (task != null) { + taskMap.put(task.getId(), task); + } + } + } + return taskMap; + } + + private Map> buildSnapshotMap(Map taskMap) { + Map> out = new LinkedHashMap<>(); + for (Map.Entry entry : taskMap.entrySet()) { + List snapshots = parseTaskSnapshots(entry.getValue().getResultJson()); + if (snapshots.isEmpty()) { + snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, ShopDataCrawlResultItemVo.class); + } + out.put(entry.getKey(), indexSnapshotByResultId(snapshots)); + } + return out; + } + + private ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, ShopDataCrawlResultItemVo snapshot) { + return toHistoryItem(entity, task, snapshot, null); + } + + private ShopDataCrawlResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, ShopDataCrawlResultItemVo snapshot, TaskFileJobEntity job) { + ShopDataCrawlResultItemVo item = snapshot != null ? snapshot : new ShopDataCrawlResultItemVo(); + item.setResultId(entity.getId()); + item.setTaskId(entity.getTaskId()); + item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename())); + item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl())); + item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus()); + item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess())); + item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError()); + item.setCreatedAt(entity.getCreatedAt()); + item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt()); + item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename())); + item.setDownloadUrl(null); + attachFileJobState(item, entity, job); + if (item.getCountryResults() == null) { + item.setCountryResults(new ArrayList<>()); + } + if (item.getCountryCodes() == null) { + item.setCountryCodes(new ArrayList<>()); + } + return item; + } + + private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity) { + attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId())); + } + + private void attachFileJobState(ShopDataCrawlResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) { + item.setFileReady(!blank(entity.getResultFileUrl())); + if (job == null) { + item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); + return; + } + item.setFileJobId(job.getId()); + item.setFileStatus(job.getStatus()); + item.setFileError(job.getErrorMessage()); + } + + private List dedupeItems(List items) { + LinkedHashMap map = new LinkedHashMap<>(); + for (ShopDataCrawlTaskItemDto item : items) { + if (item == null) { + continue; + } + String normalizedShopName = ziniaoShopSwitchService.normalizeShopName(item.getShopName()); + if (blank(normalizedShopName)) { + continue; + } + item.setShopName(normalizedShopName); + String key = normalizedShopName + "::" + Objects.toString(item.getShopId(), ""); + map.put(key, item); + } + return new ArrayList<>(map.values()); + } + + private ShopDataCrawlResultItemVo toSnapshotVo(FileResultEntity result, ShopDataCrawlTaskItemDto item, String taskStatus, LocalDateTime finishedAt) { + ShopDataCrawlResultItemVo vo = new ShopDataCrawlResultItemVo(); + vo.setResultId(result.getId()); + vo.setTaskId(result.getTaskId()); + vo.setShopName(item.getShopName()); + vo.setShopId(item.getShopId()); + vo.setPlatform(item.getPlatform()); + vo.setCompanyName(item.getCompanyName()); + vo.setMatched(item.isMatched()); + vo.setMatchStatus(item.getMatchStatus()); + vo.setMatchMessage(item.getMatchMessage()); + vo.setTaskStatus(taskStatus); + vo.setSuccess(toSuccessFlag(result.getSuccess(), null)); + vo.setError(result.getErrorMessage()); + vo.setCreatedAt(result.getCreatedAt()); + vo.setFinishedAt(finishedAt); + vo.setCountryResults(new ArrayList<>()); + vo.setCountryCodes(item.getCountryCodes() == null ? new ArrayList<>() : new ArrayList<>(item.getCountryCodes())); + vo.setOutputFilename(result.getResultFilename()); + vo.setDownloadUrl(null); + return vo; + } + + private void persistTaskJson(FileTaskEntity task, List requestItems, + List countryCodes, List snapshots) { + try { + Map requestSnapshot = new LinkedHashMap<>(); + requestSnapshot.put("ownerInstanceId", currentInstanceId()); + requestSnapshot.put("items", requestItems); + requestSnapshot.put("countryCodes", countryCodes); + task.setRequestJson(objectMapper.writeValueAsString(requestSnapshot)); + task.setResultJson(objectMapper.writeValueAsString(snapshots)); + syncSnapshotTables(task, snapshots); + fileTaskMapper.updateById(task); + } catch (Exception ex) { + throw new BusinessException("店铺数据抓取任务快照保存失败"); + } + } + + private List buildSnapshotFromDb(FileTaskEntity task, List rows) { + List existing = parseTaskSnapshots(task.getResultJson()); + Map snapshotByResultId = indexSnapshotByResultId(existing); + List list = new ArrayList<>(); + for (FileResultEntity row : rows) { + list.add(toHistoryItem(row, task, snapshotByResultId.get(row.getId()))); + } + return list; + } + + private List buildProgressItems(FileTaskEntity task, List rows) { + List list = new ArrayList<>(); + for (FileResultEntity row : rows) { + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setResultId(row.getId()); + item.setTaskId(row.getTaskId()); + item.setShopName(row.getSourceFilename()); + item.setShopId(row.getSourceFileUrl()); + item.setTaskStatus(task == null ? null : task.getStatus()); + item.setSuccess(toSuccessFlag(row.getSuccess(), null)); + item.setError(row.getErrorMessage()); + item.setCreatedAt(row.getCreatedAt()); + item.setFinishedAt(task == null ? null : task.getFinishedAt()); + item.setOutputFilename(row.getResultFilename()); + item.setDownloadUrl(null); + attachFileJobState(item, row); + list.add(item); + } + return list; + } + + private Map indexSnapshotByResultId(List snapshots) { + Map map = new LinkedHashMap<>(); + if (snapshots == null) { + return map; + } + for (ShopDataCrawlResultItemVo snapshot : snapshots) { + if (snapshot != null && snapshot.getResultId() != null) { + map.put(snapshot.getResultId(), snapshot); + } + } + return map; + } + + private void updateTaskStatusFromRows(FileTaskEntity task, List rows) { + long successCount = rows.stream().filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())).count(); + long failedCount = rows.stream().filter(row -> Integer.valueOf(RESULT_FAILED).equals(row.getSuccess())).count(); + long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count(); + if (pendingCount == 0 && successCount > 0 && isTaskWorkbookPending(task, rows)) { + pendingCount = 1; + } + task.setSuccessFileCount((int) successCount); + task.setFailedFileCount((int) failedCount); + task.setUpdatedAt(LocalDateTime.now()); + if (pendingCount > 0) { + task.setStatus("RUNNING"); + task.setFinishedAt(null); + task.setErrorMessage(null); + return; + } + task.setStatus(failedCount > 0 ? "FAILED" : "SUCCESS"); + task.setFinishedAt(LocalDateTime.now()); + if (failedCount > 0) { + List errors = new ArrayList<>(); + for (FileResultEntity row : rows) { + if (!blank(row.getErrorMessage())) { + errors.add(row.getSourceFilename() + ": " + row.getErrorMessage()); + } + } + task.setErrorMessage(String.join("; ", errors)); + } else { + task.setErrorMessage(null); + } + } + + private ShopDataCrawlShopPayloadDto mergeShopPayload(Long taskId, String shopKey, ShopDataCrawlShopPayloadDto incoming) { + ShopDataCrawlShopPayloadDto merged = taskCacheService.getShopMergedPayload(taskId, shopKey); + if (merged == null) { + merged = new ShopDataCrawlShopPayloadDto(); + merged.setShopName(shopKey); + merged.setCountryResults(new ArrayList<>()); + } + + merged.setShopName(firstNonBlank(incoming.getShopName(), merged.getShopName())); + merged.setSubmissionId(firstNonBlank(incoming.getSubmissionId(), merged.getSubmissionId())); + + if (!blank(incoming.getError())) { + merged.setError(incoming.getError().trim()); + merged.setShopDone(Boolean.TRUE); + taskCacheService.saveShopMergedPayload(taskId, shopKey, merged); + return merged; + } + + merged.setCountryResults(mergeCountryResults(merged.getCountryResults(), incoming.getCountryResults())); + if (Boolean.TRUE.equals(incoming.getShopDone())) { + merged.setShopDone(Boolean.TRUE); + } + taskCacheService.saveShopMergedPayload(taskId, shopKey, merged); + return merged; + } + + private ResultChunkReceipt persistResultChunk(Long taskId, + String shopKey, + ShopDataCrawlShopPayloadDto incoming) { + int chunkIndex = incoming.getChunkIndex(); + int chunkTotal = incoming.getChunkTotal(); + validateChunkMetadata(chunkIndex, chunkTotal); + + String scopeKey = resultChunkScopeKey(shopKey); + String scopeHash = DigestUtil.sha256Hex(scopeKey); + TaskScopeStateEntity scope = findResultScope(taskId, scopeHash); + validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal); + + List countryResults = copyCountryResults(incoming.getCountryResults()); + String payloadJson = writeJson(countryResults, "序列化店铺数据抓取结果分片失败"); + String payloadHash = DigestUtil.sha256Hex(payloadJson); + TaskChunkEntity existing = findResultChunk(taskId, scopeHash, chunkIndex); + if (existing != null) { + validateExistingChunk(existing, chunkTotal, payloadHash); + int receivedChunkCount = countResultChunks(taskId, scopeHash); + persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount); + return new ResultChunkReceipt(scopeHash, chunkTotal, receivedChunkCount >= chunkTotal); + } + + ensureRustfsPayloadStorageEnabled(); + String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned( + MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + requireRustfsPayload(storedPayload, "店铺数据抓取结果分片必须写入 RustFS"); + + LocalDateTime now = LocalDateTime.now(); + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setTaskId(taskId); + chunk.setModuleType(MODULE_TYPE); + chunk.setScopeKey(scopeKey); + chunk.setScopeHash(scopeHash); + chunk.setChunkIndex(chunkIndex); + chunk.setChunkTotal(chunkTotal); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(payloadHash); + chunk.setCreatedAt(now); + chunk.setUpdatedAt(now); + try { + taskChunkMapper.insert(chunk); + } catch (DuplicateKeyException ex) { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex); + if (winner == null) { + throw new BusinessException("店铺数据抓取结果分片并发写入失败，请重试"); + } + validateExistingChunk(winner, chunkTotal, payloadHash); + } catch (RuntimeException ex) { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + throw ex; + } + + int receivedChunkCount = countResultChunks(taskId, scopeHash); + persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount); + log.info("[shop-data-crawl] result chunk received taskId={} shop={} chunk={}/{} received={}", + taskId, shopKey, chunkIndex, chunkTotal, receivedChunkCount); + return new ResultChunkReceipt(scopeHash, chunkTotal, receivedChunkCount >= chunkTotal); + } + + private void validateChunkMetadata(int chunkIndex, int chunkTotal) { + if (chunkIndex <= 0) { + throw new BusinessException("chunk_index 必须从 1 开始"); + } + if (chunkTotal <= 0) { + throw new BusinessException("chunk_total 必须大于 0"); + } + if (chunkIndex > chunkTotal) { + throw new BusinessException("chunk_index 不能大于 chunk_total"); + } + } + + private TaskScopeStateEntity findResultScope(Long taskId, String scopeHash) { + return taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + } + + private TaskChunkEntity findResultChunk(Long taskId, String scopeHash, int chunkIndex) { + return taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + } + + private void validateExistingChunk(TaskChunkEntity existing, int chunkTotal, String payloadHash) { + validateChunkTotal(existing.getChunkTotal(), chunkTotal); + if (!Objects.equals(existing.getPayloadHash(), payloadHash)) { + throw new BusinessException("同一 chunk_index 已回传不同内容，拒绝覆盖"); + } + } + + private void validateChunkTotal(Integer existingChunkTotal, int chunkTotal) { + if (existingChunkTotal != null && existingChunkTotal > 0 && existingChunkTotal != chunkTotal) { + throw new BusinessException("同一店铺的 chunk_total 必须保持一致"); + } + } + + private int countResultChunks(Long taskId, String scopeHash) { + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash)); + return count == null ? 0 : count.intValue(); + } + + private void persistResultScope(Long taskId, + String scopeKey, + String scopeHash, + int chunkTotal, + int receivedChunkCount) { + TaskScopeStateEntity scope = findResultScope(taskId, scopeHash); + validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal); + LocalDateTime now = LocalDateTime.now(); + if (scope == null) { + scope = new TaskScopeStateEntity(); + scope.setTaskId(taskId); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(scopeKey); + scope.setScopeHash(scopeHash); + scope.setCreatedAt(now); + } + boolean completed = receivedChunkCount >= chunkTotal; + scope.setChunkTotal(chunkTotal); + scope.setReceivedChunkCount(receivedChunkCount); + scope.setCompleted(completed ? 1 : 0); + scope.setLastChunkAt(now); + scope.setLastError(null); + scope.setUpdatedAt(now); + if (scope.getId() != null) { + taskScopeStateMapper.updateById(scope); + return; + } + try { + taskScopeStateMapper.insert(scope); + } catch (DuplicateKeyException ex) { + TaskScopeStateEntity winner = findResultScope(taskId, scopeHash); + if (winner == null) { + throw new BusinessException("店铺数据抓取结果分片状态写入失败，请重试"); + } + validateChunkTotal(winner.getChunkTotal(), chunkTotal); + winner.setChunkTotal(chunkTotal); + winner.setReceivedChunkCount(receivedChunkCount); + winner.setCompleted(completed ? 1 : 0); + winner.setLastChunkAt(now); + winner.setLastError(null); + winner.setUpdatedAt(now); + taskScopeStateMapper.updateById(winner); + } + } + + private ShopDataCrawlShopPayloadDto loadCompleteResultPayload(Long taskId, + String shopKey, + ResultChunkReceipt receipt) { + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, receipt.scopeHash()) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + if (chunks == null || chunks.size() != receipt.chunkTotal()) { + throw new BusinessException("店铺数据抓取结果分片尚未完整，暂不能合并"); + } + + List mergedResults = new ArrayList<>(); + TypeReference> resultType = new TypeReference<>() {}; + for (int i = 0; i < chunks.size(); i++) { + TaskChunkEntity chunk = chunks.get(i); + int expectedIndex = i + 1; + if (!Objects.equals(chunk.getChunkIndex(), expectedIndex)) { + throw new BusinessException("店铺数据抓取结果缺少第 " + expectedIndex + " 个分片"); + } + validateChunkTotal(chunk.getChunkTotal(), receipt.chunkTotal()); + mergedResults = mergeCountryResults(mergedResults, readResultChunk(chunk, resultType)); + } + + ShopDataCrawlShopPayloadDto merged = new ShopDataCrawlShopPayloadDto(); + merged.setShopName(shopKey); + merged.setCountryResults(mergedResults); + merged.setShopDone(Boolean.TRUE); + return merged; + } + + private ShopDataCrawlShopPayloadDto loadCompletedResultPayloadIfPresent(Long taskId, String shopKey) { + String scopeHash = DigestUtil.sha256Hex(resultChunkScopeKey(shopKey)); + TaskScopeStateEntity scope = findResultScope(taskId, scopeHash); + if (scope == null || !Integer.valueOf(1).equals(scope.getCompleted()) + || scope.getChunkTotal() == null || scope.getChunkTotal() <= 0) { + return null; + } + return loadCompleteResultPayload(taskId, shopKey, + new ResultChunkReceipt(scopeHash, scope.getChunkTotal(), true)); + } + + private ShopDataCrawlShopPayloadDto loadReceivedResultPayloadIfPresent(Long taskId, String shopKey) { + String scopeHash = DigestUtil.sha256Hex(resultChunkScopeKey(shopKey)); + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + if (chunks == null || chunks.isEmpty()) { + return null; + } + + List mergedResults = new ArrayList<>(); + TypeReference> resultType = new TypeReference<>() {}; + for (TaskChunkEntity chunk : chunks) { + try { + mergedResults = mergeCountryResults(mergedResults, readResultChunk(chunk, resultType)); + } catch (Exception ex) { + log.warn("[shop-data-crawl] skip unreadable stale result chunk taskId={} shop={} chunkIndex={} msg={}", + taskId, shopKey, chunk.getChunkIndex(), safeMessage(ex)); + } + } + if (!hasResultRows(mergedResults)) { + return null; + } + + ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto(); + payload.setShopName(shopKey); + payload.setCountryResults(mergedResults); + payload.setShopDone(Boolean.TRUE); + return payload; + } + + private List readResultChunk( + TaskChunkEntity chunk, + TypeReference> resultType) { + int chunkIndex = chunk.getChunkIndex() == null ? 0 : chunk.getChunkIndex(); + String pointer = transientPayloadStorageService.extractPointer(chunk.getPayloadJson()); + if (pointer == null || !pointer.startsWith("rustfs:")) { + throw new BusinessException("店铺数据抓取结果分片不是 RustFS 数据，拒绝合并"); + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + chunk.getPayloadJson(), "读取店铺数据抓取结果分片失败"); + List rows = objectMapper.readValue(payloadJson, resultType); + return rows == null ? List.of() : rows; + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("读取店铺数据抓取结果第 " + chunkIndex + " 个分片失败: " + + safeMessage(ex)); + } + } + + private void ensureRustfsPayloadStorageEnabled() { + if (!transientPayloadStorageService.isSharedWriteEnabled()) { + throw new BusinessException("RustFS 未配置，店铺数据抓取结果分片暂不可接收"); + } + } + + private void requireRustfsPayload(String storedPayload, String message) { + String pointer = transientPayloadStorageService.extractPointer(storedPayload); + if (pointer != null && pointer.startsWith("rustfs:")) { + return; + } + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + throw new BusinessException(message); + } + + private String resultChunkScopeKey(String shopKey) { + return RESULT_CHUNK_SCOPE_PREFIX + trim(shopKey); + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } + + private String safeMessage(Exception ex) { + return ex == null || blank(ex.getMessage()) ? ex == null ? "未知错误" : ex.getClass().getSimpleName() : ex.getMessage(); + } + + private void mergePayloadIntoSnapshot(ShopDataCrawlResultItemVo snapshot, ShopDataCrawlShopPayloadDto payload) { + if (snapshot == null || payload == null) { + return; + } + snapshot.setShopName(firstNonBlank(payload.getShopName(), snapshot.getShopName())); + snapshot.setCountryResults(copyCountryResults(payload.getCountryResults())); + if (!blank(payload.getError())) { + snapshot.setError(payload.getError().trim()); + } + } + + private void applySuccessToSnapshot(ShopDataCrawlResultItemVo snapshot) { + if (snapshot == null) { + return; + } + snapshot.setSuccess(Boolean.TRUE); + snapshot.setError(null); + } + + private void applyPartialSuccessToSnapshot(ShopDataCrawlResultItemVo snapshot) { + if (snapshot == null) { + return; + } + snapshot.setSuccess(Boolean.TRUE); + snapshot.setError(PARTIAL_RESULT_MESSAGE); + } + + private void applyFailureToSnapshot(ShopDataCrawlResultItemVo snapshot, String error) { + if (snapshot == null) { + return; + } + snapshot.setSuccess(Boolean.FALSE); + snapshot.setError(blankToNull(error)); + } + + private List mergeCountryResults(List base, List incoming) { + Map map = new LinkedHashMap<>(); + for (ShopDataCrawlCountryResultDto item : copyCountryResults(base)) { + map.put(item.getCountry(), item); + } + for (ShopDataCrawlCountryResultDto item : copyCountryResults(incoming)) { + if (blank(item.getCountry())) { + continue; + } + ShopDataCrawlCountryResultDto existing = map.get(item.getCountry()); + if (existing == null) { + map.put(item.getCountry(), item); + continue; + } + List merged = new ArrayList<>(existing.getItems() == null ? List.of() : existing.getItems()); + for (ShopDataCrawlRowDto row : item.getItems() == null ? List.of() : item.getItems()) { + if (row != null && merged.stream().noneMatch(old -> sameRow(old, row))) merged.add(copyRow(row)); + } + existing.setItems(merged); + } + return new ArrayList<>(map.values()); + } + + private boolean hasResultRows(ShopDataCrawlShopPayloadDto payload) { + return payload != null && hasResultRows(payload.getCountryResults()); + } + + private boolean hasResultRows(List countryResults) { + if (countryResults == null) { + return false; + } + return countryResults.stream() + .filter(Objects::nonNull) + .anyMatch(country -> country.getItems() != null + && country.getItems().stream().anyMatch(Objects::nonNull)); + } + + private void finalizeTaskWorkbook(FileTaskEntity task, List rows, List snapshots) { + List successItems = snapshots.stream() + .filter(item -> Boolean.TRUE.equals(item.getSuccess())) + .toList(); + if (!successItems.isEmpty()) { + String filename = buildTaskWorkbookFilename(task); + int rowCount = excelAssemblyService.countRows(successItems); + FileResultEntity firstSuccessRow = null; + for (FileResultEntity row : rows) { + if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { + row.setResultFilename(filename); + row.setResultFileUrl(null); + row.setResultFileSize(0L); + row.setResultContentType(CONTENT_TYPE_XLSX); + row.setRowCount(rowCount); + fileResultMapper.updateById(row); + if (firstSuccessRow == null) { + firstSuccessRow = row; + } + } + } + if (firstSuccessRow != null) { + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), ownerScopeKey(task.getId())); + snapshots = buildSnapshotFromDb(task, rows); + updateTaskStatusFromRows(task, rows); + } + } + + persistSnapshotJson(task, snapshots); + fileTaskMapper.updateById(task); + taskCacheService.deleteTaskCache(task.getId()); + if (successItems.isEmpty()) { + cleanupResultChunksQuietly(task.getId(), "terminal task without workbook"); + } + } + + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + ensureTaskOwnedByCurrentInstance(task, "assemble shop data crawl result"); + List rows = listTaskRows(job.getTaskId()); + List snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, ShopDataCrawlResultItemVo.class); + if (snapshots.isEmpty()) { + snapshots = buildSnapshotFromDb(task, rows); + } + List successItems = snapshots.stream() + .filter(item -> Boolean.TRUE.equals(item.getSuccess())) + .toList(); + if (successItems.isEmpty()) { + throw new BusinessException("没有可生成的店铺数据抓取结果"); + } + File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "shop-data-crawl-result", String.valueOf(task.getId()))); + String filename = buildTaskWorkbookFilename(task); + File xlsx = FileUtil.file(workRoot, filename); + try { + excelAssemblyService.writeWorkbook(xlsx, successItems); + String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); + long fileSize = xlsx.length(); + int rowCount = excelAssemblyService.countRows(successItems); + for (FileResultEntity row : rows) { + if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { + row.setResultFilename(filename); + row.setResultFileUrl(objectKey); + row.setResultFileSize(fileSize); + row.setResultContentType(CONTENT_TYPE_XLSX); + row.setRowCount(rowCount); + fileResultMapper.updateById(row); + } + } + updateTaskStatusFromRows(task, rows); + persistSnapshotJson(task, buildSnapshotFromDb(task, rows)); + fileTaskMapper.updateById(task); + } finally { + FileUtil.del(xlsx); + } + } + + public void cleanupResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + return; + } + deleteTransientResultChunks(job.getTaskId()); + } + + private void deleteTransientResultChunks(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); + } + } + taskChunkMapper.delete(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskScopeStateMapper.delete(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .likeRight(TaskScopeStateEntity::getScopeKey, RESULT_CHUNK_SCOPE_PREFIX)); + } + + private void cleanupResultChunksQuietly(Long taskId, String reason) { + try { + deleteTransientResultChunks(taskId); + } catch (Exception ex) { + log.warn("[shop-data-crawl] result chunk cleanup failed taskId={} reason={} msg={}", + taskId, reason, safeMessage(ex)); + } + } + + private boolean isTaskWorkbookPending(FileTaskEntity task, List rows) { + if (task == null || task.getId() == null || rows == null || rows.isEmpty()) { + return false; + } + FileResultEntity firstSuccess = rows.stream() + .filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) + .findFirst() + .orElse(null); + if (firstSuccess == null) { + return false; + } + if (!blank(firstSuccess.getResultFileUrl())) { + return false; + } + TaskFileJobEntity job = taskFileJobService.findAssembleJob(task.getId(), MODULE_TYPE, firstSuccess.getId()); + return job != null && !"SUCCESS".equals(job.getStatus()); + } + + private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) { + TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId); + if (lockHandle == null) { + throw new BusinessException(40901, "任务正在处理中，请稍后再试"); + } + return lockHandle; + } + + private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) { + if (taskId == null || taskId <= 0) { + return null; + } + return taskDistributedLockService.acquire(MODULE_TYPE, taskId); + } + + private void markResultSuccess(FileResultEntity row) { + row.setSuccess(RESULT_SUCCESS); + row.setErrorMessage(null); + fileResultMapper.updateById(row); + } + + private void markResultPartialSuccess(FileResultEntity row) { + row.setSuccess(RESULT_SUCCESS); + row.setErrorMessage(PARTIAL_RESULT_MESSAGE); + fileResultMapper.updateById(row); + } + + private void markResultFailed(FileResultEntity row, String message) { + row.setSuccess(RESULT_FAILED); + row.setErrorMessage(blankToNull(message)); + row.setResultFilename(null); + row.setResultFileUrl(null); + row.setResultFileSize(0L); + row.setResultContentType(null); + row.setRowCount(0); + fileResultMapper.updateById(row); + } + + private boolean isResultFinished(FileResultEntity row) { + return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess()); + } + + private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) { + if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) { + return fallback; + } + return Integer.valueOf(RESULT_SUCCESS).equals(dbValue); + } + + private List parseTaskSnapshots(String json) { + if (blank(json)) { + return new ArrayList<>(); + } + try { + return objectMapper.readValue(json, new TypeReference>() {}); + } catch (Exception ex) { + log.warn("[shop-data-crawl] parse task snapshot failed: {}", ex.getMessage()); + return new ArrayList<>(); + } + } + + private void persistSnapshotJson(FileTaskEntity task, List snapshots) { + try { + task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots)); + syncSnapshotTables(task, snapshots); + } catch (Exception ex) { + throw new BusinessException("店铺数据抓取任务快照保存失败"); + } + } + + private void syncSnapshotTables(FileTaskEntity task, List snapshots) { + List safe = snapshots == null ? List.of() : snapshots; + taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() { + @Override + public Long resultId(Object snapshot) { + return ((ShopDataCrawlResultItemVo) snapshot).getResultId(); + } + + @Override + public String scopeKey(Object snapshot) { + ShopDataCrawlResultItemVo item = (ShopDataCrawlResultItemVo) snapshot; + return firstNonBlank(item.getShopName(), "result:" + item.getResultId()); + } + }); + int successCount = 0; + int failedCount = 0; + for (ShopDataCrawlResultItemVo item : safe) { + if (Boolean.TRUE.equals(item.getSuccess())) { + successCount++; + } else if (Boolean.FALSE.equals(item.getSuccess())) { + failedCount++; + } + } + taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"), + safe.size(), successCount, failedCount, null, task.getErrorMessage(), null); + } + + private List copyCountryResults(List results) { + List copy = new ArrayList<>(); + if (results == null) { + return copy; + } + for (ShopDataCrawlCountryResultDto source : results) { + if (source == null || blank(source.getCountry())) { + continue; + } + ShopDataCrawlCountryResultDto item = new ShopDataCrawlCountryResultDto(); + item.setCountry(normalizeCountry(source.getCountry())); + List rows = new ArrayList<>(); + if (source.getItems() != null) { + for (ShopDataCrawlRowDto sourceRow : source.getItems()) { + if (sourceRow != null && !rowEmpty(sourceRow) && rows.stream().noneMatch(old -> sameRow(old, sourceRow))) { + rows.add(copyRow(sourceRow)); + } + } + } + item.setItems(rows); + copy.add(item); + } + return copy; + } + + private ShopDataCrawlRowDto copyRow(ShopDataCrawlRowDto source) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate(trim(source.getDate())); + row.setAsin(trim(source.getAsin())); + row.setInventorySales(trim(source.getInventorySales())); + row.setSalesRank(trim(source.getSalesRank())); + row.setPageViews(trim(source.getPageViews())); + row.setUnitsSold(trim(source.getUnitsSold())); + row.setPrice(trim(source.getPrice())); + row.setRecommendedOffer(trim(source.getRecommendedOffer())); + return row; + } + + public void handleResultFileJobFailure(TaskFileJobEntity job, String message) { + if (job == null || job.getTaskId() == null) return; + try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(job.getTaskId())) { + FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) return; + ensureTaskOwnedByCurrentInstance(task, "fail shop data crawl result file job"); + FileResultEntity row = job.getResultId() == null ? null : fileResultMapper.selectById(job.getResultId()); + if (row != null && MODULE_TYPE.equals(row.getModuleType())) markResultFailed(row, message); + List rows = listTaskRows(job.getTaskId()); + List snapshots = buildSnapshotFromDb(task, rows); + for (ShopDataCrawlResultItemVo snapshot : snapshots) { + if (Objects.equals(snapshot.getResultId(), job.getResultId())) applyFailureToSnapshot(snapshot, message); + } + updateTaskStatusFromRows(task, rows); + task.setStatus("FAILED"); + task.setErrorMessage(firstNonBlank(message, "结果文件生成失败")); + task.setFinishedAt(LocalDateTime.now()); + persistSnapshotJson(task, snapshots); + fileTaskMapper.updateById(task); + taskCacheService.deleteTaskCache(task.getId()); + cleanupResultChunksQuietly(task.getId(), "result file job exhausted"); + } + } + + private boolean sameRow(ShopDataCrawlRowDto left, ShopDataCrawlRowDto right) { + return left != null && right != null + && Objects.equals(trim(left.getDate()), trim(right.getDate())) + && Objects.equals(trim(left.getAsin()), trim(right.getAsin())) + && Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales())) + && Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank())) + && Objects.equals(trim(left.getPageViews()), trim(right.getPageViews())) + && Objects.equals(trim(left.getUnitsSold()), trim(right.getUnitsSold())) + && Objects.equals(trim(left.getPrice()), trim(right.getPrice())) + && Objects.equals(trim(left.getRecommendedOffer()), trim(right.getRecommendedOffer())); + } + + private boolean rowEmpty(ShopDataCrawlRowDto row) { + return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getInventorySales()) + && blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold()) + && blank(row.getPrice()) && blank(row.getRecommendedOffer())); + } + + private String trim(String value) { + return value == null ? "" : value.trim(); + } + + void deleteResultObjectIfUnreferenced(String resultFileUrl) { + if (blank(resultFileUrl)) return; + Long references = fileResultMapper.selectCount(new LambdaQueryWrapper() + .eq(FileResultEntity::getResultFileUrl, resultFileUrl)); + if (references == null || references == 0L) ossStorageService.deleteObject(resultFileUrl); + } + + public void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) { + String owner = ownerFromTask(task); + if (blank(owner) || Objects.equals(owner, currentInstanceId())) return; + throw new TaskOwnerMismatchException(task == null ? null : task.getId(), operation, owner, currentInstanceId()); + } + + private String ownerFromTask(FileTaskEntity task) { + if (task == null || blank(task.getRequestJson())) return null; + try { + String owner = objectMapper.readTree(task.getRequestJson()).path("ownerInstanceId").asText(""); + return blank(owner) ? null : owner; + } catch (Exception ex) { + log.warn("[shop-data-crawl] read task owner failed taskId={} msg={}", task.getId(), ex.getMessage()); + return null; + } + } + + private String currentInstanceId() { + String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId(); + return blank(instanceId) ? "unknown-instance" : instanceId; + } + + private String ownerScopeKey(Long taskId) { + return "task:" + taskId + ":owner:" + currentInstanceId(); + } + + private String firstNonBlank(String first, String second) { + return !blank(first) ? first : second; + } + + private String normalizeCountry(String country) { + if (country == null) { + return ""; + } + String value = country.trim().toUpperCase(); + return switch (value) { + case "德国" -> "DE"; + case "英国" -> "UK"; + case "法国" -> "FR"; + case "意大利" -> "IT"; + case "西班牙" -> "ES"; + default -> value; + }; + } + + private String buildTaskWorkbookFilename(FileTaskEntity task) { + Long taskId = task == null ? null : task.getId(); + List rows = taskId == null ? List.of() : listTaskRows(taskId); + String shopName = rows.isEmpty() ? "result" : rows.get(0).getSourceFilename(); + return safeFileStem("店铺数据抓取-" + shopName + "-" + (taskId == null ? "result" : taskId)) + ".xlsx"; + } + + private String safeFileStem(String value) { + String raw = value == null ? "result" : value.trim(); + String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_"); + return safe.isBlank() ? "result" : safe; + } + + private boolean blank(String value) { + return value == null || value.isBlank(); + } + + private static long elapsedMs(long startInclusive, long endExclusive) { + return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L); + } + + private String blankToNull(String value) { + return blank(value) ? null : value.trim(); + } + + private void validateUserId(Long userId) { + if (userId == null || userId <= 0) { + throw new BusinessException("user_id 不合法"); + } + } + + private record ResultChunkReceipt(String scopeHash, int chunkTotal, boolean completed) {} +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java index d55210be..a0b6eb66 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java @@ -14,7 +14,7 @@ public class SimilarAsinHistoryItemVo { private String sourceFilename; @Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx") private String resultFilename; - @Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/similar_asin/xxx/17-result.xlsx") + @Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "https://oss.aishufu.top/nanri-ai-images/result/similar_asin/xxx/17-result.xlsx") private String downloadUrl; private Long fileJobId; private String fileStatus; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index d20044cb..b134ad32 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -4027,6 +4027,10 @@ public class SimilarAsinTaskService { int safeCompleted = Math.max(0, Math.min(completed, safeTotal)); TaskProgressSnapshotEntity existing = taskProgressSnapshotService.find(task.getId(), MODULE_TYPE); int displayPercent = calculateDisplayProgressPercent(safeCompleted, safeTotal, job, LocalDateTime.now()); + boolean terminalProgress = isTerminalFileBuildProgress(task.getStatus(), safeCompleted, safeTotal); + if (terminalProgress) { + displayPercent = 100; + } int previousDisplayPercent = calculateSnapshotDisplayPercent(existing, job); if (previousDisplayPercent > displayPercent && safeCompleted < safeTotal) { displayPercent = previousDisplayPercent; @@ -4034,7 +4038,7 @@ public class SimilarAsinTaskService { taskProgressSnapshotService.save( task.getId(), MODULE_TYPE, - STATUS_RUNNING, + terminalProgress ? task.getStatus() : STATUS_RUNNING, safeTotal, safeCompleted, 0, @@ -4044,6 +4048,12 @@ public class SimilarAsinTaskService { ); } + static boolean isTerminalFileBuildProgress(String taskStatus, int completed, int total) { + return total > 0 + && completed >= total + && (STATUS_SUCCESS.equals(taskStatus) || STATUS_FAILED.equals(taskStatus)); + } + private int calculateDisplayProgressPercent(int current, int total, TaskFileJobEntity job, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java index 54763bba..c3d8047c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.similarasin.util; import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import okhttp3.Dns; @@ -39,6 +40,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutorCompletionService; @@ -102,12 +104,14 @@ public class SimilarAsinImageEmbedder { private final int downloadPoolSize; private final OkHttpClient httpClient; private final ExecutorService downloadPool; + private final OssStorageService ossStorageService; - public SimilarAsinImageEmbedder(SimilarAsinProperties properties) { + public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) { int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds(); int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize(); this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS; this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE; + this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null"); this.httpClient = new OkHttpClient.Builder() .connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds)) .readTimeout(Duration.ofSeconds(downloadTimeoutSeconds)) @@ -178,8 +182,9 @@ public class SimilarAsinImageEmbedder { if (taskImageCache.containsKey(url)) { return; } - byte[] raw = doFetch(url); - ResizedImage thumb = resizeImage(url, raw); + String downloadUrl = normalizeAndValidateDownloadUrl(url); + byte[] raw = doFetch(downloadUrl); + ResizedImage thumb = resizeImage(downloadUrl, raw); taskImageCache.putIfAbsent(url, thumb); } catch (Exception ex) { // 预下载失败不抛出：embed() 时同 url 会再次尝试并走原有兜底链路。 @@ -345,8 +350,8 @@ public class SimilarAsinImageEmbedder { } private byte[] downloadWithRetry(String url) throws IOException, TimeoutException { - validateHttpsUrl(url); - List candidates = downloadCandidates(url); + String downloadUrl = normalizeAndValidateDownloadUrl(url); + List candidates = downloadCandidates(downloadUrl); IOException last = null; TimeoutException lastTimeout = null; long waitSeconds = downloadTimeoutSeconds * 2L; @@ -355,9 +360,9 @@ public class SimilarAsinImageEmbedder { Future future = downloadPool.submit(() -> doFetch(attemptUrl)); try { byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS); - if (!attemptUrl.equals(url)) { + if (!attemptUrl.equals(downloadUrl)) { log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}", - url, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1); + downloadUrl, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1); } return bytes; } catch (java.util.concurrent.ExecutionException ee) { @@ -396,6 +401,12 @@ public class SimilarAsinImageEmbedder { throw last != null ? last : new IOException("image download failed without cause"); } + String normalizeAndValidateDownloadUrl(String url) { + String downloadUrl = ossStorageService.normalizeManagedPublicUrl(url); + validateHttpsUrl(downloadUrl); + return downloadUrl; + } + static List downloadCandidates(String url) { List single = List.of(url); URI uri; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index eb5696b2..ae1146f5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -95,7 +95,7 @@ public class TaskFileJobService { List ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper() .in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED")) .lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT) - .in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH")) + .in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL")) .like(TaskFileJobEntity::getScopeKey, ownerMarker) .orderByAsc(TaskFileJobEntity::getUpdatedAt) .last("limit " + safeLimit)); @@ -108,7 +108,7 @@ public class TaskFileJobService { .in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED")) .lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT) .and(wrapper -> wrapper - .notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH")) + .notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL")) .or() .isNull(TaskFileJobEntity::getScopeKey) .or() @@ -265,6 +265,11 @@ public class TaskFileJobService { return taskFileJobMapper.selectById(jobId); } + public boolean isRetryExhausted(Long jobId) { + TaskFileJobEntity job = findById(jobId); + return job != null && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT; + } + public void markFailed(TaskFileJobEntity job, String message) { int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1; taskFileJobMapper.update(null, new LambdaUpdateWrapper() diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java index 397736ca..3dc07396 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java @@ -12,6 +12,8 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService; import com.nanri.aiimage.modules.publish.service.PublishTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService; import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; @@ -41,6 +43,7 @@ public class TaskHeartbeatService { private static final String MODULE_SHOP_MATCH = "SHOP_MATCH"; private static final String MODULE_PATROL_DELETE = "PATROL_DELETE"; private static final String MODULE_QUERY_ASIN = "QUERY_ASIN"; + private static final String MODULE_SHOP_DATA_CRAWL = "SHOP_DATA_CRAWL"; private static final String MODULE_WITHDRAW = "WITHDRAW"; private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT"; private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN"; @@ -54,6 +57,8 @@ public class TaskHeartbeatService { private final ShopMatchTaskCacheService shopMatchTaskCacheService; private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService; private final QueryAsinTaskCacheService queryAsinTaskCacheService; + private final ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService; + private final ShopDataCrawlTaskService shopDataCrawlTaskService; private final WithdrawTaskCacheService withdrawTaskCacheService; private final AppearancePatentTaskCacheService appearancePatentTaskCacheService; private final SimilarAsinTaskCacheService similarAsinTaskCacheService; @@ -106,6 +111,9 @@ public class TaskHeartbeatService { if (PublishTaskService.MODULE_TYPE.equals(moduleType)) { publishTaskService.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat"); } + if (MODULE_SHOP_DATA_CRAWL.equals(moduleType)) { + shopDataCrawlTaskService.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat"); + } String status = task.getStatus(); if (!STATUS_RUNNING.equals(status)) { log.warn("[task-heartbeat] file task is not running taskId={} actualUserId={} moduleType={} status={}", @@ -179,6 +187,7 @@ public class TaskHeartbeatService { case MODULE_QUERY_ASIN -> { queryAsinTaskCacheService.touchTaskHeartbeat(taskId); } + case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.touchTaskHeartbeat(taskId); case MODULE_WITHDRAW -> { withdrawTaskCacheService.touchTaskHeartbeat(taskId); } @@ -216,6 +225,7 @@ public class TaskHeartbeatService { case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task); case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task); case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task); + case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.saveTaskCache(task); case MODULE_WITHDRAW -> withdrawTaskCacheService.saveTaskCache(task); case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task); default -> { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index 28c5e192..587efdd3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService; import com.nanri.aiimage.modules.publish.service.PublishTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService; import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; @@ -44,6 +45,7 @@ public class TaskResultFileJobWorker { private final ProductRiskTaskService productRiskTaskService; private final PublishTaskService publishTaskService; private final QueryAsinTaskService queryAsinTaskService; + private final ShopDataCrawlTaskService shopDataCrawlTaskService; private final WithdrawTaskService withdrawTaskService; private final PatrolDeleteTaskService patrolDeleteTaskService; private final AppearancePatentTaskService appearancePatentTaskService; @@ -192,6 +194,9 @@ public class TaskResultFileJobWorker { log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}", job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message); taskFileJobService.markFailed(job, message); + if ("SHOP_DATA_CRAWL".equals(job.getModuleType()) && taskFileJobService.isRetryExhausted(job.getId())) { + shopDataCrawlTaskService.handleResultFileJobFailure(job, message); + } } } @@ -225,7 +230,8 @@ public class TaskResultFileJobWorker { } return "APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType()) - || "PUBLISH".equals(job.getModuleType()); + || "PUBLISH".equals(job.getModuleType()) + || "SHOP_DATA_CRAWL".equals(job.getModuleType()); } private boolean isOwnedByCurrentInstance(TaskFileJobEntity job) { @@ -273,6 +279,10 @@ public class TaskResultFileJobWorker { queryAsinTaskService.processResultFileJob(job); return true; } + if ("SHOP_DATA_CRAWL".equals(moduleType)) { + shopDataCrawlTaskService.processResultFileJob(job); + return true; + } if ("WITHDRAW".equals(moduleType)) { withdrawTaskService.processResultFileJob(job); return true; @@ -304,6 +314,11 @@ public class TaskResultFileJobWorker { private void cleanupAfterSuccess(TaskFileJobEntity job) { String moduleType = job.getModuleType(); + if ("SHOP_DATA_CRAWL".equals(moduleType)) { + taskResultPayloadService.deleteLatest(job.getTaskId(), moduleType, job.getScopeKey()); + shopDataCrawlTaskService.cleanupResultFileJob(job); + return; + } if ("SHOP_MATCH".equals(moduleType) || "PRICE_TRACK".equals(moduleType) || "PRODUCT_RISK_RESOLVE".equals(moduleType) @@ -321,6 +336,10 @@ public class TaskResultFileJobWorker { similarAsinTaskService.cleanupResultFileJob(job); return; } + if ("PUBLISH".equals(moduleType)) { + publishTaskService.cleanupResultFileJob(job); + return; + } if ("DELETE_BRAND".equals(moduleType)) { deleteBrandRunService.cleanupResultFileJob(job); return; diff --git a/backend-java/src/main/resources/application-local.example.yml b/backend-java/src/main/resources/application-local.example.yml index b278ef98..b5743de2 100644 --- a/backend-java/src/main/resources/application-local.example.yml +++ b/backend-java/src/main/resources/application-local.example.yml @@ -7,14 +7,15 @@ SPRING_PROFILES_ACTIVE=local AIIMAGE_INSTANCE_ID=local-121 AIIMAGE_SERVER_PORT=18080 +AIIMAGE_INTERNAL_TOKEN=change-me-local-internal-token AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false AIIMAGE_DB_USERNAME=change-me AIIMAGE_DB_PASSWORD=change-me AIIMAGE_OSS_REGION=us-east-1 -AIIMAGE_OSS_ENDPOINT=http://47.110.241.161:9000 -AIIMAGE_OSS_PUBLIC_ENDPOINT=http://47.110.241.161:9000 +AIIMAGE_OSS_ENDPOINT=https://oss.aishufu.top +AIIMAGE_OSS_PUBLIC_ENDPOINT=https://oss.aishufu.top AIIMAGE_OSS_BUCKET=nanri-ai-images AIIMAGE_IMAGE_VIDEO_OSS_BUCKET=shufu-video AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET=nanri-ai-digital-human diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index a423c90a..1cbdfc3c 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -85,8 +85,8 @@ aiimage: request-body-cache-limit-bytes: ${AIIMAGE_INSTANCE_ROUTE_REQUEST_BODY_CACHE_LIMIT_BYTES:104857600} oss: region: ${AIIMAGE_OSS_REGION:us-east-1} - endpoint: ${AIIMAGE_OSS_ENDPOINT:http://47.110.241.161:9000} - public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:http://47.110.241.161:9000} + endpoint: ${AIIMAGE_OSS_ENDPOINT:https://oss.aishufu.top} + public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:https://oss.aishufu.top} bucket: ${AIIMAGE_OSS_BUCKET:nanri-ai-images} image-video-bucket: ${AIIMAGE_IMAGE_VIDEO_OSS_BUCKET:shufu-video} digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human} @@ -153,7 +153,7 @@ aiimage: enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true} cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *} retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7} - module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA} + module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,SHOP_DATA_CRAWL,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA} permission-schema-init: enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false} task-pressure: @@ -247,6 +247,7 @@ aiimage: security: shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key} internal-token: ${AIIMAGE_INTERNAL_TOKEN:} + internal-token-file: ${AIIMAGE_INTERNAL_TOKEN_FILE:} auth: jwt-secret: ${AIIMAGE_JWT_SECRET:please-change-this-secret-please-rotate-at-least-32-bytes} jwt-ttl-hours: ${AIIMAGE_AUTH_JWT_TTL_HOURS:168} diff --git a/backend-java/src/main/resources/db/V79__shop_data_crawl.sql b/backend-java/src/main/resources/db/V79__shop_data_crawl.sql new file mode 100644 index 00000000..aa4cf59b --- /dev/null +++ b/backend-java/src/main/resources/db/V79__shop_data_crawl.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS `biz_shop_data_crawl_shop_candidate` ( + `id` BIGINT NOT NULL AUTO_INCREMENT, + `user_id` BIGINT NOT NULL, + `shop_name` VARCHAR(255) NOT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_shop_data_crawl_candidate_user_shop` (`user_id`, `shop_name`), + KEY `idx_shop_data_crawl_candidate_user` (`user_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS `biz_shop_data_crawl_country_pref` ( + `user_id` BIGINT NOT NULL, + `country_codes_json` VARCHAR(256) NOT NULL, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`user_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`) +SELECT '店铺数据抓取', 'shop_data_crawl', 'app', 'shop-data-crawl', 139 +WHERE NOT EXISTS ( + SELECT 1 FROM `columns` WHERE `column_key` = 'shop_data_crawl' +); + +UPDATE `columns` +SET `name` = '店铺数据抓取', + `menu_type` = 'app', + `route_path` = 'shop-data-crawl', + `sort_order` = 139 +WHERE `column_key` = 'shop_data_crawl'; diff --git a/backend-java/src/main/resources/db/V80__permission_menu_parent_hierarchy.sql b/backend-java/src/main/resources/db/V80__permission_menu_parent_hierarchy.sql new file mode 100644 index 00000000..0e39534a --- /dev/null +++ b/backend-java/src/main/resources/db/V80__permission_menu_parent_hierarchy.sql @@ -0,0 +1,220 @@ +-- Hierarchical menu permissions. user_column_permission intentionally keeps +-- only direct grants; descendants are expanded by PermissionMenuService. +SET @permission_db_name = DATABASE(); + +SET @permission_parent_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = @permission_db_name + AND TABLE_NAME = 'columns' + AND COLUMN_NAME = 'parent_id' +); +SET @permission_sql = IF( + @permission_parent_column_exists = 0, + 'ALTER TABLE `columns` ADD COLUMN `parent_id` INT NULL COMMENT ''直接父菜单 ID'' AFTER `id`', + 'SELECT 1' +); +PREPARE permission_stmt FROM @permission_sql; +EXECUTE permission_stmt; +DEALLOCATE PREPARE permission_stmt; + +SET @permission_parent_index_exists = ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @permission_db_name + AND TABLE_NAME = 'columns' + AND INDEX_NAME = 'idx_columns_parent_sort_id' +); +SET @permission_sql = IF( + @permission_parent_index_exists = 0, + 'ALTER TABLE `columns` ADD INDEX `idx_columns_parent_sort_id` (`parent_id`, `sort_order`, `id`)', + 'SELECT 1' +); +PREPARE permission_stmt FROM @permission_sql; +EXECUTE permission_stmt; +DEALLOCATE PREPARE permission_stmt; + +SET @permission_parent_fk_exists = ( + SELECT COUNT(*) + FROM information_schema.TABLE_CONSTRAINTS + WHERE CONSTRAINT_SCHEMA = @permission_db_name + AND TABLE_NAME = 'columns' + AND CONSTRAINT_NAME = 'fk_columns_parent_id' + AND CONSTRAINT_TYPE = 'FOREIGN KEY' +); +SET @permission_sql = IF( + @permission_parent_fk_exists = 0, + 'ALTER TABLE `columns` ADD CONSTRAINT `fk_columns_parent_id` FOREIGN KEY (`parent_id`) REFERENCES `columns` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT', + 'SELECT 1' +); +PREPARE permission_stmt FROM @permission_sql; +EXECUTE permission_stmt; +DEALLOCATE PREPARE permission_stmt; + +-- Normalize older underscore keys. If both spellings already exist, retain the +-- canonical row and move direct grants before deleting the duplicate. +INSERT IGNORE INTO `user_column_permission` (`user_id`, `column_id`) +SELECT direct_grant.`user_id`, canonical.`id` +FROM `user_column_permission` AS direct_grant +JOIN `columns` AS legacy ON legacy.`id` = direct_grant.`column_id` +JOIN ( + SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key + UNION ALL SELECT 'appearance_patent', 'appearance-patent' + UNION ALL SELECT 'similar_asin', 'similar-asin' +) AS key_map ON key_map.legacy_key = legacy.`column_key` +JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key; + +DELETE legacy +FROM `columns` AS legacy +JOIN ( + SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key + UNION ALL SELECT 'appearance_patent', 'appearance-patent' + UNION ALL SELECT 'similar_asin', 'similar-asin' +) AS key_map ON key_map.legacy_key = legacy.`column_key` +JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key; + +UPDATE `columns` AS legacy +JOIN ( + SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key + UNION ALL SELECT 'appearance_patent', 'appearance-patent' + UNION ALL SELECT 'similar_asin', 'similar-asin' +) AS key_map ON key_map.legacy_key = legacy.`column_key` +LEFT JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key +SET legacy.`column_key` = key_map.canonical_key +WHERE canonical.`id` IS NULL; + +-- Video/image workspace. Keep the legacy `wb` key because existing user +-- grants reference that row; its route is the current image-video page. +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '视频', 'wb', 'app', 'image-video', 100, NULL +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'wb'); +UPDATE `columns` +SET `name` = '视频', `menu_type` = 'app', `route_path` = 'image-video', `sort_order` = 100, `parent_id` = NULL +WHERE `column_key` = 'wb'; + +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '数字人', 'digital-human', 'app', 'digital-human', 101, + (SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1) +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'digital-human'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '带货视频', 'delivery-video', 'app', 'delivery-video', 102, + (SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1) +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'delivery-video'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '混剪', 'mix-video', 'app', 'mix-video', 103, + (SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1) +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'mix-video'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '图片', 'image', 'app', 'image', 104, + (SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1) +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'image'); + +UPDATE `columns` AS child +JOIN `columns` AS parent ON parent.`column_key` = 'wb' +SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app', + child.`name` = CASE child.`column_key` + WHEN 'digital-human' THEN '数字人' WHEN 'delivery-video' THEN '带货视频' + WHEN 'mix-video' THEN '混剪' WHEN 'image' THEN '图片' ELSE child.`name` END, + child.`route_path` = CASE child.`column_key` + WHEN 'digital-human' THEN 'digital-human' WHEN 'delivery-video' THEN 'delivery-video' + WHEN 'mix-video' THEN 'mix-video' WHEN 'image' THEN 'image' ELSE child.`route_path` END, + child.`sort_order` = CASE child.`column_key` + WHEN 'digital-human' THEN 101 WHEN 'delivery-video' THEN 102 + WHEN 'mix-video' THEN 103 WHEN 'image' THEN 104 ELSE child.`sort_order` END +WHERE child.`column_key` IN ('digital-human', 'delivery-video', 'mix-video', 'image'); + +-- Three APP navigation groups. +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '前端工具', 'brand_front_tools', 'app', 'brand-front-tools', 110, NULL +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_front_tools'); +UPDATE `columns` SET `name` = '前端工具', `menu_type` = 'app', `route_path` = 'brand-front-tools', `sort_order` = 110, `parent_id` = NULL +WHERE `column_key` = 'brand_front_tools'; + +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '运营工具', 'brand_operation_tools', 'app', 'brand-operation-tools', 120, NULL +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_operation_tools'); +UPDATE `columns` SET `name` = '运营工具', `menu_type` = 'app', `route_path` = 'brand-operation-tools', `sort_order` = 120, `parent_id` = NULL +WHERE `column_key` = 'brand_operation_tools'; + +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '后勤工具', 'brand_logistics_tools', 'app', 'brand-logistics-tools', 130, NULL +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_logistics_tools'); +UPDATE `columns` SET `name` = '后勤工具', `menu_type` = 'app', `route_path` = 'brand-logistics-tools', `sort_order` = 130, `parent_id` = NULL +WHERE `column_key` = 'brand_logistics_tools'; + +-- APP children. The key follows the frontend item key (except the existing +-- shop_data_crawl compatibility key); route_path remains the route slug. +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) +SELECT '采集数据', 'collect-data', 'app', 'collect-data', 111, + (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) +WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'collect-data'); +UPDATE `columns` AS child +JOIN `columns` AS parent ON parent.`column_key` = 'brand_front_tools' +SET child.`name` = '采集数据', child.`menu_type` = 'app', child.`route_path` = 'collect-data', + child.`sort_order` = 111, child.`parent_id` = parent.`id` +WHERE child.`column_key` = 'collect-data'; + +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '品牌检测', 'brand', 'app', 'brand', 112, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'brand'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '外观专利检测', 'appearance-patent', 'app', 'appearance-patent', 113, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'appearance-patent'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '货源查询', 'similar-asin', 'app', 'similar-asin', 114, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'similar-asin'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '数据去重', 'dedupe', 'app', 'dedupe', 115, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'dedupe'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '数据拆分', 'split', 'app', 'split', 116, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'split'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '格式转换', 'convert', 'app', 'convert', 117, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'convert'); + +UPDATE `columns` AS child +JOIN `columns` AS parent ON parent.`column_key` = 'brand_front_tools' +SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app', + child.`name` = CASE child.`column_key` + WHEN 'brand' THEN '品牌检测' WHEN 'appearance-patent' THEN '外观专利检测' + WHEN 'similar-asin' THEN '货源查询' WHEN 'dedupe' THEN '数据去重' + WHEN 'split' THEN '数据拆分' WHEN 'convert' THEN '格式转换' ELSE child.`name` END, + child.`route_path` = CASE child.`column_key` + WHEN 'brand' THEN 'brand' WHEN 'appearance-patent' THEN 'appearance-patent' + WHEN 'similar-asin' THEN 'similar-asin' WHEN 'dedupe' THEN 'dedupe' + WHEN 'split' THEN 'split' WHEN 'convert' THEN 'convert' ELSE child.`route_path` END, + child.`sort_order` = CASE child.`column_key` + WHEN 'brand' THEN 112 WHEN 'appearance-patent' THEN 113 WHEN 'similar-asin' THEN 114 + WHEN 'dedupe' THEN 115 WHEN 'split' THEN 116 WHEN 'convert' THEN 117 ELSE child.`sort_order` END +WHERE child.`column_key` IN ('brand', 'appearance-patent', 'similar-asin', 'dedupe', 'split', 'convert'); + +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '上架', 'publish', 'app', 'publish', 121, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'publish'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '删除ASIN', 'delete-brand', 'app', 'delete-brand', 122, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'delete-brand'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '商品风险解决', 'product-risk', 'app', 'product-risk', 123, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'product-risk'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '定时匹配', 'shop-match', 'app', 'shop-match', 124, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'shop-match'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '跟价', 'pricing', 'app', 'price-track', 125, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'pricing'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '巡店删除', 'patrol-delete', 'app', 'patrol-delete', 126, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'patrol-delete'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '查询ASIN', 'query-asin', 'app', 'query-asin', 127, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'query-asin'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '店铺数据抓取', 'shop_data_crawl', 'app', 'shop-data-crawl', 128, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'shop_data_crawl'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '取款', 'withdraw', 'app', 'withdraw', 129, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'withdraw'); +UPDATE `columns` AS child +JOIN `columns` AS parent ON parent.`column_key` = 'brand_operation_tools' +SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app', + child.`name` = CASE child.`column_key` + WHEN 'publish' THEN '上架' WHEN 'delete-brand' THEN '删除ASIN' + WHEN 'product-risk' THEN '商品风险解决' WHEN 'shop-match' THEN '定时匹配' + WHEN 'pricing' THEN '跟价' WHEN 'patrol-delete' THEN '巡店删除' + WHEN 'query-asin' THEN '查询ASIN' WHEN 'shop_data_crawl' THEN '店铺数据抓取' + WHEN 'withdraw' THEN '取款' ELSE child.`name` END, + child.`route_path` = CASE child.`column_key` + WHEN 'publish' THEN 'publish' WHEN 'delete-brand' THEN 'delete-brand' + WHEN 'product-risk' THEN 'product-risk' WHEN 'shop-match' THEN 'shop-match' + WHEN 'pricing' THEN 'price-track' WHEN 'patrol-delete' THEN 'patrol-delete' + WHEN 'query-asin' THEN 'query-asin' WHEN 'shop_data_crawl' THEN 'shop-data-crawl' + WHEN 'withdraw' THEN 'withdraw' ELSE child.`route_path` END, + child.`sort_order` = CASE child.`column_key` + WHEN 'publish' THEN 121 WHEN 'delete-brand' THEN 122 WHEN 'product-risk' THEN 123 WHEN 'shop-match' THEN 124 + WHEN 'pricing' THEN 125 WHEN 'patrol-delete' THEN 126 WHEN 'query-asin' THEN 127 WHEN 'shop_data_crawl' THEN 128 WHEN 'withdraw' THEN 129 ELSE child.`sort_order` END +WHERE child.`column_key` IN ('publish', 'delete-brand', 'product-risk', 'shop-match', 'pricing', 'patrol-delete', 'query-asin', 'shop_data_crawl', 'withdraw'); + +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '采购', 'purchase', 'app', 'purchase', 131, (SELECT id FROM `columns` WHERE column_key = 'brand_logistics_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'purchase'); +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT 'ERP', 'erp', 'app', 'erp', 132, (SELECT id FROM `columns` WHERE column_key = 'brand_logistics_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'erp'); +UPDATE `columns` AS child +JOIN `columns` AS parent ON parent.`column_key` = 'brand_logistics_tools' +SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app', + child.`name` = CASE child.`column_key` + WHEN 'purchase' THEN '采购' WHEN 'erp' THEN 'ERP' ELSE child.`name` END, + child.`route_path` = CASE child.`column_key` + WHEN 'purchase' THEN 'purchase' WHEN 'erp' THEN 'erp' ELSE child.`route_path` END, + child.`sort_order` = CASE child.`column_key` + WHEN 'purchase' THEN 131 WHEN 'erp' THEN 132 ELSE child.`sort_order` END +WHERE child.`column_key` IN ('purchase', 'erp'); diff --git a/backend-java/src/main/resources/db/V81__expand_publish_item_status.sql b/backend-java/src/main/resources/db/V81__expand_publish_item_status.sql new file mode 100644 index 00000000..cde24c55 --- /dev/null +++ b/backend-java/src/main/resources/db/V81__expand_publish_item_status.sql @@ -0,0 +1,2 @@ +ALTER TABLE biz_publish_item + MODIFY COLUMN status_value TEXT NULL COMMENT 'status column'; diff --git a/backend-java/src/main/resources/templates/shop-data-crawl/文档格式.xlsx b/backend-java/src/main/resources/templates/shop-data-crawl/文档格式.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..d83167324d7a4005a2e5c0ebddac6559e3bce94d GIT binary patch literal 11754 zcmeHtWmH|uvNi4mch}(V4hilS+}Xk1-Q6t&CwOpocY*~CAvnR^3Hoi4drwYy=ic$Y z|F6ej0ee?f_nK9!R(17MkOqf<19=)Da%)0QzyCd8fqxj=7%AA>*g7yO0L9ROAHe(+ zBVpj*cLoIk82|$TLH@g#fvqirtCeL&JimMo6LPQ%`9r`)o6~TzLJ|atK(GI{!ko^Q z5*DTa4Wdx_rdRRY;l|CwxZhnU)bmE;%qVha5vHuJFx?LxuTJ175HGU=U?E}zJq^UgZ zv`%R&GRU|2V!>I>Q*;K?1yHAP0&*WWRn$3oY-r;p@o9~mV%regmuv=HVhHWr71IJY zX6;ARz{J#e0wHWNMC;_Yh=tHjB|Xyf`(fD(??6|?ec;^ABh5egz|SE z8QItao}AU3Q*b?ak`6a^Da(nNh3wa8mv_W$XPpjw?)&z2^6|itS@RQrYWWf zr7~0gb{KP=X|;fy(TSy|cLtVygy;=%FKdw+q7)Z?4g?01YUnPmR-n-z#VZK&)g>4>EJ#JU| zUJk>K#q|TaAS2P2`yzvU`4&{)@F{O|>&3|mdo90KUd*(a8+KVGdtR~{$zUJw6m&09 zN(5VImgAV}itsJtgayGwc$HpqIT02_Gi{|$+leTr@vE%t3yfkPa5e92WM0{$^(Vv^ zSf*CPwlQTwiZ0)74Pa&ciFPzk@#oh-v;%=?qy2%llY^s;)f3)z@jCJWPk77TA?00t zR%*x6#{rP{!$*xR;wpWbh0qikc4U5V;kEN#FmJUu;}~}v+mTEj_c9@vD{4f?u_b_@ zqb#y)oCB;FD1T+HOe_~wc2TNCz)t}izB?stBiQH^8%?Q35Gg{~uOq}nrfD|uD#J&E zPpGxux3DT-fAOh)BQYsnjvo!zFi=F|Wj??LDT|Bg3={fZ%b09uNY09Lplf?`2l~#m zu5D8mO81j*BET06uL8}mWsrWqSDF;B4`vlCfkvZD&yPKw&`3Y5nX^o9c^TW1RZ&N# z)uP^#Z972Mu_s~DC|c3i3XAt`@QWAgL1YB(FEgVb%0LSM!4R4bW-v{WuAxIDq?EfMAQ-%4-jZrFE zHs6S`y|QW_eO)UUcPOmT3wPrIWwnws&BLr+SwRpz@tevgrYAXsf#w%;C%XCS{A5c3vWUF%tliM>3YxNkb_^phX z4!OAgE>Ir!(=sI~9cw%Ls4?$vbvPRlwQTI#^~ld$W3~o)_otAh2j=WUW&7vMLo=eQ z#$WAQMwW%l{lLCu>1F80cE8UL-T{rMLTRJ@HfCBMWHVwZY84g;wkMZEO;J#%ckLn- zqPl+}#@%XhU)uW;snS3)Fqk^g{vKj++li?2fNer1zzx~wBuF%Zmh9LL-bPcbj%AE# zwbe_j%1Pm*Vfz?{@d5PDv{ND;;ZFh#+Hi>f7_=-ugEmpeChaNrygU$8Z)YTuoo7Q) za2l$Xo6j>}9H;{i(eInfv=aHI*I`6OP1UlEap`uFd*wMdbf8Q>h!VLWRlxRwR7z;G zc*Qgyy&^L}D<|w*i6He-^3}*7`DOXG?8mIUiG&bh3Yl}{LS z4b&r=UU-XPa<{)={{8x3a-GbAv@&?zHrb^|Md7DyxI(Iix>3Wy89E z%NE087jh$~+_!q+606u@HLP5)z6j)n<|?@vyiM%U_;PS0%8__~BMh&-Fh7=jmy=zA z0_>xQ>(LK&x$GzO%hXQC-Cs!a<4pW~7qGIJ?(hu|Me-b3)7Izi4zj(Y29JC3E1)0h z@e-h4!BJ_DEog=bw%V#nB|C=RD;)42du`atnRH=L6x;brUqgu_yAtO|B)DXmy~CbC zP4~xcGu<>{(_PHoT7W)c3MY@aHNvD=MX}nbm@lltl&`x`Z%H>#tKvvA&l;HbEVde$ zpHaDGnJq8G<=CSS>63(KKKd}|MZ)&2$K~y%wkBzIZ)>i;aM(`C*RqUB9rV3s!!@(|84IMEF?Es4}KpSXG&&qj9)zD2Kv zteuA&KmJUtzw5jIMOd-^Us(NLSp9#7Rawcad2nF1()iEWitSfeY1yO+AqOqfUI=D; zA;H@j4qzhZx3eW_QWv|s=`sdrxG_4qSjGEffr5cHA-SC%dT!coj?4|+*Ti?>l%k0h zlD{BB_1&bzx`~Zn?r>IfA-B!kNtetepMj$ESPX*Zc`=_*+O~-WNr-!f;qwo}i@r&|qq~E(4=$Nx#dCP{<*^8<+ed z?TX8E9|^{W(tceJL9&RnXax6@(I7#JM)omuf$w4yWAlJy>E`tFkJmw;eY%xK2<2xm z;MC?8bR|~x=AAuWtt>n3How^W6r|nJgbj(#NOgTNIQBvLfn^)?Tl`Sn_6GURlteE0NH}33E+)WJvnMDsQ^* z8aTBE!b6&MO>Zhsr)=nid|xVml$y(S8fPDLX-<64Eq4T}yO`Eplow8o>!(>GXMR8$ zbDHc|=Mn1yOU}0q%9lw4F3E9#r)%)9I@#qPZAE9x-x#jy{+-WL;jBC-H#%^EG$vA7=6htgco6bKtQ5 zLp}=j&JXh~mVW0nNvS$DtHWyx^X9J0@@4I#_W-hzg8KyR{#l^fk=r=z&V5AucaT}w zT-Mar%oN%1%5ZI?+gZzd5xi3ATg~4mO$WDWx!kjS9un*O5490jU};Q~C+2V8cD<)H zS6C1&0FBCHd1Du&)&eN6>Ks0#p!f>FUuNkGDroEO_I}-?l0>Bs2yX*B@IZl&_c%W? zpJ)_5$iIu#HFJw-dwR{O@pJD`d@*@Qq$;&5I8`)2-z&>-;Wjyw%qr$WbAL(KcVNA& zooyR1Y88n7P-o%71o0qSX!WhgN?T<+H$qWpDAG^wFoV;)kH64uUUp@unvWi`09#C+ zP6fwH0y74-&;2UG1hfnAPFVR&+M63^}vJ6{TklnE!6fw~`G!h%_) zsuMP^`L-)s*xjo00>h8;^>BZvCAN=I!4X(=$lSshYI!V7m23y2)?6u>UV}o%UWX}QMS>du1=5lSLNCK zIO`D5#986|(=bNVHEkIL%z*~v4AlsQ$5tb*{^YHvLt~-2te?|9XV|09IHSIvWRg$> zGe62a@4gQjdjZ+^5#3y0!cE?+m$q}FHcud;$yAXropc^X_$oBx;shKA3LO@f2-hIb zXLal$iJ&$aUa?7dHhynck0BX8vH-yUG7BMGk~Yaa-&tY%3kb~@yA1cj*^d}EY#@js zD+N`36HQtbwW-{*U@77;hhx@F3r@%H9;7WeT!u#jiuPC72e&IqSNCS#wydqO53ZD0 zTuUp`n!mxn$2C5#WA`l7G*jh$p&MW*HMxV!6aJ}Aj$IrpmN_>>*R@S~m$2618Wqx# zswlqzyGF%5>|OS4q|0iFh$6Wp?x0)nfGJF^6^Rs7I)u0}sx&+Fe9z#Gb#Snum4qx- zD9Otd%Rr*HFmMJRFJ{h;boqU7TiqYN->uf1g$QN}x;s7`L+1{o@V(yY#q|6!TIl0> zp?G{dRiP*Fun>dA@sNBZ`0%4}{7#4}CtsAY-Su>@<|SkM-S-9?B99eld_+svEOK=S z&SMSR2#S@o7m*g+gvVbJwg*DFP^q9631r)xfop(`Is<2Mi)>n=}Du->^#Ev zfhx(=Uzra4B1)F|($ysf`DOWl@nDL+5zYdcFdB-D#EtO^vKh6b1nS`Mf(dcneAwto zM)pwA_#TIVQ=9^FsC(p*(Qe=gZmMMBwaTTU$bl~r3C?Zg{^*a-{Z4^_GV)9SVUxh1 zhrJsjYe+{MHvw5`3xcClq25dK#rf7z zgrj+9GQO2wExr)iQFpXMpShfGrtdQKi^4A`c6CC$zkKLvXn$o>yy4T%$f(8aXjX^A z_RXs8huodV5%Y)Ims{vWux#+0OA*ieM~ zA@GrUqKc=Xd8DnQ2R2u!vsDusJt5qbI8>-%83V+h7ZsuL+-r% zfbwonj8G0bbS;jBIl1z_GD~l&&@~u7iy^VZ7X2}%G2gT*-Zpi9r_jt6$_1T9#DM9w z1{r>j9|BGJWw zLpk=_kYt-v)6AfUh`;VU>A(fen1AsgU<|{zr948#xiU=bO6qwR9b}40_Dbiy z;VS?ZK3|8q;-iAKrMb24697Z;g;Je_=#IUyuI@{W%bARD5UxTIfC-qtVxq6# z-XPYd<#0e+a=Lmzm?oSRF3+B>qD3$w-j{&`kxzJySMg#KfTXumD3I|3md3%-?u(x+ z%HM18ZjMxL)6ssU-+7NQylTk2*zG)AEE;NX(NM;c4l&>98Y#wgCCqNS4o-I>O+-3V z)+*3w(K=C_#NZ6(oTvs?pFYAYtI-OR2bi~?X17`;w(UZ!u$Zqu7?2`2hp(fxyWO(_ zom0pwo@+vHvp5@r{QgD*{?&TlrF@O(W10RVWGB?shvC}QPflX_Oawv!myZyC79rfF zx?_;QBE%N>^R!h0z9<|_4eSBN%8vGC)+P>51&CGbs1!Iey4aQf5tFeEYq&%lxhOMKVn(sCdwPaV_1#K(NQ_;HbhQB{`XciBaQU z$rg%mJLItrMe){92D72@1xmn>p|x(_;;EmilQ4d6VnHu!`KsMF&@`v(%VP*(D27ne z0PgW9QE3427z?>Lzr0vdZD_C4x|DQ(Ry|Y3BOdXPcA$;Xc1d)_$3UdJ-^fd(dza8$ zZYukB3d!S4SJq@o{`Ooozn(RBzS2MJ+ zvG`T8@!L!ZVGp>Fw+A?R*n3;8QORL&RuxUFsKQXo3|5Vwe5@aDZdoDj?FTX3?)6m{ z4)p_NW&+Kkv=W3e;g!qqk1agozw+_EYR$L}+2e}OB>BW7Ntayw4nkvv+i=C!_EA@m?A%azFo+xkwnk{sZ9X8@jU-%WmXcbkS5nAy-@Yne zdQd4`J1Yq0gv-B}j;M3450(m+e?CyyAH>7-b*pM5s;&rCxHdz#35lGB=owa@D6rp& zc|ZrSV)UcBQpRXZ@)Z{BL`h9%k6mEncQT6G;Q>?n&LVKhEZz>uRz-o+%#JOTQ!=f) zDv_R=S_;NBm$VTXE~}fxEr!!lx1~kh6DD8nuz&U$%zR)PvbdIC$Q8$a0!DQYn;U5s zTU5+nwNy#zwcFX{x6K~;iQLxG0a9eq*E2n90fe_waz!Pa9p3vTGpWs?9p&ve57v70 zQk6y)E?+sby!G_F+q}=(F(Y*yJ|FRDw>2+#-nF{j=WoV-sSR%*>41?SKWK_`5zF_a zo@B$&V0R$bJVWb`P}Nkl^uwy3r~{0uXf@|@(d#g8yt5+OrZ@U-DkhFY5QVq}V5izk z$KKz}OI^nHNMu4it`|2{k6zS6D|IwjjnZzy$&`M^~;Y@t)0_RYx9A7Cgc7a^FWfDs+A< z`P^N;(g)E1!9Sy|q>V__99Zqa2!enh{_7!ibh8Bfd^-EI4r0DZ;5P%rLl^6bd*d=wP{Qvsm8v{;?%vMbg>R{EfJd8iPl;McuF9?hF{>c?P9T+UP8 z=2IY9lXZ>d`u2`xoP53a+uV@YJl7>($JW7B|Hae?`bFlpstq1ljjyyr@jH%- znHvbHr!F=)L8)|6_t8^(bLB=H8>|j=dNyA77Ex6`Z__yCWu}rrx^Xx+(3Xh~up-tE z!?^b&uNLBVI+NP>5P1de&Y}8F23dLAm{D-&Me9_G(L)G^wNn_D1@=-y}X+$jY*S~ zW3p5w;kavGvMLkY|CvZy$7StWRwG7UvMY(x!&WISzjLSl8#t@A^m2T;*X?}Wq}2|h z{apaq!^J3(y!K<1$g!xF=T<#A7MUuxnocib;KS9-`U*&iMsr{tgARgR9*c+;8|iej zn&xp##>(dkJ-vgx_RtPvx+vYic1QMe){`4Jqi_Rft*G|BO7aqlK^E>Q^GF}k#hp0? zO1L3~FLgTdr+x>q0QdeKgwgE2^EEZbEWx}oIi4l#aB}2nNBih^5_9&Zpaw%jhS_A- zf+S@S^Q~QW${dlXcQ{aSnkChVVcob?rHVzR=;g-@$BfcP(RN%G5Gm4`@1&CBIovR3 zgNslpE6ii%X_BZD2Ab5yzVXZ9A?jpyeXsqXAMBKDxBI42iIpu^lca037$RO9-c1DE z{KpAIAB&3VLJ{FR7Wz|mLfmu4;n?!Q<;&KVNqQ1MScNS@u%cAGnbwEWrC?s+0@X?- zjr{Zu6N6?JiuS>emS(~s@J84v>h%Oj66{ftsw*-jpaDk9=lI~>600X=H7JoP_}Bvp z@w89}v$ce-8LXoDSBZ{bMqldPd&7m(qGNK}A*I?DDcxgw;NMNL%FALiscpY63Pq5! z(9S7$ZqZ$iXPEIFcXnK)N&sH#XvM{DO?jJrab^y7czBAoMtdqJsVf}vJW9n*=});d z=SybyE>e{~&X_ytDA!kXAl&R}mgy>XMJg?+=v?Z7U(DDyy?L`+T>gC#GVrG1`&*4A ztJJuMj69s3Wy#c??3a(`<@;Qt5|46|k>^L`qciD8ZU%*li*|n#B3F2rL8#f{ZQ~@n z=apog>_}j=HQyNZShRyXgkt)TqZjg z$#Od{seDrTQ4fn;$wUng_?MkITlGlt92-*D5{*jRnZ$cEb=UAszn+k3@FdSc#B7Pw zU>+qY1&<&O&>#tU%!;6aaUzj zy{OiNHO!oC8$?os`}*B|9-TDIk@__lwQ?wV^>LgmLg599WZoGHY~DGFr1s1}Ca)jl zxA!s9Q_G}`Up~>@HobYhvZI+&+g?sBSw-06>%_?;kEHW;Z#utHsnK2Ui1mwS5{`ij ze1^-H+Jm&bw0m8_!!D=EpsdRYe-2A_;)o36(RBRo*8o>+$1hzmZ*q;az)ehfDg({k z+h-`?D|N2Tph^k~?;+JQ7!AfdCj-t0WmkSLso3`MR1z8Von|0xJnGQNp>gXh|aKJ2--9JONX9VE5uYM?M79 z=1r!opIu~=9I3Q^dGZ`Ob_nTIbx&u?EI~9g)ljrGHgHKeLqaA*7pae89G-P<9jLyFwWW z%$7txm2th`M&79<8jExkBfH%fVCHny=)&J^(A@Hu5@ zNxL_z}v}xPp-45mxwU(iX{)+kj3$% ztK_EO6(dpm`}-TaypVHx8Ok}&cI4olt*lYT_GVAVliePdt%%b+c!N_%VGG5reLbHx zzDHsuiDoED5F=UP_D%UmmitlYn&qrsff|@aFoM9mqUKu&Z4v?WcVP}O_Ly~Uk^=i6`6?MNJWqi2XT~nm%b^z-~ z2cxf%KN@>DcfDD0nYz7tOXPzyeA=oI6b47tK`tGlMWSzz+Wm$i6rllNj_6^!)z z^!dqIBmc5Rk%*LxU`Ob;GfRaa%f{B*h~)9ieo{Ux6%wohJ#Xg60sfWi^PSvbd4bf9 zN1R;x{j3d7RLm;Pn_!jLH{HkDe!RuR5jXDGnS!aYf=L%le1$94)Rkf@a@xr>@^o`8 zbRZIuQV6f0F(Zw-q^17#t`LU5OF!ePgr26b+d8 zOxQ_G&~m%=T{q1|HYvq|x{gO?f<`cGDGoK*Gi#2cH&h#eUygNa27^!ntpuAelybr9 zqvoRa2w~e6I~g2Ng~EL&Z(NR5lSTRJ2(;DoB)%&$_LnW$NVovMYyHB&5scT6RG4Cr z*-XB%HNPew{=j-W;jBXlIGA0i5qJznX%tqpvi&?RKySdodr=%kq^_Kb~)AYYOtQOpBB`D$s7H zYA{R>Lx%)aF9~q*vF%dSO+Vlm2e*>8`xI|iT;V!@)GdXs2=6&d$T^C!a@F_-pMewI45JuQrCm133!|iTjTN$YP<0H&z(3ZU{R)L53qcG{O-Z!-1-RoZW*{$fd5N| z;YqSAeq5@T3Az8s{|I@rUDgEyh#@2CzO1pA?|YfY&y;BK8VBud!isd$&CoF^KQ_~r zJ&jxN7KwG43!+>7ofQ3rYzCd%6^}h#)ffFGO50|@-BfJ5AzF4%4Q~SjHu)0g8OY;w zJe5APgz!=)m`&`cC(3%okmc)Jq6zhAfwJQYYW-nj zQ@l5}!k?1+YMhn!yGmS!lVvPMWPoSxTnF-nC=?X;xV@9W@DcFXx3ugnJ))>~(0k?Z zg$?ZvJGztsZ8@TFitN!JLQ_*)-Cc1LTD8LxJb@HBdynKS1HIqhxe2iAN!Xv#m$>|A zEdSNj{3kfatno*nfg9CB;I0ezR~`II?1}&I@4eSgblVb#&4FAk(W{_`fcIYX-`r{w z2frqY)R*0Yb`uJQ(&LPtuXa?}lGqU!rL9D*jAeVoh>5h^%P)s9I!k#1cBc3go&R{Rnf9>z(?9<@UBUiKhT9!BVm?0RQZKS6rQH5>JB7EPs--Tw+ zeN?wu^F=e)E-nCfKx=1{FjmJk{ME-!0;#O5m;m{Qq_DbiGIU!OIFZ4&I+Jc01hL;q?D zYrV3ub_7^E>Z!Qd0vvRnHm;*d!`9tTxFy{n#WkoJW+SO$_=yq?V}3jb*Y+Sr3A1uT z2h*9|CW_3$_5_NG(m0gk{;9Vo!EwZdUedIwC6bKxd^Qy zHX^kaNVWITX%4TpdS_@cdjZ|ql7MYAOMsHTA)Tov4vrd}e1r8!Su7ee)RI}VjTd@iH71chorKu7^jls#-BGrbuEtO`A?LRG$m ztw0FTR-2_6{&Yx%iHhnMO1X)&Y5|85102KXz;XEK6x-~jwp>nByr~HuWH?q3J5!Fj(7{4-JLDMS4lPt?XYZbW;646U`xgcCIYiItfWNhefpPxVr2i%dK39LvZ~Lvz4P;*Z zR{x9b_TO&$iL3Q%Jl%9TFhv1-Y5z^(Pn2>fek%Ms?fr?;^ThWw&o7jQ{;u{v(%|zO ze%|f-Z3iFgpL>7LEj({U{I<{q%*apO!@n96|NDC%0V4L-3Dxr%pC4uYp7HGOYQGP* zo*Q`HtNU$08F)GdocHgr{kKIwk?nqsCl5ch{{E%>7xnH>uRSNfqDGT75udFlN>BLIgb8tfP9;`+pa@FINBn literal 0 HcmV?d00001 diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java new file mode 100644 index 00000000..89b81bd9 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/admin/service/AdminUserServiceTest.java @@ -0,0 +1,99 @@ +package com.nanri.aiimage.modules.admin.service; + +import com.nanri.aiimage.modules.admin.model.dto.AdminUserCreateRequest; +import com.nanri.aiimage.modules.admin.model.dto.AdminUserUpdateRequest; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; +import com.nanri.aiimage.modules.auth.util.WerkzeugPasswordEncoder; +import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; +import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class AdminUserServiceTest { + + @Test + void createUserStoresSuppliedDirectPermissionsInSameTransactionalMethod() throws Exception { + AdminUserMapper userMapper = mock(AdminUserMapper.class); + WerkzeugPasswordEncoder passwordEncoder = mock(WerkzeugPasswordEncoder.class); + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService permissionService = mock(PermissionMenuService.class); + AdminUserService service = new AdminUserService( + userMapper, passwordEncoder, authSupport, permissionService); + AdminUserEntity operator = user(10L, "admin", 1, null); + AdminUserCreateRequest request = new AdminUserCreateRequest(); + request.setUsername("created-user"); + request.setPassword("secret1"); + request.setRole("normal"); + request.setColumnIds(List.of(11L, 12L)); + + when(authSupport.currentRole(operator)).thenReturn("admin"); + when(passwordEncoder.hash("secret1")).thenReturn("hashed"); + when(userMapper.insert(any(AdminUserEntity.class))).thenAnswer(invocation -> { + invocation.getArgument(0).setId(20L); + return 1; + }); + + Long createdId = service.createUser(operator, request); + + assertThat(createdId).isEqualTo(20L); + ArgumentCaptor permissions = + ArgumentCaptor.forClass(UserColumnPermissionUpdateRequest.class); + verify(permissionService).updateUserColumnPermissions( + eq(operator), eq(20L), permissions.capture(), eq(PermissionMenuService.MENU_TYPE_APP)); + assertThat(permissions.getValue().getColumnIds()).containsExactly(11L, 12L); + assertThat(AdminUserService.class + .getMethod("createUser", AdminUserEntity.class, AdminUserCreateRequest.class) + .isAnnotationPresent(Transactional.class)).isTrue(); + } + + @Test + void updateUserCanReplaceOnlyDirectPermissionsTransactionally() throws Exception { + AdminUserMapper userMapper = mock(AdminUserMapper.class); + WerkzeugPasswordEncoder passwordEncoder = mock(WerkzeugPasswordEncoder.class); + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService permissionService = mock(PermissionMenuService.class); + AdminUserService service = new AdminUserService( + userMapper, passwordEncoder, authSupport, permissionService); + AdminUserEntity operator = user(10L, "admin", 1, null); + AdminUserEntity target = user(20L, "normal", 0, 10L); + AdminUserUpdateRequest request = new AdminUserUpdateRequest(); + request.setColumnIds(List.of(21L)); + + when(authSupport.currentRole(operator)).thenReturn("admin"); + when(userMapper.selectById(20L)).thenReturn(target); + + service.updateUser(operator, 20L, request); + + verify(userMapper, never()).update(any(), any()); + ArgumentCaptor permissions = + ArgumentCaptor.forClass(UserColumnPermissionUpdateRequest.class); + verify(permissionService).updateUserColumnPermissions( + eq(operator), eq(20L), permissions.capture(), eq(PermissionMenuService.MENU_TYPE_APP)); + assertThat(permissions.getValue().getColumnIds()).containsExactly(21L); + assertThat(AdminUserService.class + .getMethod("updateUser", AdminUserEntity.class, Long.class, AdminUserUpdateRequest.class) + .isAnnotationPresent(Transactional.class)).isTrue(); + } + + private AdminUserEntity user(Long id, String role, int isAdmin, Long createdById) { + AdminUserEntity user = new AdminUserEntity(); + user.setId(id); + user.setRole(role); + user.setIsAdmin(isAdmin); + user.setCreatedById(createdById); + return user; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupportTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupportTest.java new file mode 100644 index 00000000..422155b0 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/admin/support/AdminAuthSupportTest.java @@ -0,0 +1,52 @@ +package com.nanri.aiimage.modules.admin.support; + +import com.nanri.aiimage.modules.auth.config.AuthProperties; +import com.nanri.aiimage.modules.auth.service.JwtService; +import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +class AdminAuthSupportTest { + + @Test + void explicitLowestIdAdminIsNotPromotedToSuperAdmin() { + AdminUserMapper userMapper = mock(AdminUserMapper.class); + AdminAuthSupport support = new AdminAuthSupport( + mock(JwtService.class), userMapper, mock(AuthProperties.class)); + AdminUserEntity user = user(1L, "admin", 1, null); + + assertThat(support.currentRole(user)).isEqualTo("admin"); + verify(userMapper, never()).selectOne(any()); + } + + @Test + void legacyBlankRoleRootRemainsSuperAdmin() { + AdminAuthSupport support = new AdminAuthSupport( + mock(JwtService.class), mock(AdminUserMapper.class), mock(AuthProperties.class)); + + assertThat(support.currentRole(user(1L, null, 1, null))).isEqualTo("super_admin"); + } + + @Test + void explicitNormalRoleIsNotPromotedByLegacyAdminFields() { + AdminAuthSupport support = new AdminAuthSupport( + mock(JwtService.class), mock(AdminUserMapper.class), mock(AuthProperties.class)); + + assertThat(support.currentRole(user(1L, "normal", 1, null))).isNull(); + } + + private AdminUserEntity user(Long id, String role, int isAdmin, Long createdById) { + AdminUserEntity user = new AdminUserEntity(); + user.setId(id); + user.setRole(role); + user.setIsAdmin(isAdmin); + user.setCreatedById(createdById); + return user; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java new file mode 100644 index 00000000..1283384c --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java @@ -0,0 +1,91 @@ +package com.nanri.aiimage.modules.file.service.oss; + +import com.nanri.aiimage.config.OssProperties; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class OssStorageServiceTest { + + private OssStorageService storageService; + + @BeforeEach + void setUp() { + OssProperties properties = new OssProperties(); + properties.setEndpoint("https://oss.aishufu.top"); + properties.setPublicEndpoint("https://oss.aishufu.top"); + properties.setBucket("nanri-ai-images"); + properties.setImageVideoBucket("shufu-video"); + properties.setDigitalHumanBucket("nanri-ai-digital-human"); + properties.setAccessKeyId("test-access-key"); + properties.setAccessKeySecret("test-secret-key"); + storageService = new OssStorageService(properties); + } + + @Test + void normalizesLegacyPathStyleUrlToCurrentPublicEndpoint() { + assertEquals( + "https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg", + storageService.normalizeManagedPublicUrl( + "http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg")); + } + + @Test + void keepsCurrentUrlAndNormalizesVirtualHostUrl() { + String current = "https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg"; + assertEquals(current, storageService.normalizeManagedPublicUrl(current)); + assertEquals( + current, + storageService.normalizeManagedPublicUrl( + "https://nanri-ai-images.oss.aishufu.top/supply_images/main.jpg")); + } + + @Test + void supportsAllConfiguredBuckets() { + assertEquals( + "https://oss.aishufu.top/shufu-video/result/image_video/demo.mp4", + storageService.normalizeManagedPublicUrl( + "http://47.110.241.161:9000/shufu-video/result/image_video/demo.mp4")); + assertEquals( + "https://oss.aishufu.top/nanri-ai-digital-human/digital-human/versions/demo.mp4", + storageService.normalizeManagedPublicUrl( + "https://nanri-ai-digital-human.oss.aishufu.top/digital-human/versions/demo.mp4")); + } + + @Test + void leavesExternalImageUrlsUnchanged() { + List externalUrls = List.of( + "https://m.media-amazon.com/nanri-ai-images/main.jpg?size=large", + "https://cbu01.alicdn.com/img/ibank/main.jpg", + "https://lf9-bot-platform-tos-sign.coze.cn/file.jpg?x-expires=1&x-signature=abc", + "https://nanri-ai-images.evil.example/supply_images/main.jpg"); + + for (String url : externalUrls) { + assertEquals(url, storageService.normalizeManagedPublicUrl(url)); + } + } + + @Test + void leavesInvalidOrUnmanagedValuesUnchanged() { + List values = List.of( + "http://47.110.241.161:9001/nanri-ai-images/supply_images/main.jpg", + "http://47.110.241.161.evil.example:9000/nanri-ai-images/supply_images/main.jpg", + "http://47.110.241.161:9000/nanri-ai-images/../other-bucket/main.jpg", + "http://47.110.241.161:9000/nanri-ai-images/%2e%2e/other-bucket/main.jpg", + "https://oss.aishufu.top/not-a-bucket/main.jpg", + "https://oss.aishufu.top/nanri-ai-images", + "supply_images/main.jpg", + "/nanri-ai-images/supply_images/main.jpg", + "not a url"); + + for (String value : values) { + assertEquals(value, storageService.normalizeManagedPublicUrl(value)); + } + assertNull(storageService.normalizeManagedPublicUrl(null)); + assertEquals(" ", storageService.normalizeManagedPublicUrl(" ")); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java index ed973af7..48e34ef1 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java @@ -142,6 +142,7 @@ class ImageVideoAsyncTaskServiceTest { assertEquals("FAILED", task.getStatus()); assertEquals("FAIL", task.getCozeStatus()); + assertEquals("model unavailable", task.getErrorMessage()); assertTrue(task.getResultJson().contains("model unavailable")); verify(archiveService).capturePollResponse(task, cozeResult); verify(taskMapper).updateById(task); @@ -294,7 +295,7 @@ class ImageVideoAsyncTaskServiceTest { task.setSubmittedAt(LocalDateTime.now().minusHours(2)); Map cozeResult = Map.of("data", List.of(Map.of( "execute_status", "Success", - "output", "{\"Output\":\"https://example.com/result.mp4\"}"))); + "output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"video_url\\\":\\\"https://example.com/result.mp4\\\"}\"}"))); when(taskMapper.selectList(any())).thenReturn(List.of(task)); when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1); @@ -309,6 +310,145 @@ class ImageVideoAsyncTaskServiceTest { assertNotNull(task.getCompletedAt()); } + @Test + void successfulWorkflowWithEmptyOutputVideoUrlIsMarkedFailedEvenWhenInputVideoExists() { + ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class); + ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class); + ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class); + ImageVideoArchiveService archiveService = mock(ImageVideoArchiveService.class); + ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService( + taskMapper, + cozeService, + workflowConfigService, + archiveService, + new ObjectMapper(), + Runnable::run, + instanceMetadata("server-121")); + ImageVideoAsyncTaskEntity task = waitingWorkflowTask(); + Map cozeResult = Map.of("data", List.of(Map.of( + "execute_status", "Success", + "video_info", Map.of("video_url", "https://example.com/input-reference.mp4"), + "output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"video_url\\\":\\\"\\\"}\"}"))); + + when(taskMapper.selectList(any())).thenReturn(List.of(task)); + when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1); + when(taskMapper.selectById(98L)).thenReturn(task); + when(workflowConfigService.imageVideoWorkflowId()).thenReturn("workflow-1"); + when(cozeService.getWorkflowResult(1L, "workflow-1", "exec-98")).thenReturn(cozeResult); + + service.pollWaitingTasks(); + + assertEquals("FAILED", task.getStatus()); + assertEquals("SUCCESS", task.getCozeStatus()); + assertEquals("Coze workflow succeeded but video_url is empty", task.getErrorMessage()); + assertNotNull(task.getCompletedAt()); + assertTrue(task.getResultJson().contains("video_url")); + verify(archiveService, never()).enrichCompletedTask(any(), any()); + verify(taskMapper).updateById(task); + } + + @Test + void successfulVoiceListWithNestedBusinessErrorIsMarkedFailed() { + ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class); + ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class); + ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class); + ImageVideoArchiveService archiveService = mock(ImageVideoArchiveService.class); + ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService( + taskMapper, + cozeService, + workflowConfigService, + archiveService, + new ObjectMapper(), + Runnable::run, + instanceMetadata("server-121")); + ImageVideoAsyncTaskEntity task = waitingVoiceListTask(); + String error = "Query voices failed: invalid API key (status: 1004)"; + Map cozeResult = Map.of("data", List.of(Map.of( + "execute_status", "Success", + "output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":null,\\\"error\\\":\\\"" + error + "\\\",\\\"system_voice\\\":[],\\\"voice_cloning\\\":[]}}\"}"))); + + when(taskMapper.selectList(any())).thenReturn(List.of(task)); + when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1); + when(taskMapper.selectById(163L)).thenReturn(task); + when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow"); + when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec")) + .thenReturn(cozeResult); + + service.pollWaitingTasks(); + + assertEquals("FAILED", task.getStatus()); + assertEquals("SUCCESS", task.getCozeStatus()); + assertEquals(error, task.getErrorMessage()); + assertTrue(task.getResultJson().contains("status: 1004")); + verify(archiveService, never()).enrichCompletedTask(any(), any()); + verify(taskMapper).updateById(task); + } + + @Test + void successfulVoiceListWithBlankBusinessErrorRemainsSuccessful() { + ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class); + ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class); + ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class); + ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService( + taskMapper, + cozeService, + workflowConfigService, + mock(ImageVideoArchiveService.class), + new ObjectMapper(), + Runnable::run, + instanceMetadata("server-121")); + ImageVideoAsyncTaskEntity task = waitingVoiceListTask(); + Map cozeResult = Map.of("data", List.of(Map.of( + "execute_status", "Success", + "output", "{\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":{\\\"status_code\\\":0,\\\"status_msg\\\":\\\"success\\\"},\\\"error\\\":\\\"\\\",\\\"system_voice\\\":[]}}\"}"))); + + when(taskMapper.selectList(any())).thenReturn(List.of(task)); + when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1); + when(taskMapper.selectById(163L)).thenReturn(task); + when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow"); + when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec")) + .thenReturn(cozeResult); + + service.pollWaitingTasks(); + + assertEquals("SUCCESS", task.getStatus()); + assertEquals("SUCCESS", task.getCozeStatus()); + assertEquals(null, task.getErrorMessage()); + verify(taskMapper).updateById(task); + } + + @Test + void successfulVoiceListWithNonzeroBusinessStatusIsMarkedFailed() { + ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class); + ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class); + ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class); + ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService( + taskMapper, + cozeService, + workflowConfigService, + mock(ImageVideoArchiveService.class), + new ObjectMapper(), + Runnable::run, + instanceMetadata("server-121")); + ImageVideoAsyncTaskEntity task = waitingVoiceListTask(); + Map cozeResult = Map.of("data", List.of(Map.of( + "execute_status", "Success", + "output", "{\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":{\\\"status_code\\\":1004,\\\"status_msg\\\":\\\"invalid API key\\\"},\\\"system_voice\\\":[]}}\"}"))); + + when(taskMapper.selectList(any())).thenReturn(List.of(task)); + when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1); + when(taskMapper.selectById(163L)).thenReturn(task); + when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow"); + when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec")) + .thenReturn(cozeResult); + + service.pollWaitingTasks(); + + assertEquals("FAILED", task.getStatus()); + assertEquals("invalid API key (status_code: 1004)", task.getErrorMessage()); + verify(taskMapper).updateById(task); + } + private ImageVideoAsyncTaskEntity waitingDouyinTask() { ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity(); task.setId(79L); @@ -331,6 +471,15 @@ class ImageVideoAsyncTaskServiceTest { return task; } + private ImageVideoAsyncTaskEntity waitingVoiceListTask() { + ImageVideoAsyncTaskEntity task = waitingDouyinTask(); + task.setId(163L); + task.setUserId(665L); + task.setTaskType("VOICE_LIST"); + task.setCozeExecuteId("voice-list-exec"); + return task; + } + private InstanceMetadata instanceMetadata(String instanceId) { return new InstanceMetadata(instanceId); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuControllerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuControllerTest.java new file mode 100644 index 00000000..573750f9 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuControllerTest.java @@ -0,0 +1,251 @@ +package com.nanri.aiimage.modules.permission.controller; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; +import com.nanri.aiimage.modules.permission.model.dto.ImageVideoDataPermissionUpdateRequest; +import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo; +import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo; +import com.nanri.aiimage.modules.permission.service.PermissionMenuService; +import jakarta.servlet.http.HttpServletRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.test.util.ReflectionTestUtils; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class PermissionMenuControllerTest { + + @Test + void effectivePermissionEndpointAllowsAuthenticatedNormalUserToReadSelf() { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + AdminUserEntity currentUser = new AdminUserEntity(); + currentUser.setId(9L); + currentUser.setRole("normal"); + PermissionMenuItemVo item = new PermissionMenuItemVo(); + item.setId(1L); + + when(authSupport.requireUser(request)).thenReturn(currentUser); + when(service.getUserColumnPermissions(currentUser, 9L, "app")).thenReturn(List.of(item)); + + var response = controller.getUserColumnPermissions(request, 9L, "app"); + + assertThat(response.getData()).extracting(PermissionMenuItemVo::getId).containsExactly(1L); + verify(authSupport).requireUser(request); + verify(authSupport, never()).requireAdmin(request); + } + + @Test + void internalProxyAllowsDatabaseBackedNormalUserToReadOwnEffectivePermissions() { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + ReflectionTestUtils.setField(controller, "internalToken", "shared-token"); + AdminUserEntity currentUser = new AdminUserEntity(); + currentUser.setId(9L); + currentUser.setRole("normal"); + + when(authSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录")); + when(request.getHeader("X-Internal-Token")).thenReturn("shared-token"); + when(request.getParameter("operatorId")).thenReturn("9"); + when(service.requireUserOperator(9L)).thenReturn(currentUser); + when(service.getUserColumnPermissions(currentUser, 9L, "app")).thenReturn(List.of()); + + controller.getUserColumnPermissions(request, 9L, "app"); + + verify(service).requireUserOperator(9L); + verify(service, never()).requireAdminOperator(anyLong()); + verify(service).getUserColumnPermissions(currentUser, 9L, "app"); + } + + @Test + void directIdEndpointStillRequiresAdminForSelf() { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + BusinessException authFailure = new BusinessException(403, "需要管理员权限"); + + when(authSupport.requireAdmin(request)).thenThrow(authFailure); + + assertThatThrownBy(() -> controller.getUserColumnIds(request, 9L, "app")) + .isSameAs(authFailure); + verify(authSupport, never()).requireUser(request); + verifyNoInteractions(service); + } + + @Test + void directIdUpdateForwardsMenuTypeScope() { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + AdminUserEntity operator = new AdminUserEntity(); + operator.setId(1L); + operator.setRole("super_admin"); + UserColumnPermissionUpdateRequest body = new UserColumnPermissionUpdateRequest(); + body.setColumnIds(List.of(12L)); + when(authSupport.requireAdmin(request)).thenReturn(operator); + + controller.updateUserColumnIds(request, 9L, "app", body); + + verify(service).updateUserColumnPermissions(eq(operator), eq(9L), eq(body), eq("app")); + } + + @Test + void internalProxyUsesSharedTokenAndDatabaseBackedOperator() { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + ReflectionTestUtils.setField(controller, "internalToken", "shared-token"); + AdminUserEntity operator = new AdminUserEntity(); + operator.setId(12L); + operator.setRole("admin"); + + when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "未登录")); + when(request.getHeader("X-Internal-Token")).thenReturn("shared-token"); + when(request.getParameter("operatorId")).thenReturn("12"); + when(service.requireAdminOperator(12L)).thenReturn(operator); + when(service.list("app")).thenReturn(List.of()); + + controller.listMenus(request, "app"); + + verify(service).requireAdminOperator(12L); + verify(service).list("app"); + verify(request, never()).getParameter("superAdmin"); + } + + @Test + void internalProxyRejectsMismatchedSharedToken() { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + ReflectionTestUtils.setField(controller, "internalToken", "shared-token"); + BusinessException authFailure = new BusinessException(401, "未登录"); + + when(authSupport.requireAdmin(request)).thenThrow(authFailure); + when(request.getHeader("X-Internal-Token")).thenReturn("wrong-token"); + + assertThatThrownBy(() -> controller.listMenus(request, "app")) + .isSameAs(authFailure); + verify(service, never()).requireAdminOperator(anyLong()); + verify(service, never()).list("app"); + } + + @Test + void internalProxyUsesSharedTokenFileWhenEnvironmentTokenIsUnconfigured(@TempDir Path tempDir) throws Exception { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + ReflectionTestUtils.setField(controller, "internalToken", ""); + Path tokenFile = tempDir.resolve("internal-token"); + Files.writeString(tokenFile, "file-token\n"); + ReflectionTestUtils.setField(controller, "internalTokenFile", tokenFile.toString()); + AdminUserEntity operator = new AdminUserEntity(); + operator.setId(12L); + operator.setRole("admin"); + + when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "未登录")); + when(request.getHeader("X-Internal-Token")).thenReturn("file-token"); + when(request.getParameter("operatorId")).thenReturn("12"); + when(service.requireAdminOperator(12L)).thenReturn(operator); + when(service.list("admin")).thenReturn(List.of()); + + controller.listMenus(request, "admin"); + + verify(service).requireAdminOperator(12L); + verify(service).list("admin"); + } + + @Test + void internalProxyRejectsMissingSharedTokenFileWhenEnvironmentTokenIsUnconfigured(@TempDir Path tempDir) { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + ReflectionTestUtils.setField(controller, "internalToken", ""); + ReflectionTestUtils.setField(controller, "internalTokenFile", tempDir.resolve("missing-token").toString()); + BusinessException authFailure = new BusinessException(401, "未登录"); + + when(authSupport.requireAdmin(request)).thenThrow(authFailure); + when(request.getHeader("X-Internal-Token")).thenReturn("guessed-token"); + when(request.getRemoteAddr()).thenReturn("127.0.0.1"); + when(request.getParameter("operatorId")).thenReturn("12"); + + assertThatThrownBy(() -> controller.listMenus(request, "admin")) + .isSameAs(authFailure); + verify(service, never()).requireAdminOperator(anyLong()); + verify(service, never()).list("admin"); + } + + @Test + void configuredInternalTokenTakesPrecedenceOverTokenFile(@TempDir Path tempDir) throws Exception { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + Path tokenFile = tempDir.resolve("internal-token"); + Files.writeString(tokenFile, "file-token\n"); + ReflectionTestUtils.setField(controller, "internalToken", "configured-token"); + ReflectionTestUtils.setField(controller, "internalTokenFile", tokenFile.toString()); + BusinessException authFailure = new BusinessException(401, "未登录"); + + when(authSupport.requireAdmin(request)).thenThrow(authFailure); + when(request.getHeader("X-Internal-Token")).thenReturn("file-token"); + when(request.getParameter("operatorId")).thenReturn("12"); + + assertThatThrownBy(() -> controller.listMenus(request, "admin")) + .isSameAs(authFailure); + verify(service, never()).requireAdminOperator(anyLong()); + verify(service, never()).list("admin"); + } + + @Test + void imageVideoPermissionEndpointsDelegateAuthenticatedOperator() { + AdminAuthSupport authSupport = mock(AdminAuthSupport.class); + PermissionMenuService service = mock(PermissionMenuService.class); + HttpServletRequest request = mock(HttpServletRequest.class); + PermissionMenuController controller = new PermissionMenuController(authSupport, service); + AdminUserEntity operator = new AdminUserEntity(); + operator.setId(1L); + operator.setRole("super_admin"); + ImageVideoDataPermissionUserVo permissionUser = new ImageVideoDataPermissionUserVo(); + permissionUser.setId(20L); + ImageVideoDataPermissionUpdateRequest body = new ImageVideoDataPermissionUpdateRequest(); + body.setUserIds(List.of(20L)); + + when(authSupport.requireAdmin(request)).thenReturn(operator); + when(service.listImageVideoDataPermissionUsers(operator)).thenReturn(List.of(permissionUser)); + when(service.updateImageVideoDataPermissionUsers(operator, List.of(20L))).thenReturn(1); + + var listResponse = controller.listImageVideoDataPermissionUsers(request); + var updateResponse = controller.updateImageVideoDataPermissionUsers(request, body); + + assertThat(listResponse.getData()).containsExactly(permissionUser); + assertThat(updateResponse.getData()).isEqualTo(1); + verify(authSupport, times(2)).requireAdmin(request); + verify(service).listImageVideoDataPermissionUsers(operator); + verify(service).updateImageVideoDataPermissionUsers(operator, List.of(20L)); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java index 2dd4a72f..956d279a 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java @@ -1,20 +1,33 @@ package com.nanri.aiimage.modules.permission.service; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper; import com.nanri.aiimage.modules.permission.mapper.PermissionMenuMapper; import com.nanri.aiimage.modules.permission.mapper.UserColumnPermissionMapper; +import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest; +import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest; import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest; import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity; import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity; +import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo; +import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo; +import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo; import org.junit.jupiter.api.Test; +import org.apache.ibatis.builder.MapperBuilderAssistant; import org.mockito.ArgumentCaptor; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -61,10 +74,505 @@ class PermissionMenuServiceTest { verify(permissionMapper, times(0)).insert(any(UserColumnPermissionEntity.class)); } + @Test + void expandsDirectParentGrantToDescendantsWithoutPersistingChildren() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0)); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L))); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(1L, null, "app", 10), + menu(2L, 1L, "app", 11), + menu(3L, 2L, "app", 12), + menu(4L, null, "app", 13))); + + List effective = service.getUserColumnPermissions(9L, "app"); + + assertThat(effective).extracting(PermissionMenuItemVo::getId) + .containsExactly(1L, 2L, 3L); + verify(permissionMapper, never()).insert(any(UserColumnPermissionEntity.class)); + } + + @Test + void directIdEndpointDoesNotReturnExpandedChildren() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0)); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L))); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(1L, null, "app", 10), + menu(2L, 1L, "app", 11))); + + UserColumnIdsVo direct = service.getUserColumnIds(9L, "app"); + + assertThat(direct.getColumnIds()).containsExactly(1L); + } + + @Test + void leafOnlyEffectivePermissionIncludesRootKeyWithoutGrantingAncestor() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0)); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 2L))); + PermissionMenuEntity root = menu(1L, null, "app", 1); + root.setColumnKey("brand_front_tools"); + PermissionMenuEntity leaf = menu(2L, 1L, "app", 2); + leaf.setColumnKey("brand"); + when(menuMapper.selectList(any())).thenReturn(List.of(root, leaf)); + + List effective = service.getUserColumnPermissions(9L, "app"); + + assertThat(effective).extracting(PermissionMenuItemVo::getId).containsExactly(2L); + assertThat(effective.getFirst().getRootColumnKey()).isEqualTo("brand_front_tools"); + } + + @Test + void menuCatalogIncludesRootKeyFromLoadedHierarchy() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + PermissionMenuEntity root = menu(1L, null, "app", 1); + root.setColumnKey("brand_front_tools"); + PermissionMenuEntity leaf = menu(2L, 1L, "app", 2); + leaf.setColumnKey("brand"); + when(menuMapper.selectList(any())).thenReturn(List.of(root, leaf)); + + List result = service.list("app"); + + assertThat(result).extracting(PermissionMenuItemVo::getId).containsExactly(1L, 2L); + assertThat(result).extracting(PermissionMenuItemVo::getRootColumnKey) + .containsExactly("brand_front_tools", "brand_front_tools"); + } + + @Test + void superAdminDirectIdEndpointReturnsOnlyPersistedIds() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(userMapper.selectById(1L)).thenReturn(user(1L, "super_admin", 1)); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 2L))); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(1L, null, "app", 1), + menu(2L, 1L, "app", 2))); + + UserColumnIdsVo direct = service.getUserColumnIds(1L, "app"); + + assertThat(direct.getColumnIds()).containsExactly(2L); + } + + @Test + void superAdminEffectivePermissionsRemainVirtualFullCatalog() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(userMapper.selectById(1L)).thenReturn(user(1L, "super_admin", 1)); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(1L, null, "app", 1), + menu(2L, 1L, "app", 2))); + + List effective = service.getUserColumnPermissions(1L, "app"); + + assertThat(effective).extracting(PermissionMenuItemVo::getId).containsExactly(1L, 2L); + verify(permissionMapper, never()).selectList(any()); + } + + @Test + void normalUserCanReadOwnEffectivePermissions() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + AdminUserEntity currentUser = user(9L, "normal", 0); + when(userMapper.selectById(9L)).thenReturn(currentUser); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L))); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(1L, null, "app", 10), + menu(2L, 1L, "app", 11))); + + List effective = service.getUserColumnPermissions(currentUser, 9L, "app"); + + assertThat(effective).extracting(PermissionMenuItemVo::getId) + .containsExactly(1L, 2L); + } + + @Test + void normalUserCannotReadAnotherUsersEffectivePermissions() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + AdminUserEntity currentUser = user(9L, "normal", 0); + when(userMapper.selectById(10L)).thenReturn(user(10L, "normal", 0)); + + assertThatThrownBy(() -> service.getUserColumnPermissions(currentUser, 10L, "app")) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("管理员权限"); + verify(permissionMapper, never()).selectList(any()); + } + + @Test + void rejectsParentFromAnotherMenuType() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + PermissionMenuEntity parent = menu(7L, null, "admin", 1); + when(menuMapper.selectById(7L)).thenReturn(parent); + PermissionMenuCreateRequest request = createRequest(7L, "app"); + + assertThatThrownBy(() -> service.create(request)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("同一类型"); + verify(menuMapper, never()).insert(any(PermissionMenuEntity.class)); + } + + @Test + void rejectsParentCycleWhenUpdatingMenu() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + PermissionMenuEntity current = menu(1L, null, "app", 1); + PermissionMenuEntity parent = menu(2L, 3L, "app", 2); + PermissionMenuEntity ancestor = menu(3L, 1L, "app", 3); + when(menuMapper.selectById(1L)).thenReturn(current); + when(menuMapper.selectById(2L)).thenReturn(parent); + when(menuMapper.selectById(3L)).thenReturn(ancestor); + + PermissionMenuUpdateRequest request = new PermissionMenuUpdateRequest(); + request.setName("changed"); + request.setColumnKey("changed"); + request.setMenuType("app"); + request.setRoutePath("changed"); + request.setParentId(2L); + + assertThatThrownBy(() -> service.update(1L, request)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("自己的子级"); + verify(menuMapper, never()).updateById(any(PermissionMenuEntity.class)); + } + + @Test + void rejectsChangingMenuTypeWhileChildrenRemain() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(menuMapper.selectById(1L)).thenReturn(menu(1L, null, "app", 1)); + when(menuMapper.selectCount(any())).thenReturn(1L); + PermissionMenuUpdateRequest request = new PermissionMenuUpdateRequest(); + request.setName("changed"); + request.setColumnKey("changed"); + request.setMenuType("admin"); + request.setRoutePath("changed"); + + assertThatThrownBy(() -> service.update(1L, request)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("子菜单"); + verify(menuMapper, never()).updateById(any(PermissionMenuEntity.class)); + } + + @Test + void refusesDeletingParentMenuUntilChildrenAreRemoved() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(menuMapper.selectById(1L)).thenReturn(menu(1L, null, "app", 1)); + when(menuMapper.selectCount(any())).thenReturn(1L); + + assertThatThrownBy(() -> service.delete(1L)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("子菜单"); + verify(menuMapper, never()).deleteById(any(Long.class)); + verify(permissionMapper, never()).delete(any()); + } + + @Test + void ordinaryAdminCannotAssignMenuOutsideOwnEffectiveSet() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + AdminUserEntity operator = user(10L, "admin", 1); + AdminUserEntity target = user(20L, "normal", 0); + target.setCreatedById(10L); + when(userMapper.selectById(20L)).thenReturn(target); + when(userMapper.selectById(10L)).thenReturn(operator); + when(userMapper.selectOne(any())).thenReturn(null); + when(menuMapper.selectCount(any())).thenReturn(1L); + when(menuMapper.selectOne(any())).thenReturn(null); + when(menuMapper.selectList(any())).thenReturn(List.of(menu(1L, null, "app", 1))); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L))); + + UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest(); + request.setColumnIds(List.of(2L)); + + assertThatThrownBy(() -> service.updateUserColumnPermissions(operator, 20L, request)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("自己已有"); + verify(permissionMapper, never()).delete(any()); + } + + @Test + void ordinaryAdminCanAssignDescendantGrantedThroughParent() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + AdminUserEntity operator = user(10L, "admin", 1); + AdminUserEntity target = user(20L, "normal", 0); + target.setCreatedById(10L); + when(userMapper.selectById(20L)).thenReturn(target); + when(userMapper.selectOne(any())).thenReturn(null); + when(menuMapper.selectCount(any())).thenReturn(1L); + when(menuMapper.selectOne(any())).thenReturn(null); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(1L, null, "app", 1), + menu(2L, 1L, "app", 2))); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L))); + UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest(); + request.setColumnIds(List.of(2L)); + + service.updateUserColumnPermissions(operator, 20L, request); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class); + verify(permissionMapper).insert(captor.capture()); + assertThat(captor.getValue().getColumnId()).isEqualTo(2L); + } + + @Test + void ordinaryAdminReplacementPreservesDirectGrantsOutsideEffectiveScope() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + AdminUserEntity operator = user(10L, "admin", 1); + AdminUserEntity target = user(20L, "normal", 0); + target.setCreatedById(10L); + when(userMapper.selectById(20L)).thenReturn(target); + when(menuMapper.selectCount(any())).thenReturn(1L); + when(menuMapper.selectOne(any())).thenReturn(null); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(1L, null, "app", 1), + menu(2L, null, "app", 2))); + when(permissionMapper.selectList(any())) + .thenReturn(List.of(grant(10L, 1L)), List.of(grant(20L, 2L))); + UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest(); + request.setColumnIds(List.of(1L)); + + service.updateUserColumnPermissions(operator, 20L, request); + + ArgumentCaptor inserted = + ArgumentCaptor.forClass(UserColumnPermissionEntity.class); + verify(permissionMapper, times(2)).insert(inserted.capture()); + assertThat(inserted.getAllValues()) + .extracting(UserColumnPermissionEntity::getColumnId) + .containsExactly(1L, 2L); + } + + @Test + void replacementStoresOnlyDirectRequestedParentId() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0)); + when(menuMapper.selectCount(any())).thenReturn(1L); + when(menuMapper.selectOne(any())).thenReturn(null); + UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest(); + request.setColumnIds(List.of(1L)); + + service.updateUserColumnPermissions(9L, request); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class); + verify(permissionMapper).insert(captor.capture()); + assertThat(captor.getValue().getColumnId()).isEqualTo(1L); + } + + @Test + void appScopedReplacementDeletesOnlyAppDirectGrants() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + + AdminUserEntity operator = user(1L, "super_admin", 1); + when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0)); + when(menuMapper.selectList(any())).thenReturn(List.of( + menu(11L, null, "app", 1), + menu(12L, 11L, "app", 2))); + when(menuMapper.selectOne(any())).thenReturn(null); + UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest(); + request.setColumnIds(List.of(12L)); + + service.updateUserColumnPermissions(operator, 9L, request, PermissionMenuService.MENU_TYPE_APP); + + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + UserColumnPermissionEntity.class); + ArgumentCaptor> deleted = + ArgumentCaptor.forClass(LambdaUpdateWrapper.class); + verify(permissionMapper).delete(deleted.capture()); + assertThat(deleted.getValue().getSqlSegment()).contains("column_id", "IN"); + assertThat(deleted.getValue().getParamNameValuePairs().values()) + .contains(9L, 11L, 12L); + ArgumentCaptor inserted = + ArgumentCaptor.forClass(UserColumnPermissionEntity.class); + verify(permissionMapper).insert(inserted.capture()); + assertThat(inserted.getValue().getColumnId()).isEqualTo(12L); + } + + @Test + void explicitLowestIdAdminCannotManageImageVideoDataPermissions() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + AdminUserEntity explicitAdmin = user(1L, "admin", 1); + + assertThatThrownBy(() -> service.listImageVideoDataPermissionUsers(explicitAdmin)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("超级管理员"); + assertThatThrownBy(() -> service.updateImageVideoDataPermissionUsers(explicitAdmin, List.of(2L))) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("超级管理员"); + verify(menuMapper, never()).selectOne(any()); + verify(permissionMapper, never()).delete(any()); + } + + @Test + void explicitNormalRoleCannotActAsInternalAdminDespiteLegacyFlag() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + when(userMapper.selectById(1L)).thenReturn(user(1L, "normal", 1)); + + assertThatThrownBy(() -> service.requireAdminOperator(1L)) + .isInstanceOf(BusinessException.class) + .hasMessageContaining("管理员权限"); + } + + @Test + void superAdminCanListImageVideoDataPermissionUsers() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + AdminUserEntity operator = user(99L, "super_admin", 1); + AdminUserEntity explicitAdmin = user(1L, "admin", 1); + explicitAdmin.setUsername("admin"); + AdminUserEntity normal = user(2L, "normal", 0); + normal.setUsername("normal"); + + when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission()); + when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 75L))); + when(userMapper.selectList(any())).thenReturn(List.of(operator, explicitAdmin, normal)); + + List result = service.listImageVideoDataPermissionUsers(operator); + + assertThat(result).extracting(ImageVideoDataPermissionUserVo::getId).containsExactly(1L, 2L); + assertThat(result.get(0).isGranted()).isTrue(); + assertThat(result.get(1).isGranted()).isFalse(); + } + + @Test + void imageVideoUpdateDeletesOnlyProtectedColumnGrants() { + PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); + UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class); + AdminUserMapper userMapper = mock(AdminUserMapper.class); + PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper); + AdminUserEntity operator = user(99L, "super_admin", 1); + AdminUserEntity explicitAdmin = user(1L, "admin", 1); + AdminUserEntity normal = user(2L, "normal", 0); + + when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission()); + when(userMapper.selectList(any())).thenReturn(List.of(operator, explicitAdmin, normal)); + + int grantedCount = service.updateImageVideoDataPermissionUsers(operator, List.of(2L)); + + assertThat(grantedCount).isEqualTo(1); + verify(permissionMapper).deleteByMap(Map.of("column_id", 75L)); + + ArgumentCaptor inserted = + ArgumentCaptor.forClass(UserColumnPermissionEntity.class); + verify(permissionMapper).insert(inserted.capture()); + assertThat(inserted.getValue().getUserId()).isEqualTo(2L); + assertThat(inserted.getValue().getColumnId()).isEqualTo(75L); + } + + private PermissionMenuCreateRequest createRequest(Long parentId, String menuType) { + PermissionMenuCreateRequest request = new PermissionMenuCreateRequest(); + request.setName("child"); + request.setColumnKey("child"); + request.setMenuType(menuType); + request.setRoutePath("child"); + request.setParentId(parentId); + return request; + } + + private PermissionMenuEntity menu(Long id, Long parentId, String menuType, int sortOrder) { + PermissionMenuEntity entity = new PermissionMenuEntity(); + entity.setId(id); + entity.setParentId(parentId); + entity.setMenuType(menuType); + entity.setColumnKey("menu-" + id); + entity.setRoutePath("route-" + id); + entity.setSortOrder(sortOrder); + return entity; + } + + private UserColumnPermissionEntity grant(Long userId, Long columnId) { + UserColumnPermissionEntity entity = new UserColumnPermissionEntity(); + entity.setUserId(userId); + entity.setColumnId(columnId); + return entity; + } + + private AdminUserEntity user(Long id, String role, int isAdmin) { + AdminUserEntity entity = new AdminUserEntity(); + entity.setId(id); + entity.setRole(role); + entity.setIsAdmin(isAdmin); + return entity; + } + private PermissionMenuEntity imageVideoPermission() { PermissionMenuEntity entity = new PermissionMenuEntity(); entity.setId(75L); entity.setColumnKey("admin_image_video_tasks"); return entity; } + + private PermissionMenuEntity imageVideoDataPermission() { + PermissionMenuEntity entity = new PermissionMenuEntity(); + entity.setId(75L); + entity.setColumnKey("admin_image_video_task_data"); + return entity; + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServiceTest.java index c3a91aaa..8d45eac9 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServiceTest.java @@ -1,5 +1,6 @@ package com.nanri.aiimage.modules.publish.service; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.MybatisConfiguration; import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; import com.fasterxml.jackson.databind.ObjectMapper; @@ -19,10 +20,15 @@ import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity; import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; import org.apache.ibatis.builder.MapperBuilderAssistant; @@ -32,6 +38,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; @@ -39,7 +46,12 @@ import org.springframework.transaction.support.TransactionTemplate; import java.time.LocalDateTime; import java.io.File; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -68,6 +80,8 @@ class PublishTaskServiceTest { TableInfoHelper.initTableInfo(assistant, FileResultEntity.class); TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class); TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class); } @Mock private LocalFileStorageService localFileStorageService; @@ -77,17 +91,26 @@ class PublishTaskServiceTest { @Mock private PublishItemMapper publishItemMapper; @Mock private FileTaskMapper fileTaskMapper; @Mock private FileResultMapper fileResultMapper; + @Mock private TaskChunkMapper taskChunkMapper; + @Mock private TaskScopeStateMapper taskScopeStateMapper; @Mock private TaskFileJobService taskFileJobService; @Mock private TaskDistributedLockService taskDistributedLockService; + @Mock private TransientPayloadStorageService transientPayloadStorageService; @Mock private OssStorageService ossStorageService; - @Mock private ObjectMapper objectMapper; + @Spy private ObjectMapper objectMapper = new ObjectMapper(); @Mock private TransactionTemplate transactionTemplate; @Mock private InstanceMetadata instanceMetadata; @InjectMocks private PublishTaskService service; + private final List storedChunks = new ArrayList<>(); + private final List storedScopes = new ArrayList<>(); + private final Map rustfsPayloads = new LinkedHashMap<>(); + private int nextPayloadId; + @BeforeEach void executeTransactionsInline() { + configureChunkStorage(); lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); lenient().when(transactionTemplate.execute(any())).thenAnswer(invocation -> { TransactionCallback callback = invocation.getArgument(0); @@ -159,6 +182,62 @@ class PublishTaskServiceTest { verifyNoInteractions(publishItemMapper); } + @Test + void failedTaskOwnedByAnotherInstanceCanBeDeletedLocally() throws Exception { + long taskId = 110L; + FileTaskEntity task = task(taskId, 7L, "FAILED"); + task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}"); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(fileResultMapper.selectList(any())).thenReturn(List.of()); + + service.deleteTask(taskId, 7L); + + verify(taskFileJobService).deleteTaskJobs(taskId, PublishTaskService.MODULE_TYPE); + verify(publishItemMapper).delete(any()); + verify(publishFileMapper).delete(any()); + verify(fileResultMapper).delete(any()); + verify(fileTaskMapper).deleteById(taskId); + verify(objectMapper, never()).readTree(any(String.class)); + } + + @Test + void runningTaskOwnedByAnotherInstanceStillRequiresRouting() throws Exception { + long taskId = 111L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}"); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(objectMapper.readTree(task.getResultJson())).thenReturn( + new ObjectMapper().readTree(task.getResultJson())); + + TaskOwnerMismatchException error = assertThrows(TaskOwnerMismatchException.class, + () -> service.deleteTask(taskId, 7L)); + + assertEquals(taskId, error.getTaskId()); + assertEquals("instance-b", error.getOwnerInstanceId()); + verifyNoInteractions(taskFileJobService, publishItemMapper, publishFileMapper, fileResultMapper); + verify(fileTaskMapper, never()).deleteById(taskId); + } + + @Test + void historyForFailedTaskOwnedByAnotherInstanceCanBeDeletedLocally() throws Exception { + long taskId = 112L; + long resultId = 212L; + FileTaskEntity task = task(taskId, 7L, "FAILED"); + task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}"); + FileResultEntity result = new FileResultEntity(); + result.setId(resultId); + result.setTaskId(taskId); + result.setModuleType(PublishTaskService.MODULE_TYPE); + when(fileResultMapper.selectById(resultId)).thenReturn(result); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(fileResultMapper.selectList(any())).thenReturn(List.of(result)); + + service.deleteHistory(resultId, 7L); + + verify(fileTaskMapper).deleteById(taskId); + verify(objectMapper, never()).readTree(any(String.class)); + } + @Test void resultCallbackRejectsAnotherUserBeforeReadingOrReplacingRows() { long taskId = 101L; @@ -258,6 +337,242 @@ class PublishTaskServiceTest { verify(lock).close(); } + @Test + void resultCallbackMergesOutOfOrderChunksOnlyAfterAllChunksArrive() { + long taskId = 113L; + long fileId = 213L; + long resultId = 313L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "分片.xlsx"); + FileResultEntity result = result(taskId, resultId); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(publishFileMapper.selectList(any())).thenReturn(List.of(file)); + when(fileResultMapper.selectOne(any())).thenReturn(result); + when(publishItemMapper.selectCount(any())).thenReturn(2L); + + service.submitResult(taskId, chunkResultRequest(7L, fileId, 2, 2, List.of(row("2")))); + + assertEquals("RUNNING", file.getStatus()); + assertEquals(1, storedChunks.size()); + assertEquals(1, storedScopes.getFirst().getReceivedChunkCount()); + assertEquals(0, storedScopes.getFirst().getCompleted()); + verify(publishItemMapper, never()).delete(any()); + verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any()); + + service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))); + + assertEquals("SUCCESS", file.getStatus()); + assertEquals(2, storedChunks.size()); + assertEquals(2, storedScopes.getFirst().getReceivedChunkCount()); + assertEquals(1, storedScopes.getFirst().getCompleted()); + org.mockito.ArgumentCaptor> rowsCaptor = + org.mockito.ArgumentCaptor.forClass(List.class); + verify(publishItemMapper).insertBatch(rowsCaptor.capture()); + assertEquals(List.of("1", "2"), rowsCaptor.getValue().stream() + .map(PublishItemEntity::getSourceId) + .toList()); + verify(taskFileJobService).enqueueAssembleResult( + taskId, PublishTaskService.MODULE_TYPE, resultId, + "task:" + taskId + ":owner:instance-a"); + } + + @Test + void identicalResultChunkRetryIsIdempotent() { + long taskId = 114L; + long fileId = 214L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "重试.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + PublishSubmitResultRequest request = chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(publishFileMapper.selectList(any())).thenReturn(List.of(file)); + when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 314L)); + + service.submitResult(taskId, request); + service.submitResult(taskId, request); + + assertEquals(1, storedChunks.size()); + assertEquals(1, rustfsPayloads.size()); + verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class)); + verify(publishItemMapper, never()).delete(any()); + } + + @Test + void resultChunkRetryWithDifferentContentIsRejected() { + long taskId = 115L; + long fileId = 215L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "冲突.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(publishFileMapper.selectList(any())).thenReturn(List.of(file)); + when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 315L)); + + service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))); + BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult( + taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("different"))))); + + assertTrue(error.getMessage().contains("不同内容")); + assertEquals(1, storedChunks.size()); + assertEquals(1, rustfsPayloads.size()); + verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class)); + } + + @Test + void resultChunksRejectChangedChunkTotal() { + long taskId = 116L; + long fileId = 216L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "总片数冲突.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(publishFileMapper.selectList(any())).thenReturn(List.of(file)); + when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 316L)); + + service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))); + BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult( + taskId, chunkResultRequest(7L, fileId, 2, 3, List.of(row("2"))))); + + assertTrue(error.getMessage().contains("chunk_total")); + assertEquals(1, storedChunks.size()); + } + + @Test + void resultChunkRejectsCallbackWhenRustfsIsUnavailable() { + long taskId = 117L; + long fileId = 217L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "RustFS未配置.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(false); + + BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult( + taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))))); + + assertTrue(error.getMessage().contains("RustFS 未配置")); + assertEquals(0, storedChunks.size()); + verify(publishItemMapper, never()).delete(any()); + } + + @Test + void resultChunkRejectsLocalFallbackPointer() { + long taskId = 118L; + long fileId = 218L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "本地回退.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + String localPointer = "local:instance-a/publish/chunk-1"; + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(transientPayloadStorageService.storeChunkPayloadVersioned(any(), any(), any(), any(), any())) + .thenReturn(localPointer); + when(transientPayloadStorageService.extractPointer(localPointer)).thenReturn(localPointer); + + BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult( + taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))))); + + assertTrue(error.getMessage().contains("必须写入 RustFS")); + assertEquals(0, storedChunks.size()); + verify(transientPayloadStorageService).deletePayloadIfPresent(localPointer); + } + + @Test + void resultChunkDeletesRustfsPayloadWhenMetadataInsertFails() { + long taskId = 120L; + long fileId = 220L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "元数据失败.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(taskChunkMapper.insert(any(TaskChunkEntity.class))) + .thenThrow(new IllegalStateException("database unavailable")); + + assertThrows(IllegalStateException.class, () -> service.submitResult( + taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))))); + + assertTrue(rustfsPayloads.isEmpty()); + assertTrue(storedChunks.isEmpty()); + verify(transientPayloadStorageService).deletePayloadIfPresent(any()); + } + + @Test + void deletingTaskRemovesPublishChunksAndRustfsPayloads() { + long taskId = 119L; + long fileId = 219L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "待删除.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(publishFileMapper.selectList(any())).thenReturn(List.of(file)); + when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 319L)); + + service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))); + String pointer = storedChunks.getFirst().getPayloadJson(); + assertEquals(1, storedChunks.size()); + assertEquals(1, storedScopes.size()); + assertEquals(1, rustfsPayloads.size()); + + when(fileResultMapper.selectList(any())).thenReturn(List.of()); + service.deleteTask(taskId, 7L); + + assertTrue(storedChunks.isEmpty()); + assertTrue(storedScopes.isEmpty()); + assertTrue(rustfsPayloads.isEmpty()); + verify(transientPayloadStorageService).deletePayloadIfPresent(pointer); + verify(fileTaskMapper).deleteById(taskId); + } + + @Test + void allFailedTaskRemovesPreviouslyUploadedChunksAfterCommit() { + long taskId = 121L; + long fileId = 221L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "执行失败.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(publishFileMapper.selectList(any())).thenReturn(List.of(file)); + when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 321L)); + + service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))); + assertEquals(1, storedChunks.size()); + + service.submitResult(taskId, errorResultRequest(7L, fileId, "Python 执行失败")); + + assertEquals("FAILED", task.getStatus()); + assertEquals("FAILED", file.getStatus()); + assertTrue(storedChunks.isEmpty()); + assertTrue(storedScopes.isEmpty()); + assertTrue(rustfsPayloads.isEmpty()); + } + @Test void activateFileRejectsSecondRunningFileWhileHoldingTaskLock() { long taskId = 104L; @@ -346,6 +661,53 @@ class PublishTaskServiceTest { verify(lock).close(); } + @Test + void staleTaskRecoversReceivedChunksAndEnqueuesAssembly() { + long taskId = 118L; + long fileId = 218L; + long resultId = 318L; + FileTaskEntity task = task(taskId, 7L, "RUNNING"); + task.setSourceFileCount(1); + task.setUpdatedAt(LocalDateTime.now().minusHours(1)); + PublishFileEntity file = file(taskId, fileId, "RUNNING", "部分结果.xlsx"); + file.setTotalRows(2); + FileResultEntity result = result(taskId, resultId); + TaskDistributedLockService.LockHandle submitLock = mock(TaskDistributedLockService.LockHandle.class); + TaskDistributedLockService.LockHandle staleLock = mock(TaskDistributedLockService.LockHandle.class); + + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(submitLock); + when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId, 0L)).thenReturn(staleLock); + when(fileTaskMapper.selectList(any())).thenReturn(List.of(task)); + when(fileTaskMapper.selectById(taskId)).thenReturn(task); + when(publishFileMapper.selectById(fileId)).thenReturn(file); + when(publishFileMapper.selectList(any())).thenReturn(List.of(file)); + when(fileResultMapper.selectOne(any())).thenReturn(result); + when(taskFileJobService.countUnfinishedAssembleJobs(taskId, PublishTaskService.MODULE_TYPE)) + .thenReturn(0L); + + service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))); + task.setUpdatedAt(LocalDateTime.now().minusHours(1)); + service.failStaleTasks(); + + assertEquals("SUCCESS", file.getStatus()); + assertEquals(1, file.getTotalRows()); + assertEquals(1, file.getProcessedRows()); + assertTrue(file.getErrorMessage().contains("已保留 1 行")); + assertEquals(1, task.getSuccessFileCount()); + assertEquals(0, task.getFailedFileCount()); + org.mockito.ArgumentCaptor> rowsCaptor = + org.mockito.ArgumentCaptor.forClass(List.class); + verify(publishItemMapper).insertBatch(rowsCaptor.capture()); + assertEquals(List.of("1"), rowsCaptor.getValue().stream() + .map(PublishItemEntity::getSourceId) + .toList()); + verify(taskFileJobService).enqueueAssembleResult( + taskId, PublishTaskService.MODULE_TYPE, resultId, + "task:" + taskId + ":owner:instance-a"); + verify(submitLock).close(); + verify(staleLock).close(); + } + @Test void staleTaskWithoutSuccessfulFilesFailsTaskAndResult() { long taskId = 107L; @@ -386,6 +748,140 @@ class PublishTaskServiceTest { verify(lock).close(); } + private void configureChunkStorage() { + storedChunks.clear(); + storedScopes.clear(); + rustfsPayloads.clear(); + nextPayloadId = 0; + + lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true); + lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned( + any(), any(), any(), any(), any())).thenAnswer(invocation -> { + String pointer = "rustfs:test/publish/chunk-" + (++nextPayloadId); + rustfsPayloads.put(pointer, invocation.getArgument(4)); + return pointer; + }); + lenient().when(transientPayloadStorageService.extractPointer(any())).thenAnswer(invocation -> { + String value = invocation.getArgument(0); + return value != null && value.startsWith("rustfs:") ? value : null; + }); + lenient().when(transientPayloadStorageService.resolvePayload(any(), any())).thenAnswer(invocation -> { + String pointer = invocation.getArgument(0); + String payload = rustfsPayloads.get(pointer); + if (payload == null) { + throw new IllegalStateException("missing test RustFS payload: " + pointer); + } + return payload; + }); + lenient().doAnswer(invocation -> { + rustfsPayloads.remove(invocation.getArgument(0)); + return null; + }).when(transientPayloadStorageService).deletePayloadIfPresent(any()); + + lenient().when(taskChunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> { + TaskChunkEntity chunk = invocation.getArgument(0); + chunk.setId((long) storedChunks.size() + 1L); + storedChunks.add(chunk); + return 1; + }); + lenient().when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + Integer chunkIndex = queryInteger(query); + return storedChunks.stream() + .filter(chunk -> matchesChunk(chunk, taskId, scopeHash, chunkIndex)) + .findFirst() + .orElse(null); + }); + lenient().when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + return storedChunks.stream() + .filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null)) + .count(); + }); + lenient().when(taskChunkMapper.selectList(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + return storedChunks.stream() + .filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null)) + .sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex)) + .toList(); + }); + lenient().when(taskChunkMapper.delete(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + int before = storedChunks.size(); + storedChunks.removeIf(chunk -> taskId == null || Objects.equals(taskId, chunk.getTaskId())); + return before - storedChunks.size(); + }); + + lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> { + TaskScopeStateEntity scope = invocation.getArgument(0); + scope.setId((long) storedScopes.size() + 1L); + storedScopes.add(scope); + return 1; + }); + lenient().when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + return storedScopes.stream() + .filter(scope -> Objects.equals(taskId, scope.getTaskId())) + .filter(scope -> Objects.equals(scopeHash, scope.getScopeHash())) + .findFirst() + .orElse(null); + }); + lenient().when(taskScopeStateMapper.updateById(any(TaskScopeStateEntity.class))).thenReturn(1); + lenient().when(taskScopeStateMapper.delete(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + int before = storedScopes.size(); + storedScopes.removeIf(scope -> taskId == null || Objects.equals(taskId, scope.getTaskId())); + return before - storedScopes.size(); + }); + } + + private boolean matchesChunk(TaskChunkEntity chunk, + Long taskId, + String scopeHash, + Integer chunkIndex) { + return (taskId == null || Objects.equals(taskId, chunk.getTaskId())) + && (scopeHash == null || Objects.equals(scopeHash, chunk.getScopeHash())) + && (chunkIndex == null || Objects.equals(chunkIndex, chunk.getChunkIndex())); + } + + private Long queryLong(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(Long.class::isInstance) + .map(Long.class::cast) + .findFirst() + .orElse(null); + } + + private Integer queryInteger(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(Integer.class::isInstance) + .map(Integer.class::cast) + .findFirst() + .orElse(null); + } + + private String queryScopeHash(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .filter(value -> value.length() == 64) + .findFirst() + .orElse(null); + } + private FileTaskEntity task(long taskId, long userId, String status) { FileTaskEntity task = new FileTaskEntity(); task.setId(taskId); @@ -407,11 +903,30 @@ class PublishTaskServiceTest { return file; } + private FileResultEntity result(long taskId, long resultId) { + FileResultEntity result = new FileResultEntity(); + result.setId(resultId); + result.setTaskId(taskId); + result.setModuleType(PublishTaskService.MODULE_TYPE); + result.setSuccess(0); + return result; + } + private PublishSubmitResultRequest resultRequest(long userId, - long fileId, - List rows) { + long fileId, + List rows) { + return chunkResultRequest(userId, fileId, null, null, rows); + } + + private PublishSubmitResultRequest chunkResultRequest(long userId, + long fileId, + Integer chunkIndex, + Integer chunkTotal, + List rows) { PublishResultFileDto file = new PublishResultFileDto(); file.setFileId(fileId); + file.setChunkIndex(chunkIndex); + file.setChunkTotal(chunkTotal); file.setRows(rows); PublishSubmitResultRequest request = new PublishSubmitResultRequest(); request.setUserId(userId); @@ -419,6 +934,18 @@ class PublishTaskServiceTest { return request; } + private PublishSubmitResultRequest errorResultRequest(long userId, + long fileId, + String error) { + PublishResultFileDto file = new PublishResultFileDto(); + file.setFileId(fileId); + file.setError(error); + PublishSubmitResultRequest request = new PublishSubmitResultRequest(); + request.setUserId(userId); + request.setFiles(List.of(file)); + return request; + } + private PublishRowDto row(String sourceId) { PublishRowDto row = new PublishRowDto(); row.setSourceId(sourceId); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java new file mode 100644 index 00000000..331969b2 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java @@ -0,0 +1,56 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ShopDataCrawlExcelAssemblyServiceTest { + @TempDir Path tempDir; + + @Test + void writesRowsIntoStrictTemplateWithoutChangingSheetOrderOrHeaders() throws Exception { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate("2026-07-25"); + row.setAsin("B012345678"); + row.setInventorySales("11"); + row.setSalesRank("22"); + row.setPageViews("33"); + row.setUnitsSold("44"); + row.setPrice("12.50"); + row.setRecommendedOffer("12.00"); + ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto(); + country.setCountry("UK"); + country.setItems(List.of(row)); + ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo(); + item.setSuccess(true); + item.setCountryResults(List.of(country)); + + File output = tempDir.resolve("result.xlsx").toFile(); + new ShopDataCrawlExcelAssemblyService().writeWorkbook(output, List.of(item)); + + try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { + assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS, + java.util.stream.IntStream.range(0, workbook.getNumberOfSheets()) + .mapToObj(i -> workbook.getSheetAt(i).getSheetName()).toList()); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + for (int columnIndex = 0; columnIndex < ShopDataCrawlExcelAssemblyService.HEADERS.size(); columnIndex++) { + assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(columnIndex), + workbook.getSheetAt(i).getRow(0).getCell(columnIndex).getStringCellValue()); + } + } + assertEquals("2026-07-25", workbook.getSheet("英国").getRow(1).getCell(0).getStringCellValue()); + assertEquals("B012345678", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue()); + assertEquals(0, workbook.getSheet("德国").getLastRowNum()); + } + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveServiceTest.java new file mode 100644 index 00000000..d556d0f4 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveServiceTest.java @@ -0,0 +1,39 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.modules.shopkey.service.ShopManageService; +import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ShopDataCrawlResolveServiceTest { + @Mock com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper candidateMapper; + @Mock com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper countryPrefMapper; + @Mock com.fasterxml.jackson.databind.ObjectMapper objectMapper; + @Mock ZiniaoShopSwitchService ziniaoShopSwitchService; + @Mock ShopManageService shopManageService; + @InjectMocks ShopDataCrawlResolveService service; + + @Test + void requiresManagedShopBeforeLookingUpZiniaoIndex() { + ZiniaoShopMatchResultVo match = new ZiniaoShopMatchResultVo(); + match.setMatched(true); + when(ziniaoShopSwitchService.normalizeShopName(" Demo ")).thenReturn("Demo"); + when(ziniaoShopSwitchService.findIndexedStoreByName("Demo", false)).thenReturn(match); + + assertSame(match, service.requireMatchedShop(" Demo ")); + + var order = inOrder(shopManageService, ziniaoShopSwitchService); + order.verify(ziniaoShopSwitchService).normalizeShopName(" Demo "); + order.verify(shopManageService).requireShopByName("Demo"); + order.verify(ziniaoShopSwitchService).findIndexedStoreByName("Demo", false); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java new file mode 100644 index 00000000..80b7fa09 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java @@ -0,0 +1,539 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.TaskPressureProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TaskResultItemService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ShopDataCrawlTaskServiceChunkTest { + + private static final String MODULE_TYPE = "SHOP_DATA_CRAWL"; + private static final String SHOP_NAME = "Demo Shop"; + + @BeforeAll + static void initializeMybatisMetadata() { + MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), ""); + TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class); + TableInfoHelper.initTableInfo(assistant, FileResultEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class); + } + + @Mock private FileTaskMapper fileTaskMapper; + @Mock private FileResultMapper fileResultMapper; + @Mock private ShopDataCrawlResolveService shopDataCrawlResolveService; + @Mock private ShopDataCrawlExcelAssemblyService excelAssemblyService; + @Mock private ShopDataCrawlTaskCacheService taskCacheService; + @Mock private OssStorageService ossStorageService; + @Mock private ZiniaoShopSwitchService ziniaoShopSwitchService; + @Mock private TaskPressureProperties taskPressureProperties; + @Mock private TaskFileJobService taskFileJobService; + @Mock private TaskResultItemService taskResultItemService; + @Mock private TaskProgressSnapshotService taskProgressSnapshotService; + @Mock private TaskDistributedLockService taskDistributedLockService; + @Mock private TaskChunkMapper taskChunkMapper; + @Mock private TaskScopeStateMapper taskScopeStateMapper; + @Mock private TransientPayloadStorageService transientPayloadStorageService; + @Mock private InstanceMetadata instanceMetadata; + @Spy private ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); + + private ShopDataCrawlTaskService service; + + private final List storedChunks = new ArrayList<>(); + private final List storedScopes = new ArrayList<>(); + private final Map rustfsPayloads = new LinkedHashMap<>(); + private FileTaskEntity task; + private FileResultEntity result; + private int nextPayloadId; + + @BeforeEach + void configureStorage() { + service = new ShopDataCrawlTaskService( + fileTaskMapper, + fileResultMapper, + shopDataCrawlResolveService, + excelAssemblyService, + taskCacheService, + ossStorageService, + ziniaoShopSwitchService, + objectMapper, + taskPressureProperties, + taskFileJobService, + taskResultItemService, + taskProgressSnapshotService, + taskDistributedLockService, + taskChunkMapper, + taskScopeStateMapper, + transientPayloadStorageService, + instanceMetadata); + + storedChunks.clear(); + storedScopes.clear(); + rustfsPayloads.clear(); + nextPayloadId = 0; + + lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + lenient().when(taskDistributedLockService.acquire(eq(MODULE_TYPE), anyLong())) + .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); + lenient().when(taskCacheService.getTaskCacheBatch(any())).thenReturn(Map.of()); + lenient().when(taskCacheService.getAllShopMergedPayload(anyLong())).thenReturn(Map.of()); + lenient().when(taskResultItemService.listResultSnapshots(anyLong(), eq(MODULE_TYPE), any())).thenReturn(List.of()); + lenient().when(taskFileJobService.findAssembleJob(anyLong(), eq(MODULE_TYPE), anyLong())).thenReturn(null); + lenient().when(excelAssemblyService.countRows(any())).thenReturn(1); + + lenient().when(fileTaskMapper.selectById(anyLong())).thenAnswer(invocation -> { + Long taskId = invocation.getArgument(0); + return task != null && Objects.equals(taskId, task.getId()) ? task : null; + }); + lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1); + lenient().when(fileResultMapper.selectById(anyLong())).thenAnswer(invocation -> { + Long resultId = invocation.getArgument(0); + return result != null && Objects.equals(resultId, result.getId()) ? result : null; + }); + lenient().when(fileResultMapper.selectList(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + return result != null && Objects.equals(taskId, result.getTaskId()) ? List.of(result) : List.of(); + }); + lenient().when(fileResultMapper.updateById(any(FileResultEntity.class))).thenReturn(1); + + lenient().when(ziniaoShopSwitchService.normalizeShopName(nullable(String.class))).thenAnswer(invocation -> { + String value = invocation.getArgument(0); + return value == null ? "" : value.trim(); + }); + + configureTransientPayloadStorage(); + configureChunkMapper(); + configureScopeMapper(); + } + + @Test + void completesChunkedResultWhenSecondChunkArrivesBeforeFirstChunk() { + givenRunningTask(101L, 201L); + + service.submitResult(task.getId(), request(chunk(2, 2, "UK", row("2026-07-26", "B002")))); + + assertEquals(-1, result.getSuccess()); + assertEquals(1, storedChunks.size()); + verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), eq(MODULE_TYPE), anyLong(), anyString()); + + service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001")))); + + assertEquals(1, result.getSuccess()); + assertEquals(2, storedChunks.size()); + assertTrue(task.getResultJson().indexOf("B001") < task.getResultJson().indexOf("B002")); + verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), + "task:" + task.getId() + ":owner:instance-a"); + } + + @Test + void identicalResultChunkRetryIsIdempotent() { + givenRunningTask(102L, 202L); + ShopDataCrawlSubmitResultRequest request = request(chunk(1, 2, "DE", row("2026-07-25", "B001"))); + + service.submitResult(task.getId(), request); + service.submitResult(task.getId(), request); + + assertEquals(1, storedChunks.size()); + assertEquals(1, rustfsPayloads.size()); + verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class)); + assertEquals(-1, result.getSuccess()); + } + + @Test + void resultChunkRetryWithDifferentContentIsRejected() { + givenRunningTask(103L, 203L); + + service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001")))); + BusinessException error = assertThrows(BusinessException.class, () -> + service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B099"))))); + + assertTrue(error.getMessage().contains("不同内容")); + assertEquals(1, storedChunks.size()); + assertEquals(1, rustfsPayloads.size()); + } + + @Test + void resultChunksRejectChangedChunkTotal() { + givenRunningTask(104L, 204L); + + service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001")))); + BusinessException error = assertThrows(BusinessException.class, () -> + service.submitResult(task.getId(), request(chunk(2, 3, "UK", row("2026-07-26", "B002"))))); + + assertTrue(error.getMessage().contains("chunk_total")); + assertEquals(1, storedChunks.size()); + } + + @Test + void missingChunkDoesNotGenerateResultFile() { + givenRunningTask(105L, 205L); + + service.submitResult(task.getId(), request(chunk(1, 3, "DE", row("2026-07-25", "B001")))); + service.submitResult(task.getId(), request(chunk(3, 3, "UK", row("2026-07-27", "B003")))); + + assertEquals(-1, result.getSuccess()); + assertEquals("RUNNING", task.getStatus()); + assertEquals(2, storedChunks.size()); + verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), eq(MODULE_TYPE), anyLong(), anyString()); + } + + @Test + void staleCompensationRecoversReceivedResultChunks() { + givenRunningTask(110L, 210L); + + service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001")))); + boolean finalized = service.tryFinalizeTask(task.getId(), true); + + assertTrue(finalized); + assertEquals(1, result.getSuccess()); + assertTrue(result.getErrorMessage().contains("已保留")); + assertTrue(task.getResultJson().contains("B001")); + assertTrue(task.getResultJson().contains("已保留")); + verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), + "task:" + task.getId() + ":owner:instance-a"); + } + + @Test + void staleCompensationRecoversLegacyMergedPayload() { + givenRunningTask(111L, 211L); + ShopDataCrawlShopPayloadDto cached = legacyChunk(false, "UK", row("2026-07-26", "B002")); + when(taskCacheService.getAllShopMergedPayload(task.getId())).thenReturn(Map.of(SHOP_NAME, cached)); + + boolean finalized = service.tryFinalizeTask(task.getId(), true); + + assertTrue(finalized); + assertEquals(1, result.getSuccess()); + assertTrue(result.getErrorMessage().contains("已保留")); + assertTrue(task.getResultJson().contains("B002")); + verify(taskCacheService).removeShopMergedPayload(task.getId(), SHOP_NAME); + verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), + "task:" + task.getId() + ":owner:instance-a"); + } + + @Test + void resultChunkRejectsCallbackWhenRustfsIsUnavailable() { + givenRunningTask(106L, 206L); + when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(false); + + BusinessException error = assertThrows(BusinessException.class, () -> + service.submitResult(task.getId(), request(chunk(1, 1, "DE", row("2026-07-25", "B001"))))); + + assertTrue(error.getMessage().contains("RustFS")); + assertEquals(0, storedChunks.size()); + } + + @Test + void resultChunkRejectsLocalFallbackPointer() { + givenRunningTask(107L, 207L); + when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), anyLong(), anyString(), any(), anyString())) + .thenReturn("local:instance-a/chunk-1"); + when(transientPayloadStorageService.extractPointer("local:instance-a/chunk-1")) + .thenReturn("local:instance-a/chunk-1"); + + BusinessException error = assertThrows(BusinessException.class, () -> + service.submitResult(task.getId(), request(chunk(1, 1, "DE", row("2026-07-25", "B001"))))); + + assertTrue(error.getMessage().contains("RustFS")); + assertEquals(0, storedChunks.size()); + verify(transientPayloadStorageService).deletePayloadIfPresent("local:instance-a/chunk-1"); + } + + @Test + void legacyShopDoneFalseThenTrueUsesMergedPayloadCache() { + givenRunningTask(108L, 208L); + ShopDataCrawlShopPayloadDto cached = legacyChunk(false, "DE", row("2026-07-25", "B001")); + when(taskCacheService.getShopMergedPayload(task.getId(), SHOP_NAME)).thenReturn(null, cached); + + service.submitResult(task.getId(), request(legacyChunk(false, "DE", row("2026-07-25", "B001")))); + + assertEquals(-1, result.getSuccess()); + verify(taskCacheService).saveShopMergedPayload(eq(task.getId()), eq(SHOP_NAME), any()); + + service.submitResult(task.getId(), request(legacyChunk(true, "UK", row("2026-07-26", "B002")))); + + assertEquals(1, result.getSuccess()); + assertTrue(task.getResultJson().contains("B001")); + assertTrue(task.getResultJson().contains("B002")); + assertEquals(0, storedChunks.size()); + verify(taskCacheService).removeShopMergedPayload(task.getId(), SHOP_NAME); + } + + @Test + void cleanupResultFileJobDeletesTransientChunkObjectsAndRows() { + givenRunningTask(109L, 209L); + service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001")))); + String storedPayload = storedChunks.get(0).getPayloadJson(); + TaskFileJobEntity job = new TaskFileJobEntity(); + job.setTaskId(task.getId()); + job.setModuleType(MODULE_TYPE); + job.setResultId(result.getId()); + + service.cleanupResultFileJob(job); + + assertFalse(rustfsPayloads.containsKey(storedPayload)); + assertEquals(0, storedChunks.size()); + assertEquals(0, storedScopes.size()); + verify(taskChunkMapper).delete(any()); + verify(taskScopeStateMapper).delete(any()); + } + + private void configureTransientPayloadStorage() { + lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true); + lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned( + anyString(), anyLong(), anyString(), any(), anyString())).thenAnswer(invocation -> { + String pointer = "rustfs:payload-" + (++nextPayloadId); + rustfsPayloads.put(pointer, invocation.getArgument(4)); + return pointer; + }); + lenient().when(transientPayloadStorageService.extractPointer(any())).thenAnswer(invocation -> { + String value = invocation.getArgument(0); + if (value == null) { + return null; + } + return value.startsWith("rustfs:") || value.startsWith("local:") || value.startsWith("oss:") + ? value : null; + }); + lenient().when(transientPayloadStorageService.resolvePayload(any(), any())).thenAnswer(invocation -> { + String pointer = invocation.getArgument(0); + String payload = rustfsPayloads.get(pointer); + if (payload == null) { + throw new IllegalStateException("missing test RustFS payload: " + pointer); + } + return payload; + }); + lenient().doAnswer(invocation -> { + rustfsPayloads.remove(invocation.getArgument(0)); + return null; + }).when(transientPayloadStorageService).deletePayloadIfPresent(any()); + } + + private void configureChunkMapper() { + lenient().when(taskChunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> { + TaskChunkEntity chunk = invocation.getArgument(0); + chunk.setId((long) storedChunks.size() + 1L); + storedChunks.add(chunk); + return 1; + }); + lenient().when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + Integer chunkIndex = queryInteger(query); + return storedChunks.stream() + .filter(chunk -> matchesChunk(chunk, taskId, scopeHash, chunkIndex)) + .findFirst() + .orElse(null); + }); + lenient().when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + return storedChunks.stream() + .filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null)) + .count(); + }); + lenient().when(taskChunkMapper.selectList(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + return storedChunks.stream() + .filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null)) + .sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex)) + .toList(); + }); + lenient().when(taskChunkMapper.delete(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + int before = storedChunks.size(); + storedChunks.removeIf(chunk -> taskId == null || Objects.equals(taskId, chunk.getTaskId())); + return before - storedChunks.size(); + }); + } + + private void configureScopeMapper() { + lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> { + TaskScopeStateEntity scope = invocation.getArgument(0); + scope.setId((long) storedScopes.size() + 1L); + storedScopes.add(scope); + return 1; + }); + lenient().when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + String scopeHash = queryScopeHash(query); + return storedScopes.stream() + .filter(scope -> Objects.equals(taskId, scope.getTaskId())) + .filter(scope -> scopeHash == null || Objects.equals(scopeHash, scope.getScopeHash())) + .findFirst() + .orElse(null); + }); + lenient().when(taskScopeStateMapper.updateById(any(TaskScopeStateEntity.class))).thenReturn(1); + lenient().when(taskScopeStateMapper.delete(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + Long taskId = queryLong(query); + int before = storedScopes.size(); + storedScopes.removeIf(scope -> taskId == null || Objects.equals(taskId, scope.getTaskId())); + return before - storedScopes.size(); + }); + } + + private boolean matchesChunk(TaskChunkEntity chunk, Long taskId, String scopeHash, Integer chunkIndex) { + return (taskId == null || Objects.equals(taskId, chunk.getTaskId())) + && (scopeHash == null || Objects.equals(scopeHash, chunk.getScopeHash())) + && (chunkIndex == null || Objects.equals(chunkIndex, chunk.getChunkIndex())); + } + + private Long queryLong(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(Long.class::isInstance) + .map(Long.class::cast) + .findFirst() + .orElse(null); + } + + private Integer queryInteger(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(Integer.class::isInstance) + .map(Integer.class::cast) + .findFirst() + .orElse(null); + } + + private String queryScopeHash(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .filter(value -> value.length() == 64) + .findFirst() + .orElse(null); + } + + private void givenRunningTask(long taskId, long resultId) { + task = new FileTaskEntity(); + task.setId(taskId); + task.setUserId(7L); + task.setModuleType(MODULE_TYPE); + task.setStatus("RUNNING"); + task.setRequestJson("{\"ownerInstanceId\":\"instance-a\"}"); + task.setResultJson("[]"); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + + result = new FileResultEntity(); + result.setId(resultId); + result.setTaskId(taskId); + result.setUserId(7L); + result.setModuleType(MODULE_TYPE); + result.setSourceFilename(SHOP_NAME); + result.setSourceFileUrl("shop-1"); + result.setSuccess(-1); + result.setCreatedAt(LocalDateTime.now()); + } + + private ShopDataCrawlSubmitResultRequest request(ShopDataCrawlShopPayloadDto payload) { + ShopDataCrawlSubmitResultRequest request = new ShopDataCrawlSubmitResultRequest(); + request.setShops(List.of(payload)); + return request; + } + + private ShopDataCrawlShopPayloadDto chunk(int chunkIndex, + int chunkTotal, + String country, + ShopDataCrawlRowDto row) { + ShopDataCrawlShopPayloadDto payload = legacyChunk(false, country, row); + payload.setChunkIndex(chunkIndex); + payload.setChunkTotal(chunkTotal); + return payload; + } + + private ShopDataCrawlShopPayloadDto legacyChunk(boolean shopDone, + String country, + ShopDataCrawlRowDto row) { + ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto(); + payload.setShopName(SHOP_NAME); + payload.setCountryResults(List.of(country(country, row))); + payload.setShopDone(shopDone); + return payload; + } + + private ShopDataCrawlCountryResultDto country(String country, ShopDataCrawlRowDto row) { + ShopDataCrawlCountryResultDto result = new ShopDataCrawlCountryResultDto(); + result.setCountry(country); + result.setItems(List.of(row)); + return result; + } + + private ShopDataCrawlRowDto row(String date, String asin) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate(date); + row.setAsin(asin); + row.setInventorySales("10"); + row.setSalesRank("20"); + row.setPageViews("30"); + row.setUnitsSold("40"); + row.setPrice("50"); + row.setRecommendedOffer("60"); + return row; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java new file mode 100644 index 00000000..be847d4f --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceOwnerTest.java @@ -0,0 +1,99 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.TaskOwnerMismatchException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ShopDataCrawlTaskServiceOwnerTest { + @Mock InstanceMetadata instanceMetadata; + @Mock com.nanri.aiimage.modules.task.mapper.FileTaskMapper fileTaskMapper; + @Mock com.nanri.aiimage.modules.task.mapper.FileResultMapper fileResultMapper; + @Mock ShopDataCrawlResolveService resolveService; + @Mock ShopDataCrawlExcelAssemblyService excelAssemblyService; + @Mock ShopDataCrawlTaskCacheService cacheService; + @Mock com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService; + @Mock com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService ziniaoShopSwitchService; + @Mock com.nanri.aiimage.config.TaskPressureProperties taskPressureProperties; + @Mock com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService; + @Mock com.nanri.aiimage.modules.task.service.TaskResultItemService taskResultItemService; + @Mock com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService; + @Mock com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService; + @Mock com.nanri.aiimage.modules.task.mapper.TaskChunkMapper taskChunkMapper; + @Mock com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper taskScopeStateMapper; + @Mock com.nanri.aiimage.modules.task.service.TransientPayloadStorageService transientPayloadStorageService; + @Spy private final ObjectMapper objectMapper = new ObjectMapper(); + @InjectMocks ShopDataCrawlTaskService service; + + @Test + void rejectsTaskOwnedByAnotherInstance() { + when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + FileTaskEntity task = task("instance-b"); + assertThrows(TaskOwnerMismatchException.class, + () -> service.ensureTaskOwnedByCurrentInstance(task, "callback")); + } + + @Test + void acceptsTaskOwnedByCurrentInstance() { + when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + assertDoesNotThrow(() -> service.ensureTaskOwnedByCurrentInstance(task("instance-a"), "callback")); + } + + @Test + void callbackWithoutShopNameBindsOnlyTaskShopAndWrongNameFails() { + var payload = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto(); + when(ziniaoShopSwitchService.normalizeShopName(null)).thenReturn(""); + assertEquals(payload, service.normalizePayloadByShop(java.util.List.of(payload), "Demo").get("Demo")); + + var wrong = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto(); + wrong.setShopName("Wrong"); + when(ziniaoShopSwitchService.normalizeShopName("Wrong")).thenReturn("Wrong"); + assertThrows(com.nanri.aiimage.common.exception.BusinessException.class, + () -> service.normalizePayloadByShop(java.util.List.of(wrong), "Demo")); + } + + @Test + void partialPayloadIsNotCompleteUntilShopDoneIsTrue() { + var payload = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto(); + assertFalse(service.isShopPayloadComplete(payload)); + payload.setShopDone(false); + assertFalse(service.isShopPayloadComplete(payload)); + payload.setShopDone(true); + assertTrue(service.isShopPayloadComplete(payload)); + } + + @Test + @SuppressWarnings("unchecked") + void deletesResultObjectOnlyAfterLastDatabaseReferenceIsGone() { + when(fileResultMapper.selectCount(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class))) + .thenReturn(1L, 0L); + service.deleteResultObjectIfUnreferenced("result/shared.xlsx"); + verify(ossStorageService, never()).deleteObject("result/shared.xlsx"); + service.deleteResultObjectIfUnreferenced("result/shared.xlsx"); + verify(ossStorageService).deleteObject("result/shared.xlsx"); + } + + private FileTaskEntity task(String owner) { + FileTaskEntity task = new FileTaskEntity(); + task.setId(1L); + task.setRequestJson("{\"ownerInstanceId\":\"" + owner + "\"}"); + return task; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java index fb17f55b..2e2a3342 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java @@ -7,6 +7,8 @@ import java.lang.reflect.Field; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; class SimilarAsinTaskServiceTest { @@ -50,6 +52,14 @@ class SimilarAsinTaskServiceTest { assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", "")); } + @Test + void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() { + assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3)); + assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("FAILED", 3, 3)); + assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("RUNNING", 3, 3)); + assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3)); + } + private int staticIntField(String name) throws Exception { Field field = SimilarAsinTaskService.class.getDeclaredField(name); field.setAccessible(true); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderTest.java index 091d9d27..f763125e 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderTest.java @@ -1,5 +1,7 @@ package com.nanri.aiimage.modules.similarasin.util; +import com.nanri.aiimage.config.OssProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import org.junit.jupiter.api.Test; import javax.imageio.ImageIO; @@ -24,7 +26,33 @@ import static org.junit.jupiter.api.Assertions.assertFalse; class SimilarAsinImageEmbedderTest { // properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底。 - private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null); + private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService()); + + private static OssStorageService createOssStorageService() { + OssProperties properties = new OssProperties(); + properties.setEndpoint("https://oss.aishufu.top"); + properties.setPublicEndpoint("https://oss.aishufu.top"); + properties.setBucket("nanri-ai-images"); + properties.setImageVideoBucket("shufu-video"); + properties.setDigitalHumanBucket("nanri-ai-digital-human"); + properties.setAccessKeyId("test-access-key"); + properties.setAccessKeySecret("test-secret-key"); + return new OssStorageService(properties); + } + + @Test + void normalizesLegacyMinioUrlBeforeHttpsValidation() { + String normalized = embedder.normalizeAndValidateDownloadUrl( + "http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg"); + + assertEquals("https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg", normalized); + } + + @Test + void leavesUnmanagedHttpUrlBlocked() { + assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class, + () -> embedder.normalizeAndValidateDownloadUrl("http://example.com/main.jpg")); + } @Test void resizeImageProducesThumbnailUnderHardCap() throws Exception { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java index 487a5d4c..4aa2bdd1 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java @@ -13,6 +13,8 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService; import com.nanri.aiimage.modules.publish.service.PublishTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService; import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; @@ -55,6 +57,8 @@ class TaskHeartbeatServiceTest { @Mock private ShopMatchTaskCacheService shopMatchTaskCacheService; @Mock private PatrolDeleteTaskCacheService patrolDeleteTaskCacheService; @Mock private QueryAsinTaskCacheService queryAsinTaskCacheService; + @Mock private ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService; + @Mock private ShopDataCrawlTaskService shopDataCrawlTaskService; @Mock private WithdrawTaskCacheService withdrawTaskCacheService; @Mock private AppearancePatentTaskCacheService appearancePatentTaskCacheService; @Mock private SimilarAsinTaskCacheService similarAsinTaskCacheService; @@ -91,4 +95,27 @@ class TaskHeartbeatServiceTest { routingBeforeUpdate.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class)); verify(publishTaskService).touchHeartbeat(taskId, request); } + + @Test + @SuppressWarnings("unchecked") + void shopDataCrawlHeartbeatChecksOwnerBeforeDatabaseUpdate() { + long taskId = 20143L; + FileTaskEntity task = new FileTaskEntity(); + task.setId(taskId); + task.setModuleType("SHOP_DATA_CRAWL"); + task.setStatus("RUNNING"); + when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); + when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + + TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest()); + + assertTrue(result.isAlive()); + InOrder order = inOrder(shopDataCrawlTaskService, fileTaskMapper); + order.verify(shopDataCrawlTaskService) + .ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat"); + order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class)); + verify(shopDataCrawlTaskCacheService).touchTaskHeartbeat(taskId); + verify(shopDataCrawlTaskCacheService).saveTaskCache(task); + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerTest.java index b0aedf03..a471d202 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerTest.java @@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService; import com.nanri.aiimage.modules.publish.service.PublishTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService; import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; @@ -24,6 +25,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -43,6 +45,7 @@ class TaskResultFileJobWorkerTest { @Mock private ProductRiskTaskService productRiskTaskService; @Mock private PublishTaskService publishTaskService; @Mock private QueryAsinTaskService queryAsinTaskService; + @Mock private ShopDataCrawlTaskService shopDataCrawlTaskService; @Mock private WithdrawTaskService withdrawTaskService; @Mock private PatrolDeleteTaskService patrolDeleteTaskService; @Mock private AppearancePatentTaskService appearancePatentTaskService; @@ -109,8 +112,10 @@ class TaskResultFileJobWorkerTest { worker.process(job); - verify(publishTaskService).processResultFileJob(job); - verify(taskFileJobService).markSuccess(job, "result/publish/20141.xlsx"); + InOrder order = inOrder(publishTaskService, taskFileJobService); + order.verify(publishTaskService).processResultFileJob(job); + order.verify(taskFileJobService).markSuccess(job, "result/publish/20141.xlsx"); + order.verify(publishTaskService).cleanupResultFileJob(job); verifyNoInteractions(taskResultPayloadService); } @@ -128,4 +133,56 @@ class TaskResultFileJobWorkerTest { verifyNoInteractions(taskFileJobService, taskDistributedLockService, publishTaskService); } + + @Test + void shopDataCrawlOwnerJobDelegatesAndCleansPayload() { + long jobId = 13643L; + long taskId = 20143L; + long resultId = 22931L; + TaskFileJobEntity job = new TaskFileJobEntity(); + job.setId(jobId); + job.setTaskId(taskId); + job.setResultId(resultId); + job.setModuleType("SHOP_DATA_CRAWL"); + job.setScopeKey("task:20143:owner:instance-a"); + FileResultEntity result = new FileResultEntity(); + result.setResultFileUrl("result/shop-data-crawl/20143.xlsx"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + + when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + when(taskFileJobService.markRunning(jobId)).thenReturn(true); + when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", taskId, + TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock); + when(fileResultMapper.selectById(resultId)).thenReturn(result); + + worker.process(job); + + verify(shopDataCrawlTaskService).processResultFileJob(job); + verify(taskFileJobService).markSuccess(job, "result/shop-data-crawl/20143.xlsx"); + verify(taskResultPayloadService).deleteLatest(taskId, "SHOP_DATA_CRAWL", job.getScopeKey()); + verify(shopDataCrawlTaskService).cleanupResultFileJob(job); + } + + @Test + void exhaustedShopDataCrawlFileJobFailsOwningTask() { + TaskFileJobEntity job = new TaskFileJobEntity(); + job.setId(13644L); + job.setTaskId(20144L); + job.setResultId(22932L); + job.setModuleType("SHOP_DATA_CRAWL"); + job.setScopeKey("task:20144:owner:instance-a"); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + when(taskFileJobService.markRunning(job.getId())).thenReturn(true); + when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", job.getTaskId(), + TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock); + doThrow(new IllegalStateException("upload failed")) + .when(shopDataCrawlTaskService).processResultFileJob(job); + when(taskFileJobService.isRetryExhausted(job.getId())).thenReturn(true); + + worker.process(job); + + verify(taskFileJobService).markFailed(job, "upload failed"); + verify(shopDataCrawlTaskService).handleResultFileJobFailure(job, "upload failed"); + } } diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index d147ae0f..19eb982a 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -4,15 +4,27 @@ import json import os import re +import secrets import tempfile import threading import zipfile from datetime import datetime +from pathlib import Path from urllib.parse import quote import requests from requests.adapters import HTTPAdapter -from flask import Blueprint, request, jsonify, session, current_app, g, Response, send_file +from flask import ( + Blueprint, + request, + jsonify, + session, + current_app, + g, + Response, + send_file, + has_request_context, +) import pymysql from werkzeug.security import generate_password_hash @@ -20,26 +32,6 @@ from werkzeug.security import generate_password_hash from utils.db import get_db from utils.auth import admin_required, login_required, get_current_admin_role - -ALLOWED_DEDUPE_TOTAL_DATA_ADMIN_USERNAMES = {'刘丽泓'} - - -def _can_access_dedupe_total_data(role, current_row): - _, _, denied = _ensure_admin_menu_access('dedupe-total-data') - return denied is None - if role == 'super_admin': - return True - if role != 'admin' or not current_row: - return False - return (current_row.get('username') or '').strip() in ALLOWED_DEDUPE_TOTAL_DATA_ADMIN_USERNAMES - - -def _ensure_dedupe_total_data_access(): - return _ensure_backend_menu_access('dedupe-total-data') - role, current_row = get_current_admin_role() - if not _can_access_dedupe_total_data(role, current_row): - return None, None, (jsonify({'success': False, 'error': '无权访问数据去重汇总数据'}), 403) - return role, current_row, None from ali_oss import upload_file as oss_upload_file try: @@ -50,11 +42,7 @@ except ImportError: admin_api = Blueprint('admin_api', __name__, url_prefix='/api/admin') _backend_java_session_local = threading.local() -_column_sort_schema_checked = False -_column_sort_schema_lock = threading.Lock() -_product_category_schema_checked = False -_product_category_schema_lock = threading.Lock() -IMAGE_VIDEO_PERMISSION_KEY = 'admin_image_video_tasks' +_internal_token_lock = threading.Lock() IMAGE_VIDEO_DATA_PERMISSION_KEY = 'admin_image_video_task_data' ADMIN_MENU_ACCESS_CONFIG = { @@ -147,11 +135,45 @@ def _get_backend_java_session(): return http_session -def _proxy_backend_java(method, path, *, params=None, json_data=None, files=None, data=None, timeout=10): +def _backend_java_forward_headers(): + """Forward the caller's Java auth material through the Flask compatibility proxy.""" + if not has_request_context(): + return {} + headers = {} + for name in ('Authorization', 'Cookie', 'X-Device-Id'): + value = request.headers.get(name) + if value: + headers[name] = value + return headers + + +def _proxy_backend_java( + method, + path, + *, + params=None, + json_data=None, + files=None, + data=None, + timeout=10, + headers=None, +): url = f"{backend_java_base_url}{path}" + forwarded_headers = _backend_java_forward_headers() + if headers: + forwarded_headers.update(headers) try: requester = requests if files else _get_backend_java_session() - resp = requester.request(method=method, url=url, params=params, json=json_data, files=files, data=data, timeout=timeout) + resp = requester.request( + method=method, + url=url, + params=params, + json=json_data, + files=files, + data=data, + headers=forwarded_headers or None, + timeout=timeout, + ) except requests.RequestException: return None, jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502 try: @@ -165,11 +187,23 @@ def _proxy_backend_java(method, path, *, params=None, json_data=None, files=None if not isinstance(data, dict): return None, jsonify({'success': False, 'error': 'backend-java 返回格式错误'}), 502 if not data.get('success'): - return data, jsonify({'success': False, 'error': data.get('message') or '操作失败'}), 200 + raw_code = data.get('code') + try: + error_status = int(raw_code) if not isinstance(raw_code, bool) else None + except (TypeError, ValueError): + error_status = None + status = ( + error_status + if resp.status_code == 200 and error_status is not None and 400 <= error_status <= 599 + else 200 + ) + return data, jsonify({'success': False, 'error': data.get('message') or '操作失败'}), status return data, None, 200 def _format_permission_item(item): + if not isinstance(item, dict): + return {} created_at = item.get('createdAt') if created_at in (None, ''): created_at = item.get('created_at') @@ -180,7 +214,11 @@ def _format_permission_item(item): sort_order = item.get('sortOrder') if sort_order is None: sort_order = item.get('sort_order') - return { + parent_id = item.get('parentId') + if parent_id is None: + parent_id = item.get('parent_id') + root_column_key = item.get('rootColumnKey') or item.get('root_column_key') or '' + formatted = { 'id': item.get('id'), 'name': item.get('name') or '', 'column_key': item.get('columnKey') or item.get('column_key') or '', @@ -188,7 +226,249 @@ def _format_permission_item(item): 'route_path': item.get('routePath') or item.get('route_path') or '', 'sort_order': sort_order if sort_order is not None else 0, 'created_at': created_at_text, + # Keep both spellings while old Flask pages migrate to the Java schema. + 'parent_id': parent_id, + 'parentId': parent_id, + 'root_column_key': root_column_key, + 'rootColumnKey': root_column_key, } + children = item.get('children') + if isinstance(children, list): + formatted['children'] = [_format_permission_item(child) for child in children] + return formatted + + +def _iter_permission_items(items): + """Yield a flat view of a possibly recursive Java permission response.""" + for item in items or []: + if not isinstance(item, dict): + continue + yield item + children = item.get('children') + if isinstance(children, list): + yield from _iter_permission_items(children) + + +def _permission_response_items(result): + """Read menu lists from both ApiResponse and legacy Flask-shaped payloads.""" + if not isinstance(result, dict): + return [] + payload = result.get('data') + if isinstance(payload, list): + return payload + if isinstance(payload, dict): + for key in ('items', 'menus', 'columns', 'permissions'): + value = payload.get(key) + if isinstance(value, list): + return value + for key in ('items', 'menus', 'columns', 'permissions'): + value = result.get(key) + if isinstance(value, list): + return value + return [] + + +def _permission_response_column_ids(result): + """Read direct IDs from Java's `data.columnIds` or old snake_case aliases.""" + if not isinstance(result, dict): + return [] + payload = result.get('data') + candidates = [payload, result] + if isinstance(payload, dict): + candidates.extend(payload.get(key) for key in ('data', 'items') if isinstance(payload.get(key), dict)) + for candidate in candidates: + if not isinstance(candidate, dict): + continue + for key in ('columnIds', 'column_ids', 'ids'): + value = candidate.get(key) + if isinstance(value, list): + normalized = [] + for raw in value: + try: + value_int = int(raw) + except (TypeError, ValueError): + continue + if value_int > 0 and value_int not in normalized: + normalized.append(value_int) + return normalized + return [] + + +class _PermissionProxyError(Exception): + """Carry a Flask response when a Java permission call cannot be completed.""" + + def __init__(self, response, status): + super().__init__('backend-java permission request failed') + self.response = response + self.status = status + + +def _proxy_permission_java(method, path, *, params=None, json_data=None, current_row=None): + """Call Java permission APIs using either forwarded JWT or trusted Flask identity.""" + proxy_params = {} + request_row = getattr(g, '_current_user_row', None) if has_request_context() else None + current_row = current_row or request_row or {} + operator_id = current_row.get('id') + if not operator_id and has_request_context(): + operator_id = session.get('user_id') + if operator_id: + proxy_params['operatorId'] = operator_id + if params: + proxy_params.update(params) + internal_token = _resolve_internal_token() + headers = {'X-Internal-Token': internal_token} if internal_token else None + result, error_response, status = _proxy_backend_java( + method, + path, + params=proxy_params or None, + json_data=json_data, + headers=headers, + ) + return result, error_response, status + + +def _resolve_internal_token(): + configured_token = (os.environ.get('AIIMAGE_INTERNAL_TOKEN') or '').strip() + if configured_token: + return configured_token + configured_path = (os.environ.get('AIIMAGE_INTERNAL_TOKEN_FILE') or '').strip() + token_path = Path(configured_path).expanduser() if configured_path else Path.home() / '.aiimage' / 'internal-token' + if configured_path and not token_path.is_absolute(): + return '' + with _internal_token_lock: + try: + existing_token = token_path.read_text(encoding='utf-8').strip() + if existing_token: + return existing_token + except FileNotFoundError: + pass + except OSError: + return '' + + try: + token_path.parent.mkdir(parents=True, exist_ok=True) + generated_token = secrets.token_urlsafe(48) + file_descriptor = os.open(str(token_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(file_descriptor, 'w', encoding='utf-8') as token_file: + token_file.write(generated_token) + token_file.write('\n') + return generated_token + except FileExistsError: + try: + return token_path.read_text(encoding='utf-8').strip() + except OSError: + return '' + except OSError: + return '' + + +def _java_permission_items(*, menu_type=None, user_id=None, effective=False, current_row=None, role=None): + """Fetch permission menus from Java and normalize its response for Flask callers.""" + params = {} + if menu_type: + params['menuType'] = menu_type + if effective: + if user_id is None: + user_id = _get_current_admin_id(current_row) + if user_id is None: + return [], None, 200 + path = f'/api/admin/permission-users/{int(user_id)}/column-permissions' + else: + path = '/api/admin/permission-menus' + result, error_response, status = _proxy_permission_java( + 'GET', + path, + params=params, + current_row=current_row, + ) + if error_response is not None: + return [], error_response, status + raw_items = _permission_response_items(result) + return [_format_permission_item(item) for item in raw_items], None, status + + +def _java_user_direct_column_ids(user_id, *, menu_type=None, current_row=None, role=None): + params = {} + if menu_type: + params['menuType'] = menu_type + result, error_response, status = _proxy_permission_java( + 'GET', + f'/api/admin/permission-users/{int(user_id)}/columns', + params=params, + current_row=current_row, + ) + if error_response is not None: + return [], error_response, status + return _permission_response_column_ids(result), None, status + + +def _java_update_user_direct_column_ids( + user_id, + column_ids, + *, + menu_type=None, + current_row=None, + role=None, +): + normalized = _normalize_column_ids(column_ids) + params = {'menuType': menu_type} if menu_type else None + result, error_response, status = _proxy_permission_java( + 'PUT', + f'/api/admin/permission-users/{int(user_id)}/columns', + params=params, + json_data={'columnIds': normalized}, + current_row=current_row, + ) + return result, error_response, status + + +def _normalize_column_ids(raw_ids): + if raw_ids is None: + return [] + if not isinstance(raw_ids, (list, tuple, set)): + raise ValueError('column_ids 必须是数组') + normalized = [] + for raw in raw_ids: + if isinstance(raw, bool): + raise ValueError('column_ids 包含无效 ID') + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError('column_ids 包含无效 ID') from exc + if value <= 0: + raise ValueError('column_ids 包含无效 ID') + if value not in normalized: + normalized.append(value) + return normalized + + +def _effective_permission_sets(user_id, *, menu_type='admin', current_row=None, role=None): + items, error_response, status = _java_permission_items( + menu_type=menu_type, + user_id=user_id, + effective=True, + current_row=current_row, + role=role, + ) + if error_response is not None: + raise _PermissionProxyError(error_response, status) + key_set = set() + route_set = set() + id_set = set() + for item in _iter_permission_items(items): + try: + item_id = int(item.get('id')) + except (TypeError, ValueError): + item_id = None + if item_id and item_id > 0: + id_set.add(item_id) + column_key = str(item.get('column_key') or '').strip() + route_path = str(item.get('route_path') or '').strip() + if column_key: + key_set.add(column_key) + if route_path: + route_set.add(route_path) + return id_set, key_set, route_set def _parse_optional_int(value): @@ -317,144 +597,6 @@ def _image_video_admin_item(row, include_json=False): return item -def _ensure_column_sort_schema(): - global _column_sort_schema_checked - if _column_sort_schema_checked: - return - with _column_sort_schema_lock: - if _column_sort_schema_checked: - return - conn = get_db() - try: - with conn.cursor() as cur: - try: - cur.execute("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path") - except Exception: - pass - try: - cur.execute("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0") - except Exception: - pass - conn.commit() - _column_sort_schema_checked = True - finally: - conn.close() - return - conn = get_db() - try: - with conn.cursor() as cur: - try: - cur.execute("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path") - except Exception: - pass - try: - cur.execute("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0") - except Exception: - pass - conn.commit() - finally: - conn.close() - - -def _load_local_column_sort_map(): - _ensure_column_sort_schema() - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute("SELECT id, sort_order FROM columns") - rows = cur.fetchall() - return {int(row['id']): int(row.get('sort_order') or 0) for row in rows if row.get('id') is not None} - finally: - conn.close() - - -def _load_local_columns(menu_type=None): - _ensure_column_sort_schema() - conn = get_db() - try: - with conn.cursor() as cur: - sql = """ - SELECT id, name, column_key, menu_type, route_path, sort_order, created_at - FROM columns - """ - params = [] - if menu_type: - sql += " WHERE menu_type = %s" - params.append(menu_type) - sql += " ORDER BY sort_order ASC, id ASC" - cur.execute(sql, params) - return cur.fetchall() - finally: - conn.close() - - -def _sync_local_columns_from_backend(items): - if not items: - return - _ensure_column_sort_schema() - conn = get_db() - try: - with conn.cursor() as cur: - for item in items: - item_id = item.get('id') - if item_id is None: - continue - sort_order = item.get('sort_order') - cur.execute( - """ - INSERT INTO columns (id, name, column_key, menu_type, route_path, sort_order, created_at) - VALUES (%s, %s, %s, %s, %s, %s, COALESCE(%s, NOW())) - ON DUPLICATE KEY UPDATE - name = VALUES(name), - column_key = VALUES(column_key), - menu_type = VALUES(menu_type), - route_path = VALUES(route_path), - sort_order = CASE - WHEN columns.sort_order IS NULL OR columns.sort_order = 0 THEN VALUES(sort_order) - ELSE columns.sort_order - END - """, - ( - int(item_id), - item.get('name') or '', - item.get('column_key') or '', - item.get('menu_type') or 'app', - item.get('route_path') or '', - int(sort_order if sort_order not in (None, '') else item_id), - item.get('created_at') or None, - ), - ) - conn.commit() - finally: - conn.close() - - -def _set_local_column_sort_order(column_id, sort_order): - _ensure_column_sort_schema() - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute( - "UPDATE columns SET sort_order = %s WHERE id = %s", - (int(sort_order), int(column_id)), - ) - conn.commit() - finally: - conn.close() - - -def _get_next_local_column_sort_order(): - _ensure_column_sort_schema() - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute("SELECT COALESCE(MAX(sort_order), 0) AS max_sort FROM columns") - row = cur.fetchone() or {} - return int(row.get('max_sort') or 0) + 1 - finally: - conn.close() - - PRODUCT_CATEGORY_DEFAULT_TREE = [ { 'name': '护肤品', @@ -508,108 +650,26 @@ PRODUCT_CATEGORY_DEFAULT_TREE = [ ] -def _ensure_local_admin_menu_item(name, column_key, route_path, sort_order): - _ensure_column_sort_schema() - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute( - """ - INSERT INTO columns (name, column_key, menu_type, route_path, sort_order) - VALUES (%s, %s, 'admin', %s, %s) - ON DUPLICATE KEY UPDATE - name = VALUES(name), - menu_type = 'admin', - route_path = VALUES(route_path), - sort_order = VALUES(sort_order) - """, - (name, column_key, route_path, int(sort_order)), - ) - conn.commit() - finally: - conn.close() - - -def _ensure_product_category_schema(): - global _product_category_schema_checked - if _product_category_schema_checked: - return - with _product_category_schema_lock: - if _product_category_schema_checked: - return - _ensure_local_admin_menu_item('商品类目', 'admin_product_categories', 'product-categories', 66) - _product_category_schema_checked = True - - -def _swap_local_column_sort_order(column_id, target_id): - _ensure_column_sort_schema() - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute("SELECT id, sort_order FROM columns WHERE id IN (%s, %s) ORDER BY id", (int(column_id), int(target_id))) - rows = cur.fetchall() - if len(rows) != 2: - raise ValueError('菜单不存在') - by_id = {int(row['id']): int(row.get('sort_order') or 0) for row in rows} - left_sort = by_id.get(int(column_id), 0) or int(column_id) - right_sort = by_id.get(int(target_id), 0) or int(target_id) - cur.execute("UPDATE columns SET sort_order = %s WHERE id = %s", (right_sort, int(column_id))) - cur.execute("UPDATE columns SET sort_order = %s WHERE id = %s", (left_sort, int(target_id))) - conn.commit() - finally: - conn.close() - - -def _sync_user_column_permissions(user_id, column_ids): - _, error_response, status = _proxy_backend_java( - 'PUT', - f'/api/admin/permission-users/{user_id}/columns', - json_data={'columnIds': [int(x) for x in (column_ids or []) if str(x).isdigit()]}, - ) - return error_response, status - - def _get_current_admin_id(current_row=None): if current_row and current_row.get('id'): return current_row.get('id') return session.get('user_id') -def _get_current_admin_permission_sets(): - user_id = _get_current_admin_id() - if not user_id: - return set(), set() - cache_key = f'_admin_permission_sets_{user_id}' - cached = getattr(g, cache_key, None) - if cached is not None: - return cached - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute( - """SELECT c.column_key, c.route_path - FROM user_column_permission ucp - INNER JOIN columns c ON c.id = ucp.column_id - WHERE ucp.user_id = %s AND c.menu_type = 'admin'""", - (user_id,), - ) - rows = cur.fetchall() - finally: - conn.close() - key_set = {(row.get('column_key') or '').strip() for row in rows if (row.get('column_key') or '').strip()} - route_set = {(row.get('route_path') or '').strip() for row in rows if (row.get('route_path') or '').strip()} - cached = (key_set, route_set) - setattr(g, cache_key, cached) - return cached - - def _ensure_admin_menu_access(*menu_names): role, current_row = get_current_admin_role() if role == 'super_admin': return role, current_row, None if role != 'admin' or not current_row: return role, current_row, (jsonify({'success': False, 'error': '需要管理员权限'}), 403) - key_set, route_set = _get_current_admin_permission_sets() + try: + _, key_set, route_set = _effective_permission_sets( + _get_current_admin_id(current_row), + current_row=current_row, + role=role, + ) + except _PermissionProxyError as exc: + return role, current_row, (exc.response, exc.status) for menu_name in menu_names: config = ADMIN_MENU_ACCESS_CONFIG.get(menu_name) or {} column_key = (config.get('column_key') or '').strip() @@ -626,39 +686,22 @@ def _ensure_dedupe_total_data_access(): return _ensure_backend_menu_access('dedupe-total-data') -def _load_current_admin_menu_items(): - role, current_row = get_current_admin_role() - if not role or not current_row: - return None, None, None, (jsonify({'success': False, 'error': '需要管理员权限'}), 403) - if role == 'super_admin': - items = [ - _format_permission_item(item) - for item in _load_local_columns('admin') - if (item.get('column_key') or '') != IMAGE_VIDEO_DATA_PERMISSION_KEY - ] - return role, current_row, items, None - user_id = _get_current_admin_id(current_row) - if not user_id: - return role, current_row, None, (jsonify({'success': False, 'error': '当前登录用户缺少有效ID'}), 400) - items = [_format_permission_item(item) for item in _get_local_user_column_permission_items(user_id, 'admin')] - return role, current_row, items, None - - def _load_current_backend_menu_items(): role, current_row = get_current_admin_role() if not role or not current_row: return None, None, None, (jsonify({'success': False, 'error': '需要登录'}), 403) - if role == 'super_admin': - items = [ - _format_permission_item(item) - for item in _load_local_columns('admin') - if (item.get('column_key') or '') != IMAGE_VIDEO_DATA_PERMISSION_KEY - ] - return role, current_row, items, None user_id = _get_current_admin_id(current_row) if not user_id: - return role, current_row, None, (jsonify({'success': False, 'error': '当前登录用户缺少有效ID'}), 400) - items = [_format_permission_item(item) for item in _get_local_user_column_permission_items(user_id, 'admin')] + return role, current_row, None, (jsonify({'success': False, 'error': '当前登录用户缺少有效 ID'}), 400) + items, error_response, status = _java_permission_items( + menu_type='admin', + user_id=user_id, + effective=True, + current_row=current_row, + role=role, + ) + if error_response is not None: + return role, current_row, None, (error_response, status) return role, current_row, items, None @@ -668,7 +711,14 @@ def _ensure_backend_menu_access(*menu_names): return role, current_row, None if not role or not current_row: return role, current_row, (jsonify({'success': False, 'error': '需要登录'}), 403) - key_set, route_set = _get_current_admin_permission_sets() + try: + _, key_set, route_set = _effective_permission_sets( + _get_current_admin_id(current_row), + current_row=current_row, + role=role, + ) + except _PermissionProxyError as exc: + return role, current_row, (exc.response, exc.status) for menu_name in menu_names: config = ADMIN_MENU_ACCESS_CONFIG.get(menu_name) or {} column_key = (config.get('column_key') or '').strip() @@ -687,20 +737,16 @@ def _ensure_image_video_data_access(): return role, current_row, None if not role or not current_row: return role, current_row, (jsonify({'success': False, 'error': '需要登录'}), 403) - conn = get_db() try: - with conn.cursor() as cur: - data_column_id = _image_video_permission_column_id(cur) - cur.execute( - "SELECT 1 FROM user_column_permission ucp " - "WHERE ucp.user_id = %s AND ucp.column_id = %s LIMIT 1", - (int(current_row['id']), data_column_id), - ) - granted = cur.fetchone() - conn.commit() - finally: - conn.close() - if granted: + _, key_set, route_set = _effective_permission_sets( + _get_current_admin_id(current_row), + menu_type=None, + current_row=current_row, + role=role, + ) + except _PermissionProxyError as exc: + return role, current_row, (exc.response, exc.status) + if IMAGE_VIDEO_DATA_PERMISSION_KEY in key_set or 'image-video-task-data' in route_set: return role, current_row, None return role, current_row, (jsonify({'success': False, 'error': '无权查看视频任务数据'}), 403) @@ -723,18 +769,15 @@ def _get_column_ids_by_route_paths(route_paths, menu_type='admin'): normalized_paths = [(path or '').strip() for path in (route_paths or []) if (path or '').strip()] if not normalized_paths: return [] - conn = get_db() - try: - with conn.cursor() as cur: - placeholders = ','.join(['%s'] * len(normalized_paths)) - cur.execute( - f"SELECT id FROM columns WHERE menu_type = %s AND route_path IN ({placeholders}) ORDER BY id ASC", - [menu_type] + normalized_paths, - ) - rows = cur.fetchall() - return [int(row['id']) for row in rows if row.get('id') is not None] - finally: - conn.close() + items, error_response, _ = _java_permission_items(menu_type=menu_type) + if error_response is not None: + return [] + wanted = set(normalized_paths) + return [ + int(item['id']) + for item in _iter_permission_items(items) + if item.get('id') is not None and (item.get('route_path') or '').strip() in wanted + ] def _grant_backend_menu_permissions(user_ids, route_paths): @@ -742,18 +785,26 @@ def _grant_backend_menu_permissions(user_ids, route_paths): column_ids = _get_column_ids_by_route_paths(route_paths, 'admin') if not normalized_user_ids or not column_ids: return - conn = get_db() - try: - with conn.cursor() as cur: - for user_id in normalized_user_ids: - for column_id in column_ids: - cur.execute( - "INSERT IGNORE INTO user_column_permission (user_id, column_id) VALUES (%s, %s)", - (user_id, column_id), - ) - conn.commit() - finally: - conn.close() + role, current_row = get_current_admin_role() + for user_id in normalized_user_ids: + try: + direct_ids, error_response, _ = _java_user_direct_column_ids( + user_id, + current_row=current_row, + role=role, + ) + if error_response is not None: + continue + _, error_response, _ = _java_update_user_direct_column_ids( + user_id, + sorted(set(direct_ids).union(column_ids)), + current_row=current_row, + role=role, + ) + if error_response is not None: + continue + except (TypeError, ValueError): + continue @admin_api.route('/current-user') @@ -775,7 +826,6 @@ def get_admin_current_user(): @admin_api.route('/current-user/menus') @login_required def get_admin_current_user_menus(): - _ensure_product_category_schema() _, _, items, denied = _load_current_backend_menu_items() if denied: return denied @@ -958,7 +1008,11 @@ def create_user(): want_created_by = current_row['id'] is_admin = 1 if want_role in ('super_admin', 'admin') else 0 pwd_hash = generate_password_hash(password, method='pbkdf2:sha256') - column_ids = data.get('column_ids') or [] + raw_column_ids = data.get('column_ids') if 'column_ids' in data else data.get('columnIds') + try: + column_ids = _normalize_column_ids(raw_column_ids) + except ValueError as exc: + return jsonify({'success': False, 'error': str(exc)}), 400 try: conn = get_db() with conn.cursor() as cur: @@ -967,16 +1021,29 @@ def create_user(): (username, pwd_hash, is_admin, want_role, want_created_by), ) new_uid = cur.lastrowid - _validate_admin_granted_columns(cur, role, current_row, column_ids) - _set_user_column_permissions( - cur, + _validate_admin_granted_columns(role, current_row, column_ids) + # Commit the user row before Java validates and stores direct grants. + conn.commit() + _set_user_column_permissions_via_java( new_uid, column_ids, - preserve_column_keys=(IMAGE_VIDEO_DATA_PERMISSION_KEY,), + role=role, + current_row=current_row, ) - conn.commit() conn.close() return jsonify({'success': True, 'msg': '用户创建成功'}) + except _PermissionProxyError as exc: + try: + conn.close() + except Exception: + pass + return exc.response, exc.status + except ValueError as exc: + try: + conn.close() + except Exception: + pass + return jsonify({'success': False, 'error': str(exc)}), 400 except PermissionError as exc: try: conn.rollback() @@ -1003,7 +1070,7 @@ def update_user(uid): role, current_row = get_current_admin_role() if not role: return jsonify({'success': False, 'error': '需要管理员权限'}), 403 - if want_role is None and not password and 'column_ids' not in data: + if want_role is None and not password and 'column_ids' not in data and 'columnIds' not in data: return jsonify({'success': False, 'error': '请提供要修改的内容'}) try: conn = get_db() @@ -1038,17 +1105,35 @@ def update_user(uid): "UPDATE users SET is_admin = %s, role = %s WHERE id = %s", (is_admin, want_role, uid), ) - if 'column_ids' in data: - _validate_admin_granted_columns(cur, role, current_row, data.get('column_ids')) - _set_user_column_permissions( - cur, + if 'column_ids' in data or 'columnIds' in data: + try: + raw_column_ids = data.get('column_ids') if 'column_ids' in data else data.get('columnIds') + normalized_column_ids = _normalize_column_ids(raw_column_ids) + except ValueError as exc: + raise exc + _validate_admin_granted_columns(role, current_row, normalized_column_ids) + conn.commit() + _set_user_column_permissions_via_java( uid, - data.get('column_ids'), - preserve_column_keys=(IMAGE_VIDEO_DATA_PERMISSION_KEY,), + normalized_column_ids, + role=role, + current_row=current_row, ) conn.commit() conn.close() return jsonify({'success': True, 'msg': '更新成功'}) + except _PermissionProxyError as exc: + try: + conn.close() + except Exception: + pass + return exc.response, exc.status + except ValueError as exc: + try: + conn.close() + except Exception: + pass + return jsonify({'success': False, 'error': str(exc)}), 400 except PermissionError as exc: try: conn.rollback() @@ -1198,113 +1283,29 @@ def _parse_admin_datetime_arg(name): raise ValueError(f'{name} 时间格式无效') from exc -def _image_video_permission_column_id(cur): - cur.execute("SELECT id FROM columns WHERE column_key = %s LIMIT 1", (IMAGE_VIDEO_DATA_PERMISSION_KEY,)) - row = cur.fetchone() - if not row or row.get('id') is None: - # 兼容 V74 尚未执行的旧库：创建数据权限栏目，并继承已有菜单授权。 - cur.execute( - "INSERT INTO columns (name, column_key, menu_type, route_path, sort_order) " - "SELECT %s, %s, 'internal', 'image-video-task-data', 0 " - "WHERE NOT EXISTS (SELECT 1 FROM columns WHERE column_key = %s)", - ('视频任务数据查看', IMAGE_VIDEO_DATA_PERMISSION_KEY, IMAGE_VIDEO_DATA_PERMISSION_KEY), - ) - cur.execute("SELECT id FROM columns WHERE column_key = %s LIMIT 1", (IMAGE_VIDEO_DATA_PERMISSION_KEY,)) - row = cur.fetchone() - if not row or row.get('id') is None: - raise ValueError('视频任务数据权限栏目不存在') - cur.execute( - "INSERT IGNORE INTO user_column_permission (user_id, column_id) " - "SELECT old_perm.user_id, %s FROM user_column_permission old_perm " - "INNER JOIN columns old_col ON old_col.id = old_perm.column_id " - "WHERE old_col.column_key = %s", - (int(row['id']), IMAGE_VIDEO_PERMISSION_KEY), - ) - return int(row['id']) - - -def _is_stored_super_admin_sql(alias='u'): - return ( - f"(LOWER(COALESCE({alias}.role, '')) = 'super_admin' OR " - f"((COALESCE({alias}.role, '') = '') AND {alias}.is_admin = 1 AND {alias}.created_by_id IS NULL))" - ) - - @admin_api.route('/image-video-task-permissions', methods=['GET', 'PUT']) @login_required def manage_image_video_task_permissions(): - role, _ = get_current_admin_role() - if role != 'super_admin': - return jsonify({'success': False, 'error': '仅超级管理员可以配置视频任务权限'}), 403 + json_data = None + if request.method == 'PUT': + data = request.get_json(silent=True) or {} + raw_user_ids = data.get('user_ids') if 'user_ids' in data else data.get('userIds') + json_data = {'userIds': raw_user_ids} - conn = get_db() - try: - with conn.cursor() as cur: - column_id = _image_video_permission_column_id(cur) - if request.method == 'GET': - cur.execute( - "SELECT u.id, u.username, u.role, u.is_admin, u.created_by_id, " - "CASE WHEN ucp.user_id IS NULL THEN 0 ELSE 1 END AS granted " - "FROM users u LEFT JOIN user_column_permission ucp " - "ON ucp.user_id = u.id AND ucp.column_id = %s " - f"WHERE NOT {_is_stored_super_admin_sql('u')} " - "ORDER BY u.username ASC, u.id ASC", - (column_id,), - ) - items = [] - for row in cur.fetchall(): - stored_role = (row.get('role') or '').strip().lower() - effective_role = stored_role or ('admin' if row.get('is_admin') else 'normal') - items.append({ - 'id': int(row['id']), - 'username': row.get('username') or '', - 'role': effective_role, - 'granted': bool(row.get('granted')), - }) - return jsonify({'success': True, 'items': items}) - - data = request.get_json(silent=True) or {} - raw_user_ids = data.get('user_ids') - if not isinstance(raw_user_ids, list): - return jsonify({'success': False, 'error': 'user_ids 必须是数组'}), 400 - try: - user_ids = sorted({int(value) for value in raw_user_ids if not isinstance(value, bool) and int(value) > 0}) - except (TypeError, ValueError): - return jsonify({'success': False, 'error': 'user_ids 包含无效用户 ID'}), 400 - if len(user_ids) != len(raw_user_ids): - return jsonify({'success': False, 'error': 'user_ids 包含重复或无效用户 ID'}), 400 - - if user_ids: - placeholders = ','.join(['%s'] * len(user_ids)) - cur.execute( - f"SELECT id FROM users u WHERE id IN ({placeholders}) AND NOT {_is_stored_super_admin_sql('u')}", - tuple(user_ids), - ) - valid_ids = {int(row['id']) for row in cur.fetchall()} - if valid_ids != set(user_ids): - return jsonify({'success': False, 'error': '包含不存在或不可授权的用户'}), 400 - - cur.execute( - "DELETE ucp FROM user_column_permission ucp " - "INNER JOIN users u ON u.id = ucp.user_id " - f"WHERE ucp.column_id = %s AND NOT {_is_stored_super_admin_sql('u')}", - (column_id,), - ) - for user_id in user_ids: - cur.execute( - "INSERT INTO user_column_permission (user_id, column_id) VALUES (%s, %s)", - (user_id, column_id), - ) - conn.commit() - return jsonify({'success': True, 'granted_count': len(user_ids), 'msg': '视频任务权限已更新'}) - except ValueError as exc: - return jsonify({'success': False, 'error': str(exc)}), 400 - except Exception as exc: - if request.method == 'PUT': - conn.rollback() - return jsonify({'success': False, 'error': str(exc)}), 500 - finally: - conn.close() + result, error_response, status = _proxy_permission_java( + request.method, + '/api/admin/image-video-task-permissions', + json_data=json_data, + ) + if error_response is not None: + return error_response, status + if request.method == 'GET': + return jsonify({'success': True, 'items': _permission_response_items(result)}) + return jsonify({ + 'success': True, + 'granted_count': result.get('data'), + 'msg': result.get('message') or '视频任务权限已更新', + }) @admin_api.route('/image-video-tasks/download-zip', methods=['POST']) @@ -1518,23 +1519,16 @@ def list_columns(): _, _, denied = _ensure_admin_menu_access('columns', 'users') if denied: return denied - _ensure_product_category_schema() - menu_type = (request.args.get('menu_type') or '').strip() - local_rows = _load_local_columns(menu_type or None) + menu_type = (request.args.get('menu_type') or request.args.get('menuType') or '').strip() + items, error_response, status = _java_permission_items(menu_type=menu_type or None) + if error_response is not None: + return error_response, status items = [ - { - 'id': row.get('id'), - 'name': row.get('name') or '', - 'column_key': row.get('column_key') or '', - 'menu_type': row.get('menu_type') or 'app', - 'route_path': row.get('route_path') or '', - 'sort_order': int(row.get('sort_order') or 0), - 'created_at': str(row.get('created_at') or '')[:16].replace('T', ' '), - } - for row in local_rows - if (row.get('column_key') or '') != IMAGE_VIDEO_DATA_PERMISSION_KEY + item for item in items + if (item.get('column_key') or '').strip() != IMAGE_VIDEO_DATA_PERMISSION_KEY ] - return jsonify({'success': True, 'items': items}) + # Keep the legacy `items` field; some Java-aware callers use `data`/`columns`. + return jsonify({'success': True, 'items': items, 'columns': items, 'data': items}) @admin_api.route('/column', methods=['POST']) @@ -1545,11 +1539,13 @@ def create_column(): return denied data = request.get_json() or {} name = (data.get('name') or '').strip() - column_key = (data.get('column_key') or '').strip() - menu_type = (data.get('menu_type') or '').strip() or 'admin' - route_path = (data.get('route_path') or '').strip() + column_key = (data.get('column_key') or data.get('columnKey') or '').strip() + menu_type = (data.get('menu_type') or data.get('menuType') or '').strip() or 'admin' + route_path = (data.get('route_path') or data.get('routePath') or '').strip() try: - sort_order = _parse_optional_int(data.get('sort_order')) + sort_order = _parse_optional_int( + data.get('sort_order') if 'sort_order' in data else data.get('sortOrder') + ) except (TypeError, ValueError): return jsonify({'success': False, 'error': '排序必须是数字'}) if not name: @@ -1558,7 +1554,7 @@ def create_column(): return jsonify({'success': False, 'error': '栏目标识不能为空'}) if not route_path: return jsonify({'success': False, 'error': '菜单路由不能为空'}) - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'POST', '/api/admin/permission-menus', json_data={ @@ -1566,15 +1562,19 @@ def create_column(): 'columnKey': column_key, 'menuType': menu_type, 'routePath': route_path, + 'sortOrder': sort_order, + 'parentId': data.get('parent_id') if data.get('parent_id') not in ('', None) else data.get('parentId'), }, ) if error_response is not None: return error_response, status item = result.get('data') or {} - _sync_local_columns_from_backend([_format_permission_item(item)]) - if item.get('id') is not None: - _set_local_column_sort_order(item.get('id'), sort_order if sort_order is not None else _get_next_local_column_sort_order()) - return jsonify({'success': True, 'msg': result.get('message') or '创建成功', 'id': item.get('id')}) + return jsonify({ + 'success': True, + 'msg': result.get('message') or '创建成功', + 'id': item.get('id'), + 'item': _format_permission_item(item), + }) @admin_api.route('/column/', methods=['PUT']) @@ -1585,11 +1585,13 @@ def update_column(cid): return denied data = request.get_json() or {} name = (data.get('name') or '').strip() - column_key = (data.get('column_key') or '').strip() - menu_type = (data.get('menu_type') or '').strip() or 'admin' - route_path = (data.get('route_path') or '').strip() + column_key = (data.get('column_key') or data.get('columnKey') or '').strip() + menu_type = (data.get('menu_type') or data.get('menuType') or '').strip() or 'admin' + route_path = (data.get('route_path') or data.get('routePath') or '').strip() try: - sort_order = _parse_optional_int(data.get('sort_order')) + sort_order = _parse_optional_int( + data.get('sort_order') if 'sort_order' in data else data.get('sortOrder') + ) except (TypeError, ValueError): return jsonify({'success': False, 'error': '排序必须是数字'}) if not name: @@ -1598,7 +1600,7 @@ def update_column(cid): return jsonify({'success': False, 'error': '栏目标识不能为空'}) if not route_path: return jsonify({'success': False, 'error': '菜单路由不能为空'}) - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'PUT', f'/api/admin/permission-menus/{cid}', json_data={ @@ -1606,22 +1608,18 @@ def update_column(cid): 'columnKey': column_key, 'menuType': menu_type, 'routePath': route_path, + 'sortOrder': sort_order, + 'parentId': data.get('parent_id') if data.get('parent_id') not in ('', None) else data.get('parentId'), }, ) if error_response is not None: return error_response, status - _sync_local_columns_from_backend([{ - 'id': cid, - 'name': name, - 'column_key': column_key, - 'menu_type': menu_type, - 'route_path': route_path, - 'sort_order': sort_order if sort_order is not None else cid, - 'created_at': None, - }]) - if sort_order is not None: - _set_local_column_sort_order(cid, sort_order) - return jsonify({'success': True, 'msg': result.get('message') or '更新成功'}) + item = result.get('data') or {} + return jsonify({ + 'success': True, + 'msg': result.get('message') or '更新成功', + 'item': _format_permission_item(item), + }) @admin_api.route('/column/reorder', methods=['POST']) @@ -1638,12 +1636,45 @@ def reorder_column(): return jsonify({'success': False, 'error': '菜单重排参数无效'}), 400 if column_id == target_id: return jsonify({'success': True, 'msg': '排序未变化'}) - try: - _swap_local_column_sort_order(column_id, target_id) - except ValueError as exc: - return jsonify({'success': False, 'error': str(exc)}), 400 - except Exception as exc: - return jsonify({'success': False, 'error': str(exc)}), 500 + menu_type = (data.get('menu_type') or data.get('menuType') or 'admin').strip() or 'admin' + items, error_response, status = _java_permission_items(menu_type=menu_type) + if error_response is not None: + return error_response, status + by_id = {} + for item in _iter_permission_items(items): + try: + item_id = int(item.get('id')) + except (TypeError, ValueError): + continue + by_id[item_id] = item + left = by_id.get(column_id) + right = by_id.get(target_id) + if not left or not right: + return jsonify({'success': False, 'error': '栏目不存在'}), 400 + + def _reorder_payload(item, sort_order): + return { + 'name': item.get('name') or '', + 'columnKey': item.get('column_key') or '', + 'menuType': item.get('menu_type') or menu_type, + 'routePath': item.get('route_path') or '', + 'sortOrder': int(sort_order if sort_order is not None else 0), + 'parentId': item.get('parent_id'), + } + + left_order = left.get('sort_order') or column_id + right_order = right.get('sort_order') or target_id + for item_id, item, sort_order in ( + (column_id, left, right_order), + (target_id, right, left_order), + ): + _, error_response, status = _proxy_permission_java( + 'PUT', + f'/api/admin/permission-menus/{item_id}', + json_data=_reorder_payload(item, sort_order), + ) + if error_response is not None: + return error_response, status return jsonify({'success': True, 'msg': '排序更新成功'}) @@ -1653,30 +1684,63 @@ def delete_column(cid): _, _, denied = _ensure_admin_menu_access('columns') if denied: return denied - result, error_response, status = _proxy_backend_java( + result, error_response, status = _proxy_permission_java( 'DELETE', f'/api/admin/permission-menus/{cid}', ) if error_response is not None: return error_response, status - conn = get_db() - try: - with conn.cursor() as cur: - cur.execute("DELETE FROM columns WHERE id = %s", (cid,)) - conn.commit() - finally: - conn.close() return jsonify({'success': True, 'msg': result.get('message') or '删除成功'}) -@admin_api.route('/user//columns') +@admin_api.route('/user//columns', methods=['GET', 'PUT']) @admin_required def get_user_columns(uid): _, _, denied = _ensure_admin_menu_access('users') if denied: return denied - menu_type = (request.args.get('menu_type') or '').strip() - return jsonify({'success': True, 'column_ids': _get_local_user_column_ids(uid, menu_type or None)}) + menu_type = (request.args.get('menu_type') or request.args.get('menuType') or '').strip() + if request.method == 'PUT': + data = request.get_json(silent=True) or {} + raw_ids = data.get('column_ids') if 'column_ids' in data else data.get('columnIds') + role, current_row = get_current_admin_role() + try: + normalized_ids = _normalize_column_ids(raw_ids) + if role == 'admin': + allowed_ids, _, _ = _effective_permission_sets( + _get_current_admin_id(current_row), + menu_type=None, + current_row=current_row, + role=role, + ) + if not set(normalized_ids).issubset(allowed_ids): + return jsonify({'success': False, 'error': '普通管理员只能分配自己的有效菜单权限'}), 403 + _, error_response, status = _java_update_user_direct_column_ids( + uid, + normalized_ids, + menu_type=menu_type or None, + current_row=current_row, + role=role, + ) + if error_response is not None: + return error_response, status + except _PermissionProxyError as exc: + return exc.response, exc.status + except ValueError as exc: + return jsonify({'success': False, 'error': str(exc)}), 400 + return jsonify({'success': True, 'column_ids': normalized_ids, 'columnIds': normalized_ids}) + column_ids, error_response, status = _java_user_direct_column_ids( + uid, + menu_type=menu_type or None, + ) + if error_response is not None: + return error_response, status + return jsonify({ + 'success': True, + 'column_ids': column_ids, + 'columnIds': column_ids, + 'data': {'columnIds': column_ids, 'column_ids': column_ids}, + }) @admin_api.route('/user//column-permissions') @@ -1685,55 +1749,31 @@ def get_user_column_permissions(uid): _, _, denied = _ensure_admin_menu_access('users') if denied: return denied - menu_type = (request.args.get('menu_type') or '').strip() - items = _get_local_user_column_permission_items(uid, menu_type or None) - return jsonify({'success': True, 'items': [_format_permission_item(item) for item in items]}) + menu_type = (request.args.get('menu_type') or request.args.get('menuType') or '').strip() + items, error_response, status = _java_permission_items( + menu_type=menu_type or None, + user_id=uid, + effective=True, + ) + if error_response is not None: + return error_response, status + return jsonify({'success': True, 'items': items, 'data': items}) -def _set_user_column_permissions(cur, user_id, column_ids, preserve_column_keys=()): - """设置用户栏目权限：先删后插。cur 为已打开的游标。""" - requested_ids = {int(x) for x in (column_ids or []) if x} - preserved_ids = set() - if preserve_column_keys: - placeholders = ','.join(['%s'] * len(preserve_column_keys)) - cur.execute( - f"SELECT id FROM columns WHERE column_key IN ({placeholders})", - tuple(preserve_column_keys), - ) - protected_ids = {int(row['id']) for row in cur.fetchall() if row.get('id') is not None} - requested_ids.difference_update(protected_ids) - if protected_ids: - id_placeholders = ','.join(['%s'] * len(protected_ids)) - cur.execute( - f"SELECT column_id FROM user_column_permission WHERE user_id = %s AND column_id IN ({id_placeholders})", - (user_id, *sorted(protected_ids)), - ) - preserved_ids = {int(row['column_id']) for row in cur.fetchall() if row.get('column_id') is not None} - cur.execute("DELETE FROM user_column_permission WHERE user_id = %s", (user_id,)) - for cid in sorted(requested_ids | preserved_ids): - cur.execute( - "INSERT INTO user_column_permission (user_id, column_id) VALUES (%s, %s)", - (user_id, cid), - ) - - -def _validate_admin_granted_columns(cur, role, current_row, column_ids): +def _validate_admin_granted_columns(role, current_row, column_ids): """普通管理员只能把自己已拥有的栏目授予其直接管理的普通用户。""" if role != 'admin' or not current_row or not column_ids: return - requested_ids = {int(value) for value in column_ids if str(value).isdigit() and int(value) > 0} - if not requested_ids: - return - placeholders = ','.join(['%s'] * len(requested_ids)) - cur.execute( - f"SELECT column_id FROM user_column_permission " - f"WHERE user_id = %s AND column_id IN ({placeholders})", - (int(current_row['id']), *sorted(requested_ids)), + requested_ids = set(_normalize_column_ids(column_ids)) + allowed_ids, _, _ = _effective_permission_sets( + _get_current_admin_id(current_row), + menu_type=None, + current_row=current_row, + role=role, ) - owned_ids = {int(row['column_id']) for row in cur.fetchall() if row.get('column_id') is not None} - denied_ids = requested_ids - owned_ids + denied_ids = requested_ids - allowed_ids if denied_ids: - raise PermissionError('普通管理员只能分配自己已有的菜单权限') + raise PermissionError('普通管理员只能分配自己的有效菜单权限') # ---------- 版本管理（web_config） ---------- @@ -1741,6 +1781,18 @@ def _validate_admin_granted_columns(cur, role, current_row, column_ids): # ---------- 商品类目 ---------- +def _set_user_column_permissions_via_java(user_id, column_ids, *, role=None, current_row=None): + """Persist direct grants in Java; never mirror them in Flask's legacy tables.""" + _, error_response, status = _java_update_user_direct_column_ids( + user_id, + column_ids, + current_row=current_row, + role=role, + ) + if error_response is not None: + raise _PermissionProxyError(error_response, status) + + def _format_product_category_item(item, include_children=True): formatted = { 'id': item.get('id'), @@ -1779,7 +1831,6 @@ def _build_product_category_payload_for_java(data): @admin_api.route('/product-categories') @login_required def list_product_categories(): - _ensure_product_category_schema() _, _, denied = _ensure_product_category_access() if denied: return denied @@ -1803,7 +1854,6 @@ def list_product_categories(): @admin_api.route('/product-categories/children') @login_required def list_product_category_children(): - _ensure_product_category_schema() _, _, denied = _ensure_product_category_access() if denied: return denied @@ -1837,7 +1887,6 @@ def list_product_category_children(): @admin_api.route('/product-categories/search') @login_required def search_product_categories(): - _ensure_product_category_schema() _, _, denied = _ensure_product_category_access() if denied: return denied @@ -1869,7 +1918,6 @@ def search_product_categories(): @admin_api.route('/product-categories/export') @login_required def export_product_categories(): - _ensure_product_category_schema() _, _, denied = _ensure_product_category_access() if denied: return denied @@ -1909,7 +1957,6 @@ def export_product_categories(): @admin_api.route('/product-category', methods=['POST']) @login_required def create_product_category(): - _ensure_product_category_schema() _, _, denied = _ensure_product_category_access() if denied: return denied @@ -1930,7 +1977,6 @@ def create_product_category(): @admin_api.route('/product-category/', methods=['PUT']) @login_required def update_product_category(item_id): - _ensure_product_category_schema() _, _, denied = _ensure_product_category_access() if denied: return denied @@ -1951,7 +1997,6 @@ def update_product_category(item_id): @admin_api.route('/product-category/', methods=['DELETE']) @login_required def delete_product_category(item_id): - _ensure_product_category_schema() _, _, denied = _ensure_product_category_access() if denied: return denied @@ -3684,46 +3729,3 @@ def update_query_asin_country(item_id, country): 'msg': result.get('message') or '保存成功', 'item': _format_query_asin_item(result.get('data') or {}), }) - - -def _get_local_user_column_ids(user_id, menu_type=None): - conn = get_db() - try: - with conn.cursor() as cur: - sql = """ - SELECT c.id - FROM user_column_permission ucp - INNER JOIN columns c ON c.id = ucp.column_id - WHERE ucp.user_id = %s - """ - params = [user_id] - if menu_type: - sql += " AND c.menu_type = %s" - params.append(menu_type) - sql += " ORDER BY c.sort_order ASC, c.id ASC" - cur.execute(sql, params) - rows = cur.fetchall() - return [int(row['id']) for row in rows if row.get('id') is not None] - finally: - conn.close() - - -def _get_local_user_column_permission_items(user_id, menu_type=None): - conn = get_db() - try: - with conn.cursor() as cur: - sql = """ - SELECT c.id, c.name, c.column_key, c.menu_type, c.route_path, c.sort_order, c.created_at - FROM user_column_permission ucp - INNER JOIN columns c ON c.id = ucp.column_id - WHERE ucp.user_id = %s - """ - params = [user_id] - if menu_type: - sql += " AND c.menu_type = %s" - params.append(menu_type) - sql += " ORDER BY c.sort_order ASC, c.id ASC" - cur.execute(sql, params) - return cur.fetchall() - finally: - conn.close() diff --git a/backend/static/admin.js b/backend/static/admin.js index 3402e278..a267d36b 100644 --- a/backend/static/admin.js +++ b/backend/static/admin.js @@ -382,115 +382,322 @@ }); } var allColumnsList = []; + var columnPermissionCatalogReady = false; + var columnCardsContainerMap = { createColumnPermissionWrap: 'createColumnCards', editColumnPermissionWrap: 'editColumnCards' }; + // Hierarchical permission editor: only direct IDs are submitted; descendants are visual inheritance. + function columnParentId(item) { + var value = item && item.parent_id != null ? item.parent_id : item && item.parentId; + var parsed = value == null || value === '' ? null : Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : null; + } + function columnId(item) { return Number(item && item.id); } + function columnMenuType(item) { + return String(item && item.menu_type || 'app').toLowerCase() === 'admin' ? 'admin' : 'app'; + } + function columnDescendantIds(id) { + var result = [], pending = [Number(id)]; + while (pending.length) { + var parent = pending.shift(); + allColumnsList.forEach(function (item) { + if (columnParentId(item) !== parent) return; + var childId = columnId(item); + if (result.indexOf(childId) < 0) { + result.push(childId); + pending.push(childId); + } + }); + } + return result; + } + function columnTreeHasDirectDescendant(id, directIds) { + return columnDescendantIds(id).some(function (descendantId) { return !!directIds[descendantId]; }); + } + function columnTreeFullyCovered(id, directIds) { + if (directIds[id]) return true; + var children = allColumnsList.filter(function (item) { return columnParentId(item) === Number(id); }); + return children.length > 0 && children.every(function (child) { + return columnTreeFullyCovered(columnId(child), directIds); + }); + } function loadColumnsForPermission() { - var ownedKeys = {}; - var ownedRoutes = {}; + columnPermissionCatalogReady = false; + var ownedKeys = {}, ownedRoutes = {}; var ownPermissionRequest = (currentUserRole === 'admin' && currentUserId) ? fetch('/api/admin/user/' + currentUserId + '/column-permissions') .then(function (r) { return r.json(); }) .then(function (res) { + if (!res.success) throw new Error(res.error || '当前用户权限加载失败'); (res.items || []).forEach(function (item) { if (item.column_key) ownedKeys[String(item.column_key).trim()] = true; if (item.route_path) ownedRoutes[String(item.route_path).trim()] = true; }); }) - .catch(function () { }) : Promise.resolve(); - ownPermissionRequest.then(function () { - return fetch('/api/admin/columns'); - }).then(function (r) { return r.json(); }) + ownPermissionRequest.then(function () { return fetch('/api/admin/columns'); }) + .then(function (r) { return r.json(); }) .then(function (res) { if (!res.success) return; - allColumnsList = (res.items || []).filter(function (item) { + var availableColumns = (res.items || []).filter(function (item) { if (item.column_key === 'admin_image_video_task_data') return false; - if (currentUserRole !== 'admin') return true; - return !!ownedKeys[String(item.column_key || '').trim()] || - !!ownedRoutes[String(item.route_path || '').trim()]; + return true; }); + if (currentUserRole === 'admin') { + var columnsById = {}, grantableIds = {}, visibleIds = {}; + availableColumns.forEach(function (item) { + var id = columnId(item); + columnsById[id] = item; + if (ownedKeys[String(item.column_key || '').trim()] || ownedRoutes[String(item.route_path || '').trim()]) { + grantableIds[id] = true; + } + }); + Object.keys(grantableIds).forEach(function (id) { + var currentId = Number(id), visited = {}; + while (currentId > 0 && !visited[currentId] && columnsById[currentId]) { + visited[currentId] = true; + visibleIds[currentId] = true; + currentId = columnParentId(columnsById[currentId]); + } + }); + allColumnsList = availableColumns.filter(function (item) { + return !!visibleIds[columnId(item)]; + }).map(function (item) { + item._structureOnly = !grantableIds[columnId(item)]; + return item; + }); + } else { + allColumnsList = availableColumns.map(function (item) { + item._structureOnly = false; + return item; + }); + } + columnPermissionCatalogReady = true; renderColumnPermissionWrap('createColumnPermissionWrap'); renderColumnPermissionWrap('editColumnPermissionWrap'); - var selCreate = document.getElementById('createColumnPermissionWrap'); - var selEdit = document.getElementById('editColumnPermissionWrap'); - if (selCreate && !selCreate._colCardsBound) { - selCreate._colCardsBound = true; - selCreate.onchange = function () { renderColumnCards('createColumnPermissionWrap'); }; - } - if (selEdit && !selEdit._colCardsBound) { - selEdit._colCardsBound = true; - selEdit.onchange = function () { renderColumnCards('editColumnPermissionWrap'); }; - } - }) - .catch(function () { }); + populateColumnParentSelects(); + }).catch(function () { + columnPermissionCatalogReady = false; + allColumnsList = []; + var editPermissionWrap = document.getElementById('editColumnPermissionWrap'); + if (editPermissionWrap) editPermissionWrap._permissionsReady = false; + renderColumnPermissionWrap('createColumnPermissionWrap'); + renderColumnPermissionWrap('editColumnPermissionWrap'); + }); } - var columnCardsContainerMap = { createColumnPermissionWrap: 'createColumnCards', editColumnPermissionWrap: 'editColumnCards' }; - function renderColumnCards(selectId) { - var sel = document.getElementById(selectId); - var cardsId = columnCardsContainerMap[selectId]; - var cardsEl = cardsId ? document.getElementById(cardsId) : null; - if (!sel || sel.tagName !== 'SELECT' || !cardsEl) return; + function renderColumnCards(wrapId) { + var wrap = document.getElementById(wrapId); + var cardsEl = document.getElementById(columnCardsContainerMap[wrapId]); + if (!wrap || !cardsEl) return; + var directIds = wrap._directIds || {}; cardsEl.innerHTML = ''; - for (var i = 0; i < sel.options.length; i++) { - var opt = sel.options[i]; - if (opt.disabled || !opt.selected) continue; + allColumnsList.forEach(function (item) { + var id = columnId(item); + if (!directIds[id] || item._structureOnly) return; var card = document.createElement('span'); card.className = 'column-permission-card'; - card.textContent = opt.textContent; + card.textContent = (item.name || '') + ' (' + (item.column_key || '') + ')'; var btn = document.createElement('button'); btn.type = 'button'; btn.className = 'col-card-remove'; btn.setAttribute('aria-label', '移除'); btn.textContent = '×'; - (function (option, seldId) { - btn.onclick = function () { - option.selected = false; - renderColumnCards(seldId); - }; - })(opt, selectId); + btn.onclick = function () { + delete directIds[id]; + renderColumnPermissionWrap(wrapId); + }; card.appendChild(btn); cardsEl.appendChild(card); - } + }); } function renderColumnPermissionWrap(wrapId) { - var sel = document.getElementById(wrapId); - if (!sel || sel.tagName !== 'SELECT') return; - sel.innerHTML = ''; - allColumnsList.forEach(function (c) { - var opt = document.createElement('option'); - opt.value = c.id; - opt.textContent = c.name + ' (' + c.column_key + ')'; - sel.appendChild(opt); - }); - if (allColumnsList.length === 0) { - var opt = document.createElement('option'); - opt.disabled = true; - opt.textContent = '暂无菜单，请先在“栏目权限配置”中新增菜单'; - sel.appendChild(opt); + var wrap = document.getElementById(wrapId); + if (!wrap) return; + var directIds = wrap._directIds || {}; + var collapsedIds = wrap._collapsedIds || {}; + var collapsedMenuTypes = wrap._collapsedMenuTypes || {}; + var previousScrollTop = wrap.scrollTop; + wrap._directIds = directIds; + wrap._collapsedIds = collapsedIds; + wrap._collapsedMenuTypes = collapsedMenuTypes; + wrap.innerHTML = ''; + if (!allColumnsList.length) { + wrap.textContent = '暂无可分配菜单'; + renderColumnCards(wrapId); + return; } + var byParent = {}; + allColumnsList.forEach(function (item) { + var parent = columnParentId(item); + var key = parent == null ? 'root' : String(parent); + (byParent[key] || (byParent[key] = [])).push(item); + }); + if (!wrap._collapseStateInitialized) { + allColumnsList.forEach(function (item) { + var id = columnId(item); + if ((byParent[String(id)] || []).length) collapsedIds[id] = true; + }); + wrap._collapseStateInitialized = true; + } + if (!wrap._menuTypeCollapseInitialized) { + collapsedMenuTypes.admin = true; + collapsedMenuTypes.app = true; + wrap._menuTypeCollapseInitialized = true; + } + function renderNode(item, depth, inherited) { + var id = columnId(item); + var itemMenuType = columnMenuType(item); + var children = (byParent[String(id)] || []).filter(function (child) { + return columnMenuType(child) === itemMenuType; + }); + var structureOnly = item._structureOnly === true; + var effective = inherited || !!directIds[id]; + var covered = effective || columnTreeFullyCovered(id, directIds); + var node = document.createElement('div'); + node.className = 'column-tree-node'; + node.setAttribute('data-depth', String(depth)); + var row = document.createElement('div'); + row.className = 'column-tree-row'; + if (children.length) { + var toggle = document.createElement('button'); + var collapsed = !!collapsedIds[id]; + toggle.type = 'button'; + toggle.className = 'column-tree-toggle'; + toggle.setAttribute('aria-expanded', String(!collapsed)); + toggle.setAttribute('aria-label', (collapsed ? '展开' : '收起') + (item.name || '菜单')); + toggle.title = collapsed ? '展开子菜单' : '收起子菜单'; + toggle.onclick = function () { + if (collapsedIds[id]) delete collapsedIds[id]; + else collapsedIds[id] = true; + renderColumnPermissionWrap(wrapId); + }; + row.appendChild(toggle); + } else { + var spacer = document.createElement('span'); + spacer.className = 'column-tree-toggle-spacer'; + spacer.setAttribute('aria-hidden', 'true'); + row.appendChild(spacer); + } + var label = document.createElement('label'); + var checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + checkbox.checked = covered; + checkbox.indeterminate = !covered && columnTreeHasDirectDescendant(id, directIds); + checkbox.disabled = inherited || structureOnly; + checkbox.onchange = function () { + if (structureOnly) return; + if (checkbox.checked) { + directIds[id] = true; + columnDescendantIds(id).forEach(function (descendantId) { delete directIds[descendantId]; }); + } else { + delete directIds[id]; + columnDescendantIds(id).forEach(function (descendantId) { delete directIds[descendantId]; }); + } + renderColumnPermissionWrap(wrapId); + }; + var text = document.createElement('span'); + text.textContent = (item.name || '') + ' (' + (item.column_key || '') + ')'; + if (inherited || structureOnly) text.className = 'inherited-label'; + label.appendChild(checkbox); + label.appendChild(text); + row.appendChild(label); + node.appendChild(row); + if (children.length) { + var childrenWrap = document.createElement('div'); + childrenWrap.className = 'column-tree-children'; + childrenWrap.hidden = !!collapsedIds[id]; + children.forEach(function (child) { + childrenWrap.appendChild(renderNode(child, depth + 1, effective)); + }); + node.appendChild(childrenWrap); + } + return node; + } + [ + { key: 'admin', label: '后台菜单' }, + { key: 'app', label: 'APP（软件）菜单' } + ].forEach(function (group) { + var groupItems = allColumnsList.filter(function (item) { + return columnMenuType(item) === group.key; + }); + if (!groupItems.length) return; + var section = document.createElement('div'); + section.className = 'column-tree-group'; + var groupToggle = document.createElement('button'); + var groupCollapsed = !!collapsedMenuTypes[group.key]; + groupToggle.type = 'button'; + groupToggle.className = 'column-tree-group-toggle'; + groupToggle.setAttribute('aria-expanded', String(!groupCollapsed)); + groupToggle.setAttribute('aria-label', (groupCollapsed ? '展开' : '收起') + group.label); + var groupName = document.createElement('span'); + groupName.className = 'column-tree-group-name'; + groupName.textContent = group.label; + var groupCount = document.createElement('span'); + groupCount.className = 'column-tree-group-count'; + groupCount.textContent = groupItems.length + ' 项'; + groupToggle.appendChild(groupName); + groupToggle.appendChild(groupCount); + groupToggle.onclick = function () { + if (collapsedMenuTypes[group.key]) delete collapsedMenuTypes[group.key]; + else collapsedMenuTypes[group.key] = true; + renderColumnPermissionWrap(wrapId); + }; + section.appendChild(groupToggle); + var groupBody = document.createElement('div'); + groupBody.className = 'column-tree-group-body'; + groupBody.hidden = groupCollapsed; + groupItems.filter(function (item) { + var parentId = columnParentId(item); + if (parentId == null) return true; + var parent = allColumnsList.find(function (candidate) { + return columnId(candidate) === parentId; + }); + return !parent || columnMenuType(parent) !== group.key; + }).forEach(function (item) { + groupBody.appendChild(renderNode(item, 0, false)); + }); + section.appendChild(groupBody); + wrap.appendChild(section); + }); renderColumnCards(wrapId); + wrap.scrollTop = previousScrollTop; } function getSelectedColumnIds(wrapId) { - var sel = document.getElementById(wrapId); - if (!sel || sel.tagName !== 'SELECT') return []; - var ids = []; - for (var i = 0; i < sel.options.length; i++) { - if (sel.options[i].selected) { - var v = parseInt(sel.options[i].value, 10); - if (!isNaN(v)) ids.push(v); - } - } - return ids; + var wrap = document.getElementById(wrapId); + return wrap && wrap._directIds ? Object.keys(wrap._directIds).map(Number).filter(function (id) { + var item = allColumnsList.find(function (column) { return columnId(column) === id; }); + return id > 0 && item && !item._structureOnly; + }) : []; } function setColumnPermissionCheckboxes(wrapId, columnIds) { - var sel = document.getElementById(wrapId); - if (!sel || sel.tagName !== 'SELECT') return; - var set = {}; - (columnIds || []).forEach(function (id) { set[id] = true; }); - for (var i = 0; i < sel.options.length; i++) { - var opt = sel.options[i]; - if (opt.disabled) continue; - opt.selected = set[parseInt(opt.value, 10)] || false; - } - renderColumnCards(wrapId); + var wrap = document.getElementById(wrapId); + if (!wrap) return; + wrap._directIds = {}; + (columnIds || []).forEach(function (id) { + var parsed = Number(id); + if (Number.isFinite(parsed) && parsed > 0) wrap._directIds[parsed] = true; + }); + wrap._permissionsReady = true; + renderColumnPermissionWrap(wrapId); + } + function populateColumnParentSelects() { + ['columnParentId', 'editColumnParentId'].forEach(function (selectId) { + var select = document.getElementById(selectId); + if (!select) return; + var current = select.value; + var typeSelect = document.getElementById(selectId === 'editColumnParentId' ? 'editColumnMenuType' : 'columnMenuType'); + var menuType = typeSelect ? (typeSelect.value || 'app') : 'app'; + var editingId = selectId === 'editColumnParentId' ? Number(document.getElementById('editColumnId').value || 0) : 0; + var blockedIds = editingId > 0 ? [editingId].concat(columnDescendantIds(editingId)) : []; + select.innerHTML = ''; + allColumnsList.filter(function (item) { + return !item._structureOnly && String(item.menu_type || 'app') === String(menuType) && blockedIds.indexOf(columnId(item)) < 0; + }).forEach(function (item) { + var option = document.createElement('option'); + option.value = item.id; + option.textContent = (item.name || '') + ' (' + (item.column_key || '') + ')'; + select.appendChild(option); + }); + select.value = current; + }); } function updateCreateFormByRole() { var roleSel = document.getElementById('createRole'); @@ -531,14 +738,25 @@ editCreatorName.value = u.creator_username || ''; if (u.role !== 'super_admin') { editRole.value = u.role || 'normal'; } document.getElementById('msgEdit').textContent = ''; - if (allColumnsList.length) { - renderColumnPermissionWrap('editColumnPermissionWrap'); - fetch('/api/admin/user/' + (u.id) + '/columns') - .then(function (r) { return r.json(); }) - .then(function (res) { - if (res.success && res.column_ids) setColumnPermissionCheckboxes('editColumnPermissionWrap', res.column_ids); - }); - } + var editPermissionWrap = document.getElementById('editColumnPermissionWrap'); + editPermissionWrap._directIds = {}; + editPermissionWrap._permissionsReady = false; + renderColumnPermissionWrap('editColumnPermissionWrap'); + fetch('/api/admin/user/' + (u.id) + '/columns') + .then(function (r) { return r.json(); }) + .then(function (res) { + if (String(document.getElementById('editUserId').value || '') !== String(u.id || '')) return; + if (!res.success || !Array.isArray(res.column_ids)) { + throw new Error(res.error || '权限加载失败'); + } + setColumnPermissionCheckboxes('editColumnPermissionWrap', res.column_ids); + }) + .catch(function (error) { + if (String(document.getElementById('editUserId').value || '') !== String(u.id || '')) return; + var msgEl = document.getElementById('msgEdit'); + msgEl.textContent = error.message || '权限加载失败'; + msgEl.className = 'msg err'; + }); document.getElementById('editUserModal').classList.add('show'); }; }); @@ -575,6 +793,11 @@ msgEl.classList.add('err'); return; } + if (!columnPermissionCatalogReady) { + msgEl.textContent = '菜单权限尚未加载，请稍后重试'; + msgEl.classList.add('err'); + return; + } var body = { username: username, password: password, role: role }; if (role === 'normal' && currentUserRole === 'super_admin' && createdById) body.created_by_id = createdById; body.column_ids = getSelectedColumnIds('createColumnPermissionWrap'); @@ -609,6 +832,12 @@ var msgEl = document.getElementById('msgEdit'); msgEl.textContent = ''; msgEl.className = 'msg'; + var editPermissionWrap = document.getElementById('editColumnPermissionWrap'); + if (!columnPermissionCatalogReady || !editPermissionWrap._permissionsReady) { + msgEl.textContent = '用户权限尚未加载，不能保存'; + msgEl.classList.add('err'); + return; + } var body = {}; if (password) body.password = password; if (currentUserRole === 'super_admin' && editRoleEl && editRoleEl.offsetParent !== null) @@ -4483,114 +4712,6 @@ // ========== 栏目权限配置 ========== - function loadColumns() { - fetch('/api/admin/columns') - .then(function (r) { return r.json(); }) - .then(function (res) { - var tbody = document.getElementById('columnListBody'); - if (!res.success) { - tbody.innerHTML = '加载失败: ' + (res.error || '') + ''; - return; - } - allColumnsList = res.items || []; - if (allColumnsList.length === 0) { - tbody.innerHTML = '暂无菜单，请在上方新增'; - } else { - tbody.innerHTML = allColumnsList.map(function (c) { - return '' + c.id + '' + (c.name || '') + '' + (c.column_key || '') + '' + (c.created_at || '') + '' + - ' ' + - ''; - }).join(''); - } - bindColumnActions(); - }) - .catch(function () { - document.getElementById('columnListBody').innerHTML = '请求失败'; - }); - } - function bindColumnActions() { - document.querySelectorAll('[data-column-edit]').forEach(function (btn) { - btn.onclick = function () { - document.getElementById('editColumnId').value = btn.dataset.columnEdit || ''; - document.getElementById('editColumnName').value = (btn.dataset.name || '').replace(/"/g, '"'); - document.getElementById('editColumnKey').value = (btn.dataset.key || '').replace(/"/g, '"'); - document.getElementById('msgEditColumn').textContent = ''; - document.getElementById('editColumnModal').classList.add('show'); - }; - }); - document.querySelectorAll('[data-column-delete]').forEach(function (btn) { - btn.onclick = function () { - if (!confirm('确定删除菜单“' + (btn.dataset.name || '').replace(/"/g, '"') + '”吗？')) return; - fetch('/api/admin/column/' + btn.dataset.columnDelete, { method: 'DELETE' }) - .then(function (r) { return r.json(); }) - .then(function (res) { - if (res.success) { loadColumns(); loadColumnsForPermission(); loadAdminMenus(getActiveAdminTabName()); } - else { alert(res.error || '删除失败'); } - }); - }; - }); - } - document.getElementById('btnAddColumn').onclick = function () { - var name = (document.getElementById('columnName').value || '').trim(); - var key = (document.getElementById('columnKey').value || '').trim(); - var msgEl = document.getElementById('msgColumn'); - msgEl.textContent = ''; - msgEl.className = 'msg'; - if (!name) { msgEl.textContent = '请填写菜单名称'; msgEl.classList.add('err'); return; } - if (!key) { - msgEl.textContent = '请填写栏目标识'; msgEl.classList.add('err'); return; - } - fetch('/api/admin/column', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: name, column_key: key }) - }) - .then(function (r) { return r.json(); }) - .then(function (res) { - if (res.success) { - msgEl.textContent = res.msg || '新增成功'; - msgEl.classList.add('ok'); - document.getElementById('columnName').value = ''; - document.getElementById('columnKey').value = ''; - loadColumns(); - loadColumnsForPermission(); - loadAdminMenus(getActiveAdminTabName()); - } else { - msgEl.textContent = res.error || '新增失败'; - msgEl.classList.add('err'); - } - }) - .catch(function () { msgEl.textContent = '请求失败'; msgEl.classList.add('err'); }); - }; - document.getElementById('btnSaveColumn').onclick = function () { - var cid = document.getElementById('editColumnId').value; - var name = (document.getElementById('editColumnName').value || '').trim(); - var key = (document.getElementById('editColumnKey').value || '').trim(); - var msgEl = document.getElementById('msgEditColumn'); - msgEl.textContent = ''; - msgEl.className = 'msg'; - if (!name) { msgEl.textContent = '请填写菜单名称'; msgEl.classList.add('err'); return; } - if (!key) { - msgEl.textContent = '请填写栏目标识'; msgEl.classList.add('err'); return; - } - fetch('/api/admin/column/' + cid, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: name, column_key: key }) - }) - .then(function (r) { return r.json(); }) - .then(function (res) { - if (res.success) { - document.getElementById('editColumnModal').classList.remove('show'); - loadColumns(); - loadColumnsForPermission(); - loadAdminMenus(getActiveAdminTabName()); - } else { - msgEl.textContent = res.error || '保存失败'; - msgEl.classList.add('err'); - } - }); - }; document.getElementById('btnCloseEditColumn').onclick = function () { document.getElementById('editColumnModal').classList.remove('show'); }; @@ -4601,29 +4722,39 @@ .then(function (res) { var tbody = document.getElementById('columnListBody'); if (!res.success) { - tbody.innerHTML = '加载失败: ' + (res.error || '') + ''; + tbody.innerHTML = '加载失败: ' + (res.error || '') + ''; return; } allColumnsList = res.items || []; if (allColumnsList.length === 0) { - tbody.innerHTML = '暂无菜单，请先在上方新增'; + tbody.innerHTML = '暂无菜单，请先在上方新增'; } else { - tbody.innerHTML = allColumnsList.map(function (c, index) { + tbody.innerHTML = allColumnsList.map(function (c) { + var siblings = columnSiblingItems(c); + var siblingIndex = siblings.findIndex(function (item) { return String(item.id) === String(c.id); }); var moveButtons = - ' ' + - ' '; - return '' + c.id + '' + (c.name || '') + '' + (c.column_key || '') + '' + ((c.menu_type || '') === 'admin' ? '后台(admin)' : '软件(app)') + '' + (c.sort_order != null ? c.sort_order : 0) + '' + (c.route_path || '') + '' + (c.created_at || '') + '' + + ' ' + + ' '; + var parent = allColumnsList.find(function (item) { return String(item.id) === String(c.parent_id || c.parentId || ''); }); + return '' + c.id + '' + (c.name || '') + '' + (c.column_key || '') + '' + ((c.menu_type || '') === 'admin' ? '后台(admin)' : '软件(app)') + '' + (parent ? (parent.name || '') : '-') + '' + (c.sort_order != null ? c.sort_order : 0) + '' + (c.route_path || '') + '' + (c.created_at || '') + '' + moveButtons + - ' ' + + ' ' + ''; }).join(''); } bindColumnActions(); }) .catch(function () { - document.getElementById('columnListBody').innerHTML = '请求失败'; + document.getElementById('columnListBody').innerHTML = '请求失败'; }); } + function columnSiblingItems(item) { + var menuType = String(item && item.menu_type || 'app'); + var parentId = columnParentId(item); + return allColumnsList.filter(function (candidate) { + return String(candidate.menu_type || 'app') === menuType && columnParentId(candidate) === parentId; + }); + } function bindColumnActions() { document.querySelectorAll('[data-column-move-up]').forEach(function (btn) { btn.onclick = function () { @@ -4645,6 +4776,8 @@ document.getElementById('editColumnRoutePath').value = (btn.dataset.route || '').replace(/"/g, '"'); document.getElementById('editColumnMenuType').value = ((btn.dataset.menuType || 'app').replace(/"/g, '"') || 'app'); document.getElementById('editColumnSortOrder').value = ((btn.dataset.sortOrder || '0').replace(/"/g, '"') || '0'); + populateColumnParentSelects(); + document.getElementById('editColumnParentId').value = btn.dataset.parentId || ''; document.getElementById('msgEditColumn').textContent = ''; document.getElementById('editColumnModal').classList.add('show'); }; @@ -4662,26 +4795,22 @@ }); } function moveColumnItem(columnId, delta, triggerBtn) { - var currentIndex = -1; - for (var i = 0; i < allColumnsList.length; i++) { - if (String(allColumnsList[i].id) === String(columnId)) { - currentIndex = i; - break; - } - } + var currentItem = allColumnsList.find(function (item) { return String(item.id) === String(columnId); }); + if (!currentItem) return; + var siblings = columnSiblingItems(currentItem); + var currentIndex = siblings.findIndex(function (item) { return String(item.id) === String(columnId); }); if (currentIndex < 0) return; var targetIndex = currentIndex + delta; - if (targetIndex < 0 || targetIndex >= allColumnsList.length) return; - var currentItem = allColumnsList[currentIndex]; - var targetItem = allColumnsList[targetIndex]; - if (!currentItem || !targetItem) return; + if (targetIndex < 0 || targetIndex >= siblings.length) return; + var targetItem = siblings[targetIndex]; if (triggerBtn) triggerBtn.disabled = true; fetch('/api/admin/column/reorder', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ column_id: currentItem.id, - target_id: targetItem.id + target_id: targetItem.id, + menu_type: currentItem.menu_type || 'app' }) }).then(function (r) { return r.json(); }) .then(function (res) { @@ -4704,6 +4833,7 @@ var routePath = (document.getElementById('columnRoutePath').value || '').trim(); var sortOrderValue = (document.getElementById('columnSortOrder').value || '').trim(); var menuType = (document.getElementById('columnMenuType').value || 'admin').trim() || 'admin'; + var parentIdValue = (document.getElementById('columnParentId').value || '').trim(); var sortOrder = sortOrderValue === '' ? null : parseInt(sortOrderValue, 10); var msgEl = document.getElementById('msgColumn'); msgEl.textContent = ''; @@ -4721,7 +4851,7 @@ fetch('/api/admin/column', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder }) + body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder, parent_id: parentIdValue ? Number(parentIdValue) : null }) }) .then(function (r) { return r.json(); }) .then(function (res) { @@ -4733,6 +4863,7 @@ document.getElementById('columnRoutePath').value = ''; document.getElementById('columnSortOrder').value = ''; document.getElementById('columnMenuType').value = 'admin'; + document.getElementById('columnParentId').value = ''; loadColumns(); loadColumnsForPermission(); loadAdminMenus(getActiveAdminTabName()); @@ -4750,6 +4881,7 @@ var routePath = (document.getElementById('editColumnRoutePath').value || '').trim(); var sortOrderValue = (document.getElementById('editColumnSortOrder').value || '').trim(); var menuType = (document.getElementById('editColumnMenuType').value || 'admin').trim() || 'admin'; + var parentIdValue = (document.getElementById('editColumnParentId').value || '').trim(); var sortOrder = sortOrderValue === '' ? null : parseInt(sortOrderValue, 10); var msgEl = document.getElementById('msgEditColumn'); msgEl.textContent = ''; @@ -4767,7 +4899,7 @@ fetch('/api/admin/column/' + cid, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder }) + body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder, parent_id: parentIdValue ? Number(parentIdValue) : null }) }) .then(function (r) { return r.json(); }) .then(function (res) { @@ -4782,6 +4914,8 @@ } }); }; + if (document.getElementById('columnMenuType')) document.getElementById('columnMenuType').onchange = populateColumnParentSelects; + if (document.getElementById('editColumnMenuType')) document.getElementById('editColumnMenuType').onchange = populateColumnParentSelects; // ========== 分页 ========== function renderPagination(elId, total, page, pageSize, onPage) { diff --git a/backend/tests/test_dedupe_total_data_admin.py b/backend/tests/test_dedupe_total_data_admin.py deleted file mode 100644 index 45000cde..00000000 --- a/backend/tests/test_dedupe_total_data_admin.py +++ /dev/null @@ -1,160 +0,0 @@ -import io -import unittest -from unittest.mock import Mock, patch - -from flask import jsonify - -from app import app -from blueprints import admin_api as admin_module - - -class DedupeTotalDataAdminTest(unittest.TestCase): - def setUp(self): - app.config.update(TESTING=True, SECRET_KEY='test') - self.client = app.test_client() - - def _login(self, user_id=23): - with self.client.session_transaction() as session: - session['user_id'] = user_id - - def test_list_forwards_session_operator_and_username_search(self): - self._login() - java_response = { - 'success': True, - 'data': { - 'items': [{ - 'id': 91, - 'dataValue': 'B012345678', - 'uploaderUserId': 23, - 'username': 'member-a', - 'createdAt': '2026-07-20T12:30:00', - }], - 'total': 1, - 'page': 1, - 'pageSize': 15, - }, - } - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_dedupe_total_data_access', - return_value=('normal', {'id': 23, 'username': 'member-a'}, None)), \ - patch.object(admin_module, '_proxy_backend_java', - return_value=(java_response, None, 200)) as proxy: - response = self.client.get( - '/api/admin/dedupe-total-data?keyword=B01&username=member&page=1&page_size=15') - - self.assertEqual(200, response.status_code) - payload = response.get_json() - self.assertTrue(payload['success']) - self.assertEqual('member-a', payload['items'][0]['username']) - self.assertEqual(23, payload['items'][0]['uploader_user_id']) - self.assertEqual({ - 'page': 1, - 'pageSize': 15, - 'keyword': 'B01', - 'username': 'member', - 'operatorId': 23, - }, proxy.call_args.kwargs['params']) - - def test_import_binds_session_operator_to_java_request(self): - self._login(31) - java_response = {'success': True, 'message': '开始导入', 'data': {'importId': 'import-1'}} - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_dedupe_total_data_access', - return_value=('normal', {'id': 31, 'username': 'member-b'}, None)), \ - patch.object(admin_module, '_proxy_backend_java', - return_value=(java_response, None, 200)) as proxy: - response = self.client.post( - '/api/admin/dedupe-total-data/import', - data={'file': (io.BytesIO(b'excel'), 'data.xlsx')}, - content_type='multipart/form-data') - - self.assertEqual(200, response.status_code) - self.assertEqual('import-1', response.get_json()['import_id']) - self.assertEqual({'operatorId': 31}, proxy.call_args.kwargs['data']) - - def test_export_forwards_filters_and_session_operator(self): - self._login(31) - java_response = Mock( - status_code=200, - content=b'xlsx-data', - headers={ - 'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'Content-Disposition': 'attachment; filename="dedupe-total-data.xlsx"', - }, - ) - java_session = Mock() - java_session.get.return_value = java_response - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_dedupe_total_data_access', - return_value=('normal', {'id': 31, 'username': 'member-b'}, None)), \ - patch.object(admin_module, '_get_backend_java_session', return_value=java_session): - response = self.client.get( - '/api/admin/dedupe-total-data/export' - '?username=member&start_date=2026-07-01&end_date=2026-07-20') - - self.assertEqual(200, response.status_code) - self.assertEqual(b'xlsx-data', response.data) - self.assertEqual({ - 'operatorId': 31, - 'username': 'member', - 'startDate': '2026-07-01', - 'endDate': '2026-07-20', - }, java_session.get.call_args.kwargs['params']) - self.assertEqual(60, java_session.get.call_args.kwargs['timeout']) - - def test_delete_ignores_client_operator_and_uses_session_user(self): - self._login(41) - java_response = {'success': True, 'message': '删除成功', 'data': None} - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_dedupe_total_data_access', - return_value=('admin', {'id': 41, 'username': 'leader'}, None)), \ - patch.object(admin_module, '_proxy_backend_java', - return_value=(java_response, None, 200)) as proxy: - response = self.client.delete('/api/admin/dedupe-total-data/91?operatorId=1') - - self.assertEqual(200, response.status_code) - self.assertEqual({'operatorId': 41}, proxy.call_args.kwargs['params']) - - def test_menu_permission_is_still_required_for_normal_user(self): - self._login() - - def deny_access(): - return 'normal', {'id': 23}, (jsonify({'success': False, 'error': '无权访问'}), 403) - - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_dedupe_total_data_access', side_effect=deny_access), \ - patch.object(admin_module, '_proxy_backend_java') as proxy: - response = self.client.get('/api/admin/dedupe-total-data') - - self.assertEqual(403, response.status_code) - self.assertFalse(response.get_json()['success']) - proxy.assert_not_called() - - def test_dedupe_access_uses_login_capable_backend_menu_check(self): - expected = ('normal', {'id': 23}, None) - with patch.object(admin_module, '_ensure_backend_menu_access', return_value=expected) as backend_access, \ - patch.object(admin_module, '_ensure_admin_menu_access') as admin_access: - result = admin_module._ensure_dedupe_total_data_access() - - self.assertEqual(expected, result) - backend_access.assert_called_once_with('dedupe-total-data') - admin_access.assert_not_called() - - def test_group_list_accepts_dedupe_menu_as_access_source(self): - self._login() - access_result = ('normal', {'id': 23, 'username': 'member-a'}, None) - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_backend_menu_access', - return_value=access_result) as backend_access, \ - patch.object(admin_module, '_load_expanded_shop_manage_groups', - return_value=([], None, 200)): - response = self.client.get('/api/admin/shop-manage-groups') - - self.assertEqual(200, response.status_code) - self.assertTrue(response.get_json()['success']) - backend_access.assert_called_once_with( - 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data') - - -if __name__ == '__main__': - unittest.main() diff --git a/backend/tests/test_image_video_admin.py b/backend/tests/test_image_video_admin.py deleted file mode 100644 index 0dc78c46..00000000 --- a/backend/tests/test_image_video_admin.py +++ /dev/null @@ -1,340 +0,0 @@ -import io -import json -import unittest -import zipfile -from datetime import datetime -from unittest.mock import MagicMock, patch - -from app import app -from blueprints import admin_api as admin_module - - -class FakeCursor: - def __init__(self, rows): - self.rows = rows - self.last_sql = '' - self.executions = [] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return False - - def execute(self, sql, params=()): - self.last_sql = sql - self.executions.append((sql, params)) - - def fetchall(self): - return self.rows - - def fetchone(self): - if 'COUNT(*)' in self.last_sql: - return {'total': len(self.rows)} - return self.rows[0] if self.rows else None - - -class FakeConnection: - def __init__(self, rows): - self.cursor_value = FakeCursor(rows) - - def cursor(self): - return self.cursor_value - - def close(self): - pass - - -class FakeDownloadResponse: - def __init__(self, content=b'video-bytes', error=None): - self.content = content - self.error = error - self.closed = False - - def raise_for_status(self): - if self.error: - raise self.error - - def iter_content(self, chunk_size): - yield self.content - - def close(self): - self.closed = True - - -class ImageVideoAdminTest(unittest.TestCase): - def setUp(self): - app.config.update(TESTING=True, SECRET_KEY='test') - self.client = app.test_client() - - def test_masks_nested_secrets_and_resolves_mode(self): - payload = { - 'parameters': { - 'api_key_info': {'ai_conductor_key': 'secret', 't8_video_key': 'secret-2'}, - 'video_info': {'mode': '2'}, - }, - 'token': 'secret-3', - } - masked = admin_module._mask_image_video_secrets(payload) - self.assertEqual('******', masked['parameters']['api_key_info']['ai_conductor_key']) - self.assertEqual('******', masked['parameters']['api_key_info']['t8_video_key']) - self.assertEqual('******', masked['token']) - self.assertEqual('视频复刻', admin_module._image_video_mode(payload)) - - def test_admin_access_clause_is_limited_to_self_and_direct_users(self): - sql, params = admin_module._image_video_access_sql('admin', {'id': 17}) - self.assertIn('created_by_id = %s', sql) - self.assertEqual([17, 17], params) - - def test_list_api_applies_pagination_and_returns_task_summary(self): - row = { - 'id': 91, - 'user_id': 21, - 'username': 'demo', - 'group_name': '测试分组、第二分组', - 'status': 'SUCCESS', - 'request_json': json.dumps({'parameters': {'video_info': {'mode': '1'}}}), - 'submit_response_json': '{}', - 'result_json': '{}', - 'video_urls_json': json.dumps(['https://coze.example/video.mp4']), - 'debug_url': 'https://coze.example/debug', - 'archived_videos_json': json.dumps([{ - 'sourceUrl': 'https://coze.example/video.mp4', - 'url': 'https://oss.example/video.mp4', - 'objectKey': 'result/image_video/1/video.mp4', - 'status': 'SUCCESS', - }]), - 'archive_status': 'SUCCESS', - 'archive_error': None, - 'archive_attempt_count': 1, - 'archived_at': datetime(2026, 7, 15, 12, 5), - 'error_message': None, - 'coze_execute_id': 'exec-91', - 'coze_status': 'SUCCESS', - 'submitted_at': datetime(2026, 7, 15, 12, 0), - 'completed_at': datetime(2026, 7, 15, 12, 4), - } - connection = FakeConnection([row]) - with self.client.session_transaction() as session: - session['user_id'] = 17 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch('utils.auth.get_current_admin_role', return_value=('admin', {'id': 17})), \ - patch.object(admin_module, '_ensure_backend_menu_access', return_value=('admin', {'id': 17}, None)), \ - patch.object(admin_module, '_ensure_image_video_data_access', return_value=('admin', {'id': 17}, None)), \ - patch.object(admin_module, 'get_db', return_value=connection): - response = self.client.get('/api/admin/image-video-tasks?page=2&page_size=20&status=success') - - self.assertEqual(200, response.status_code) - data = response.get_json() - self.assertTrue(data['success']) - self.assertEqual(2, data['page']) - self.assertEqual('测试分组、第二分组', data['items'][0]['group_name']) - self.assertEqual('图生视频', data['items'][0]['mode']) - self.assertEqual('https://oss.example/video.mp4', data['items'][0]['video_url']) - self.assertEqual([{ - 'source_url': 'https://coze.example/video.mp4', - 'archived_url': 'https://oss.example/video.mp4', - 'object_key': 'result/image_video/1/video.mp4', - 'archive_status': 'SUCCESS', - 'archive_error': '', - 'display_url': 'https://oss.example/video.mp4', - }], data['items'][0]['videos']) - self.assertNotIn('request', data['items'][0]) - self.assertNotIn('submit_response', data['items'][0]) - self.assertNotIn('result', data['items'][0]) - list_sql, list_params = connection.cursor_value.executions[0] - self.assertIn("t.task_type = 'IMAGE_VIDEO_WORKFLOW'", list_sql) - self.assertIn('t.submitted_at >= DATE_SUB(NOW(), INTERVAL 3 DAY)', list_sql) - self.assertNotIn('created_by_id = %s', list_sql) - self.assertEqual(('SUCCESS', 20, 20), list_params) - - def test_authorized_normal_user_receives_all_video_tasks(self): - connection = FakeConnection([]) - with self.client.session_transaction() as session: - session['user_id'] = 23 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 23}, None)), \ - patch.object(admin_module, '_ensure_image_video_data_access', return_value=('normal', {'id': 23}, None)), \ - patch.object(admin_module, 'get_db', return_value=connection): - response = self.client.get('/api/admin/image-video-tasks') - - self.assertEqual(200, response.status_code) - list_sql, list_params = connection.cursor_value.executions[0] - self.assertNotIn('t.user_id = %s', list_sql) - self.assertEqual((20, 0), list_params) - - def test_batch_download_returns_single_zip_with_selected_video(self): - row = { - 'id': 91, - 'video_urls_json': json.dumps(['https://coze.example/video.mp4']), - 'archived_videos_json': json.dumps([{'url': 'https://oss.example/video.mp4'}]), - } - connection = FakeConnection([row]) - remote_response = FakeDownloadResponse() - with self.client.session_transaction() as session: - session['user_id'] = 23 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 23}, None)), \ - patch.object(admin_module, '_ensure_image_video_data_access', return_value=('normal', {'id': 23}, None)), \ - patch.object(admin_module, 'get_db', return_value=connection), \ - patch.object(admin_module.requests, 'get', return_value=remote_response): - response = self.client.post('/api/admin/image-video-tasks/download-zip', json={ - 'items': [{'task_id': 91, 'video_index': 0}], - }) - - self.assertEqual(200, response.status_code) - self.assertEqual('application/zip', response.mimetype) - self.assertEqual('1', response.headers['X-Archive-File-Count']) - with zipfile.ZipFile(io.BytesIO(response.get_data())) as archive: - self.assertEqual(['task-91-video-1.mp4'], archive.namelist()) - self.assertEqual(b'video-bytes', archive.read('task-91-video-1.mp4')) - response.close() - self.assertTrue(remote_response.closed) - - def test_batch_download_writes_failures_into_zip(self): - row = { - 'id': 91, - 'video_urls_json': json.dumps(['https://coze.example/video.mp4']), - 'archived_videos_json': '[]', - } - connection = FakeConnection([row]) - remote_response = FakeDownloadResponse(error=admin_module.requests.RequestException('upstream failed')) - with self.client.session_transaction() as session: - session['user_id'] = 23 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 23}, None)), \ - patch.object(admin_module, '_ensure_image_video_data_access', return_value=('normal', {'id': 23}, None)), \ - patch.object(admin_module, 'get_db', return_value=connection), \ - patch.object(admin_module.requests, 'get', return_value=remote_response): - response = self.client.post('/api/admin/image-video-tasks/download-zip', json={ - 'items': [{'task_id': 91, 'video_index': 0}], - }) - - self.assertEqual(200, response.status_code) - self.assertEqual('1', response.headers['X-Archive-Error-Count']) - with zipfile.ZipFile(io.BytesIO(response.get_data())) as archive: - self.assertEqual(['download-errors.txt'], archive.namelist()) - self.assertIn('下载失败', archive.read('download-errors.txt').decode('utf-8')) - response.close() - - def test_unassigned_normal_user_is_denied_video_tasks(self): - denied = (app.response_class('{"success":false}', content_type='application/json'), 403) - with self.client.session_transaction() as session: - session['user_id'] = 24 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 24}, denied)): - response = self.client.get('/api/admin/image-video-tasks') - - self.assertEqual(403, response.status_code) - - def test_only_super_admin_can_manage_image_video_permissions(self): - with self.client.session_transaction() as session: - session['user_id'] = 17 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, 'get_current_admin_role', return_value=('admin', {'id': 17})): - response = self.client.get('/api/admin/image-video-task-permissions') - - self.assertEqual(403, response.status_code) - - def test_super_admin_lists_image_video_permission_users(self): - cursor = MagicMock() - cursor.__enter__.return_value = cursor - cursor.__exit__.return_value = False - cursor.fetchone.return_value = {'id': 75} - cursor.fetchall.return_value = [ - {'id': 2, 'username': 'admin-a', 'role': 'admin', 'is_admin': 1, 'created_by_id': 1, 'granted': 1}, - {'id': 3, 'username': 'user-a', 'role': 'normal', 'is_admin': 0, 'created_by_id': 2, 'granted': 0}, - ] - connection = MagicMock() - connection.cursor.return_value = cursor - with self.client.session_transaction() as session: - session['user_id'] = 1 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, 'get_current_admin_role', return_value=('super_admin', {'id': 1})), \ - patch.object(admin_module, 'get_db', return_value=connection): - response = self.client.get('/api/admin/image-video-task-permissions') - - self.assertEqual(200, response.status_code) - self.assertEqual([True, False], [item['granted'] for item in response.get_json()['items']]) - - def test_super_admin_replaces_only_image_video_permissions(self): - cursor = MagicMock() - cursor.__enter__.return_value = cursor - cursor.__exit__.return_value = False - cursor.fetchone.return_value = {'id': 75} - cursor.fetchall.return_value = [{'id': 2}, {'id': 3}] - connection = MagicMock() - connection.cursor.return_value = cursor - with self.client.session_transaction() as session: - session['user_id'] = 1 - with patch('utils.auth.is_session_user_valid', return_value=True), \ - patch.object(admin_module, 'get_current_admin_role', return_value=('super_admin', {'id': 1})), \ - patch.object(admin_module, 'get_db', return_value=connection): - response = self.client.put('/api/admin/image-video-task-permissions', json={'user_ids': [2, 3]}) - - self.assertEqual(200, response.status_code) - self.assertEqual(2, response.get_json()['granted_count']) - statements = [call.args[0] for call in cursor.execute.call_args_list] - self.assertTrue(any('DELETE ucp FROM user_column_permission' in sql for sql in statements)) - self.assertEqual(2, sum('INSERT INTO user_column_permission' in sql for sql in statements)) - connection.commit.assert_called_once() - - def test_generic_permission_update_preserves_video_permission(self): - cursor = MagicMock() - cursor.fetchall.side_effect = [[{'id': 75}], [{'column_id': 75}]] - - admin_module._set_user_column_permissions( - cursor, - 7, - [2, 75], - preserve_column_keys=(admin_module.IMAGE_VIDEO_PERMISSION_KEY,), - ) - - inserts = [call.args[1] for call in cursor.execute.call_args_list if call.args[0].startswith('INSERT INTO')] - self.assertEqual([(7, 2), (7, 75)], inserts) - - def test_video_items_fall_back_to_source_and_handle_empty_values(self): - row = { - 'video_urls_json': json.dumps([ - 'https://coze.example/first.mp4', - 'https://coze.example/second.mp4', - ]), - 'archived_videos_json': json.dumps([{ - 'sourceUrl': 'https://coze.example/first.mp4', - 'status': 'FAILED', - 'error': 'archive failed', - }]), - } - - videos = admin_module._image_video_urls(row) - - self.assertEqual(2, len(videos)) - self.assertEqual('https://coze.example/first.mp4', videos[0]['display_url']) - self.assertEqual('FAILED', videos[0]['archive_status']) - self.assertEqual('archive failed', videos[0]['archive_error']) - self.assertEqual('https://coze.example/second.mp4', videos[1]['display_url']) - self.assertEqual([], admin_module._image_video_urls({ - 'video_urls_json': None, - 'archived_videos_json': 'not-json', - })) - - def test_detail_item_returns_masked_request_and_responses(self): - row = { - 'id': 8, - 'request_json': json.dumps({'token': 'secret', 'name': 'demo'}), - 'submit_response_json': json.dumps({'execute_id': 'exec-8'}), - 'result_json': json.dumps({'status': 'SUCCESS'}), - 'video_urls_json': '[]', - 'archived_videos_json': '[]', - } - - item = admin_module._image_video_admin_item(row, include_json=True) - - self.assertEqual('******', item['request']['token']) - self.assertEqual('demo', item['request']['name']) - self.assertEqual({'execute_id': 'exec-8'}, item['submit_response']) - self.assertEqual({'status': 'SUCCESS'}, item['result']) - - -if __name__ == '__main__': - unittest.main() diff --git a/backend/utils/db.py b/backend/utils/db.py index ae5a6943..05b9fb61 100644 --- a/backend/utils/db.py +++ b/backend/utils/db.py @@ -129,44 +129,6 @@ def init_db(): created_at DATETIME DEFAULT CURRENT_TIMESTAMP ) """) - cur.execute(""" - CREATE TABLE IF NOT EXISTS columns ( - id INT AUTO_INCREMENT PRIMARY KEY, - name VARCHAR(128) NOT NULL COMMENT '栏目名', - column_key VARCHAR(64) NOT NULL COMMENT '栏目标识', - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - UNIQUE KEY uk_column_key (column_key) - ) - """) - try: - cur.execute("ALTER TABLE columns ADD COLUMN menu_type VARCHAR(20) NOT NULL DEFAULT 'app' COMMENT '菜单类型: app/admin' AFTER column_key") - except Exception: - pass - try: - cur.execute("ALTER TABLE columns ADD COLUMN route_path VARCHAR(255) NOT NULL DEFAULT '' COMMENT '菜单路由或页面标识' AFTER menu_type") - except Exception: - pass - try: - cur.execute("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path") - except Exception: - pass - try: - cur.execute("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''") - except Exception: - pass - try: - cur.execute("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0") - except Exception: - pass - cur.execute(""" - CREATE TABLE IF NOT EXISTS user_column_permission ( - user_id INT NOT NULL, - column_id INT NOT NULL, - PRIMARY KEY (user_id, column_id), - FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, - FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE - ) - """) try: cur.execute("UPDATE users SET role = 'normal' WHERE (role IS NULL OR role = '') AND (is_admin = 0 OR is_admin IS NULL)") cur.execute("SELECT MIN(id) AS mid FROM users WHERE is_admin = 1") diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 77ce0d36..993ad94c 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -713,6 +713,136 @@ overflow: hidden; } + .column-permission-tree { + max-height: 280px; + overflow: auto; + border: 1px solid #ddd; + border-radius: 6px; + padding: 8px 10px; + background: #fff; + } + + .column-tree-node { + display: block; + padding: 4px 0; + } + + .column-tree-group + .column-tree-group { + margin-top: 8px; + } + + .column-tree-group-toggle { + display: flex; + align-items: center; + width: 100%; + min-height: 36px; + padding: 6px 10px; + border: 0; + border-radius: 4px; + background: #f3f5f8; + color: #24292f; + cursor: pointer; + text-align: left; + } + + .column-tree-group-toggle::before { + content: '\203A'; + flex: 0 0 20px; + font-size: 20px; + line-height: 1; + color: #667085; + transition: transform 0.15s ease; + } + + .column-tree-group-toggle[aria-expanded="true"]::before { + transform: rotate(90deg); + } + + .column-tree-group-toggle:hover { + background: #e9edf3; + } + + .column-tree-group-toggle:focus-visible { + outline: 2px solid #3159d8; + outline-offset: -2px; + } + + .column-tree-group-name { + font-size: 13px; + font-weight: 600; + } + + .column-tree-group-count { + margin-left: auto; + font-size: 12px; + color: #667085; + } + + .column-tree-group-body { + padding-top: 4px; + } + + .column-tree-row { + display: flex; + align-items: center; + min-height: 28px; + } + + .column-tree-node label { + display: flex; + align-items: center; + gap: 7px; + margin: 0; + min-height: 28px; + font-size: 13px; + flex: 1; + min-width: 0; + } + + .column-tree-toggle, + .column-tree-toggle-spacer { + flex: 0 0 24px; + width: 24px; + height: 28px; + } + + .column-tree-toggle { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0; + border: 0; + background: transparent; + color: #666; + cursor: pointer; + } + + .column-tree-toggle::before { + content: '\203A'; + display: block; + font-size: 20px; + line-height: 1; + transition: transform 0.15s ease; + } + + .column-tree-toggle[aria-expanded="true"]::before { + transform: rotate(90deg); + } + + .column-tree-toggle:hover { + color: #3159d8; + } + + .column-tree-toggle:focus-visible { + outline: 2px solid #3159d8; + outline-offset: -2px; + } + + .column-tree-node[data-depth="1"] > .column-tree-row { padding-left: 22px; } + .column-tree-node[data-depth="2"] > .column-tree-row { padding-left: 44px; } + .column-tree-node input[type="checkbox"] { width: auto; } + .column-tree-node .inherited-label { color: #999; } + .multi-select-native { position: absolute; opacity: 0; @@ -1416,9 +1546,9 @@
- +
- +

@@ -1484,6 +1614,10 @@ +
+ + +

@@ -1497,6 +1631,7 @@ 菜单名称 栏目标识 菜单类型 + 父菜单 排序 菜单路由 创建时间 @@ -2233,9 +2368,9 @@
- +
- +

@@ -2311,6 +2446,10 @@
+
+ + +

@@ -2607,7 +2746,7 @@
- + diff --git a/frontend-vue/shop-data-crawl.html b/frontend-vue/shop-data-crawl.html new file mode 100644 index 00000000..c6c3def5 --- /dev/null +++ b/frontend-vue/shop-data-crawl.html @@ -0,0 +1,12 @@ + + + + + + 店铺数据抓取 - 数富AI + + +
+ + + diff --git a/frontend-vue/src/pages/brand/components/BrandPublishTab.vue b/frontend-vue/src/pages/brand/components/BrandPublishTab.vue index 9190596c..200b48ec 100644 --- a/frontend-vue/src/pages/brand/components/BrandPublishTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandPublishTab.vue @@ -128,8 +128,7 @@ >
- {{ detail.task.taskNo || `任务 ${detail.task.id}` }} - 任务 ID：{{ detail.task.id }} + 上架任务 #{{ detail.task.id }} 创建时间：{{ formatDateTime(detail.task.createdAt) }}
@@ -144,6 +143,14 @@ > 下载结果 +
@@ -163,17 +170,19 @@ 平台：{{ file.platform }} 数据：{{ fileProgressCurrent(file) }}/{{ fileProgressTotal(file) }} -
- {{ file.progressMessage || statusText(file.status) }} - {{ fileProgressPercent(file) }}% -
-
-
-
+
{{ fileErrorText(file) }}
@@ -191,7 +200,7 @@ + + diff --git a/frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue b/frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue index f98c4208..8104590c 100644 --- a/frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue @@ -792,7 +792,9 @@ function isResultPreparing(item: SimilarAsinHistoryItem) { } function normalizeTaskStatus(item: SimilarAsinHistoryItem) { - return (item.taskStatus || '').toUpperCase() + const taskStatus = (item.taskStatus || '').toUpperCase() + if (taskStatus !== 'FAILED' && isResultFileComplete(item)) return 'SUCCESS' + return taskStatus } function isResultBuildFailed(item: SimilarAsinHistoryItem) { @@ -821,7 +823,16 @@ function canDownload(item: SimilarAsinHistoryItem) { return Boolean(item.resultId && (item.fileReady || item.downloadUrl)) } +function isResultFileComplete(item: SimilarAsinHistoryItem) { + return Boolean( + item.fileReady + || item.downloadUrl + || (item.fileStatus || '').toUpperCase() === 'SUCCESS', + ) +} + function fileProgressPercent(item: SimilarAsinHistoryItem) { + if (isResultFileComplete(item)) return 100 const percent = Number(item.fileProgressPercent || 0) if (Number.isFinite(percent) && percent > 0) { return Math.max(0, Math.min(100, Math.round(percent))) diff --git a/frontend-vue/src/pages/brand/components/BrandTopBar.vue b/frontend-vue/src/pages/brand/components/BrandTopBar.vue index 536b205f..87b61e71 100644 --- a/frontend-vue/src/pages/brand/components/BrandTopBar.vue +++ b/frontend-vue/src/pages/brand/components/BrandTopBar.vue @@ -52,6 +52,7 @@ type ActiveNavKey = | 'pricing' | 'patrol-delete' | 'query-asin' + | 'shop-data-crawl' | 'withdraw' | 'collect-data' | 'image-video' @@ -85,7 +86,7 @@ const active = props.active const showNav = props.showNav const showHomeLink = props.showHomeLink const showActions = props.showActions -const allowedColumnKeys = ref(null) +const allowedColumnKeys = ref([]) const navGroups: ReadonlyArray = [ { @@ -114,6 +115,7 @@ const navGroups: ReadonlyArray = [ { key: 'pricing', label: '跟价', href: '/new_web_source/price-track.html', aliases: ['price-track'] }, { key: 'patrol-delete', label: '巡店删除', href: '/new_web_source/patrol-delete.html' }, { key: 'query-asin', label: '查询ASIN', href: '/new_web_source/query-asin.html' }, + { key: 'shop-data-crawl', label: '店铺数据抓取', href: '/new_web_source/shop-data-crawl.html', columnKey: 'shop_data_crawl', aliases: ['shop-data-crawl'] }, { key: 'withdraw', label: '取款', href: '/new_web_source/withdraw.html' }, { key: 'shop-status', label: '店铺状态查询' }, ], @@ -135,9 +137,6 @@ function getItemPermissionKeys(item: NavItem) { } const visibleNavGroups = computed(() => { - if (allowedColumnKeys.value === null) { - return navGroups - } const allowedSet = new Set(allowedColumnKeys.value) const groups = navGroups.flatMap((group) => { @@ -150,14 +149,14 @@ const visibleNavGroups = computed(() => { return items.length ? [{ ...group, items }] : [] }) - return groups.length ? groups : navGroups + return groups }) onMounted(async () => { try { allowedColumnKeys.value = await getCurrentUserAppColumnKeys() } catch (_error) { - allowedColumnKeys.value = null + allowedColumnKeys.value = [] } }) diff --git a/frontend-vue/src/pages/image-video/ImageVideoPage.vue b/frontend-vue/src/pages/image-video/ImageVideoPage.vue index 45ae0685..4209a5ac 100644 --- a/frontend-vue/src/pages/image-video/ImageVideoPage.vue +++ b/frontend-vue/src/pages/image-video/ImageVideoPage.vue @@ -5,8 +5,9 @@ 返回首页 -
+
- - + + 图片 +
@@ -64,17 +68,20 @@