菜单修改优化

This commit is contained in:
supernijia
2026-07-28 13:40:19 +08:00
parent 4ddb8b47b0
commit 7c3c9e53a0
90 changed files with 9388 additions and 1545 deletions
+5 -2
View File
@@ -8,6 +8,7 @@ Requirements:
- Put JVM options before `-jar`
- Pass `AIIMAGE_INSTANCE_ID` and Redis address from the startup command
- Result file job MQ is expected to stay enabled in packaged `server` deployments
- Use `https://oss.aishufu.top` for the MinIO SDK and public object URLs; `https://minio.aishufu.top` is the management console only
1Panel server 121:
@@ -22,6 +23,8 @@ AIIMAGE_REDIS_PASSWORD=B6COTcY094TYe545
AIIMAGE_REDIS_DATABASE=0
AIIMAGE_SERVER_PORT=18080
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=/app/data/tmp
AIIMAGE_OSS_ENDPOINT=https://oss.aishufu.top
AIIMAGE_OSS_PUBLIC_ENDPOINT=https://oss.aishufu.top
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://121.196.149.225:9000
@@ -36,13 +39,13 @@ java -Xmx2048M -Xms2048M -jar /app/aiimage-backend-0.0.1-SNAPSHOT.jar
Baota server 111:
```bash
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-111 --spring.data.redis.host=47.111.163.154 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-111 --spring.data.redis.host=47.111.163.154 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp --aiimage.oss.endpoint=https://oss.aishufu.top --aiimage.oss.public-endpoint=https://oss.aishufu.top
```
Optional Baota server 121 command:
```bash
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-121 --spring.data.redis.host=192.168.0.172 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-121 --spring.data.redis.host=192.168.0.172 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp --aiimage.oss.endpoint=https://oss.aishufu.top --aiimage.oss.public-endpoint=https://oss.aishufu.top
```
Successful startup should show a line similar to:
@@ -12,5 +12,5 @@ public class ModuleCleanupProperties {
private boolean enabled = true;
private String cron = "0 0 0 * * *";
private long retentionDays = 7;
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
}
@@ -48,11 +48,11 @@ public class AdminUserController {
@PostMapping("/user")
@Operation(summary = "创建用户")
public ApiResponse<Void> createUser(HttpServletRequest request,
public ApiResponse<Long> createUser(HttpServletRequest request,
@RequestBody AdminUserCreateRequest body) {
AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request);
adminUserService.createUser(currentUser, body);
return ApiResponse.success("用户创建成功", null);
Long userId = adminUserService.createUser(currentUser, body);
return ApiResponse.success("用户创建成功", userId);
}
@PutMapping("/user/{uid}")
@@ -2,10 +2,13 @@ package com.nanri.aiimage.modules.admin.model.dto;
import lombok.Data;
import java.util.List;
@Data
public class AdminUserCreateRequest {
private String username;
private String password;
private String role;
private Long createdById;
private List<Long> columnIds;
}
@@ -2,8 +2,11 @@ package com.nanri.aiimage.modules.admin.model.dto;
import lombok.Data;
import java.util.List;
@Data
public class AdminUserUpdateRequest {
private String password;
private String role;
private List<Long> columnIds;
}
@@ -11,10 +11,13 @@ import com.nanri.aiimage.modules.admin.model.vo.AdminUserListVo;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.auth.util.WerkzeugPasswordEncoder;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@@ -35,6 +38,7 @@ public class AdminUserService {
private final AdminUserMapper adminUserMapper;
private final WerkzeugPasswordEncoder passwordEncoder;
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
public AdminUserListVo listUsers(AdminUserEntity currentUser, Integer page, Integer pageSize,
String username, Long createdById) {
@@ -97,7 +101,8 @@ public class AdminUserService {
return vo;
}
public void createUser(AdminUserEntity currentUser, AdminUserCreateRequest request) {
@Transactional
public Long createUser(AdminUserEntity currentUser, AdminUserCreateRequest request) {
String role = adminAuthSupport.currentRole(currentUser);
if (role == null) {
throw new BusinessException(403, "需要管理员权限");
@@ -150,8 +155,11 @@ public class AdminUserService {
} catch (DuplicateKeyException e) {
throw new BusinessException("用户名已存在");
}
replaceDirectPermissions(currentUser, entity.getId(), request.getColumnIds());
return entity.getId();
}
@Transactional
public void updateUser(AdminUserEntity currentUser, Long uid, AdminUserUpdateRequest request) {
String role = adminAuthSupport.currentRole(currentUser);
if (role == null) {
@@ -160,7 +168,9 @@ public class AdminUserService {
String password = request.getPassword();
String wantRole = request.getRole() == null ? null : request.getRole().trim();
if ((password == null || password.isEmpty()) && (wantRole == null || wantRole.isEmpty())) {
if ((password == null || password.isEmpty())
&& (wantRole == null || wantRole.isEmpty())
&& request.getColumnIds() == null) {
throw new BusinessException("请提供要修改的内容");
}
@@ -206,6 +216,16 @@ public class AdminUserService {
if (dirty) {
adminUserMapper.update(null, update);
}
if (request.getColumnIds() != null) {
replaceDirectPermissions(currentUser, uid, request.getColumnIds());
}
}
private void replaceDirectPermissions(AdminUserEntity operator, Long userId, List<Long> columnIds) {
UserColumnPermissionUpdateRequest permissionRequest = new UserColumnPermissionUpdateRequest();
permissionRequest.setColumnIds(columnIds == null ? List.of() : columnIds);
permissionMenuService.updateUserColumnPermissions(
operator, userId, permissionRequest, PermissionMenuService.MENU_TYPE_APP);
}
public void deleteUser(AdminUserEntity currentUser, Long uid) {
@@ -1,6 +1,5 @@
package com.nanri.aiimage.modules.admin.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.auth.service.JwtService;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
@@ -65,27 +64,15 @@ public class AdminAuthSupport {
return "super_admin";
}
if ("admin".equals(storedRole)) {
return resolveAdminRole(user);
return "admin";
}
boolean isAdminFlag = user.getIsAdmin() != null && user.getIsAdmin() == 1;
if (isAdminFlag) {
return resolveAdminRole(user);
if (storedRole.isEmpty() && isAdminFlag) {
return user.getCreatedById() == null ? "super_admin" : "admin";
}
return null;
}
/** 数据库里 role='admin' 中 id 最小者视作超级管理员(与 Python 行为一致)。 */
private String resolveAdminRole(AdminUserEntity user) {
AdminUserEntity superAdmin = adminUserMapper.selectOne(new LambdaQueryWrapper<AdminUserEntity>()
.eq(AdminUserEntity::getRole, "admin")
.orderByAsc(AdminUserEntity::getId)
.last("LIMIT 1"));
if (superAdmin != null && superAdmin.getId().equals(user.getId())) {
return "super_admin";
}
return "admin";
}
private String resolveToken(HttpServletRequest request) {
String authHeader = request.getHeader(HttpHeaders.AUTHORIZATION);
if (authHeader != null && authHeader.startsWith("Bearer ")) {
@@ -14,7 +14,7 @@ public class AppearancePatentHistoryItemVo {
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/appearance_patent/xxx/17-result.xlsx")
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "https://oss.aishufu.top/nanri-ai-images/result/appearance_patent/xxx/17-result.xlsx")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
@@ -9,7 +9,7 @@ import lombok.Data;
public class BrandSourceFileDto {
@NotBlank(message = "fileUrl 不能为空")
@Schema(description = "对象存储文件下载链接,Java 将先下载文件再读取内容。", example = "http://47.110.241.161:9000/nanri-ai-images/source/brand.xlsx")
@Schema(description = "对象存储文件下载链接,Java 将先下载文件再读取内容。", example = "https://oss.aishufu.top/nanri-ai-images/source/brand.xlsx")
private String fileUrl;
@Schema(description = "原始文件名,用于生成任务描述和结果文件名。", example = "品牌样例.xlsx")
@@ -27,6 +27,7 @@ public class OssStorageService {
private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO";
private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/";
private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000";
private final OssProperties ossProperties;
private final MinioClient minioClient;
@@ -164,6 +165,52 @@ public class OssStorageService {
return location == null ? null : getPublicUrl(location.objectKey(), location.bucket());
}
/** Rewrites managed MinIO URLs to the current public endpoint without touching external URLs. */
public String normalizeManagedPublicUrl(String value) {
if (value == null || value.isBlank()) {
return value;
}
URI source;
try {
source = new URI(value.trim());
} catch (Exception ignored) {
return value;
}
if (!isHttpUrl(source) || source.getHost() == null || source.getUserInfo() != null) {
return value;
}
String objectPath;
try {
objectPath = trimLeadingSlash(decodePath(source.getRawPath()));
} catch (Exception ignored) {
return value;
}
if (objectPath.isBlank() || hasDotPathSegment(objectPath)) {
return value;
}
List<URI> endpoints = managedEndpoints();
for (URI endpoint : endpoints) {
for (String bucket : configuredBuckets()) {
if (matchesVirtualHostEndpoint(source, endpoint, bucket)) {
return getPublicUrl(objectPath, bucket);
}
}
}
if (endpoints.stream().noneMatch(endpoint -> matchesEndpoint(source, endpoint))) {
return value;
}
String bucket = bucketPrefix(objectPath);
if (bucket == null) {
return value;
}
String objectKey = trimLeadingSlash(objectPath.substring(bucket.length()));
return objectKey.isBlank() ? value : getPublicUrl(objectKey, bucket);
}
public String resolveObjectKey(String value) {
StorageLocation location = resolveStorageLocation(value);
return location == null ? value : location.objectKey();
@@ -271,6 +318,58 @@ public class OssStorageService {
.toList();
}
private List<URI> managedEndpoints() {
return Stream.of(ossProperties.getEndpoint(), ossProperties.getPublicEndpoint(), LEGACY_MINIO_ENDPOINT)
.filter(Objects::nonNull)
.map(String::trim)
.filter(endpoint -> !endpoint.isBlank())
.map(this::parseEndpoint)
.filter(Objects::nonNull)
.distinct()
.toList();
}
private URI parseEndpoint(String value) {
try {
URI endpoint = new URI(withScheme(value));
return endpoint.getHost() == null ? null : endpoint;
} catch (Exception ignored) {
return null;
}
}
private boolean matchesEndpoint(URI source, URI endpoint) {
return sameSchemeAndPort(source, endpoint)
&& source.getHost().equalsIgnoreCase(endpoint.getHost());
}
private boolean matchesVirtualHostEndpoint(URI source, URI endpoint, String bucket) {
String expectedHost = bucket + "." + endpoint.getHost();
return sameSchemeAndPort(source, endpoint)
&& source.getHost().equalsIgnoreCase(expectedHost);
}
private boolean sameSchemeAndPort(URI left, URI right) {
return left.getScheme().equalsIgnoreCase(right.getScheme())
&& effectivePort(left) == effectivePort(right);
}
private int effectivePort(URI uri) {
if (uri.getPort() >= 0) {
return uri.getPort();
}
return "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80;
}
private boolean isHttpUrl(URI uri) {
return "http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme());
}
private boolean hasDotPathSegment(String path) {
return Stream.of(path.split("/", -1))
.anyMatch(segment -> ".".equals(segment) || "..".equals(segment));
}
private MinioClient buildClient() {
return minioClient;
}
@@ -49,6 +49,15 @@ public class ImageVideoAsyncTaskService {
private static final Set<String> COZE_PRIMARY_STATUS_FIELDS = Set.of(
"execute_status", "executeStatus", "workflow_status", "workflowStatus"
);
private static final Set<String> COZE_BUSINESS_ERROR_FIELDS = Set.of(
"error", "error_message", "errorMessage"
);
private static final Set<String> COZE_BUSINESS_STATUS_CODE_FIELDS = Set.of(
"status_code", "statusCode"
);
private static final Set<String> COZE_BUSINESS_STATUS_MESSAGE_FIELDS = Set.of(
"status_msg", "statusMessage"
);
private final ImageVideoAsyncTaskMapper taskMapper;
private final ImageVideoCozeService cozeService;
@@ -233,7 +242,10 @@ public class ImageVideoAsyncTaskService {
if (FAILED_STATUSES.contains(cozeStatus)) {
task.setCozeStatus(cozeStatus);
task.setResultJson(writeJson(result));
failTask(task, new BusinessException("Coze workflow finished with status " + cozeStatus));
String businessError = resolveCozeBusinessError(result);
failTask(task, new BusinessException(businessError.isBlank()
? "Coze workflow finished with status " + cozeStatus
: businessError));
} else {
completeTask(task, transformResult(type, result), cozeStatus);
}
@@ -315,6 +327,19 @@ public class ImageVideoAsyncTaskService {
}
private void completeTask(ImageVideoAsyncTaskEntity task, Object result, String cozeStatus) {
String businessError = resolveCozeBusinessError(result);
if (!businessError.isBlank()) {
task.setCozeStatus(normalizeStatus(cozeStatus));
task.setResultJson(writeJson(result));
failTask(task, new BusinessException(businessError));
return;
}
if (requiresVideoResult(task) && !hasFinalVideoUrl(result)) {
task.setCozeStatus(normalizeStatus(cozeStatus));
task.setResultJson(writeJson(result));
failTask(task, new BusinessException("Coze workflow succeeded but video_url is empty"));
return;
}
task.setStatus(TaskStatus.SUCCESS.name());
task.setResultJson(writeJson(result));
archiveService.enrichCompletedTask(task, result);
@@ -511,6 +536,60 @@ public class ImageVideoAsyncTaskService {
return value;
}
private String resolveCozeBusinessError(Object result) {
String error = findText(result, COZE_BUSINESS_ERROR_FIELDS);
if (!error.isBlank()) {
return error;
}
String statusCode = findText(result, COZE_BUSINESS_STATUS_CODE_FIELDS);
if (statusCode.isBlank() || "0".equals(statusCode)) {
return "";
}
String statusMessage = findText(result, COZE_BUSINESS_STATUS_MESSAGE_FIELDS);
return statusMessage.isBlank()
? "Coze workflow returned business status " + statusCode
: statusMessage + " (status_code: " + statusCode + ")";
}
private boolean hasFinalVideoUrl(Object result) {
Object data = parsePossiblyJson(childValue(result, "data"));
Object scope = data == null ? parsePossiblyJson(result) : data;
if (scope == null) {
return false;
}
Collection<?> records = scope instanceof Collection<?> collection ? collection : List.of(scope);
boolean hasRunOutput = false;
for (Object record : records) {
Object output = childValue(parsePossiblyJson(record), "output");
if (output == null) {
continue;
}
hasRunOutput = true;
if (hasVideoUrlInOutput(output)) {
return true;
}
}
return !hasRunOutput && hasVideoUrlInOutput(scope);
}
private boolean hasVideoUrlInOutput(Object output) {
Object envelope = parsePossiblyJson(output);
Object finalOutput = childValue(envelope, "Output");
if (finalOutput != null) {
return hasRecognizedVideoUrl(parsePossiblyJson(finalOutput));
}
if (childValue(envelope, "node_status") != null) {
return false;
}
return hasRecognizedVideoUrl(envelope);
}
private boolean hasRecognizedVideoUrl(Object payload) {
return !ImageVideoArchiveService.findUrls(payload, Set.of("videourl")).isEmpty();
}
private String normalizeStatus(String value) {
return value == null ? "" : value.trim().toUpperCase();
}
@@ -529,6 +608,12 @@ public class ImageVideoAsyncTaskService {
&& task.getSubmittedAt().plusHours(1).isBefore(LocalDateTime.now());
}
private boolean requiresVideoResult(ImageVideoAsyncTaskEntity task) {
String taskType = task == null ? "" : task.getTaskType();
return TaskType.IMAGE_VIDEO_WORKFLOW.name().equals(taskType)
|| TaskType.WORKFLOW_RESULT.name().equals(taskType);
}
private void ensureTaskOwnedByCurrentInstance(ImageVideoAsyncTaskEntity task, String operation) {
String owner = task == null ? null : task.getOwnerInstanceId();
if (owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId())) {
@@ -1,17 +1,23 @@
package com.nanri.aiimage.modules.permission.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.ImageVideoDataPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -22,6 +28,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
@RestController
@@ -30,54 +39,188 @@ import java.util.List;
@Tag(name = "菜单权限管理", description = "统一维护软件端与后台管理端的菜单权限配置")
public class PermissionMenuController {
@Value("${aiimage.security.internal-token:}")
private String internalToken;
@Value("${aiimage.security.internal-token-file:}")
private String internalTokenFile;
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
@GetMapping("/permission-menus")
@Operation(summary = "查询菜单权限列表")
public ApiResponse<List<PermissionMenuItemVo>> listMenus(
public ApiResponse<List<PermissionMenuItemVo>> listMenus(HttpServletRequest request,
@Parameter(description = "菜单类型: app/admin") @RequestParam(required = false) String menuType) {
requireAdmin(request);
return ApiResponse.success(permissionMenuService.list(menuType));
}
@PostMapping("/permission-menus")
@Operation(summary = "新增菜单权限项")
public ApiResponse<PermissionMenuItemVo> createMenu(@Valid @RequestBody PermissionMenuCreateRequest request) {
return ApiResponse.success("创建成功", permissionMenuService.create(request));
public ApiResponse<PermissionMenuItemVo> createMenu(HttpServletRequest httpRequest,
@Valid @RequestBody PermissionMenuCreateRequest request) {
return ApiResponse.success("创建成功", permissionMenuService.create(
requireAdmin(httpRequest), request));
}
@PutMapping("/permission-menus/{id}")
@Operation(summary = "编辑菜单权限项")
public ApiResponse<PermissionMenuItemVo> updateMenu(@PathVariable Long id,
public ApiResponse<PermissionMenuItemVo> updateMenu(HttpServletRequest httpRequest,
@PathVariable Long id,
@Valid @RequestBody PermissionMenuUpdateRequest request) {
return ApiResponse.success("更新成功", permissionMenuService.update(id, request));
return ApiResponse.success("更新成功", permissionMenuService.update(
requireAdmin(httpRequest), id, request));
}
@DeleteMapping("/permission-menus/{id}")
@Operation(summary = "删除菜单权限项")
public ApiResponse<Void> deleteMenu(@PathVariable Long id) {
permissionMenuService.delete(id);
public ApiResponse<Void> deleteMenu(HttpServletRequest httpRequest, @PathVariable Long id) {
permissionMenuService.delete(requireAdmin(httpRequest), id);
return ApiResponse.success("删除成功", null);
}
@GetMapping("/permission-users/{userId}/columns")
@Operation(summary = "查询用户菜单权限 ID 列表")
public ApiResponse<UserColumnIdsVo> getUserColumnIds(@PathVariable Long userId,
@Operation(summary = "查询用户直接授权菜单 ID 列表",
description = "columnIds 仅表示数据库保存的直接授权,所有角色都不包含递归或虚拟权限")
public ApiResponse<UserColumnIdsVo> getUserColumnIds(HttpServletRequest httpRequest,
@PathVariable Long userId,
@RequestParam(required = false) String menuType) {
return ApiResponse.success(permissionMenuService.getUserColumnIds(userId, menuType));
return ApiResponse.success(permissionMenuService.getUserColumnIds(
requireAdmin(httpRequest), userId, menuType));
}
@PutMapping("/permission-users/{userId}/columns")
@Operation(summary = "更新用户菜单权限")
public ApiResponse<Void> updateUserColumnIds(@PathVariable Long userId,
public ApiResponse<Void> updateUserColumnIds(HttpServletRequest httpRequest,
@PathVariable Long userId,
@RequestParam(required = false) String menuType,
@RequestBody(required = false) UserColumnPermissionUpdateRequest request) {
permissionMenuService.updateUserColumnPermissions(userId, request);
permissionMenuService.updateUserColumnPermissions(
requireAdmin(httpRequest), userId, request, menuType);
return ApiResponse.success("保存成功", null);
}
@GetMapping("/permission-users/{userId}/column-permissions")
@Operation(summary = "查询用户菜单权限详情")
public ApiResponse<List<PermissionMenuItemVo>> getUserColumnPermissions(@PathVariable Long userId,
@Operation(summary = "查询用户有效菜单权限详情",
description = "返回直接授权及其递归展开的全部子菜单;不会改变数据库中的直接授权")
public ApiResponse<List<PermissionMenuItemVo>> getUserColumnPermissions(HttpServletRequest httpRequest,
@PathVariable Long userId,
@RequestParam(required = false) String menuType) {
return ApiResponse.success(permissionMenuService.getUserColumnPermissions(userId, menuType));
return ApiResponse.success(permissionMenuService.getUserColumnPermissions(
requireUser(httpRequest), userId, menuType));
}
@GetMapping("/image-video-task-permissions")
@Operation(summary = "查询视频任务数据权限用户")
public ApiResponse<List<ImageVideoDataPermissionUserVo>> listImageVideoDataPermissionUsers(
HttpServletRequest request) {
return ApiResponse.success(permissionMenuService.listImageVideoDataPermissionUsers(requireAdmin(request)));
}
@PutMapping("/image-video-task-permissions")
@Operation(summary = "更新视频任务数据权限用户")
public ApiResponse<Integer> updateImageVideoDataPermissionUsers(
HttpServletRequest request,
@RequestBody(required = false) ImageVideoDataPermissionUpdateRequest body) {
int grantedCount = permissionMenuService.updateImageVideoDataPermissionUsers(
requireAdmin(request), body == null ? List.of() : body.getUserIds());
return ApiResponse.success("视频任务权限已更新", grantedCount);
}
private AdminUserEntity requireAdmin(HttpServletRequest request) {
try {
return adminAuthSupport.requireAdmin(request);
} catch (com.nanri.aiimage.common.exception.BusinessException authFailure) {
AdminUserEntity internalOperator = resolveInternalOperator(request, true);
if (internalOperator != null) {
return internalOperator;
}
throw authFailure;
}
}
private AdminUserEntity requireUser(HttpServletRequest request) {
try {
return adminAuthSupport.requireUser(request);
} catch (com.nanri.aiimage.common.exception.BusinessException authFailure) {
AdminUserEntity internalOperator = resolveInternalOperator(request, false);
if (internalOperator != null) {
return internalOperator;
}
throw authFailure;
}
}
/**
* Flask's legacy session cannot mint a Java JWT. It may identify its
* already-authenticated user only with a shared secret supplied explicitly
* or through the per-user local token file. Admin routes still revalidate
* the database role, and the superAdmin query flag is deliberately ignored.
*/
private AdminUserEntity resolveInternalOperator(HttpServletRequest request, boolean requireAdmin) {
String suppliedToken = request.getHeader("X-Internal-Token");
if (!isTrustedInternalRequest(request, suppliedToken)) {
return null;
}
String rawOperatorId = request.getParameter("operatorId");
if (rawOperatorId == null || rawOperatorId.isBlank()) {
rawOperatorId = request.getParameter("operator_id");
}
if (rawOperatorId == null || rawOperatorId.isBlank()) {
return null;
}
try {
Long operatorId = Long.parseLong(rawOperatorId.trim());
return requireAdmin
? permissionMenuService.requireAdminOperator(operatorId)
: permissionMenuService.requireUserOperator(operatorId);
} catch (NumberFormatException ex) {
return null;
}
}
private boolean isTrustedInternalRequest(HttpServletRequest request, String suppliedToken) {
String expectedToken = resolveExpectedInternalToken();
if (expectedToken.isBlank() || suppliedToken == null || suppliedToken.isBlank()) {
return false;
}
return expectedToken.equals(suppliedToken);
}
private String resolveExpectedInternalToken() {
if (internalToken != null && !internalToken.isBlank()) {
return internalToken.trim();
}
Path path = resolveInternalTokenFile();
if (path == null || !Files.isRegularFile(path)) {
return "";
}
try {
return Files.readString(path, StandardCharsets.UTF_8).trim();
} catch (Exception ignored) {
return "";
}
}
private Path resolveInternalTokenFile() {
String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim();
if (!configuredPath.isEmpty()) {
if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) {
String userHome = System.getProperty("user.home", "").trim();
if (userHome.isEmpty()) {
return null;
}
configuredPath = configuredPath.length() == 1
? userHome
: Path.of(userHome, configuredPath.substring(2)).toString();
}
Path configuredTokenPath = Path.of(configuredPath);
return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null;
}
String userHome = System.getProperty("user.home", "").trim();
return userHome.isEmpty()
? null
: Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize();
}
}
@@ -0,0 +1,12 @@
package com.nanri.aiimage.modules.permission.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import lombok.Data;
import java.util.List;
@Data
public class ImageVideoDataPermissionUpdateRequest {
@JsonAlias("user_ids")
private List<Long> userIds;
}
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.permission.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@@ -12,6 +13,10 @@ public class PermissionMenuCreateRequest {
@NotBlank(message = "菜单标识不能为空")
private String columnKey;
/** 直接父菜单;null 表示根菜单。 */
@JsonAlias("parent_id")
private Long parentId;
@NotBlank(message = "菜单类型不能为空")
private String menuType;
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.permission.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@@ -12,6 +13,10 @@ public class PermissionMenuUpdateRequest {
@NotBlank(message = "菜单标识不能为空")
private String columnKey;
/** 直接父菜单;null 表示根菜单。 */
@JsonAlias("parent_id")
private Long parentId;
@NotBlank(message = "菜单类型不能为空")
private String menuType;
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.permission.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@@ -15,6 +16,8 @@ public class PermissionMenuEntity {
private Long id;
private String name;
private String columnKey;
@TableField("parent_id")
private Long parentId;
private String menuType;
private String routePath;
private Integer sortOrder;
@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.permission.model.vo;
import lombok.Data;
@Data
public class ImageVideoDataPermissionUserVo {
private Long id;
private String username;
private String role;
private boolean granted;
}
@@ -12,6 +12,10 @@ public class PermissionMenuItemVo {
private String name;
@JsonProperty("column_key")
private String columnKey;
@JsonProperty("parent_id")
private Long parentId;
@JsonProperty("root_column_key")
private String rootColumnKey;
@JsonProperty("menu_type")
private String menuType;
@JsonProperty("route_path")
@@ -7,5 +7,9 @@ import java.util.List;
@Data
public class UserColumnIdsVo {
/**
* 仅返回数据库中保存的直接授权 ID;不包含递归子菜单,
* 也不会将超级管理员的虚拟全量权限写入响应。
*/
private List<Long> columnIds;
}
@@ -22,8 +22,29 @@ public class PermissionMenuSchemaInitializer {
private static final List<DefaultAppMenu> DEFAULT_APP_MENUS = List.of(
new DefaultAppMenu("前端工具", "brand_front_tools", "brand-front-tools", 110),
new DefaultAppMenu("运营工具", "brand_operation_tools", "brand-operation-tools", 120),
new DefaultAppMenu("后勤工具", "brand_logistics_tools", "brand-logistics-tools", 130),
new DefaultAppMenu("取款", "withdraw", "withdraw", 140)
new DefaultAppMenu("后勤工具", "brand_logistics_tools", "brand-logistics-tools", 130)
);
private static final List<DefaultAppChildMenu> DEFAULT_APP_CHILD_MENUS = List.of(
new DefaultAppChildMenu("采集数据", "collect-data", "collect-data", 111, "brand_front_tools"),
new DefaultAppChildMenu("品牌检测", "brand", "brand", 112, "brand_front_tools"),
new DefaultAppChildMenu("外观专利检测", "appearance-patent", "appearance-patent", 113, "brand_front_tools"),
new DefaultAppChildMenu("货源查询", "similar-asin", "similar-asin", 114, "brand_front_tools"),
new DefaultAppChildMenu("数据去重", "dedupe", "dedupe", 115, "brand_front_tools"),
new DefaultAppChildMenu("数据拆分", "split", "split", 116, "brand_front_tools"),
new DefaultAppChildMenu("格式转换", "convert", "convert", 117, "brand_front_tools"),
new DefaultAppChildMenu("上架", "publish", "publish", 121, "brand_operation_tools"),
new DefaultAppChildMenu("删除ASIN", "delete-brand", "delete-brand", 122, "brand_operation_tools"),
new DefaultAppChildMenu("商品风险解决", "product-risk", "product-risk", 123, "brand_operation_tools"),
new DefaultAppChildMenu("定时匹配", "shop-match", "shop-match", 124, "brand_operation_tools"),
new DefaultAppChildMenu("跟价", "pricing", "price-track", 125, "brand_operation_tools"),
new DefaultAppChildMenu("巡店删除", "patrol-delete", "patrol-delete", 126, "brand_operation_tools"),
new DefaultAppChildMenu("查询ASIN", "query-asin", "query-asin", 127, "brand_operation_tools"),
// Keep this key for compatibility with the existing Java/Flask module.
new DefaultAppChildMenu("店铺数据抓取", "shop_data_crawl", "shop-data-crawl", 128, "brand_operation_tools"),
new DefaultAppChildMenu("取款", "withdraw", "withdraw", 129, "brand_operation_tools"),
new DefaultAppChildMenu("采购", "purchase", "purchase", 131, "brand_logistics_tools"),
new DefaultAppChildMenu("ERP", "erp", "erp", 132, "brand_logistics_tools")
);
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
@@ -52,11 +73,13 @@ public class PermissionMenuSchemaInitializer {
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(128) NOT NULL COMMENT '菜单标题',
column_key VARCHAR(64) NOT NULL COMMENT '菜单唯一标识',
parent_id INT NULL COMMENT '直接父菜单 ID',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uk_column_key (column_key)
)
""");
executeQuietly("ALTER TABLE columns ADD COLUMN menu_type VARCHAR(20) NOT NULL DEFAULT 'app' COMMENT '菜单类型: app/admin' AFTER column_key");
executeQuietly("ALTER TABLE columns ADD COLUMN parent_id INT NULL COMMENT '直接父菜单 ID' AFTER id");
executeQuietly("ALTER TABLE columns ADD COLUMN route_path VARCHAR(255) NOT NULL DEFAULT '' COMMENT '菜单路由或页面标识' AFTER menu_type");
executeQuietly("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path");
executeQuietly("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''");
@@ -64,6 +87,8 @@ public class PermissionMenuSchemaInitializer {
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
ensureDefaultAdminMenus();
ensureDefaultAppMenus();
ensureDefaultAppChildMenus();
ensureInternalDataPermissions();
executeQuietly("""
CREATE TABLE IF NOT EXISTS user_column_permission (
user_id INT NOT NULL,
@@ -86,7 +111,7 @@ public class PermissionMenuSchemaInitializer {
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
executeQuietly("""
UPDATE columns
SET sort_order = %d
SET sort_order = %d, parent_id = NULL
WHERE column_key = '%s' AND (sort_order IS NULL OR sort_order = 0)
""".formatted(menu.sortOrder(), menu.columnKey()));
}
@@ -103,12 +128,50 @@ public class PermissionMenuSchemaInitializer {
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
executeQuietly("""
UPDATE columns
SET sort_order = %d, route_path = '%s', menu_type = 'app'
SET sort_order = %d, route_path = '%s', menu_type = 'app', parent_id = NULL
WHERE column_key = '%s'
""".formatted(menu.sortOrder(), menu.routePath(), menu.columnKey()));
}
}
private void ensureDefaultAppChildMenus() {
// Older migrations used underscore keys for a few app pages. Normalize
// them once so the frontend item key and permission key stay aligned.
executeQuietly("UPDATE columns SET column_key = 'collect-data' WHERE column_key = 'collect_data'");
executeQuietly("UPDATE columns SET column_key = 'appearance-patent' WHERE column_key = 'appearance_patent'");
executeQuietly("UPDATE columns SET column_key = 'similar-asin' WHERE column_key = 'similar_asin'");
for (DefaultAppChildMenu menu : DEFAULT_APP_CHILD_MENUS) {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order, parent_id)
SELECT '%s', '%s', 'app', '%s', %d, parent.id
FROM columns parent
WHERE parent.column_key = '%s'
AND NOT EXISTS (
SELECT 1 FROM columns existing WHERE existing.column_key = '%s'
)
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(),
menu.parentKey(), menu.columnKey()));
executeQuietly("""
UPDATE columns child
JOIN columns parent ON parent.column_key = '%s'
SET child.name = '%s', child.menu_type = 'app', child.route_path = '%s',
child.sort_order = %d, child.parent_id = parent.id
WHERE child.column_key = '%s'
""".formatted(menu.parentKey(), menu.name(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
}
}
private void ensureInternalDataPermissions() {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '视频任务数据查看', 'admin_image_video_task_data', 'internal', 'image-video-task-data', 0
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = 'admin_image_video_task_data'
)
""");
}
private void executeQuietly(String sql) {
try {
jdbcTemplate.execute(sql);
@@ -122,4 +185,8 @@ public class PermissionMenuSchemaInitializer {
private record DefaultAppMenu(String name, String columnKey, String routePath, int sortOrder) {
}
private record DefaultAppChildMenu(String name, String columnKey, String routePath, int sortOrder,
String parentKey) {
}
}
@@ -12,22 +12,34 @@ import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdate
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Menu hierarchy and user-column permissions.
*
* <p>The assignment table stores direct grants only. A parent grant is
* expanded to all descendants when effective permissions are read or checked;
* descendants are never inserted as a side effect of an update.</p>
*/
@Service
@RequiredArgsConstructor
public class PermissionMenuService {
@@ -40,28 +52,37 @@ public class PermissionMenuService {
private final UserColumnPermissionMapper userColumnPermissionMapper;
private final AdminUserMapper adminUserMapper;
/** Returns the flat menu catalog, including each item's direct parent ID. */
public List<PermissionMenuItemVo> list(String menuType) {
LambdaQueryWrapper<PermissionMenuEntity> query = new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(isValidMenuType(menuType), PermissionMenuEntity::getMenuType, normalizeMenuType(menuType))
.orderByAsc(PermissionMenuEntity::getSortOrder)
.orderByAsc(PermissionMenuEntity::getId);
return permissionMenuMapper.selectList(query).stream()
.map(this::toItemVo)
.toList();
List<PermissionMenuEntity> menus = loadMenus(menuType);
return toItemVos(menus, menus);
}
@Transactional
public PermissionMenuItemVo create(PermissionMenuCreateRequest request) {
return create(null, request);
}
/** Controller-facing variant; the caller may pass the authenticated admin. */
@Transactional
public PermissionMenuItemVo create(AdminUserEntity operator, PermissionMenuCreateRequest request) {
ensureAdminOperatorIfPresent(operator);
if (request == null) {
throw new BusinessException("菜单参数不能为空");
}
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
String menuType = normalizeMenuTypeRequired(request.getMenuType());
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
Long parentId = normalizeParentId(request.getParentId());
ensureParentValid(parentId, null, menuType);
ensureUniqueColumnKey(columnKey, null);
ensureUniqueRoutePath(menuType, routePath, null);
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setName(name);
entity.setColumnKey(columnKey);
entity.setParentId(parentId);
entity.setMenuType(menuType);
entity.setRoutePath(routePath);
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), null));
@@ -71,16 +92,30 @@ public class PermissionMenuService {
@Transactional
public PermissionMenuItemVo update(Long id, PermissionMenuUpdateRequest request) {
return update(null, id, request);
}
/** Controller-facing variant; validates a parent change against the full ancestor chain. */
@Transactional
public PermissionMenuItemVo update(AdminUserEntity operator, Long id, PermissionMenuUpdateRequest request) {
ensureAdminOperatorIfPresent(operator);
PermissionMenuEntity entity = getMenuById(id);
if (request == null) {
throw new BusinessException("菜单参数不能为空");
}
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
String menuType = normalizeMenuTypeRequired(request.getMenuType());
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
Long parentId = normalizeParentId(request.getParentId());
ensureParentValid(parentId, id, menuType);
ensureMenuTypeChangeAllowed(entity, menuType);
ensureUniqueColumnKey(columnKey, id);
ensureUniqueRoutePath(menuType, routePath, id);
entity.setName(name);
entity.setColumnKey(columnKey);
entity.setParentId(parentId);
entity.setMenuType(menuType);
entity.setRoutePath(routePath);
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), id));
@@ -90,102 +125,224 @@ public class PermissionMenuService {
@Transactional
public void delete(Long id) {
delete(null, id);
}
/**
* Deletes a leaf menu only. Refusing a parent deletion prevents orphaned
* hierarchy rows and makes the direct-grant meaning deterministic.
*/
@Transactional
public void delete(AdminUserEntity operator, Long id) {
ensureAdminOperatorIfPresent(operator);
PermissionMenuEntity entity = getMenuById(id);
Long childCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getParentId, entity.getId()));
if (childCount != null && childCount > 0) {
throw new BusinessException("请先删除该菜单下的子菜单");
}
// Do this explicitly even when the database FK is configured, so old
// installations without the FK are cleaned up as well.
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getColumnId, entity.getId()));
permissionMenuMapper.deleteById(entity.getId());
}
/** Returns only persisted direct grant IDs (never recursively expanded). */
public UserColumnIdsVo getUserColumnIds(Long userId, String menuType) {
return getUserColumnIds(null, userId, menuType);
}
/** Controller-facing variant with target-user scope checks. */
public UserColumnIdsVo getUserColumnIds(AdminUserEntity operator, Long userId, String menuType) {
AdminUserEntity user = getUserById(userId);
List<Long> columnIds;
if (isSuperAdmin(user)) {
columnIds = list(menuType).stream().map(PermissionMenuItemVo::getId).toList();
} else {
List<Long> assignedIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId))
.stream()
.map(UserColumnPermissionEntity::getColumnId)
.distinct()
.toList();
if (assignedIds.isEmpty()) {
columnIds = List.of();
} else if (isValidMenuType(menuType)) {
Set<Long> allowedIds = list(menuType).stream()
.map(PermissionMenuItemVo::getId)
.collect(Collectors.toSet());
columnIds = assignedIds.stream().filter(allowedIds::contains).toList();
} else {
columnIds = assignedIds;
}
}
ensureTargetAccessible(operator, user);
List<PermissionMenuEntity> menus = loadMenus(menuType);
Map<Long, PermissionMenuEntity> menuMap = menus.stream()
.filter(menu -> menu.getId() != null)
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity(), (left, right) -> left));
UserColumnIdsVo vo = new UserColumnIdsVo();
vo.setColumnIds(columnIds);
vo.setColumnIds(sortDirectIds(loadDirectColumnIds(userId), menuMap));
return vo;
}
/**
* Returns effective menu items: each directly granted menu plus every
* descendant in the same menu type. The response is flat and sorted.
*/
public List<PermissionMenuItemVo> getUserColumnPermissions(Long userId, String menuType) {
return getUserColumnPermissions(null, userId, menuType);
}
/** Controller-facing variant with target-user scope checks. */
public List<PermissionMenuItemVo> getUserColumnPermissions(AdminUserEntity operator,
Long userId,
String menuType) {
AdminUserEntity user = getUserById(userId);
ensureTargetAccessible(operator, user);
List<PermissionMenuEntity> menus = loadMenus(menuType);
if (isSuperAdmin(user)) {
return list(menuType);
return toItemVos(menus, menus);
}
List<Long> assignedIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId))
.stream()
.map(UserColumnPermissionEntity::getColumnId)
.distinct()
.toList();
if (assignedIds.isEmpty()) {
Set<Long> directIds = new LinkedHashSet<>(loadDirectColumnIds(userId));
if (directIds.isEmpty() || menus.isEmpty()) {
return List.of();
}
List<PermissionMenuEntity> menus = permissionMenuMapper.selectBatchIds(assignedIds);
Map<Long, PermissionMenuEntity> menuMap = menus.stream()
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity()));
String safeMenuType = normalizeMenuType(menuType);
return assignedIds.stream()
.map(menuMap::get)
.filter(menu -> menu != null)
.filter(menu -> safeMenuType == null || safeMenuType.equals(menu.getMenuType()))
.sorted(Comparator
.comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo))
.thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo)))
.map(this::toItemVo)
Set<Long> effectiveIds = expandDescendantIds(directIds, menus);
List<PermissionMenuEntity> effectiveMenus = menus.stream()
.filter(menu -> menu.getId() != null && effectiveIds.contains(menu.getId()))
.toList();
return toItemVos(effectiveMenus, menus);
}
/** Effective authorization check used by backend callers that need a boolean decision. */
public boolean hasEffectiveColumnPermission(Long userId, Long columnId) {
if (columnId == null || columnId <= 0) {
return false;
}
AdminUserEntity user = getUserById(userId);
PermissionMenuEntity target = permissionMenuMapper.selectById(columnId);
if (target == null) {
return false;
}
if (isSuperAdmin(user)) {
return true;
}
List<PermissionMenuEntity> menus = loadMenus(target.getMenuType());
return expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(userId)), menus)
.contains(columnId);
}
public List<ImageVideoDataPermissionUserVo> listImageVideoDataPermissionUsers(AdminUserEntity operator) {
ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
Set<Long> grantedUserIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getColumnId, dataPermission.getId()))
.stream()
.map(UserColumnPermissionEntity::getUserId)
.filter(id -> id != null && id > 0)
.collect(Collectors.toSet());
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
.orderByAsc(AdminUserEntity::getUsername)
.orderByAsc(AdminUserEntity::getId))
.stream()
.filter(user -> !isSuperAdmin(user))
.map(user -> toImageVideoDataPermissionUserVo(user, grantedUserIds.contains(user.getId())))
.toList();
}
@Transactional
public int updateImageVideoDataPermissionUsers(AdminUserEntity operator, List<Long> userIds) {
ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
List<Long> requestedIds = normalizeColumnIds(userIds);
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>());
Map<Long, AdminUserEntity> grantableUsers = users.stream()
.filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
throw new BusinessException("包含不存在或不可授权的用户");
}
userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
for (Long userId : requestedIds) {
UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
grant.setUserId(userId);
grant.setColumnId(dataPermission.getId());
userColumnPermissionMapper.insert(grant);
}
return requestedIds.size();
}
@Transactional
public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) {
getUserById(userId);
updateUserColumnPermissions(null, userId, request);
}
/**
* Replaces direct grants. A non-super admin may only grant effective menus
* that the operator already has, and only to themself or a directly-created
* normal user.
*/
@Transactional
public void updateUserColumnPermissions(AdminUserEntity operator,
Long userId,
UserColumnPermissionUpdateRequest request) {
updateUserColumnPermissions(operator, userId, request, null);
}
/** Replaces direct grants only within menuType; a blank type keeps legacy full-replacement behavior. */
@Transactional
public void updateUserColumnPermissions(AdminUserEntity operator,
Long userId,
UserColumnPermissionUpdateRequest request,
String menuType) {
AdminUserEntity target = getUserById(userId);
ensureTargetAccessible(operator, target);
List<Long> requestedIds = request == null ? List.of() : normalizeColumnIds(request.getColumnIds());
if (!requestedIds.isEmpty()) {
Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.in(PermissionMenuEntity::getId, requestedIds));
if (validCount == null || validCount != requestedIds.size()) {
throw new BusinessException("存在无效的菜单权限项");
String normalizedType = menuType == null || menuType.isBlank()
? null
: normalizeMenuTypeRequired(menuType);
Set<Long> scopedMenuIds = new LinkedHashSet<>();
if (normalizedType == null) {
if (!requestedIds.isEmpty()) {
Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.in(PermissionMenuEntity::getId, requestedIds));
if (validCount == null || validCount != requestedIds.size()) {
throw new BusinessException("存在无效的菜单权限项");
}
}
} else {
scopedMenuIds.addAll(loadMenus(normalizedType).stream()
.map(PermissionMenuEntity::getId)
.filter(Objects::nonNull)
.toList());
if (!scopedMenuIds.containsAll(requestedIds)) {
throw new BusinessException("存在无效或不属于当前类型的菜单权限项");
}
}
PermissionMenuEntity imageVideoDataPermission = permissionMenuMapper.selectOne(
new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY)
.last("LIMIT 1"));
if (imageVideoDataPermission != null) {
requestedIds = new ArrayList<>(requestedIds);
PermissionMenuEntity imageVideoDataPermission = normalizedType == null
? findImageVideoDataPermission()
: null;
Long protectedId = imageVideoDataPermission == null ? null : imageVideoDataPermission.getId();
List<Long> grantIds = requestedIds;
if (normalizedType == null && protectedId != null) {
grantIds = requestedIds.stream()
.filter(id -> !protectedId.equals(id))
.toList();
}
Set<Long> operatorEffectiveIds = ensureGrantable(operator, grantIds);
LinkedHashSet<Long> finalGrantIds = new LinkedHashSet<>(grantIds);
if (normalizedType == null && protectedId != null) {
Long existingCount = userColumnPermissionMapper.selectCount(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId)
.eq(UserColumnPermissionEntity::getColumnId, imageVideoDataPermission.getId()));
requestedIds.remove(imageVideoDataPermission.getId());
.eq(UserColumnPermissionEntity::getColumnId, protectedId));
if (existingCount != null && existingCount > 0) {
requestedIds.add(imageVideoDataPermission.getId());
finalGrantIds.add(protectedId);
}
}
if (operatorEffectiveIds != null) {
loadDirectColumnIds(userId).stream()
.filter(id -> normalizedType == null || scopedMenuIds.contains(id))
.filter(id -> !operatorEffectiveIds.contains(id))
.forEach(finalGrantIds::add);
}
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId));
for (Long columnId : requestedIds) {
LambdaUpdateWrapper<UserColumnPermissionEntity> delete =
new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId);
if (normalizedType == null) {
userColumnPermissionMapper.delete(delete);
} else if (!scopedMenuIds.isEmpty()) {
userColumnPermissionMapper.delete(delete.in(
UserColumnPermissionEntity::getColumnId, scopedMenuIds));
}
for (Long columnId : finalGrantIds) {
UserColumnPermissionEntity entity = new UserColumnPermissionEntity();
entity.setUserId(userId);
entity.setColumnId(columnId);
@@ -193,6 +350,75 @@ public class PermissionMenuService {
}
}
private List<PermissionMenuEntity> loadMenus(String menuType) {
String normalizedType = normalizeMenuType(menuType);
LambdaQueryWrapper<PermissionMenuEntity> query = new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(normalizedType != null, PermissionMenuEntity::getMenuType, normalizedType)
.orderByAsc(PermissionMenuEntity::getSortOrder)
.orderByAsc(PermissionMenuEntity::getId);
List<PermissionMenuEntity> rows = permissionMenuMapper.selectList(query);
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream()
.filter(row -> row != null && row.getId() != null)
.sorted(menuComparator())
.toList();
}
private List<Long> loadDirectColumnIds(Long userId) {
List<UserColumnPermissionEntity> rows = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId));
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream()
.map(UserColumnPermissionEntity::getColumnId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
}
private List<Long> sortDirectIds(List<Long> ids, Map<Long, PermissionMenuEntity> menuMap) {
return ids.stream()
.filter(menuMap::containsKey)
.sorted(Comparator.comparing(menuMap::get, menuComparator()))
.toList();
}
private Set<Long> expandDescendantIds(Set<Long> directIds, List<PermissionMenuEntity> menus) {
Map<Long, List<PermissionMenuEntity>> childrenByParent = new HashMap<>();
Map<Long, PermissionMenuEntity> menuMap = new HashMap<>();
for (PermissionMenuEntity menu : menus) {
if (menu.getId() == null) {
continue;
}
menuMap.put(menu.getId(), menu);
if (menu.getParentId() != null) {
childrenByParent.computeIfAbsent(menu.getParentId(), ignored -> new ArrayList<>()).add(menu);
}
}
childrenByParent.values().forEach(children -> children.sort(menuComparator()));
LinkedHashSet<Long> effective = new LinkedHashSet<>();
ArrayDeque<Long> queue = new ArrayDeque<>();
for (Long directId : directIds) {
if (directId != null && menuMap.containsKey(directId) && effective.add(directId)) {
queue.addLast(directId);
}
}
while (!queue.isEmpty()) {
Long parentId = queue.removeFirst();
for (PermissionMenuEntity child : childrenByParent.getOrDefault(parentId, List.of())) {
if (effective.add(child.getId())) {
queue.addLast(child.getId());
}
}
}
return effective;
}
private PermissionMenuEntity getMenuById(Long id) {
PermissionMenuEntity entity = permissionMenuMapper.selectById(id);
if (entity == null) {
@@ -209,8 +435,175 @@ public class PermissionMenuService {
return entity;
}
/** Resolves and validates an operator supplied by a trusted internal proxy. */
public AdminUserEntity requireAdminOperator(Long userId) {
AdminUserEntity operator = getUserById(userId);
ensureAdminOperatorIfPresent(operator);
return operator;
}
/** Resolves any authenticated user supplied by a trusted internal proxy. */
public AdminUserEntity requireUserOperator(Long userId) {
return getUserById(userId);
}
private void ensureAdminOperatorIfPresent(AdminUserEntity operator) {
if (operator == null) {
return;
}
if (!isAdmin(operator)) {
throw new BusinessException(403, "需要管理员权限");
}
}
private void ensureTargetAccessible(AdminUserEntity operator, AdminUserEntity target) {
if (operator == null) {
return;
}
boolean self = operator.getId() != null && operator.getId().equals(target.getId());
if (self || isSuperAdmin(operator)) {
return;
}
if (!isAdmin(operator)) {
throw new BusinessException(403, "需要管理员权限");
}
boolean ownedNormal = isNormalUser(target)
&& operator.getId() != null
&& operator.getId().equals(target.getCreatedById());
if (!ownedNormal) {
throw new BusinessException(403, "普通管理员只能管理自己创建的普通用户");
}
}
private Set<Long> ensureGrantable(AdminUserEntity operator, List<Long> requestedIds) {
if (operator == null || isSuperAdmin(operator)) {
return null;
}
List<PermissionMenuEntity> menus = loadMenus(null);
Set<Long> effective = expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(operator.getId())), menus);
Set<Long> denied = requestedIds.stream()
.filter(id -> !effective.contains(id))
.collect(Collectors.toCollection(LinkedHashSet::new));
if (!denied.isEmpty()) {
throw new BusinessException(403, "普通管理员只能分配自己已有的菜单权限");
}
return effective;
}
private boolean isAdmin(AdminUserEntity user) {
if (user == null) {
return false;
}
String role = normalizeRole(user.getRole());
if (!role.isEmpty()) {
return "super_admin".equals(role) || "admin".equals(role);
}
return Integer.valueOf(1).equals(user.getIsAdmin());
}
private boolean isSuperAdmin(AdminUserEntity user) {
return user != null && "super_admin".equalsIgnoreCase(normalizeRequired(user.getRole(), ""));
if (user == null) {
return false;
}
String role = normalizeRole(user.getRole());
if ("super_admin".equals(role)) {
return true;
}
return role.isEmpty()
&& Integer.valueOf(1).equals(user.getIsAdmin())
&& user.getCreatedById() == null;
}
private boolean isNormalUser(AdminUserEntity user) {
if (user == null) {
return false;
}
String role = normalizeRole(user.getRole());
if (!role.isEmpty()) {
return !"admin".equals(role) && !"super_admin".equals(role);
}
return !Integer.valueOf(1).equals(user.getIsAdmin());
}
private PermissionMenuEntity findImageVideoDataPermission() {
return permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY)
.last("LIMIT 1"));
}
private PermissionMenuEntity requireImageVideoDataPermission() {
PermissionMenuEntity permission = findImageVideoDataPermission();
if (permission == null || permission.getId() == null) {
throw new BusinessException("视频任务数据权限尚未初始化");
}
return permission;
}
private void ensureSuperAdminOperator(AdminUserEntity operator) {
ensureAdminOperatorIfPresent(operator);
if (operator == null || !isSuperAdmin(operator)) {
throw new BusinessException(403, "仅超级管理员可以配置视频任务权限");
}
}
private ImageVideoDataPermissionUserVo toImageVideoDataPermissionUserVo(AdminUserEntity user,
boolean granted) {
ImageVideoDataPermissionUserVo vo = new ImageVideoDataPermissionUserVo();
vo.setId(user.getId());
vo.setUsername(user.getUsername());
String role = normalizeRole(user.getRole());
vo.setRole(role.isEmpty()
? (Integer.valueOf(1).equals(user.getIsAdmin()) ? "admin" : "normal")
: role);
vo.setGranted(granted);
return vo;
}
private void ensureParentValid(Long parentId, Long currentId, String menuType) {
if (parentId == null) {
return;
}
if (currentId != null && currentId.equals(parentId)) {
throw new BusinessException("父级菜单不能选择自己");
}
PermissionMenuEntity parent = permissionMenuMapper.selectById(parentId);
if (parent == null) {
throw new BusinessException("父级菜单不存在");
}
String parentType = normalizeMenuType(parent.getMenuType());
if (!menuType.equals(parentType)) {
throw new BusinessException("父级菜单必须与当前菜单属于同一类型");
}
Set<Long> visited = new HashSet<>();
PermissionMenuEntity cursor = parent;
while (cursor != null && cursor.getId() != null) {
if (!visited.add(cursor.getId())) {
throw new BusinessException("菜单层级存在循环");
}
if (currentId != null && currentId.equals(cursor.getId())) {
throw new BusinessException("父级菜单不能选择自己的子级");
}
Long ancestorId = cursor.getParentId();
if (ancestorId == null) {
break;
}
cursor = permissionMenuMapper.selectById(ancestorId);
if (cursor == null) {
throw new BusinessException("父级菜单不存在");
}
}
}
private void ensureMenuTypeChangeAllowed(PermissionMenuEntity entity, String menuType) {
if (menuType.equals(normalizeMenuType(entity.getMenuType()))) {
return;
}
Long childCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getParentId, entity.getId()));
if (childCount != null && childCount > 0) {
throw new BusinessException("请先移出该菜单下的子菜单再修改菜单类型");
}
}
private void ensureUniqueColumnKey(String columnKey, Long excludeId) {
@@ -240,8 +633,12 @@ public class PermissionMenuService {
return normalized;
}
private boolean isValidMenuType(String menuType) {
return normalizeMenuType(menuType) != null;
private Long normalizeParentId(Long value) {
return value == null || value <= 0 ? null : value;
}
private String normalizeRole(String role) {
return role == null ? "" : role.trim().toLowerCase(Locale.ROOT);
}
private String normalizeMenuTypeRequired(String menuType) {
@@ -260,6 +657,8 @@ public class PermissionMenuService {
if (MENU_TYPE_APP.equals(normalized) || MENU_TYPE_ADMIN.equals(normalized)) {
return normalized;
}
// Keep the internal data permission visible to legacy callers that do
// not pass a menu type, but never treat an unknown filter as a type.
return null;
}
@@ -276,11 +675,34 @@ public class PermissionMenuService {
return new ArrayList<>(uniqueIds);
}
private Comparator<PermissionMenuEntity> menuComparator() {
return Comparator
.comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo))
.thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo));
}
private List<PermissionMenuItemVo> toItemVos(List<PermissionMenuEntity> visibleMenus,
List<PermissionMenuEntity> hierarchyMenus) {
Map<Long, PermissionMenuEntity> menuMap = hierarchyMenus.stream()
.filter(menu -> menu.getId() != null)
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity(), (left, right) -> left));
return visibleMenus.stream()
.map(menu -> toItemVo(menu, menuMap))
.toList();
}
private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity) {
return toItemVos(List.of(entity), loadMenus(entity.getMenuType())).getFirst();
}
private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity,
Map<Long, PermissionMenuEntity> menuMap) {
PermissionMenuItemVo vo = new PermissionMenuItemVo();
vo.setId(entity.getId());
vo.setName(entity.getName());
vo.setColumnKey(entity.getColumnKey());
vo.setParentId(entity.getParentId());
vo.setRootColumnKey(resolveRootColumnKey(entity, menuMap));
vo.setMenuType(entity.getMenuType());
vo.setRoutePath(entity.getRoutePath());
vo.setSortOrder(entity.getSortOrder());
@@ -288,6 +710,20 @@ public class PermissionMenuService {
return vo;
}
private String resolveRootColumnKey(PermissionMenuEntity entity,
Map<Long, PermissionMenuEntity> menuMap) {
PermissionMenuEntity root = entity;
Set<Long> visited = new HashSet<>();
while (root.getId() != null && visited.add(root.getId()) && root.getParentId() != null) {
PermissionMenuEntity parent = menuMap.get(root.getParentId());
if (parent == null) {
break;
}
root = parent;
}
return root.getColumnKey();
}
private Integer resolveSortOrder(Integer sortOrder, Long excludeId) {
if (sortOrder != null) {
return sortOrder;
@@ -20,7 +20,7 @@ public class ProductRiskMatchShopsRequest {
@NotEmpty(message = "shop_names 不能为空")
@JsonProperty("shop_names")
@Schema(
description = "待匹配的店铺名称列表,至少 1 个;服务端按列表顺序逐项查索引,返回体 `items` 与列表顺序一一对应",
description = "待匹配的店铺名称列表,至少 1 个;服务端规范化名称、过滤空项并按首次出现顺序去重",
requiredMode = Schema.RequiredMode.REQUIRED,
example = "[\"店铺甲\",\"店铺乙\"]")
private List<String> shopNames;
@@ -12,6 +12,6 @@ import java.util.List;
public class ProductRiskCountryPreferenceVo {
@JsonProperty("country_codes")
@Schema(description = "已选国家代码,顺序即处理顺序;未持久化时服务端返回默认德国→英国→法国→意大利→西班牙(全选)")
@Schema(description = "已选国家代码,顺序即处理顺序;未保存偏好时返回当前业务模块配置的默认顺序")
private List<String> countryCodes = new ArrayList<>();
}
@@ -7,9 +7,9 @@ import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "批量匹配响应体:items 与请求 shop_names 顺序一致")
@Schema(description = "批量匹配响应体:按规范化、过滤空项并去重后的有效店铺顺序返回")
public class ProductRiskMatchShopsVo {
@Schema(description = "各店匹配结果列表,一项对应请求中的一个店")
@Schema(description = "各店匹配结果列表,一项对应一个处理后的有效店铺")
private List<ProductRiskShopQueueItemVo> items = new ArrayList<>();
}
@@ -106,8 +106,8 @@ public class PublishController {
@PostMapping("/tasks/{taskId}/result")
@Operation(
summary = "Python 按文件回传当前店铺完整数据",
description = "请求只需 taskId 和 filesuser_id 为兼容旧客户端的可选字段;后端按 taskId 反查任务所属用户,传入 user_id 时会校验归属。成功回传必须包含该文件全部原始行,可使用 rows 数组,或在 rows 为空时使用 countries 按国家分组;行数少于原始数据、包含 null 行或八列全空白对象时会被拒绝且不会覆盖已解析数据。error 非空时将文件标记为 FAILED。全部文件进入终态后,只要至少一个文件成功就异步组装结果;全部失败则不生成结果文件。")
summary = "Python 按文件分片回传上架结果",
description = "请求只需 taskId 和 filesuser_id 为兼容旧客户端的可选字段;后端始终按 taskId 处理任务数据。每个文件通过 chunk_index/chunk_total 声明从 1 开始的分片,不传时按 1/1 兼容旧客户端。正常分片先写入 RustFS,同一文件的全部分片到齐后才按序合并、校验总行数并覆盖解析数据;重复分片内容相同则幂等接受,内容不同或 chunk_total 不一致则拒绝。error 非空时直接将文件标记为 FAILED。全部文件进入终态后,只要至少一个文件成功就异步组装结果;全部失败则不生成结果文件。")
public ApiResponse<Void> submitResult(
@Parameter(description = "上架任务 ID", required = true, example = "9001")
@PathVariable Long taskId,
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.publish.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import lombok.Data;
@@ -11,29 +12,39 @@ import java.util.List;
import java.util.Map;
@Data
@Schema(description = "单个文件的 Python 处理结果。fileIdfileKeysourceFilename 至少提供一个用于定位文件。成功时必须提交完整 rows 或 countries;失败时填写 error。同一次请求中,不同定位字段指向同一任务文件也视为重复提交")
@Schema(description = "单个文件的一次 Python 分片回传。fileId/file_key/source_filename 三者可用于定位文件,rows 或 countries 提供当前分片数据,error 用于返回失败")
public class PublishResultFileDto {
@JsonAlias("file_id")
@Schema(description = "任务内文件 ID;同时兼容 file_id,优先使用该字段定位", example = "9101")
private Long fileId;
@JsonAlias("file_key")
@Schema(description = "文件标识;同时兼容 file_key可在 fileId 缺失时定位文件", example = "uploads/20260724/uuid/郭亚庆.xlsx")
@Schema(description = "任务内文件标识;兼容 file_keyfileId 缺失时可通过该字段定位", example = "uploads/20260724/uuid/郭亚庆.xlsx")
private String fileKey;
@JsonAlias("source_filename")
@Schema(description = "原始文件名;同时兼容 source_filename可在 fileId/fileKey 缺失时定位文件", example = "郭亚庆.xlsx")
@Schema(description = "原始文件名;兼容 source_filenamefileId/fileKey 缺失时定位文件", example = "郭亚庆.xlsx")
private String sourceFilename;
@Schema(description = "文件级失败原因。非空时文件标记为 FAILEDrows/countries 不会覆盖原始数据", example = "打开店铺失败")
@JsonProperty("chunk_index")
@JsonAlias("chunkIndex")
@Schema(description = "分片序号,从 1 开始;不传时与 chunk_total 同时未传表示 1/1,兼容旧版。", example = "1", requiredMode = Schema.RequiredMode.NOT_REQUIRED)
private Integer chunkIndex;
@JsonProperty("chunk_total")
@JsonAlias("chunkTotal")
@Schema(description = "该文件的分片总数;不传时按 1 处理。分片先暂存到 RustFS,全部到齐后再按序合并。", example = "36", requiredMode = Schema.RequiredMode.NOT_REQUIRED)
private Integer chunkTotal;
@Schema(description = "文件处理结果;FAILED 时通过 error 携带错误信息,rows/countries 可为空", example = "打开文件失败")
private String error;
@Valid
@JsonAlias({"items", "data"})
@Schema(description = "当前店铺的完整结果行;同时兼容 items/data。成功时行数不能少于原始 Excel 数据行数,且不能包含 null 行或八列全空白对象")
@Schema(description = "当前分片的结果行,可为一条或多条;rows 中每条均不能为空对象")
private List<PublishRowDto> rows = new ArrayList<>();
@Valid
@Schema(description = "按国家名称或代码分组的完整结果,可替代 rows。仅当 rows 为空时读取;缺少国家字段的行会使用当前 Map key;分组内不能包含 null 行或八列全空白对象")
@Schema(description = "当前分片按国家分组的结果;rows 为空时可使用 countrieskey 为国家或店铺名")
private Map<String, List<PublishRowDto>> countries = new LinkedHashMap<>();
}
@@ -2,8 +2,10 @@ package com.nanri.aiimage.modules.publish.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
@@ -31,18 +33,24 @@ import com.nanri.aiimage.modules.publish.model.vo.PublishTaskDetailVo;
import com.nanri.aiimage.modules.publish.model.vo.PublishTaskVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -84,8 +92,11 @@ public class PublishTaskService {
private final PublishItemMapper publishItemMapper;
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskFileJobService taskFileJobService;
private final TaskDistributedLockService taskDistributedLockService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
@@ -239,7 +250,23 @@ public class PublishTaskService {
if (lock == null) {
throw new BusinessException("task lock is busy");
}
transactionTemplate.executeWithoutResult(status -> submitResultLocked(taskId, request));
List<String> storedPayloads = new ArrayList<>();
boolean[] cleanupAfterCommit = {false};
try {
transactionTemplate.executeWithoutResult(
status -> cleanupAfterCommit[0] = submitResultLocked(taskId, request, storedPayloads));
} catch (RuntimeException ex) {
deleteRolledBackPayloads(storedPayloads);
throw ex;
}
if (cleanupAfterCommit[0]) {
try {
deleteTransientResultChunks(taskId);
} catch (Exception ex) {
log.warn("[publish] failed-task chunk cleanup failed taskId={} msg={}",
taskId, safeMessage(ex));
}
}
}
}
@@ -455,7 +482,7 @@ public class PublishTaskService {
@Transactional
public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = requireTask(taskId, userId);
FileTaskEntity task = requireTaskForDeletion(taskId, userId);
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
@@ -470,6 +497,7 @@ public class PublishTaskService {
}
}
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
deleteTransientResultChunks(taskId);
publishItemMapper.delete(new LambdaQueryWrapper<PublishItemEntity>()
.eq(PublishItemEntity::getTaskId, taskId));
publishFileMapper.delete(new LambdaQueryWrapper<PublishFileEntity>()
@@ -486,8 +514,49 @@ public class PublishTaskService {
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果不存在");
}
FileTaskEntity task = requireTask(result.getTaskId(), userId);
deleteTask(task.getId(), userId);
deleteTask(result.getTaskId(), userId);
}
public void cleanupResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
return;
}
deleteTransientResultChunks(job.getTaskId());
}
private void deleteTransientResultChunks(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
private void deleteRolledBackPayloads(List<String> storedPayloads) {
if (storedPayloads == null || storedPayloads.isEmpty()) {
return;
}
for (String storedPayload : storedPayloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
} catch (Exception ex) {
log.warn("[publish] rolled-back RustFS payload cleanup failed pointer={} msg={}",
transientPayloadStorageService.extractPointer(storedPayload), safeMessage(ex));
}
}
}
private PreparedFile prepareFile(PublishSourceFileDto source) {
@@ -538,6 +607,26 @@ public class PublishTaskService {
}
LocalDateTime now = LocalDateTime.now();
String error = "任务心跳超时";
List<PublishFileEntity> files = listTaskFiles(taskId);
for (PublishFileEntity file : files) {
if (!STATUS_PENDING.equals(file.getStatus()) && !STATUS_RUNNING.equals(file.getStatus())) {
continue;
}
List<PublishRowDto> recoveredRows = loadReceivedResultRows(taskId, file.getId());
if (recoveredRows.isEmpty()) {
continue;
}
replaceRows(taskId, file.getId(), recoveredRows);
file.setStatus(STATUS_SUCCESS);
file.setTotalRows(recoveredRows.size());
file.setProcessedRows(recoveredRows.size());
file.setErrorMessage("任务心跳超时,已保留 " + recoveredRows.size() + " 行已回传数据");
file.setUpdatedAt(now);
file.setFinishedAt(now);
publishFileMapper.updateById(file);
log.info("[publish] recovered partial result from stale task taskId={} fileId={} rows={}",
taskId, file.getId(), recoveredRows.size());
}
publishFileMapper.update(null, new LambdaUpdateWrapper<PublishFileEntity>()
.eq(PublishFileEntity::getTaskId, taskId)
.in(PublishFileEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING))
@@ -545,7 +634,7 @@ public class PublishTaskService {
.set(PublishFileEntity::getErrorMessage, error)
.set(PublishFileEntity::getUpdatedAt, now)
.set(PublishFileEntity::getFinishedAt, now));
List<PublishFileEntity> files = listTaskFiles(taskId);
files = listTaskFiles(taskId);
int successCount = (int) files.stream().filter(file -> STATUS_SUCCESS.equals(file.getStatus())).count();
task.setSuccessFileCount(successCount);
task.setFailedFileCount(Math.max(0, files.size() - successCount));
@@ -615,10 +704,12 @@ public class PublishTaskService {
return new PersistedTask(task, result, savedFiles);
}
private void submitResultLocked(Long taskId, PublishSubmitResultRequest request) {
private boolean submitResultLocked(Long taskId,
PublishSubmitResultRequest request,
List<String> storedPayloads) {
FileTaskEntity task = requireTask(taskId, request.getUserId());
if (STATUS_SUCCESS.equals(task.getStatus())) {
return;
return false;
}
if (STATUS_FAILED.equals(task.getStatus())) {
throw new BusinessException("任务已失败,拒绝继续回传");
@@ -629,6 +720,9 @@ public class PublishTaskService {
Set<Long> submittedFileIds = new LinkedHashSet<>();
for (PublishResultFileDto incoming : request.getFiles()) {
if (incoming == null) {
throw new BusinessException("files 不能包含空对象");
}
PublishFileEntity file = findCallbackFile(taskId, incoming);
if (!submittedFileIds.add(file.getId())) {
throw new BusinessException("同一文件不能在一次请求中重复提交");
@@ -641,7 +735,16 @@ public class PublishTaskService {
file.setProcessedRows(0);
file.setErrorMessage(incoming.getError().trim());
} else {
List<PublishRowDto> rows = flattenRows(incoming);
ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads);
if (!receipt.completed()) {
file.setStatus(STATUS_RUNNING);
file.setErrorMessage(null);
file.setUpdatedAt(LocalDateTime.now());
file.setFinishedAt(null);
publishFileMapper.updateById(file);
continue;
}
List<PublishRowDto> rows = loadCompleteResultRows(taskId, receipt);
validateCompleteResultRows(taskId, file.getId(), rows);
replaceRows(taskId, file.getId(), rows);
file.setStatus(STATUS_SUCCESS);
@@ -666,11 +769,11 @@ public class PublishTaskService {
if (terminalCount < files.size()) {
task.setStatus(STATUS_RUNNING);
fileTaskMapper.updateById(task);
return;
return false;
}
if (successCount <= 0) {
markTaskAndResultFailed(task, result, "全部文件处理失败");
return;
return true;
}
task.setStatus(STATUS_RUNNING);
@@ -678,6 +781,7 @@ public class PublishTaskService {
task.setFinishedAt(null);
fileTaskMapper.updateById(task);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId));
return false;
}
private List<PublishTaskDetailVo> loadTaskDetails(List<FileTaskEntity> tasks) {
@@ -805,6 +909,20 @@ public class PublishTaskService {
}
private FileTaskEntity requireTask(Long taskId, Long userId) {
FileTaskEntity task = requireTaskRecord(taskId, userId);
ensureTaskOwnedByCurrentInstance(task, "access publish task");
return task;
}
private FileTaskEntity requireTaskForDeletion(Long taskId, Long userId) {
FileTaskEntity task = requireTaskRecord(taskId, userId);
if (!isTerminal(task.getStatus())) {
ensureTaskOwnedByCurrentInstance(task, "delete publish task");
}
return task;
}
private FileTaskEntity requireTaskRecord(Long taskId, Long userId) {
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId 不合法");
}
@@ -813,7 +931,6 @@ public class PublishTaskService {
|| (userId != null && !userId.equals(task.getUserId()))) {
throw new BusinessException("任务不存在");
}
ensureTaskOwnedByCurrentInstance(task, "access publish task");
return task;
}
@@ -888,6 +1005,257 @@ public class PublishTaskService {
return file;
}
private ResultChunkReceipt persistResultChunk(Long taskId,
PublishFileEntity file,
PublishResultFileDto incoming,
List<String> storedPayloads) {
int chunkIndex = incoming.getChunkIndex() == null ? 1 : incoming.getChunkIndex();
int chunkTotal = incoming.getChunkTotal() == null ? 1 : incoming.getChunkTotal();
validateChunkMetadata(chunkIndex, chunkTotal);
String scopeKey = "file:" + file.getId();
String scopeHash = DigestUtil.sha256Hex(scopeKey);
TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
List<PublishRowDto> rows = flattenRows(incoming);
String payloadJson = writeJson(rows, "序列化上架结果分片失败");
String payloadHash = DigestUtil.sha256Hex(payloadJson);
TaskChunkEntity existing = findResultChunk(taskId, scopeHash, chunkIndex);
if (existing != null) {
validateExistingChunk(existing, chunkTotal, payloadHash);
int receivedChunkCount = countResultChunks(taskId, scopeHash);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
receivedChunkCount >= chunkTotal);
}
ensureRustfsPayloadStorageEnabled();
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
requireRustfsPayload(storedPayload, "上架结果分片必须写入 RustFS");
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(taskId);
chunk.setModuleType(MODULE_TYPE);
chunk.setScopeKey(scopeKey);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setChunkTotal(chunkTotal);
chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(payloadHash);
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
try {
taskChunkMapper.insert(chunk);
storedPayloads.add(storedPayload);
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
if (winner == null) {
throw new BusinessException("上架结果分片并发写入失败,请重试");
}
validateExistingChunk(winner, chunkTotal, payloadHash);
} catch (RuntimeException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw ex;
}
int receivedChunkCount = countResultChunks(taskId, scopeHash);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} received={}",
taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
receivedChunkCount >= chunkTotal);
}
private void validateChunkMetadata(int chunkIndex, int chunkTotal) {
if (chunkIndex <= 0) {
throw new BusinessException("chunk_index 必须从 1 开始");
}
if (chunkTotal <= 0) {
throw new BusinessException("chunk_total 必须大于 0");
}
if (chunkIndex > chunkTotal) {
throw new BusinessException("chunk_index 不能大于 chunk_total");
}
}
private TaskScopeStateEntity findResultScope(Long taskId, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private TaskChunkEntity findResultChunk(Long taskId, String scopeHash, int chunkIndex) {
return taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
}
private void validateExistingChunk(TaskChunkEntity existing, int chunkTotal, String payloadHash) {
validateChunkTotal(existing.getChunkTotal(), chunkTotal);
if (!Objects.equals(existing.getPayloadHash(), payloadHash)) {
throw new BusinessException("同一 chunk_index 已回传不同内容,拒绝覆盖");
}
}
private void validateChunkTotal(Integer existingChunkTotal, int chunkTotal) {
if (existingChunkTotal != null && existingChunkTotal > 0 && existingChunkTotal != chunkTotal) {
throw new BusinessException("同一文件的 chunk_total 必须保持一致");
}
}
private int countResultChunks(Long taskId, String scopeHash) {
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
return count == null ? 0 : count.intValue();
}
private void persistResultScope(Long taskId,
String scopeKey,
String scopeHash,
int chunkTotal,
int receivedChunkCount) {
TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
LocalDateTime now = LocalDateTime.now();
if (scope == null) {
scope = new TaskScopeStateEntity();
scope.setTaskId(taskId);
scope.setModuleType(MODULE_TYPE);
scope.setScopeKey(scopeKey);
scope.setScopeHash(scopeHash);
scope.setCreatedAt(now);
}
boolean completed = receivedChunkCount >= chunkTotal;
scope.setChunkTotal(chunkTotal);
scope.setReceivedChunkCount(receivedChunkCount);
scope.setCompleted(completed ? 1 : 0);
scope.setLastChunkAt(now);
scope.setLastError(null);
scope.setStateJson(completed ? "{\"phase\":\"COMPLETE\"}" : "{\"phase\":\"RECEIVING\"}");
scope.setUpdatedAt(now);
if (scope.getId() != null) {
taskScopeStateMapper.updateById(scope);
return;
}
try {
taskScopeStateMapper.insert(scope);
} catch (DuplicateKeyException ex) {
TaskScopeStateEntity winner = findResultScope(taskId, scopeHash);
if (winner == null) {
throw new BusinessException("上架结果分片状态写入失败,请重试");
}
validateChunkTotal(winner.getChunkTotal(), chunkTotal);
winner.setChunkTotal(chunkTotal);
winner.setReceivedChunkCount(receivedChunkCount);
winner.setCompleted(completed ? 1 : 0);
winner.setLastChunkAt(now);
winner.setLastError(null);
winner.setStateJson(scope.getStateJson());
winner.setUpdatedAt(now);
taskScopeStateMapper.updateById(winner);
}
}
private List<PublishRowDto> loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, receipt.scopeHash())
.orderByAsc(TaskChunkEntity::getChunkIndex));
if (chunks == null || chunks.size() != receipt.chunkTotal()) {
throw new BusinessException("上架结果分片尚未完整,暂不能合并");
}
List<PublishRowDto> rows = new ArrayList<>();
TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
};
for (int i = 0; i < chunks.size(); i++) {
TaskChunkEntity chunk = chunks.get(i);
int expectedIndex = i + 1;
if (!Objects.equals(chunk.getChunkIndex(), expectedIndex)) {
throw new BusinessException("上架结果缺少第 " + expectedIndex + " 个分片");
}
validateChunkTotal(chunk.getChunkTotal(), receipt.chunkTotal());
List<PublishRowDto> chunkRows = readResultChunkRows(chunk, listType);
for (PublishRowDto row : chunkRows) {
rows.add(copyRequiredRow(row));
}
}
return rows;
}
private List<PublishRowDto> loadReceivedResultRows(Long taskId, Long fileId) {
String scopeHash = DigestUtil.sha256Hex("file:" + fileId);
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByAsc(TaskChunkEntity::getChunkIndex));
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
List<PublishRowDto> rows = new ArrayList<>();
TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
};
for (TaskChunkEntity chunk : chunks) {
try {
for (PublishRowDto row : readResultChunkRows(chunk, listType)) {
rows.add(copyRequiredRow(row));
}
} catch (Exception ex) {
log.warn("[publish] skip unreadable stale result chunk taskId={} fileId={} chunkIndex={} msg={}",
taskId, fileId, chunk.getChunkIndex(), safeMessage(ex));
}
}
return rows;
}
private List<PublishRowDto> readResultChunkRows(TaskChunkEntity chunk,
TypeReference<List<PublishRowDto>> listType) {
int chunkIndex = chunk.getChunkIndex() == null ? 0 : chunk.getChunkIndex();
String pointer = transientPayloadStorageService.extractPointer(chunk.getPayloadJson());
if (pointer == null || !pointer.startsWith("rustfs:")) {
throw new BusinessException("上架结果分片不是 RustFS 数据,拒绝合并");
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
chunk.getPayloadJson(), "读取上架结果分片失败");
List<PublishRowDto> rows = objectMapper.readValue(payloadJson, listType);
return rows == null ? List.of() : rows;
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("读取上架结果第 " + chunkIndex + " 个分片失败: "
+ safeMessage(ex));
}
}
private void ensureRustfsPayloadStorageEnabled() {
if (!transientPayloadStorageService.isSharedWriteEnabled()) {
throw new BusinessException("RustFS 未配置,上架结果分片暂不可接收");
}
}
private void requireRustfsPayload(String storedPayload, String message) {
String pointer = transientPayloadStorageService.extractPointer(storedPayload);
if (pointer != null && pointer.startsWith("rustfs:")) {
return;
}
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new BusinessException(message);
}
private List<PublishRowDto> flattenRows(PublishResultFileDto incoming) {
if (incoming.getRows() != null && !incoming.getRows().isEmpty()) {
return incoming.getRows().stream().map(this::copyRequiredRow).toList();
@@ -1266,4 +1634,9 @@ public class PublishTaskService {
private record TaskOptions(String publishCountry, List<String> syncCountries) {
}
private record ResultChunkReceipt(String scopeHash,
int chunkTotal,
boolean completed) {
}
}
@@ -0,0 +1,308 @@
package com.nanri.aiimage.modules.shopdatacrawl.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskBatchRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.util.List;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/shop-data-crawl")
@Tag(
name = "店铺数据抓取",
description = "运营工具中的店铺数据抓取接口:管理待处理店铺、匹配店铺、保存五国处理顺序、创建并轮询串行任务、接收 Python 分片结果及下载最终 Excel。任务相关操作均以 taskId 为准。")
public class ShopDataCrawlTaskController {
private final ShopDataCrawlResolveService resolveService;
private final ShopDataCrawlTaskService taskService;
@GetMapping("/candidates")
@Operation(
summary = "查询待处理店铺",
description = "返回当前用户已添加到店铺数据抓取管理区的店铺,按新增时间倒序排列。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<List<ProductRiskCandidateVo>> candidates(
@Parameter(name = "user_id", description = "当前用户 ID,店铺数据按用户隔离", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(resolveService.listCandidates(userId));
}
@PostMapping("/candidates")
@Operation(
summary = "添加待处理店铺",
description = "先校验店铺已在后台管理中维护且能够命中紫鸟店铺索引,再添加到当前用户的管理区;重复添加时返回已有记录。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "添加成功"))
public ApiResponse<ProductRiskCandidateVo> addCandidate(
@Valid @RequestBody ProductRiskCandidateAddRequest request) {
return ApiResponse.success(resolveService.addCandidate(request));
}
@DeleteMapping("/candidates/{id}")
@Operation(
summary = "删除待处理店铺",
description = "从当前用户的店铺数据抓取管理区删除指定记录,不会删除后台管理中的店铺。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"))
public ApiResponse<Void> deleteCandidate(
@Parameter(description = "待处理店铺记录主键", required = true, example = "10")
@PathVariable Long id,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验记录归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
resolveService.deleteCandidate(userId, id);
return ApiResponse.success(null);
}
@GetMapping("/country-preference")
@Operation(
summary = "查询国家处理顺序",
description = "返回当前用户勾选的国家代码,数组顺序就是抓取顺序。尚未保存时默认返回 UK、DE、FR、ES、IT。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ProductRiskCountryPreferenceVo> countryPreference(
@Parameter(name = "user_id", description = "当前用户 ID", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(resolveService.getCountryPreference(userId));
}
@PutMapping("/country-preference")
@Operation(
summary = "保存国家处理顺序",
description = "保存当前用户勾选并排序后的国家列表。支持 DE、FR、ES、IT、UK,至少选择 1 个且不能重复。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "保存成功"))
public ApiResponse<ProductRiskCountryPreferenceVo> saveCountryPreference(
@Valid @RequestBody ProductRiskCountryPreferenceSaveRequest request) {
return ApiResponse.success(resolveService.saveCountryPreference(request));
}
@PostMapping("/match-shops")
@Operation(
summary = "批量匹配店铺",
description = "店铺名称规范化、去重后逐一匹配。先确认店铺已在后台管理中维护,再查询紫鸟店铺索引,返回店铺 ID、平台、公司名称和匹配状态。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "匹配完成"))
public ApiResponse<ProductRiskMatchShopsVo> matchShops(
@Valid @RequestBody ProductRiskMatchShopsRequest request) {
return ApiResponse.success(resolveService.matchShops(request));
}
@GetMapping("/dashboard")
@Operation(
summary = "查询任务统计",
description = "返回当前用户的待处理店铺数、已结束任务数、成功任务数和失败任务数。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ProductRiskDashboardVo> dashboard(
@Parameter(name = "user_id", description = "当前用户 ID", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(taskService.dashboard(userId));
}
@GetMapping("/history")
@Operation(
summary = "查询抓取记录",
description = "返回当前用户最近 100 条店铺抓取结果,按创建时间倒序排列,包含执行状态和异步结果文件状态。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ShopDataCrawlHistoryVo> history(
@Parameter(name = "user_id", description = "当前用户 ID", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(taskService.listHistory(userId));
}
@PostMapping("/tasks/progress/batch")
@Operation(
summary = "批量查询任务进度",
description = "供前端轮询任务队列。对有效正数 taskId 去重后最多查询前 50 个;不存在、无结果或非本模块的任务写入 missingTaskIds。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ShopDataCrawlTaskBatchVo> progress(
@Valid @RequestBody ShopDataCrawlTaskBatchRequest request) {
return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds()));
}
@PostMapping("/tasks")
@Operation(
summary = "创建店铺数据抓取任务",
description = "每次必须且只能提交一个已匹配店铺,并按 country_codes 的顺序抓取。任务创建后状态为 RUNNING,返回 taskId 和该店铺的初始结果项。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功"))
public ApiResponse<ShopDataCrawlCreateTaskVo> createTask(
@Valid @RequestBody ShopDataCrawlCreateTaskRequest request) {
return ApiResponse.success(taskService.createTask(request));
}
@PostMapping("/tasks/{taskId}/result")
@Operation(
summary = "回传店铺抓取结果",
description = "供 Python 按 taskId 回传,不依赖 user_id。每次 shops 必须且只能包含任务中的一个店铺。新协议要求同时传 chunk_index/chunk_total,分片从 1 开始、允许乱序和同内容重试,数据严格暂存到 RustFS,全部到齐后按序合并;旧客户端未传分片字段时仍按 shopDone 累加。error 非空时店铺立即失败;成功结束后异步生成 Excel。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "回传成功"))
public ApiResponse<Void> submitResult(
@Parameter(description = "店铺数据抓取任务 ID;服务端据此路由到任务归属实例,任务必须尚未结束", required = true,
example = "20363")
@PathVariable Long taskId,
@io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "单个店铺的一次结果分片。chunk_index 与 chunk_total 必须同时传入。结果行依次包含日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价。",
required = true,
content = @Content(
mediaType = "application/json",
schema = @Schema(implementation = ShopDataCrawlSubmitResultRequest.class),
examples = {
@ExampleObject(
name = "中间分片",
summary = "继续等待后续分片",
value = """
{
"shops": [
{
"shopName": "示例店铺",
"chunk_index": 1,
"chunk_total": 2,
"countryResults": [
{
"country": "DE",
"items": [
{
"date": "2026-07-25",
"asin": "B0EXAMPLE1",
"inventorySales": "120",
"salesRank": "35",
"pageViews": "860",
"unitsSold": "18",
"price": "29.99",
"recommendedOffer": "27.99"
}
]
}
],
"submissionId": "shop-data-crawl:20363:part-1"
}
]
}
"""),
@ExampleObject(
name = "最终分片",
summary = "完成该店铺并触发异步生成 Excel",
value = """
{
"shops": [
{
"shopName": "示例店铺",
"chunk_index": 2,
"chunk_total": 2,
"countryResults": [
{
"country": "UK",
"items": []
}
],
"submissionId": "shop-data-crawl:20363:final"
}
]
}
"""),
@ExampleObject(
name = "失败结果",
summary = "错误信息非空时直接将店铺标记为失败",
value = """
{
"shops": [
{
"shopName": "示例店铺",
"error": "紫鸟页面加载超时",
"submissionId": "shop-data-crawl:20363:error"
}
]
}
""")
}))
@Valid @RequestBody ShopDataCrawlSubmitResultRequest request) {
taskService.submitResult(taskId, request);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(
summary = "下载抓取结果 Excel",
description = "校验 resultId 属于当前 user_id 后,从对象存储读取最终 XLSX 并以附件返回。结果文件异步生成,只有列表中的 fileReady=true 后才能下载。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200",
description = "店铺数据抓取结果 Excel 文件",
content = @Content(
mediaType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
schema = @Schema(type = "string", format = "binary"))))
public void download(
@Parameter(description = "结果记录主键", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验结果归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = taskService.resolveResultDownloadUrl(resultId, userId);
String filename = taskService.resolveResultDownloadFilename(resultId, userId);
try {
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream input = URI.create(url).toURL().openStream()) {
input.transferTo(response.getOutputStream());
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
@DeleteMapping("/tasks/{taskId}")
@Operation(
summary = "删除整条抓取任务",
description = "校验任务归属后,删除任务、结果记录、文件生成作业、进度快照和不再被引用的结果文件。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"))
public ApiResponse<Void> deleteTask(
@Parameter(description = "店铺数据抓取任务 ID", required = true, example = "20363")
@PathVariable Long taskId,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验任务归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
taskService.deleteTask(taskId, userId);
return ApiResponse.success(null);
}
@DeleteMapping("/history/{resultId}")
@Operation(
summary = "删除单条抓取记录",
description = "校验结果归属后,删除指定结果及其文件作业和快照,并重新计算父任务状态;任务没有其他结果时会一并删除父任务。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"))
public ApiResponse<Void> deleteHistory(
@Parameter(description = "结果记录主键", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验结果归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
taskService.deleteHistory(resultId, userId);
return ApiResponse.success(null);
}
}
@@ -0,0 +1,8 @@
package com.nanri.aiimage.modules.shopdatacrawl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ShopDataCrawlCountryPrefMapper extends BaseMapper<ShopDataCrawlCountryPrefEntity> {}
@@ -0,0 +1,10 @@
package com.nanri.aiimage.modules.shopdatacrawl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ShopDataCrawlShopCandidateMapper extends BaseMapper<ShopDataCrawlShopCandidateEntity> {
}
@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个国家站点的抓取结果分片")
public class ShopDataCrawlCountryResultDto {
@Schema(
description = "国家站点代码;生成结果文件时仅 DE、FR、ES、IT、UK 会写入对应工作表",
example = "UK",
allowableValues = {"DE", "FR", "ES", "IT", "UK"})
private String country;
@Schema(description = "该国家站点的抓取数据行;多次分片回传时后端会合并并去重")
private List<ShopDataCrawlRowDto> items = new ArrayList<>();
}
@@ -0,0 +1,35 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "创建店铺数据抓取任务请求;每个任务必须且只能处理一个已匹配店铺")
public class ShopDataCrawlCreateTaskRequest {
@NotNull
@JsonProperty("user_id")
@Schema(description = "任务所属用户 ID,必须为正数", example = "10001", requiredMode = Schema.RequiredMode.REQUIRED)
private Long userId;
@Valid
@NotEmpty
@Size(min = 1, max = 1, message = "items 必须且只能包含一个店铺")
@Schema(description = "待抓取店铺列表,必须且只能包含一个有效且已匹配的店铺", requiredMode = Schema.RequiredMode.REQUIRED)
private List<ShopDataCrawlTaskItemDto> items = new ArrayList<>();
@NotEmpty
@JsonProperty("country_codes")
@Schema(
description = "本任务处理的国家代码,必须提交 1-5 个且不得重复,仅支持 DE、FR、ES、IT、UK;列表顺序就是处理顺序。用户未保存偏好时,偏好接口默认返回 UK、DE、FR、ES、IT,但创建任务仍需显式提交本字段",
example = "[\"UK\", \"DE\", \"FR\", \"ES\", \"IT\"]",
allowableValues = {"DE", "FR", "ES", "IT", "UK"},
requiredMode = Schema.RequiredMode.REQUIRED)
private List<String> countryCodes = new ArrayList<>();
}
@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "店铺数据抓取结果行;生成 Excel 时按日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价的固定列顺序写入")
public class ShopDataCrawlRowDto {
@JsonAlias("日期")
@Schema(description = "日期列,按来源文本原样保留", example = "2026-07-25")
private String date;
@JsonAlias("ASIN")
@Schema(description = "亚马逊商品 ASIN", example = "B0CJ8SNXXV")
private String asin;
@JsonAlias({"库存销量", "inventory_sales"})
@Schema(description = "库存销量列,按来源文本原样保留", example = "128")
private String inventorySales;
@JsonAlias({"销售排名", "sales_rank"})
@Schema(description = "销售排名列,按来源文本原样保留", example = "#1,245")
private String salesRank;
@JsonAlias({"页面浏览量", "page_views"})
@Schema(description = "页面浏览量列,按来源文本原样保留", example = "3560")
private String pageViews;
@JsonAlias({"售出件数", "units_sold"})
@Schema(description = "售出件数列,按来源文本原样保留", example = "42")
private String unitsSold;
@JsonAlias("价格")
@Schema(description = "价格列,含货币格式时按来源文本原样保留", example = "GBP 19.99")
private String price;
@JsonAlias({"推荐报价", "recommended_offer"})
@Schema(description = "推荐报价列,含货币格式时按来源文本原样保留", example = "GBP 18.99")
private String recommendedOffer;
}
@@ -0,0 +1,44 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个店铺的一次抓取结果分片;新客户端通过 chunk_index/chunk_total 回传,旧客户端仍可使用 shopDone 增量协议")
public class ShopDataCrawlShopPayloadDto {
@JsonProperty("shopName")
@Schema(description = "店铺名称,必须与任务中的唯一店铺一致;为空时后端使用任务店铺名称", example = "示例英国店")
private String shopName;
@Schema(description = "店铺处理错误信息;只要是非空文本,后端就会忽略 shopDone 并立即将店铺及任务标记失败", example = "紫鸟页面加载超时,未能完成数据抓取")
private String error;
@JsonProperty("countryResults")
@Schema(description = "本分片的国家结果;显式分片暂存到 RustFS,全部到齐后按 chunk_index 顺序合并")
private List<ShopDataCrawlCountryResultDto> countryResults = new ArrayList<>();
@JsonProperty("chunk_index")
@JsonAlias("chunkIndex")
@Schema(description = "分片序号,从 1 开始;必须与 chunk_total 同时传入。不传时沿用旧版 shopDone 增量协议。", example = "1")
private Integer chunkIndex;
@JsonProperty("chunk_total")
@JsonAlias("chunkTotal")
@Schema(description = "该店铺本次结果的分片总数;必须与 chunk_index 同时传入,且同一任务内保持一致。", example = "10")
private Integer chunkTotal;
@JsonProperty("shopDone")
@Schema(description = "旧版增量协议的完成标志。显式传入 chunk_index/chunk_total 时,后端只按分片是否全部到齐判断完成,不会被 shopDone 提前结束。", example = "true")
private Boolean shopDone;
@JsonProperty("submissionId")
@Schema(description = "本次店铺提交批次标识,便于问题排查与日志追踪;不参与业务去重", example = "shop-data-crawl:3089:示例英国店:1784941200000")
private String submissionId;
}
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取结果回传请求;支持同一店铺分多次回传,但每次请求只能包含一个店铺结果")
public class ShopDataCrawlSubmitResultRequest {
@Valid
@NotEmpty
@Schema(description = "本次提交的店铺结果列表,必须且只能包含一个与任务匹配的店铺;中间分片和最后一片均遵循此限制", requiredMode = Schema.RequiredMode.REQUIRED)
private List<ShopDataCrawlShopPayloadDto> shops = new ArrayList<>();
}
@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "批量查询店铺数据抓取任务进度的请求")
public class ShopDataCrawlTaskBatchRequest {
@NotEmpty(message = "taskIds 不能为空")
@Schema(
description = "待查询的任务 ID 列表;后端按输入顺序过滤 null、零和负数,去重后最多处理前 50 个有效 ID",
example = "[12001, 12002, 12001, -1]",
requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds = new ArrayList<>();
}
@@ -0,0 +1,44 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取任务中的单店铺项;创建任务时后端会按店铺名称重新校验匹配信息")
public class ShopDataCrawlTaskItemDto {
@JsonProperty("shopName")
@Schema(description = "店铺名称,是创建任务时重新匹配紫鸟店铺的依据,不得为空", example = "示例英国店", requiredMode = Schema.RequiredMode.REQUIRED)
private String shopName;
@Schema(description = "是否已命中紫鸟店铺索引;创建成功后由后端写为 true", example = "true")
private boolean matched;
@JsonProperty("shopId")
@Schema(description = "紫鸟侧店铺 ID;创建任务时以后端重新匹配到的值为准", example = "3089")
private String shopId;
@Schema(description = "店铺所属平台;创建任务时以后端重新匹配到的值为准", example = "Amazon")
private String platform;
@JsonProperty("companyName")
@Schema(description = "店铺所属公司名称;创建任务时以后端重新匹配到的值为准", example = "示例跨境贸易有限公司")
private String companyName;
@JsonProperty("matchStatus")
@Schema(description = "店铺匹配状态码;创建任务时以后端重新匹配结果为准", example = "MATCHED")
private String matchStatus;
@JsonProperty("matchMessage")
@Schema(description = "店铺匹配状态说明;创建任务时以后端重新匹配结果为准", example = "店铺索引匹配成功")
private String matchMessage;
@JsonProperty("countryCodes")
@Schema(
description = "该店铺实际处理的国家代码及顺序;创建任务后由顶层 country_codes 覆盖",
example = "[\"UK\", \"DE\"]",
allowableValues = {"DE", "FR", "ES", "IT", "UK"})
private List<String> countryCodes = new ArrayList<>();
}
@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_shop_data_crawl_country_pref")
public class ShopDataCrawlCountryPrefEntity {
@TableId(type = IdType.INPUT) private Long userId;
private String countryCodesJson;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_shop_data_crawl_shop_candidate")
public class ShopDataCrawlShopCandidateEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private String shopName;
private LocalDateTime createdAt;
}
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取任务创建结果")
public class ShopDataCrawlCreateTaskVo {
@Schema(description = "新创建的任务 ID,用于轮询进度和回传抓取结果", example = "12001")
private Long taskId;
@Schema(description = "任务创建后的单店铺初始快照;由于每个任务只允许一个店铺,本列表固定包含一项")
private List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "当前用户的店铺数据抓取历史记录")
public class ShopDataCrawlHistoryVo {
@Schema(description = "历史记录项,按创建时间倒序返回,最多返回最近 100 条")
private List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
}
@@ -0,0 +1,97 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个店铺的数据抓取任务结果快照,同时用于任务创建、进度查询和历史记录")
public class ShopDataCrawlResultItemVo {
@Schema(description = "结果记录 ID,用于删除历史记录和下载结果文件", example = "45001")
private Long resultId;
@Schema(description = "所属店铺数据抓取任务 ID", example = "12001")
private Long taskId;
@JsonProperty("shopName")
@Schema(description = "店铺名称", example = "示例英国店")
private String shopName;
@JsonProperty("shopId")
@Schema(description = "紫鸟侧店铺 ID", example = "3089")
private String shopId;
@Schema(description = "店铺所属平台", example = "Amazon")
private String platform;
@JsonProperty("companyName")
@Schema(description = "店铺所属公司名称", example = "示例跨境贸易有限公司")
private String companyName;
@Schema(description = "创建任务时是否成功命中紫鸟店铺索引", example = "true")
private boolean matched;
@JsonProperty("matchStatus")
@Schema(description = "创建任务时保存的店铺匹配状态码", example = "MATCHED")
private String matchStatus;
@JsonProperty("matchMessage")
@Schema(description = "创建任务时保存的店铺匹配说明", example = "店铺索引匹配成功")
private String matchMessage;
@JsonProperty("taskStatus")
@Schema(
description = "任务状态:RUNNING=抓取或结果文件生成中,SUCCESS=任务成功且文件生成完成,FAILED=抓取或任务处理失败",
example = "RUNNING",
allowableValues = {"RUNNING", "SUCCESS", "FAILED"})
private String taskStatus;
@Schema(description = "店铺抓取结果:null=尚未完成,true=抓取成功,false=抓取失败;true 时结果文件仍可能在异步生成", example = "true")
private Boolean success;
@Schema(description = "抓取或任务失败原因;无错误时为空", example = "紫鸟页面加载超时")
private String error;
@Schema(description = "结果记录创建时间,ISO 本地日期时间", example = "2026-07-25T10:00:00")
private LocalDateTime createdAt;
@Schema(description = "任务结束时间;任务仍在运行时为空,ISO 本地日期时间", example = "2026-07-25T10:05:00")
private LocalDateTime finishedAt;
@Schema(description = "结果 Excel 文件名;文件异步生成期间可能已返回文件名,但不能据此判断是否可下载", example = "店铺数据抓取-示例英国店-12001.xlsx")
private String outputFilename;
@Schema(description = "兼容保留的下载地址字段,当前接口通常返回空;fileReady=true 后应通过结果下载接口获取文件")
private String downloadUrl;
@Schema(description = "异步结果文件生成任务 ID;尚未创建文件任务时为空", example = "88001")
private Long fileJobId;
@Schema(
description = "异步结果文件生成状态;为空表示尚未创建文件任务,PENDING=等待生成,RUNNING=生成中,SUCCESS=生成成功,FAILED=生成失败",
example = "PENDING",
allowableValues = {"PENDING", "RUNNING", "SUCCESS", "FAILED"})
private String fileStatus;
@Schema(description = "异步结果文件生成失败原因;文件任务未失败时为空", example = "上传结果文件失败")
private String fileError;
@Schema(description = "结果文件是否已生成并可下载;只有 true 时才能调用结果下载接口", example = "false")
private Boolean fileReady;
@JsonProperty("countryCodes")
@Schema(
description = "任务实际处理的国家代码,顺序与创建任务时提交的 country_codes 一致",
example = "[\"UK\", \"DE\"]",
allowableValues = {"DE", "FR", "ES", "IT", "UK"})
private List<String> countryCodes = new ArrayList<>();
@JsonProperty("countryResults")
@Schema(description = "按国家累计合并后的抓取结果;中间分片阶段可能只包含部分国家或部分数据行")
private List<ShopDataCrawlCountryResultDto> countryResults = new ArrayList<>();
}
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取任务的批量进度查询结果")
public class ShopDataCrawlTaskBatchVo {
@Schema(description = "查询到的店铺结果快照,按规范化后的 taskIds 顺序返回;每个任务当前只包含一个店铺结果")
private List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
@Schema(description = "未找到任务或任务没有结果记录的任务 ID,顺序与规范化后的请求顺序一致", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}
@@ -0,0 +1,110 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
public class ShopDataCrawlExcelAssemblyService {
static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
static final List<String> HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
public void writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
XSSFWorkbook workbook = new XSSFWorkbook(input);
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
for (int i = 0; i < COUNTRIES.size(); i++) {
writeSheet(workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)));
}
workbook.write(output);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("生成店铺数据抓取 Excel 失败: " + ex.getMessage());
}
}
public int countRows(List<ShopDataCrawlResultItemVo> items) {
return rowsByCountry(items).values().stream().mapToInt(List::size).sum();
}
void validateTemplate(XSSFWorkbook workbook) {
if (workbook.getNumberOfSheets() != SHEETS.size()) {
throw new BusinessException("店铺数据抓取模板工作表数量不正确");
}
for (int i = 0; i < SHEETS.size(); i++) {
Sheet sheet = workbook.getSheetAt(i);
if (!SHEETS.get(i).equals(sheet.getSheetName())) {
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
}
Row header = sheet.getRow(0);
for (int column = 0; column < HEADERS.size(); column++) {
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
if (!HEADERS.get(column).equals(actual)) {
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
}
}
}
}
private void writeSheet(Sheet sheet, List<ShopDataCrawlRowDto> rows) {
Row styleRow = sheet.getRow(1);
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
Cell cell = styleRow == null ? null : styleRow.getCell(column);
styles[column] = cell == null ? null : cell.getCellStyle();
}
int last = sheet.getLastRowNum();
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
Row row = sheet.getRow(rowIndex);
if (row != null) {
sheet.removeRow(row);
}
}
int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
Row row = sheet.createRow(rowIndex++);
String[] values = {value.getDate(), value.getAsin(), value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
}
}
private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>();
COUNTRIES.forEach(country -> result.put(country, new ArrayList<>()));
if (items == null) return result;
for (ShopDataCrawlResultItemVo item : items) {
if (item == null || Boolean.FALSE.equals(item.getSuccess()) || item.getCountryResults() == null) continue;
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) {
String country = countryResult == null || countryResult.getCountry() == null ? "" : countryResult.getCountry().trim().toUpperCase();
if (result.containsKey(country) && countryResult.getItems() != null) result.get(country).addAll(countryResult.getItems());
}
}
return result;
}
}
@@ -0,0 +1,313 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity;
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
import com.nanri.aiimage.modules.shopkey.service.ShopManageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
@Service
@RequiredArgsConstructor
public class ShopDataCrawlResolveService {
/**
* 默认国家偏好顺序:英国 -> 德国 -> 法国 -> 西班牙 -> 意大利,与前端和结果模板保持一致。
*/
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("UK", "DE", "FR", "ES", "IT");
private final ShopDataCrawlShopCandidateMapper candidateMapper;
private final ShopDataCrawlCountryPrefMapper countryPrefMapper;
private final ObjectMapper objectMapper;
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final ShopManageService shopManageService;
public List<ProductRiskCandidateVo> listCandidates(Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
}
List<ShopDataCrawlShopCandidateEntity> rows = candidateMapper.selectList(
new LambdaQueryWrapper<ShopDataCrawlShopCandidateEntity>()
.eq(ShopDataCrawlShopCandidateEntity::getUserId, userId)
.orderByDesc(ShopDataCrawlShopCandidateEntity::getId));
List<ProductRiskCandidateVo> list = new ArrayList<>();
for (ShopDataCrawlShopCandidateEntity row : rows) {
if (row == null) {
continue;
}
ProductRiskCandidateVo vo = new ProductRiskCandidateVo();
vo.setId(row.getId());
vo.setShopName(row.getShopName());
vo.setCreatedAt(row.getCreatedAt());
list.add(vo);
}
return list;
}
public long countCandidates(Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
Long count = candidateMapper.selectCount(new LambdaQueryWrapper<ShopDataCrawlShopCandidateEntity>()
.eq(ShopDataCrawlShopCandidateEntity::getUserId, userId));
return count == null ? 0L : count;
}
@Transactional
public ProductRiskCandidateVo addCandidate(ProductRiskCandidateAddRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法");
}
String normalized = ziniaoShopSwitchService.normalizeShopName(request.getShopName());
if (normalized.isBlank()) {
throw new BusinessException("店铺名不能为空");
}
shopManageService.requireShopByName(normalized);
ZiniaoShopMatchResultVo indexHit = ziniaoShopSwitchService.findIndexedStoreByName(normalized, false);
if (indexHit == null || !indexHit.isMatched()) {
String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank()
? indexHit.getMatchMessage()
: "店铺索引未命中,无法加入待处理列表";
throw new BusinessException(hint);
}
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) {
throw new BusinessException(indexHit.getMatchMessage() != null ? indexHit.getMatchMessage() : "存在多个同名店铺,请人工确认");
}
ShopDataCrawlShopCandidateEntity existing = candidateMapper.selectOne(
new LambdaQueryWrapper<ShopDataCrawlShopCandidateEntity>()
.eq(ShopDataCrawlShopCandidateEntity::getUserId, request.getUserId())
.eq(ShopDataCrawlShopCandidateEntity::getShopName, normalized)
.last("LIMIT 1"));
if (existing != null) {
ProductRiskCandidateVo vo = new ProductRiskCandidateVo();
vo.setId(existing.getId());
vo.setShopName(existing.getShopName());
vo.setCreatedAt(existing.getCreatedAt());
return vo;
}
ShopDataCrawlShopCandidateEntity entity = new ShopDataCrawlShopCandidateEntity();
entity.setUserId(request.getUserId());
entity.setShopName(normalized);
entity.setCreatedAt(LocalDateTime.now());
candidateMapper.insert(entity);
ProductRiskCandidateVo vo = new ProductRiskCandidateVo();
vo.setId(entity.getId());
vo.setShopName(entity.getShopName());
vo.setCreatedAt(entity.getCreatedAt());
return vo;
}
@Transactional
public void deleteCandidate(Long userId, Long id) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
}
if (id == null || id <= 0) {
throw new BusinessException("id 不合法");
}
ShopDataCrawlShopCandidateEntity row = candidateMapper.selectById(id);
if (row == null || !userId.equals(row.getUserId())) {
throw new BusinessException("记录不存在");
}
candidateMapper.deleteById(id);
}
public ProductRiskMatchShopsVo matchShops(ProductRiskMatchShopsRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法");
}
LinkedHashSet<String> ordered = new LinkedHashSet<>();
for (String raw : request.getShopNames()) {
String n = ziniaoShopSwitchService.normalizeShopName(raw);
if (!n.isBlank()) {
ordered.add(n);
}
}
if (ordered.isEmpty()) {
throw new BusinessException("shop_names 没有有效店铺名");
}
ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo();
for (String shopName : ordered) {
vo.getItems().add(matchOneShop(shopName));
}
return vo;
}
public ZiniaoShopMatchResultVo requireMatchedShop(String shopName) {
String normalized = ziniaoShopSwitchService.normalizeShopName(shopName);
shopManageService.requireShopByName(normalized);
ZiniaoShopMatchResultVo matched = ziniaoShopSwitchService.findIndexedStoreByName(normalized, false);
if (matched == null || !matched.isMatched()) {
throw new BusinessException(matched != null && matched.getMatchMessage() != null
? matched.getMatchMessage() : "店铺索引未命中");
}
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(matched.getMatchStatus())) {
throw new BusinessException(matched.getMatchMessage() == null ? "存在多个同名店铺,请人工确认" : matched.getMatchMessage());
}
return matched;
}
public ProductRiskCountryPreferenceVo getCountryPreference(Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
}
ShopDataCrawlCountryPrefEntity row = countryPrefMapper.selectById(userId);
ProductRiskCountryPreferenceVo vo = new ProductRiskCountryPreferenceVo();
if (row == null || row.getCountryCodesJson() == null || row.getCountryCodesJson().isBlank()) {
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
return vo;
}
try {
List<String> parsed = objectMapper.readValue(row.getCountryCodesJson(), new TypeReference<List<String>>() {
});
vo.getCountryCodes().addAll(sanitizeStoredCodes(parsed));
} catch (Exception ex) {
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
}
return vo;
}
@Transactional
public ProductRiskCountryPreferenceVo saveCountryPreference(ProductRiskCountryPreferenceSaveRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法");
}
List<String> normalized = validateCountryCodesForSave(request.getCountryCodes());
String json;
try {
json = objectMapper.writeValueAsString(normalized);
} catch (Exception ex) {
throw new BusinessException("保存偏好失败");
}
ShopDataCrawlCountryPrefEntity row = countryPrefMapper.selectById(request.getUserId());
if (row == null) {
row = new ShopDataCrawlCountryPrefEntity();
row.setUserId(request.getUserId());
row.setCountryCodesJson(json);
countryPrefMapper.insert(row);
} else {
row.setCountryCodesJson(json);
countryPrefMapper.updateById(row);
}
ProductRiskCountryPreferenceVo vo = new ProductRiskCountryPreferenceVo();
vo.getCountryCodes().addAll(normalized);
return vo;
}
private static List<String> sanitizeStoredCodes(List<String> raw) {
List<String> parsed = parseValidCountryCodes(raw);
if (parsed.isEmpty()) {
return new ArrayList<>(DEFAULT_COUNTRY_PREFERENCE_ORDER);
}
return parsed;
}
private static List<String> validateCountryCodesForSave(List<String> raw) {
if (raw == null || raw.isEmpty()) {
throw new BusinessException("country_codes 至少选择 1 个国家");
}
LinkedHashSet<String> seen = new LinkedHashSet<>();
List<String> out = new ArrayList<>();
for (String s : raw) {
if (s == null || s.isBlank()) {
throw new BusinessException("country_codes 含空项");
}
String u = s.trim().toUpperCase();
try {
ProductRiskCountryCode.valueOf(u);
} catch (IllegalArgumentException ex) {
throw new BusinessException("非法国家代码: " + s);
}
if (!seen.add(u)) {
throw new BusinessException("country_codes 存在重复: " + u);
}
out.add(u);
}
if (out.size() > ProductRiskCountryCode.values().length) {
throw new BusinessException("country_codes 最多 5 项");
}
return out;
}
public List<String> validateCountryCodes(List<String> raw) {
return validateCountryCodesForSave(raw);
}
/**
* 仅保留合法国家代码,去重并保持顺序;非法值静默忽略,用于读取库存量数据。
*/
private static List<String> parseValidCountryCodes(List<String> raw) {
if (raw == null || raw.isEmpty()) {
return new ArrayList<>();
}
LinkedHashSet<String> seen = new LinkedHashSet<>();
List<String> out = new ArrayList<>();
for (String s : raw) {
if (s == null || s.isBlank()) {
continue;
}
String u = s.trim().toUpperCase();
try {
ProductRiskCountryCode.valueOf(u);
} catch (IllegalArgumentException ex) {
continue;
}
if (seen.add(u)) {
out.add(u);
}
}
return out;
}
private ProductRiskShopQueueItemVo matchOneShop(String shopName) {
ProductRiskShopQueueItemVo item = new ProductRiskShopQueueItemVo();
item.setShopName(shopName);
try {
shopManageService.requireShopByName(shopName);
ZiniaoShopMatchResultVo m = ziniaoShopSwitchService.findIndexedStoreByName(shopName, false);
if (m == null) {
item.setMatched(false);
item.setMatchStatus("PENDING");
item.setMatchMessage("匹配结果为空");
return item;
}
item.setMatched(m.isMatched());
item.setShopId(m.getShopId());
item.setPlatform(m.getPlatform());
item.setCompanyName(m.getCompanyName());
item.setMatchedUserId(m.getMatchedUserId());
item.setMatchStatus(m.getMatchStatus());
item.setMatchMessage(m.getMatchMessage());
} catch (BusinessException ex) {
item.setMatched(false);
item.setMatchStatus("PENDING");
item.setMatchMessage(ex.getMessage());
} catch (Exception ex) {
item.setMatched(false);
item.setMatchStatus("PENDING");
item.setMatchMessage(Objects.toString(ex.getMessage(), "匹配异常"));
}
return item;
}
}
@@ -0,0 +1,226 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class ShopDataCrawlTaskCacheService {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public ShopDataCrawlShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopDataCrawlShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, ShopDataCrawlShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, ShopDataCrawlShopPayloadDto> getAllShopMergedPayload(Long taskId) {
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ShopDataCrawlShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public long countShopMergedPayload(Long taskId) {
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public void touchTaskHeartbeat(Long taskId) {
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
if (task == null || task.getId() == null) {
return;
}
long now = System.currentTimeMillis();
taskEntityLocalCache.put(task.getId(), new LocalTaskEntityCacheEntry(
now,
objectMapper.convertValue(task, FileTaskEntity.class)
));
try {
stringRedisTemplate.opsForValue().set(
buildTaskEntityKey(task.getId()),
objectMapper.writeValueAsString(task),
Duration.ofHours(PAYLOAD_TTL_HOURS)
);
} catch (Exception ignored) {
}
}
public Map<Long, FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
Map<Long, FileTaskEntity> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
java.util.List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
long now = System.currentTimeMillis();
java.util.List<Long> missingIds = new ArrayList<>();
for (Long taskId : normalized) {
LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId);
if (isLocalCacheFresh(cached, now)) {
result.put(taskId, objectMapper.convertValue(cached.task(), FileTaskEntity.class));
} else {
missingIds.add(taskId);
}
}
if (missingIds.isEmpty()) {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
if (val == null || val.isBlank()) {
continue;
}
try {
FileTaskEntity task = objectMapper.readValue(val, FileTaskEntity.class);
result.put(taskId, task);
taskEntityLocalCache.put(taskId, new LocalTaskEntityCacheEntry(now, task));
} catch (Exception ignored) {
}
}
return result;
}
private String buildTaskHeartbeatKey(Long taskId) {
return "shop-data-crawl:task:heartbeat:" + taskId;
}
private String buildTaskEntityKey(Long taskId) {
return "shop-data-crawl:task:entity:" + taskId;
}
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
return cached != null
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());
}
private record LocalTaskEntityCacheEntry(long cachedAtMillis, FileTaskEntity task) {}
}
@@ -14,7 +14,7 @@ public class SimilarAsinHistoryItemVo {
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/similar_asin/xxx/17-result.xlsx")
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "https://oss.aishufu.top/nanri-ai-images/result/similar_asin/xxx/17-result.xlsx")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
@@ -4027,6 +4027,10 @@ public class SimilarAsinTaskService {
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
TaskProgressSnapshotEntity existing = taskProgressSnapshotService.find(task.getId(), MODULE_TYPE);
int displayPercent = calculateDisplayProgressPercent(safeCompleted, safeTotal, job, LocalDateTime.now());
boolean terminalProgress = isTerminalFileBuildProgress(task.getStatus(), safeCompleted, safeTotal);
if (terminalProgress) {
displayPercent = 100;
}
int previousDisplayPercent = calculateSnapshotDisplayPercent(existing, job);
if (previousDisplayPercent > displayPercent && safeCompleted < safeTotal) {
displayPercent = previousDisplayPercent;
@@ -4034,7 +4038,7 @@ public class SimilarAsinTaskService {
taskProgressSnapshotService.save(
task.getId(),
MODULE_TYPE,
STATUS_RUNNING,
terminalProgress ? task.getStatus() : STATUS_RUNNING,
safeTotal,
safeCompleted,
0,
@@ -4044,6 +4048,12 @@ public class SimilarAsinTaskService {
);
}
static boolean isTerminalFileBuildProgress(String taskStatus, int completed, int total) {
return total > 0
&& completed >= total
&& (STATUS_SUCCESS.equals(taskStatus) || STATUS_FAILED.equals(taskStatus));
}
private int calculateDisplayProgressPercent(int current,
int total,
TaskFileJobEntity job,
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Dns;
@@ -39,6 +40,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
@@ -102,12 +104,14 @@ public class SimilarAsinImageEmbedder {
private final int downloadPoolSize;
private final OkHttpClient httpClient;
private final ExecutorService downloadPool;
private final OssStorageService ossStorageService;
public SimilarAsinImageEmbedder(SimilarAsinProperties properties) {
public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) {
int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds();
int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize();
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null");
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
@@ -178,8 +182,9 @@ public class SimilarAsinImageEmbedder {
if (taskImageCache.containsKey(url)) {
return;
}
byte[] raw = doFetch(url);
ResizedImage thumb = resizeImage(url, raw);
String downloadUrl = normalizeAndValidateDownloadUrl(url);
byte[] raw = doFetch(downloadUrl);
ResizedImage thumb = resizeImage(downloadUrl, raw);
taskImageCache.putIfAbsent(url, thumb);
} catch (Exception ex) {
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
@@ -345,8 +350,8 @@ public class SimilarAsinImageEmbedder {
}
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
validateHttpsUrl(url);
List<String> candidates = downloadCandidates(url);
String downloadUrl = normalizeAndValidateDownloadUrl(url);
List<String> candidates = downloadCandidates(downloadUrl);
IOException last = null;
TimeoutException lastTimeout = null;
long waitSeconds = downloadTimeoutSeconds * 2L;
@@ -355,9 +360,9 @@ public class SimilarAsinImageEmbedder {
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
try {
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
if (!attemptUrl.equals(url)) {
if (!attemptUrl.equals(downloadUrl)) {
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
url, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
downloadUrl, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
}
return bytes;
} catch (java.util.concurrent.ExecutionException ee) {
@@ -396,6 +401,12 @@ public class SimilarAsinImageEmbedder {
throw last != null ? last : new IOException("image download failed without cause");
}
String normalizeAndValidateDownloadUrl(String url) {
String downloadUrl = ossStorageService.normalizeManagedPublicUrl(url);
validateHttpsUrl(downloadUrl);
return downloadUrl;
}
static List<String> downloadCandidates(String url) {
List<String> single = List.of(url);
URI uri;
@@ -95,7 +95,7 @@ public class TaskFileJobService {
List<TaskFileJobEntity> ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH"))
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.like(TaskFileJobEntity::getScopeKey, ownerMarker)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + safeLimit));
@@ -108,7 +108,7 @@ public class TaskFileJobService {
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.and(wrapper -> wrapper
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH"))
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.or()
.isNull(TaskFileJobEntity::getScopeKey)
.or()
@@ -265,6 +265,11 @@ public class TaskFileJobService {
return taskFileJobMapper.selectById(jobId);
}
public boolean isRetryExhausted(Long jobId) {
TaskFileJobEntity job = findById(jobId);
return job != null && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT;
}
public void markFailed(TaskFileJobEntity job, String message) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
@@ -12,6 +12,8 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
@@ -41,6 +43,7 @@ public class TaskHeartbeatService {
private static final String MODULE_SHOP_MATCH = "SHOP_MATCH";
private static final String MODULE_PATROL_DELETE = "PATROL_DELETE";
private static final String MODULE_QUERY_ASIN = "QUERY_ASIN";
private static final String MODULE_SHOP_DATA_CRAWL = "SHOP_DATA_CRAWL";
private static final String MODULE_WITHDRAW = "WITHDRAW";
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
@@ -54,6 +57,8 @@ public class TaskHeartbeatService {
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
private final QueryAsinTaskCacheService queryAsinTaskCacheService;
private final ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService;
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
private final WithdrawTaskCacheService withdrawTaskCacheService;
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
@@ -106,6 +111,9 @@ public class TaskHeartbeatService {
if (PublishTaskService.MODULE_TYPE.equals(moduleType)) {
publishTaskService.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat");
}
if (MODULE_SHOP_DATA_CRAWL.equals(moduleType)) {
shopDataCrawlTaskService.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat");
}
String status = task.getStatus();
if (!STATUS_RUNNING.equals(status)) {
log.warn("[task-heartbeat] file task is not running taskId={} actualUserId={} moduleType={} status={}",
@@ -179,6 +187,7 @@ public class TaskHeartbeatService {
case MODULE_QUERY_ASIN -> {
queryAsinTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.touchTaskHeartbeat(taskId);
case MODULE_WITHDRAW -> {
withdrawTaskCacheService.touchTaskHeartbeat(taskId);
}
@@ -216,6 +225,7 @@ public class TaskHeartbeatService {
case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task);
case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task);
case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task);
case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.saveTaskCache(task);
case MODULE_WITHDRAW -> withdrawTaskCacheService.saveTaskCache(task);
case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task);
default -> {
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
@@ -44,6 +45,7 @@ public class TaskResultFileJobWorker {
private final ProductRiskTaskService productRiskTaskService;
private final PublishTaskService publishTaskService;
private final QueryAsinTaskService queryAsinTaskService;
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
private final WithdrawTaskService withdrawTaskService;
private final PatrolDeleteTaskService patrolDeleteTaskService;
private final AppearancePatentTaskService appearancePatentTaskService;
@@ -192,6 +194,9 @@ public class TaskResultFileJobWorker {
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
taskFileJobService.markFailed(job, message);
if ("SHOP_DATA_CRAWL".equals(job.getModuleType()) && taskFileJobService.isRetryExhausted(job.getId())) {
shopDataCrawlTaskService.handleResultFileJobFailure(job, message);
}
}
}
@@ -225,7 +230,8 @@ public class TaskResultFileJobWorker {
}
return "APPEARANCE_PATENT".equals(job.getModuleType())
|| "SIMILAR_ASIN".equals(job.getModuleType())
|| "PUBLISH".equals(job.getModuleType());
|| "PUBLISH".equals(job.getModuleType())
|| "SHOP_DATA_CRAWL".equals(job.getModuleType());
}
private boolean isOwnedByCurrentInstance(TaskFileJobEntity job) {
@@ -273,6 +279,10 @@ public class TaskResultFileJobWorker {
queryAsinTaskService.processResultFileJob(job);
return true;
}
if ("SHOP_DATA_CRAWL".equals(moduleType)) {
shopDataCrawlTaskService.processResultFileJob(job);
return true;
}
if ("WITHDRAW".equals(moduleType)) {
withdrawTaskService.processResultFileJob(job);
return true;
@@ -304,6 +314,11 @@ public class TaskResultFileJobWorker {
private void cleanupAfterSuccess(TaskFileJobEntity job) {
String moduleType = job.getModuleType();
if ("SHOP_DATA_CRAWL".equals(moduleType)) {
taskResultPayloadService.deleteLatest(job.getTaskId(), moduleType, job.getScopeKey());
shopDataCrawlTaskService.cleanupResultFileJob(job);
return;
}
if ("SHOP_MATCH".equals(moduleType)
|| "PRICE_TRACK".equals(moduleType)
|| "PRODUCT_RISK_RESOLVE".equals(moduleType)
@@ -321,6 +336,10 @@ public class TaskResultFileJobWorker {
similarAsinTaskService.cleanupResultFileJob(job);
return;
}
if ("PUBLISH".equals(moduleType)) {
publishTaskService.cleanupResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job);
return;
@@ -7,14 +7,15 @@
SPRING_PROFILES_ACTIVE=local
AIIMAGE_INSTANCE_ID=local-121
AIIMAGE_SERVER_PORT=18080
AIIMAGE_INTERNAL_TOKEN=change-me-local-internal-token
AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
AIIMAGE_DB_USERNAME=change-me
AIIMAGE_DB_PASSWORD=change-me
AIIMAGE_OSS_REGION=us-east-1
AIIMAGE_OSS_ENDPOINT=http://47.110.241.161:9000
AIIMAGE_OSS_PUBLIC_ENDPOINT=http://47.110.241.161:9000
AIIMAGE_OSS_ENDPOINT=https://oss.aishufu.top
AIIMAGE_OSS_PUBLIC_ENDPOINT=https://oss.aishufu.top
AIIMAGE_OSS_BUCKET=nanri-ai-images
AIIMAGE_IMAGE_VIDEO_OSS_BUCKET=shufu-video
AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET=nanri-ai-digital-human
@@ -85,8 +85,8 @@ aiimage:
request-body-cache-limit-bytes: ${AIIMAGE_INSTANCE_ROUTE_REQUEST_BODY_CACHE_LIMIT_BYTES:104857600}
oss:
region: ${AIIMAGE_OSS_REGION:us-east-1}
endpoint: ${AIIMAGE_OSS_ENDPOINT:http://47.110.241.161:9000}
public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:http://47.110.241.161:9000}
endpoint: ${AIIMAGE_OSS_ENDPOINT:https://oss.aishufu.top}
public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:https://oss.aishufu.top}
bucket: ${AIIMAGE_OSS_BUCKET:nanri-ai-images}
image-video-bucket: ${AIIMAGE_IMAGE_VIDEO_OSS_BUCKET:shufu-video}
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
@@ -153,7 +153,7 @@ aiimage:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,SHOP_DATA_CRAWL,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
permission-schema-init:
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
task-pressure:
@@ -247,6 +247,7 @@ aiimage:
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
internal-token-file: ${AIIMAGE_INTERNAL_TOKEN_FILE:}
auth:
jwt-secret: ${AIIMAGE_JWT_SECRET:please-change-this-secret-please-rotate-at-least-32-bytes}
jwt-ttl-hours: ${AIIMAGE_AUTH_JWT_TTL_HOURS:168}
@@ -0,0 +1,29 @@
CREATE TABLE IF NOT EXISTS `biz_shop_data_crawl_shop_candidate` (
`id` BIGINT NOT NULL AUTO_INCREMENT,
`user_id` BIGINT NOT NULL,
`shop_name` VARCHAR(255) NOT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uk_shop_data_crawl_candidate_user_shop` (`user_id`, `shop_name`),
KEY `idx_shop_data_crawl_candidate_user` (`user_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `biz_shop_data_crawl_country_pref` (
`user_id` BIGINT NOT NULL,
`country_codes_json` VARCHAR(256) NOT NULL,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '店铺数据抓取', 'shop_data_crawl', 'app', 'shop-data-crawl', 139
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'shop_data_crawl'
);
UPDATE `columns`
SET `name` = '店铺数据抓取',
`menu_type` = 'app',
`route_path` = 'shop-data-crawl',
`sort_order` = 139
WHERE `column_key` = 'shop_data_crawl';
@@ -0,0 +1,220 @@
-- Hierarchical menu permissions. user_column_permission intentionally keeps
-- only direct grants; descendants are expanded by PermissionMenuService.
SET @permission_db_name = DATABASE();
SET @permission_parent_column_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @permission_db_name
AND TABLE_NAME = 'columns'
AND COLUMN_NAME = 'parent_id'
);
SET @permission_sql = IF(
@permission_parent_column_exists = 0,
'ALTER TABLE `columns` ADD COLUMN `parent_id` INT NULL COMMENT ''直接父菜单 ID'' AFTER `id`',
'SELECT 1'
);
PREPARE permission_stmt FROM @permission_sql;
EXECUTE permission_stmt;
DEALLOCATE PREPARE permission_stmt;
SET @permission_parent_index_exists = (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @permission_db_name
AND TABLE_NAME = 'columns'
AND INDEX_NAME = 'idx_columns_parent_sort_id'
);
SET @permission_sql = IF(
@permission_parent_index_exists = 0,
'ALTER TABLE `columns` ADD INDEX `idx_columns_parent_sort_id` (`parent_id`, `sort_order`, `id`)',
'SELECT 1'
);
PREPARE permission_stmt FROM @permission_sql;
EXECUTE permission_stmt;
DEALLOCATE PREPARE permission_stmt;
SET @permission_parent_fk_exists = (
SELECT COUNT(*)
FROM information_schema.TABLE_CONSTRAINTS
WHERE CONSTRAINT_SCHEMA = @permission_db_name
AND TABLE_NAME = 'columns'
AND CONSTRAINT_NAME = 'fk_columns_parent_id'
AND CONSTRAINT_TYPE = 'FOREIGN KEY'
);
SET @permission_sql = IF(
@permission_parent_fk_exists = 0,
'ALTER TABLE `columns` ADD CONSTRAINT `fk_columns_parent_id` FOREIGN KEY (`parent_id`) REFERENCES `columns` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT',
'SELECT 1'
);
PREPARE permission_stmt FROM @permission_sql;
EXECUTE permission_stmt;
DEALLOCATE PREPARE permission_stmt;
-- Normalize older underscore keys. If both spellings already exist, retain the
-- canonical row and move direct grants before deleting the duplicate.
INSERT IGNORE INTO `user_column_permission` (`user_id`, `column_id`)
SELECT direct_grant.`user_id`, canonical.`id`
FROM `user_column_permission` AS direct_grant
JOIN `columns` AS legacy ON legacy.`id` = direct_grant.`column_id`
JOIN (
SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key
UNION ALL SELECT 'appearance_patent', 'appearance-patent'
UNION ALL SELECT 'similar_asin', 'similar-asin'
) AS key_map ON key_map.legacy_key = legacy.`column_key`
JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key;
DELETE legacy
FROM `columns` AS legacy
JOIN (
SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key
UNION ALL SELECT 'appearance_patent', 'appearance-patent'
UNION ALL SELECT 'similar_asin', 'similar-asin'
) AS key_map ON key_map.legacy_key = legacy.`column_key`
JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key;
UPDATE `columns` AS legacy
JOIN (
SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key
UNION ALL SELECT 'appearance_patent', 'appearance-patent'
UNION ALL SELECT 'similar_asin', 'similar-asin'
) AS key_map ON key_map.legacy_key = legacy.`column_key`
LEFT JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key
SET legacy.`column_key` = key_map.canonical_key
WHERE canonical.`id` IS NULL;
-- Video/image workspace. Keep the legacy `wb` key because existing user
-- grants reference that row; its route is the current image-video page.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '视频', 'wb', 'app', 'image-video', 100, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'wb');
UPDATE `columns`
SET `name` = '视频', `menu_type` = 'app', `route_path` = 'image-video', `sort_order` = 100, `parent_id` = NULL
WHERE `column_key` = 'wb';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '数字人', 'digital-human', 'app', 'digital-human', 101,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'digital-human');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '带货视频', 'delivery-video', 'app', 'delivery-video', 102,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'delivery-video');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '混剪', 'mix-video', 'app', 'mix-video', 103,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'mix-video');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '图片', 'image', 'app', 'image', 104,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'image');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'wb'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'digital-human' THEN '数字人' WHEN 'delivery-video' THEN '带货视频'
WHEN 'mix-video' THEN '混剪' WHEN 'image' THEN '图片' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'digital-human' THEN 'digital-human' WHEN 'delivery-video' THEN 'delivery-video'
WHEN 'mix-video' THEN 'mix-video' WHEN 'image' THEN 'image' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'digital-human' THEN 101 WHEN 'delivery-video' THEN 102
WHEN 'mix-video' THEN 103 WHEN 'image' THEN 104 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('digital-human', 'delivery-video', 'mix-video', 'image');
-- Three APP navigation groups.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '前端工具', 'brand_front_tools', 'app', 'brand-front-tools', 110, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_front_tools');
UPDATE `columns` SET `name` = '前端工具', `menu_type` = 'app', `route_path` = 'brand-front-tools', `sort_order` = 110, `parent_id` = NULL
WHERE `column_key` = 'brand_front_tools';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '运营工具', 'brand_operation_tools', 'app', 'brand-operation-tools', 120, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_operation_tools');
UPDATE `columns` SET `name` = '运营工具', `menu_type` = 'app', `route_path` = 'brand-operation-tools', `sort_order` = 120, `parent_id` = NULL
WHERE `column_key` = 'brand_operation_tools';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '后勤工具', 'brand_logistics_tools', 'app', 'brand-logistics-tools', 130, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_logistics_tools');
UPDATE `columns` SET `name` = '后勤工具', `menu_type` = 'app', `route_path` = 'brand-logistics-tools', `sort_order` = 130, `parent_id` = NULL
WHERE `column_key` = 'brand_logistics_tools';
-- APP children. The key follows the frontend item key (except the existing
-- shop_data_crawl compatibility key); route_path remains the route slug.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '采集数据', 'collect-data', 'app', 'collect-data', 111,
(SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'collect-data');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_front_tools'
SET child.`name` = '采集数据', child.`menu_type` = 'app', child.`route_path` = 'collect-data',
child.`sort_order` = 111, child.`parent_id` = parent.`id`
WHERE child.`column_key` = 'collect-data';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '品牌检测', 'brand', 'app', 'brand', 112, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'brand');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '外观专利检测', 'appearance-patent', 'app', 'appearance-patent', 113, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'appearance-patent');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '货源查询', 'similar-asin', 'app', 'similar-asin', 114, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'similar-asin');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '数据去重', 'dedupe', 'app', 'dedupe', 115, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'dedupe');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '数据拆分', 'split', 'app', 'split', 116, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'split');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '格式转换', 'convert', 'app', 'convert', 117, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'convert');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_front_tools'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'brand' THEN '品牌检测' WHEN 'appearance-patent' THEN '外观专利检测'
WHEN 'similar-asin' THEN '货源查询' WHEN 'dedupe' THEN '数据去重'
WHEN 'split' THEN '数据拆分' WHEN 'convert' THEN '格式转换' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'brand' THEN 'brand' WHEN 'appearance-patent' THEN 'appearance-patent'
WHEN 'similar-asin' THEN 'similar-asin' WHEN 'dedupe' THEN 'dedupe'
WHEN 'split' THEN 'split' WHEN 'convert' THEN 'convert' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'brand' THEN 112 WHEN 'appearance-patent' THEN 113 WHEN 'similar-asin' THEN 114
WHEN 'dedupe' THEN 115 WHEN 'split' THEN 116 WHEN 'convert' THEN 117 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('brand', 'appearance-patent', 'similar-asin', 'dedupe', 'split', 'convert');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '上架', 'publish', 'app', 'publish', 121, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'publish');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '删除ASIN', 'delete-brand', 'app', 'delete-brand', 122, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'delete-brand');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '商品风险解决', 'product-risk', 'app', 'product-risk', 123, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'product-risk');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '定时匹配', 'shop-match', 'app', 'shop-match', 124, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'shop-match');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '跟价', 'pricing', 'app', 'price-track', 125, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'pricing');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '巡店删除', 'patrol-delete', 'app', 'patrol-delete', 126, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'patrol-delete');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '查询ASIN', 'query-asin', 'app', 'query-asin', 127, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'query-asin');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '店铺数据抓取', 'shop_data_crawl', 'app', 'shop-data-crawl', 128, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'shop_data_crawl');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '取款', 'withdraw', 'app', 'withdraw', 129, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'withdraw');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_operation_tools'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'publish' THEN '上架' WHEN 'delete-brand' THEN '删除ASIN'
WHEN 'product-risk' THEN '商品风险解决' WHEN 'shop-match' THEN '定时匹配'
WHEN 'pricing' THEN '跟价' WHEN 'patrol-delete' THEN '巡店删除'
WHEN 'query-asin' THEN '查询ASIN' WHEN 'shop_data_crawl' THEN '店铺数据抓取'
WHEN 'withdraw' THEN '取款' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'publish' THEN 'publish' WHEN 'delete-brand' THEN 'delete-brand'
WHEN 'product-risk' THEN 'product-risk' WHEN 'shop-match' THEN 'shop-match'
WHEN 'pricing' THEN 'price-track' WHEN 'patrol-delete' THEN 'patrol-delete'
WHEN 'query-asin' THEN 'query-asin' WHEN 'shop_data_crawl' THEN 'shop-data-crawl'
WHEN 'withdraw' THEN 'withdraw' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'publish' THEN 121 WHEN 'delete-brand' THEN 122 WHEN 'product-risk' THEN 123 WHEN 'shop-match' THEN 124
WHEN 'pricing' THEN 125 WHEN 'patrol-delete' THEN 126 WHEN 'query-asin' THEN 127 WHEN 'shop_data_crawl' THEN 128 WHEN 'withdraw' THEN 129 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('publish', 'delete-brand', 'product-risk', 'shop-match', 'pricing', 'patrol-delete', 'query-asin', 'shop_data_crawl', 'withdraw');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '采购', 'purchase', 'app', 'purchase', 131, (SELECT id FROM `columns` WHERE column_key = 'brand_logistics_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'purchase');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT 'ERP', 'erp', 'app', 'erp', 132, (SELECT id FROM `columns` WHERE column_key = 'brand_logistics_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'erp');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_logistics_tools'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'purchase' THEN '采购' WHEN 'erp' THEN 'ERP' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'purchase' THEN 'purchase' WHEN 'erp' THEN 'erp' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'purchase' THEN 131 WHEN 'erp' THEN 132 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('purchase', 'erp');
@@ -0,0 +1,2 @@
ALTER TABLE biz_publish_item
MODIFY COLUMN status_value TEXT NULL COMMENT 'status column';
@@ -0,0 +1,99 @@
package com.nanri.aiimage.modules.admin.service;
import com.nanri.aiimage.modules.admin.model.dto.AdminUserCreateRequest;
import com.nanri.aiimage.modules.admin.model.dto.AdminUserUpdateRequest;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.auth.util.WerkzeugPasswordEncoder;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class AdminUserServiceTest {
@Test
void createUserStoresSuppliedDirectPermissionsInSameTransactionalMethod() throws Exception {
AdminUserMapper userMapper = mock(AdminUserMapper.class);
WerkzeugPasswordEncoder passwordEncoder = mock(WerkzeugPasswordEncoder.class);
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService permissionService = mock(PermissionMenuService.class);
AdminUserService service = new AdminUserService(
userMapper, passwordEncoder, authSupport, permissionService);
AdminUserEntity operator = user(10L, "admin", 1, null);
AdminUserCreateRequest request = new AdminUserCreateRequest();
request.setUsername("created-user");
request.setPassword("secret1");
request.setRole("normal");
request.setColumnIds(List.of(11L, 12L));
when(authSupport.currentRole(operator)).thenReturn("admin");
when(passwordEncoder.hash("secret1")).thenReturn("hashed");
when(userMapper.insert(any(AdminUserEntity.class))).thenAnswer(invocation -> {
invocation.<AdminUserEntity>getArgument(0).setId(20L);
return 1;
});
Long createdId = service.createUser(operator, request);
assertThat(createdId).isEqualTo(20L);
ArgumentCaptor<UserColumnPermissionUpdateRequest> permissions =
ArgumentCaptor.forClass(UserColumnPermissionUpdateRequest.class);
verify(permissionService).updateUserColumnPermissions(
eq(operator), eq(20L), permissions.capture(), eq(PermissionMenuService.MENU_TYPE_APP));
assertThat(permissions.getValue().getColumnIds()).containsExactly(11L, 12L);
assertThat(AdminUserService.class
.getMethod("createUser", AdminUserEntity.class, AdminUserCreateRequest.class)
.isAnnotationPresent(Transactional.class)).isTrue();
}
@Test
void updateUserCanReplaceOnlyDirectPermissionsTransactionally() throws Exception {
AdminUserMapper userMapper = mock(AdminUserMapper.class);
WerkzeugPasswordEncoder passwordEncoder = mock(WerkzeugPasswordEncoder.class);
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService permissionService = mock(PermissionMenuService.class);
AdminUserService service = new AdminUserService(
userMapper, passwordEncoder, authSupport, permissionService);
AdminUserEntity operator = user(10L, "admin", 1, null);
AdminUserEntity target = user(20L, "normal", 0, 10L);
AdminUserUpdateRequest request = new AdminUserUpdateRequest();
request.setColumnIds(List.of(21L));
when(authSupport.currentRole(operator)).thenReturn("admin");
when(userMapper.selectById(20L)).thenReturn(target);
service.updateUser(operator, 20L, request);
verify(userMapper, never()).update(any(), any());
ArgumentCaptor<UserColumnPermissionUpdateRequest> permissions =
ArgumentCaptor.forClass(UserColumnPermissionUpdateRequest.class);
verify(permissionService).updateUserColumnPermissions(
eq(operator), eq(20L), permissions.capture(), eq(PermissionMenuService.MENU_TYPE_APP));
assertThat(permissions.getValue().getColumnIds()).containsExactly(21L);
assertThat(AdminUserService.class
.getMethod("updateUser", AdminUserEntity.class, Long.class, AdminUserUpdateRequest.class)
.isAnnotationPresent(Transactional.class)).isTrue();
}
private AdminUserEntity user(Long id, String role, int isAdmin, Long createdById) {
AdminUserEntity user = new AdminUserEntity();
user.setId(id);
user.setRole(role);
user.setIsAdmin(isAdmin);
user.setCreatedById(createdById);
return user;
}
}
@@ -0,0 +1,52 @@
package com.nanri.aiimage.modules.admin.support;
import com.nanri.aiimage.modules.auth.config.AuthProperties;
import com.nanri.aiimage.modules.auth.service.JwtService;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
class AdminAuthSupportTest {
@Test
void explicitLowestIdAdminIsNotPromotedToSuperAdmin() {
AdminUserMapper userMapper = mock(AdminUserMapper.class);
AdminAuthSupport support = new AdminAuthSupport(
mock(JwtService.class), userMapper, mock(AuthProperties.class));
AdminUserEntity user = user(1L, "admin", 1, null);
assertThat(support.currentRole(user)).isEqualTo("admin");
verify(userMapper, never()).selectOne(any());
}
@Test
void legacyBlankRoleRootRemainsSuperAdmin() {
AdminAuthSupport support = new AdminAuthSupport(
mock(JwtService.class), mock(AdminUserMapper.class), mock(AuthProperties.class));
assertThat(support.currentRole(user(1L, null, 1, null))).isEqualTo("super_admin");
}
@Test
void explicitNormalRoleIsNotPromotedByLegacyAdminFields() {
AdminAuthSupport support = new AdminAuthSupport(
mock(JwtService.class), mock(AdminUserMapper.class), mock(AuthProperties.class));
assertThat(support.currentRole(user(1L, "normal", 1, null))).isNull();
}
private AdminUserEntity user(Long id, String role, int isAdmin, Long createdById) {
AdminUserEntity user = new AdminUserEntity();
user.setId(id);
user.setRole(role);
user.setIsAdmin(isAdmin);
user.setCreatedById(createdById);
return user;
}
}
@@ -0,0 +1,91 @@
package com.nanri.aiimage.modules.file.service.oss;
import com.nanri.aiimage.config.OssProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
class OssStorageServiceTest {
private OssStorageService storageService;
@BeforeEach
void setUp() {
OssProperties properties = new OssProperties();
properties.setEndpoint("https://oss.aishufu.top");
properties.setPublicEndpoint("https://oss.aishufu.top");
properties.setBucket("nanri-ai-images");
properties.setImageVideoBucket("shufu-video");
properties.setDigitalHumanBucket("nanri-ai-digital-human");
properties.setAccessKeyId("test-access-key");
properties.setAccessKeySecret("test-secret-key");
storageService = new OssStorageService(properties);
}
@Test
void normalizesLegacyPathStyleUrlToCurrentPublicEndpoint() {
assertEquals(
"https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg",
storageService.normalizeManagedPublicUrl(
"http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg"));
}
@Test
void keepsCurrentUrlAndNormalizesVirtualHostUrl() {
String current = "https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg";
assertEquals(current, storageService.normalizeManagedPublicUrl(current));
assertEquals(
current,
storageService.normalizeManagedPublicUrl(
"https://nanri-ai-images.oss.aishufu.top/supply_images/main.jpg"));
}
@Test
void supportsAllConfiguredBuckets() {
assertEquals(
"https://oss.aishufu.top/shufu-video/result/image_video/demo.mp4",
storageService.normalizeManagedPublicUrl(
"http://47.110.241.161:9000/shufu-video/result/image_video/demo.mp4"));
assertEquals(
"https://oss.aishufu.top/nanri-ai-digital-human/digital-human/versions/demo.mp4",
storageService.normalizeManagedPublicUrl(
"https://nanri-ai-digital-human.oss.aishufu.top/digital-human/versions/demo.mp4"));
}
@Test
void leavesExternalImageUrlsUnchanged() {
List<String> externalUrls = List.of(
"https://m.media-amazon.com/nanri-ai-images/main.jpg?size=large",
"https://cbu01.alicdn.com/img/ibank/main.jpg",
"https://lf9-bot-platform-tos-sign.coze.cn/file.jpg?x-expires=1&x-signature=abc",
"https://nanri-ai-images.evil.example/supply_images/main.jpg");
for (String url : externalUrls) {
assertEquals(url, storageService.normalizeManagedPublicUrl(url));
}
}
@Test
void leavesInvalidOrUnmanagedValuesUnchanged() {
List<String> values = List.of(
"http://47.110.241.161:9001/nanri-ai-images/supply_images/main.jpg",
"http://47.110.241.161.evil.example:9000/nanri-ai-images/supply_images/main.jpg",
"http://47.110.241.161:9000/nanri-ai-images/../other-bucket/main.jpg",
"http://47.110.241.161:9000/nanri-ai-images/%2e%2e/other-bucket/main.jpg",
"https://oss.aishufu.top/not-a-bucket/main.jpg",
"https://oss.aishufu.top/nanri-ai-images",
"supply_images/main.jpg",
"/nanri-ai-images/supply_images/main.jpg",
"not a url");
for (String value : values) {
assertEquals(value, storageService.normalizeManagedPublicUrl(value));
}
assertNull(storageService.normalizeManagedPublicUrl(null));
assertEquals(" ", storageService.normalizeManagedPublicUrl(" "));
}
}
@@ -142,6 +142,7 @@ class ImageVideoAsyncTaskServiceTest {
assertEquals("FAILED", task.getStatus());
assertEquals("FAIL", task.getCozeStatus());
assertEquals("model unavailable", task.getErrorMessage());
assertTrue(task.getResultJson().contains("model unavailable"));
verify(archiveService).capturePollResponse(task, cozeResult);
verify(taskMapper).updateById(task);
@@ -294,7 +295,7 @@ class ImageVideoAsyncTaskServiceTest {
task.setSubmittedAt(LocalDateTime.now().minusHours(2));
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"Output\":\"https://example.com/result.mp4\"}")));
"output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"video_url\\\":\\\"https://example.com/result.mp4\\\"}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1);
@@ -309,6 +310,145 @@ class ImageVideoAsyncTaskServiceTest {
assertNotNull(task.getCompletedAt());
}
@Test
void successfulWorkflowWithEmptyOutputVideoUrlIsMarkedFailedEvenWhenInputVideoExists() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoArchiveService archiveService = mock(ImageVideoArchiveService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
archiveService,
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingWorkflowTask();
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"video_info", Map.of("video_url", "https://example.com/input-reference.mp4"),
"output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"video_url\\\":\\\"\\\"}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1);
when(taskMapper.selectById(98L)).thenReturn(task);
when(workflowConfigService.imageVideoWorkflowId()).thenReturn("workflow-1");
when(cozeService.getWorkflowResult(1L, "workflow-1", "exec-98")).thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("FAILED", task.getStatus());
assertEquals("SUCCESS", task.getCozeStatus());
assertEquals("Coze workflow succeeded but video_url is empty", task.getErrorMessage());
assertNotNull(task.getCompletedAt());
assertTrue(task.getResultJson().contains("video_url"));
verify(archiveService, never()).enrichCompletedTask(any(), any());
verify(taskMapper).updateById(task);
}
@Test
void successfulVoiceListWithNestedBusinessErrorIsMarkedFailed() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoArchiveService archiveService = mock(ImageVideoArchiveService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
archiveService,
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingVoiceListTask();
String error = "Query voices failed: invalid API key (status: 1004)";
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":null,\\\"error\\\":\\\"" + error + "\\\",\\\"system_voice\\\":[],\\\"voice_cloning\\\":[]}}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1);
when(taskMapper.selectById(163L)).thenReturn(task);
when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow");
when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec"))
.thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("FAILED", task.getStatus());
assertEquals("SUCCESS", task.getCozeStatus());
assertEquals(error, task.getErrorMessage());
assertTrue(task.getResultJson().contains("status: 1004"));
verify(archiveService, never()).enrichCompletedTask(any(), any());
verify(taskMapper).updateById(task);
}
@Test
void successfulVoiceListWithBlankBusinessErrorRemainsSuccessful() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
mock(ImageVideoArchiveService.class),
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingVoiceListTask();
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":{\\\"status_code\\\":0,\\\"status_msg\\\":\\\"success\\\"},\\\"error\\\":\\\"\\\",\\\"system_voice\\\":[]}}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1);
when(taskMapper.selectById(163L)).thenReturn(task);
when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow");
when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec"))
.thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("SUCCESS", task.getStatus());
assertEquals("SUCCESS", task.getCozeStatus());
assertEquals(null, task.getErrorMessage());
verify(taskMapper).updateById(task);
}
@Test
void successfulVoiceListWithNonzeroBusinessStatusIsMarkedFailed() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
mock(ImageVideoArchiveService.class),
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingVoiceListTask();
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":{\\\"status_code\\\":1004,\\\"status_msg\\\":\\\"invalid API key\\\"},\\\"system_voice\\\":[]}}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1);
when(taskMapper.selectById(163L)).thenReturn(task);
when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow");
when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec"))
.thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("FAILED", task.getStatus());
assertEquals("invalid API key (status_code: 1004)", task.getErrorMessage());
verify(taskMapper).updateById(task);
}
private ImageVideoAsyncTaskEntity waitingDouyinTask() {
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
task.setId(79L);
@@ -331,6 +471,15 @@ class ImageVideoAsyncTaskServiceTest {
return task;
}
private ImageVideoAsyncTaskEntity waitingVoiceListTask() {
ImageVideoAsyncTaskEntity task = waitingDouyinTask();
task.setId(163L);
task.setUserId(665L);
task.setTaskType("VOICE_LIST");
task.setCozeExecuteId("voice-list-exec");
return task;
}
private InstanceMetadata instanceMetadata(String instanceId) {
return new InstanceMetadata(instanceId);
}
@@ -0,0 +1,251 @@
package com.nanri.aiimage.modules.permission.controller;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.dto.ImageVideoDataPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.test.util.ReflectionTestUtils;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class PermissionMenuControllerTest {
@Test
void effectivePermissionEndpointAllowsAuthenticatedNormalUserToReadSelf() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
AdminUserEntity currentUser = new AdminUserEntity();
currentUser.setId(9L);
currentUser.setRole("normal");
PermissionMenuItemVo item = new PermissionMenuItemVo();
item.setId(1L);
when(authSupport.requireUser(request)).thenReturn(currentUser);
when(service.getUserColumnPermissions(currentUser, 9L, "app")).thenReturn(List.of(item));
var response = controller.getUserColumnPermissions(request, 9L, "app");
assertThat(response.getData()).extracting(PermissionMenuItemVo::getId).containsExactly(1L);
verify(authSupport).requireUser(request);
verify(authSupport, never()).requireAdmin(request);
}
@Test
void internalProxyAllowsDatabaseBackedNormalUserToReadOwnEffectivePermissions() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
AdminUserEntity currentUser = new AdminUserEntity();
currentUser.setId(9L);
currentUser.setRole("normal");
when(authSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录"));
when(request.getHeader("X-Internal-Token")).thenReturn("shared-token");
when(request.getParameter("operatorId")).thenReturn("9");
when(service.requireUserOperator(9L)).thenReturn(currentUser);
when(service.getUserColumnPermissions(currentUser, 9L, "app")).thenReturn(List.of());
controller.getUserColumnPermissions(request, 9L, "app");
verify(service).requireUserOperator(9L);
verify(service, never()).requireAdminOperator(anyLong());
verify(service).getUserColumnPermissions(currentUser, 9L, "app");
}
@Test
void directIdEndpointStillRequiresAdminForSelf() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
BusinessException authFailure = new BusinessException(403, "需要管理员权限");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
assertThatThrownBy(() -> controller.getUserColumnIds(request, 9L, "app"))
.isSameAs(authFailure);
verify(authSupport, never()).requireUser(request);
verifyNoInteractions(service);
}
@Test
void directIdUpdateForwardsMenuTypeScope() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
AdminUserEntity operator = new AdminUserEntity();
operator.setId(1L);
operator.setRole("super_admin");
UserColumnPermissionUpdateRequest body = new UserColumnPermissionUpdateRequest();
body.setColumnIds(List.of(12L));
when(authSupport.requireAdmin(request)).thenReturn(operator);
controller.updateUserColumnIds(request, 9L, "app", body);
verify(service).updateUserColumnPermissions(eq(operator), eq(9L), eq(body), eq("app"));
}
@Test
void internalProxyUsesSharedTokenAndDatabaseBackedOperator() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
AdminUserEntity operator = new AdminUserEntity();
operator.setId(12L);
operator.setRole("admin");
when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "未登录"));
when(request.getHeader("X-Internal-Token")).thenReturn("shared-token");
when(request.getParameter("operatorId")).thenReturn("12");
when(service.requireAdminOperator(12L)).thenReturn(operator);
when(service.list("app")).thenReturn(List.of());
controller.listMenus(request, "app");
verify(service).requireAdminOperator(12L);
verify(service).list("app");
verify(request, never()).getParameter("superAdmin");
}
@Test
void internalProxyRejectsMismatchedSharedToken() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
BusinessException authFailure = new BusinessException(401, "未登录");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
when(request.getHeader("X-Internal-Token")).thenReturn("wrong-token");
assertThatThrownBy(() -> controller.listMenus(request, "app"))
.isSameAs(authFailure);
verify(service, never()).requireAdminOperator(anyLong());
verify(service, never()).list("app");
}
@Test
void internalProxyUsesSharedTokenFileWhenEnvironmentTokenIsUnconfigured(@TempDir Path tempDir) throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "");
Path tokenFile = tempDir.resolve("internal-token");
Files.writeString(tokenFile, "file-token\n");
ReflectionTestUtils.setField(controller, "internalTokenFile", tokenFile.toString());
AdminUserEntity operator = new AdminUserEntity();
operator.setId(12L);
operator.setRole("admin");
when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "未登录"));
when(request.getHeader("X-Internal-Token")).thenReturn("file-token");
when(request.getParameter("operatorId")).thenReturn("12");
when(service.requireAdminOperator(12L)).thenReturn(operator);
when(service.list("admin")).thenReturn(List.of());
controller.listMenus(request, "admin");
verify(service).requireAdminOperator(12L);
verify(service).list("admin");
}
@Test
void internalProxyRejectsMissingSharedTokenFileWhenEnvironmentTokenIsUnconfigured(@TempDir Path tempDir) {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "");
ReflectionTestUtils.setField(controller, "internalTokenFile", tempDir.resolve("missing-token").toString());
BusinessException authFailure = new BusinessException(401, "未登录");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
when(request.getHeader("X-Internal-Token")).thenReturn("guessed-token");
when(request.getRemoteAddr()).thenReturn("127.0.0.1");
when(request.getParameter("operatorId")).thenReturn("12");
assertThatThrownBy(() -> controller.listMenus(request, "admin"))
.isSameAs(authFailure);
verify(service, never()).requireAdminOperator(anyLong());
verify(service, never()).list("admin");
}
@Test
void configuredInternalTokenTakesPrecedenceOverTokenFile(@TempDir Path tempDir) throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
Path tokenFile = tempDir.resolve("internal-token");
Files.writeString(tokenFile, "file-token\n");
ReflectionTestUtils.setField(controller, "internalToken", "configured-token");
ReflectionTestUtils.setField(controller, "internalTokenFile", tokenFile.toString());
BusinessException authFailure = new BusinessException(401, "未登录");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
when(request.getHeader("X-Internal-Token")).thenReturn("file-token");
when(request.getParameter("operatorId")).thenReturn("12");
assertThatThrownBy(() -> controller.listMenus(request, "admin"))
.isSameAs(authFailure);
verify(service, never()).requireAdminOperator(anyLong());
verify(service, never()).list("admin");
}
@Test
void imageVideoPermissionEndpointsDelegateAuthenticatedOperator() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
AdminUserEntity operator = new AdminUserEntity();
operator.setId(1L);
operator.setRole("super_admin");
ImageVideoDataPermissionUserVo permissionUser = new ImageVideoDataPermissionUserVo();
permissionUser.setId(20L);
ImageVideoDataPermissionUpdateRequest body = new ImageVideoDataPermissionUpdateRequest();
body.setUserIds(List.of(20L));
when(authSupport.requireAdmin(request)).thenReturn(operator);
when(service.listImageVideoDataPermissionUsers(operator)).thenReturn(List.of(permissionUser));
when(service.updateImageVideoDataPermissionUsers(operator, List.of(20L))).thenReturn(1);
var listResponse = controller.listImageVideoDataPermissionUsers(request);
var updateResponse = controller.updateImageVideoDataPermissionUsers(request, body);
assertThat(listResponse.getData()).containsExactly(permissionUser);
assertThat(updateResponse.getData()).isEqualTo(1);
verify(authSupport, times(2)).requireAdmin(request);
verify(service).listImageVideoDataPermissionUsers(operator);
verify(service).updateImageVideoDataPermissionUsers(operator, List.of(20L));
}
}
@@ -1,20 +1,33 @@
package com.nanri.aiimage.modules.permission.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.mapper.PermissionMenuMapper;
import com.nanri.aiimage.modules.permission.mapper.UserColumnPermissionMapper;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import org.junit.jupiter.api.Test;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -61,10 +74,505 @@ class PermissionMenuServiceTest {
verify(permissionMapper, times(0)).insert(any(UserColumnPermissionEntity.class));
}
@Test
void expandsDirectParentGrantToDescendantsWithoutPersistingChildren() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11),
menu(3L, 2L, "app", 12),
menu(4L, null, "app", 13)));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(9L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId)
.containsExactly(1L, 2L, 3L);
verify(permissionMapper, never()).insert(any(UserColumnPermissionEntity.class));
}
@Test
void directIdEndpointDoesNotReturnExpandedChildren() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11)));
UserColumnIdsVo direct = service.getUserColumnIds(9L, "app");
assertThat(direct.getColumnIds()).containsExactly(1L);
}
@Test
void leafOnlyEffectivePermissionIncludesRootKeyWithoutGrantingAncestor() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 2L)));
PermissionMenuEntity root = menu(1L, null, "app", 1);
root.setColumnKey("brand_front_tools");
PermissionMenuEntity leaf = menu(2L, 1L, "app", 2);
leaf.setColumnKey("brand");
when(menuMapper.selectList(any())).thenReturn(List.of(root, leaf));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(9L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId).containsExactly(2L);
assertThat(effective.getFirst().getRootColumnKey()).isEqualTo("brand_front_tools");
}
@Test
void menuCatalogIncludesRootKeyFromLoadedHierarchy() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
PermissionMenuEntity root = menu(1L, null, "app", 1);
root.setColumnKey("brand_front_tools");
PermissionMenuEntity leaf = menu(2L, 1L, "app", 2);
leaf.setColumnKey("brand");
when(menuMapper.selectList(any())).thenReturn(List.of(root, leaf));
List<PermissionMenuItemVo> result = service.list("app");
assertThat(result).extracting(PermissionMenuItemVo::getId).containsExactly(1L, 2L);
assertThat(result).extracting(PermissionMenuItemVo::getRootColumnKey)
.containsExactly("brand_front_tools", "brand_front_tools");
}
@Test
void superAdminDirectIdEndpointReturnsOnlyPersistedIds() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(1L)).thenReturn(user(1L, "super_admin", 1));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 2L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
UserColumnIdsVo direct = service.getUserColumnIds(1L, "app");
assertThat(direct.getColumnIds()).containsExactly(2L);
}
@Test
void superAdminEffectivePermissionsRemainVirtualFullCatalog() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(1L)).thenReturn(user(1L, "super_admin", 1));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(1L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId).containsExactly(1L, 2L);
verify(permissionMapper, never()).selectList(any());
}
@Test
void normalUserCanReadOwnEffectivePermissions() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity currentUser = user(9L, "normal", 0);
when(userMapper.selectById(9L)).thenReturn(currentUser);
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11)));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(currentUser, 9L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId)
.containsExactly(1L, 2L);
}
@Test
void normalUserCannotReadAnotherUsersEffectivePermissions() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity currentUser = user(9L, "normal", 0);
when(userMapper.selectById(10L)).thenReturn(user(10L, "normal", 0));
assertThatThrownBy(() -> service.getUserColumnPermissions(currentUser, 10L, "app"))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("管理员权限");
verify(permissionMapper, never()).selectList(any());
}
@Test
void rejectsParentFromAnotherMenuType() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
PermissionMenuEntity parent = menu(7L, null, "admin", 1);
when(menuMapper.selectById(7L)).thenReturn(parent);
PermissionMenuCreateRequest request = createRequest(7L, "app");
assertThatThrownBy(() -> service.create(request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("同一类型");
verify(menuMapper, never()).insert(any(PermissionMenuEntity.class));
}
@Test
void rejectsParentCycleWhenUpdatingMenu() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
PermissionMenuEntity current = menu(1L, null, "app", 1);
PermissionMenuEntity parent = menu(2L, 3L, "app", 2);
PermissionMenuEntity ancestor = menu(3L, 1L, "app", 3);
when(menuMapper.selectById(1L)).thenReturn(current);
when(menuMapper.selectById(2L)).thenReturn(parent);
when(menuMapper.selectById(3L)).thenReturn(ancestor);
PermissionMenuUpdateRequest request = new PermissionMenuUpdateRequest();
request.setName("changed");
request.setColumnKey("changed");
request.setMenuType("app");
request.setRoutePath("changed");
request.setParentId(2L);
assertThatThrownBy(() -> service.update(1L, request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("自己的子级");
verify(menuMapper, never()).updateById(any(PermissionMenuEntity.class));
}
@Test
void rejectsChangingMenuTypeWhileChildrenRemain() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(menuMapper.selectById(1L)).thenReturn(menu(1L, null, "app", 1));
when(menuMapper.selectCount(any())).thenReturn(1L);
PermissionMenuUpdateRequest request = new PermissionMenuUpdateRequest();
request.setName("changed");
request.setColumnKey("changed");
request.setMenuType("admin");
request.setRoutePath("changed");
assertThatThrownBy(() -> service.update(1L, request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("子菜单");
verify(menuMapper, never()).updateById(any(PermissionMenuEntity.class));
}
@Test
void refusesDeletingParentMenuUntilChildrenAreRemoved() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(menuMapper.selectById(1L)).thenReturn(menu(1L, null, "app", 1));
when(menuMapper.selectCount(any())).thenReturn(1L);
assertThatThrownBy(() -> service.delete(1L))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("子菜单");
verify(menuMapper, never()).deleteById(any(Long.class));
verify(permissionMapper, never()).delete(any());
}
@Test
void ordinaryAdminCannotAssignMenuOutsideOwnEffectiveSet() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(10L, "admin", 1);
AdminUserEntity target = user(20L, "normal", 0);
target.setCreatedById(10L);
when(userMapper.selectById(20L)).thenReturn(target);
when(userMapper.selectById(10L)).thenReturn(operator);
when(userMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectList(any())).thenReturn(List.of(menu(1L, null, "app", 1)));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
assertThatThrownBy(() -> service.updateUserColumnPermissions(operator, 20L, request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("自己已有");
verify(permissionMapper, never()).delete(any());
}
@Test
void ordinaryAdminCanAssignDescendantGrantedThroughParent() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(10L, "admin", 1);
AdminUserEntity target = user(20L, "normal", 0);
target.setCreatedById(10L);
when(userMapper.selectById(20L)).thenReturn(target);
when(userMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
service.updateUserColumnPermissions(operator, 20L, request);
ArgumentCaptor<UserColumnPermissionEntity> captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(captor.capture());
assertThat(captor.getValue().getColumnId()).isEqualTo(2L);
}
@Test
void ordinaryAdminReplacementPreservesDirectGrantsOutsideEffectiveScope() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(10L, "admin", 1);
AdminUserEntity target = user(20L, "normal", 0);
target.setCreatedById(10L);
when(userMapper.selectById(20L)).thenReturn(target);
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, null, "app", 2)));
when(permissionMapper.selectList(any()))
.thenReturn(List.of(grant(10L, 1L)), List.of(grant(20L, 2L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(1L));
service.updateUserColumnPermissions(operator, 20L, request);
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper, times(2)).insert(inserted.capture());
assertThat(inserted.getAllValues())
.extracting(UserColumnPermissionEntity::getColumnId)
.containsExactly(1L, 2L);
}
@Test
void replacementStoresOnlyDirectRequestedParentId() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(1L));
service.updateUserColumnPermissions(9L, request);
ArgumentCaptor<UserColumnPermissionEntity> captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(captor.capture());
assertThat(captor.getValue().getColumnId()).isEqualTo(1L);
}
@Test
void appScopedReplacementDeletesOnlyAppDirectGrants() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(1L, "super_admin", 1);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(11L, null, "app", 1),
menu(12L, 11L, "app", 2)));
when(menuMapper.selectOne(any())).thenReturn(null);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(12L));
service.updateUserColumnPermissions(operator, 9L, request, PermissionMenuService.MENU_TYPE_APP);
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
UserColumnPermissionEntity.class);
ArgumentCaptor<LambdaUpdateWrapper<UserColumnPermissionEntity>> deleted =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(permissionMapper).delete(deleted.capture());
assertThat(deleted.getValue().getSqlSegment()).contains("column_id", "IN");
assertThat(deleted.getValue().getParamNameValuePairs().values())
.contains(9L, 11L, 12L);
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(inserted.capture());
assertThat(inserted.getValue().getColumnId()).isEqualTo(12L);
}
@Test
void explicitLowestIdAdminCannotManageImageVideoDataPermissions() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
assertThatThrownBy(() -> service.listImageVideoDataPermissionUsers(explicitAdmin))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
assertThatThrownBy(() -> service.updateImageVideoDataPermissionUsers(explicitAdmin, List.of(2L)))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
verify(menuMapper, never()).selectOne(any());
verify(permissionMapper, never()).delete(any());
}
@Test
void explicitNormalRoleCannotActAsInternalAdminDespiteLegacyFlag() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(1L)).thenReturn(user(1L, "normal", 1));
assertThatThrownBy(() -> service.requireAdminOperator(1L))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("管理员权限");
}
@Test
void superAdminCanListImageVideoDataPermissionUsers() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(99L, "super_admin", 1);
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
explicitAdmin.setUsername("admin");
AdminUserEntity normal = user(2L, "normal", 0);
normal.setUsername("normal");
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission());
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 75L)));
when(userMapper.selectList(any())).thenReturn(List.of(operator, explicitAdmin, normal));
List<ImageVideoDataPermissionUserVo> result = service.listImageVideoDataPermissionUsers(operator);
assertThat(result).extracting(ImageVideoDataPermissionUserVo::getId).containsExactly(1L, 2L);
assertThat(result.get(0).isGranted()).isTrue();
assertThat(result.get(1).isGranted()).isFalse();
}
@Test
void imageVideoUpdateDeletesOnlyProtectedColumnGrants() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(99L, "super_admin", 1);
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
AdminUserEntity normal = user(2L, "normal", 0);
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission());
when(userMapper.selectList(any())).thenReturn(List.of(operator, explicitAdmin, normal));
int grantedCount = service.updateImageVideoDataPermissionUsers(operator, List.of(2L));
assertThat(grantedCount).isEqualTo(1);
verify(permissionMapper).deleteByMap(Map.of("column_id", 75L));
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(inserted.capture());
assertThat(inserted.getValue().getUserId()).isEqualTo(2L);
assertThat(inserted.getValue().getColumnId()).isEqualTo(75L);
}
private PermissionMenuCreateRequest createRequest(Long parentId, String menuType) {
PermissionMenuCreateRequest request = new PermissionMenuCreateRequest();
request.setName("child");
request.setColumnKey("child");
request.setMenuType(menuType);
request.setRoutePath("child");
request.setParentId(parentId);
return request;
}
private PermissionMenuEntity menu(Long id, Long parentId, String menuType, int sortOrder) {
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setId(id);
entity.setParentId(parentId);
entity.setMenuType(menuType);
entity.setColumnKey("menu-" + id);
entity.setRoutePath("route-" + id);
entity.setSortOrder(sortOrder);
return entity;
}
private UserColumnPermissionEntity grant(Long userId, Long columnId) {
UserColumnPermissionEntity entity = new UserColumnPermissionEntity();
entity.setUserId(userId);
entity.setColumnId(columnId);
return entity;
}
private AdminUserEntity user(Long id, String role, int isAdmin) {
AdminUserEntity entity = new AdminUserEntity();
entity.setId(id);
entity.setRole(role);
entity.setIsAdmin(isAdmin);
return entity;
}
private PermissionMenuEntity imageVideoPermission() {
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setId(75L);
entity.setColumnKey("admin_image_video_tasks");
return entity;
}
private PermissionMenuEntity imageVideoDataPermission() {
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setId(75L);
entity.setColumnKey("admin_image_video_task_data");
return entity;
}
}
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.publish.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -19,10 +20,15 @@ import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import org.apache.ibatis.builder.MapperBuilderAssistant;
@@ -32,6 +38,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
@@ -39,7 +46,12 @@ import org.springframework.transaction.support.TransactionTemplate;
import java.time.LocalDateTime;
import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -68,6 +80,8 @@ class PublishTaskServiceTest {
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@Mock private LocalFileStorageService localFileStorageService;
@@ -77,17 +91,26 @@ class PublishTaskServiceTest {
@Mock private PublishItemMapper publishItemMapper;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private ObjectMapper objectMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private TransactionTemplate transactionTemplate;
@Mock private InstanceMetadata instanceMetadata;
@InjectMocks private PublishTaskService service;
private final List<TaskChunkEntity> storedChunks = new ArrayList<>();
private final List<TaskScopeStateEntity> storedScopes = new ArrayList<>();
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
private int nextPayloadId;
@BeforeEach
void executeTransactionsInline() {
configureChunkStorage();
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(transactionTemplate.execute(any())).thenAnswer(invocation -> {
TransactionCallback<?> callback = invocation.getArgument(0);
@@ -159,6 +182,62 @@ class PublishTaskServiceTest {
verifyNoInteractions(publishItemMapper);
}
@Test
void failedTaskOwnedByAnotherInstanceCanBeDeletedLocally() throws Exception {
long taskId = 110L;
FileTaskEntity task = task(taskId, 7L, "FAILED");
task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}");
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(fileResultMapper.selectList(any())).thenReturn(List.of());
service.deleteTask(taskId, 7L);
verify(taskFileJobService).deleteTaskJobs(taskId, PublishTaskService.MODULE_TYPE);
verify(publishItemMapper).delete(any());
verify(publishFileMapper).delete(any());
verify(fileResultMapper).delete(any());
verify(fileTaskMapper).deleteById(taskId);
verify(objectMapper, never()).readTree(any(String.class));
}
@Test
void runningTaskOwnedByAnotherInstanceStillRequiresRouting() throws Exception {
long taskId = 111L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}");
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(objectMapper.readTree(task.getResultJson())).thenReturn(
new ObjectMapper().readTree(task.getResultJson()));
TaskOwnerMismatchException error = assertThrows(TaskOwnerMismatchException.class,
() -> service.deleteTask(taskId, 7L));
assertEquals(taskId, error.getTaskId());
assertEquals("instance-b", error.getOwnerInstanceId());
verifyNoInteractions(taskFileJobService, publishItemMapper, publishFileMapper, fileResultMapper);
verify(fileTaskMapper, never()).deleteById(taskId);
}
@Test
void historyForFailedTaskOwnedByAnotherInstanceCanBeDeletedLocally() throws Exception {
long taskId = 112L;
long resultId = 212L;
FileTaskEntity task = task(taskId, 7L, "FAILED");
task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}");
FileResultEntity result = new FileResultEntity();
result.setId(resultId);
result.setTaskId(taskId);
result.setModuleType(PublishTaskService.MODULE_TYPE);
when(fileResultMapper.selectById(resultId)).thenReturn(result);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
service.deleteHistory(resultId, 7L);
verify(fileTaskMapper).deleteById(taskId);
verify(objectMapper, never()).readTree(any(String.class));
}
@Test
void resultCallbackRejectsAnotherUserBeforeReadingOrReplacingRows() {
long taskId = 101L;
@@ -258,6 +337,242 @@ class PublishTaskServiceTest {
verify(lock).close();
}
@Test
void resultCallbackMergesOutOfOrderChunksOnlyAfterAllChunksArrive() {
long taskId = 113L;
long fileId = 213L;
long resultId = 313L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "分片.xlsx");
FileResultEntity result = result(taskId, resultId);
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result);
when(publishItemMapper.selectCount(any())).thenReturn(2L);
service.submitResult(taskId, chunkResultRequest(7L, fileId, 2, 2, List.of(row("2"))));
assertEquals("RUNNING", file.getStatus());
assertEquals(1, storedChunks.size());
assertEquals(1, storedScopes.getFirst().getReceivedChunkCount());
assertEquals(0, storedScopes.getFirst().getCompleted());
verify(publishItemMapper, never()).delete(any());
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
assertEquals("SUCCESS", file.getStatus());
assertEquals(2, storedChunks.size());
assertEquals(2, storedScopes.getFirst().getReceivedChunkCount());
assertEquals(1, storedScopes.getFirst().getCompleted());
org.mockito.ArgumentCaptor<List<PublishItemEntity>> rowsCaptor =
org.mockito.ArgumentCaptor.forClass(List.class);
verify(publishItemMapper).insertBatch(rowsCaptor.capture());
assertEquals(List.of("1", "2"), rowsCaptor.getValue().stream()
.map(PublishItemEntity::getSourceId)
.toList());
verify(taskFileJobService).enqueueAssembleResult(
taskId, PublishTaskService.MODULE_TYPE, resultId,
"task:" + taskId + ":owner:instance-a");
}
@Test
void identicalResultChunkRetryIsIdempotent() {
long taskId = 114L;
long fileId = 214L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "重试.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
PublishSubmitResultRequest request = chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")));
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 314L));
service.submitResult(taskId, request);
service.submitResult(taskId, request);
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
verify(publishItemMapper, never()).delete(any());
}
@Test
void resultChunkRetryWithDifferentContentIsRejected() {
long taskId = 115L;
long fileId = 215L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "冲突.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 315L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("different")))));
assertTrue(error.getMessage().contains("不同内容"));
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
}
@Test
void resultChunksRejectChangedChunkTotal() {
long taskId = 116L;
long fileId = 216L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "总片数冲突.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 316L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 2, 3, List.of(row("2")))));
assertTrue(error.getMessage().contains("chunk_total"));
assertEquals(1, storedChunks.size());
}
@Test
void resultChunkRejectsCallbackWhenRustfsIsUnavailable() {
long taskId = 117L;
long fileId = 217L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "RustFS未配置.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(false);
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))));
assertTrue(error.getMessage().contains("RustFS 未配置"));
assertEquals(0, storedChunks.size());
verify(publishItemMapper, never()).delete(any());
}
@Test
void resultChunkRejectsLocalFallbackPointer() {
long taskId = 118L;
long fileId = 218L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "本地回退.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
String localPointer = "local:instance-a/publish/chunk-1";
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(transientPayloadStorageService.storeChunkPayloadVersioned(any(), any(), any(), any(), any()))
.thenReturn(localPointer);
when(transientPayloadStorageService.extractPointer(localPointer)).thenReturn(localPointer);
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))));
assertTrue(error.getMessage().contains("必须写入 RustFS"));
assertEquals(0, storedChunks.size());
verify(transientPayloadStorageService).deletePayloadIfPresent(localPointer);
}
@Test
void resultChunkDeletesRustfsPayloadWhenMetadataInsertFails() {
long taskId = 120L;
long fileId = 220L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "元数据失败.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(taskChunkMapper.insert(any(TaskChunkEntity.class)))
.thenThrow(new IllegalStateException("database unavailable"));
assertThrows(IllegalStateException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))));
assertTrue(rustfsPayloads.isEmpty());
assertTrue(storedChunks.isEmpty());
verify(transientPayloadStorageService).deletePayloadIfPresent(any());
}
@Test
void deletingTaskRemovesPublishChunksAndRustfsPayloads() {
long taskId = 119L;
long fileId = 219L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "待删除.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 319L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
String pointer = storedChunks.getFirst().getPayloadJson();
assertEquals(1, storedChunks.size());
assertEquals(1, storedScopes.size());
assertEquals(1, rustfsPayloads.size());
when(fileResultMapper.selectList(any())).thenReturn(List.of());
service.deleteTask(taskId, 7L);
assertTrue(storedChunks.isEmpty());
assertTrue(storedScopes.isEmpty());
assertTrue(rustfsPayloads.isEmpty());
verify(transientPayloadStorageService).deletePayloadIfPresent(pointer);
verify(fileTaskMapper).deleteById(taskId);
}
@Test
void allFailedTaskRemovesPreviouslyUploadedChunksAfterCommit() {
long taskId = 121L;
long fileId = 221L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "执行失败.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 321L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
assertEquals(1, storedChunks.size());
service.submitResult(taskId, errorResultRequest(7L, fileId, "Python 执行失败"));
assertEquals("FAILED", task.getStatus());
assertEquals("FAILED", file.getStatus());
assertTrue(storedChunks.isEmpty());
assertTrue(storedScopes.isEmpty());
assertTrue(rustfsPayloads.isEmpty());
}
@Test
void activateFileRejectsSecondRunningFileWhileHoldingTaskLock() {
long taskId = 104L;
@@ -346,6 +661,53 @@ class PublishTaskServiceTest {
verify(lock).close();
}
@Test
void staleTaskRecoversReceivedChunksAndEnqueuesAssembly() {
long taskId = 118L;
long fileId = 218L;
long resultId = 318L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
task.setSourceFileCount(1);
task.setUpdatedAt(LocalDateTime.now().minusHours(1));
PublishFileEntity file = file(taskId, fileId, "RUNNING", "部分结果.xlsx");
file.setTotalRows(2);
FileResultEntity result = result(taskId, resultId);
TaskDistributedLockService.LockHandle submitLock = mock(TaskDistributedLockService.LockHandle.class);
TaskDistributedLockService.LockHandle staleLock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(submitLock);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId, 0L)).thenReturn(staleLock);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result);
when(taskFileJobService.countUnfinishedAssembleJobs(taskId, PublishTaskService.MODULE_TYPE))
.thenReturn(0L);
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
task.setUpdatedAt(LocalDateTime.now().minusHours(1));
service.failStaleTasks();
assertEquals("SUCCESS", file.getStatus());
assertEquals(1, file.getTotalRows());
assertEquals(1, file.getProcessedRows());
assertTrue(file.getErrorMessage().contains("已保留 1 行"));
assertEquals(1, task.getSuccessFileCount());
assertEquals(0, task.getFailedFileCount());
org.mockito.ArgumentCaptor<List<PublishItemEntity>> rowsCaptor =
org.mockito.ArgumentCaptor.forClass(List.class);
verify(publishItemMapper).insertBatch(rowsCaptor.capture());
assertEquals(List.of("1"), rowsCaptor.getValue().stream()
.map(PublishItemEntity::getSourceId)
.toList());
verify(taskFileJobService).enqueueAssembleResult(
taskId, PublishTaskService.MODULE_TYPE, resultId,
"task:" + taskId + ":owner:instance-a");
verify(submitLock).close();
verify(staleLock).close();
}
@Test
void staleTaskWithoutSuccessfulFilesFailsTaskAndResult() {
long taskId = 107L;
@@ -386,6 +748,140 @@ class PublishTaskServiceTest {
verify(lock).close();
}
private void configureChunkStorage() {
storedChunks.clear();
storedScopes.clear();
rustfsPayloads.clear();
nextPayloadId = 0;
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
any(), any(), any(), any(), any())).thenAnswer(invocation -> {
String pointer = "rustfs:test/publish/chunk-" + (++nextPayloadId);
rustfsPayloads.put(pointer, invocation.getArgument(4));
return pointer;
});
lenient().when(transientPayloadStorageService.extractPointer(any())).thenAnswer(invocation -> {
String value = invocation.getArgument(0);
return value != null && value.startsWith("rustfs:") ? value : null;
});
lenient().when(transientPayloadStorageService.resolvePayload(any(), any())).thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
String payload = rustfsPayloads.get(pointer);
if (payload == null) {
throw new IllegalStateException("missing test RustFS payload: " + pointer);
}
return payload;
});
lenient().doAnswer(invocation -> {
rustfsPayloads.remove(invocation.getArgument(0));
return null;
}).when(transientPayloadStorageService).deletePayloadIfPresent(any());
lenient().when(taskChunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> {
TaskChunkEntity chunk = invocation.getArgument(0);
chunk.setId((long) storedChunks.size() + 1L);
storedChunks.add(chunk);
return 1;
});
lenient().when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
Integer chunkIndex = queryInteger(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, chunkIndex))
.findFirst()
.orElse(null);
});
lenient().when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.count();
});
lenient().when(taskChunkMapper.selectList(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex))
.toList();
});
lenient().when(taskChunkMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedChunks.size();
storedChunks.removeIf(chunk -> taskId == null || Objects.equals(taskId, chunk.getTaskId()));
return before - storedChunks.size();
});
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
scope.setId((long) storedScopes.size() + 1L);
storedScopes.add(scope);
return 1;
});
lenient().when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedScopes.stream()
.filter(scope -> Objects.equals(taskId, scope.getTaskId()))
.filter(scope -> Objects.equals(scopeHash, scope.getScopeHash()))
.findFirst()
.orElse(null);
});
lenient().when(taskScopeStateMapper.updateById(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedScopes.size();
storedScopes.removeIf(scope -> taskId == null || Objects.equals(taskId, scope.getTaskId()));
return before - storedScopes.size();
});
}
private boolean matchesChunk(TaskChunkEntity chunk,
Long taskId,
String scopeHash,
Integer chunkIndex) {
return (taskId == null || Objects.equals(taskId, chunk.getTaskId()))
&& (scopeHash == null || Objects.equals(scopeHash, chunk.getScopeHash()))
&& (chunkIndex == null || Objects.equals(chunkIndex, chunk.getChunkIndex()));
}
private Long queryLong(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Long.class::isInstance)
.map(Long.class::cast)
.findFirst()
.orElse(null);
}
private Integer queryInteger(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Integer.class::isInstance)
.map(Integer.class::cast)
.findFirst()
.orElse(null);
}
private String queryScopeHash(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.filter(value -> value.length() == 64)
.findFirst()
.orElse(null);
}
private FileTaskEntity task(long taskId, long userId, String status) {
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
@@ -407,11 +903,30 @@ class PublishTaskServiceTest {
return file;
}
private FileResultEntity result(long taskId, long resultId) {
FileResultEntity result = new FileResultEntity();
result.setId(resultId);
result.setTaskId(taskId);
result.setModuleType(PublishTaskService.MODULE_TYPE);
result.setSuccess(0);
return result;
}
private PublishSubmitResultRequest resultRequest(long userId,
long fileId,
List<PublishRowDto> rows) {
long fileId,
List<PublishRowDto> rows) {
return chunkResultRequest(userId, fileId, null, null, rows);
}
private PublishSubmitResultRequest chunkResultRequest(long userId,
long fileId,
Integer chunkIndex,
Integer chunkTotal,
List<PublishRowDto> rows) {
PublishResultFileDto file = new PublishResultFileDto();
file.setFileId(fileId);
file.setChunkIndex(chunkIndex);
file.setChunkTotal(chunkTotal);
file.setRows(rows);
PublishSubmitResultRequest request = new PublishSubmitResultRequest();
request.setUserId(userId);
@@ -419,6 +934,18 @@ class PublishTaskServiceTest {
return request;
}
private PublishSubmitResultRequest errorResultRequest(long userId,
long fileId,
String error) {
PublishResultFileDto file = new PublishResultFileDto();
file.setFileId(fileId);
file.setError(error);
PublishSubmitResultRequest request = new PublishSubmitResultRequest();
request.setUserId(userId);
request.setFiles(List.of(file));
return request;
}
private PublishRowDto row(String sourceId) {
PublishRowDto row = new PublishRowDto();
row.setSourceId(sourceId);
@@ -0,0 +1,56 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
class ShopDataCrawlExcelAssemblyServiceTest {
@TempDir Path tempDir;
@Test
void writesRowsIntoStrictTemplateWithoutChangingSheetOrderOrHeaders() throws Exception {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate("2026-07-25");
row.setAsin("B012345678");
row.setInventorySales("11");
row.setSalesRank("22");
row.setPageViews("33");
row.setUnitsSold("44");
row.setPrice("12.50");
row.setRecommendedOffer("12.00");
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
country.setCountry("UK");
country.setItems(List.of(row));
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
item.setSuccess(true);
item.setCountryResults(List.of(country));
File output = tempDir.resolve("result.xlsx").toFile();
new ShopDataCrawlExcelAssemblyService().writeWorkbook(output, List.of(item));
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS,
java.util.stream.IntStream.range(0, workbook.getNumberOfSheets())
.mapToObj(i -> workbook.getSheetAt(i).getSheetName()).toList());
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
for (int columnIndex = 0; columnIndex < ShopDataCrawlExcelAssemblyService.HEADERS.size(); columnIndex++) {
assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(columnIndex),
workbook.getSheetAt(i).getRow(0).getCell(columnIndex).getStringCellValue());
}
}
assertEquals("2026-07-25", workbook.getSheet("英国").getRow(1).getCell(0).getStringCellValue());
assertEquals("B012345678", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue());
assertEquals(0, workbook.getSheet("德国").getLastRowNum());
}
}
}
@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.modules.shopkey.service.ShopManageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ShopDataCrawlResolveServiceTest {
@Mock com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper candidateMapper;
@Mock com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper countryPrefMapper;
@Mock com.fasterxml.jackson.databind.ObjectMapper objectMapper;
@Mock ZiniaoShopSwitchService ziniaoShopSwitchService;
@Mock ShopManageService shopManageService;
@InjectMocks ShopDataCrawlResolveService service;
@Test
void requiresManagedShopBeforeLookingUpZiniaoIndex() {
ZiniaoShopMatchResultVo match = new ZiniaoShopMatchResultVo();
match.setMatched(true);
when(ziniaoShopSwitchService.normalizeShopName(" Demo ")).thenReturn("Demo");
when(ziniaoShopSwitchService.findIndexedStoreByName("Demo", false)).thenReturn(match);
assertSame(match, service.requireMatchedShop(" Demo "));
var order = inOrder(shopManageService, ziniaoShopSwitchService);
order.verify(ziniaoShopSwitchService).normalizeShopName(" Demo ");
order.verify(shopManageService).requireShopByName("Demo");
order.verify(ziniaoShopSwitchService).findIndexedStoreByName("Demo", false);
}
}
@@ -0,0 +1,539 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ShopDataCrawlTaskServiceChunkTest {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
private static final String SHOP_NAME = "Demo Shop";
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private ShopDataCrawlResolveService shopDataCrawlResolveService;
@Mock private ShopDataCrawlExcelAssemblyService excelAssemblyService;
@Mock private ShopDataCrawlTaskCacheService taskCacheService;
@Mock private OssStorageService ossStorageService;
@Mock private ZiniaoShopSwitchService ziniaoShopSwitchService;
@Mock private TaskPressureProperties taskPressureProperties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskResultItemService taskResultItemService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private InstanceMetadata instanceMetadata;
@Spy private ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
private ShopDataCrawlTaskService service;
private final List<TaskChunkEntity> storedChunks = new ArrayList<>();
private final List<TaskScopeStateEntity> storedScopes = new ArrayList<>();
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
private FileTaskEntity task;
private FileResultEntity result;
private int nextPayloadId;
@BeforeEach
void configureStorage() {
service = new ShopDataCrawlTaskService(
fileTaskMapper,
fileResultMapper,
shopDataCrawlResolveService,
excelAssemblyService,
taskCacheService,
ossStorageService,
ziniaoShopSwitchService,
objectMapper,
taskPressureProperties,
taskFileJobService,
taskResultItemService,
taskProgressSnapshotService,
taskDistributedLockService,
taskChunkMapper,
taskScopeStateMapper,
transientPayloadStorageService,
instanceMetadata);
storedChunks.clear();
storedScopes.clear();
rustfsPayloads.clear();
nextPayloadId = 0;
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(eq(MODULE_TYPE), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(taskCacheService.getTaskCacheBatch(any())).thenReturn(Map.of());
lenient().when(taskCacheService.getAllShopMergedPayload(anyLong())).thenReturn(Map.of());
lenient().when(taskResultItemService.listResultSnapshots(anyLong(), eq(MODULE_TYPE), any())).thenReturn(List.of());
lenient().when(taskFileJobService.findAssembleJob(anyLong(), eq(MODULE_TYPE), anyLong())).thenReturn(null);
lenient().when(excelAssemblyService.countRows(any())).thenReturn(1);
lenient().when(fileTaskMapper.selectById(anyLong())).thenAnswer(invocation -> {
Long taskId = invocation.getArgument(0);
return task != null && Objects.equals(taskId, task.getId()) ? task : null;
});
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
lenient().when(fileResultMapper.selectById(anyLong())).thenAnswer(invocation -> {
Long resultId = invocation.getArgument(0);
return result != null && Objects.equals(resultId, result.getId()) ? result : null;
});
lenient().when(fileResultMapper.selectList(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<FileResultEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
return result != null && Objects.equals(taskId, result.getTaskId()) ? List.of(result) : List.of();
});
lenient().when(fileResultMapper.updateById(any(FileResultEntity.class))).thenReturn(1);
lenient().when(ziniaoShopSwitchService.normalizeShopName(nullable(String.class))).thenAnswer(invocation -> {
String value = invocation.getArgument(0);
return value == null ? "" : value.trim();
});
configureTransientPayloadStorage();
configureChunkMapper();
configureScopeMapper();
}
@Test
void completesChunkedResultWhenSecondChunkArrivesBeforeFirstChunk() {
givenRunningTask(101L, 201L);
service.submitResult(task.getId(), request(chunk(2, 2, "UK", row("2026-07-26", "B002"))));
assertEquals(-1, result.getSuccess());
assertEquals(1, storedChunks.size());
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), eq(MODULE_TYPE), anyLong(), anyString());
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
assertEquals(1, result.getSuccess());
assertEquals(2, storedChunks.size());
assertTrue(task.getResultJson().indexOf("B001") < task.getResultJson().indexOf("B002"));
verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(),
"task:" + task.getId() + ":owner:instance-a");
}
@Test
void identicalResultChunkRetryIsIdempotent() {
givenRunningTask(102L, 202L);
ShopDataCrawlSubmitResultRequest request = request(chunk(1, 2, "DE", row("2026-07-25", "B001")));
service.submitResult(task.getId(), request);
service.submitResult(task.getId(), request);
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
assertEquals(-1, result.getSuccess());
}
@Test
void resultChunkRetryWithDifferentContentIsRejected() {
givenRunningTask(103L, 203L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B099")))));
assertTrue(error.getMessage().contains("不同内容"));
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
}
@Test
void resultChunksRejectChangedChunkTotal() {
givenRunningTask(104L, 204L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(2, 3, "UK", row("2026-07-26", "B002")))));
assertTrue(error.getMessage().contains("chunk_total"));
assertEquals(1, storedChunks.size());
}
@Test
void missingChunkDoesNotGenerateResultFile() {
givenRunningTask(105L, 205L);
service.submitResult(task.getId(), request(chunk(1, 3, "DE", row("2026-07-25", "B001"))));
service.submitResult(task.getId(), request(chunk(3, 3, "UK", row("2026-07-27", "B003"))));
assertEquals(-1, result.getSuccess());
assertEquals("RUNNING", task.getStatus());
assertEquals(2, storedChunks.size());
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), eq(MODULE_TYPE), anyLong(), anyString());
}
@Test
void staleCompensationRecoversReceivedResultChunks() {
givenRunningTask(110L, 210L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
boolean finalized = service.tryFinalizeTask(task.getId(), true);
assertTrue(finalized);
assertEquals(1, result.getSuccess());
assertTrue(result.getErrorMessage().contains("已保留"));
assertTrue(task.getResultJson().contains("B001"));
assertTrue(task.getResultJson().contains("已保留"));
verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(),
"task:" + task.getId() + ":owner:instance-a");
}
@Test
void staleCompensationRecoversLegacyMergedPayload() {
givenRunningTask(111L, 211L);
ShopDataCrawlShopPayloadDto cached = legacyChunk(false, "UK", row("2026-07-26", "B002"));
when(taskCacheService.getAllShopMergedPayload(task.getId())).thenReturn(Map.of(SHOP_NAME, cached));
boolean finalized = service.tryFinalizeTask(task.getId(), true);
assertTrue(finalized);
assertEquals(1, result.getSuccess());
assertTrue(result.getErrorMessage().contains("已保留"));
assertTrue(task.getResultJson().contains("B002"));
verify(taskCacheService).removeShopMergedPayload(task.getId(), SHOP_NAME);
verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(),
"task:" + task.getId() + ":owner:instance-a");
}
@Test
void resultChunkRejectsCallbackWhenRustfsIsUnavailable() {
givenRunningTask(106L, 206L);
when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(false);
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(1, 1, "DE", row("2026-07-25", "B001")))));
assertTrue(error.getMessage().contains("RustFS"));
assertEquals(0, storedChunks.size());
}
@Test
void resultChunkRejectsLocalFallbackPointer() {
givenRunningTask(107L, 207L);
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), anyLong(), anyString(), any(), anyString()))
.thenReturn("local:instance-a/chunk-1");
when(transientPayloadStorageService.extractPointer("local:instance-a/chunk-1"))
.thenReturn("local:instance-a/chunk-1");
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(1, 1, "DE", row("2026-07-25", "B001")))));
assertTrue(error.getMessage().contains("RustFS"));
assertEquals(0, storedChunks.size());
verify(transientPayloadStorageService).deletePayloadIfPresent("local:instance-a/chunk-1");
}
@Test
void legacyShopDoneFalseThenTrueUsesMergedPayloadCache() {
givenRunningTask(108L, 208L);
ShopDataCrawlShopPayloadDto cached = legacyChunk(false, "DE", row("2026-07-25", "B001"));
when(taskCacheService.getShopMergedPayload(task.getId(), SHOP_NAME)).thenReturn(null, cached);
service.submitResult(task.getId(), request(legacyChunk(false, "DE", row("2026-07-25", "B001"))));
assertEquals(-1, result.getSuccess());
verify(taskCacheService).saveShopMergedPayload(eq(task.getId()), eq(SHOP_NAME), any());
service.submitResult(task.getId(), request(legacyChunk(true, "UK", row("2026-07-26", "B002"))));
assertEquals(1, result.getSuccess());
assertTrue(task.getResultJson().contains("B001"));
assertTrue(task.getResultJson().contains("B002"));
assertEquals(0, storedChunks.size());
verify(taskCacheService).removeShopMergedPayload(task.getId(), SHOP_NAME);
}
@Test
void cleanupResultFileJobDeletesTransientChunkObjectsAndRows() {
givenRunningTask(109L, 209L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
String storedPayload = storedChunks.get(0).getPayloadJson();
TaskFileJobEntity job = new TaskFileJobEntity();
job.setTaskId(task.getId());
job.setModuleType(MODULE_TYPE);
job.setResultId(result.getId());
service.cleanupResultFileJob(job);
assertFalse(rustfsPayloads.containsKey(storedPayload));
assertEquals(0, storedChunks.size());
assertEquals(0, storedScopes.size());
verify(taskChunkMapper).delete(any());
verify(taskScopeStateMapper).delete(any());
}
private void configureTransientPayloadStorage() {
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
anyString(), anyLong(), anyString(), any(), anyString())).thenAnswer(invocation -> {
String pointer = "rustfs:payload-" + (++nextPayloadId);
rustfsPayloads.put(pointer, invocation.getArgument(4));
return pointer;
});
lenient().when(transientPayloadStorageService.extractPointer(any())).thenAnswer(invocation -> {
String value = invocation.getArgument(0);
if (value == null) {
return null;
}
return value.startsWith("rustfs:") || value.startsWith("local:") || value.startsWith("oss:")
? value : null;
});
lenient().when(transientPayloadStorageService.resolvePayload(any(), any())).thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
String payload = rustfsPayloads.get(pointer);
if (payload == null) {
throw new IllegalStateException("missing test RustFS payload: " + pointer);
}
return payload;
});
lenient().doAnswer(invocation -> {
rustfsPayloads.remove(invocation.getArgument(0));
return null;
}).when(transientPayloadStorageService).deletePayloadIfPresent(any());
}
private void configureChunkMapper() {
lenient().when(taskChunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> {
TaskChunkEntity chunk = invocation.getArgument(0);
chunk.setId((long) storedChunks.size() + 1L);
storedChunks.add(chunk);
return 1;
});
lenient().when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
Integer chunkIndex = queryInteger(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, chunkIndex))
.findFirst()
.orElse(null);
});
lenient().when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.count();
});
lenient().when(taskChunkMapper.selectList(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex))
.toList();
});
lenient().when(taskChunkMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedChunks.size();
storedChunks.removeIf(chunk -> taskId == null || Objects.equals(taskId, chunk.getTaskId()));
return before - storedChunks.size();
});
}
private void configureScopeMapper() {
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
scope.setId((long) storedScopes.size() + 1L);
storedScopes.add(scope);
return 1;
});
lenient().when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedScopes.stream()
.filter(scope -> Objects.equals(taskId, scope.getTaskId()))
.filter(scope -> scopeHash == null || Objects.equals(scopeHash, scope.getScopeHash()))
.findFirst()
.orElse(null);
});
lenient().when(taskScopeStateMapper.updateById(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedScopes.size();
storedScopes.removeIf(scope -> taskId == null || Objects.equals(taskId, scope.getTaskId()));
return before - storedScopes.size();
});
}
private boolean matchesChunk(TaskChunkEntity chunk, Long taskId, String scopeHash, Integer chunkIndex) {
return (taskId == null || Objects.equals(taskId, chunk.getTaskId()))
&& (scopeHash == null || Objects.equals(scopeHash, chunk.getScopeHash()))
&& (chunkIndex == null || Objects.equals(chunkIndex, chunk.getChunkIndex()));
}
private Long queryLong(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Long.class::isInstance)
.map(Long.class::cast)
.findFirst()
.orElse(null);
}
private Integer queryInteger(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Integer.class::isInstance)
.map(Integer.class::cast)
.findFirst()
.orElse(null);
}
private String queryScopeHash(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.filter(value -> value.length() == 64)
.findFirst()
.orElse(null);
}
private void givenRunningTask(long taskId, long resultId) {
task = new FileTaskEntity();
task.setId(taskId);
task.setUserId(7L);
task.setModuleType(MODULE_TYPE);
task.setStatus("RUNNING");
task.setRequestJson("{\"ownerInstanceId\":\"instance-a\"}");
task.setResultJson("[]");
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
result = new FileResultEntity();
result.setId(resultId);
result.setTaskId(taskId);
result.setUserId(7L);
result.setModuleType(MODULE_TYPE);
result.setSourceFilename(SHOP_NAME);
result.setSourceFileUrl("shop-1");
result.setSuccess(-1);
result.setCreatedAt(LocalDateTime.now());
}
private ShopDataCrawlSubmitResultRequest request(ShopDataCrawlShopPayloadDto payload) {
ShopDataCrawlSubmitResultRequest request = new ShopDataCrawlSubmitResultRequest();
request.setShops(List.of(payload));
return request;
}
private ShopDataCrawlShopPayloadDto chunk(int chunkIndex,
int chunkTotal,
String country,
ShopDataCrawlRowDto row) {
ShopDataCrawlShopPayloadDto payload = legacyChunk(false, country, row);
payload.setChunkIndex(chunkIndex);
payload.setChunkTotal(chunkTotal);
return payload;
}
private ShopDataCrawlShopPayloadDto legacyChunk(boolean shopDone,
String country,
ShopDataCrawlRowDto row) {
ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto();
payload.setShopName(SHOP_NAME);
payload.setCountryResults(List.of(country(country, row)));
payload.setShopDone(shopDone);
return payload;
}
private ShopDataCrawlCountryResultDto country(String country, ShopDataCrawlRowDto row) {
ShopDataCrawlCountryResultDto result = new ShopDataCrawlCountryResultDto();
result.setCountry(country);
result.setItems(List.of(row));
return result;
}
private ShopDataCrawlRowDto row(String date, String asin) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setInventorySales("10");
row.setSalesRank("20");
row.setPageViews("30");
row.setUnitsSold("40");
row.setPrice("50");
row.setRecommendedOffer("60");
return row;
}
}
@@ -0,0 +1,99 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ShopDataCrawlTaskServiceOwnerTest {
@Mock InstanceMetadata instanceMetadata;
@Mock com.nanri.aiimage.modules.task.mapper.FileTaskMapper fileTaskMapper;
@Mock com.nanri.aiimage.modules.task.mapper.FileResultMapper fileResultMapper;
@Mock ShopDataCrawlResolveService resolveService;
@Mock ShopDataCrawlExcelAssemblyService excelAssemblyService;
@Mock ShopDataCrawlTaskCacheService cacheService;
@Mock com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService ziniaoShopSwitchService;
@Mock com.nanri.aiimage.config.TaskPressureProperties taskPressureProperties;
@Mock com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@Mock com.nanri.aiimage.modules.task.service.TaskResultItemService taskResultItemService;
@Mock com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock com.nanri.aiimage.modules.task.mapper.TaskChunkMapper taskChunkMapper;
@Mock com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper taskScopeStateMapper;
@Mock com.nanri.aiimage.modules.task.service.TransientPayloadStorageService transientPayloadStorageService;
@Spy private final ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks ShopDataCrawlTaskService service;
@Test
void rejectsTaskOwnedByAnotherInstance() {
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
FileTaskEntity task = task("instance-b");
assertThrows(TaskOwnerMismatchException.class,
() -> service.ensureTaskOwnedByCurrentInstance(task, "callback"));
}
@Test
void acceptsTaskOwnedByCurrentInstance() {
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
assertDoesNotThrow(() -> service.ensureTaskOwnedByCurrentInstance(task("instance-a"), "callback"));
}
@Test
void callbackWithoutShopNameBindsOnlyTaskShopAndWrongNameFails() {
var payload = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto();
when(ziniaoShopSwitchService.normalizeShopName(null)).thenReturn("");
assertEquals(payload, service.normalizePayloadByShop(java.util.List.of(payload), "Demo").get("Demo"));
var wrong = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto();
wrong.setShopName("Wrong");
when(ziniaoShopSwitchService.normalizeShopName("Wrong")).thenReturn("Wrong");
assertThrows(com.nanri.aiimage.common.exception.BusinessException.class,
() -> service.normalizePayloadByShop(java.util.List.of(wrong), "Demo"));
}
@Test
void partialPayloadIsNotCompleteUntilShopDoneIsTrue() {
var payload = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto();
assertFalse(service.isShopPayloadComplete(payload));
payload.setShopDone(false);
assertFalse(service.isShopPayloadComplete(payload));
payload.setShopDone(true);
assertTrue(service.isShopPayloadComplete(payload));
}
@Test
@SuppressWarnings("unchecked")
void deletesResultObjectOnlyAfterLastDatabaseReferenceIsGone() {
when(fileResultMapper.selectCount(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
.thenReturn(1L, 0L);
service.deleteResultObjectIfUnreferenced("result/shared.xlsx");
verify(ossStorageService, never()).deleteObject("result/shared.xlsx");
service.deleteResultObjectIfUnreferenced("result/shared.xlsx");
verify(ossStorageService).deleteObject("result/shared.xlsx");
}
private FileTaskEntity task(String owner) {
FileTaskEntity task = new FileTaskEntity();
task.setId(1L);
task.setRequestJson("{\"ownerInstanceId\":\"" + owner + "\"}");
return task;
}
}
@@ -7,6 +7,8 @@ import java.lang.reflect.Field;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SimilarAsinTaskServiceTest {
@@ -50,6 +52,14 @@ class SimilarAsinTaskServiceTest {
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
}
@Test
void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() {
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3));
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("FAILED", 3, 3));
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("RUNNING", 3, 3));
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3));
}
private int staticIntField(String name) throws Exception {
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
field.setAccessible(true);
@@ -1,5 +1,7 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
@@ -24,7 +26,33 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
class SimilarAsinImageEmbedderTest {
// properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null);
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService());
private static OssStorageService createOssStorageService() {
OssProperties properties = new OssProperties();
properties.setEndpoint("https://oss.aishufu.top");
properties.setPublicEndpoint("https://oss.aishufu.top");
properties.setBucket("nanri-ai-images");
properties.setImageVideoBucket("shufu-video");
properties.setDigitalHumanBucket("nanri-ai-digital-human");
properties.setAccessKeyId("test-access-key");
properties.setAccessKeySecret("test-secret-key");
return new OssStorageService(properties);
}
@Test
void normalizesLegacyMinioUrlBeforeHttpsValidation() {
String normalized = embedder.normalizeAndValidateDownloadUrl(
"http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg");
assertEquals("https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg", normalized);
}
@Test
void leavesUnmanagedHttpUrlBlocked() {
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> embedder.normalizeAndValidateDownloadUrl("http://example.com/main.jpg"));
}
@Test
void resizeImageProducesThumbnailUnderHardCap() throws Exception {
@@ -13,6 +13,8 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
@@ -55,6 +57,8 @@ class TaskHeartbeatServiceTest {
@Mock private ShopMatchTaskCacheService shopMatchTaskCacheService;
@Mock private PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
@Mock private QueryAsinTaskCacheService queryAsinTaskCacheService;
@Mock private ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService;
@Mock private ShopDataCrawlTaskService shopDataCrawlTaskService;
@Mock private WithdrawTaskCacheService withdrawTaskCacheService;
@Mock private AppearancePatentTaskCacheService appearancePatentTaskCacheService;
@Mock private SimilarAsinTaskCacheService similarAsinTaskCacheService;
@@ -91,4 +95,27 @@ class TaskHeartbeatServiceTest {
routingBeforeUpdate.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
verify(publishTaskService).touchHeartbeat(taskId, request);
}
@Test
@SuppressWarnings("unchecked")
void shopDataCrawlHeartbeatChecksOwnerBeforeDatabaseUpdate() {
long taskId = 20143L;
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
task.setModuleType("SHOP_DATA_CRAWL");
task.setStatus("RUNNING");
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest());
assertTrue(result.isAlive());
InOrder order = inOrder(shopDataCrawlTaskService, fileTaskMapper);
order.verify(shopDataCrawlTaskService)
.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat");
order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
verify(shopDataCrawlTaskCacheService).touchTaskHeartbeat(taskId);
verify(shopDataCrawlTaskCacheService).saveTaskCache(task);
}
}
@@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
@@ -24,6 +25,7 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
@@ -43,6 +45,7 @@ class TaskResultFileJobWorkerTest {
@Mock private ProductRiskTaskService productRiskTaskService;
@Mock private PublishTaskService publishTaskService;
@Mock private QueryAsinTaskService queryAsinTaskService;
@Mock private ShopDataCrawlTaskService shopDataCrawlTaskService;
@Mock private WithdrawTaskService withdrawTaskService;
@Mock private PatrolDeleteTaskService patrolDeleteTaskService;
@Mock private AppearancePatentTaskService appearancePatentTaskService;
@@ -109,8 +112,10 @@ class TaskResultFileJobWorkerTest {
worker.process(job);
verify(publishTaskService).processResultFileJob(job);
verify(taskFileJobService).markSuccess(job, "result/publish/20141.xlsx");
InOrder order = inOrder(publishTaskService, taskFileJobService);
order.verify(publishTaskService).processResultFileJob(job);
order.verify(taskFileJobService).markSuccess(job, "result/publish/20141.xlsx");
order.verify(publishTaskService).cleanupResultFileJob(job);
verifyNoInteractions(taskResultPayloadService);
}
@@ -128,4 +133,56 @@ class TaskResultFileJobWorkerTest {
verifyNoInteractions(taskFileJobService, taskDistributedLockService, publishTaskService);
}
@Test
void shopDataCrawlOwnerJobDelegatesAndCleansPayload() {
long jobId = 13643L;
long taskId = 20143L;
long resultId = 22931L;
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(jobId);
job.setTaskId(taskId);
job.setResultId(resultId);
job.setModuleType("SHOP_DATA_CRAWL");
job.setScopeKey("task:20143:owner:instance-a");
FileResultEntity result = new FileResultEntity();
result.setResultFileUrl("result/shop-data-crawl/20143.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", taskId,
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
when(fileResultMapper.selectById(resultId)).thenReturn(result);
worker.process(job);
verify(shopDataCrawlTaskService).processResultFileJob(job);
verify(taskFileJobService).markSuccess(job, "result/shop-data-crawl/20143.xlsx");
verify(taskResultPayloadService).deleteLatest(taskId, "SHOP_DATA_CRAWL", job.getScopeKey());
verify(shopDataCrawlTaskService).cleanupResultFileJob(job);
}
@Test
void exhaustedShopDataCrawlFileJobFailsOwningTask() {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(13644L);
job.setTaskId(20144L);
job.setResultId(22932L);
job.setModuleType("SHOP_DATA_CRAWL");
job.setScopeKey("task:20144:owner:instance-a");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(taskFileJobService.markRunning(job.getId())).thenReturn(true);
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", job.getTaskId(),
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
doThrow(new IllegalStateException("upload failed"))
.when(shopDataCrawlTaskService).processResultFileJob(job);
when(taskFileJobService.isRetryExhausted(job.getId())).thenReturn(true);
worker.process(job);
verify(taskFileJobService).markFailed(job, "upload failed");
verify(shopDataCrawlTaskService).handleResultFileJobFailure(job, "upload failed");
}
}
File diff suppressed because it is too large Load Diff
+345 -211
View File
@@ -382,115 +382,322 @@
});
}
var allColumnsList = [];
var columnPermissionCatalogReady = false;
var columnCardsContainerMap = { createColumnPermissionWrap: 'createColumnCards', editColumnPermissionWrap: 'editColumnCards' };
// Hierarchical permission editor: only direct IDs are submitted; descendants are visual inheritance.
function columnParentId(item) {
var value = item && item.parent_id != null ? item.parent_id : item && item.parentId;
var parsed = value == null || value === '' ? null : Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
}
function columnId(item) { return Number(item && item.id); }
function columnMenuType(item) {
return String(item && item.menu_type || 'app').toLowerCase() === 'admin' ? 'admin' : 'app';
}
function columnDescendantIds(id) {
var result = [], pending = [Number(id)];
while (pending.length) {
var parent = pending.shift();
allColumnsList.forEach(function (item) {
if (columnParentId(item) !== parent) return;
var childId = columnId(item);
if (result.indexOf(childId) < 0) {
result.push(childId);
pending.push(childId);
}
});
}
return result;
}
function columnTreeHasDirectDescendant(id, directIds) {
return columnDescendantIds(id).some(function (descendantId) { return !!directIds[descendantId]; });
}
function columnTreeFullyCovered(id, directIds) {
if (directIds[id]) return true;
var children = allColumnsList.filter(function (item) { return columnParentId(item) === Number(id); });
return children.length > 0 && children.every(function (child) {
return columnTreeFullyCovered(columnId(child), directIds);
});
}
function loadColumnsForPermission() {
var ownedKeys = {};
var ownedRoutes = {};
columnPermissionCatalogReady = false;
var ownedKeys = {}, ownedRoutes = {};
var ownPermissionRequest = (currentUserRole === 'admin' && currentUserId)
? fetch('/api/admin/user/' + currentUserId + '/column-permissions')
.then(function (r) { return r.json(); })
.then(function (res) {
if (!res.success) throw new Error(res.error || '当前用户权限加载失败');
(res.items || []).forEach(function (item) {
if (item.column_key) ownedKeys[String(item.column_key).trim()] = true;
if (item.route_path) ownedRoutes[String(item.route_path).trim()] = true;
});
})
.catch(function () { })
: Promise.resolve();
ownPermissionRequest.then(function () {
return fetch('/api/admin/columns');
}).then(function (r) { return r.json(); })
ownPermissionRequest.then(function () { return fetch('/api/admin/columns'); })
.then(function (r) { return r.json(); })
.then(function (res) {
if (!res.success) return;
allColumnsList = (res.items || []).filter(function (item) {
var availableColumns = (res.items || []).filter(function (item) {
if (item.column_key === 'admin_image_video_task_data') return false;
if (currentUserRole !== 'admin') return true;
return !!ownedKeys[String(item.column_key || '').trim()] ||
!!ownedRoutes[String(item.route_path || '').trim()];
return true;
});
if (currentUserRole === 'admin') {
var columnsById = {}, grantableIds = {}, visibleIds = {};
availableColumns.forEach(function (item) {
var id = columnId(item);
columnsById[id] = item;
if (ownedKeys[String(item.column_key || '').trim()] || ownedRoutes[String(item.route_path || '').trim()]) {
grantableIds[id] = true;
}
});
Object.keys(grantableIds).forEach(function (id) {
var currentId = Number(id), visited = {};
while (currentId > 0 && !visited[currentId] && columnsById[currentId]) {
visited[currentId] = true;
visibleIds[currentId] = true;
currentId = columnParentId(columnsById[currentId]);
}
});
allColumnsList = availableColumns.filter(function (item) {
return !!visibleIds[columnId(item)];
}).map(function (item) {
item._structureOnly = !grantableIds[columnId(item)];
return item;
});
} else {
allColumnsList = availableColumns.map(function (item) {
item._structureOnly = false;
return item;
});
}
columnPermissionCatalogReady = true;
renderColumnPermissionWrap('createColumnPermissionWrap');
renderColumnPermissionWrap('editColumnPermissionWrap');
var selCreate = document.getElementById('createColumnPermissionWrap');
var selEdit = document.getElementById('editColumnPermissionWrap');
if (selCreate && !selCreate._colCardsBound) {
selCreate._colCardsBound = true;
selCreate.onchange = function () { renderColumnCards('createColumnPermissionWrap'); };
}
if (selEdit && !selEdit._colCardsBound) {
selEdit._colCardsBound = true;
selEdit.onchange = function () { renderColumnCards('editColumnPermissionWrap'); };
}
})
.catch(function () { });
populateColumnParentSelects();
}).catch(function () {
columnPermissionCatalogReady = false;
allColumnsList = [];
var editPermissionWrap = document.getElementById('editColumnPermissionWrap');
if (editPermissionWrap) editPermissionWrap._permissionsReady = false;
renderColumnPermissionWrap('createColumnPermissionWrap');
renderColumnPermissionWrap('editColumnPermissionWrap');
});
}
var columnCardsContainerMap = { createColumnPermissionWrap: 'createColumnCards', editColumnPermissionWrap: 'editColumnCards' };
function renderColumnCards(selectId) {
var sel = document.getElementById(selectId);
var cardsId = columnCardsContainerMap[selectId];
var cardsEl = cardsId ? document.getElementById(cardsId) : null;
if (!sel || sel.tagName !== 'SELECT' || !cardsEl) return;
function renderColumnCards(wrapId) {
var wrap = document.getElementById(wrapId);
var cardsEl = document.getElementById(columnCardsContainerMap[wrapId]);
if (!wrap || !cardsEl) return;
var directIds = wrap._directIds || {};
cardsEl.innerHTML = '';
for (var i = 0; i < sel.options.length; i++) {
var opt = sel.options[i];
if (opt.disabled || !opt.selected) continue;
allColumnsList.forEach(function (item) {
var id = columnId(item);
if (!directIds[id] || item._structureOnly) return;
var card = document.createElement('span');
card.className = 'column-permission-card';
card.textContent = opt.textContent;
card.textContent = (item.name || '') + ' (' + (item.column_key || '') + ')';
var btn = document.createElement('button');
btn.type = 'button';
btn.className = 'col-card-remove';
btn.setAttribute('aria-label', '移除');
btn.textContent = '×';
(function (option, seldId) {
btn.onclick = function () {
option.selected = false;
renderColumnCards(seldId);
};
})(opt, selectId);
btn.onclick = function () {
delete directIds[id];
renderColumnPermissionWrap(wrapId);
};
card.appendChild(btn);
cardsEl.appendChild(card);
}
});
}
function renderColumnPermissionWrap(wrapId) {
var sel = document.getElementById(wrapId);
if (!sel || sel.tagName !== 'SELECT') return;
sel.innerHTML = '';
allColumnsList.forEach(function (c) {
var opt = document.createElement('option');
opt.value = c.id;
opt.textContent = c.name + ' (' + c.column_key + ')';
sel.appendChild(opt);
});
if (allColumnsList.length === 0) {
var opt = document.createElement('option');
opt.disabled = true;
opt.textContent = '暂无菜单,请先在“栏目权限配置”中新增菜单';
sel.appendChild(opt);
var wrap = document.getElementById(wrapId);
if (!wrap) return;
var directIds = wrap._directIds || {};
var collapsedIds = wrap._collapsedIds || {};
var collapsedMenuTypes = wrap._collapsedMenuTypes || {};
var previousScrollTop = wrap.scrollTop;
wrap._directIds = directIds;
wrap._collapsedIds = collapsedIds;
wrap._collapsedMenuTypes = collapsedMenuTypes;
wrap.innerHTML = '';
if (!allColumnsList.length) {
wrap.textContent = '暂无可分配菜单';
renderColumnCards(wrapId);
return;
}
var byParent = {};
allColumnsList.forEach(function (item) {
var parent = columnParentId(item);
var key = parent == null ? 'root' : String(parent);
(byParent[key] || (byParent[key] = [])).push(item);
});
if (!wrap._collapseStateInitialized) {
allColumnsList.forEach(function (item) {
var id = columnId(item);
if ((byParent[String(id)] || []).length) collapsedIds[id] = true;
});
wrap._collapseStateInitialized = true;
}
if (!wrap._menuTypeCollapseInitialized) {
collapsedMenuTypes.admin = true;
collapsedMenuTypes.app = true;
wrap._menuTypeCollapseInitialized = true;
}
function renderNode(item, depth, inherited) {
var id = columnId(item);
var itemMenuType = columnMenuType(item);
var children = (byParent[String(id)] || []).filter(function (child) {
return columnMenuType(child) === itemMenuType;
});
var structureOnly = item._structureOnly === true;
var effective = inherited || !!directIds[id];
var covered = effective || columnTreeFullyCovered(id, directIds);
var node = document.createElement('div');
node.className = 'column-tree-node';
node.setAttribute('data-depth', String(depth));
var row = document.createElement('div');
row.className = 'column-tree-row';
if (children.length) {
var toggle = document.createElement('button');
var collapsed = !!collapsedIds[id];
toggle.type = 'button';
toggle.className = 'column-tree-toggle';
toggle.setAttribute('aria-expanded', String(!collapsed));
toggle.setAttribute('aria-label', (collapsed ? '展开' : '收起') + (item.name || '菜单'));
toggle.title = collapsed ? '展开子菜单' : '收起子菜单';
toggle.onclick = function () {
if (collapsedIds[id]) delete collapsedIds[id];
else collapsedIds[id] = true;
renderColumnPermissionWrap(wrapId);
};
row.appendChild(toggle);
} else {
var spacer = document.createElement('span');
spacer.className = 'column-tree-toggle-spacer';
spacer.setAttribute('aria-hidden', 'true');
row.appendChild(spacer);
}
var label = document.createElement('label');
var checkbox = document.createElement('input');
checkbox.type = 'checkbox';
checkbox.checked = covered;
checkbox.indeterminate = !covered && columnTreeHasDirectDescendant(id, directIds);
checkbox.disabled = inherited || structureOnly;
checkbox.onchange = function () {
if (structureOnly) return;
if (checkbox.checked) {
directIds[id] = true;
columnDescendantIds(id).forEach(function (descendantId) { delete directIds[descendantId]; });
} else {
delete directIds[id];
columnDescendantIds(id).forEach(function (descendantId) { delete directIds[descendantId]; });
}
renderColumnPermissionWrap(wrapId);
};
var text = document.createElement('span');
text.textContent = (item.name || '') + ' (' + (item.column_key || '') + ')';
if (inherited || structureOnly) text.className = 'inherited-label';
label.appendChild(checkbox);
label.appendChild(text);
row.appendChild(label);
node.appendChild(row);
if (children.length) {
var childrenWrap = document.createElement('div');
childrenWrap.className = 'column-tree-children';
childrenWrap.hidden = !!collapsedIds[id];
children.forEach(function (child) {
childrenWrap.appendChild(renderNode(child, depth + 1, effective));
});
node.appendChild(childrenWrap);
}
return node;
}
[
{ key: 'admin', label: '后台菜单' },
{ key: 'app', label: 'APP(软件)菜单' }
].forEach(function (group) {
var groupItems = allColumnsList.filter(function (item) {
return columnMenuType(item) === group.key;
});
if (!groupItems.length) return;
var section = document.createElement('div');
section.className = 'column-tree-group';
var groupToggle = document.createElement('button');
var groupCollapsed = !!collapsedMenuTypes[group.key];
groupToggle.type = 'button';
groupToggle.className = 'column-tree-group-toggle';
groupToggle.setAttribute('aria-expanded', String(!groupCollapsed));
groupToggle.setAttribute('aria-label', (groupCollapsed ? '展开' : '收起') + group.label);
var groupName = document.createElement('span');
groupName.className = 'column-tree-group-name';
groupName.textContent = group.label;
var groupCount = document.createElement('span');
groupCount.className = 'column-tree-group-count';
groupCount.textContent = groupItems.length + ' 项';
groupToggle.appendChild(groupName);
groupToggle.appendChild(groupCount);
groupToggle.onclick = function () {
if (collapsedMenuTypes[group.key]) delete collapsedMenuTypes[group.key];
else collapsedMenuTypes[group.key] = true;
renderColumnPermissionWrap(wrapId);
};
section.appendChild(groupToggle);
var groupBody = document.createElement('div');
groupBody.className = 'column-tree-group-body';
groupBody.hidden = groupCollapsed;
groupItems.filter(function (item) {
var parentId = columnParentId(item);
if (parentId == null) return true;
var parent = allColumnsList.find(function (candidate) {
return columnId(candidate) === parentId;
});
return !parent || columnMenuType(parent) !== group.key;
}).forEach(function (item) {
groupBody.appendChild(renderNode(item, 0, false));
});
section.appendChild(groupBody);
wrap.appendChild(section);
});
renderColumnCards(wrapId);
wrap.scrollTop = previousScrollTop;
}
function getSelectedColumnIds(wrapId) {
var sel = document.getElementById(wrapId);
if (!sel || sel.tagName !== 'SELECT') return [];
var ids = [];
for (var i = 0; i < sel.options.length; i++) {
if (sel.options[i].selected) {
var v = parseInt(sel.options[i].value, 10);
if (!isNaN(v)) ids.push(v);
}
}
return ids;
var wrap = document.getElementById(wrapId);
return wrap && wrap._directIds ? Object.keys(wrap._directIds).map(Number).filter(function (id) {
var item = allColumnsList.find(function (column) { return columnId(column) === id; });
return id > 0 && item && !item._structureOnly;
}) : [];
}
function setColumnPermissionCheckboxes(wrapId, columnIds) {
var sel = document.getElementById(wrapId);
if (!sel || sel.tagName !== 'SELECT') return;
var set = {};
(columnIds || []).forEach(function (id) { set[id] = true; });
for (var i = 0; i < sel.options.length; i++) {
var opt = sel.options[i];
if (opt.disabled) continue;
opt.selected = set[parseInt(opt.value, 10)] || false;
}
renderColumnCards(wrapId);
var wrap = document.getElementById(wrapId);
if (!wrap) return;
wrap._directIds = {};
(columnIds || []).forEach(function (id) {
var parsed = Number(id);
if (Number.isFinite(parsed) && parsed > 0) wrap._directIds[parsed] = true;
});
wrap._permissionsReady = true;
renderColumnPermissionWrap(wrapId);
}
function populateColumnParentSelects() {
['columnParentId', 'editColumnParentId'].forEach(function (selectId) {
var select = document.getElementById(selectId);
if (!select) return;
var current = select.value;
var typeSelect = document.getElementById(selectId === 'editColumnParentId' ? 'editColumnMenuType' : 'columnMenuType');
var menuType = typeSelect ? (typeSelect.value || 'app') : 'app';
var editingId = selectId === 'editColumnParentId' ? Number(document.getElementById('editColumnId').value || 0) : 0;
var blockedIds = editingId > 0 ? [editingId].concat(columnDescendantIds(editingId)) : [];
select.innerHTML = '<option value="">无(根菜单)</option>';
allColumnsList.filter(function (item) {
return !item._structureOnly && String(item.menu_type || 'app') === String(menuType) && blockedIds.indexOf(columnId(item)) < 0;
}).forEach(function (item) {
var option = document.createElement('option');
option.value = item.id;
option.textContent = (item.name || '') + ' (' + (item.column_key || '') + ')';
select.appendChild(option);
});
select.value = current;
});
}
function updateCreateFormByRole() {
var roleSel = document.getElementById('createRole');
@@ -531,14 +738,25 @@
editCreatorName.value = u.creator_username || '';
if (u.role !== 'super_admin') { editRole.value = u.role || 'normal'; }
document.getElementById('msgEdit').textContent = '';
if (allColumnsList.length) {
renderColumnPermissionWrap('editColumnPermissionWrap');
fetch('/api/admin/user/' + (u.id) + '/columns')
.then(function (r) { return r.json(); })
.then(function (res) {
if (res.success && res.column_ids) setColumnPermissionCheckboxes('editColumnPermissionWrap', res.column_ids);
});
}
var editPermissionWrap = document.getElementById('editColumnPermissionWrap');
editPermissionWrap._directIds = {};
editPermissionWrap._permissionsReady = false;
renderColumnPermissionWrap('editColumnPermissionWrap');
fetch('/api/admin/user/' + (u.id) + '/columns')
.then(function (r) { return r.json(); })
.then(function (res) {
if (String(document.getElementById('editUserId').value || '') !== String(u.id || '')) return;
if (!res.success || !Array.isArray(res.column_ids)) {
throw new Error(res.error || '权限加载失败');
}
setColumnPermissionCheckboxes('editColumnPermissionWrap', res.column_ids);
})
.catch(function (error) {
if (String(document.getElementById('editUserId').value || '') !== String(u.id || '')) return;
var msgEl = document.getElementById('msgEdit');
msgEl.textContent = error.message || '权限加载失败';
msgEl.className = 'msg err';
});
document.getElementById('editUserModal').classList.add('show');
};
});
@@ -575,6 +793,11 @@
msgEl.classList.add('err');
return;
}
if (!columnPermissionCatalogReady) {
msgEl.textContent = '菜单权限尚未加载,请稍后重试';
msgEl.classList.add('err');
return;
}
var body = { username: username, password: password, role: role };
if (role === 'normal' && currentUserRole === 'super_admin' && createdById) body.created_by_id = createdById;
body.column_ids = getSelectedColumnIds('createColumnPermissionWrap');
@@ -609,6 +832,12 @@
var msgEl = document.getElementById('msgEdit');
msgEl.textContent = '';
msgEl.className = 'msg';
var editPermissionWrap = document.getElementById('editColumnPermissionWrap');
if (!columnPermissionCatalogReady || !editPermissionWrap._permissionsReady) {
msgEl.textContent = '用户权限尚未加载,不能保存';
msgEl.classList.add('err');
return;
}
var body = {};
if (password) body.password = password;
if (currentUserRole === 'super_admin' && editRoleEl && editRoleEl.offsetParent !== null)
@@ -4483,114 +4712,6 @@
// ========== 栏目权限配置 ==========
function loadColumns() {
fetch('/api/admin/columns')
.then(function (r) { return r.json(); })
.then(function (res) {
var tbody = document.getElementById('columnListBody');
if (!res.success) {
tbody.innerHTML = '<tr><td colspan="5" class="empty-tip">加载失败: ' + (res.error || '') + '</td></tr>';
return;
}
allColumnsList = res.items || [];
if (allColumnsList.length === 0) {
tbody.innerHTML = '<tr><td colspan="5" class="empty-tip">暂无菜单,请在上方新增</td></tr>';
} else {
tbody.innerHTML = allColumnsList.map(function (c) {
return '<tr><td>' + c.id + '</td><td>' + (c.name || '') + '</td><td>' + (c.column_key || '') + '</td><td>' + (c.created_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-column-edit="' + c.id + '" data-name="' + (c.name || '').replace(/"/g, '&quot;') + '" data-key="' + (c.column_key || '').replace(/"/g, '&quot;') + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-column-delete="' + c.id + '" data-name="' + (c.name || '').replace(/"/g, '&quot;') + '">删除</button></td></tr>';
}).join('');
}
bindColumnActions();
})
.catch(function () {
document.getElementById('columnListBody').innerHTML = '<tr><td colspan="5" class="empty-tip">请求失败</td></tr>';
});
}
function bindColumnActions() {
document.querySelectorAll('[data-column-edit]').forEach(function (btn) {
btn.onclick = function () {
document.getElementById('editColumnId').value = btn.dataset.columnEdit || '';
document.getElementById('editColumnName').value = (btn.dataset.name || '').replace(/&quot;/g, '"');
document.getElementById('editColumnKey').value = (btn.dataset.key || '').replace(/&quot;/g, '"');
document.getElementById('msgEditColumn').textContent = '';
document.getElementById('editColumnModal').classList.add('show');
};
});
document.querySelectorAll('[data-column-delete]').forEach(function (btn) {
btn.onclick = function () {
if (!confirm('确定删除菜单“' + (btn.dataset.name || '').replace(/&quot;/g, '"') + '”吗?')) return;
fetch('/api/admin/column/' + btn.dataset.columnDelete, { method: 'DELETE' })
.then(function (r) { return r.json(); })
.then(function (res) {
if (res.success) { loadColumns(); loadColumnsForPermission(); loadAdminMenus(getActiveAdminTabName()); }
else { alert(res.error || '删除失败'); }
});
};
});
}
document.getElementById('btnAddColumn').onclick = function () {
var name = (document.getElementById('columnName').value || '').trim();
var key = (document.getElementById('columnKey').value || '').trim();
var msgEl = document.getElementById('msgColumn');
msgEl.textContent = '';
msgEl.className = 'msg';
if (!name) { msgEl.textContent = '请填写菜单名称'; msgEl.classList.add('err'); return; }
if (!key) {
msgEl.textContent = '请填写栏目标识'; msgEl.classList.add('err'); return;
}
fetch('/api/admin/column', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: name, column_key: key })
})
.then(function (r) { return r.json(); })
.then(function (res) {
if (res.success) {
msgEl.textContent = res.msg || '新增成功';
msgEl.classList.add('ok');
document.getElementById('columnName').value = '';
document.getElementById('columnKey').value = '';
loadColumns();
loadColumnsForPermission();
loadAdminMenus(getActiveAdminTabName());
} else {
msgEl.textContent = res.error || '新增失败';
msgEl.classList.add('err');
}
})
.catch(function () { msgEl.textContent = '请求失败'; msgEl.classList.add('err'); });
};
document.getElementById('btnSaveColumn').onclick = function () {
var cid = document.getElementById('editColumnId').value;
var name = (document.getElementById('editColumnName').value || '').trim();
var key = (document.getElementById('editColumnKey').value || '').trim();
var msgEl = document.getElementById('msgEditColumn');
msgEl.textContent = '';
msgEl.className = 'msg';
if (!name) { msgEl.textContent = '请填写菜单名称'; msgEl.classList.add('err'); return; }
if (!key) {
msgEl.textContent = '请填写栏目标识'; msgEl.classList.add('err'); return;
}
fetch('/api/admin/column/' + cid, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: name, column_key: key })
})
.then(function (r) { return r.json(); })
.then(function (res) {
if (res.success) {
document.getElementById('editColumnModal').classList.remove('show');
loadColumns();
loadColumnsForPermission();
loadAdminMenus(getActiveAdminTabName());
} else {
msgEl.textContent = res.error || '保存失败';
msgEl.classList.add('err');
}
});
};
document.getElementById('btnCloseEditColumn').onclick = function () {
document.getElementById('editColumnModal').classList.remove('show');
};
@@ -4601,29 +4722,39 @@
.then(function (res) {
var tbody = document.getElementById('columnListBody');
if (!res.success) {
tbody.innerHTML = '<tr><td colspan="7" class="empty-tip">加载失败: ' + (res.error || '') + '</td></tr>';
tbody.innerHTML = '<tr><td colspan="9" class="empty-tip">加载失败: ' + (res.error || '') + '</td></tr>';
return;
}
allColumnsList = res.items || [];
if (allColumnsList.length === 0) {
tbody.innerHTML = '<tr><td colspan="8" class="empty-tip">暂无菜单,请先在上方新增</td></tr>';
tbody.innerHTML = '<tr><td colspan="9" class="empty-tip">暂无菜单,请先在上方新增</td></tr>';
} else {
tbody.innerHTML = allColumnsList.map(function (c, index) {
tbody.innerHTML = allColumnsList.map(function (c) {
var siblings = columnSiblingItems(c);
var siblingIndex = siblings.findIndex(function (item) { return String(item.id) === String(c.id); });
var moveButtons =
'<button class="btn btn-sm btn-secondary" data-column-move-up="' + c.id + '"' + (index === 0 ? ' disabled' : '') + '>上移</button> ' +
'<button class="btn btn-sm btn-secondary" data-column-move-down="' + c.id + '"' + (index === allColumnsList.length - 1 ? ' disabled' : '') + '>下移</button> ';
return '<tr><td>' + c.id + '</td><td>' + (c.name || '') + '</td><td>' + (c.column_key || '') + '</td><td>' + ((c.menu_type || '') === 'admin' ? '后台(admin)' : '软件(app)') + '</td><td>' + (c.sort_order != null ? c.sort_order : 0) + '</td><td>' + (c.route_path || '') + '</td><td>' + (c.created_at || '') + '</td><td>' +
'<button class="btn btn-sm btn-secondary" data-column-move-up="' + c.id + '"' + (siblingIndex <= 0 ? ' disabled' : '') + '>上移</button> ' +
'<button class="btn btn-sm btn-secondary" data-column-move-down="' + c.id + '"' + (siblingIndex < 0 || siblingIndex === siblings.length - 1 ? ' disabled' : '') + '>下移</button> ';
var parent = allColumnsList.find(function (item) { return String(item.id) === String(c.parent_id || c.parentId || ''); });
return '<tr><td>' + c.id + '</td><td>' + (c.name || '') + '</td><td>' + (c.column_key || '') + '</td><td>' + ((c.menu_type || '') === 'admin' ? '后台(admin)' : '软件(app)') + '</td><td>' + (parent ? (parent.name || '') : '-') + '</td><td>' + (c.sort_order != null ? c.sort_order : 0) + '</td><td>' + (c.route_path || '') + '</td><td>' + (c.created_at || '') + '</td><td>' +
moveButtons +
'<button class="btn btn-sm" data-column-edit="' + c.id + '" data-name="' + (c.name || '').replace(/"/g, '&quot;') + '" data-key="' + (c.column_key || '').replace(/"/g, '&quot;') + '" data-route="' + (c.route_path || '').replace(/"/g, '&quot;') + '" data-menu-type="' + (c.menu_type || 'app').replace(/"/g, '&quot;') + '" data-sort-order="' + (c.sort_order != null ? String(c.sort_order) : '0').replace(/"/g, '&quot;') + '">编辑</button> ' +
'<button class="btn btn-sm" data-column-edit="' + c.id + '" data-name="' + (c.name || '').replace(/"/g, '&quot;') + '" data-key="' + (c.column_key || '').replace(/"/g, '&quot;') + '" data-route="' + (c.route_path || '').replace(/"/g, '&quot;') + '" data-menu-type="' + (c.menu_type || 'app').replace(/"/g, '&quot;') + '" data-sort-order="' + (c.sort_order != null ? String(c.sort_order) : '0').replace(/"/g, '&quot;') + '" data-parent-id="' + String(c.parent_id || c.parentId || '').replace(/"/g, '&quot;') + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-column-delete="' + c.id + '" data-name="' + (c.name || '').replace(/"/g, '&quot;') + '">删除</button></td></tr>';
}).join('');
}
bindColumnActions();
})
.catch(function () {
document.getElementById('columnListBody').innerHTML = '<tr><td colspan="8" class="empty-tip">请求失败</td></tr>';
document.getElementById('columnListBody').innerHTML = '<tr><td colspan="9" class="empty-tip">请求失败</td></tr>';
});
}
function columnSiblingItems(item) {
var menuType = String(item && item.menu_type || 'app');
var parentId = columnParentId(item);
return allColumnsList.filter(function (candidate) {
return String(candidate.menu_type || 'app') === menuType && columnParentId(candidate) === parentId;
});
}
function bindColumnActions() {
document.querySelectorAll('[data-column-move-up]').forEach(function (btn) {
btn.onclick = function () {
@@ -4645,6 +4776,8 @@
document.getElementById('editColumnRoutePath').value = (btn.dataset.route || '').replace(/&quot;/g, '"');
document.getElementById('editColumnMenuType').value = ((btn.dataset.menuType || 'app').replace(/&quot;/g, '"') || 'app');
document.getElementById('editColumnSortOrder').value = ((btn.dataset.sortOrder || '0').replace(/&quot;/g, '"') || '0');
populateColumnParentSelects();
document.getElementById('editColumnParentId').value = btn.dataset.parentId || '';
document.getElementById('msgEditColumn').textContent = '';
document.getElementById('editColumnModal').classList.add('show');
};
@@ -4662,26 +4795,22 @@
});
}
function moveColumnItem(columnId, delta, triggerBtn) {
var currentIndex = -1;
for (var i = 0; i < allColumnsList.length; i++) {
if (String(allColumnsList[i].id) === String(columnId)) {
currentIndex = i;
break;
}
}
var currentItem = allColumnsList.find(function (item) { return String(item.id) === String(columnId); });
if (!currentItem) return;
var siblings = columnSiblingItems(currentItem);
var currentIndex = siblings.findIndex(function (item) { return String(item.id) === String(columnId); });
if (currentIndex < 0) return;
var targetIndex = currentIndex + delta;
if (targetIndex < 0 || targetIndex >= allColumnsList.length) return;
var currentItem = allColumnsList[currentIndex];
var targetItem = allColumnsList[targetIndex];
if (!currentItem || !targetItem) return;
if (targetIndex < 0 || targetIndex >= siblings.length) return;
var targetItem = siblings[targetIndex];
if (triggerBtn) triggerBtn.disabled = true;
fetch('/api/admin/column/reorder', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
column_id: currentItem.id,
target_id: targetItem.id
target_id: targetItem.id,
menu_type: currentItem.menu_type || 'app'
})
}).then(function (r) { return r.json(); })
.then(function (res) {
@@ -4704,6 +4833,7 @@
var routePath = (document.getElementById('columnRoutePath').value || '').trim();
var sortOrderValue = (document.getElementById('columnSortOrder').value || '').trim();
var menuType = (document.getElementById('columnMenuType').value || 'admin').trim() || 'admin';
var parentIdValue = (document.getElementById('columnParentId').value || '').trim();
var sortOrder = sortOrderValue === '' ? null : parseInt(sortOrderValue, 10);
var msgEl = document.getElementById('msgColumn');
msgEl.textContent = '';
@@ -4721,7 +4851,7 @@
fetch('/api/admin/column', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder })
body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder, parent_id: parentIdValue ? Number(parentIdValue) : null })
})
.then(function (r) { return r.json(); })
.then(function (res) {
@@ -4733,6 +4863,7 @@
document.getElementById('columnRoutePath').value = '';
document.getElementById('columnSortOrder').value = '';
document.getElementById('columnMenuType').value = 'admin';
document.getElementById('columnParentId').value = '';
loadColumns();
loadColumnsForPermission();
loadAdminMenus(getActiveAdminTabName());
@@ -4750,6 +4881,7 @@
var routePath = (document.getElementById('editColumnRoutePath').value || '').trim();
var sortOrderValue = (document.getElementById('editColumnSortOrder').value || '').trim();
var menuType = (document.getElementById('editColumnMenuType').value || 'admin').trim() || 'admin';
var parentIdValue = (document.getElementById('editColumnParentId').value || '').trim();
var sortOrder = sortOrderValue === '' ? null : parseInt(sortOrderValue, 10);
var msgEl = document.getElementById('msgEditColumn');
msgEl.textContent = '';
@@ -4767,7 +4899,7 @@
fetch('/api/admin/column/' + cid, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder })
body: JSON.stringify({ name: name, column_key: key, route_path: routePath, menu_type: menuType, sort_order: sortOrder, parent_id: parentIdValue ? Number(parentIdValue) : null })
})
.then(function (r) { return r.json(); })
.then(function (res) {
@@ -4782,6 +4914,8 @@
}
});
};
if (document.getElementById('columnMenuType')) document.getElementById('columnMenuType').onchange = populateColumnParentSelects;
if (document.getElementById('editColumnMenuType')) document.getElementById('editColumnMenuType').onchange = populateColumnParentSelects;
// ========== 分页 ==========
function renderPagination(elId, total, page, pageSize, onPage) {
@@ -1,160 +0,0 @@
import io
import unittest
from unittest.mock import Mock, patch
from flask import jsonify
from app import app
from blueprints import admin_api as admin_module
class DedupeTotalDataAdminTest(unittest.TestCase):
def setUp(self):
app.config.update(TESTING=True, SECRET_KEY='test')
self.client = app.test_client()
def _login(self, user_id=23):
with self.client.session_transaction() as session:
session['user_id'] = user_id
def test_list_forwards_session_operator_and_username_search(self):
self._login()
java_response = {
'success': True,
'data': {
'items': [{
'id': 91,
'dataValue': 'B012345678',
'uploaderUserId': 23,
'username': 'member-a',
'createdAt': '2026-07-20T12:30:00',
}],
'total': 1,
'page': 1,
'pageSize': 15,
},
}
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_dedupe_total_data_access',
return_value=('normal', {'id': 23, 'username': 'member-a'}, None)), \
patch.object(admin_module, '_proxy_backend_java',
return_value=(java_response, None, 200)) as proxy:
response = self.client.get(
'/api/admin/dedupe-total-data?keyword=B01&username=member&page=1&page_size=15')
self.assertEqual(200, response.status_code)
payload = response.get_json()
self.assertTrue(payload['success'])
self.assertEqual('member-a', payload['items'][0]['username'])
self.assertEqual(23, payload['items'][0]['uploader_user_id'])
self.assertEqual({
'page': 1,
'pageSize': 15,
'keyword': 'B01',
'username': 'member',
'operatorId': 23,
}, proxy.call_args.kwargs['params'])
def test_import_binds_session_operator_to_java_request(self):
self._login(31)
java_response = {'success': True, 'message': '开始导入', 'data': {'importId': 'import-1'}}
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_dedupe_total_data_access',
return_value=('normal', {'id': 31, 'username': 'member-b'}, None)), \
patch.object(admin_module, '_proxy_backend_java',
return_value=(java_response, None, 200)) as proxy:
response = self.client.post(
'/api/admin/dedupe-total-data/import',
data={'file': (io.BytesIO(b'excel'), 'data.xlsx')},
content_type='multipart/form-data')
self.assertEqual(200, response.status_code)
self.assertEqual('import-1', response.get_json()['import_id'])
self.assertEqual({'operatorId': 31}, proxy.call_args.kwargs['data'])
def test_export_forwards_filters_and_session_operator(self):
self._login(31)
java_response = Mock(
status_code=200,
content=b'xlsx-data',
headers={
'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'Content-Disposition': 'attachment; filename="dedupe-total-data.xlsx"',
},
)
java_session = Mock()
java_session.get.return_value = java_response
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_dedupe_total_data_access',
return_value=('normal', {'id': 31, 'username': 'member-b'}, None)), \
patch.object(admin_module, '_get_backend_java_session', return_value=java_session):
response = self.client.get(
'/api/admin/dedupe-total-data/export'
'?username=member&start_date=2026-07-01&end_date=2026-07-20')
self.assertEqual(200, response.status_code)
self.assertEqual(b'xlsx-data', response.data)
self.assertEqual({
'operatorId': 31,
'username': 'member',
'startDate': '2026-07-01',
'endDate': '2026-07-20',
}, java_session.get.call_args.kwargs['params'])
self.assertEqual(60, java_session.get.call_args.kwargs['timeout'])
def test_delete_ignores_client_operator_and_uses_session_user(self):
self._login(41)
java_response = {'success': True, 'message': '删除成功', 'data': None}
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_dedupe_total_data_access',
return_value=('admin', {'id': 41, 'username': 'leader'}, None)), \
patch.object(admin_module, '_proxy_backend_java',
return_value=(java_response, None, 200)) as proxy:
response = self.client.delete('/api/admin/dedupe-total-data/91?operatorId=1')
self.assertEqual(200, response.status_code)
self.assertEqual({'operatorId': 41}, proxy.call_args.kwargs['params'])
def test_menu_permission_is_still_required_for_normal_user(self):
self._login()
def deny_access():
return 'normal', {'id': 23}, (jsonify({'success': False, 'error': '无权访问'}), 403)
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_dedupe_total_data_access', side_effect=deny_access), \
patch.object(admin_module, '_proxy_backend_java') as proxy:
response = self.client.get('/api/admin/dedupe-total-data')
self.assertEqual(403, response.status_code)
self.assertFalse(response.get_json()['success'])
proxy.assert_not_called()
def test_dedupe_access_uses_login_capable_backend_menu_check(self):
expected = ('normal', {'id': 23}, None)
with patch.object(admin_module, '_ensure_backend_menu_access', return_value=expected) as backend_access, \
patch.object(admin_module, '_ensure_admin_menu_access') as admin_access:
result = admin_module._ensure_dedupe_total_data_access()
self.assertEqual(expected, result)
backend_access.assert_called_once_with('dedupe-total-data')
admin_access.assert_not_called()
def test_group_list_accepts_dedupe_menu_as_access_source(self):
self._login()
access_result = ('normal', {'id': 23, 'username': 'member-a'}, None)
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_backend_menu_access',
return_value=access_result) as backend_access, \
patch.object(admin_module, '_load_expanded_shop_manage_groups',
return_value=([], None, 200)):
response = self.client.get('/api/admin/shop-manage-groups')
self.assertEqual(200, response.status_code)
self.assertTrue(response.get_json()['success'])
backend_access.assert_called_once_with(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data')
if __name__ == '__main__':
unittest.main()
-340
View File
@@ -1,340 +0,0 @@
import io
import json
import unittest
import zipfile
from datetime import datetime
from unittest.mock import MagicMock, patch
from app import app
from blueprints import admin_api as admin_module
class FakeCursor:
def __init__(self, rows):
self.rows = rows
self.last_sql = ''
self.executions = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, sql, params=()):
self.last_sql = sql
self.executions.append((sql, params))
def fetchall(self):
return self.rows
def fetchone(self):
if 'COUNT(*)' in self.last_sql:
return {'total': len(self.rows)}
return self.rows[0] if self.rows else None
class FakeConnection:
def __init__(self, rows):
self.cursor_value = FakeCursor(rows)
def cursor(self):
return self.cursor_value
def close(self):
pass
class FakeDownloadResponse:
def __init__(self, content=b'video-bytes', error=None):
self.content = content
self.error = error
self.closed = False
def raise_for_status(self):
if self.error:
raise self.error
def iter_content(self, chunk_size):
yield self.content
def close(self):
self.closed = True
class ImageVideoAdminTest(unittest.TestCase):
def setUp(self):
app.config.update(TESTING=True, SECRET_KEY='test')
self.client = app.test_client()
def test_masks_nested_secrets_and_resolves_mode(self):
payload = {
'parameters': {
'api_key_info': {'ai_conductor_key': 'secret', 't8_video_key': 'secret-2'},
'video_info': {'mode': '2'},
},
'token': 'secret-3',
}
masked = admin_module._mask_image_video_secrets(payload)
self.assertEqual('******', masked['parameters']['api_key_info']['ai_conductor_key'])
self.assertEqual('******', masked['parameters']['api_key_info']['t8_video_key'])
self.assertEqual('******', masked['token'])
self.assertEqual('视频复刻', admin_module._image_video_mode(payload))
def test_admin_access_clause_is_limited_to_self_and_direct_users(self):
sql, params = admin_module._image_video_access_sql('admin', {'id': 17})
self.assertIn('created_by_id = %s', sql)
self.assertEqual([17, 17], params)
def test_list_api_applies_pagination_and_returns_task_summary(self):
row = {
'id': 91,
'user_id': 21,
'username': 'demo',
'group_name': '测试分组、第二分组',
'status': 'SUCCESS',
'request_json': json.dumps({'parameters': {'video_info': {'mode': '1'}}}),
'submit_response_json': '{}',
'result_json': '{}',
'video_urls_json': json.dumps(['https://coze.example/video.mp4']),
'debug_url': 'https://coze.example/debug',
'archived_videos_json': json.dumps([{
'sourceUrl': 'https://coze.example/video.mp4',
'url': 'https://oss.example/video.mp4',
'objectKey': 'result/image_video/1/video.mp4',
'status': 'SUCCESS',
}]),
'archive_status': 'SUCCESS',
'archive_error': None,
'archive_attempt_count': 1,
'archived_at': datetime(2026, 7, 15, 12, 5),
'error_message': None,
'coze_execute_id': 'exec-91',
'coze_status': 'SUCCESS',
'submitted_at': datetime(2026, 7, 15, 12, 0),
'completed_at': datetime(2026, 7, 15, 12, 4),
}
connection = FakeConnection([row])
with self.client.session_transaction() as session:
session['user_id'] = 17
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch('utils.auth.get_current_admin_role', return_value=('admin', {'id': 17})), \
patch.object(admin_module, '_ensure_backend_menu_access', return_value=('admin', {'id': 17}, None)), \
patch.object(admin_module, '_ensure_image_video_data_access', return_value=('admin', {'id': 17}, None)), \
patch.object(admin_module, 'get_db', return_value=connection):
response = self.client.get('/api/admin/image-video-tasks?page=2&page_size=20&status=success')
self.assertEqual(200, response.status_code)
data = response.get_json()
self.assertTrue(data['success'])
self.assertEqual(2, data['page'])
self.assertEqual('测试分组、第二分组', data['items'][0]['group_name'])
self.assertEqual('图生视频', data['items'][0]['mode'])
self.assertEqual('https://oss.example/video.mp4', data['items'][0]['video_url'])
self.assertEqual([{
'source_url': 'https://coze.example/video.mp4',
'archived_url': 'https://oss.example/video.mp4',
'object_key': 'result/image_video/1/video.mp4',
'archive_status': 'SUCCESS',
'archive_error': '',
'display_url': 'https://oss.example/video.mp4',
}], data['items'][0]['videos'])
self.assertNotIn('request', data['items'][0])
self.assertNotIn('submit_response', data['items'][0])
self.assertNotIn('result', data['items'][0])
list_sql, list_params = connection.cursor_value.executions[0]
self.assertIn("t.task_type = 'IMAGE_VIDEO_WORKFLOW'", list_sql)
self.assertIn('t.submitted_at >= DATE_SUB(NOW(), INTERVAL 3 DAY)', list_sql)
self.assertNotIn('created_by_id = %s', list_sql)
self.assertEqual(('SUCCESS', 20, 20), list_params)
def test_authorized_normal_user_receives_all_video_tasks(self):
connection = FakeConnection([])
with self.client.session_transaction() as session:
session['user_id'] = 23
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 23}, None)), \
patch.object(admin_module, '_ensure_image_video_data_access', return_value=('normal', {'id': 23}, None)), \
patch.object(admin_module, 'get_db', return_value=connection):
response = self.client.get('/api/admin/image-video-tasks')
self.assertEqual(200, response.status_code)
list_sql, list_params = connection.cursor_value.executions[0]
self.assertNotIn('t.user_id = %s', list_sql)
self.assertEqual((20, 0), list_params)
def test_batch_download_returns_single_zip_with_selected_video(self):
row = {
'id': 91,
'video_urls_json': json.dumps(['https://coze.example/video.mp4']),
'archived_videos_json': json.dumps([{'url': 'https://oss.example/video.mp4'}]),
}
connection = FakeConnection([row])
remote_response = FakeDownloadResponse()
with self.client.session_transaction() as session:
session['user_id'] = 23
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 23}, None)), \
patch.object(admin_module, '_ensure_image_video_data_access', return_value=('normal', {'id': 23}, None)), \
patch.object(admin_module, 'get_db', return_value=connection), \
patch.object(admin_module.requests, 'get', return_value=remote_response):
response = self.client.post('/api/admin/image-video-tasks/download-zip', json={
'items': [{'task_id': 91, 'video_index': 0}],
})
self.assertEqual(200, response.status_code)
self.assertEqual('application/zip', response.mimetype)
self.assertEqual('1', response.headers['X-Archive-File-Count'])
with zipfile.ZipFile(io.BytesIO(response.get_data())) as archive:
self.assertEqual(['task-91-video-1.mp4'], archive.namelist())
self.assertEqual(b'video-bytes', archive.read('task-91-video-1.mp4'))
response.close()
self.assertTrue(remote_response.closed)
def test_batch_download_writes_failures_into_zip(self):
row = {
'id': 91,
'video_urls_json': json.dumps(['https://coze.example/video.mp4']),
'archived_videos_json': '[]',
}
connection = FakeConnection([row])
remote_response = FakeDownloadResponse(error=admin_module.requests.RequestException('upstream failed'))
with self.client.session_transaction() as session:
session['user_id'] = 23
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 23}, None)), \
patch.object(admin_module, '_ensure_image_video_data_access', return_value=('normal', {'id': 23}, None)), \
patch.object(admin_module, 'get_db', return_value=connection), \
patch.object(admin_module.requests, 'get', return_value=remote_response):
response = self.client.post('/api/admin/image-video-tasks/download-zip', json={
'items': [{'task_id': 91, 'video_index': 0}],
})
self.assertEqual(200, response.status_code)
self.assertEqual('1', response.headers['X-Archive-Error-Count'])
with zipfile.ZipFile(io.BytesIO(response.get_data())) as archive:
self.assertEqual(['download-errors.txt'], archive.namelist())
self.assertIn('下载失败', archive.read('download-errors.txt').decode('utf-8'))
response.close()
def test_unassigned_normal_user_is_denied_video_tasks(self):
denied = (app.response_class('{"success":false}', content_type='application/json'), 403)
with self.client.session_transaction() as session:
session['user_id'] = 24
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, '_ensure_backend_menu_access', return_value=('normal', {'id': 24}, denied)):
response = self.client.get('/api/admin/image-video-tasks')
self.assertEqual(403, response.status_code)
def test_only_super_admin_can_manage_image_video_permissions(self):
with self.client.session_transaction() as session:
session['user_id'] = 17
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, 'get_current_admin_role', return_value=('admin', {'id': 17})):
response = self.client.get('/api/admin/image-video-task-permissions')
self.assertEqual(403, response.status_code)
def test_super_admin_lists_image_video_permission_users(self):
cursor = MagicMock()
cursor.__enter__.return_value = cursor
cursor.__exit__.return_value = False
cursor.fetchone.return_value = {'id': 75}
cursor.fetchall.return_value = [
{'id': 2, 'username': 'admin-a', 'role': 'admin', 'is_admin': 1, 'created_by_id': 1, 'granted': 1},
{'id': 3, 'username': 'user-a', 'role': 'normal', 'is_admin': 0, 'created_by_id': 2, 'granted': 0},
]
connection = MagicMock()
connection.cursor.return_value = cursor
with self.client.session_transaction() as session:
session['user_id'] = 1
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, 'get_current_admin_role', return_value=('super_admin', {'id': 1})), \
patch.object(admin_module, 'get_db', return_value=connection):
response = self.client.get('/api/admin/image-video-task-permissions')
self.assertEqual(200, response.status_code)
self.assertEqual([True, False], [item['granted'] for item in response.get_json()['items']])
def test_super_admin_replaces_only_image_video_permissions(self):
cursor = MagicMock()
cursor.__enter__.return_value = cursor
cursor.__exit__.return_value = False
cursor.fetchone.return_value = {'id': 75}
cursor.fetchall.return_value = [{'id': 2}, {'id': 3}]
connection = MagicMock()
connection.cursor.return_value = cursor
with self.client.session_transaction() as session:
session['user_id'] = 1
with patch('utils.auth.is_session_user_valid', return_value=True), \
patch.object(admin_module, 'get_current_admin_role', return_value=('super_admin', {'id': 1})), \
patch.object(admin_module, 'get_db', return_value=connection):
response = self.client.put('/api/admin/image-video-task-permissions', json={'user_ids': [2, 3]})
self.assertEqual(200, response.status_code)
self.assertEqual(2, response.get_json()['granted_count'])
statements = [call.args[0] for call in cursor.execute.call_args_list]
self.assertTrue(any('DELETE ucp FROM user_column_permission' in sql for sql in statements))
self.assertEqual(2, sum('INSERT INTO user_column_permission' in sql for sql in statements))
connection.commit.assert_called_once()
def test_generic_permission_update_preserves_video_permission(self):
cursor = MagicMock()
cursor.fetchall.side_effect = [[{'id': 75}], [{'column_id': 75}]]
admin_module._set_user_column_permissions(
cursor,
7,
[2, 75],
preserve_column_keys=(admin_module.IMAGE_VIDEO_PERMISSION_KEY,),
)
inserts = [call.args[1] for call in cursor.execute.call_args_list if call.args[0].startswith('INSERT INTO')]
self.assertEqual([(7, 2), (7, 75)], inserts)
def test_video_items_fall_back_to_source_and_handle_empty_values(self):
row = {
'video_urls_json': json.dumps([
'https://coze.example/first.mp4',
'https://coze.example/second.mp4',
]),
'archived_videos_json': json.dumps([{
'sourceUrl': 'https://coze.example/first.mp4',
'status': 'FAILED',
'error': 'archive failed',
}]),
}
videos = admin_module._image_video_urls(row)
self.assertEqual(2, len(videos))
self.assertEqual('https://coze.example/first.mp4', videos[0]['display_url'])
self.assertEqual('FAILED', videos[0]['archive_status'])
self.assertEqual('archive failed', videos[0]['archive_error'])
self.assertEqual('https://coze.example/second.mp4', videos[1]['display_url'])
self.assertEqual([], admin_module._image_video_urls({
'video_urls_json': None,
'archived_videos_json': 'not-json',
}))
def test_detail_item_returns_masked_request_and_responses(self):
row = {
'id': 8,
'request_json': json.dumps({'token': 'secret', 'name': 'demo'}),
'submit_response_json': json.dumps({'execute_id': 'exec-8'}),
'result_json': json.dumps({'status': 'SUCCESS'}),
'video_urls_json': '[]',
'archived_videos_json': '[]',
}
item = admin_module._image_video_admin_item(row, include_json=True)
self.assertEqual('******', item['request']['token'])
self.assertEqual('demo', item['request']['name'])
self.assertEqual({'execute_id': 'exec-8'}, item['submit_response'])
self.assertEqual({'status': 'SUCCESS'}, item['result'])
if __name__ == '__main__':
unittest.main()
-38
View File
@@ -129,44 +129,6 @@ def init_db():
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS columns (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(128) NOT NULL COMMENT '栏目名',
column_key VARCHAR(64) NOT NULL COMMENT '栏目标识',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uk_column_key (column_key)
)
""")
try:
cur.execute("ALTER TABLE columns ADD COLUMN menu_type VARCHAR(20) NOT NULL DEFAULT 'app' COMMENT '菜单类型: app/admin' AFTER column_key")
except Exception:
pass
try:
cur.execute("ALTER TABLE columns ADD COLUMN route_path VARCHAR(255) NOT NULL DEFAULT '' COMMENT '菜单路由或页面标识' AFTER menu_type")
except Exception:
pass
try:
cur.execute("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path")
except Exception:
pass
try:
cur.execute("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''")
except Exception:
pass
try:
cur.execute("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0")
except Exception:
pass
cur.execute("""
CREATE TABLE IF NOT EXISTS user_column_permission (
user_id INT NOT NULL,
column_id INT NOT NULL,
PRIMARY KEY (user_id, column_id),
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
)
""")
try:
cur.execute("UPDATE users SET role = 'normal' WHERE (role IS NULL OR role = '') AND (is_admin = 0 OR is_admin IS NULL)")
cur.execute("SELECT MIN(id) AS mid FROM users WHERE is_admin = 1")
+144 -5
View File
@@ -713,6 +713,136 @@
overflow: hidden;
}
.column-permission-tree {
max-height: 280px;
overflow: auto;
border: 1px solid #ddd;
border-radius: 6px;
padding: 8px 10px;
background: #fff;
}
.column-tree-node {
display: block;
padding: 4px 0;
}
.column-tree-group + .column-tree-group {
margin-top: 8px;
}
.column-tree-group-toggle {
display: flex;
align-items: center;
width: 100%;
min-height: 36px;
padding: 6px 10px;
border: 0;
border-radius: 4px;
background: #f3f5f8;
color: #24292f;
cursor: pointer;
text-align: left;
}
.column-tree-group-toggle::before {
content: '\203A';
flex: 0 0 20px;
font-size: 20px;
line-height: 1;
color: #667085;
transition: transform 0.15s ease;
}
.column-tree-group-toggle[aria-expanded="true"]::before {
transform: rotate(90deg);
}
.column-tree-group-toggle:hover {
background: #e9edf3;
}
.column-tree-group-toggle:focus-visible {
outline: 2px solid #3159d8;
outline-offset: -2px;
}
.column-tree-group-name {
font-size: 13px;
font-weight: 600;
}
.column-tree-group-count {
margin-left: auto;
font-size: 12px;
color: #667085;
}
.column-tree-group-body {
padding-top: 4px;
}
.column-tree-row {
display: flex;
align-items: center;
min-height: 28px;
}
.column-tree-node label {
display: flex;
align-items: center;
gap: 7px;
margin: 0;
min-height: 28px;
font-size: 13px;
flex: 1;
min-width: 0;
}
.column-tree-toggle,
.column-tree-toggle-spacer {
flex: 0 0 24px;
width: 24px;
height: 28px;
}
.column-tree-toggle {
display: inline-flex;
align-items: center;
justify-content: center;
padding: 0;
border: 0;
background: transparent;
color: #666;
cursor: pointer;
}
.column-tree-toggle::before {
content: '\203A';
display: block;
font-size: 20px;
line-height: 1;
transition: transform 0.15s ease;
}
.column-tree-toggle[aria-expanded="true"]::before {
transform: rotate(90deg);
}
.column-tree-toggle:hover {
color: #3159d8;
}
.column-tree-toggle:focus-visible {
outline: 2px solid #3159d8;
outline-offset: -2px;
}
.column-tree-node[data-depth="1"] > .column-tree-row { padding-left: 22px; }
.column-tree-node[data-depth="2"] > .column-tree-row { padding-left: 44px; }
.column-tree-node input[type="checkbox"] { width: auto; }
.column-tree-node .inherited-label { color: #999; }
.multi-select-native {
position: absolute;
opacity: 0;
@@ -1416,9 +1546,9 @@
</select>
</div>
<div class="form-group" id="formGroupCreateColumns">
<label>菜单权限(可多选,Ctrl/Shift 点选</label>
<label>菜单权限(勾选父菜单会继承全部子菜单</label>
<div id="createColumnCards" class="column-permission-cards"></div>
<select id="createColumnPermissionWrap" class="column-permission-select" multiple></select>
<div id="createColumnPermissionWrap" class="column-permission-tree"></div>
</div>
<button class="btn" id="btnCreate">创建用户</button>
<p class="msg" id="msgCreate"></p>
@@ -1484,6 +1614,10 @@
<option value="app">软件(app)</option>
</select>
</div>
<div class="form-group" style="min-width:220px;">
<label>父菜单(可选)</label>
<select id="columnParentId"><option value="">无(根菜单)</option></select>
</div>
<button class="btn" id="btnAddColumn">新增菜单</button>
</div>
<p class="msg" id="msgColumn"></p>
@@ -1497,6 +1631,7 @@
<th>菜单名称</th>
<th>栏目标识</th>
<th>菜单类型</th>
<th>父菜单</th>
<th>排序</th>
<th>菜单路由</th>
<th>创建时间</th>
@@ -2233,9 +2368,9 @@
<input type="text" id="editCreatorName" readonly style="background:#f5f5f5;">
</div>
<div class="form-group">
<label>菜单权限(可多选,Ctrl/Shift 点选</label>
<label>菜单权限(勾选父菜单会继承全部子菜单</label>
<div id="editColumnCards" class="column-permission-cards"></div>
<select id="editColumnPermissionWrap" class="column-permission-select" multiple></select>
<div id="editColumnPermissionWrap" class="column-permission-tree"></div>
</div>
<p class="msg" id="msgEdit"></p>
<div style="margin-top:16px;display:flex;gap:8px;">
@@ -2311,6 +2446,10 @@
<option value="app">软件(app)</option>
</select>
</div>
<div class="form-group">
<label>父菜单(可选)</label>
<select id="editColumnParentId"><option value="">无(根菜单)</option></select>
</div>
<p class="msg" id="msgEditColumn"></p>
<div style="margin-top:16px;display:flex;gap:8px;">
<button class="btn" id="btnSaveColumn">保存</button>
@@ -2607,7 +2746,7 @@
</div>
</div>
</div>
<script src="/static/admin.js?v=shop-key-remark-1"></script>
<script src="/static/admin.js?v=permission-tree-groups-1"></script>
</body>
</html>
+12
View File
@@ -0,0 +1,12 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>店铺数据抓取 - 数富AI</title>
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/shop-data-crawl-main.ts"></script>
</body>
</html>
@@ -128,8 +128,7 @@
>
<div class="task-section-header">
<div class="task-title-wrap">
<strong>{{ detail.task.taskNo || `任务 ${detail.task.id}` }}</strong>
<span class="task-meta">任务 ID{{ detail.task.id }}</span>
<strong>上架任务 #{{ detail.task.id }}</strong>
<span class="task-meta">创建时间{{ formatDateTime(detail.task.createdAt) }}</span>
</div>
<div class="task-actions">
@@ -144,6 +143,14 @@
>
下载结果
</button>
<button
type="button"
class="btn-delete"
:disabled="isDeletingTask(detail.task.id)"
@click="deleteTaskRecord(detail)"
>
{{ isDeletingTask(detail.task.id) ? '删除中...' : '删除' }}
</button>
</div>
</div>
@@ -163,17 +170,19 @@
<span v-if="file.platform">平台{{ file.platform }}</span>
<span>数据{{ fileProgressCurrent(file) }}/{{ fileProgressTotal(file) }}</span>
</div>
<div class="file-progress-header">
<span>{{ file.progressMessage || statusText(file.status) }}</span>
<span>{{ fileProgressPercent(file) }}%</span>
</div>
<div class="file-progress-track">
<div
class="file-progress-fill"
:class="statusClass(file.status)"
:style="{ width: `${fileProgressPercent(file)}%` }"
/>
</div>
<template v-if="shouldShowFileProgress(file)">
<div class="file-progress-header">
<span>{{ file.progressMessage || statusText(file.status) }}</span>
<span>{{ fileProgressPercent(file) }}%</span>
</div>
<div class="file-progress-track">
<div
class="file-progress-fill"
:class="statusClass(file.status)"
:style="{ width: `${fileProgressPercent(file)}%` }"
/>
</div>
</template>
<div v-if="fileErrorText(file)" class="file-error">{{ fileErrorText(file) }}</div>
</div>
<span class="status file-status" :class="statusClass(file.status)">
@@ -191,7 +200,7 @@
<script setup lang="ts">
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue'
import { ElMessage } from 'element-plus'
import { ElMessage, ElMessageBox } from 'element-plus'
import BrandTopBar from './BrandTopBar.vue'
import ZiniaoVersionSetting from '@/shared/components/ZiniaoVersionSetting.vue'
@@ -199,6 +208,7 @@ import { expandBrandFolderRecursive, type BrandExpandFolderItem } from '@/shared
import {
activatePublishFile,
activatePublishTask,
deletePublishTask,
getPublishDashboard,
getPublishHistory,
getPublishItemsPageUrl,
@@ -261,6 +271,7 @@ const queuedBatches = ref<PublishQueueBatch[]>([])
const taskSnapshots = ref<Record<number, PublishTaskDetailVo>>({})
const historyItems = ref<PublishTaskDetailVo[]>([])
const missingTaskIds = ref<number[]>([])
const deletingTaskIds = ref<number[]>([])
const dashboard = ref<PublishDashboardVo>({
pendingCount: 0,
runningCount: 0,
@@ -990,6 +1001,10 @@ function fileErrorText(file: PublishFileItem) {
return file.errorMessage || file.error || ''
}
function shouldShowFileProgress(file: PublishFileItem) {
return ['RUNNING', 'SUCCESS', 'COMPLETED', 'FAILED', 'CANCELLED'].includes(normalizeStatus(file.status))
}
function fileProgressCurrent(file: PublishFileItem) {
return Math.max(0, Number(file.progressCurrent ?? file.processedRows ?? 0))
}
@@ -1038,6 +1053,58 @@ async function downloadResult(detail: PublishTaskDetailVo) {
else if (saved.error && saved.error !== '用户取消') ElMessage.error(saved.error)
}
function isDeletingTask(taskId: number) {
return deletingTaskIds.value.includes(taskId)
}
function removeTaskLocally(taskId: number) {
progressLoop.remove(taskId)
historyItems.value = historyItems.value.filter((detail) => detail.task?.id !== taskId)
const snapshots = { ...taskSnapshots.value }
delete snapshots[taskId]
taskSnapshots.value = snapshots
missingTaskIds.value = Array.from(new Set([...missingTaskIds.value, taskId]))
queuedBatches.value = queuedBatches.value.filter((batch) => batch.taskId !== taskId)
if (currentTaskId.value === taskId) {
currentTaskId.value = null
currentFiles.value = []
pendingFileIds.value = []
activeFileId.value = null
dispatchOptions.value = null
}
saveQueueState()
}
async function deleteTaskRecord(detail: PublishTaskDetailVo) {
const taskId = detail.task?.id
if (!taskId || isDeletingTask(taskId)) return
try {
await ElMessageBox.confirm(
`确定删除上架任务 #${taskId}?任务明细和结果文件也会一并删除。`,
'删除任务',
{
confirmButtonText: '删除',
cancelButtonText: '取消',
type: 'warning',
},
)
} catch {
return
}
deletingTaskIds.value = [...deletingTaskIds.value, taskId]
try {
await deletePublishTask(taskId)
removeTaskLocally(taskId)
ElMessage.success('任务已删除')
await Promise.all([loadDashboard(), loadHistory()])
} catch (error) {
ElMessage.error(error instanceof Error ? error.message : '删除任务失败')
} finally {
deletingTaskIds.value = deletingTaskIds.value.filter((id) => id !== taskId)
}
}
onMounted(async () => {
loadQueueState()
if (currentTaskId.value) progressLoop.add(currentTaskId.value)
@@ -1069,7 +1136,7 @@ onBeforeUnmount(() => {
.hint, .loading-msg, .task-meta, .file-info, .history-count { color: #888; font-size: 12px; line-height: 1.5; }
.btns, .run-row, .task-actions { display: flex; align-items: center; flex-wrap: wrap; gap: 10px; }
.btns { justify-content: center; }
.opt-btn, .btn-run, .download { border: 0; border-radius: 6px; cursor: pointer; transition: background-color .15s ease, color .15s ease, opacity .15s ease; }
.opt-btn, .btn-run, .download, .btn-delete { border: 0; border-radius: 6px; cursor: pointer; transition: background-color .15s ease, color .15s ease, opacity .15s ease; }
.opt-btn { padding: 8px 16px; border: 1px solid #3a3a3a; background: #2a2a2a; color: #ccc; font-size: 13px; }
.opt-btn:hover:not(:disabled) { border-color: #3498db; color: #67b7ef; }
.opt-btn:disabled, .btn-run:disabled { opacity: .55; cursor: not-allowed; }
@@ -1105,6 +1172,9 @@ onBeforeUnmount(() => {
.task-error { padding: 10px 16px 0; }
.download { padding: 7px 12px; background: #2d6b46; color: #dff7e8; font-size: 12px; white-space: nowrap; }
.download:hover { background: #367d54; }
.btn-delete { padding: 7px 12px; background: #472929; color: #efaaaa; font-size: 12px; white-space: nowrap; }
.btn-delete:hover:not(:disabled) { background: #5a3030; color: #ffd0d0; }
.btn-delete:disabled { cursor: wait; opacity: .55; }
.status { display: inline-flex; align-items: center; justify-content: center; min-width: 58px; min-height: 26px; padding: 0 8px; border-radius: 4px; font-size: 12px; white-space: nowrap; }
.status.pending { background: #3a3424; color: #e4c56a; }
.status.running { background: #24394a; color: #75bff1; }
@@ -0,0 +1,604 @@
<template>
<div class="page-shell module-page">
<BrandTopBar active="shop-data-crawl" />
<div class="main-content">
<aside class="left-panel">
<div class="section-title">店铺输入</div>
<div class="input-zone">
<div class="input-row">
<el-input v-model="shopInput" clearable placeholder="请输入店铺名" @keyup.enter="confirmAdd" />
<button type="button" class="opt-btn" :disabled="adding" @click="confirmAdd">
{{ adding ? '添加中...' : '添加' }}
</button>
</div>
</div>
<div class="section-title">备选区</div>
<div v-if="!candidates.length" class="empty-candidates">暂无备选店铺</div>
<div v-else class="candidate-table-scroll">
<el-table :data="candidates" row-key="id" height="250" class="candidate-table"
@selection-change="onSelectionChange">
<el-table-column type="selection" width="42" />
<el-table-column prop="shop_name" label="店铺名" min-width="140" show-overflow-tooltip />
<el-table-column label="操作" width="64" align="center">
<template #default="{ row }">
<button type="button" class="link-danger" @click="removeCandidate(row.id)">删除</button>
</template>
</el-table-column>
</el-table>
</div>
<div class="section-title">抓取国家与顺序</div>
<div class="country-pref-checks">
<label v-for="row in countryCheckboxRows" :key="row.code" class="country-check-row">
<input type="checkbox" class="country-check-input" :checked="isCountrySelected(row.code)"
:disabled="isCountrySelectionLocked(row.code)" @change="onCountryNativeChange(row.code, $event)" />
<span>{{ row.label }}{{ row.code }}</span>
</label>
</div>
<div class="country-order-panel">
<div class="country-order-caption">已选顺序</div>
<div class="country-order-list">
<div v-for="(code, index) in orderedCountryCodes" :key="code" class="country-drag-row"
:class="{ dragging: dragCountryIndex === index }" draggable="true"
@dragstart="dragCountryIndex = index" @dragend="dragCountryIndex = null"
@dragover.prevent @drop.prevent="onCountryDrop(index)">
<span class="drag-handle" title="拖动排序"></span>
<span>{{ countryLabel(code) }}{{ code }}</span>
</div>
</div>
</div>
<div v-if="countryPrefSaving" class="country-pref-status">保存中...</div>
<ZiniaoVersionSetting v-model="ziniaoVersion" />
<div class="run-row">
<button type="button" class="btn-run" :disabled="matching" @click="runMatch">
{{ matching ? '匹配中...' : '匹配店铺' }}
</button>
<button type="button" class="btn-run btn-queue" :disabled="isQueueBusy || !matchedRunnableItems.length"
@click="startQueue">
{{ isQueueBusy ? '串行抓取中...' : '开始串行抓取' }}
</button>
</div>
<p v-if="queueStatus" class="queue-status">{{ queueStatus }}</p>
</aside>
<section class="right-panel">
<div class="panel-header">匹配与任务</div>
<div class="task-list-wrap">
<div class="clean-result-summary">
<div class="summary-card"><span class="summary-label">备选店铺</span><strong>{{ dashboard.candidateCount }}</strong></div>
<div class="summary-card"><span class="summary-label">已处理任务</span><strong>{{ dashboard.processedTaskCount }}</strong></div>
<div class="summary-card"><span class="summary-label">成功任务</span><strong>{{ dashboard.successTaskCount }}</strong></div>
<div class="summary-card"><span class="summary-label">失败任务</span><strong>{{ dashboard.failedTaskCount }}</strong></div>
</div>
<div class="subsection-title">匹配结果</div>
<div v-if="!matchedItems.length" class="empty-tasks narrow">匹配后将在这里显示结果</div>
<el-table v-else :data="matchedItems" :row-key="rowKey" :highlight-current-row="false"
class="result-table match-table">
<el-table-column prop="shopName" label="店铺名" min-width="120" />
<el-table-column label="匹配" width="64" align="center">
<template #default="{ row }"><span :class="row.matched ? 'ok' : 'fail'">{{ row.matched ? '是' : '否' }}</span></template>
</el-table-column>
<el-table-column prop="shopId" label="店铺 ID" min-width="120" show-overflow-tooltip />
<el-table-column prop="platform" label="平台" width="88" show-overflow-tooltip />
<el-table-column prop="companyName" label="公司" min-width="120" show-overflow-tooltip />
<el-table-column label="状态" width="100"><template #default="{ row }">{{ formatMatchStatus(row.matchStatus) }}</template></el-table-column>
<el-table-column label="说明" min-width="150" show-overflow-tooltip><template #default="{ row }">{{ row.matchMessage || '-' }}</template></el-table-column>
<el-table-column label="操作" width="64" align="center"><template #default="{ row }">
<button type="button" class="link-danger" @click="removeMatchedRow(row)">删除</button>
</template></el-table-column>
</el-table>
<div class="result-list-wrap">
<div class="result-list-header"><span>任务记录</span></div>
<div v-if="!currentItems.length && !historySectionItems.length" class="empty-tasks">暂无任务记录</div>
<div v-if="currentItems.length" class="result-subsection">
<div class="result-subsection-title">当前任务</div>
<ul class="task-list">
<TaskRow v-for="item in currentItems" :key="historyKey(item)" :item="item"
@download="downloadResult" @delete="deleteTaskRecord" />
</ul>
</div>
<div v-if="historySectionItems.length" class="result-subsection">
<div class="result-subsection-title">历史记录</div>
<ul class="task-list">
<TaskRow v-for="item in historySectionItems" :key="historyKey(item)" :item="item"
@download="downloadResult" @delete="deleteTaskRecord" />
</ul>
</div>
</div>
</div>
</section>
</div>
</div>
</template>
<script setup lang="ts">
import { computed, defineComponent, h, onMounted, onUnmounted, ref } from 'vue'
import { ElMessage } from 'element-plus'
import BrandTopBar from '@/pages/brand/components/BrandTopBar.vue'
import ZiniaoVersionSetting from '@/shared/components/ZiniaoVersionSetting.vue'
import { useZiniaoVersion } from '@/shared/utils/ziniao-version'
import { getPywebviewApi } from '@/shared/bridges/pywebview'
import { getTaskPollIntervalMs } from '@/shared/task-progress-config'
import { createCategorizedTimers } from '@/shared/utils/categorized-timers'
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
import {
addShopDataCrawlCandidate,
createShopDataCrawlTask,
deleteShopDataCrawlCandidate,
deleteShopDataCrawlHistory,
deleteShopDataCrawlTask,
getShopDataCrawlCountryPreference,
getShopDataCrawlDashboard,
getShopDataCrawlHistory,
getShopDataCrawlResultDownloadUrl,
getShopDataCrawlTaskProgressBatch,
listShopDataCrawlCandidates,
matchShopDataCrawlShops,
putShopDataCrawlCountryPreference,
type ShopDataCrawlCandidateVo,
type ShopDataCrawlDashboardVo,
type ShopDataCrawlHistoryItem,
type ShopDataCrawlShopItem,
type ShopDataCrawlTaskDetailVo,
} from '@/shared/api/java-modules'
const COUNTRY_OPTIONS = [
{ code: 'UK', label: '英国' },
{ code: 'DE', label: '德国' },
{ code: 'FR', label: '法国' },
{ code: 'ES', label: '西班牙' },
{ code: 'IT', label: '意大利' },
] as const
const TaskRow = defineComponent({
props: { item: { type: Object as () => ShopDataCrawlHistoryItem, required: true } },
emits: ['download', 'delete'],
setup(props, { emit }) {
return () => h('li', { class: 'task-item' }, [
h('div', { class: 'left split-result-main' }, [
h('span', { class: 'id', title: props.item.shopName || '' }, props.item.shopName || '-'),
h('div', { class: 'files' }, `任务 ID${props.item.taskId ?? '-'}`),
props.item.platform ? h('div', { class: 'files' }, `平台:${props.item.platform}`) : null,
props.item.outputFilename ? h('div', { class: 'files' }, `文件:${props.item.outputFilename}`) : null,
props.item.createdAt ? h('div', { class: 'files' }, `创建时间:${formatDateTime(props.item.createdAt)}`) : null,
props.item.error ? h('div', { class: 'files error-text' }, `错误:${props.item.error}`) : null,
]),
h('div', { class: 'task-right' }, [
h('span', { class: ['status', statusClass(props.item.taskStatus)] }, statusText(props.item.taskStatus)),
canDownload(props.item) ? h('button', { type: 'button', class: 'download', onClick: () => emit('download', props.item) }, '下载') : null,
h('button', { type: 'button', class: 'btn-delete', onClick: () => emit('delete', props.item) }, '删除'),
]),
])
},
})
const timers = createCategorizedTimers('shop-data-crawl')
const ziniaoVersion = useZiniaoVersion()
const shopInput = ref('')
const candidates = ref<ShopDataCrawlCandidateVo[]>([])
const selectedCandidates = ref<ShopDataCrawlCandidateVo[]>([])
const matchedItems = ref<ShopDataCrawlShopItem[]>([])
const historyItems = ref<ShopDataCrawlHistoryItem[]>([])
const dashboard = ref<ShopDataCrawlDashboardVo>({ candidateCount: 0, processedTaskCount: 0, successTaskCount: 0, failedTaskCount: 0 })
const orderedCountryCodes = ref<string[]>(COUNTRY_OPTIONS.map((row) => row.code))
const dragCountryIndex = ref<number | null>(null)
const countryPrefSaving = ref(false)
const countryPrefUserTouched = ref(false)
const adding = ref(false)
const matching = ref(false)
const queueWorkerRunning = ref(false)
const queueStatus = ref('')
const pendingQueue = ref<ShopDataCrawlShopItem[]>([])
const activeTaskId = ref<number | null>(null)
const activeQueueItem = ref<ShopDataCrawlShopItem | null>(null)
const activeDispatched = ref(false)
const activeCountryCodes = ref<string[]>([])
const activeZiniaoVersion = ref<'new' | 'old'>('new')
const pollingTaskIds = ref<number[]>([])
const taskSnapshots = ref<Record<number, ShopDataCrawlTaskDetailVo>>({})
const autoQueueEnabled = ref(false)
let countryPrefSaveTimer: number | null = null
let disposed = false
const matchedRunnableItems = computed(() => matchedItems.value.filter((item) => item.matched))
const currentItems = computed(() => historyItems.value.filter((item) => !isTerminal(item.taskStatus)))
const historySectionItems = computed(() => historyItems.value.filter((item) => isTerminal(item.taskStatus)))
const isQueueBusy = computed(() => queueWorkerRunning.value || !!activeTaskId.value || pendingQueue.value.length > 0)
const countryCheckboxRows = computed(() => {
const selected = new Set(orderedCountryCodes.value)
return [
...orderedCountryCodes.value.map((code) => ({ code, label: countryLabel(code) })),
...COUNTRY_OPTIONS.filter((row) => !selected.has(row.code)),
]
})
function uid() { return typeof window === 'undefined' ? '0' : window.localStorage.getItem('uid') || '0' }
function storageKey(name: string) { return `shop-data-crawl:${name}:${uid()}` }
function rowKey(item: ShopDataCrawlShopItem) { return `${(item.shopName || '').trim()}::${item.shopId || ''}` }
function historyKey(item: ShopDataCrawlHistoryItem) { return `${item.taskId || 0}:${item.resultId || 0}:${rowKey(item)}` }
function countryLabel(code: string) { return COUNTRY_OPTIONS.find((row) => row.code === code)?.label || code }
function isCountrySelected(code: string) { return orderedCountryCodes.value.includes(code) }
function isCountrySelectionLocked(code: string) { return orderedCountryCodes.value.length === 1 && orderedCountryCodes.value[0] === code }
function isTerminal(status?: string) { return status === 'SUCCESS' || status === 'FAILED' || status === 'COMPLETED' }
function statusText(status?: string) { return status === 'SUCCESS' || status === 'COMPLETED' ? '已完成' : status === 'FAILED' ? '失败' : '执行中' }
function statusClass(status?: string) { return status === 'SUCCESS' || status === 'COMPLETED' ? 'success' : status === 'FAILED' ? 'failed' : 'running' }
function canDownload(item: ShopDataCrawlHistoryItem) { return Boolean(item.resultId && (item.fileReady || item.downloadUrl)) }
function formatDateTime(value?: string) { if (!value) return '-'; const date = new Date(value); return Number.isNaN(date.getTime()) ? value : date.toLocaleString('zh-CN', { hour12: false }) }
function formatMatchStatus(status?: string) { return ({ MATCHED: '已匹配', PENDING: '待匹配', CONFLICT: '需确认', INDEX_STALE: '索引过期' } as Record<string, string>)[status || ''] || status || '-' }
function setStorage(name: string, value: unknown) {
if (typeof window !== 'undefined') window.localStorage.setItem(storageKey(name), JSON.stringify(value))
}
function loadLocalState() {
try { matchedItems.value = JSON.parse(window.localStorage.getItem(storageKey('matched')) || '[]') } catch { matchedItems.value = [] }
try {
const state = JSON.parse(window.localStorage.getItem(storageKey('queue')) || '{}') as { pendingQueue?: ShopDataCrawlShopItem[]; activeTaskId?: number | null; activeQueueItem?: ShopDataCrawlShopItem | null; activeDispatched?: boolean; activeCountryCodes?: string[]; activeZiniaoVersion?: 'new' | 'old'; autoQueueEnabled?: boolean }
pendingQueue.value = state.pendingQueue || []
activeTaskId.value = state.activeTaskId || null
activeQueueItem.value = state.activeQueueItem || null
activeDispatched.value = state.activeDispatched || false
activeCountryCodes.value = state.activeCountryCodes || []
activeZiniaoVersion.value = state.activeZiniaoVersion || 'new'
autoQueueEnabled.value = state.autoQueueEnabled || false
} catch { pendingQueue.value = []; activeTaskId.value = null; activeQueueItem.value = null }
try { pollingTaskIds.value = JSON.parse(window.localStorage.getItem(storageKey('polling-task-ids')) || '[]') } catch { pollingTaskIds.value = [] }
try { taskSnapshots.value = JSON.parse(window.localStorage.getItem(storageKey('task-snapshots')) || '{}') } catch { taskSnapshots.value = {} }
if (activeDispatched.value && activeQueueItem.value) removeMatchedRow(activeQueueItem.value)
}
function saveQueueState() {
setStorage('queue', { pendingQueue: pendingQueue.value, activeTaskId: activeTaskId.value, activeQueueItem: activeQueueItem.value, activeDispatched: activeDispatched.value, activeCountryCodes: activeCountryCodes.value, activeZiniaoVersion: activeZiniaoVersion.value, autoQueueEnabled: autoQueueEnabled.value })
setStorage('polling-task-ids', pollingTaskIds.value)
setStorage('task-snapshots', taskSnapshots.value)
}
function mergeItems(base: ShopDataCrawlShopItem[], incoming: ShopDataCrawlShopItem[]) {
const map = new Map(base.map((item) => [rowKey(item), item]))
for (const item of incoming) map.set(rowKey(item), item)
return [...map.values()]
}
function saveMatched() { setStorage('matched', matchedItems.value) }
function onSelectionChange(rows: ShopDataCrawlCandidateVo[]) { selectedCandidates.value = rows || [] }
function onCountryNativeChange(code: string, event: Event) {
const input = event.target as HTMLInputElement
if (!input.checked && isCountrySelectionLocked(code)) { input.checked = true; ElMessage.warning('至少保留 1 个国家'); return }
countryPrefUserTouched.value = true
orderedCountryCodes.value = input.checked ? [...orderedCountryCodes.value, code] : orderedCountryCodes.value.filter((item) => item !== code)
scheduleCountryPreferenceSave()
}
function onCountryDrop(toIndex: number) {
const from = dragCountryIndex.value
dragCountryIndex.value = null
if (from == null || from === toIndex) return
const next = [...orderedCountryCodes.value]
const [item] = next.splice(from, 1)
next.splice(toIndex, 0, item)
orderedCountryCodes.value = next
countryPrefUserTouched.value = true
scheduleCountryPreferenceSave()
}
function scheduleCountryPreferenceSave() {
if (countryPrefSaveTimer) timers.clearTimer('preference-save', countryPrefSaveTimer)
countryPrefSaveTimer = timers.setTimeout('preference-save', () => { countryPrefSaveTimer = null; void persistCountryPreference() }, 450)
}
async function persistCountryPreference() {
countryPrefSaving.value = true
try { await putShopDataCrawlCountryPreference(orderedCountryCodes.value) }
catch (error) { ElMessage.error(error instanceof Error ? error.message : '保存国家顺序失败') }
finally { countryPrefSaving.value = false }
}
async function loadCandidates() { candidates.value = await listShopDataCrawlCandidates() }
async function loadDashboard() { dashboard.value = await getShopDataCrawlDashboard() }
async function loadHistory() {
const localFailures = historyItems.value.filter((item) => !item.taskId && !item.resultId && item.taskStatus === 'FAILED')
historyItems.value = [...localFailures, ...((await getShopDataCrawlHistory()).items || [])]
}
async function refreshViews() { await Promise.all([loadDashboard(), loadHistory()]) }
async function confirmAdd() {
const name = shopInput.value.trim()
if (!name) { ElMessage.warning('请输入店铺名'); return }
adding.value = true
try { await addShopDataCrawlCandidate(name); shopInput.value = ''; await Promise.all([loadCandidates(), loadDashboard()]); ElMessage.success('已加入备选区') }
catch (error) { ElMessage.error(error instanceof Error ? error.message : '添加失败') }
finally { adding.value = false }
}
async function removeCandidate(id: number) {
try { await deleteShopDataCrawlCandidate(id); selectedCandidates.value = selectedCandidates.value.filter((item) => item.id !== id); await Promise.all([loadCandidates(), loadDashboard()]); ElMessage.success('已删除') }
catch (error) { ElMessage.error(error instanceof Error ? error.message : '删除失败') }
}
async function runMatch() {
const names = selectedCandidates.value.map((item) => item.shop_name).filter(Boolean)
if (!names.length) { ElMessage.warning('请先勾选备选店铺'); return }
matching.value = true
try {
const incoming = (await matchShopDataCrawlShops(names)).items || []
matchedItems.value = mergeItems(matchedItems.value, incoming)
saveMatched()
if (autoQueueEnabled.value) {
const activeKey = activeQueueItem.value ? rowKey(activeQueueItem.value) : ''
pendingQueue.value = mergeItems(
pendingQueue.value,
incoming.filter((item) => item.matched && rowKey(item) !== activeKey),
)
saveQueueState()
void processQueue()
}
ElMessage.success(`匹配完成,共 ${incoming.length}`)
} catch (error) { ElMessage.error(error instanceof Error ? error.message : '匹配失败') }
finally { matching.value = false }
}
function removeMatchedRow(item: ShopDataCrawlShopItem) {
const key = rowKey(item)
matchedItems.value = matchedItems.value.filter((row) => rowKey(row) !== key)
pendingQueue.value = pendingQueue.value.filter((row) => rowKey(row) !== key)
saveMatched(); saveQueueState()
}
function buildTaskItem(item: ShopDataCrawlShopItem): ShopDataCrawlShopItem { return { ...item } }
function mergeProgress(detail: ShopDataCrawlTaskDetailVo) {
const taskId = detail.task?.id || detail.items?.[0]?.taskId
if (!taskId) return
taskSnapshots.value = { ...taskSnapshots.value, [taskId]: detail }
const incoming = (detail.items || []).map((item) => ({ ...item, taskId: item.taskId || taskId, taskStatus: item.taskStatus || detail.task?.status, error: item.error || detail.task?.errorMessage }))
const map = new Map(historyItems.value.map((item) => [historyKey(item), item]))
for (const item of incoming) {
const existing = [...map.values()].find((row) => row.taskId === item.taskId && (row.resultId === item.resultId || !item.resultId))
if (existing) map.set(historyKey(existing), { ...existing, ...item })
else map.set(historyKey(item), item)
}
historyItems.value = [...map.values()]
saveQueueState()
}
function isTaskDetail(row: ShopDataCrawlTaskDetailVo | ShopDataCrawlHistoryItem): row is ShopDataCrawlTaskDetailVo {
return 'task' in row || 'items' in row
}
async function refreshTaskProgress(taskId: number) {
const batch = await getShopDataCrawlTaskProgressBatch([taskId])
for (const row of batch.items || []) {
if (isTaskDetail(row)) {
mergeProgress(row)
} else {
mergeProgress({ task: { id: row.taskId, status: row.taskStatus }, items: [row] })
}
}
return !(batch.missingTaskIds || []).includes(taskId)
}
function activeTaskStatus(taskId: number) {
return taskSnapshots.value[taskId]?.task?.status || historyItems.value.find((item) => item.taskId === taskId)?.taskStatus || ''
}
function sleep(ms: number) { return disposed ? Promise.resolve() : timers.sleep('queue-wait', ms) }
function isTransientError(error: unknown) { return /network|fetch|timeout|50[0234]|load failed|connection refused|econnrefused|无法连接.*后端|后端服务.*(?:不可用|连接)|网络错误/i.test(error instanceof Error ? error.message : String(error || '')) }
async function retry<T>(action: () => Promise<T>) {
let attempt = 0
while (!disposed) {
try { return await action() } catch (error) {
if (!isTransientError(error)) throw error
attempt += 1; queueStatus.value = `服务暂时不可用,正在重试(${attempt}`; await sleep(getTaskPollIntervalMs())
}
}
throw new Error('页面已关闭')
}
async function waitForTerminal(taskId: number) {
while (!disposed) {
try {
const exists = await refreshTaskProgress(taskId)
if (!exists) return 'FAILED'
const status = activeTaskStatus(taskId)
if (isTerminal(status)) return status
} catch (error) {
if (!isTransientError(error)) throw error
queueStatus.value = `任务 ${taskId} 执行中,等待服务恢复...`
}
await sleep(getTaskPollIntervalMs())
}
return 'STOPPED'
}
function clearActiveTask() {
const taskId = activeTaskId.value
activeTaskId.value = null; activeQueueItem.value = null; activeDispatched.value = false
activeCountryCodes.value = []; activeZiniaoVersion.value = 'new'
if (taskId) pollingTaskIds.value = pollingTaskIds.value.filter((id) => id !== taskId)
saveQueueState()
}
function recordLocalFailure(item: ShopDataCrawlShopItem, error: unknown) {
historyItems.value = [{ ...item, taskStatus: 'FAILED', success: false, error: error instanceof Error ? error.message : String(error || '任务失败'), createdAt: new Date().toISOString() }, ...historyItems.value]
}
async function dispatchActiveTask(api: NonNullable<ReturnType<typeof getPywebviewApi>>) {
if (!activeTaskId.value || !activeQueueItem.value || !api.enqueue_json) return false
const payload = {
type: 'shop-data-crawl-run',
ts: Date.now(),
data: {
taskId: activeTaskId.value,
ziniao_version: activeZiniaoVersion.value,
items: [buildTaskItem(activeQueueItem.value)],
country_codes: [...activeCountryCodes.value],
},
}
const pushed = await api.enqueue_json(payload)
if (!pushed?.success) throw new Error(pushed?.error || `任务 ${activeTaskId.value} 入队失败`)
activeDispatched.value = true
saveQueueState()
return true
}
async function processQueue() {
if (disposed || queueWorkerRunning.value) return
queueWorkerRunning.value = true
try {
const api = getPywebviewApi()
if (!api?.enqueue_json) throw new Error('当前客户端不支持任务队列')
while (!disposed && (activeTaskId.value || pendingQueue.value.length)) {
if (activeTaskId.value) {
if (!activeDispatched.value) {
const taskId = activeTaskId.value
const item = activeQueueItem.value
try {
await dispatchActiveTask(api)
if (item) removeMatchedRow(item)
queueStatus.value = `任务 ${taskId} 执行中,剩余 ${pendingQueue.value.length}`
} catch (error) {
await deleteShopDataCrawlTask(taskId).catch(() => undefined)
if (item) recordLocalFailure(item, error)
clearActiveTask()
queueStatus.value = `任务 ${taskId} 入队失败,继续下一店`
continue
}
}
const taskId = activeTaskId.value
const status = await waitForTerminal(taskId)
clearActiveTask()
queueStatus.value = `任务 ${taskId} ${status === 'SUCCESS' || status === 'COMPLETED' ? '已完成' : '失败'},剩余 ${pendingQueue.value.length}`
continue
}
const next = pendingQueue.value.shift()
saveQueueState()
if (!next) break
const countries = [...orderedCountryCodes.value]
let created
try {
created = await retry(() => createShopDataCrawlTask([buildTaskItem(next)], countries))
if (!created.taskId) throw new Error('后端未返回任务 ID')
} catch (error) {
if (disposed) return
recordLocalFailure(next, error)
queueStatus.value = `${next.shopName} 创建任务失败,继续下一店`
continue
}
activeTaskId.value = created.taskId
activeQueueItem.value = next
activeDispatched.value = false
activeCountryCodes.value = countries
activeZiniaoVersion.value = ziniaoVersion.value
pollingTaskIds.value = [...new Set([...pollingTaskIds.value, created.taskId])]
const createdItems = (created.items || []).map((item) => ({ ...item, taskId: item.taskId || created.taskId }))
taskSnapshots.value = { ...taskSnapshots.value, [created.taskId]: { task: { id: created.taskId, status: createdItems[0]?.taskStatus }, items: createdItems } }
historyItems.value = [...createdItems, ...historyItems.value.filter((item) => item.taskId !== created.taskId)]
saveQueueState()
continue
}
if (!disposed) {
autoQueueEnabled.value = false
saveQueueState()
queueStatus.value = '串行抓取已完成'
await refreshViews()
ElMessage.success('店铺数据抓取队列已完成')
}
} catch (error) {
if (!disposed) { queueStatus.value = error instanceof Error ? error.message : '队列执行失败'; ElMessage.error(queueStatus.value) }
} finally { queueWorkerRunning.value = false; saveQueueState() }
}
function startQueue() {
const runnable = matchedRunnableItems.value
if (!runnable.length) { ElMessage.warning('请先匹配可用店铺'); return }
autoQueueEnabled.value = true
pendingQueue.value = mergeItems(pendingQueue.value, runnable)
saveQueueState()
queueStatus.value = `已加入 ${runnable.length} 条店铺`
void processQueue()
}
async function downloadResult(item: ShopDataCrawlHistoryItem) {
if (!item.resultId) return
const filename = item.outputFilename || `${item.shopName || 'shop-data'}.xlsx`
const result = await saveUrlWithProgress(getShopDataCrawlResultDownloadUrl(item.resultId), filename, `shop-data-crawl:${item.resultId}`)
if (result.success) ElMessage.success(`已保存:${result.path || filename}`)
else if (result.error && result.error !== '用户取消') ElMessage.error(result.error)
}
async function deleteTaskRecord(item: ShopDataCrawlHistoryItem) {
try {
if (!item.taskId && !item.resultId) {
historyItems.value = historyItems.value.filter((row) => row !== item)
ElMessage.success('已删除')
return
}
if (item.taskId && !isTerminal(item.taskStatus)) await deleteShopDataCrawlTask(item.taskId)
else if (item.resultId) await deleteShopDataCrawlHistory(item.resultId)
else if (item.taskId) await deleteShopDataCrawlTask(item.taskId)
else throw new Error('缺少记录标识')
if (item.taskId === activeTaskId.value) clearActiveTask()
historyItems.value = historyItems.value.filter((row) => historyKey(row) !== historyKey(item))
await loadDashboard(); ElMessage.success('已删除')
} catch (error) { ElMessage.error(error instanceof Error ? error.message : '删除失败') }
}
onMounted(async () => {
loadLocalState()
await Promise.allSettled([
loadCandidates(), loadDashboard(), loadHistory(),
getShopDataCrawlCountryPreference().then((preference) => {
if (!countryPrefUserTouched.value && preference.country_codes?.length) orderedCountryCodes.value = preference.country_codes
}).catch(() => undefined),
])
if (activeTaskId.value || pendingQueue.value.length) {
autoQueueEnabled.value = true
queueStatus.value = '检测到未完成队列,正在恢复...'
void processQueue()
}
})
onUnmounted(() => {
disposed = true
if (countryPrefSaveTimer) timers.clearTimer('preference-save', countryPrefSaveTimer)
timers.clearScope()
})
</script>
<style scoped>
.module-page { min-height: 100vh; background: #1a1a1a; }
.main-content { display: flex; height: calc(100vh - 56px); min-height: calc(100vh - 56px); }
.left-panel { width: 400px; padding: 20px; overflow-y: auto; border-right: 1px solid #2a2a2a; background: #1e1e1e; }
.right-panel { flex: 1; min-width: 0; background: #1a1a1a; }
.section-title { margin-bottom: 10px; color: #bbb; font-size: 13px; }
.input-zone { margin-bottom: 18px; padding: 14px; border: 1px dashed #3a3a3a; border-radius: 8px; background: #252525; }
.input-row { display: flex; gap: 10px; }
.opt-btn, .btn-run { min-height: 36px; padding: 0 16px; border: 0; border-radius: 5px; background: #409eff; color: #fff; white-space: nowrap; cursor: pointer; }
.opt-btn { flex: 0 0 auto; }
.opt-btn:disabled, .btn-run:disabled { opacity: .5; cursor: not-allowed; }
.empty-candidates, .empty-tasks { padding: 16px; border: 1px dashed #333; border-radius: 6px; color: #777; font-size: 13px; }
.candidate-table-scroll { margin-bottom: 18px; border: 1px solid #2a2a2a; border-radius: 6px; overflow: hidden; }
.candidate-table { --el-table-bg-color: #252525; --el-table-tr-bg-color: #252525; --el-table-header-bg-color: #2a2a2a; --el-table-text-color: #ccc; --el-table-border-color: #333; }
.link-danger { border: 0; background: transparent; color: #f56c6c; cursor: pointer; }
.country-pref-checks { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; margin-bottom: 10px; }
.country-check-row { display: flex; align-items: center; gap: 7px; min-height: 32px; color: #ccc; font-size: 13px; }
.country-check-input { width: 15px; height: 15px; }
.country-order-panel { margin-bottom: 8px; padding: 10px; border: 1px solid #333; border-radius: 6px; background: #242424; }
.country-order-caption { margin-bottom: 8px; color: #888; font-size: 12px; }
.country-order-list { display: flex; flex-direction: column; gap: 6px; }
.country-drag-row { display: flex; align-items: center; gap: 8px; min-height: 30px; padding: 0 9px; border: 1px solid #383838; border-radius: 4px; color: #ccc; font-size: 13px; cursor: grab; }
.country-drag-row.dragging { opacity: .5; }
.drag-handle { color: #777; }
.country-pref-status, .queue-status { color: #8dc4ff; font-size: 12px; line-height: 1.5; }
.run-row { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; margin-top: 16px; }
.btn-queue { background: #67c23a; }
.panel-header { height: 52px; padding: 16px 22px; border-bottom: 1px solid #2a2a2a; color: #eee; font-size: 15px; font-weight: 700; }
.task-list-wrap { height: calc(100% - 52px); padding: 18px 22px 28px; overflow-y: auto; }
.clean-result-summary { display: grid; grid-template-columns: repeat(4, minmax(100px, 1fr)); gap: 12px; margin-bottom: 20px; }
.summary-card { display: flex; min-height: 70px; flex-direction: column; justify-content: center; padding: 12px 16px; border: 1px solid #303030; border-radius: 6px; background: #222; }
.summary-label { margin-bottom: 4px; color: #888; font-size: 12px; }
.summary-card strong { color: #fff; font-size: 22px; }
.subsection-title, .result-subsection-title { margin: 18px 0 10px; color: #bbb; font-size: 13px; font-weight: 700; }
.match-table { margin-bottom: 20px; --el-table-bg-color: #222; --el-table-tr-bg-color: #222; --el-table-header-bg-color: #292929; --el-table-text-color: #ccc; --el-table-border-color: #333; }
.ok { color: #67c23a; }.fail, .error-text { color: #f56c6c; }
.result-list-header { padding: 13px 0; border-bottom: 1px solid #303030; color: #eee; font-weight: 700; }
.task-list { margin: 0; padding: 0; list-style: none; }
:deep(.task-item) { display: flex; align-items: center; justify-content: space-between; gap: 18px; min-height: 84px; padding: 14px 0; border-bottom: 1px solid #2b2b2b; }
:deep(.split-result-main) { min-width: 0; }
:deep(.id) { display: block; margin-bottom: 5px; color: #eee; font-weight: 700; }
:deep(.files) { margin-top: 3px; color: #888; font-size: 12px; }
:deep(.task-right) { display: flex; align-items: center; gap: 10px; }
:deep(.status) { min-width: 52px; font-size: 12px; text-align: center; }
:deep(.status.success) { color: #67c23a; }:deep(.status.failed) { color: #f56c6c; }:deep(.status.running) { color: #e6a23c; }
:deep(.download), :deep(.btn-delete) { padding: 5px 10px; border: 1px solid #444; border-radius: 4px; background: transparent; color: #ccc; cursor: pointer; }
:deep(.download) { border-color: #409eff; color: #8dc4ff; }:deep(.btn-delete) { color: #f56c6c; }
@media (max-width: 900px) { .main-content { height: auto; flex-direction: column; }.left-panel { width: 100%; border-right: 0; }.clean-result-summary { grid-template-columns: repeat(2, 1fr); } }
</style>
@@ -792,7 +792,9 @@ function isResultPreparing(item: SimilarAsinHistoryItem) {
}
function normalizeTaskStatus(item: SimilarAsinHistoryItem) {
return (item.taskStatus || '').toUpperCase()
const taskStatus = (item.taskStatus || '').toUpperCase()
if (taskStatus !== 'FAILED' && isResultFileComplete(item)) return 'SUCCESS'
return taskStatus
}
function isResultBuildFailed(item: SimilarAsinHistoryItem) {
@@ -821,7 +823,16 @@ function canDownload(item: SimilarAsinHistoryItem) {
return Boolean(item.resultId && (item.fileReady || item.downloadUrl))
}
function isResultFileComplete(item: SimilarAsinHistoryItem) {
return Boolean(
item.fileReady
|| item.downloadUrl
|| (item.fileStatus || '').toUpperCase() === 'SUCCESS',
)
}
function fileProgressPercent(item: SimilarAsinHistoryItem) {
if (isResultFileComplete(item)) return 100
const percent = Number(item.fileProgressPercent || 0)
if (Number.isFinite(percent) && percent > 0) {
return Math.max(0, Math.min(100, Math.round(percent)))
@@ -52,6 +52,7 @@ type ActiveNavKey =
| 'pricing'
| 'patrol-delete'
| 'query-asin'
| 'shop-data-crawl'
| 'withdraw'
| 'collect-data'
| 'image-video'
@@ -85,7 +86,7 @@ const active = props.active
const showNav = props.showNav
const showHomeLink = props.showHomeLink
const showActions = props.showActions
const allowedColumnKeys = ref<string[] | null>(null)
const allowedColumnKeys = ref<string[]>([])
const navGroups: ReadonlyArray<NavGroup> = [
{
@@ -114,6 +115,7 @@ const navGroups: ReadonlyArray<NavGroup> = [
{ key: 'pricing', label: '跟价', href: '/new_web_source/price-track.html', aliases: ['price-track'] },
{ key: 'patrol-delete', label: '巡店删除', href: '/new_web_source/patrol-delete.html' },
{ key: 'query-asin', label: '查询ASIN', href: '/new_web_source/query-asin.html' },
{ key: 'shop-data-crawl', label: '店铺数据抓取', href: '/new_web_source/shop-data-crawl.html', columnKey: 'shop_data_crawl', aliases: ['shop-data-crawl'] },
{ key: 'withdraw', label: '取款', href: '/new_web_source/withdraw.html' },
{ key: 'shop-status', label: '店铺状态查询' },
],
@@ -135,9 +137,6 @@ function getItemPermissionKeys(item: NavItem) {
}
const visibleNavGroups = computed(() => {
if (allowedColumnKeys.value === null) {
return navGroups
}
const allowedSet = new Set(allowedColumnKeys.value)
const groups = navGroups.flatMap((group) => {
@@ -150,14 +149,14 @@ const visibleNavGroups = computed(() => {
return items.length ? [{ ...group, items }] : []
})
return groups.length ? groups : navGroups
return groups
})
onMounted(async () => {
try {
allowedColumnKeys.value = await getCurrentUserAppColumnKeys()
} catch (_error) {
allowedColumnKeys.value = null
allowedColumnKeys.value = []
}
})
</script>
@@ -5,8 +5,9 @@
<a class="back-link" href="/home">返回首页</a>
</header>
<main class="entrances" aria-label="视频入口">
<main class="entrances" :class="{ 'is-loading': permissionsLoading }" aria-label="视频与图片入口">
<button
v-if="hasMenuPermission('digital-human')"
type="button"
class="entrance-btn"
:disabled="launching"
@@ -14,12 +15,15 @@
>
数字人
</button>
<button type="button" class="entrance-btn" @click="openDeliveryWorkspace">
<button v-if="hasMenuPermission('delivery-video')" type="button" class="entrance-btn" @click="openDeliveryWorkspace">
带货视频
</button>
<button type="button" class="entrance-btn" @click="showSoon('混剪')">
<button v-if="hasMenuPermission('mix-video')" type="button" class="entrance-btn" @click="showSoon('混剪')">
混剪
</button>
<a v-if="hasMenuPermission('image')" class="entrance-btn" href="/image">
图片
</a>
</main>
<div class="toast" :class="{ show: Boolean(statusText), error: statusType === 'error' }">
@@ -64,17 +68,20 @@
</template>
<script setup lang="ts">
import { onBeforeUnmount, ref } from 'vue'
import { onBeforeUnmount, onMounted, ref } from 'vue'
import PageShell from '@/components/layout/PageShell.vue'
import BrandTopBar from '@/pages/brand/components/BrandTopBar.vue'
import DeliveryVideoWorkspace from '@/pages/image-video/components/DeliveryVideoWorkspace.vue'
import { getCurrentUserAppColumnKeys } from '@/shared/api/permission'
import { getPywebviewApi } from '@/shared/bridges/pywebview'
import DownloadProgressPanel from '@/shared/components/DownloadProgressPanel.vue'
type ViewMode = 'menu' | 'delivery'
const currentView = ref<ViewMode>('menu')
const allowedColumnKeys = ref(new Set<string>())
const permissionsLoading = ref(true)
const launching = ref(false)
const statusText = ref('')
const statusType = ref<'normal' | 'error'>('normal')
@@ -103,6 +110,20 @@ function showSoon(name: string) {
showStatus(`${name}暂未开通,敬请期待`)
}
function hasMenuPermission(columnKey: string) {
return allowedColumnKeys.value.has('wb') || allowedColumnKeys.value.has(columnKey)
}
onMounted(async () => {
try {
allowedColumnKeys.value = new Set(await getCurrentUserAppColumnKeys())
} catch (_error) {
showStatus('菜单权限加载失败', 'error')
} finally {
permissionsLoading.value = false
}
})
function openDeliveryWorkspace() {
currentView.value = 'delivery'
}
@@ -265,6 +286,10 @@ async function launchDesktop() {
justify-content: center;
}
.entrances.is-loading {
visibility: hidden;
}
.entrance-btn {
width: 160px;
height: 100px;
+193
View File
@@ -1569,6 +1569,191 @@ export function deleteQueryAsinHistory(resultId: number) {
);
}
// ========== 店铺数据抓取 ==========
export interface ShopDataCrawlCandidateVo {
id: number;
shop_name: string;
created_at?: string;
}
export interface ShopDataCrawlCountryPreferenceVo {
country_codes: string[];
}
export interface ShopDataCrawlShopItem {
shopName: string;
matched: boolean;
shopId?: string;
platform?: string;
companyName?: string;
openStoreUrl?: string;
matchedUserId?: number;
matchStatus?: string;
matchMessage?: string;
}
export interface ShopDataCrawlMatchVo {
items: ShopDataCrawlShopItem[];
}
export interface ShopDataCrawlDashboardVo {
candidateCount: number;
processedTaskCount: number;
successTaskCount: number;
failedTaskCount: number;
}
export interface ShopDataCrawlHistoryItem extends ShopDataCrawlShopItem {
resultId?: number;
taskId?: number;
taskStatus?: string;
success?: boolean;
error?: string;
fileReady?: boolean;
fileStatus?: string;
downloadUrl?: string;
outputFilename?: string;
createdAt?: string;
finishedAt?: string;
}
export interface ShopDataCrawlHistoryVo {
items: ShopDataCrawlHistoryItem[];
}
export interface ShopDataCrawlTaskSummary {
id?: number;
status?: string;
errorMessage?: string;
createdAt?: string;
finishedAt?: string;
countryCodes?: string[];
}
export interface ShopDataCrawlTaskDetailVo {
task?: ShopDataCrawlTaskSummary;
items?: ShopDataCrawlHistoryItem[];
}
export interface ShopDataCrawlTaskBatchVo {
items: Array<ShopDataCrawlTaskDetailVo | ShopDataCrawlHistoryItem>;
missingTaskIds?: number[];
}
export interface ShopDataCrawlCreateTaskVo {
taskId: number;
items: ShopDataCrawlHistoryItem[];
}
export function listShopDataCrawlCandidates() {
return unwrapJavaResponse(
get<JavaApiResponse<ShopDataCrawlCandidateVo[]>>(`${JAVA_API_PREFIX}/shop-data-crawl/candidates`, {
params: { user_id: getCurrentUserId() },
}),
);
}
export function addShopDataCrawlCandidate(shopName: string) {
return unwrapJavaResponse(
post<JavaApiResponse<ShopDataCrawlCandidateVo>, { user_id: number; shop_name: string }>(
`${JAVA_API_PREFIX}/shop-data-crawl/candidates`,
{ user_id: getCurrentUserId(), shop_name: shopName },
),
);
}
export function deleteShopDataCrawlCandidate(id: number) {
return unwrapJavaResponse(
del<JavaApiResponse<null>>(`${JAVA_API_PREFIX}/shop-data-crawl/candidates/${id}`, {
params: { user_id: getCurrentUserId() },
}),
);
}
export function getShopDataCrawlCountryPreference() {
return unwrapJavaResponse(
get<JavaApiResponse<ShopDataCrawlCountryPreferenceVo>>(
`${JAVA_API_PREFIX}/shop-data-crawl/country-preference`,
{ params: { user_id: getCurrentUserId() } },
),
);
}
export function putShopDataCrawlCountryPreference(countryCodes: string[]) {
return unwrapJavaResponse(
put<JavaApiResponse<ShopDataCrawlCountryPreferenceVo>, { user_id: number; country_codes: string[] }>(
`${JAVA_API_PREFIX}/shop-data-crawl/country-preference`,
{ user_id: getCurrentUserId(), country_codes: countryCodes },
),
);
}
export function matchShopDataCrawlShops(shopNames: string[]) {
return unwrapJavaResponse(
post<JavaApiResponse<ShopDataCrawlMatchVo>, { user_id: number; shop_names: string[] }>(
`${JAVA_API_PREFIX}/shop-data-crawl/match-shops`,
{ user_id: getCurrentUserId(), shop_names: shopNames },
),
);
}
export function getShopDataCrawlDashboard() {
return unwrapJavaResponse(
get<JavaApiResponse<ShopDataCrawlDashboardVo>>(`${JAVA_API_PREFIX}/shop-data-crawl/dashboard`, {
params: { user_id: getCurrentUserId() },
}),
);
}
export function getShopDataCrawlHistory() {
return unwrapJavaResponse(
get<JavaApiResponse<ShopDataCrawlHistoryVo>>(`${JAVA_API_PREFIX}/shop-data-crawl/history`, {
params: { user_id: getCurrentUserId() },
}),
);
}
export function createShopDataCrawlTask(items: ShopDataCrawlShopItem[], countryCodes: string[]) {
return unwrapJavaResponse(
post<
JavaApiResponse<ShopDataCrawlCreateTaskVo>,
{ user_id: number; items: ShopDataCrawlShopItem[]; country_codes: string[] }
>(`${JAVA_API_PREFIX}/shop-data-crawl/tasks`, {
user_id: getCurrentUserId(),
items,
country_codes: countryCodes,
}),
);
}
export function getShopDataCrawlTaskProgressBatch(taskIds: number[]) {
return postTaskProgressBatch<ShopDataCrawlTaskBatchVo>(
`${JAVA_API_PREFIX}/shop-data-crawl/tasks/progress/batch`,
taskIds,
);
}
export function getShopDataCrawlResultDownloadUrl(resultId: number) {
return getJavaDownloadUrl(`/shop-data-crawl/results/${resultId}/download`);
}
export function deleteShopDataCrawlTask(taskId: number) {
return unwrapJavaResponse(
del<JavaApiResponse<null>>(`${JAVA_API_PREFIX}/shop-data-crawl/tasks/${taskId}`, {
params: { user_id: getCurrentUserId() },
}),
);
}
export function deleteShopDataCrawlHistory(resultId: number) {
return unwrapJavaResponse(
del<JavaApiResponse<null>>(`${JAVA_API_PREFIX}/shop-data-crawl/history/${resultId}`, {
params: { user_id: getCurrentUserId() },
}),
);
}
// ========== 取款 ==========
export type WithdrawCandidateVo = QueryAsinCandidateVo;
@@ -3324,6 +3509,14 @@ export function getPublishHistory() {
);
}
export function deletePublishTask(taskId: number) {
return unwrapJavaResponse(
del<JavaApiResponse<null>>(`${JAVA_API_PREFIX}/publish/tasks/${taskId}`, {
params: { user_id: getCurrentUserId() },
}),
);
}
export function getJavaDownloadUrl(path: string) {
let raw =
path.startsWith("http://") || path.startsWith("https://")
+4 -10
View File
@@ -8,6 +8,10 @@ export interface PermissionMenuItem {
route_path?: string
routePath?: string
menu_type?: string
parent_id?: number | string | null
parentId?: number | string | null
root_column_key?: string
rootColumnKey?: string
sort_order?: number
created_at?: string
}
@@ -54,16 +58,6 @@ export async function getCurrentUserAppColumnKeys() {
const uid = getCurrentUserId()
const cacheKey = getAppPermissionCacheKey(uid)
try {
const cachedItems = JSON.parse(window.localStorage.getItem(cacheKey) || 'null') as PermissionMenuItem[] | null
if (Array.isArray(cachedItems)) {
const cachedKeys = normalizeColumnKeys(cachedItems)
if (cachedKeys.length) {
return cachedKeys
}
}
} catch (_error) {}
const headers: Record<string, string> = {}
const token = getAuthToken()
if (token) {
+12
View File
@@ -0,0 +1,12 @@
import { createApp } from 'vue'
import ElementPlus from 'element-plus'
import zhCn from 'element-plus/es/locale/lang/zh-cn'
import dayjs from 'dayjs'
import 'dayjs/locale/zh-cn'
import 'element-plus/dist/index.css'
import '@/styles/main.css'
import BrandShopDataCrawlTab from '@/pages/brand/components/BrandShopDataCrawlTab.vue'
dayjs.locale('zh-cn')
createApp(BrandShopDataCrawlTab).use(ElementPlus, { locale: zhCn }).mount('#app')
+1
View File
@@ -54,6 +54,7 @@ export default defineConfig({
'price-track': resolve(__dirname, 'price-track.html'),
'patrol-delete': resolve(__dirname, 'patrol-delete.html'),
'query-asin': resolve(__dirname, 'query-asin.html'),
'shop-data-crawl': resolve(__dirname, 'shop-data-crawl.html'),
withdraw: resolve(__dirname, 'withdraw.html'),
'collect-data': resolve(__dirname, 'collect-data.html'),
'image-video': resolve(__dirname, 'image-video.html'),