菜单修改优化

This commit is contained in:
supernijia
2026-07-28 13:40:19 +08:00
parent 4ddb8b47b0
commit 7c3c9e53a0
90 changed files with 9388 additions and 1545 deletions
+5 -2
View File
@@ -8,6 +8,7 @@ Requirements:
- Put JVM options before `-jar`
- Pass `AIIMAGE_INSTANCE_ID` and Redis address from the startup command
- Result file job MQ is expected to stay enabled in packaged `server` deployments
- Use `https://oss.aishufu.top` for the MinIO SDK and public object URLs; `https://minio.aishufu.top` is the management console only
1Panel server 121:
@@ -22,6 +23,8 @@ AIIMAGE_REDIS_PASSWORD=B6COTcY094TYe545
AIIMAGE_REDIS_DATABASE=0
AIIMAGE_SERVER_PORT=18080
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=/app/data/tmp
AIIMAGE_OSS_ENDPOINT=https://oss.aishufu.top
AIIMAGE_OSS_PUBLIC_ENDPOINT=https://oss.aishufu.top
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://121.196.149.225:9000
@@ -36,13 +39,13 @@ java -Xmx2048M -Xms2048M -jar /app/aiimage-backend-0.0.1-SNAPSHOT.jar
Baota server 111:
```bash
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-111 --spring.data.redis.host=47.111.163.154 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-111 --spring.data.redis.host=47.111.163.154 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp --aiimage.oss.endpoint=https://oss.aishufu.top --aiimage.oss.public-endpoint=https://oss.aishufu.top
```
Optional Baota server 121 command:
```bash
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-121 --spring.data.redis.host=192.168.0.172 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp
/www/server/java/jdk-21.0.2/bin/java -Xmx1024M -Xms256M -jar /app/java/aiimage-backend-0.0.1-SNAPSHOT.jar --spring.profiles.active=server --aiimage.instance-id=server-121 --spring.data.redis.host=192.168.0.172 --spring.data.redis.port=16379 --spring.data.redis.password=B6COTcY094TYe545 --server.port=18080 --aiimage.storage.local-temp-dir=/app/data/tmp --aiimage.oss.endpoint=https://oss.aishufu.top --aiimage.oss.public-endpoint=https://oss.aishufu.top
```
Successful startup should show a line similar to:
@@ -12,5 +12,5 @@ public class ModuleCleanupProperties {
private boolean enabled = true;
private String cron = "0 0 0 * * *";
private long retentionDays = 7;
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
}
@@ -48,11 +48,11 @@ public class AdminUserController {
@PostMapping("/user")
@Operation(summary = "创建用户")
public ApiResponse<Void> createUser(HttpServletRequest request,
public ApiResponse<Long> createUser(HttpServletRequest request,
@RequestBody AdminUserCreateRequest body) {
AdminUserEntity currentUser = adminAuthSupport.requireAdmin(request);
adminUserService.createUser(currentUser, body);
return ApiResponse.success("用户创建成功", null);
Long userId = adminUserService.createUser(currentUser, body);
return ApiResponse.success("用户创建成功", userId);
}
@PutMapping("/user/{uid}")
@@ -2,10 +2,13 @@ package com.nanri.aiimage.modules.admin.model.dto;
import lombok.Data;
import java.util.List;
@Data
public class AdminUserCreateRequest {
private String username;
private String password;
private String role;
private Long createdById;
private List<Long> columnIds;
}
@@ -2,8 +2,11 @@ package com.nanri.aiimage.modules.admin.model.dto;
import lombok.Data;
import java.util.List;
@Data
public class AdminUserUpdateRequest {
private String password;
private String role;
private List<Long> columnIds;
}
@@ -11,10 +11,13 @@ import com.nanri.aiimage.modules.admin.model.vo.AdminUserListVo;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.auth.util.WerkzeugPasswordEncoder;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@@ -35,6 +38,7 @@ public class AdminUserService {
private final AdminUserMapper adminUserMapper;
private final WerkzeugPasswordEncoder passwordEncoder;
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
public AdminUserListVo listUsers(AdminUserEntity currentUser, Integer page, Integer pageSize,
String username, Long createdById) {
@@ -97,7 +101,8 @@ public class AdminUserService {
return vo;
}
public void createUser(AdminUserEntity currentUser, AdminUserCreateRequest request) {
@Transactional
public Long createUser(AdminUserEntity currentUser, AdminUserCreateRequest request) {
String role = adminAuthSupport.currentRole(currentUser);
if (role == null) {
throw new BusinessException(403, "需要管理员权限");
@@ -150,8 +155,11 @@ public class AdminUserService {
} catch (DuplicateKeyException e) {
throw new BusinessException("用户名已存在");
}
replaceDirectPermissions(currentUser, entity.getId(), request.getColumnIds());
return entity.getId();
}
@Transactional
public void updateUser(AdminUserEntity currentUser, Long uid, AdminUserUpdateRequest request) {
String role = adminAuthSupport.currentRole(currentUser);
if (role == null) {
@@ -160,7 +168,9 @@ public class AdminUserService {
String password = request.getPassword();
String wantRole = request.getRole() == null ? null : request.getRole().trim();
if ((password == null || password.isEmpty()) && (wantRole == null || wantRole.isEmpty())) {
if ((password == null || password.isEmpty())
&& (wantRole == null || wantRole.isEmpty())
&& request.getColumnIds() == null) {
throw new BusinessException("请提供要修改的内容");
}
@@ -206,6 +216,16 @@ public class AdminUserService {
if (dirty) {
adminUserMapper.update(null, update);
}
if (request.getColumnIds() != null) {
replaceDirectPermissions(currentUser, uid, request.getColumnIds());
}
}
private void replaceDirectPermissions(AdminUserEntity operator, Long userId, List<Long> columnIds) {
UserColumnPermissionUpdateRequest permissionRequest = new UserColumnPermissionUpdateRequest();
permissionRequest.setColumnIds(columnIds == null ? List.of() : columnIds);
permissionMenuService.updateUserColumnPermissions(
operator, userId, permissionRequest, PermissionMenuService.MENU_TYPE_APP);
}
public void deleteUser(AdminUserEntity currentUser, Long uid) {
@@ -1,6 +1,5 @@
package com.nanri.aiimage.modules.admin.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.auth.service.JwtService;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
@@ -65,27 +64,15 @@ public class AdminAuthSupport {
return "super_admin";
}
if ("admin".equals(storedRole)) {
return resolveAdminRole(user);
return "admin";
}
boolean isAdminFlag = user.getIsAdmin() != null && user.getIsAdmin() == 1;
if (isAdminFlag) {
return resolveAdminRole(user);
if (storedRole.isEmpty() && isAdminFlag) {
return user.getCreatedById() == null ? "super_admin" : "admin";
}
return null;
}
/** 数据库里 role='admin' 中 id 最小者视作超级管理员(与 Python 行为一致)。 */
private String resolveAdminRole(AdminUserEntity user) {
AdminUserEntity superAdmin = adminUserMapper.selectOne(new LambdaQueryWrapper<AdminUserEntity>()
.eq(AdminUserEntity::getRole, "admin")
.orderByAsc(AdminUserEntity::getId)
.last("LIMIT 1"));
if (superAdmin != null && superAdmin.getId().equals(user.getId())) {
return "super_admin";
}
return "admin";
}
private String resolveToken(HttpServletRequest request) {
String authHeader = request.getHeader(HttpHeaders.AUTHORIZATION);
if (authHeader != null && authHeader.startsWith("Bearer ")) {
@@ -14,7 +14,7 @@ public class AppearancePatentHistoryItemVo {
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/appearance_patent/xxx/17-result.xlsx")
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "https://oss.aishufu.top/nanri-ai-images/result/appearance_patent/xxx/17-result.xlsx")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
@@ -9,7 +9,7 @@ import lombok.Data;
public class BrandSourceFileDto {
@NotBlank(message = "fileUrl 不能为空")
@Schema(description = "对象存储文件下载链接,Java 将先下载文件再读取内容。", example = "http://47.110.241.161:9000/nanri-ai-images/source/brand.xlsx")
@Schema(description = "对象存储文件下载链接,Java 将先下载文件再读取内容。", example = "https://oss.aishufu.top/nanri-ai-images/source/brand.xlsx")
private String fileUrl;
@Schema(description = "原始文件名,用于生成任务描述和结果文件名。", example = "品牌样例.xlsx")
@@ -27,6 +27,7 @@ public class OssStorageService {
private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO";
private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/";
private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000";
private final OssProperties ossProperties;
private final MinioClient minioClient;
@@ -164,6 +165,52 @@ public class OssStorageService {
return location == null ? null : getPublicUrl(location.objectKey(), location.bucket());
}
/** Rewrites managed MinIO URLs to the current public endpoint without touching external URLs. */
public String normalizeManagedPublicUrl(String value) {
if (value == null || value.isBlank()) {
return value;
}
URI source;
try {
source = new URI(value.trim());
} catch (Exception ignored) {
return value;
}
if (!isHttpUrl(source) || source.getHost() == null || source.getUserInfo() != null) {
return value;
}
String objectPath;
try {
objectPath = trimLeadingSlash(decodePath(source.getRawPath()));
} catch (Exception ignored) {
return value;
}
if (objectPath.isBlank() || hasDotPathSegment(objectPath)) {
return value;
}
List<URI> endpoints = managedEndpoints();
for (URI endpoint : endpoints) {
for (String bucket : configuredBuckets()) {
if (matchesVirtualHostEndpoint(source, endpoint, bucket)) {
return getPublicUrl(objectPath, bucket);
}
}
}
if (endpoints.stream().noneMatch(endpoint -> matchesEndpoint(source, endpoint))) {
return value;
}
String bucket = bucketPrefix(objectPath);
if (bucket == null) {
return value;
}
String objectKey = trimLeadingSlash(objectPath.substring(bucket.length()));
return objectKey.isBlank() ? value : getPublicUrl(objectKey, bucket);
}
public String resolveObjectKey(String value) {
StorageLocation location = resolveStorageLocation(value);
return location == null ? value : location.objectKey();
@@ -271,6 +318,58 @@ public class OssStorageService {
.toList();
}
private List<URI> managedEndpoints() {
return Stream.of(ossProperties.getEndpoint(), ossProperties.getPublicEndpoint(), LEGACY_MINIO_ENDPOINT)
.filter(Objects::nonNull)
.map(String::trim)
.filter(endpoint -> !endpoint.isBlank())
.map(this::parseEndpoint)
.filter(Objects::nonNull)
.distinct()
.toList();
}
private URI parseEndpoint(String value) {
try {
URI endpoint = new URI(withScheme(value));
return endpoint.getHost() == null ? null : endpoint;
} catch (Exception ignored) {
return null;
}
}
private boolean matchesEndpoint(URI source, URI endpoint) {
return sameSchemeAndPort(source, endpoint)
&& source.getHost().equalsIgnoreCase(endpoint.getHost());
}
private boolean matchesVirtualHostEndpoint(URI source, URI endpoint, String bucket) {
String expectedHost = bucket + "." + endpoint.getHost();
return sameSchemeAndPort(source, endpoint)
&& source.getHost().equalsIgnoreCase(expectedHost);
}
private boolean sameSchemeAndPort(URI left, URI right) {
return left.getScheme().equalsIgnoreCase(right.getScheme())
&& effectivePort(left) == effectivePort(right);
}
private int effectivePort(URI uri) {
if (uri.getPort() >= 0) {
return uri.getPort();
}
return "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80;
}
private boolean isHttpUrl(URI uri) {
return "http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme());
}
private boolean hasDotPathSegment(String path) {
return Stream.of(path.split("/", -1))
.anyMatch(segment -> ".".equals(segment) || "..".equals(segment));
}
private MinioClient buildClient() {
return minioClient;
}
@@ -49,6 +49,15 @@ public class ImageVideoAsyncTaskService {
private static final Set<String> COZE_PRIMARY_STATUS_FIELDS = Set.of(
"execute_status", "executeStatus", "workflow_status", "workflowStatus"
);
private static final Set<String> COZE_BUSINESS_ERROR_FIELDS = Set.of(
"error", "error_message", "errorMessage"
);
private static final Set<String> COZE_BUSINESS_STATUS_CODE_FIELDS = Set.of(
"status_code", "statusCode"
);
private static final Set<String> COZE_BUSINESS_STATUS_MESSAGE_FIELDS = Set.of(
"status_msg", "statusMessage"
);
private final ImageVideoAsyncTaskMapper taskMapper;
private final ImageVideoCozeService cozeService;
@@ -233,7 +242,10 @@ public class ImageVideoAsyncTaskService {
if (FAILED_STATUSES.contains(cozeStatus)) {
task.setCozeStatus(cozeStatus);
task.setResultJson(writeJson(result));
failTask(task, new BusinessException("Coze workflow finished with status " + cozeStatus));
String businessError = resolveCozeBusinessError(result);
failTask(task, new BusinessException(businessError.isBlank()
? "Coze workflow finished with status " + cozeStatus
: businessError));
} else {
completeTask(task, transformResult(type, result), cozeStatus);
}
@@ -315,6 +327,19 @@ public class ImageVideoAsyncTaskService {
}
private void completeTask(ImageVideoAsyncTaskEntity task, Object result, String cozeStatus) {
String businessError = resolveCozeBusinessError(result);
if (!businessError.isBlank()) {
task.setCozeStatus(normalizeStatus(cozeStatus));
task.setResultJson(writeJson(result));
failTask(task, new BusinessException(businessError));
return;
}
if (requiresVideoResult(task) && !hasFinalVideoUrl(result)) {
task.setCozeStatus(normalizeStatus(cozeStatus));
task.setResultJson(writeJson(result));
failTask(task, new BusinessException("Coze workflow succeeded but video_url is empty"));
return;
}
task.setStatus(TaskStatus.SUCCESS.name());
task.setResultJson(writeJson(result));
archiveService.enrichCompletedTask(task, result);
@@ -511,6 +536,60 @@ public class ImageVideoAsyncTaskService {
return value;
}
private String resolveCozeBusinessError(Object result) {
String error = findText(result, COZE_BUSINESS_ERROR_FIELDS);
if (!error.isBlank()) {
return error;
}
String statusCode = findText(result, COZE_BUSINESS_STATUS_CODE_FIELDS);
if (statusCode.isBlank() || "0".equals(statusCode)) {
return "";
}
String statusMessage = findText(result, COZE_BUSINESS_STATUS_MESSAGE_FIELDS);
return statusMessage.isBlank()
? "Coze workflow returned business status " + statusCode
: statusMessage + " (status_code: " + statusCode + ")";
}
private boolean hasFinalVideoUrl(Object result) {
Object data = parsePossiblyJson(childValue(result, "data"));
Object scope = data == null ? parsePossiblyJson(result) : data;
if (scope == null) {
return false;
}
Collection<?> records = scope instanceof Collection<?> collection ? collection : List.of(scope);
boolean hasRunOutput = false;
for (Object record : records) {
Object output = childValue(parsePossiblyJson(record), "output");
if (output == null) {
continue;
}
hasRunOutput = true;
if (hasVideoUrlInOutput(output)) {
return true;
}
}
return !hasRunOutput && hasVideoUrlInOutput(scope);
}
private boolean hasVideoUrlInOutput(Object output) {
Object envelope = parsePossiblyJson(output);
Object finalOutput = childValue(envelope, "Output");
if (finalOutput != null) {
return hasRecognizedVideoUrl(parsePossiblyJson(finalOutput));
}
if (childValue(envelope, "node_status") != null) {
return false;
}
return hasRecognizedVideoUrl(envelope);
}
private boolean hasRecognizedVideoUrl(Object payload) {
return !ImageVideoArchiveService.findUrls(payload, Set.of("videourl")).isEmpty();
}
private String normalizeStatus(String value) {
return value == null ? "" : value.trim().toUpperCase();
}
@@ -529,6 +608,12 @@ public class ImageVideoAsyncTaskService {
&& task.getSubmittedAt().plusHours(1).isBefore(LocalDateTime.now());
}
private boolean requiresVideoResult(ImageVideoAsyncTaskEntity task) {
String taskType = task == null ? "" : task.getTaskType();
return TaskType.IMAGE_VIDEO_WORKFLOW.name().equals(taskType)
|| TaskType.WORKFLOW_RESULT.name().equals(taskType);
}
private void ensureTaskOwnedByCurrentInstance(ImageVideoAsyncTaskEntity task, String operation) {
String owner = task == null ? null : task.getOwnerInstanceId();
if (owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId())) {
@@ -1,17 +1,23 @@
package com.nanri.aiimage.modules.permission.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.ImageVideoDataPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -22,6 +28,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
@RestController
@@ -30,54 +39,188 @@ import java.util.List;
@Tag(name = "菜单权限管理", description = "统一维护软件端与后台管理端的菜单权限配置")
public class PermissionMenuController {
@Value("${aiimage.security.internal-token:}")
private String internalToken;
@Value("${aiimage.security.internal-token-file:}")
private String internalTokenFile;
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
@GetMapping("/permission-menus")
@Operation(summary = "查询菜单权限列表")
public ApiResponse<List<PermissionMenuItemVo>> listMenus(
public ApiResponse<List<PermissionMenuItemVo>> listMenus(HttpServletRequest request,
@Parameter(description = "菜单类型: app/admin") @RequestParam(required = false) String menuType) {
requireAdmin(request);
return ApiResponse.success(permissionMenuService.list(menuType));
}
@PostMapping("/permission-menus")
@Operation(summary = "新增菜单权限项")
public ApiResponse<PermissionMenuItemVo> createMenu(@Valid @RequestBody PermissionMenuCreateRequest request) {
return ApiResponse.success("创建成功", permissionMenuService.create(request));
public ApiResponse<PermissionMenuItemVo> createMenu(HttpServletRequest httpRequest,
@Valid @RequestBody PermissionMenuCreateRequest request) {
return ApiResponse.success("创建成功", permissionMenuService.create(
requireAdmin(httpRequest), request));
}
@PutMapping("/permission-menus/{id}")
@Operation(summary = "编辑菜单权限项")
public ApiResponse<PermissionMenuItemVo> updateMenu(@PathVariable Long id,
public ApiResponse<PermissionMenuItemVo> updateMenu(HttpServletRequest httpRequest,
@PathVariable Long id,
@Valid @RequestBody PermissionMenuUpdateRequest request) {
return ApiResponse.success("更新成功", permissionMenuService.update(id, request));
return ApiResponse.success("更新成功", permissionMenuService.update(
requireAdmin(httpRequest), id, request));
}
@DeleteMapping("/permission-menus/{id}")
@Operation(summary = "删除菜单权限项")
public ApiResponse<Void> deleteMenu(@PathVariable Long id) {
permissionMenuService.delete(id);
public ApiResponse<Void> deleteMenu(HttpServletRequest httpRequest, @PathVariable Long id) {
permissionMenuService.delete(requireAdmin(httpRequest), id);
return ApiResponse.success("删除成功", null);
}
@GetMapping("/permission-users/{userId}/columns")
@Operation(summary = "查询用户菜单权限 ID 列表")
public ApiResponse<UserColumnIdsVo> getUserColumnIds(@PathVariable Long userId,
@Operation(summary = "查询用户直接授权菜单 ID 列表",
description = "columnIds 仅表示数据库保存的直接授权,所有角色都不包含递归或虚拟权限")
public ApiResponse<UserColumnIdsVo> getUserColumnIds(HttpServletRequest httpRequest,
@PathVariable Long userId,
@RequestParam(required = false) String menuType) {
return ApiResponse.success(permissionMenuService.getUserColumnIds(userId, menuType));
return ApiResponse.success(permissionMenuService.getUserColumnIds(
requireAdmin(httpRequest), userId, menuType));
}
@PutMapping("/permission-users/{userId}/columns")
@Operation(summary = "更新用户菜单权限")
public ApiResponse<Void> updateUserColumnIds(@PathVariable Long userId,
public ApiResponse<Void> updateUserColumnIds(HttpServletRequest httpRequest,
@PathVariable Long userId,
@RequestParam(required = false) String menuType,
@RequestBody(required = false) UserColumnPermissionUpdateRequest request) {
permissionMenuService.updateUserColumnPermissions(userId, request);
permissionMenuService.updateUserColumnPermissions(
requireAdmin(httpRequest), userId, request, menuType);
return ApiResponse.success("保存成功", null);
}
@GetMapping("/permission-users/{userId}/column-permissions")
@Operation(summary = "查询用户菜单权限详情")
public ApiResponse<List<PermissionMenuItemVo>> getUserColumnPermissions(@PathVariable Long userId,
@Operation(summary = "查询用户有效菜单权限详情",
description = "返回直接授权及其递归展开的全部子菜单;不会改变数据库中的直接授权")
public ApiResponse<List<PermissionMenuItemVo>> getUserColumnPermissions(HttpServletRequest httpRequest,
@PathVariable Long userId,
@RequestParam(required = false) String menuType) {
return ApiResponse.success(permissionMenuService.getUserColumnPermissions(userId, menuType));
return ApiResponse.success(permissionMenuService.getUserColumnPermissions(
requireUser(httpRequest), userId, menuType));
}
@GetMapping("/image-video-task-permissions")
@Operation(summary = "查询视频任务数据权限用户")
public ApiResponse<List<ImageVideoDataPermissionUserVo>> listImageVideoDataPermissionUsers(
HttpServletRequest request) {
return ApiResponse.success(permissionMenuService.listImageVideoDataPermissionUsers(requireAdmin(request)));
}
@PutMapping("/image-video-task-permissions")
@Operation(summary = "更新视频任务数据权限用户")
public ApiResponse<Integer> updateImageVideoDataPermissionUsers(
HttpServletRequest request,
@RequestBody(required = false) ImageVideoDataPermissionUpdateRequest body) {
int grantedCount = permissionMenuService.updateImageVideoDataPermissionUsers(
requireAdmin(request), body == null ? List.of() : body.getUserIds());
return ApiResponse.success("视频任务权限已更新", grantedCount);
}
private AdminUserEntity requireAdmin(HttpServletRequest request) {
try {
return adminAuthSupport.requireAdmin(request);
} catch (com.nanri.aiimage.common.exception.BusinessException authFailure) {
AdminUserEntity internalOperator = resolveInternalOperator(request, true);
if (internalOperator != null) {
return internalOperator;
}
throw authFailure;
}
}
private AdminUserEntity requireUser(HttpServletRequest request) {
try {
return adminAuthSupport.requireUser(request);
} catch (com.nanri.aiimage.common.exception.BusinessException authFailure) {
AdminUserEntity internalOperator = resolveInternalOperator(request, false);
if (internalOperator != null) {
return internalOperator;
}
throw authFailure;
}
}
/**
* Flask's legacy session cannot mint a Java JWT. It may identify its
* already-authenticated user only with a shared secret supplied explicitly
* or through the per-user local token file. Admin routes still revalidate
* the database role, and the superAdmin query flag is deliberately ignored.
*/
private AdminUserEntity resolveInternalOperator(HttpServletRequest request, boolean requireAdmin) {
String suppliedToken = request.getHeader("X-Internal-Token");
if (!isTrustedInternalRequest(request, suppliedToken)) {
return null;
}
String rawOperatorId = request.getParameter("operatorId");
if (rawOperatorId == null || rawOperatorId.isBlank()) {
rawOperatorId = request.getParameter("operator_id");
}
if (rawOperatorId == null || rawOperatorId.isBlank()) {
return null;
}
try {
Long operatorId = Long.parseLong(rawOperatorId.trim());
return requireAdmin
? permissionMenuService.requireAdminOperator(operatorId)
: permissionMenuService.requireUserOperator(operatorId);
} catch (NumberFormatException ex) {
return null;
}
}
private boolean isTrustedInternalRequest(HttpServletRequest request, String suppliedToken) {
String expectedToken = resolveExpectedInternalToken();
if (expectedToken.isBlank() || suppliedToken == null || suppliedToken.isBlank()) {
return false;
}
return expectedToken.equals(suppliedToken);
}
private String resolveExpectedInternalToken() {
if (internalToken != null && !internalToken.isBlank()) {
return internalToken.trim();
}
Path path = resolveInternalTokenFile();
if (path == null || !Files.isRegularFile(path)) {
return "";
}
try {
return Files.readString(path, StandardCharsets.UTF_8).trim();
} catch (Exception ignored) {
return "";
}
}
private Path resolveInternalTokenFile() {
String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim();
if (!configuredPath.isEmpty()) {
if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) {
String userHome = System.getProperty("user.home", "").trim();
if (userHome.isEmpty()) {
return null;
}
configuredPath = configuredPath.length() == 1
? userHome
: Path.of(userHome, configuredPath.substring(2)).toString();
}
Path configuredTokenPath = Path.of(configuredPath);
return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null;
}
String userHome = System.getProperty("user.home", "").trim();
return userHome.isEmpty()
? null
: Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize();
}
}
@@ -0,0 +1,12 @@
package com.nanri.aiimage.modules.permission.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import lombok.Data;
import java.util.List;
@Data
public class ImageVideoDataPermissionUpdateRequest {
@JsonAlias("user_ids")
private List<Long> userIds;
}
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.permission.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@@ -12,6 +13,10 @@ public class PermissionMenuCreateRequest {
@NotBlank(message = "菜单标识不能为空")
private String columnKey;
/** 直接父菜单;null 表示根菜单。 */
@JsonAlias("parent_id")
private Long parentId;
@NotBlank(message = "菜单类型不能为空")
private String menuType;
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.permission.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@@ -12,6 +13,10 @@ public class PermissionMenuUpdateRequest {
@NotBlank(message = "菜单标识不能为空")
private String columnKey;
/** 直接父菜单;null 表示根菜单。 */
@JsonAlias("parent_id")
private Long parentId;
@NotBlank(message = "菜单类型不能为空")
private String menuType;
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.permission.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@@ -15,6 +16,8 @@ public class PermissionMenuEntity {
private Long id;
private String name;
private String columnKey;
@TableField("parent_id")
private Long parentId;
private String menuType;
private String routePath;
private Integer sortOrder;
@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.permission.model.vo;
import lombok.Data;
@Data
public class ImageVideoDataPermissionUserVo {
private Long id;
private String username;
private String role;
private boolean granted;
}
@@ -12,6 +12,10 @@ public class PermissionMenuItemVo {
private String name;
@JsonProperty("column_key")
private String columnKey;
@JsonProperty("parent_id")
private Long parentId;
@JsonProperty("root_column_key")
private String rootColumnKey;
@JsonProperty("menu_type")
private String menuType;
@JsonProperty("route_path")
@@ -7,5 +7,9 @@ import java.util.List;
@Data
public class UserColumnIdsVo {
/**
* 仅返回数据库中保存的直接授权 ID;不包含递归子菜单,
* 也不会将超级管理员的虚拟全量权限写入响应。
*/
private List<Long> columnIds;
}
@@ -22,8 +22,29 @@ public class PermissionMenuSchemaInitializer {
private static final List<DefaultAppMenu> DEFAULT_APP_MENUS = List.of(
new DefaultAppMenu("前端工具", "brand_front_tools", "brand-front-tools", 110),
new DefaultAppMenu("运营工具", "brand_operation_tools", "brand-operation-tools", 120),
new DefaultAppMenu("后勤工具", "brand_logistics_tools", "brand-logistics-tools", 130),
new DefaultAppMenu("取款", "withdraw", "withdraw", 140)
new DefaultAppMenu("后勤工具", "brand_logistics_tools", "brand-logistics-tools", 130)
);
private static final List<DefaultAppChildMenu> DEFAULT_APP_CHILD_MENUS = List.of(
new DefaultAppChildMenu("采集数据", "collect-data", "collect-data", 111, "brand_front_tools"),
new DefaultAppChildMenu("品牌检测", "brand", "brand", 112, "brand_front_tools"),
new DefaultAppChildMenu("外观专利检测", "appearance-patent", "appearance-patent", 113, "brand_front_tools"),
new DefaultAppChildMenu("货源查询", "similar-asin", "similar-asin", 114, "brand_front_tools"),
new DefaultAppChildMenu("数据去重", "dedupe", "dedupe", 115, "brand_front_tools"),
new DefaultAppChildMenu("数据拆分", "split", "split", 116, "brand_front_tools"),
new DefaultAppChildMenu("格式转换", "convert", "convert", 117, "brand_front_tools"),
new DefaultAppChildMenu("上架", "publish", "publish", 121, "brand_operation_tools"),
new DefaultAppChildMenu("删除ASIN", "delete-brand", "delete-brand", 122, "brand_operation_tools"),
new DefaultAppChildMenu("商品风险解决", "product-risk", "product-risk", 123, "brand_operation_tools"),
new DefaultAppChildMenu("定时匹配", "shop-match", "shop-match", 124, "brand_operation_tools"),
new DefaultAppChildMenu("跟价", "pricing", "price-track", 125, "brand_operation_tools"),
new DefaultAppChildMenu("巡店删除", "patrol-delete", "patrol-delete", 126, "brand_operation_tools"),
new DefaultAppChildMenu("查询ASIN", "query-asin", "query-asin", 127, "brand_operation_tools"),
// Keep this key for compatibility with the existing Java/Flask module.
new DefaultAppChildMenu("店铺数据抓取", "shop_data_crawl", "shop-data-crawl", 128, "brand_operation_tools"),
new DefaultAppChildMenu("取款", "withdraw", "withdraw", 129, "brand_operation_tools"),
new DefaultAppChildMenu("采购", "purchase", "purchase", 131, "brand_logistics_tools"),
new DefaultAppChildMenu("ERP", "erp", "erp", 132, "brand_logistics_tools")
);
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
@@ -52,11 +73,13 @@ public class PermissionMenuSchemaInitializer {
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(128) NOT NULL COMMENT '菜单标题',
column_key VARCHAR(64) NOT NULL COMMENT '菜单唯一标识',
parent_id INT NULL COMMENT '直接父菜单 ID',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uk_column_key (column_key)
)
""");
executeQuietly("ALTER TABLE columns ADD COLUMN menu_type VARCHAR(20) NOT NULL DEFAULT 'app' COMMENT '菜单类型: app/admin' AFTER column_key");
executeQuietly("ALTER TABLE columns ADD COLUMN parent_id INT NULL COMMENT '直接父菜单 ID' AFTER id");
executeQuietly("ALTER TABLE columns ADD COLUMN route_path VARCHAR(255) NOT NULL DEFAULT '' COMMENT '菜单路由或页面标识' AFTER menu_type");
executeQuietly("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path");
executeQuietly("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''");
@@ -64,6 +87,8 @@ public class PermissionMenuSchemaInitializer {
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
ensureDefaultAdminMenus();
ensureDefaultAppMenus();
ensureDefaultAppChildMenus();
ensureInternalDataPermissions();
executeQuietly("""
CREATE TABLE IF NOT EXISTS user_column_permission (
user_id INT NOT NULL,
@@ -86,7 +111,7 @@ public class PermissionMenuSchemaInitializer {
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
executeQuietly("""
UPDATE columns
SET sort_order = %d
SET sort_order = %d, parent_id = NULL
WHERE column_key = '%s' AND (sort_order IS NULL OR sort_order = 0)
""".formatted(menu.sortOrder(), menu.columnKey()));
}
@@ -103,12 +128,50 @@ public class PermissionMenuSchemaInitializer {
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
executeQuietly("""
UPDATE columns
SET sort_order = %d, route_path = '%s', menu_type = 'app'
SET sort_order = %d, route_path = '%s', menu_type = 'app', parent_id = NULL
WHERE column_key = '%s'
""".formatted(menu.sortOrder(), menu.routePath(), menu.columnKey()));
}
}
private void ensureDefaultAppChildMenus() {
// Older migrations used underscore keys for a few app pages. Normalize
// them once so the frontend item key and permission key stay aligned.
executeQuietly("UPDATE columns SET column_key = 'collect-data' WHERE column_key = 'collect_data'");
executeQuietly("UPDATE columns SET column_key = 'appearance-patent' WHERE column_key = 'appearance_patent'");
executeQuietly("UPDATE columns SET column_key = 'similar-asin' WHERE column_key = 'similar_asin'");
for (DefaultAppChildMenu menu : DEFAULT_APP_CHILD_MENUS) {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order, parent_id)
SELECT '%s', '%s', 'app', '%s', %d, parent.id
FROM columns parent
WHERE parent.column_key = '%s'
AND NOT EXISTS (
SELECT 1 FROM columns existing WHERE existing.column_key = '%s'
)
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(),
menu.parentKey(), menu.columnKey()));
executeQuietly("""
UPDATE columns child
JOIN columns parent ON parent.column_key = '%s'
SET child.name = '%s', child.menu_type = 'app', child.route_path = '%s',
child.sort_order = %d, child.parent_id = parent.id
WHERE child.column_key = '%s'
""".formatted(menu.parentKey(), menu.name(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
}
}
private void ensureInternalDataPermissions() {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '视频任务数据查看', 'admin_image_video_task_data', 'internal', 'image-video-task-data', 0
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = 'admin_image_video_task_data'
)
""");
}
private void executeQuietly(String sql) {
try {
jdbcTemplate.execute(sql);
@@ -122,4 +185,8 @@ public class PermissionMenuSchemaInitializer {
private record DefaultAppMenu(String name, String columnKey, String routePath, int sortOrder) {
}
private record DefaultAppChildMenu(String name, String columnKey, String routePath, int sortOrder,
String parentKey) {
}
}
@@ -12,22 +12,34 @@ import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdate
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Menu hierarchy and user-column permissions.
*
* <p>The assignment table stores direct grants only. A parent grant is
* expanded to all descendants when effective permissions are read or checked;
* descendants are never inserted as a side effect of an update.</p>
*/
@Service
@RequiredArgsConstructor
public class PermissionMenuService {
@@ -40,28 +52,37 @@ public class PermissionMenuService {
private final UserColumnPermissionMapper userColumnPermissionMapper;
private final AdminUserMapper adminUserMapper;
/** Returns the flat menu catalog, including each item's direct parent ID. */
public List<PermissionMenuItemVo> list(String menuType) {
LambdaQueryWrapper<PermissionMenuEntity> query = new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(isValidMenuType(menuType), PermissionMenuEntity::getMenuType, normalizeMenuType(menuType))
.orderByAsc(PermissionMenuEntity::getSortOrder)
.orderByAsc(PermissionMenuEntity::getId);
return permissionMenuMapper.selectList(query).stream()
.map(this::toItemVo)
.toList();
List<PermissionMenuEntity> menus = loadMenus(menuType);
return toItemVos(menus, menus);
}
@Transactional
public PermissionMenuItemVo create(PermissionMenuCreateRequest request) {
return create(null, request);
}
/** Controller-facing variant; the caller may pass the authenticated admin. */
@Transactional
public PermissionMenuItemVo create(AdminUserEntity operator, PermissionMenuCreateRequest request) {
ensureAdminOperatorIfPresent(operator);
if (request == null) {
throw new BusinessException("菜单参数不能为空");
}
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
String menuType = normalizeMenuTypeRequired(request.getMenuType());
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
Long parentId = normalizeParentId(request.getParentId());
ensureParentValid(parentId, null, menuType);
ensureUniqueColumnKey(columnKey, null);
ensureUniqueRoutePath(menuType, routePath, null);
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setName(name);
entity.setColumnKey(columnKey);
entity.setParentId(parentId);
entity.setMenuType(menuType);
entity.setRoutePath(routePath);
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), null));
@@ -71,16 +92,30 @@ public class PermissionMenuService {
@Transactional
public PermissionMenuItemVo update(Long id, PermissionMenuUpdateRequest request) {
return update(null, id, request);
}
/** Controller-facing variant; validates a parent change against the full ancestor chain. */
@Transactional
public PermissionMenuItemVo update(AdminUserEntity operator, Long id, PermissionMenuUpdateRequest request) {
ensureAdminOperatorIfPresent(operator);
PermissionMenuEntity entity = getMenuById(id);
if (request == null) {
throw new BusinessException("菜单参数不能为空");
}
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
String menuType = normalizeMenuTypeRequired(request.getMenuType());
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
Long parentId = normalizeParentId(request.getParentId());
ensureParentValid(parentId, id, menuType);
ensureMenuTypeChangeAllowed(entity, menuType);
ensureUniqueColumnKey(columnKey, id);
ensureUniqueRoutePath(menuType, routePath, id);
entity.setName(name);
entity.setColumnKey(columnKey);
entity.setParentId(parentId);
entity.setMenuType(menuType);
entity.setRoutePath(routePath);
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), id));
@@ -90,102 +125,224 @@ public class PermissionMenuService {
@Transactional
public void delete(Long id) {
delete(null, id);
}
/**
* Deletes a leaf menu only. Refusing a parent deletion prevents orphaned
* hierarchy rows and makes the direct-grant meaning deterministic.
*/
@Transactional
public void delete(AdminUserEntity operator, Long id) {
ensureAdminOperatorIfPresent(operator);
PermissionMenuEntity entity = getMenuById(id);
Long childCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getParentId, entity.getId()));
if (childCount != null && childCount > 0) {
throw new BusinessException("请先删除该菜单下的子菜单");
}
// Do this explicitly even when the database FK is configured, so old
// installations without the FK are cleaned up as well.
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getColumnId, entity.getId()));
permissionMenuMapper.deleteById(entity.getId());
}
/** Returns only persisted direct grant IDs (never recursively expanded). */
public UserColumnIdsVo getUserColumnIds(Long userId, String menuType) {
return getUserColumnIds(null, userId, menuType);
}
/** Controller-facing variant with target-user scope checks. */
public UserColumnIdsVo getUserColumnIds(AdminUserEntity operator, Long userId, String menuType) {
AdminUserEntity user = getUserById(userId);
List<Long> columnIds;
if (isSuperAdmin(user)) {
columnIds = list(menuType).stream().map(PermissionMenuItemVo::getId).toList();
} else {
List<Long> assignedIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId))
.stream()
.map(UserColumnPermissionEntity::getColumnId)
.distinct()
.toList();
if (assignedIds.isEmpty()) {
columnIds = List.of();
} else if (isValidMenuType(menuType)) {
Set<Long> allowedIds = list(menuType).stream()
.map(PermissionMenuItemVo::getId)
.collect(Collectors.toSet());
columnIds = assignedIds.stream().filter(allowedIds::contains).toList();
} else {
columnIds = assignedIds;
}
}
ensureTargetAccessible(operator, user);
List<PermissionMenuEntity> menus = loadMenus(menuType);
Map<Long, PermissionMenuEntity> menuMap = menus.stream()
.filter(menu -> menu.getId() != null)
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity(), (left, right) -> left));
UserColumnIdsVo vo = new UserColumnIdsVo();
vo.setColumnIds(columnIds);
vo.setColumnIds(sortDirectIds(loadDirectColumnIds(userId), menuMap));
return vo;
}
/**
* Returns effective menu items: each directly granted menu plus every
* descendant in the same menu type. The response is flat and sorted.
*/
public List<PermissionMenuItemVo> getUserColumnPermissions(Long userId, String menuType) {
return getUserColumnPermissions(null, userId, menuType);
}
/** Controller-facing variant with target-user scope checks. */
public List<PermissionMenuItemVo> getUserColumnPermissions(AdminUserEntity operator,
Long userId,
String menuType) {
AdminUserEntity user = getUserById(userId);
ensureTargetAccessible(operator, user);
List<PermissionMenuEntity> menus = loadMenus(menuType);
if (isSuperAdmin(user)) {
return list(menuType);
return toItemVos(menus, menus);
}
List<Long> assignedIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId))
.stream()
.map(UserColumnPermissionEntity::getColumnId)
.distinct()
.toList();
if (assignedIds.isEmpty()) {
Set<Long> directIds = new LinkedHashSet<>(loadDirectColumnIds(userId));
if (directIds.isEmpty() || menus.isEmpty()) {
return List.of();
}
List<PermissionMenuEntity> menus = permissionMenuMapper.selectBatchIds(assignedIds);
Map<Long, PermissionMenuEntity> menuMap = menus.stream()
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity()));
String safeMenuType = normalizeMenuType(menuType);
return assignedIds.stream()
.map(menuMap::get)
.filter(menu -> menu != null)
.filter(menu -> safeMenuType == null || safeMenuType.equals(menu.getMenuType()))
.sorted(Comparator
.comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo))
.thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo)))
.map(this::toItemVo)
Set<Long> effectiveIds = expandDescendantIds(directIds, menus);
List<PermissionMenuEntity> effectiveMenus = menus.stream()
.filter(menu -> menu.getId() != null && effectiveIds.contains(menu.getId()))
.toList();
return toItemVos(effectiveMenus, menus);
}
/** Effective authorization check used by backend callers that need a boolean decision. */
public boolean hasEffectiveColumnPermission(Long userId, Long columnId) {
if (columnId == null || columnId <= 0) {
return false;
}
AdminUserEntity user = getUserById(userId);
PermissionMenuEntity target = permissionMenuMapper.selectById(columnId);
if (target == null) {
return false;
}
if (isSuperAdmin(user)) {
return true;
}
List<PermissionMenuEntity> menus = loadMenus(target.getMenuType());
return expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(userId)), menus)
.contains(columnId);
}
public List<ImageVideoDataPermissionUserVo> listImageVideoDataPermissionUsers(AdminUserEntity operator) {
ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
Set<Long> grantedUserIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getColumnId, dataPermission.getId()))
.stream()
.map(UserColumnPermissionEntity::getUserId)
.filter(id -> id != null && id > 0)
.collect(Collectors.toSet());
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
.orderByAsc(AdminUserEntity::getUsername)
.orderByAsc(AdminUserEntity::getId))
.stream()
.filter(user -> !isSuperAdmin(user))
.map(user -> toImageVideoDataPermissionUserVo(user, grantedUserIds.contains(user.getId())))
.toList();
}
@Transactional
public int updateImageVideoDataPermissionUsers(AdminUserEntity operator, List<Long> userIds) {
ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
List<Long> requestedIds = normalizeColumnIds(userIds);
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>());
Map<Long, AdminUserEntity> grantableUsers = users.stream()
.filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
throw new BusinessException("包含不存在或不可授权的用户");
}
userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
for (Long userId : requestedIds) {
UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
grant.setUserId(userId);
grant.setColumnId(dataPermission.getId());
userColumnPermissionMapper.insert(grant);
}
return requestedIds.size();
}
@Transactional
public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) {
getUserById(userId);
updateUserColumnPermissions(null, userId, request);
}
/**
* Replaces direct grants. A non-super admin may only grant effective menus
* that the operator already has, and only to themself or a directly-created
* normal user.
*/
@Transactional
public void updateUserColumnPermissions(AdminUserEntity operator,
Long userId,
UserColumnPermissionUpdateRequest request) {
updateUserColumnPermissions(operator, userId, request, null);
}
/** Replaces direct grants only within menuType; a blank type keeps legacy full-replacement behavior. */
@Transactional
public void updateUserColumnPermissions(AdminUserEntity operator,
Long userId,
UserColumnPermissionUpdateRequest request,
String menuType) {
AdminUserEntity target = getUserById(userId);
ensureTargetAccessible(operator, target);
List<Long> requestedIds = request == null ? List.of() : normalizeColumnIds(request.getColumnIds());
if (!requestedIds.isEmpty()) {
Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.in(PermissionMenuEntity::getId, requestedIds));
if (validCount == null || validCount != requestedIds.size()) {
throw new BusinessException("存在无效的菜单权限项");
String normalizedType = menuType == null || menuType.isBlank()
? null
: normalizeMenuTypeRequired(menuType);
Set<Long> scopedMenuIds = new LinkedHashSet<>();
if (normalizedType == null) {
if (!requestedIds.isEmpty()) {
Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.in(PermissionMenuEntity::getId, requestedIds));
if (validCount == null || validCount != requestedIds.size()) {
throw new BusinessException("存在无效的菜单权限项");
}
}
} else {
scopedMenuIds.addAll(loadMenus(normalizedType).stream()
.map(PermissionMenuEntity::getId)
.filter(Objects::nonNull)
.toList());
if (!scopedMenuIds.containsAll(requestedIds)) {
throw new BusinessException("存在无效或不属于当前类型的菜单权限项");
}
}
PermissionMenuEntity imageVideoDataPermission = permissionMenuMapper.selectOne(
new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY)
.last("LIMIT 1"));
if (imageVideoDataPermission != null) {
requestedIds = new ArrayList<>(requestedIds);
PermissionMenuEntity imageVideoDataPermission = normalizedType == null
? findImageVideoDataPermission()
: null;
Long protectedId = imageVideoDataPermission == null ? null : imageVideoDataPermission.getId();
List<Long> grantIds = requestedIds;
if (normalizedType == null && protectedId != null) {
grantIds = requestedIds.stream()
.filter(id -> !protectedId.equals(id))
.toList();
}
Set<Long> operatorEffectiveIds = ensureGrantable(operator, grantIds);
LinkedHashSet<Long> finalGrantIds = new LinkedHashSet<>(grantIds);
if (normalizedType == null && protectedId != null) {
Long existingCount = userColumnPermissionMapper.selectCount(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId)
.eq(UserColumnPermissionEntity::getColumnId, imageVideoDataPermission.getId()));
requestedIds.remove(imageVideoDataPermission.getId());
.eq(UserColumnPermissionEntity::getColumnId, protectedId));
if (existingCount != null && existingCount > 0) {
requestedIds.add(imageVideoDataPermission.getId());
finalGrantIds.add(protectedId);
}
}
if (operatorEffectiveIds != null) {
loadDirectColumnIds(userId).stream()
.filter(id -> normalizedType == null || scopedMenuIds.contains(id))
.filter(id -> !operatorEffectiveIds.contains(id))
.forEach(finalGrantIds::add);
}
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId));
for (Long columnId : requestedIds) {
LambdaUpdateWrapper<UserColumnPermissionEntity> delete =
new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId);
if (normalizedType == null) {
userColumnPermissionMapper.delete(delete);
} else if (!scopedMenuIds.isEmpty()) {
userColumnPermissionMapper.delete(delete.in(
UserColumnPermissionEntity::getColumnId, scopedMenuIds));
}
for (Long columnId : finalGrantIds) {
UserColumnPermissionEntity entity = new UserColumnPermissionEntity();
entity.setUserId(userId);
entity.setColumnId(columnId);
@@ -193,6 +350,75 @@ public class PermissionMenuService {
}
}
private List<PermissionMenuEntity> loadMenus(String menuType) {
String normalizedType = normalizeMenuType(menuType);
LambdaQueryWrapper<PermissionMenuEntity> query = new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(normalizedType != null, PermissionMenuEntity::getMenuType, normalizedType)
.orderByAsc(PermissionMenuEntity::getSortOrder)
.orderByAsc(PermissionMenuEntity::getId);
List<PermissionMenuEntity> rows = permissionMenuMapper.selectList(query);
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream()
.filter(row -> row != null && row.getId() != null)
.sorted(menuComparator())
.toList();
}
private List<Long> loadDirectColumnIds(Long userId) {
List<UserColumnPermissionEntity> rows = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId));
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream()
.map(UserColumnPermissionEntity::getColumnId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
}
private List<Long> sortDirectIds(List<Long> ids, Map<Long, PermissionMenuEntity> menuMap) {
return ids.stream()
.filter(menuMap::containsKey)
.sorted(Comparator.comparing(menuMap::get, menuComparator()))
.toList();
}
private Set<Long> expandDescendantIds(Set<Long> directIds, List<PermissionMenuEntity> menus) {
Map<Long, List<PermissionMenuEntity>> childrenByParent = new HashMap<>();
Map<Long, PermissionMenuEntity> menuMap = new HashMap<>();
for (PermissionMenuEntity menu : menus) {
if (menu.getId() == null) {
continue;
}
menuMap.put(menu.getId(), menu);
if (menu.getParentId() != null) {
childrenByParent.computeIfAbsent(menu.getParentId(), ignored -> new ArrayList<>()).add(menu);
}
}
childrenByParent.values().forEach(children -> children.sort(menuComparator()));
LinkedHashSet<Long> effective = new LinkedHashSet<>();
ArrayDeque<Long> queue = new ArrayDeque<>();
for (Long directId : directIds) {
if (directId != null && menuMap.containsKey(directId) && effective.add(directId)) {
queue.addLast(directId);
}
}
while (!queue.isEmpty()) {
Long parentId = queue.removeFirst();
for (PermissionMenuEntity child : childrenByParent.getOrDefault(parentId, List.of())) {
if (effective.add(child.getId())) {
queue.addLast(child.getId());
}
}
}
return effective;
}
private PermissionMenuEntity getMenuById(Long id) {
PermissionMenuEntity entity = permissionMenuMapper.selectById(id);
if (entity == null) {
@@ -209,8 +435,175 @@ public class PermissionMenuService {
return entity;
}
/** Resolves and validates an operator supplied by a trusted internal proxy. */
public AdminUserEntity requireAdminOperator(Long userId) {
AdminUserEntity operator = getUserById(userId);
ensureAdminOperatorIfPresent(operator);
return operator;
}
/** Resolves any authenticated user supplied by a trusted internal proxy. */
public AdminUserEntity requireUserOperator(Long userId) {
return getUserById(userId);
}
private void ensureAdminOperatorIfPresent(AdminUserEntity operator) {
if (operator == null) {
return;
}
if (!isAdmin(operator)) {
throw new BusinessException(403, "需要管理员权限");
}
}
private void ensureTargetAccessible(AdminUserEntity operator, AdminUserEntity target) {
if (operator == null) {
return;
}
boolean self = operator.getId() != null && operator.getId().equals(target.getId());
if (self || isSuperAdmin(operator)) {
return;
}
if (!isAdmin(operator)) {
throw new BusinessException(403, "需要管理员权限");
}
boolean ownedNormal = isNormalUser(target)
&& operator.getId() != null
&& operator.getId().equals(target.getCreatedById());
if (!ownedNormal) {
throw new BusinessException(403, "普通管理员只能管理自己创建的普通用户");
}
}
private Set<Long> ensureGrantable(AdminUserEntity operator, List<Long> requestedIds) {
if (operator == null || isSuperAdmin(operator)) {
return null;
}
List<PermissionMenuEntity> menus = loadMenus(null);
Set<Long> effective = expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(operator.getId())), menus);
Set<Long> denied = requestedIds.stream()
.filter(id -> !effective.contains(id))
.collect(Collectors.toCollection(LinkedHashSet::new));
if (!denied.isEmpty()) {
throw new BusinessException(403, "普通管理员只能分配自己已有的菜单权限");
}
return effective;
}
private boolean isAdmin(AdminUserEntity user) {
if (user == null) {
return false;
}
String role = normalizeRole(user.getRole());
if (!role.isEmpty()) {
return "super_admin".equals(role) || "admin".equals(role);
}
return Integer.valueOf(1).equals(user.getIsAdmin());
}
private boolean isSuperAdmin(AdminUserEntity user) {
return user != null && "super_admin".equalsIgnoreCase(normalizeRequired(user.getRole(), ""));
if (user == null) {
return false;
}
String role = normalizeRole(user.getRole());
if ("super_admin".equals(role)) {
return true;
}
return role.isEmpty()
&& Integer.valueOf(1).equals(user.getIsAdmin())
&& user.getCreatedById() == null;
}
private boolean isNormalUser(AdminUserEntity user) {
if (user == null) {
return false;
}
String role = normalizeRole(user.getRole());
if (!role.isEmpty()) {
return !"admin".equals(role) && !"super_admin".equals(role);
}
return !Integer.valueOf(1).equals(user.getIsAdmin());
}
private PermissionMenuEntity findImageVideoDataPermission() {
return permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY)
.last("LIMIT 1"));
}
private PermissionMenuEntity requireImageVideoDataPermission() {
PermissionMenuEntity permission = findImageVideoDataPermission();
if (permission == null || permission.getId() == null) {
throw new BusinessException("视频任务数据权限尚未初始化");
}
return permission;
}
private void ensureSuperAdminOperator(AdminUserEntity operator) {
ensureAdminOperatorIfPresent(operator);
if (operator == null || !isSuperAdmin(operator)) {
throw new BusinessException(403, "仅超级管理员可以配置视频任务权限");
}
}
private ImageVideoDataPermissionUserVo toImageVideoDataPermissionUserVo(AdminUserEntity user,
boolean granted) {
ImageVideoDataPermissionUserVo vo = new ImageVideoDataPermissionUserVo();
vo.setId(user.getId());
vo.setUsername(user.getUsername());
String role = normalizeRole(user.getRole());
vo.setRole(role.isEmpty()
? (Integer.valueOf(1).equals(user.getIsAdmin()) ? "admin" : "normal")
: role);
vo.setGranted(granted);
return vo;
}
private void ensureParentValid(Long parentId, Long currentId, String menuType) {
if (parentId == null) {
return;
}
if (currentId != null && currentId.equals(parentId)) {
throw new BusinessException("父级菜单不能选择自己");
}
PermissionMenuEntity parent = permissionMenuMapper.selectById(parentId);
if (parent == null) {
throw new BusinessException("父级菜单不存在");
}
String parentType = normalizeMenuType(parent.getMenuType());
if (!menuType.equals(parentType)) {
throw new BusinessException("父级菜单必须与当前菜单属于同一类型");
}
Set<Long> visited = new HashSet<>();
PermissionMenuEntity cursor = parent;
while (cursor != null && cursor.getId() != null) {
if (!visited.add(cursor.getId())) {
throw new BusinessException("菜单层级存在循环");
}
if (currentId != null && currentId.equals(cursor.getId())) {
throw new BusinessException("父级菜单不能选择自己的子级");
}
Long ancestorId = cursor.getParentId();
if (ancestorId == null) {
break;
}
cursor = permissionMenuMapper.selectById(ancestorId);
if (cursor == null) {
throw new BusinessException("父级菜单不存在");
}
}
}
private void ensureMenuTypeChangeAllowed(PermissionMenuEntity entity, String menuType) {
if (menuType.equals(normalizeMenuType(entity.getMenuType()))) {
return;
}
Long childCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getParentId, entity.getId()));
if (childCount != null && childCount > 0) {
throw new BusinessException("请先移出该菜单下的子菜单再修改菜单类型");
}
}
private void ensureUniqueColumnKey(String columnKey, Long excludeId) {
@@ -240,8 +633,12 @@ public class PermissionMenuService {
return normalized;
}
private boolean isValidMenuType(String menuType) {
return normalizeMenuType(menuType) != null;
private Long normalizeParentId(Long value) {
return value == null || value <= 0 ? null : value;
}
private String normalizeRole(String role) {
return role == null ? "" : role.trim().toLowerCase(Locale.ROOT);
}
private String normalizeMenuTypeRequired(String menuType) {
@@ -260,6 +657,8 @@ public class PermissionMenuService {
if (MENU_TYPE_APP.equals(normalized) || MENU_TYPE_ADMIN.equals(normalized)) {
return normalized;
}
// Keep the internal data permission visible to legacy callers that do
// not pass a menu type, but never treat an unknown filter as a type.
return null;
}
@@ -276,11 +675,34 @@ public class PermissionMenuService {
return new ArrayList<>(uniqueIds);
}
private Comparator<PermissionMenuEntity> menuComparator() {
return Comparator
.comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo))
.thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo));
}
private List<PermissionMenuItemVo> toItemVos(List<PermissionMenuEntity> visibleMenus,
List<PermissionMenuEntity> hierarchyMenus) {
Map<Long, PermissionMenuEntity> menuMap = hierarchyMenus.stream()
.filter(menu -> menu.getId() != null)
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity(), (left, right) -> left));
return visibleMenus.stream()
.map(menu -> toItemVo(menu, menuMap))
.toList();
}
private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity) {
return toItemVos(List.of(entity), loadMenus(entity.getMenuType())).getFirst();
}
private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity,
Map<Long, PermissionMenuEntity> menuMap) {
PermissionMenuItemVo vo = new PermissionMenuItemVo();
vo.setId(entity.getId());
vo.setName(entity.getName());
vo.setColumnKey(entity.getColumnKey());
vo.setParentId(entity.getParentId());
vo.setRootColumnKey(resolveRootColumnKey(entity, menuMap));
vo.setMenuType(entity.getMenuType());
vo.setRoutePath(entity.getRoutePath());
vo.setSortOrder(entity.getSortOrder());
@@ -288,6 +710,20 @@ public class PermissionMenuService {
return vo;
}
private String resolveRootColumnKey(PermissionMenuEntity entity,
Map<Long, PermissionMenuEntity> menuMap) {
PermissionMenuEntity root = entity;
Set<Long> visited = new HashSet<>();
while (root.getId() != null && visited.add(root.getId()) && root.getParentId() != null) {
PermissionMenuEntity parent = menuMap.get(root.getParentId());
if (parent == null) {
break;
}
root = parent;
}
return root.getColumnKey();
}
private Integer resolveSortOrder(Integer sortOrder, Long excludeId) {
if (sortOrder != null) {
return sortOrder;
@@ -20,7 +20,7 @@ public class ProductRiskMatchShopsRequest {
@NotEmpty(message = "shop_names 不能为空")
@JsonProperty("shop_names")
@Schema(
description = "待匹配的店铺名称列表,至少 1 个;服务端按列表顺序逐项查索引,返回体 `items` 与列表顺序一一对应",
description = "待匹配的店铺名称列表,至少 1 个;服务端规范化名称、过滤空项并按首次出现顺序去重",
requiredMode = Schema.RequiredMode.REQUIRED,
example = "[\"店铺甲\",\"店铺乙\"]")
private List<String> shopNames;
@@ -12,6 +12,6 @@ import java.util.List;
public class ProductRiskCountryPreferenceVo {
@JsonProperty("country_codes")
@Schema(description = "已选国家代码,顺序即处理顺序;未持久化时服务端返回默认德国→英国→法国→意大利→西班牙(全选)")
@Schema(description = "已选国家代码,顺序即处理顺序;未保存偏好时返回当前业务模块配置的默认顺序")
private List<String> countryCodes = new ArrayList<>();
}
@@ -7,9 +7,9 @@ import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "批量匹配响应体:items 与请求 shop_names 顺序一致")
@Schema(description = "批量匹配响应体:按规范化、过滤空项并去重后的有效店铺顺序返回")
public class ProductRiskMatchShopsVo {
@Schema(description = "各店匹配结果列表,一项对应请求中的一个店")
@Schema(description = "各店匹配结果列表,一项对应一个处理后的有效店铺")
private List<ProductRiskShopQueueItemVo> items = new ArrayList<>();
}
@@ -106,8 +106,8 @@ public class PublishController {
@PostMapping("/tasks/{taskId}/result")
@Operation(
summary = "Python 按文件回传当前店铺完整数据",
description = "请求只需 taskId 和 filesuser_id 为兼容旧客户端的可选字段;后端按 taskId 反查任务所属用户,传入 user_id 时会校验归属。成功回传必须包含该文件全部原始行,可使用 rows 数组,或在 rows 为空时使用 countries 按国家分组;行数少于原始数据、包含 null 行或八列全空白对象时会被拒绝且不会覆盖已解析数据。error 非空时将文件标记为 FAILED。全部文件进入终态后,只要至少一个文件成功就异步组装结果;全部失败则不生成结果文件。")
summary = "Python 按文件分片回传上架结果",
description = "请求只需 taskId 和 filesuser_id 为兼容旧客户端的可选字段;后端始终按 taskId 处理任务数据。每个文件通过 chunk_index/chunk_total 声明从 1 开始的分片,不传时按 1/1 兼容旧客户端。正常分片先写入 RustFS,同一文件的全部分片到齐后才按序合并、校验总行数并覆盖解析数据;重复分片内容相同则幂等接受,内容不同或 chunk_total 不一致则拒绝。error 非空时直接将文件标记为 FAILED。全部文件进入终态后,只要至少一个文件成功就异步组装结果;全部失败则不生成结果文件。")
public ApiResponse<Void> submitResult(
@Parameter(description = "上架任务 ID", required = true, example = "9001")
@PathVariable Long taskId,
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.publish.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import lombok.Data;
@@ -11,29 +12,39 @@ import java.util.List;
import java.util.Map;
@Data
@Schema(description = "单个文件的 Python 处理结果。fileIdfileKeysourceFilename 至少提供一个用于定位文件。成功时必须提交完整 rows 或 countries;失败时填写 error。同一次请求中,不同定位字段指向同一任务文件也视为重复提交")
@Schema(description = "单个文件的一次 Python 分片回传。fileId/file_key/source_filename 三者可用于定位文件,rows 或 countries 提供当前分片数据,error 用于返回失败")
public class PublishResultFileDto {
@JsonAlias("file_id")
@Schema(description = "任务内文件 ID;同时兼容 file_id,优先使用该字段定位", example = "9101")
private Long fileId;
@JsonAlias("file_key")
@Schema(description = "文件标识;同时兼容 file_key可在 fileId 缺失时定位文件", example = "uploads/20260724/uuid/郭亚庆.xlsx")
@Schema(description = "任务内文件标识;兼容 file_keyfileId 缺失时可通过该字段定位", example = "uploads/20260724/uuid/郭亚庆.xlsx")
private String fileKey;
@JsonAlias("source_filename")
@Schema(description = "原始文件名;同时兼容 source_filename可在 fileId/fileKey 缺失时定位文件", example = "郭亚庆.xlsx")
@Schema(description = "原始文件名;兼容 source_filenamefileId/fileKey 缺失时定位文件", example = "郭亚庆.xlsx")
private String sourceFilename;
@Schema(description = "文件级失败原因。非空时文件标记为 FAILEDrows/countries 不会覆盖原始数据", example = "打开店铺失败")
@JsonProperty("chunk_index")
@JsonAlias("chunkIndex")
@Schema(description = "分片序号,从 1 开始;不传时与 chunk_total 同时未传表示 1/1,兼容旧版。", example = "1", requiredMode = Schema.RequiredMode.NOT_REQUIRED)
private Integer chunkIndex;
@JsonProperty("chunk_total")
@JsonAlias("chunkTotal")
@Schema(description = "该文件的分片总数;不传时按 1 处理。分片先暂存到 RustFS,全部到齐后再按序合并。", example = "36", requiredMode = Schema.RequiredMode.NOT_REQUIRED)
private Integer chunkTotal;
@Schema(description = "文件处理结果;FAILED 时通过 error 携带错误信息,rows/countries 可为空", example = "打开文件失败")
private String error;
@Valid
@JsonAlias({"items", "data"})
@Schema(description = "当前店铺的完整结果行;同时兼容 items/data。成功时行数不能少于原始 Excel 数据行数,且不能包含 null 行或八列全空白对象")
@Schema(description = "当前分片的结果行,可为一条或多条;rows 中每条均不能为空对象")
private List<PublishRowDto> rows = new ArrayList<>();
@Valid
@Schema(description = "按国家名称或代码分组的完整结果,可替代 rows。仅当 rows 为空时读取;缺少国家字段的行会使用当前 Map key;分组内不能包含 null 行或八列全空白对象")
@Schema(description = "当前分片按国家分组的结果;rows 为空时可使用 countrieskey 为国家或店铺名")
private Map<String, List<PublishRowDto>> countries = new LinkedHashMap<>();
}
@@ -2,8 +2,10 @@ package com.nanri.aiimage.modules.publish.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
@@ -31,18 +33,24 @@ import com.nanri.aiimage.modules.publish.model.vo.PublishTaskDetailVo;
import com.nanri.aiimage.modules.publish.model.vo.PublishTaskVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -84,8 +92,11 @@ public class PublishTaskService {
private final PublishItemMapper publishItemMapper;
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskFileJobService taskFileJobService;
private final TaskDistributedLockService taskDistributedLockService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
@@ -239,7 +250,23 @@ public class PublishTaskService {
if (lock == null) {
throw new BusinessException("task lock is busy");
}
transactionTemplate.executeWithoutResult(status -> submitResultLocked(taskId, request));
List<String> storedPayloads = new ArrayList<>();
boolean[] cleanupAfterCommit = {false};
try {
transactionTemplate.executeWithoutResult(
status -> cleanupAfterCommit[0] = submitResultLocked(taskId, request, storedPayloads));
} catch (RuntimeException ex) {
deleteRolledBackPayloads(storedPayloads);
throw ex;
}
if (cleanupAfterCommit[0]) {
try {
deleteTransientResultChunks(taskId);
} catch (Exception ex) {
log.warn("[publish] failed-task chunk cleanup failed taskId={} msg={}",
taskId, safeMessage(ex));
}
}
}
}
@@ -455,7 +482,7 @@ public class PublishTaskService {
@Transactional
public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = requireTask(taskId, userId);
FileTaskEntity task = requireTaskForDeletion(taskId, userId);
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
@@ -470,6 +497,7 @@ public class PublishTaskService {
}
}
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
deleteTransientResultChunks(taskId);
publishItemMapper.delete(new LambdaQueryWrapper<PublishItemEntity>()
.eq(PublishItemEntity::getTaskId, taskId));
publishFileMapper.delete(new LambdaQueryWrapper<PublishFileEntity>()
@@ -486,8 +514,49 @@ public class PublishTaskService {
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果不存在");
}
FileTaskEntity task = requireTask(result.getTaskId(), userId);
deleteTask(task.getId(), userId);
deleteTask(result.getTaskId(), userId);
}
public void cleanupResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
return;
}
deleteTransientResultChunks(job.getTaskId());
}
private void deleteTransientResultChunks(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
private void deleteRolledBackPayloads(List<String> storedPayloads) {
if (storedPayloads == null || storedPayloads.isEmpty()) {
return;
}
for (String storedPayload : storedPayloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
} catch (Exception ex) {
log.warn("[publish] rolled-back RustFS payload cleanup failed pointer={} msg={}",
transientPayloadStorageService.extractPointer(storedPayload), safeMessage(ex));
}
}
}
private PreparedFile prepareFile(PublishSourceFileDto source) {
@@ -538,6 +607,26 @@ public class PublishTaskService {
}
LocalDateTime now = LocalDateTime.now();
String error = "任务心跳超时";
List<PublishFileEntity> files = listTaskFiles(taskId);
for (PublishFileEntity file : files) {
if (!STATUS_PENDING.equals(file.getStatus()) && !STATUS_RUNNING.equals(file.getStatus())) {
continue;
}
List<PublishRowDto> recoveredRows = loadReceivedResultRows(taskId, file.getId());
if (recoveredRows.isEmpty()) {
continue;
}
replaceRows(taskId, file.getId(), recoveredRows);
file.setStatus(STATUS_SUCCESS);
file.setTotalRows(recoveredRows.size());
file.setProcessedRows(recoveredRows.size());
file.setErrorMessage("任务心跳超时,已保留 " + recoveredRows.size() + " 行已回传数据");
file.setUpdatedAt(now);
file.setFinishedAt(now);
publishFileMapper.updateById(file);
log.info("[publish] recovered partial result from stale task taskId={} fileId={} rows={}",
taskId, file.getId(), recoveredRows.size());
}
publishFileMapper.update(null, new LambdaUpdateWrapper<PublishFileEntity>()
.eq(PublishFileEntity::getTaskId, taskId)
.in(PublishFileEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING))
@@ -545,7 +634,7 @@ public class PublishTaskService {
.set(PublishFileEntity::getErrorMessage, error)
.set(PublishFileEntity::getUpdatedAt, now)
.set(PublishFileEntity::getFinishedAt, now));
List<PublishFileEntity> files = listTaskFiles(taskId);
files = listTaskFiles(taskId);
int successCount = (int) files.stream().filter(file -> STATUS_SUCCESS.equals(file.getStatus())).count();
task.setSuccessFileCount(successCount);
task.setFailedFileCount(Math.max(0, files.size() - successCount));
@@ -615,10 +704,12 @@ public class PublishTaskService {
return new PersistedTask(task, result, savedFiles);
}
private void submitResultLocked(Long taskId, PublishSubmitResultRequest request) {
private boolean submitResultLocked(Long taskId,
PublishSubmitResultRequest request,
List<String> storedPayloads) {
FileTaskEntity task = requireTask(taskId, request.getUserId());
if (STATUS_SUCCESS.equals(task.getStatus())) {
return;
return false;
}
if (STATUS_FAILED.equals(task.getStatus())) {
throw new BusinessException("任务已失败,拒绝继续回传");
@@ -629,6 +720,9 @@ public class PublishTaskService {
Set<Long> submittedFileIds = new LinkedHashSet<>();
for (PublishResultFileDto incoming : request.getFiles()) {
if (incoming == null) {
throw new BusinessException("files 不能包含空对象");
}
PublishFileEntity file = findCallbackFile(taskId, incoming);
if (!submittedFileIds.add(file.getId())) {
throw new BusinessException("同一文件不能在一次请求中重复提交");
@@ -641,7 +735,16 @@ public class PublishTaskService {
file.setProcessedRows(0);
file.setErrorMessage(incoming.getError().trim());
} else {
List<PublishRowDto> rows = flattenRows(incoming);
ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads);
if (!receipt.completed()) {
file.setStatus(STATUS_RUNNING);
file.setErrorMessage(null);
file.setUpdatedAt(LocalDateTime.now());
file.setFinishedAt(null);
publishFileMapper.updateById(file);
continue;
}
List<PublishRowDto> rows = loadCompleteResultRows(taskId, receipt);
validateCompleteResultRows(taskId, file.getId(), rows);
replaceRows(taskId, file.getId(), rows);
file.setStatus(STATUS_SUCCESS);
@@ -666,11 +769,11 @@ public class PublishTaskService {
if (terminalCount < files.size()) {
task.setStatus(STATUS_RUNNING);
fileTaskMapper.updateById(task);
return;
return false;
}
if (successCount <= 0) {
markTaskAndResultFailed(task, result, "全部文件处理失败");
return;
return true;
}
task.setStatus(STATUS_RUNNING);
@@ -678,6 +781,7 @@ public class PublishTaskService {
task.setFinishedAt(null);
fileTaskMapper.updateById(task);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId));
return false;
}
private List<PublishTaskDetailVo> loadTaskDetails(List<FileTaskEntity> tasks) {
@@ -805,6 +909,20 @@ public class PublishTaskService {
}
private FileTaskEntity requireTask(Long taskId, Long userId) {
FileTaskEntity task = requireTaskRecord(taskId, userId);
ensureTaskOwnedByCurrentInstance(task, "access publish task");
return task;
}
private FileTaskEntity requireTaskForDeletion(Long taskId, Long userId) {
FileTaskEntity task = requireTaskRecord(taskId, userId);
if (!isTerminal(task.getStatus())) {
ensureTaskOwnedByCurrentInstance(task, "delete publish task");
}
return task;
}
private FileTaskEntity requireTaskRecord(Long taskId, Long userId) {
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId 不合法");
}
@@ -813,7 +931,6 @@ public class PublishTaskService {
|| (userId != null && !userId.equals(task.getUserId()))) {
throw new BusinessException("任务不存在");
}
ensureTaskOwnedByCurrentInstance(task, "access publish task");
return task;
}
@@ -888,6 +1005,257 @@ public class PublishTaskService {
return file;
}
private ResultChunkReceipt persistResultChunk(Long taskId,
PublishFileEntity file,
PublishResultFileDto incoming,
List<String> storedPayloads) {
int chunkIndex = incoming.getChunkIndex() == null ? 1 : incoming.getChunkIndex();
int chunkTotal = incoming.getChunkTotal() == null ? 1 : incoming.getChunkTotal();
validateChunkMetadata(chunkIndex, chunkTotal);
String scopeKey = "file:" + file.getId();
String scopeHash = DigestUtil.sha256Hex(scopeKey);
TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
List<PublishRowDto> rows = flattenRows(incoming);
String payloadJson = writeJson(rows, "序列化上架结果分片失败");
String payloadHash = DigestUtil.sha256Hex(payloadJson);
TaskChunkEntity existing = findResultChunk(taskId, scopeHash, chunkIndex);
if (existing != null) {
validateExistingChunk(existing, chunkTotal, payloadHash);
int receivedChunkCount = countResultChunks(taskId, scopeHash);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
receivedChunkCount >= chunkTotal);
}
ensureRustfsPayloadStorageEnabled();
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
requireRustfsPayload(storedPayload, "上架结果分片必须写入 RustFS");
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(taskId);
chunk.setModuleType(MODULE_TYPE);
chunk.setScopeKey(scopeKey);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setChunkTotal(chunkTotal);
chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(payloadHash);
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
try {
taskChunkMapper.insert(chunk);
storedPayloads.add(storedPayload);
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
if (winner == null) {
throw new BusinessException("上架结果分片并发写入失败,请重试");
}
validateExistingChunk(winner, chunkTotal, payloadHash);
} catch (RuntimeException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw ex;
}
int receivedChunkCount = countResultChunks(taskId, scopeHash);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} received={}",
taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
receivedChunkCount >= chunkTotal);
}
private void validateChunkMetadata(int chunkIndex, int chunkTotal) {
if (chunkIndex <= 0) {
throw new BusinessException("chunk_index 必须从 1 开始");
}
if (chunkTotal <= 0) {
throw new BusinessException("chunk_total 必须大于 0");
}
if (chunkIndex > chunkTotal) {
throw new BusinessException("chunk_index 不能大于 chunk_total");
}
}
private TaskScopeStateEntity findResultScope(Long taskId, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private TaskChunkEntity findResultChunk(Long taskId, String scopeHash, int chunkIndex) {
return taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
}
private void validateExistingChunk(TaskChunkEntity existing, int chunkTotal, String payloadHash) {
validateChunkTotal(existing.getChunkTotal(), chunkTotal);
if (!Objects.equals(existing.getPayloadHash(), payloadHash)) {
throw new BusinessException("同一 chunk_index 已回传不同内容,拒绝覆盖");
}
}
private void validateChunkTotal(Integer existingChunkTotal, int chunkTotal) {
if (existingChunkTotal != null && existingChunkTotal > 0 && existingChunkTotal != chunkTotal) {
throw new BusinessException("同一文件的 chunk_total 必须保持一致");
}
}
private int countResultChunks(Long taskId, String scopeHash) {
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
return count == null ? 0 : count.intValue();
}
private void persistResultScope(Long taskId,
String scopeKey,
String scopeHash,
int chunkTotal,
int receivedChunkCount) {
TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
LocalDateTime now = LocalDateTime.now();
if (scope == null) {
scope = new TaskScopeStateEntity();
scope.setTaskId(taskId);
scope.setModuleType(MODULE_TYPE);
scope.setScopeKey(scopeKey);
scope.setScopeHash(scopeHash);
scope.setCreatedAt(now);
}
boolean completed = receivedChunkCount >= chunkTotal;
scope.setChunkTotal(chunkTotal);
scope.setReceivedChunkCount(receivedChunkCount);
scope.setCompleted(completed ? 1 : 0);
scope.setLastChunkAt(now);
scope.setLastError(null);
scope.setStateJson(completed ? "{\"phase\":\"COMPLETE\"}" : "{\"phase\":\"RECEIVING\"}");
scope.setUpdatedAt(now);
if (scope.getId() != null) {
taskScopeStateMapper.updateById(scope);
return;
}
try {
taskScopeStateMapper.insert(scope);
} catch (DuplicateKeyException ex) {
TaskScopeStateEntity winner = findResultScope(taskId, scopeHash);
if (winner == null) {
throw new BusinessException("上架结果分片状态写入失败,请重试");
}
validateChunkTotal(winner.getChunkTotal(), chunkTotal);
winner.setChunkTotal(chunkTotal);
winner.setReceivedChunkCount(receivedChunkCount);
winner.setCompleted(completed ? 1 : 0);
winner.setLastChunkAt(now);
winner.setLastError(null);
winner.setStateJson(scope.getStateJson());
winner.setUpdatedAt(now);
taskScopeStateMapper.updateById(winner);
}
}
private List<PublishRowDto> loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, receipt.scopeHash())
.orderByAsc(TaskChunkEntity::getChunkIndex));
if (chunks == null || chunks.size() != receipt.chunkTotal()) {
throw new BusinessException("上架结果分片尚未完整,暂不能合并");
}
List<PublishRowDto> rows = new ArrayList<>();
TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
};
for (int i = 0; i < chunks.size(); i++) {
TaskChunkEntity chunk = chunks.get(i);
int expectedIndex = i + 1;
if (!Objects.equals(chunk.getChunkIndex(), expectedIndex)) {
throw new BusinessException("上架结果缺少第 " + expectedIndex + " 个分片");
}
validateChunkTotal(chunk.getChunkTotal(), receipt.chunkTotal());
List<PublishRowDto> chunkRows = readResultChunkRows(chunk, listType);
for (PublishRowDto row : chunkRows) {
rows.add(copyRequiredRow(row));
}
}
return rows;
}
private List<PublishRowDto> loadReceivedResultRows(Long taskId, Long fileId) {
String scopeHash = DigestUtil.sha256Hex("file:" + fileId);
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByAsc(TaskChunkEntity::getChunkIndex));
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
List<PublishRowDto> rows = new ArrayList<>();
TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
};
for (TaskChunkEntity chunk : chunks) {
try {
for (PublishRowDto row : readResultChunkRows(chunk, listType)) {
rows.add(copyRequiredRow(row));
}
} catch (Exception ex) {
log.warn("[publish] skip unreadable stale result chunk taskId={} fileId={} chunkIndex={} msg={}",
taskId, fileId, chunk.getChunkIndex(), safeMessage(ex));
}
}
return rows;
}
private List<PublishRowDto> readResultChunkRows(TaskChunkEntity chunk,
TypeReference<List<PublishRowDto>> listType) {
int chunkIndex = chunk.getChunkIndex() == null ? 0 : chunk.getChunkIndex();
String pointer = transientPayloadStorageService.extractPointer(chunk.getPayloadJson());
if (pointer == null || !pointer.startsWith("rustfs:")) {
throw new BusinessException("上架结果分片不是 RustFS 数据,拒绝合并");
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
chunk.getPayloadJson(), "读取上架结果分片失败");
List<PublishRowDto> rows = objectMapper.readValue(payloadJson, listType);
return rows == null ? List.of() : rows;
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("读取上架结果第 " + chunkIndex + " 个分片失败: "
+ safeMessage(ex));
}
}
private void ensureRustfsPayloadStorageEnabled() {
if (!transientPayloadStorageService.isSharedWriteEnabled()) {
throw new BusinessException("RustFS 未配置,上架结果分片暂不可接收");
}
}
private void requireRustfsPayload(String storedPayload, String message) {
String pointer = transientPayloadStorageService.extractPointer(storedPayload);
if (pointer != null && pointer.startsWith("rustfs:")) {
return;
}
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new BusinessException(message);
}
private List<PublishRowDto> flattenRows(PublishResultFileDto incoming) {
if (incoming.getRows() != null && !incoming.getRows().isEmpty()) {
return incoming.getRows().stream().map(this::copyRequiredRow).toList();
@@ -1266,4 +1634,9 @@ public class PublishTaskService {
private record TaskOptions(String publishCountry, List<String> syncCountries) {
}
private record ResultChunkReceipt(String scopeHash,
int chunkTotal,
boolean completed) {
}
}
@@ -0,0 +1,308 @@
package com.nanri.aiimage.modules.shopdatacrawl.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskBatchRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.util.List;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/shop-data-crawl")
@Tag(
name = "店铺数据抓取",
description = "运营工具中的店铺数据抓取接口:管理待处理店铺、匹配店铺、保存五国处理顺序、创建并轮询串行任务、接收 Python 分片结果及下载最终 Excel。任务相关操作均以 taskId 为准。")
public class ShopDataCrawlTaskController {
private final ShopDataCrawlResolveService resolveService;
private final ShopDataCrawlTaskService taskService;
@GetMapping("/candidates")
@Operation(
summary = "查询待处理店铺",
description = "返回当前用户已添加到店铺数据抓取管理区的店铺,按新增时间倒序排列。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<List<ProductRiskCandidateVo>> candidates(
@Parameter(name = "user_id", description = "当前用户 ID,店铺数据按用户隔离", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(resolveService.listCandidates(userId));
}
@PostMapping("/candidates")
@Operation(
summary = "添加待处理店铺",
description = "先校验店铺已在后台管理中维护且能够命中紫鸟店铺索引,再添加到当前用户的管理区;重复添加时返回已有记录。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "添加成功"))
public ApiResponse<ProductRiskCandidateVo> addCandidate(
@Valid @RequestBody ProductRiskCandidateAddRequest request) {
return ApiResponse.success(resolveService.addCandidate(request));
}
@DeleteMapping("/candidates/{id}")
@Operation(
summary = "删除待处理店铺",
description = "从当前用户的店铺数据抓取管理区删除指定记录,不会删除后台管理中的店铺。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"))
public ApiResponse<Void> deleteCandidate(
@Parameter(description = "待处理店铺记录主键", required = true, example = "10")
@PathVariable Long id,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验记录归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
resolveService.deleteCandidate(userId, id);
return ApiResponse.success(null);
}
@GetMapping("/country-preference")
@Operation(
summary = "查询国家处理顺序",
description = "返回当前用户勾选的国家代码,数组顺序就是抓取顺序。尚未保存时默认返回 UK、DE、FR、ES、IT。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ProductRiskCountryPreferenceVo> countryPreference(
@Parameter(name = "user_id", description = "当前用户 ID", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(resolveService.getCountryPreference(userId));
}
@PutMapping("/country-preference")
@Operation(
summary = "保存国家处理顺序",
description = "保存当前用户勾选并排序后的国家列表。支持 DE、FR、ES、IT、UK,至少选择 1 个且不能重复。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "保存成功"))
public ApiResponse<ProductRiskCountryPreferenceVo> saveCountryPreference(
@Valid @RequestBody ProductRiskCountryPreferenceSaveRequest request) {
return ApiResponse.success(resolveService.saveCountryPreference(request));
}
@PostMapping("/match-shops")
@Operation(
summary = "批量匹配店铺",
description = "店铺名称规范化、去重后逐一匹配。先确认店铺已在后台管理中维护,再查询紫鸟店铺索引,返回店铺 ID、平台、公司名称和匹配状态。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "匹配完成"))
public ApiResponse<ProductRiskMatchShopsVo> matchShops(
@Valid @RequestBody ProductRiskMatchShopsRequest request) {
return ApiResponse.success(resolveService.matchShops(request));
}
@GetMapping("/dashboard")
@Operation(
summary = "查询任务统计",
description = "返回当前用户的待处理店铺数、已结束任务数、成功任务数和失败任务数。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ProductRiskDashboardVo> dashboard(
@Parameter(name = "user_id", description = "当前用户 ID", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(taskService.dashboard(userId));
}
@GetMapping("/history")
@Operation(
summary = "查询抓取记录",
description = "返回当前用户最近 100 条店铺抓取结果,按创建时间倒序排列,包含执行状态和异步结果文件状态。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ShopDataCrawlHistoryVo> history(
@Parameter(name = "user_id", description = "当前用户 ID", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(taskService.listHistory(userId));
}
@PostMapping("/tasks/progress/batch")
@Operation(
summary = "批量查询任务进度",
description = "供前端轮询任务队列。对有效正数 taskId 去重后最多查询前 50 个;不存在、无结果或非本模块的任务写入 missingTaskIds。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ShopDataCrawlTaskBatchVo> progress(
@Valid @RequestBody ShopDataCrawlTaskBatchRequest request) {
return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds()));
}
@PostMapping("/tasks")
@Operation(
summary = "创建店铺数据抓取任务",
description = "每次必须且只能提交一个已匹配店铺,并按 country_codes 的顺序抓取。任务创建后状态为 RUNNING,返回 taskId 和该店铺的初始结果项。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功"))
public ApiResponse<ShopDataCrawlCreateTaskVo> createTask(
@Valid @RequestBody ShopDataCrawlCreateTaskRequest request) {
return ApiResponse.success(taskService.createTask(request));
}
@PostMapping("/tasks/{taskId}/result")
@Operation(
summary = "回传店铺抓取结果",
description = "供 Python 按 taskId 回传,不依赖 user_id。每次 shops 必须且只能包含任务中的一个店铺。新协议要求同时传 chunk_index/chunk_total,分片从 1 开始、允许乱序和同内容重试,数据严格暂存到 RustFS,全部到齐后按序合并;旧客户端未传分片字段时仍按 shopDone 累加。error 非空时店铺立即失败;成功结束后异步生成 Excel。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "回传成功"))
public ApiResponse<Void> submitResult(
@Parameter(description = "店铺数据抓取任务 ID;服务端据此路由到任务归属实例,任务必须尚未结束", required = true,
example = "20363")
@PathVariable Long taskId,
@io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "单个店铺的一次结果分片。chunk_index 与 chunk_total 必须同时传入。结果行依次包含日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价。",
required = true,
content = @Content(
mediaType = "application/json",
schema = @Schema(implementation = ShopDataCrawlSubmitResultRequest.class),
examples = {
@ExampleObject(
name = "中间分片",
summary = "继续等待后续分片",
value = """
{
"shops": [
{
"shopName": "示例店铺",
"chunk_index": 1,
"chunk_total": 2,
"countryResults": [
{
"country": "DE",
"items": [
{
"date": "2026-07-25",
"asin": "B0EXAMPLE1",
"inventorySales": "120",
"salesRank": "35",
"pageViews": "860",
"unitsSold": "18",
"price": "29.99",
"recommendedOffer": "27.99"
}
]
}
],
"submissionId": "shop-data-crawl:20363:part-1"
}
]
}
"""),
@ExampleObject(
name = "最终分片",
summary = "完成该店铺并触发异步生成 Excel",
value = """
{
"shops": [
{
"shopName": "示例店铺",
"chunk_index": 2,
"chunk_total": 2,
"countryResults": [
{
"country": "UK",
"items": []
}
],
"submissionId": "shop-data-crawl:20363:final"
}
]
}
"""),
@ExampleObject(
name = "失败结果",
summary = "错误信息非空时直接将店铺标记为失败",
value = """
{
"shops": [
{
"shopName": "示例店铺",
"error": "紫鸟页面加载超时",
"submissionId": "shop-data-crawl:20363:error"
}
]
}
""")
}))
@Valid @RequestBody ShopDataCrawlSubmitResultRequest request) {
taskService.submitResult(taskId, request);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(
summary = "下载抓取结果 Excel",
description = "校验 resultId 属于当前 user_id 后,从对象存储读取最终 XLSX 并以附件返回。结果文件异步生成,只有列表中的 fileReady=true 后才能下载。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200",
description = "店铺数据抓取结果 Excel 文件",
content = @Content(
mediaType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
schema = @Schema(type = "string", format = "binary"))))
public void download(
@Parameter(description = "结果记录主键", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验结果归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = taskService.resolveResultDownloadUrl(resultId, userId);
String filename = taskService.resolveResultDownloadFilename(resultId, userId);
try {
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream input = URI.create(url).toURL().openStream()) {
input.transferTo(response.getOutputStream());
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
@DeleteMapping("/tasks/{taskId}")
@Operation(
summary = "删除整条抓取任务",
description = "校验任务归属后,删除任务、结果记录、文件生成作业、进度快照和不再被引用的结果文件。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"))
public ApiResponse<Void> deleteTask(
@Parameter(description = "店铺数据抓取任务 ID", required = true, example = "20363")
@PathVariable Long taskId,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验任务归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
taskService.deleteTask(taskId, userId);
return ApiResponse.success(null);
}
@DeleteMapping("/history/{resultId}")
@Operation(
summary = "删除单条抓取记录",
description = "校验结果归属后,删除指定结果及其文件作业和快照,并重新计算父任务状态;任务没有其他结果时会一并删除父任务。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"))
public ApiResponse<Void> deleteHistory(
@Parameter(description = "结果记录主键", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前用户 ID,用于校验结果归属", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
taskService.deleteHistory(resultId, userId);
return ApiResponse.success(null);
}
}
@@ -0,0 +1,8 @@
package com.nanri.aiimage.modules.shopdatacrawl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ShopDataCrawlCountryPrefMapper extends BaseMapper<ShopDataCrawlCountryPrefEntity> {}
@@ -0,0 +1,10 @@
package com.nanri.aiimage.modules.shopdatacrawl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ShopDataCrawlShopCandidateMapper extends BaseMapper<ShopDataCrawlShopCandidateEntity> {
}
@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个国家站点的抓取结果分片")
public class ShopDataCrawlCountryResultDto {
@Schema(
description = "国家站点代码;生成结果文件时仅 DE、FR、ES、IT、UK 会写入对应工作表",
example = "UK",
allowableValues = {"DE", "FR", "ES", "IT", "UK"})
private String country;
@Schema(description = "该国家站点的抓取数据行;多次分片回传时后端会合并并去重")
private List<ShopDataCrawlRowDto> items = new ArrayList<>();
}
@@ -0,0 +1,35 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "创建店铺数据抓取任务请求;每个任务必须且只能处理一个已匹配店铺")
public class ShopDataCrawlCreateTaskRequest {
@NotNull
@JsonProperty("user_id")
@Schema(description = "任务所属用户 ID,必须为正数", example = "10001", requiredMode = Schema.RequiredMode.REQUIRED)
private Long userId;
@Valid
@NotEmpty
@Size(min = 1, max = 1, message = "items 必须且只能包含一个店铺")
@Schema(description = "待抓取店铺列表,必须且只能包含一个有效且已匹配的店铺", requiredMode = Schema.RequiredMode.REQUIRED)
private List<ShopDataCrawlTaskItemDto> items = new ArrayList<>();
@NotEmpty
@JsonProperty("country_codes")
@Schema(
description = "本任务处理的国家代码,必须提交 1-5 个且不得重复,仅支持 DE、FR、ES、IT、UK;列表顺序就是处理顺序。用户未保存偏好时,偏好接口默认返回 UK、DE、FR、ES、IT,但创建任务仍需显式提交本字段",
example = "[\"UK\", \"DE\", \"FR\", \"ES\", \"IT\"]",
allowableValues = {"DE", "FR", "ES", "IT", "UK"},
requiredMode = Schema.RequiredMode.REQUIRED)
private List<String> countryCodes = new ArrayList<>();
}
@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "店铺数据抓取结果行;生成 Excel 时按日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价的固定列顺序写入")
public class ShopDataCrawlRowDto {
@JsonAlias("日期")
@Schema(description = "日期列,按来源文本原样保留", example = "2026-07-25")
private String date;
@JsonAlias("ASIN")
@Schema(description = "亚马逊商品 ASIN", example = "B0CJ8SNXXV")
private String asin;
@JsonAlias({"库存销量", "inventory_sales"})
@Schema(description = "库存销量列,按来源文本原样保留", example = "128")
private String inventorySales;
@JsonAlias({"销售排名", "sales_rank"})
@Schema(description = "销售排名列,按来源文本原样保留", example = "#1,245")
private String salesRank;
@JsonAlias({"页面浏览量", "page_views"})
@Schema(description = "页面浏览量列,按来源文本原样保留", example = "3560")
private String pageViews;
@JsonAlias({"售出件数", "units_sold"})
@Schema(description = "售出件数列,按来源文本原样保留", example = "42")
private String unitsSold;
@JsonAlias("价格")
@Schema(description = "价格列,含货币格式时按来源文本原样保留", example = "GBP 19.99")
private String price;
@JsonAlias({"推荐报价", "recommended_offer"})
@Schema(description = "推荐报价列,含货币格式时按来源文本原样保留", example = "GBP 18.99")
private String recommendedOffer;
}
@@ -0,0 +1,44 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个店铺的一次抓取结果分片;新客户端通过 chunk_index/chunk_total 回传,旧客户端仍可使用 shopDone 增量协议")
public class ShopDataCrawlShopPayloadDto {
@JsonProperty("shopName")
@Schema(description = "店铺名称,必须与任务中的唯一店铺一致;为空时后端使用任务店铺名称", example = "示例英国店")
private String shopName;
@Schema(description = "店铺处理错误信息;只要是非空文本,后端就会忽略 shopDone 并立即将店铺及任务标记失败", example = "紫鸟页面加载超时,未能完成数据抓取")
private String error;
@JsonProperty("countryResults")
@Schema(description = "本分片的国家结果;显式分片暂存到 RustFS,全部到齐后按 chunk_index 顺序合并")
private List<ShopDataCrawlCountryResultDto> countryResults = new ArrayList<>();
@JsonProperty("chunk_index")
@JsonAlias("chunkIndex")
@Schema(description = "分片序号,从 1 开始;必须与 chunk_total 同时传入。不传时沿用旧版 shopDone 增量协议。", example = "1")
private Integer chunkIndex;
@JsonProperty("chunk_total")
@JsonAlias("chunkTotal")
@Schema(description = "该店铺本次结果的分片总数;必须与 chunk_index 同时传入,且同一任务内保持一致。", example = "10")
private Integer chunkTotal;
@JsonProperty("shopDone")
@Schema(description = "旧版增量协议的完成标志。显式传入 chunk_index/chunk_total 时,后端只按分片是否全部到齐判断完成,不会被 shopDone 提前结束。", example = "true")
private Boolean shopDone;
@JsonProperty("submissionId")
@Schema(description = "本次店铺提交批次标识,便于问题排查与日志追踪;不参与业务去重", example = "shop-data-crawl:3089:示例英国店:1784941200000")
private String submissionId;
}
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取结果回传请求;支持同一店铺分多次回传,但每次请求只能包含一个店铺结果")
public class ShopDataCrawlSubmitResultRequest {
@Valid
@NotEmpty
@Schema(description = "本次提交的店铺结果列表,必须且只能包含一个与任务匹配的店铺;中间分片和最后一片均遵循此限制", requiredMode = Schema.RequiredMode.REQUIRED)
private List<ShopDataCrawlShopPayloadDto> shops = new ArrayList<>();
}
@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "批量查询店铺数据抓取任务进度的请求")
public class ShopDataCrawlTaskBatchRequest {
@NotEmpty(message = "taskIds 不能为空")
@Schema(
description = "待查询的任务 ID 列表;后端按输入顺序过滤 null、零和负数,去重后最多处理前 50 个有效 ID",
example = "[12001, 12002, 12001, -1]",
requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds = new ArrayList<>();
}
@@ -0,0 +1,44 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取任务中的单店铺项;创建任务时后端会按店铺名称重新校验匹配信息")
public class ShopDataCrawlTaskItemDto {
@JsonProperty("shopName")
@Schema(description = "店铺名称,是创建任务时重新匹配紫鸟店铺的依据,不得为空", example = "示例英国店", requiredMode = Schema.RequiredMode.REQUIRED)
private String shopName;
@Schema(description = "是否已命中紫鸟店铺索引;创建成功后由后端写为 true", example = "true")
private boolean matched;
@JsonProperty("shopId")
@Schema(description = "紫鸟侧店铺 ID;创建任务时以后端重新匹配到的值为准", example = "3089")
private String shopId;
@Schema(description = "店铺所属平台;创建任务时以后端重新匹配到的值为准", example = "Amazon")
private String platform;
@JsonProperty("companyName")
@Schema(description = "店铺所属公司名称;创建任务时以后端重新匹配到的值为准", example = "示例跨境贸易有限公司")
private String companyName;
@JsonProperty("matchStatus")
@Schema(description = "店铺匹配状态码;创建任务时以后端重新匹配结果为准", example = "MATCHED")
private String matchStatus;
@JsonProperty("matchMessage")
@Schema(description = "店铺匹配状态说明;创建任务时以后端重新匹配结果为准", example = "店铺索引匹配成功")
private String matchMessage;
@JsonProperty("countryCodes")
@Schema(
description = "该店铺实际处理的国家代码及顺序;创建任务后由顶层 country_codes 覆盖",
example = "[\"UK\", \"DE\"]",
allowableValues = {"DE", "FR", "ES", "IT", "UK"})
private List<String> countryCodes = new ArrayList<>();
}
@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_shop_data_crawl_country_pref")
public class ShopDataCrawlCountryPrefEntity {
@TableId(type = IdType.INPUT) private Long userId;
private String countryCodesJson;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_shop_data_crawl_shop_candidate")
public class ShopDataCrawlShopCandidateEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private String shopName;
private LocalDateTime createdAt;
}
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取任务创建结果")
public class ShopDataCrawlCreateTaskVo {
@Schema(description = "新创建的任务 ID,用于轮询进度和回传抓取结果", example = "12001")
private Long taskId;
@Schema(description = "任务创建后的单店铺初始快照;由于每个任务只允许一个店铺,本列表固定包含一项")
private List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "当前用户的店铺数据抓取历史记录")
public class ShopDataCrawlHistoryVo {
@Schema(description = "历史记录项,按创建时间倒序返回,最多返回最近 100 条")
private List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
}
@@ -0,0 +1,97 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "单个店铺的数据抓取任务结果快照,同时用于任务创建、进度查询和历史记录")
public class ShopDataCrawlResultItemVo {
@Schema(description = "结果记录 ID,用于删除历史记录和下载结果文件", example = "45001")
private Long resultId;
@Schema(description = "所属店铺数据抓取任务 ID", example = "12001")
private Long taskId;
@JsonProperty("shopName")
@Schema(description = "店铺名称", example = "示例英国店")
private String shopName;
@JsonProperty("shopId")
@Schema(description = "紫鸟侧店铺 ID", example = "3089")
private String shopId;
@Schema(description = "店铺所属平台", example = "Amazon")
private String platform;
@JsonProperty("companyName")
@Schema(description = "店铺所属公司名称", example = "示例跨境贸易有限公司")
private String companyName;
@Schema(description = "创建任务时是否成功命中紫鸟店铺索引", example = "true")
private boolean matched;
@JsonProperty("matchStatus")
@Schema(description = "创建任务时保存的店铺匹配状态码", example = "MATCHED")
private String matchStatus;
@JsonProperty("matchMessage")
@Schema(description = "创建任务时保存的店铺匹配说明", example = "店铺索引匹配成功")
private String matchMessage;
@JsonProperty("taskStatus")
@Schema(
description = "任务状态:RUNNING=抓取或结果文件生成中,SUCCESS=任务成功且文件生成完成,FAILED=抓取或任务处理失败",
example = "RUNNING",
allowableValues = {"RUNNING", "SUCCESS", "FAILED"})
private String taskStatus;
@Schema(description = "店铺抓取结果:null=尚未完成,true=抓取成功,false=抓取失败;true 时结果文件仍可能在异步生成", example = "true")
private Boolean success;
@Schema(description = "抓取或任务失败原因;无错误时为空", example = "紫鸟页面加载超时")
private String error;
@Schema(description = "结果记录创建时间,ISO 本地日期时间", example = "2026-07-25T10:00:00")
private LocalDateTime createdAt;
@Schema(description = "任务结束时间;任务仍在运行时为空,ISO 本地日期时间", example = "2026-07-25T10:05:00")
private LocalDateTime finishedAt;
@Schema(description = "结果 Excel 文件名;文件异步生成期间可能已返回文件名,但不能据此判断是否可下载", example = "店铺数据抓取-示例英国店-12001.xlsx")
private String outputFilename;
@Schema(description = "兼容保留的下载地址字段,当前接口通常返回空;fileReady=true 后应通过结果下载接口获取文件")
private String downloadUrl;
@Schema(description = "异步结果文件生成任务 ID;尚未创建文件任务时为空", example = "88001")
private Long fileJobId;
@Schema(
description = "异步结果文件生成状态;为空表示尚未创建文件任务,PENDING=等待生成,RUNNING=生成中,SUCCESS=生成成功,FAILED=生成失败",
example = "PENDING",
allowableValues = {"PENDING", "RUNNING", "SUCCESS", "FAILED"})
private String fileStatus;
@Schema(description = "异步结果文件生成失败原因;文件任务未失败时为空", example = "上传结果文件失败")
private String fileError;
@Schema(description = "结果文件是否已生成并可下载;只有 true 时才能调用结果下载接口", example = "false")
private Boolean fileReady;
@JsonProperty("countryCodes")
@Schema(
description = "任务实际处理的国家代码,顺序与创建任务时提交的 country_codes 一致",
example = "[\"UK\", \"DE\"]",
allowableValues = {"DE", "FR", "ES", "IT", "UK"})
private List<String> countryCodes = new ArrayList<>();
@JsonProperty("countryResults")
@Schema(description = "按国家累计合并后的抓取结果;中间分片阶段可能只包含部分国家或部分数据行")
private List<ShopDataCrawlCountryResultDto> countryResults = new ArrayList<>();
}
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.shopdatacrawl.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "店铺数据抓取任务的批量进度查询结果")
public class ShopDataCrawlTaskBatchVo {
@Schema(description = "查询到的店铺结果快照,按规范化后的 taskIds 顺序返回;每个任务当前只包含一个店铺结果")
private List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
@Schema(description = "未找到任务或任务没有结果记录的任务 ID,顺序与规范化后的请求顺序一致", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}
@@ -0,0 +1,110 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
public class ShopDataCrawlExcelAssemblyService {
static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
static final List<String> HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
public void writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
XSSFWorkbook workbook = new XSSFWorkbook(input);
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
for (int i = 0; i < COUNTRIES.size(); i++) {
writeSheet(workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)));
}
workbook.write(output);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("生成店铺数据抓取 Excel 失败: " + ex.getMessage());
}
}
public int countRows(List<ShopDataCrawlResultItemVo> items) {
return rowsByCountry(items).values().stream().mapToInt(List::size).sum();
}
void validateTemplate(XSSFWorkbook workbook) {
if (workbook.getNumberOfSheets() != SHEETS.size()) {
throw new BusinessException("店铺数据抓取模板工作表数量不正确");
}
for (int i = 0; i < SHEETS.size(); i++) {
Sheet sheet = workbook.getSheetAt(i);
if (!SHEETS.get(i).equals(sheet.getSheetName())) {
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
}
Row header = sheet.getRow(0);
for (int column = 0; column < HEADERS.size(); column++) {
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
if (!HEADERS.get(column).equals(actual)) {
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
}
}
}
}
private void writeSheet(Sheet sheet, List<ShopDataCrawlRowDto> rows) {
Row styleRow = sheet.getRow(1);
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
Cell cell = styleRow == null ? null : styleRow.getCell(column);
styles[column] = cell == null ? null : cell.getCellStyle();
}
int last = sheet.getLastRowNum();
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
Row row = sheet.getRow(rowIndex);
if (row != null) {
sheet.removeRow(row);
}
}
int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
Row row = sheet.createRow(rowIndex++);
String[] values = {value.getDate(), value.getAsin(), value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
}
}
private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>();
COUNTRIES.forEach(country -> result.put(country, new ArrayList<>()));
if (items == null) return result;
for (ShopDataCrawlResultItemVo item : items) {
if (item == null || Boolean.FALSE.equals(item.getSuccess()) || item.getCountryResults() == null) continue;
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) {
String country = countryResult == null || countryResult.getCountry() == null ? "" : countryResult.getCountry().trim().toUpperCase();
if (result.containsKey(country) && countryResult.getItems() != null) result.get(country).addAll(countryResult.getItems());
}
}
return result;
}
}
@@ -0,0 +1,313 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity;
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
import com.nanri.aiimage.modules.shopkey.service.ShopManageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
@Service
@RequiredArgsConstructor
public class ShopDataCrawlResolveService {
/**
* 默认国家偏好顺序:英国 -> 德国 -> 法国 -> 西班牙 -> 意大利,与前端和结果模板保持一致。
*/
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("UK", "DE", "FR", "ES", "IT");
private final ShopDataCrawlShopCandidateMapper candidateMapper;
private final ShopDataCrawlCountryPrefMapper countryPrefMapper;
private final ObjectMapper objectMapper;
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final ShopManageService shopManageService;
public List<ProductRiskCandidateVo> listCandidates(Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
}
List<ShopDataCrawlShopCandidateEntity> rows = candidateMapper.selectList(
new LambdaQueryWrapper<ShopDataCrawlShopCandidateEntity>()
.eq(ShopDataCrawlShopCandidateEntity::getUserId, userId)
.orderByDesc(ShopDataCrawlShopCandidateEntity::getId));
List<ProductRiskCandidateVo> list = new ArrayList<>();
for (ShopDataCrawlShopCandidateEntity row : rows) {
if (row == null) {
continue;
}
ProductRiskCandidateVo vo = new ProductRiskCandidateVo();
vo.setId(row.getId());
vo.setShopName(row.getShopName());
vo.setCreatedAt(row.getCreatedAt());
list.add(vo);
}
return list;
}
public long countCandidates(Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
Long count = candidateMapper.selectCount(new LambdaQueryWrapper<ShopDataCrawlShopCandidateEntity>()
.eq(ShopDataCrawlShopCandidateEntity::getUserId, userId));
return count == null ? 0L : count;
}
@Transactional
public ProductRiskCandidateVo addCandidate(ProductRiskCandidateAddRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法");
}
String normalized = ziniaoShopSwitchService.normalizeShopName(request.getShopName());
if (normalized.isBlank()) {
throw new BusinessException("店铺名不能为空");
}
shopManageService.requireShopByName(normalized);
ZiniaoShopMatchResultVo indexHit = ziniaoShopSwitchService.findIndexedStoreByName(normalized, false);
if (indexHit == null || !indexHit.isMatched()) {
String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank()
? indexHit.getMatchMessage()
: "店铺索引未命中,无法加入待处理列表";
throw new BusinessException(hint);
}
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) {
throw new BusinessException(indexHit.getMatchMessage() != null ? indexHit.getMatchMessage() : "存在多个同名店铺,请人工确认");
}
ShopDataCrawlShopCandidateEntity existing = candidateMapper.selectOne(
new LambdaQueryWrapper<ShopDataCrawlShopCandidateEntity>()
.eq(ShopDataCrawlShopCandidateEntity::getUserId, request.getUserId())
.eq(ShopDataCrawlShopCandidateEntity::getShopName, normalized)
.last("LIMIT 1"));
if (existing != null) {
ProductRiskCandidateVo vo = new ProductRiskCandidateVo();
vo.setId(existing.getId());
vo.setShopName(existing.getShopName());
vo.setCreatedAt(existing.getCreatedAt());
return vo;
}
ShopDataCrawlShopCandidateEntity entity = new ShopDataCrawlShopCandidateEntity();
entity.setUserId(request.getUserId());
entity.setShopName(normalized);
entity.setCreatedAt(LocalDateTime.now());
candidateMapper.insert(entity);
ProductRiskCandidateVo vo = new ProductRiskCandidateVo();
vo.setId(entity.getId());
vo.setShopName(entity.getShopName());
vo.setCreatedAt(entity.getCreatedAt());
return vo;
}
@Transactional
public void deleteCandidate(Long userId, Long id) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
}
if (id == null || id <= 0) {
throw new BusinessException("id 不合法");
}
ShopDataCrawlShopCandidateEntity row = candidateMapper.selectById(id);
if (row == null || !userId.equals(row.getUserId())) {
throw new BusinessException("记录不存在");
}
candidateMapper.deleteById(id);
}
public ProductRiskMatchShopsVo matchShops(ProductRiskMatchShopsRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法");
}
LinkedHashSet<String> ordered = new LinkedHashSet<>();
for (String raw : request.getShopNames()) {
String n = ziniaoShopSwitchService.normalizeShopName(raw);
if (!n.isBlank()) {
ordered.add(n);
}
}
if (ordered.isEmpty()) {
throw new BusinessException("shop_names 没有有效店铺名");
}
ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo();
for (String shopName : ordered) {
vo.getItems().add(matchOneShop(shopName));
}
return vo;
}
public ZiniaoShopMatchResultVo requireMatchedShop(String shopName) {
String normalized = ziniaoShopSwitchService.normalizeShopName(shopName);
shopManageService.requireShopByName(normalized);
ZiniaoShopMatchResultVo matched = ziniaoShopSwitchService.findIndexedStoreByName(normalized, false);
if (matched == null || !matched.isMatched()) {
throw new BusinessException(matched != null && matched.getMatchMessage() != null
? matched.getMatchMessage() : "店铺索引未命中");
}
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(matched.getMatchStatus())) {
throw new BusinessException(matched.getMatchMessage() == null ? "存在多个同名店铺,请人工确认" : matched.getMatchMessage());
}
return matched;
}
public ProductRiskCountryPreferenceVo getCountryPreference(Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
}
ShopDataCrawlCountryPrefEntity row = countryPrefMapper.selectById(userId);
ProductRiskCountryPreferenceVo vo = new ProductRiskCountryPreferenceVo();
if (row == null || row.getCountryCodesJson() == null || row.getCountryCodesJson().isBlank()) {
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
return vo;
}
try {
List<String> parsed = objectMapper.readValue(row.getCountryCodesJson(), new TypeReference<List<String>>() {
});
vo.getCountryCodes().addAll(sanitizeStoredCodes(parsed));
} catch (Exception ex) {
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
}
return vo;
}
@Transactional
public ProductRiskCountryPreferenceVo saveCountryPreference(ProductRiskCountryPreferenceSaveRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法");
}
List<String> normalized = validateCountryCodesForSave(request.getCountryCodes());
String json;
try {
json = objectMapper.writeValueAsString(normalized);
} catch (Exception ex) {
throw new BusinessException("保存偏好失败");
}
ShopDataCrawlCountryPrefEntity row = countryPrefMapper.selectById(request.getUserId());
if (row == null) {
row = new ShopDataCrawlCountryPrefEntity();
row.setUserId(request.getUserId());
row.setCountryCodesJson(json);
countryPrefMapper.insert(row);
} else {
row.setCountryCodesJson(json);
countryPrefMapper.updateById(row);
}
ProductRiskCountryPreferenceVo vo = new ProductRiskCountryPreferenceVo();
vo.getCountryCodes().addAll(normalized);
return vo;
}
private static List<String> sanitizeStoredCodes(List<String> raw) {
List<String> parsed = parseValidCountryCodes(raw);
if (parsed.isEmpty()) {
return new ArrayList<>(DEFAULT_COUNTRY_PREFERENCE_ORDER);
}
return parsed;
}
private static List<String> validateCountryCodesForSave(List<String> raw) {
if (raw == null || raw.isEmpty()) {
throw new BusinessException("country_codes 至少选择 1 个国家");
}
LinkedHashSet<String> seen = new LinkedHashSet<>();
List<String> out = new ArrayList<>();
for (String s : raw) {
if (s == null || s.isBlank()) {
throw new BusinessException("country_codes 含空项");
}
String u = s.trim().toUpperCase();
try {
ProductRiskCountryCode.valueOf(u);
} catch (IllegalArgumentException ex) {
throw new BusinessException("非法国家代码: " + s);
}
if (!seen.add(u)) {
throw new BusinessException("country_codes 存在重复: " + u);
}
out.add(u);
}
if (out.size() > ProductRiskCountryCode.values().length) {
throw new BusinessException("country_codes 最多 5 项");
}
return out;
}
public List<String> validateCountryCodes(List<String> raw) {
return validateCountryCodesForSave(raw);
}
/**
* 仅保留合法国家代码,去重并保持顺序;非法值静默忽略,用于读取库存量数据。
*/
private static List<String> parseValidCountryCodes(List<String> raw) {
if (raw == null || raw.isEmpty()) {
return new ArrayList<>();
}
LinkedHashSet<String> seen = new LinkedHashSet<>();
List<String> out = new ArrayList<>();
for (String s : raw) {
if (s == null || s.isBlank()) {
continue;
}
String u = s.trim().toUpperCase();
try {
ProductRiskCountryCode.valueOf(u);
} catch (IllegalArgumentException ex) {
continue;
}
if (seen.add(u)) {
out.add(u);
}
}
return out;
}
private ProductRiskShopQueueItemVo matchOneShop(String shopName) {
ProductRiskShopQueueItemVo item = new ProductRiskShopQueueItemVo();
item.setShopName(shopName);
try {
shopManageService.requireShopByName(shopName);
ZiniaoShopMatchResultVo m = ziniaoShopSwitchService.findIndexedStoreByName(shopName, false);
if (m == null) {
item.setMatched(false);
item.setMatchStatus("PENDING");
item.setMatchMessage("匹配结果为空");
return item;
}
item.setMatched(m.isMatched());
item.setShopId(m.getShopId());
item.setPlatform(m.getPlatform());
item.setCompanyName(m.getCompanyName());
item.setMatchedUserId(m.getMatchedUserId());
item.setMatchStatus(m.getMatchStatus());
item.setMatchMessage(m.getMatchMessage());
} catch (BusinessException ex) {
item.setMatched(false);
item.setMatchStatus("PENDING");
item.setMatchMessage(ex.getMessage());
} catch (Exception ex) {
item.setMatched(false);
item.setMatchStatus("PENDING");
item.setMatchMessage(Objects.toString(ex.getMessage(), "匹配异常"));
}
return item;
}
}
@@ -0,0 +1,226 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class ShopDataCrawlTaskCacheService {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public ShopDataCrawlShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopDataCrawlShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, ShopDataCrawlShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, ShopDataCrawlShopPayloadDto> getAllShopMergedPayload(Long taskId) {
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ShopDataCrawlShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public long countShopMergedPayload(Long taskId) {
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public void touchTaskHeartbeat(Long taskId) {
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
if (task == null || task.getId() == null) {
return;
}
long now = System.currentTimeMillis();
taskEntityLocalCache.put(task.getId(), new LocalTaskEntityCacheEntry(
now,
objectMapper.convertValue(task, FileTaskEntity.class)
));
try {
stringRedisTemplate.opsForValue().set(
buildTaskEntityKey(task.getId()),
objectMapper.writeValueAsString(task),
Duration.ofHours(PAYLOAD_TTL_HOURS)
);
} catch (Exception ignored) {
}
}
public Map<Long, FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
Map<Long, FileTaskEntity> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
java.util.List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
long now = System.currentTimeMillis();
java.util.List<Long> missingIds = new ArrayList<>();
for (Long taskId : normalized) {
LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId);
if (isLocalCacheFresh(cached, now)) {
result.put(taskId, objectMapper.convertValue(cached.task(), FileTaskEntity.class));
} else {
missingIds.add(taskId);
}
}
if (missingIds.isEmpty()) {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[shop-data-crawl-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
if (val == null || val.isBlank()) {
continue;
}
try {
FileTaskEntity task = objectMapper.readValue(val, FileTaskEntity.class);
result.put(taskId, task);
taskEntityLocalCache.put(taskId, new LocalTaskEntityCacheEntry(now, task));
} catch (Exception ignored) {
}
}
return result;
}
private String buildTaskHeartbeatKey(Long taskId) {
return "shop-data-crawl:task:heartbeat:" + taskId;
}
private String buildTaskEntityKey(Long taskId) {
return "shop-data-crawl:task:entity:" + taskId;
}
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
return cached != null
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());
}
private record LocalTaskEntityCacheEntry(long cachedAtMillis, FileTaskEntity task) {}
}
@@ -14,7 +14,7 @@ public class SimilarAsinHistoryItemVo {
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/similar_asin/xxx/17-result.xlsx")
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "https://oss.aishufu.top/nanri-ai-images/result/similar_asin/xxx/17-result.xlsx")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
@@ -4027,6 +4027,10 @@ public class SimilarAsinTaskService {
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
TaskProgressSnapshotEntity existing = taskProgressSnapshotService.find(task.getId(), MODULE_TYPE);
int displayPercent = calculateDisplayProgressPercent(safeCompleted, safeTotal, job, LocalDateTime.now());
boolean terminalProgress = isTerminalFileBuildProgress(task.getStatus(), safeCompleted, safeTotal);
if (terminalProgress) {
displayPercent = 100;
}
int previousDisplayPercent = calculateSnapshotDisplayPercent(existing, job);
if (previousDisplayPercent > displayPercent && safeCompleted < safeTotal) {
displayPercent = previousDisplayPercent;
@@ -4034,7 +4038,7 @@ public class SimilarAsinTaskService {
taskProgressSnapshotService.save(
task.getId(),
MODULE_TYPE,
STATUS_RUNNING,
terminalProgress ? task.getStatus() : STATUS_RUNNING,
safeTotal,
safeCompleted,
0,
@@ -4044,6 +4048,12 @@ public class SimilarAsinTaskService {
);
}
static boolean isTerminalFileBuildProgress(String taskStatus, int completed, int total) {
return total > 0
&& completed >= total
&& (STATUS_SUCCESS.equals(taskStatus) || STATUS_FAILED.equals(taskStatus));
}
private int calculateDisplayProgressPercent(int current,
int total,
TaskFileJobEntity job,
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Dns;
@@ -39,6 +40,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
@@ -102,12 +104,14 @@ public class SimilarAsinImageEmbedder {
private final int downloadPoolSize;
private final OkHttpClient httpClient;
private final ExecutorService downloadPool;
private final OssStorageService ossStorageService;
public SimilarAsinImageEmbedder(SimilarAsinProperties properties) {
public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) {
int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds();
int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize();
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null");
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
@@ -178,8 +182,9 @@ public class SimilarAsinImageEmbedder {
if (taskImageCache.containsKey(url)) {
return;
}
byte[] raw = doFetch(url);
ResizedImage thumb = resizeImage(url, raw);
String downloadUrl = normalizeAndValidateDownloadUrl(url);
byte[] raw = doFetch(downloadUrl);
ResizedImage thumb = resizeImage(downloadUrl, raw);
taskImageCache.putIfAbsent(url, thumb);
} catch (Exception ex) {
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
@@ -345,8 +350,8 @@ public class SimilarAsinImageEmbedder {
}
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
validateHttpsUrl(url);
List<String> candidates = downloadCandidates(url);
String downloadUrl = normalizeAndValidateDownloadUrl(url);
List<String> candidates = downloadCandidates(downloadUrl);
IOException last = null;
TimeoutException lastTimeout = null;
long waitSeconds = downloadTimeoutSeconds * 2L;
@@ -355,9 +360,9 @@ public class SimilarAsinImageEmbedder {
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
try {
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
if (!attemptUrl.equals(url)) {
if (!attemptUrl.equals(downloadUrl)) {
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
url, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
downloadUrl, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
}
return bytes;
} catch (java.util.concurrent.ExecutionException ee) {
@@ -396,6 +401,12 @@ public class SimilarAsinImageEmbedder {
throw last != null ? last : new IOException("image download failed without cause");
}
String normalizeAndValidateDownloadUrl(String url) {
String downloadUrl = ossStorageService.normalizeManagedPublicUrl(url);
validateHttpsUrl(downloadUrl);
return downloadUrl;
}
static List<String> downloadCandidates(String url) {
List<String> single = List.of(url);
URI uri;
@@ -95,7 +95,7 @@ public class TaskFileJobService {
List<TaskFileJobEntity> ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH"))
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.like(TaskFileJobEntity::getScopeKey, ownerMarker)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + safeLimit));
@@ -108,7 +108,7 @@ public class TaskFileJobService {
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.and(wrapper -> wrapper
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH"))
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.or()
.isNull(TaskFileJobEntity::getScopeKey)
.or()
@@ -265,6 +265,11 @@ public class TaskFileJobService {
return taskFileJobMapper.selectById(jobId);
}
public boolean isRetryExhausted(Long jobId) {
TaskFileJobEntity job = findById(jobId);
return job != null && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT;
}
public void markFailed(TaskFileJobEntity job, String message) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
@@ -12,6 +12,8 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
@@ -41,6 +43,7 @@ public class TaskHeartbeatService {
private static final String MODULE_SHOP_MATCH = "SHOP_MATCH";
private static final String MODULE_PATROL_DELETE = "PATROL_DELETE";
private static final String MODULE_QUERY_ASIN = "QUERY_ASIN";
private static final String MODULE_SHOP_DATA_CRAWL = "SHOP_DATA_CRAWL";
private static final String MODULE_WITHDRAW = "WITHDRAW";
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
@@ -54,6 +57,8 @@ public class TaskHeartbeatService {
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
private final QueryAsinTaskCacheService queryAsinTaskCacheService;
private final ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService;
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
private final WithdrawTaskCacheService withdrawTaskCacheService;
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
@@ -106,6 +111,9 @@ public class TaskHeartbeatService {
if (PublishTaskService.MODULE_TYPE.equals(moduleType)) {
publishTaskService.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat");
}
if (MODULE_SHOP_DATA_CRAWL.equals(moduleType)) {
shopDataCrawlTaskService.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat");
}
String status = task.getStatus();
if (!STATUS_RUNNING.equals(status)) {
log.warn("[task-heartbeat] file task is not running taskId={} actualUserId={} moduleType={} status={}",
@@ -179,6 +187,7 @@ public class TaskHeartbeatService {
case MODULE_QUERY_ASIN -> {
queryAsinTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.touchTaskHeartbeat(taskId);
case MODULE_WITHDRAW -> {
withdrawTaskCacheService.touchTaskHeartbeat(taskId);
}
@@ -216,6 +225,7 @@ public class TaskHeartbeatService {
case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task);
case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task);
case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task);
case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.saveTaskCache(task);
case MODULE_WITHDRAW -> withdrawTaskCacheService.saveTaskCache(task);
case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task);
default -> {
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
@@ -44,6 +45,7 @@ public class TaskResultFileJobWorker {
private final ProductRiskTaskService productRiskTaskService;
private final PublishTaskService publishTaskService;
private final QueryAsinTaskService queryAsinTaskService;
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
private final WithdrawTaskService withdrawTaskService;
private final PatrolDeleteTaskService patrolDeleteTaskService;
private final AppearancePatentTaskService appearancePatentTaskService;
@@ -192,6 +194,9 @@ public class TaskResultFileJobWorker {
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
taskFileJobService.markFailed(job, message);
if ("SHOP_DATA_CRAWL".equals(job.getModuleType()) && taskFileJobService.isRetryExhausted(job.getId())) {
shopDataCrawlTaskService.handleResultFileJobFailure(job, message);
}
}
}
@@ -225,7 +230,8 @@ public class TaskResultFileJobWorker {
}
return "APPEARANCE_PATENT".equals(job.getModuleType())
|| "SIMILAR_ASIN".equals(job.getModuleType())
|| "PUBLISH".equals(job.getModuleType());
|| "PUBLISH".equals(job.getModuleType())
|| "SHOP_DATA_CRAWL".equals(job.getModuleType());
}
private boolean isOwnedByCurrentInstance(TaskFileJobEntity job) {
@@ -273,6 +279,10 @@ public class TaskResultFileJobWorker {
queryAsinTaskService.processResultFileJob(job);
return true;
}
if ("SHOP_DATA_CRAWL".equals(moduleType)) {
shopDataCrawlTaskService.processResultFileJob(job);
return true;
}
if ("WITHDRAW".equals(moduleType)) {
withdrawTaskService.processResultFileJob(job);
return true;
@@ -304,6 +314,11 @@ public class TaskResultFileJobWorker {
private void cleanupAfterSuccess(TaskFileJobEntity job) {
String moduleType = job.getModuleType();
if ("SHOP_DATA_CRAWL".equals(moduleType)) {
taskResultPayloadService.deleteLatest(job.getTaskId(), moduleType, job.getScopeKey());
shopDataCrawlTaskService.cleanupResultFileJob(job);
return;
}
if ("SHOP_MATCH".equals(moduleType)
|| "PRICE_TRACK".equals(moduleType)
|| "PRODUCT_RISK_RESOLVE".equals(moduleType)
@@ -321,6 +336,10 @@ public class TaskResultFileJobWorker {
similarAsinTaskService.cleanupResultFileJob(job);
return;
}
if ("PUBLISH".equals(moduleType)) {
publishTaskService.cleanupResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job);
return;
@@ -7,14 +7,15 @@
SPRING_PROFILES_ACTIVE=local
AIIMAGE_INSTANCE_ID=local-121
AIIMAGE_SERVER_PORT=18080
AIIMAGE_INTERNAL_TOKEN=change-me-local-internal-token
AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
AIIMAGE_DB_USERNAME=change-me
AIIMAGE_DB_PASSWORD=change-me
AIIMAGE_OSS_REGION=us-east-1
AIIMAGE_OSS_ENDPOINT=http://47.110.241.161:9000
AIIMAGE_OSS_PUBLIC_ENDPOINT=http://47.110.241.161:9000
AIIMAGE_OSS_ENDPOINT=https://oss.aishufu.top
AIIMAGE_OSS_PUBLIC_ENDPOINT=https://oss.aishufu.top
AIIMAGE_OSS_BUCKET=nanri-ai-images
AIIMAGE_IMAGE_VIDEO_OSS_BUCKET=shufu-video
AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET=nanri-ai-digital-human
@@ -85,8 +85,8 @@ aiimage:
request-body-cache-limit-bytes: ${AIIMAGE_INSTANCE_ROUTE_REQUEST_BODY_CACHE_LIMIT_BYTES:104857600}
oss:
region: ${AIIMAGE_OSS_REGION:us-east-1}
endpoint: ${AIIMAGE_OSS_ENDPOINT:http://47.110.241.161:9000}
public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:http://47.110.241.161:9000}
endpoint: ${AIIMAGE_OSS_ENDPOINT:https://oss.aishufu.top}
public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:https://oss.aishufu.top}
bucket: ${AIIMAGE_OSS_BUCKET:nanri-ai-images}
image-video-bucket: ${AIIMAGE_IMAGE_VIDEO_OSS_BUCKET:shufu-video}
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
@@ -153,7 +153,7 @@ aiimage:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,SHOP_DATA_CRAWL,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
permission-schema-init:
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
task-pressure:
@@ -247,6 +247,7 @@ aiimage:
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
internal-token-file: ${AIIMAGE_INTERNAL_TOKEN_FILE:}
auth:
jwt-secret: ${AIIMAGE_JWT_SECRET:please-change-this-secret-please-rotate-at-least-32-bytes}
jwt-ttl-hours: ${AIIMAGE_AUTH_JWT_TTL_HOURS:168}
@@ -0,0 +1,29 @@
CREATE TABLE IF NOT EXISTS `biz_shop_data_crawl_shop_candidate` (
`id` BIGINT NOT NULL AUTO_INCREMENT,
`user_id` BIGINT NOT NULL,
`shop_name` VARCHAR(255) NOT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uk_shop_data_crawl_candidate_user_shop` (`user_id`, `shop_name`),
KEY `idx_shop_data_crawl_candidate_user` (`user_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `biz_shop_data_crawl_country_pref` (
`user_id` BIGINT NOT NULL,
`country_codes_json` VARCHAR(256) NOT NULL,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`user_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '店铺数据抓取', 'shop_data_crawl', 'app', 'shop-data-crawl', 139
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'shop_data_crawl'
);
UPDATE `columns`
SET `name` = '店铺数据抓取',
`menu_type` = 'app',
`route_path` = 'shop-data-crawl',
`sort_order` = 139
WHERE `column_key` = 'shop_data_crawl';
@@ -0,0 +1,220 @@
-- Hierarchical menu permissions. user_column_permission intentionally keeps
-- only direct grants; descendants are expanded by PermissionMenuService.
SET @permission_db_name = DATABASE();
SET @permission_parent_column_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @permission_db_name
AND TABLE_NAME = 'columns'
AND COLUMN_NAME = 'parent_id'
);
SET @permission_sql = IF(
@permission_parent_column_exists = 0,
'ALTER TABLE `columns` ADD COLUMN `parent_id` INT NULL COMMENT ''直接父菜单 ID'' AFTER `id`',
'SELECT 1'
);
PREPARE permission_stmt FROM @permission_sql;
EXECUTE permission_stmt;
DEALLOCATE PREPARE permission_stmt;
SET @permission_parent_index_exists = (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @permission_db_name
AND TABLE_NAME = 'columns'
AND INDEX_NAME = 'idx_columns_parent_sort_id'
);
SET @permission_sql = IF(
@permission_parent_index_exists = 0,
'ALTER TABLE `columns` ADD INDEX `idx_columns_parent_sort_id` (`parent_id`, `sort_order`, `id`)',
'SELECT 1'
);
PREPARE permission_stmt FROM @permission_sql;
EXECUTE permission_stmt;
DEALLOCATE PREPARE permission_stmt;
SET @permission_parent_fk_exists = (
SELECT COUNT(*)
FROM information_schema.TABLE_CONSTRAINTS
WHERE CONSTRAINT_SCHEMA = @permission_db_name
AND TABLE_NAME = 'columns'
AND CONSTRAINT_NAME = 'fk_columns_parent_id'
AND CONSTRAINT_TYPE = 'FOREIGN KEY'
);
SET @permission_sql = IF(
@permission_parent_fk_exists = 0,
'ALTER TABLE `columns` ADD CONSTRAINT `fk_columns_parent_id` FOREIGN KEY (`parent_id`) REFERENCES `columns` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT',
'SELECT 1'
);
PREPARE permission_stmt FROM @permission_sql;
EXECUTE permission_stmt;
DEALLOCATE PREPARE permission_stmt;
-- Normalize older underscore keys. If both spellings already exist, retain the
-- canonical row and move direct grants before deleting the duplicate.
INSERT IGNORE INTO `user_column_permission` (`user_id`, `column_id`)
SELECT direct_grant.`user_id`, canonical.`id`
FROM `user_column_permission` AS direct_grant
JOIN `columns` AS legacy ON legacy.`id` = direct_grant.`column_id`
JOIN (
SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key
UNION ALL SELECT 'appearance_patent', 'appearance-patent'
UNION ALL SELECT 'similar_asin', 'similar-asin'
) AS key_map ON key_map.legacy_key = legacy.`column_key`
JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key;
DELETE legacy
FROM `columns` AS legacy
JOIN (
SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key
UNION ALL SELECT 'appearance_patent', 'appearance-patent'
UNION ALL SELECT 'similar_asin', 'similar-asin'
) AS key_map ON key_map.legacy_key = legacy.`column_key`
JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key;
UPDATE `columns` AS legacy
JOIN (
SELECT 'collect_data' AS legacy_key, 'collect-data' AS canonical_key
UNION ALL SELECT 'appearance_patent', 'appearance-patent'
UNION ALL SELECT 'similar_asin', 'similar-asin'
) AS key_map ON key_map.legacy_key = legacy.`column_key`
LEFT JOIN `columns` AS canonical ON canonical.`column_key` = key_map.canonical_key
SET legacy.`column_key` = key_map.canonical_key
WHERE canonical.`id` IS NULL;
-- Video/image workspace. Keep the legacy `wb` key because existing user
-- grants reference that row; its route is the current image-video page.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '视频', 'wb', 'app', 'image-video', 100, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'wb');
UPDATE `columns`
SET `name` = '视频', `menu_type` = 'app', `route_path` = 'image-video', `sort_order` = 100, `parent_id` = NULL
WHERE `column_key` = 'wb';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '数字人', 'digital-human', 'app', 'digital-human', 101,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'digital-human');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '带货视频', 'delivery-video', 'app', 'delivery-video', 102,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'delivery-video');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '混剪', 'mix-video', 'app', 'mix-video', 103,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'mix-video');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '图片', 'image', 'app', 'image', 104,
(SELECT id FROM `columns` WHERE column_key = 'wb' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'image');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'wb'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'digital-human' THEN '数字人' WHEN 'delivery-video' THEN '带货视频'
WHEN 'mix-video' THEN '混剪' WHEN 'image' THEN '图片' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'digital-human' THEN 'digital-human' WHEN 'delivery-video' THEN 'delivery-video'
WHEN 'mix-video' THEN 'mix-video' WHEN 'image' THEN 'image' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'digital-human' THEN 101 WHEN 'delivery-video' THEN 102
WHEN 'mix-video' THEN 103 WHEN 'image' THEN 104 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('digital-human', 'delivery-video', 'mix-video', 'image');
-- Three APP navigation groups.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '前端工具', 'brand_front_tools', 'app', 'brand-front-tools', 110, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_front_tools');
UPDATE `columns` SET `name` = '前端工具', `menu_type` = 'app', `route_path` = 'brand-front-tools', `sort_order` = 110, `parent_id` = NULL
WHERE `column_key` = 'brand_front_tools';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '运营工具', 'brand_operation_tools', 'app', 'brand-operation-tools', 120, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_operation_tools');
UPDATE `columns` SET `name` = '运营工具', `menu_type` = 'app', `route_path` = 'brand-operation-tools', `sort_order` = 120, `parent_id` = NULL
WHERE `column_key` = 'brand_operation_tools';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '后勤工具', 'brand_logistics_tools', 'app', 'brand-logistics-tools', 130, NULL
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE `column_key` = 'brand_logistics_tools');
UPDATE `columns` SET `name` = '后勤工具', `menu_type` = 'app', `route_path` = 'brand-logistics-tools', `sort_order` = 130, `parent_id` = NULL
WHERE `column_key` = 'brand_logistics_tools';
-- APP children. The key follows the frontend item key (except the existing
-- shop_data_crawl compatibility key); route_path remains the route slug.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT '采集数据', 'collect-data', 'app', 'collect-data', 111,
(SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1)
WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'collect-data');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_front_tools'
SET child.`name` = '采集数据', child.`menu_type` = 'app', child.`route_path` = 'collect-data',
child.`sort_order` = 111, child.`parent_id` = parent.`id`
WHERE child.`column_key` = 'collect-data';
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '品牌检测', 'brand', 'app', 'brand', 112, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'brand');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '外观专利检测', 'appearance-patent', 'app', 'appearance-patent', 113, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'appearance-patent');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '货源查询', 'similar-asin', 'app', 'similar-asin', 114, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'similar-asin');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '数据去重', 'dedupe', 'app', 'dedupe', 115, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'dedupe');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '数据拆分', 'split', 'app', 'split', 116, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'split');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '格式转换', 'convert', 'app', 'convert', 117, (SELECT id FROM `columns` WHERE column_key = 'brand_front_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'convert');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_front_tools'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'brand' THEN '品牌检测' WHEN 'appearance-patent' THEN '外观专利检测'
WHEN 'similar-asin' THEN '货源查询' WHEN 'dedupe' THEN '数据去重'
WHEN 'split' THEN '数据拆分' WHEN 'convert' THEN '格式转换' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'brand' THEN 'brand' WHEN 'appearance-patent' THEN 'appearance-patent'
WHEN 'similar-asin' THEN 'similar-asin' WHEN 'dedupe' THEN 'dedupe'
WHEN 'split' THEN 'split' WHEN 'convert' THEN 'convert' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'brand' THEN 112 WHEN 'appearance-patent' THEN 113 WHEN 'similar-asin' THEN 114
WHEN 'dedupe' THEN 115 WHEN 'split' THEN 116 WHEN 'convert' THEN 117 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('brand', 'appearance-patent', 'similar-asin', 'dedupe', 'split', 'convert');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '上架', 'publish', 'app', 'publish', 121, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'publish');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '删除ASIN', 'delete-brand', 'app', 'delete-brand', 122, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'delete-brand');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '商品风险解决', 'product-risk', 'app', 'product-risk', 123, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'product-risk');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '定时匹配', 'shop-match', 'app', 'shop-match', 124, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'shop-match');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '跟价', 'pricing', 'app', 'price-track', 125, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'pricing');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '巡店删除', 'patrol-delete', 'app', 'patrol-delete', 126, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'patrol-delete');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '查询ASIN', 'query-asin', 'app', 'query-asin', 127, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'query-asin');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '店铺数据抓取', 'shop_data_crawl', 'app', 'shop-data-crawl', 128, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'shop_data_crawl');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '取款', 'withdraw', 'app', 'withdraw', 129, (SELECT id FROM `columns` WHERE column_key = 'brand_operation_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'withdraw');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_operation_tools'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'publish' THEN '上架' WHEN 'delete-brand' THEN '删除ASIN'
WHEN 'product-risk' THEN '商品风险解决' WHEN 'shop-match' THEN '定时匹配'
WHEN 'pricing' THEN '跟价' WHEN 'patrol-delete' THEN '巡店删除'
WHEN 'query-asin' THEN '查询ASIN' WHEN 'shop_data_crawl' THEN '店铺数据抓取'
WHEN 'withdraw' THEN '取款' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'publish' THEN 'publish' WHEN 'delete-brand' THEN 'delete-brand'
WHEN 'product-risk' THEN 'product-risk' WHEN 'shop-match' THEN 'shop-match'
WHEN 'pricing' THEN 'price-track' WHEN 'patrol-delete' THEN 'patrol-delete'
WHEN 'query-asin' THEN 'query-asin' WHEN 'shop_data_crawl' THEN 'shop-data-crawl'
WHEN 'withdraw' THEN 'withdraw' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'publish' THEN 121 WHEN 'delete-brand' THEN 122 WHEN 'product-risk' THEN 123 WHEN 'shop-match' THEN 124
WHEN 'pricing' THEN 125 WHEN 'patrol-delete' THEN 126 WHEN 'query-asin' THEN 127 WHEN 'shop_data_crawl' THEN 128 WHEN 'withdraw' THEN 129 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('publish', 'delete-brand', 'product-risk', 'shop-match', 'pricing', 'patrol-delete', 'query-asin', 'shop_data_crawl', 'withdraw');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT '采购', 'purchase', 'app', 'purchase', 131, (SELECT id FROM `columns` WHERE column_key = 'brand_logistics_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'purchase');
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`) SELECT 'ERP', 'erp', 'app', 'erp', 132, (SELECT id FROM `columns` WHERE column_key = 'brand_logistics_tools' LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM `columns` WHERE column_key = 'erp');
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_logistics_tools'
SET child.`parent_id` = parent.`id`, child.`menu_type` = 'app',
child.`name` = CASE child.`column_key`
WHEN 'purchase' THEN '采购' WHEN 'erp' THEN 'ERP' ELSE child.`name` END,
child.`route_path` = CASE child.`column_key`
WHEN 'purchase' THEN 'purchase' WHEN 'erp' THEN 'erp' ELSE child.`route_path` END,
child.`sort_order` = CASE child.`column_key`
WHEN 'purchase' THEN 131 WHEN 'erp' THEN 132 ELSE child.`sort_order` END
WHERE child.`column_key` IN ('purchase', 'erp');
@@ -0,0 +1,2 @@
ALTER TABLE biz_publish_item
MODIFY COLUMN status_value TEXT NULL COMMENT 'status column';
@@ -0,0 +1,99 @@
package com.nanri.aiimage.modules.admin.service;
import com.nanri.aiimage.modules.admin.model.dto.AdminUserCreateRequest;
import com.nanri.aiimage.modules.admin.model.dto.AdminUserUpdateRequest;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.auth.util.WerkzeugPasswordEncoder;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class AdminUserServiceTest {
@Test
void createUserStoresSuppliedDirectPermissionsInSameTransactionalMethod() throws Exception {
AdminUserMapper userMapper = mock(AdminUserMapper.class);
WerkzeugPasswordEncoder passwordEncoder = mock(WerkzeugPasswordEncoder.class);
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService permissionService = mock(PermissionMenuService.class);
AdminUserService service = new AdminUserService(
userMapper, passwordEncoder, authSupport, permissionService);
AdminUserEntity operator = user(10L, "admin", 1, null);
AdminUserCreateRequest request = new AdminUserCreateRequest();
request.setUsername("created-user");
request.setPassword("secret1");
request.setRole("normal");
request.setColumnIds(List.of(11L, 12L));
when(authSupport.currentRole(operator)).thenReturn("admin");
when(passwordEncoder.hash("secret1")).thenReturn("hashed");
when(userMapper.insert(any(AdminUserEntity.class))).thenAnswer(invocation -> {
invocation.<AdminUserEntity>getArgument(0).setId(20L);
return 1;
});
Long createdId = service.createUser(operator, request);
assertThat(createdId).isEqualTo(20L);
ArgumentCaptor<UserColumnPermissionUpdateRequest> permissions =
ArgumentCaptor.forClass(UserColumnPermissionUpdateRequest.class);
verify(permissionService).updateUserColumnPermissions(
eq(operator), eq(20L), permissions.capture(), eq(PermissionMenuService.MENU_TYPE_APP));
assertThat(permissions.getValue().getColumnIds()).containsExactly(11L, 12L);
assertThat(AdminUserService.class
.getMethod("createUser", AdminUserEntity.class, AdminUserCreateRequest.class)
.isAnnotationPresent(Transactional.class)).isTrue();
}
@Test
void updateUserCanReplaceOnlyDirectPermissionsTransactionally() throws Exception {
AdminUserMapper userMapper = mock(AdminUserMapper.class);
WerkzeugPasswordEncoder passwordEncoder = mock(WerkzeugPasswordEncoder.class);
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService permissionService = mock(PermissionMenuService.class);
AdminUserService service = new AdminUserService(
userMapper, passwordEncoder, authSupport, permissionService);
AdminUserEntity operator = user(10L, "admin", 1, null);
AdminUserEntity target = user(20L, "normal", 0, 10L);
AdminUserUpdateRequest request = new AdminUserUpdateRequest();
request.setColumnIds(List.of(21L));
when(authSupport.currentRole(operator)).thenReturn("admin");
when(userMapper.selectById(20L)).thenReturn(target);
service.updateUser(operator, 20L, request);
verify(userMapper, never()).update(any(), any());
ArgumentCaptor<UserColumnPermissionUpdateRequest> permissions =
ArgumentCaptor.forClass(UserColumnPermissionUpdateRequest.class);
verify(permissionService).updateUserColumnPermissions(
eq(operator), eq(20L), permissions.capture(), eq(PermissionMenuService.MENU_TYPE_APP));
assertThat(permissions.getValue().getColumnIds()).containsExactly(21L);
assertThat(AdminUserService.class
.getMethod("updateUser", AdminUserEntity.class, Long.class, AdminUserUpdateRequest.class)
.isAnnotationPresent(Transactional.class)).isTrue();
}
private AdminUserEntity user(Long id, String role, int isAdmin, Long createdById) {
AdminUserEntity user = new AdminUserEntity();
user.setId(id);
user.setRole(role);
user.setIsAdmin(isAdmin);
user.setCreatedById(createdById);
return user;
}
}
@@ -0,0 +1,52 @@
package com.nanri.aiimage.modules.admin.support;
import com.nanri.aiimage.modules.auth.config.AuthProperties;
import com.nanri.aiimage.modules.auth.service.JwtService;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
class AdminAuthSupportTest {
@Test
void explicitLowestIdAdminIsNotPromotedToSuperAdmin() {
AdminUserMapper userMapper = mock(AdminUserMapper.class);
AdminAuthSupport support = new AdminAuthSupport(
mock(JwtService.class), userMapper, mock(AuthProperties.class));
AdminUserEntity user = user(1L, "admin", 1, null);
assertThat(support.currentRole(user)).isEqualTo("admin");
verify(userMapper, never()).selectOne(any());
}
@Test
void legacyBlankRoleRootRemainsSuperAdmin() {
AdminAuthSupport support = new AdminAuthSupport(
mock(JwtService.class), mock(AdminUserMapper.class), mock(AuthProperties.class));
assertThat(support.currentRole(user(1L, null, 1, null))).isEqualTo("super_admin");
}
@Test
void explicitNormalRoleIsNotPromotedByLegacyAdminFields() {
AdminAuthSupport support = new AdminAuthSupport(
mock(JwtService.class), mock(AdminUserMapper.class), mock(AuthProperties.class));
assertThat(support.currentRole(user(1L, "normal", 1, null))).isNull();
}
private AdminUserEntity user(Long id, String role, int isAdmin, Long createdById) {
AdminUserEntity user = new AdminUserEntity();
user.setId(id);
user.setRole(role);
user.setIsAdmin(isAdmin);
user.setCreatedById(createdById);
return user;
}
}
@@ -0,0 +1,91 @@
package com.nanri.aiimage.modules.file.service.oss;
import com.nanri.aiimage.config.OssProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
class OssStorageServiceTest {
private OssStorageService storageService;
@BeforeEach
void setUp() {
OssProperties properties = new OssProperties();
properties.setEndpoint("https://oss.aishufu.top");
properties.setPublicEndpoint("https://oss.aishufu.top");
properties.setBucket("nanri-ai-images");
properties.setImageVideoBucket("shufu-video");
properties.setDigitalHumanBucket("nanri-ai-digital-human");
properties.setAccessKeyId("test-access-key");
properties.setAccessKeySecret("test-secret-key");
storageService = new OssStorageService(properties);
}
@Test
void normalizesLegacyPathStyleUrlToCurrentPublicEndpoint() {
assertEquals(
"https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg",
storageService.normalizeManagedPublicUrl(
"http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg"));
}
@Test
void keepsCurrentUrlAndNormalizesVirtualHostUrl() {
String current = "https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg";
assertEquals(current, storageService.normalizeManagedPublicUrl(current));
assertEquals(
current,
storageService.normalizeManagedPublicUrl(
"https://nanri-ai-images.oss.aishufu.top/supply_images/main.jpg"));
}
@Test
void supportsAllConfiguredBuckets() {
assertEquals(
"https://oss.aishufu.top/shufu-video/result/image_video/demo.mp4",
storageService.normalizeManagedPublicUrl(
"http://47.110.241.161:9000/shufu-video/result/image_video/demo.mp4"));
assertEquals(
"https://oss.aishufu.top/nanri-ai-digital-human/digital-human/versions/demo.mp4",
storageService.normalizeManagedPublicUrl(
"https://nanri-ai-digital-human.oss.aishufu.top/digital-human/versions/demo.mp4"));
}
@Test
void leavesExternalImageUrlsUnchanged() {
List<String> externalUrls = List.of(
"https://m.media-amazon.com/nanri-ai-images/main.jpg?size=large",
"https://cbu01.alicdn.com/img/ibank/main.jpg",
"https://lf9-bot-platform-tos-sign.coze.cn/file.jpg?x-expires=1&x-signature=abc",
"https://nanri-ai-images.evil.example/supply_images/main.jpg");
for (String url : externalUrls) {
assertEquals(url, storageService.normalizeManagedPublicUrl(url));
}
}
@Test
void leavesInvalidOrUnmanagedValuesUnchanged() {
List<String> values = List.of(
"http://47.110.241.161:9001/nanri-ai-images/supply_images/main.jpg",
"http://47.110.241.161.evil.example:9000/nanri-ai-images/supply_images/main.jpg",
"http://47.110.241.161:9000/nanri-ai-images/../other-bucket/main.jpg",
"http://47.110.241.161:9000/nanri-ai-images/%2e%2e/other-bucket/main.jpg",
"https://oss.aishufu.top/not-a-bucket/main.jpg",
"https://oss.aishufu.top/nanri-ai-images",
"supply_images/main.jpg",
"/nanri-ai-images/supply_images/main.jpg",
"not a url");
for (String value : values) {
assertEquals(value, storageService.normalizeManagedPublicUrl(value));
}
assertNull(storageService.normalizeManagedPublicUrl(null));
assertEquals(" ", storageService.normalizeManagedPublicUrl(" "));
}
}
@@ -142,6 +142,7 @@ class ImageVideoAsyncTaskServiceTest {
assertEquals("FAILED", task.getStatus());
assertEquals("FAIL", task.getCozeStatus());
assertEquals("model unavailable", task.getErrorMessage());
assertTrue(task.getResultJson().contains("model unavailable"));
verify(archiveService).capturePollResponse(task, cozeResult);
verify(taskMapper).updateById(task);
@@ -294,7 +295,7 @@ class ImageVideoAsyncTaskServiceTest {
task.setSubmittedAt(LocalDateTime.now().minusHours(2));
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"Output\":\"https://example.com/result.mp4\"}")));
"output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"video_url\\\":\\\"https://example.com/result.mp4\\\"}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1);
@@ -309,6 +310,145 @@ class ImageVideoAsyncTaskServiceTest {
assertNotNull(task.getCompletedAt());
}
@Test
void successfulWorkflowWithEmptyOutputVideoUrlIsMarkedFailedEvenWhenInputVideoExists() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoArchiveService archiveService = mock(ImageVideoArchiveService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
archiveService,
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingWorkflowTask();
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"video_info", Map.of("video_url", "https://example.com/input-reference.mp4"),
"output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"video_url\\\":\\\"\\\"}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1);
when(taskMapper.selectById(98L)).thenReturn(task);
when(workflowConfigService.imageVideoWorkflowId()).thenReturn("workflow-1");
when(cozeService.getWorkflowResult(1L, "workflow-1", "exec-98")).thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("FAILED", task.getStatus());
assertEquals("SUCCESS", task.getCozeStatus());
assertEquals("Coze workflow succeeded but video_url is empty", task.getErrorMessage());
assertNotNull(task.getCompletedAt());
assertTrue(task.getResultJson().contains("video_url"));
verify(archiveService, never()).enrichCompletedTask(any(), any());
verify(taskMapper).updateById(task);
}
@Test
void successfulVoiceListWithNestedBusinessErrorIsMarkedFailed() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoArchiveService archiveService = mock(ImageVideoArchiveService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
archiveService,
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingVoiceListTask();
String error = "Query voices failed: invalid API key (status: 1004)";
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"node_status\":\"{}\",\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":null,\\\"error\\\":\\\"" + error + "\\\",\\\"system_voice\\\":[],\\\"voice_cloning\\\":[]}}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1);
when(taskMapper.selectById(163L)).thenReturn(task);
when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow");
when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec"))
.thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("FAILED", task.getStatus());
assertEquals("SUCCESS", task.getCozeStatus());
assertEquals(error, task.getErrorMessage());
assertTrue(task.getResultJson().contains("status: 1004"));
verify(archiveService, never()).enrichCompletedTask(any(), any());
verify(taskMapper).updateById(task);
}
@Test
void successfulVoiceListWithBlankBusinessErrorRemainsSuccessful() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
mock(ImageVideoArchiveService.class),
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingVoiceListTask();
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":{\\\"status_code\\\":0,\\\"status_msg\\\":\\\"success\\\"},\\\"error\\\":\\\"\\\",\\\"system_voice\\\":[]}}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1);
when(taskMapper.selectById(163L)).thenReturn(task);
when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow");
when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec"))
.thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("SUCCESS", task.getStatus());
assertEquals("SUCCESS", task.getCozeStatus());
assertEquals(null, task.getErrorMessage());
verify(taskMapper).updateById(task);
}
@Test
void successfulVoiceListWithNonzeroBusinessStatusIsMarkedFailed() {
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
taskMapper,
cozeService,
workflowConfigService,
mock(ImageVideoArchiveService.class),
new ObjectMapper(),
Runnable::run,
instanceMetadata("server-121"));
ImageVideoAsyncTaskEntity task = waitingVoiceListTask();
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
"execute_status", "Success",
"output", "{\"Output\":\"{\\\"data\\\":{\\\"base_resp\\\":{\\\"status_code\\\":1004,\\\"status_msg\\\":\\\"invalid API key\\\"},\\\"system_voice\\\":[]}}\"}")));
when(taskMapper.selectList(any())).thenReturn(List.of(task));
when(taskMapper.claimWaiting(163L, "server-121")).thenReturn(1);
when(taskMapper.selectById(163L)).thenReturn(task);
when(workflowConfigService.voiceListWorkflowId()).thenReturn("voice-list-workflow");
when(cozeService.getWorkflowResult(665L, "voice-list-workflow", "voice-list-exec"))
.thenReturn(cozeResult);
service.pollWaitingTasks();
assertEquals("FAILED", task.getStatus());
assertEquals("invalid API key (status_code: 1004)", task.getErrorMessage());
verify(taskMapper).updateById(task);
}
private ImageVideoAsyncTaskEntity waitingDouyinTask() {
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
task.setId(79L);
@@ -331,6 +471,15 @@ class ImageVideoAsyncTaskServiceTest {
return task;
}
private ImageVideoAsyncTaskEntity waitingVoiceListTask() {
ImageVideoAsyncTaskEntity task = waitingDouyinTask();
task.setId(163L);
task.setUserId(665L);
task.setTaskType("VOICE_LIST");
task.setCozeExecuteId("voice-list-exec");
return task;
}
private InstanceMetadata instanceMetadata(String instanceId) {
return new InstanceMetadata(instanceId);
}
@@ -0,0 +1,251 @@
package com.nanri.aiimage.modules.permission.controller;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.dto.ImageVideoDataPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.test.util.ReflectionTestUtils;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class PermissionMenuControllerTest {
@Test
void effectivePermissionEndpointAllowsAuthenticatedNormalUserToReadSelf() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
AdminUserEntity currentUser = new AdminUserEntity();
currentUser.setId(9L);
currentUser.setRole("normal");
PermissionMenuItemVo item = new PermissionMenuItemVo();
item.setId(1L);
when(authSupport.requireUser(request)).thenReturn(currentUser);
when(service.getUserColumnPermissions(currentUser, 9L, "app")).thenReturn(List.of(item));
var response = controller.getUserColumnPermissions(request, 9L, "app");
assertThat(response.getData()).extracting(PermissionMenuItemVo::getId).containsExactly(1L);
verify(authSupport).requireUser(request);
verify(authSupport, never()).requireAdmin(request);
}
@Test
void internalProxyAllowsDatabaseBackedNormalUserToReadOwnEffectivePermissions() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
AdminUserEntity currentUser = new AdminUserEntity();
currentUser.setId(9L);
currentUser.setRole("normal");
when(authSupport.requireUser(request)).thenThrow(new BusinessException(401, "未登录"));
when(request.getHeader("X-Internal-Token")).thenReturn("shared-token");
when(request.getParameter("operatorId")).thenReturn("9");
when(service.requireUserOperator(9L)).thenReturn(currentUser);
when(service.getUserColumnPermissions(currentUser, 9L, "app")).thenReturn(List.of());
controller.getUserColumnPermissions(request, 9L, "app");
verify(service).requireUserOperator(9L);
verify(service, never()).requireAdminOperator(anyLong());
verify(service).getUserColumnPermissions(currentUser, 9L, "app");
}
@Test
void directIdEndpointStillRequiresAdminForSelf() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
BusinessException authFailure = new BusinessException(403, "需要管理员权限");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
assertThatThrownBy(() -> controller.getUserColumnIds(request, 9L, "app"))
.isSameAs(authFailure);
verify(authSupport, never()).requireUser(request);
verifyNoInteractions(service);
}
@Test
void directIdUpdateForwardsMenuTypeScope() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
AdminUserEntity operator = new AdminUserEntity();
operator.setId(1L);
operator.setRole("super_admin");
UserColumnPermissionUpdateRequest body = new UserColumnPermissionUpdateRequest();
body.setColumnIds(List.of(12L));
when(authSupport.requireAdmin(request)).thenReturn(operator);
controller.updateUserColumnIds(request, 9L, "app", body);
verify(service).updateUserColumnPermissions(eq(operator), eq(9L), eq(body), eq("app"));
}
@Test
void internalProxyUsesSharedTokenAndDatabaseBackedOperator() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
AdminUserEntity operator = new AdminUserEntity();
operator.setId(12L);
operator.setRole("admin");
when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "未登录"));
when(request.getHeader("X-Internal-Token")).thenReturn("shared-token");
when(request.getParameter("operatorId")).thenReturn("12");
when(service.requireAdminOperator(12L)).thenReturn(operator);
when(service.list("app")).thenReturn(List.of());
controller.listMenus(request, "app");
verify(service).requireAdminOperator(12L);
verify(service).list("app");
verify(request, never()).getParameter("superAdmin");
}
@Test
void internalProxyRejectsMismatchedSharedToken() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
BusinessException authFailure = new BusinessException(401, "未登录");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
when(request.getHeader("X-Internal-Token")).thenReturn("wrong-token");
assertThatThrownBy(() -> controller.listMenus(request, "app"))
.isSameAs(authFailure);
verify(service, never()).requireAdminOperator(anyLong());
verify(service, never()).list("app");
}
@Test
void internalProxyUsesSharedTokenFileWhenEnvironmentTokenIsUnconfigured(@TempDir Path tempDir) throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "");
Path tokenFile = tempDir.resolve("internal-token");
Files.writeString(tokenFile, "file-token\n");
ReflectionTestUtils.setField(controller, "internalTokenFile", tokenFile.toString());
AdminUserEntity operator = new AdminUserEntity();
operator.setId(12L);
operator.setRole("admin");
when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "未登录"));
when(request.getHeader("X-Internal-Token")).thenReturn("file-token");
when(request.getParameter("operatorId")).thenReturn("12");
when(service.requireAdminOperator(12L)).thenReturn(operator);
when(service.list("admin")).thenReturn(List.of());
controller.listMenus(request, "admin");
verify(service).requireAdminOperator(12L);
verify(service).list("admin");
}
@Test
void internalProxyRejectsMissingSharedTokenFileWhenEnvironmentTokenIsUnconfigured(@TempDir Path tempDir) {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
ReflectionTestUtils.setField(controller, "internalToken", "");
ReflectionTestUtils.setField(controller, "internalTokenFile", tempDir.resolve("missing-token").toString());
BusinessException authFailure = new BusinessException(401, "未登录");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
when(request.getHeader("X-Internal-Token")).thenReturn("guessed-token");
when(request.getRemoteAddr()).thenReturn("127.0.0.1");
when(request.getParameter("operatorId")).thenReturn("12");
assertThatThrownBy(() -> controller.listMenus(request, "admin"))
.isSameAs(authFailure);
verify(service, never()).requireAdminOperator(anyLong());
verify(service, never()).list("admin");
}
@Test
void configuredInternalTokenTakesPrecedenceOverTokenFile(@TempDir Path tempDir) throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
Path tokenFile = tempDir.resolve("internal-token");
Files.writeString(tokenFile, "file-token\n");
ReflectionTestUtils.setField(controller, "internalToken", "configured-token");
ReflectionTestUtils.setField(controller, "internalTokenFile", tokenFile.toString());
BusinessException authFailure = new BusinessException(401, "未登录");
when(authSupport.requireAdmin(request)).thenThrow(authFailure);
when(request.getHeader("X-Internal-Token")).thenReturn("file-token");
when(request.getParameter("operatorId")).thenReturn("12");
assertThatThrownBy(() -> controller.listMenus(request, "admin"))
.isSameAs(authFailure);
verify(service, never()).requireAdminOperator(anyLong());
verify(service, never()).list("admin");
}
@Test
void imageVideoPermissionEndpointsDelegateAuthenticatedOperator() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
AdminUserEntity operator = new AdminUserEntity();
operator.setId(1L);
operator.setRole("super_admin");
ImageVideoDataPermissionUserVo permissionUser = new ImageVideoDataPermissionUserVo();
permissionUser.setId(20L);
ImageVideoDataPermissionUpdateRequest body = new ImageVideoDataPermissionUpdateRequest();
body.setUserIds(List.of(20L));
when(authSupport.requireAdmin(request)).thenReturn(operator);
when(service.listImageVideoDataPermissionUsers(operator)).thenReturn(List.of(permissionUser));
when(service.updateImageVideoDataPermissionUsers(operator, List.of(20L))).thenReturn(1);
var listResponse = controller.listImageVideoDataPermissionUsers(request);
var updateResponse = controller.updateImageVideoDataPermissionUsers(request, body);
assertThat(listResponse.getData()).containsExactly(permissionUser);
assertThat(updateResponse.getData()).isEqualTo(1);
verify(authSupport, times(2)).requireAdmin(request);
verify(service).listImageVideoDataPermissionUsers(operator);
verify(service).updateImageVideoDataPermissionUsers(operator, List.of(20L));
}
}
@@ -1,20 +1,33 @@
package com.nanri.aiimage.modules.permission.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.mapper.PermissionMenuMapper;
import com.nanri.aiimage.modules.permission.mapper.UserColumnPermissionMapper;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import org.junit.jupiter.api.Test;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -61,10 +74,505 @@ class PermissionMenuServiceTest {
verify(permissionMapper, times(0)).insert(any(UserColumnPermissionEntity.class));
}
@Test
void expandsDirectParentGrantToDescendantsWithoutPersistingChildren() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11),
menu(3L, 2L, "app", 12),
menu(4L, null, "app", 13)));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(9L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId)
.containsExactly(1L, 2L, 3L);
verify(permissionMapper, never()).insert(any(UserColumnPermissionEntity.class));
}
@Test
void directIdEndpointDoesNotReturnExpandedChildren() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11)));
UserColumnIdsVo direct = service.getUserColumnIds(9L, "app");
assertThat(direct.getColumnIds()).containsExactly(1L);
}
@Test
void leafOnlyEffectivePermissionIncludesRootKeyWithoutGrantingAncestor() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 2L)));
PermissionMenuEntity root = menu(1L, null, "app", 1);
root.setColumnKey("brand_front_tools");
PermissionMenuEntity leaf = menu(2L, 1L, "app", 2);
leaf.setColumnKey("brand");
when(menuMapper.selectList(any())).thenReturn(List.of(root, leaf));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(9L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId).containsExactly(2L);
assertThat(effective.getFirst().getRootColumnKey()).isEqualTo("brand_front_tools");
}
@Test
void menuCatalogIncludesRootKeyFromLoadedHierarchy() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
PermissionMenuEntity root = menu(1L, null, "app", 1);
root.setColumnKey("brand_front_tools");
PermissionMenuEntity leaf = menu(2L, 1L, "app", 2);
leaf.setColumnKey("brand");
when(menuMapper.selectList(any())).thenReturn(List.of(root, leaf));
List<PermissionMenuItemVo> result = service.list("app");
assertThat(result).extracting(PermissionMenuItemVo::getId).containsExactly(1L, 2L);
assertThat(result).extracting(PermissionMenuItemVo::getRootColumnKey)
.containsExactly("brand_front_tools", "brand_front_tools");
}
@Test
void superAdminDirectIdEndpointReturnsOnlyPersistedIds() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(1L)).thenReturn(user(1L, "super_admin", 1));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 2L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
UserColumnIdsVo direct = service.getUserColumnIds(1L, "app");
assertThat(direct.getColumnIds()).containsExactly(2L);
}
@Test
void superAdminEffectivePermissionsRemainVirtualFullCatalog() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(1L)).thenReturn(user(1L, "super_admin", 1));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(1L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId).containsExactly(1L, 2L);
verify(permissionMapper, never()).selectList(any());
}
@Test
void normalUserCanReadOwnEffectivePermissions() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity currentUser = user(9L, "normal", 0);
when(userMapper.selectById(9L)).thenReturn(currentUser);
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(9L, 1L)));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 10),
menu(2L, 1L, "app", 11)));
List<PermissionMenuItemVo> effective = service.getUserColumnPermissions(currentUser, 9L, "app");
assertThat(effective).extracting(PermissionMenuItemVo::getId)
.containsExactly(1L, 2L);
}
@Test
void normalUserCannotReadAnotherUsersEffectivePermissions() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity currentUser = user(9L, "normal", 0);
when(userMapper.selectById(10L)).thenReturn(user(10L, "normal", 0));
assertThatThrownBy(() -> service.getUserColumnPermissions(currentUser, 10L, "app"))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("管理员权限");
verify(permissionMapper, never()).selectList(any());
}
@Test
void rejectsParentFromAnotherMenuType() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
PermissionMenuEntity parent = menu(7L, null, "admin", 1);
when(menuMapper.selectById(7L)).thenReturn(parent);
PermissionMenuCreateRequest request = createRequest(7L, "app");
assertThatThrownBy(() -> service.create(request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("同一类型");
verify(menuMapper, never()).insert(any(PermissionMenuEntity.class));
}
@Test
void rejectsParentCycleWhenUpdatingMenu() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
PermissionMenuEntity current = menu(1L, null, "app", 1);
PermissionMenuEntity parent = menu(2L, 3L, "app", 2);
PermissionMenuEntity ancestor = menu(3L, 1L, "app", 3);
when(menuMapper.selectById(1L)).thenReturn(current);
when(menuMapper.selectById(2L)).thenReturn(parent);
when(menuMapper.selectById(3L)).thenReturn(ancestor);
PermissionMenuUpdateRequest request = new PermissionMenuUpdateRequest();
request.setName("changed");
request.setColumnKey("changed");
request.setMenuType("app");
request.setRoutePath("changed");
request.setParentId(2L);
assertThatThrownBy(() -> service.update(1L, request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("自己的子级");
verify(menuMapper, never()).updateById(any(PermissionMenuEntity.class));
}
@Test
void rejectsChangingMenuTypeWhileChildrenRemain() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(menuMapper.selectById(1L)).thenReturn(menu(1L, null, "app", 1));
when(menuMapper.selectCount(any())).thenReturn(1L);
PermissionMenuUpdateRequest request = new PermissionMenuUpdateRequest();
request.setName("changed");
request.setColumnKey("changed");
request.setMenuType("admin");
request.setRoutePath("changed");
assertThatThrownBy(() -> service.update(1L, request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("子菜单");
verify(menuMapper, never()).updateById(any(PermissionMenuEntity.class));
}
@Test
void refusesDeletingParentMenuUntilChildrenAreRemoved() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(menuMapper.selectById(1L)).thenReturn(menu(1L, null, "app", 1));
when(menuMapper.selectCount(any())).thenReturn(1L);
assertThatThrownBy(() -> service.delete(1L))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("子菜单");
verify(menuMapper, never()).deleteById(any(Long.class));
verify(permissionMapper, never()).delete(any());
}
@Test
void ordinaryAdminCannotAssignMenuOutsideOwnEffectiveSet() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(10L, "admin", 1);
AdminUserEntity target = user(20L, "normal", 0);
target.setCreatedById(10L);
when(userMapper.selectById(20L)).thenReturn(target);
when(userMapper.selectById(10L)).thenReturn(operator);
when(userMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectList(any())).thenReturn(List.of(menu(1L, null, "app", 1)));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
assertThatThrownBy(() -> service.updateUserColumnPermissions(operator, 20L, request))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("自己已有");
verify(permissionMapper, never()).delete(any());
}
@Test
void ordinaryAdminCanAssignDescendantGrantedThroughParent() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(10L, "admin", 1);
AdminUserEntity target = user(20L, "normal", 0);
target.setCreatedById(10L);
when(userMapper.selectById(20L)).thenReturn(target);
when(userMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, 1L, "app", 2)));
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(10L, 1L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
service.updateUserColumnPermissions(operator, 20L, request);
ArgumentCaptor<UserColumnPermissionEntity> captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(captor.capture());
assertThat(captor.getValue().getColumnId()).isEqualTo(2L);
}
@Test
void ordinaryAdminReplacementPreservesDirectGrantsOutsideEffectiveScope() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(10L, "admin", 1);
AdminUserEntity target = user(20L, "normal", 0);
target.setCreatedById(10L);
when(userMapper.selectById(20L)).thenReturn(target);
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(1L, null, "app", 1),
menu(2L, null, "app", 2)));
when(permissionMapper.selectList(any()))
.thenReturn(List.of(grant(10L, 1L)), List.of(grant(20L, 2L)));
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(1L));
service.updateUserColumnPermissions(operator, 20L, request);
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper, times(2)).insert(inserted.capture());
assertThat(inserted.getAllValues())
.extracting(UserColumnPermissionEntity::getColumnId)
.containsExactly(1L, 2L);
}
@Test
void replacementStoresOnlyDirectRequestedParentId() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(null);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(1L));
service.updateUserColumnPermissions(9L, request);
ArgumentCaptor<UserColumnPermissionEntity> captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(captor.capture());
assertThat(captor.getValue().getColumnId()).isEqualTo(1L);
}
@Test
void appScopedReplacementDeletesOnlyAppDirectGrants() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(1L, "super_admin", 1);
when(userMapper.selectById(9L)).thenReturn(user(9L, "normal", 0));
when(menuMapper.selectList(any())).thenReturn(List.of(
menu(11L, null, "app", 1),
menu(12L, 11L, "app", 2)));
when(menuMapper.selectOne(any())).thenReturn(null);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(12L));
service.updateUserColumnPermissions(operator, 9L, request, PermissionMenuService.MENU_TYPE_APP);
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
UserColumnPermissionEntity.class);
ArgumentCaptor<LambdaUpdateWrapper<UserColumnPermissionEntity>> deleted =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(permissionMapper).delete(deleted.capture());
assertThat(deleted.getValue().getSqlSegment()).contains("column_id", "IN");
assertThat(deleted.getValue().getParamNameValuePairs().values())
.contains(9L, 11L, 12L);
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(inserted.capture());
assertThat(inserted.getValue().getColumnId()).isEqualTo(12L);
}
@Test
void explicitLowestIdAdminCannotManageImageVideoDataPermissions() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
assertThatThrownBy(() -> service.listImageVideoDataPermissionUsers(explicitAdmin))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
assertThatThrownBy(() -> service.updateImageVideoDataPermissionUsers(explicitAdmin, List.of(2L)))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
verify(menuMapper, never()).selectOne(any());
verify(permissionMapper, never()).delete(any());
}
@Test
void explicitNormalRoleCannotActAsInternalAdminDespiteLegacyFlag() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(1L)).thenReturn(user(1L, "normal", 1));
assertThatThrownBy(() -> service.requireAdminOperator(1L))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("管理员权限");
}
@Test
void superAdminCanListImageVideoDataPermissionUsers() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(99L, "super_admin", 1);
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
explicitAdmin.setUsername("admin");
AdminUserEntity normal = user(2L, "normal", 0);
normal.setUsername("normal");
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission());
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 75L)));
when(userMapper.selectList(any())).thenReturn(List.of(operator, explicitAdmin, normal));
List<ImageVideoDataPermissionUserVo> result = service.listImageVideoDataPermissionUsers(operator);
assertThat(result).extracting(ImageVideoDataPermissionUserVo::getId).containsExactly(1L, 2L);
assertThat(result.get(0).isGranted()).isTrue();
assertThat(result.get(1).isGranted()).isFalse();
}
@Test
void imageVideoUpdateDeletesOnlyProtectedColumnGrants() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(99L, "super_admin", 1);
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
AdminUserEntity normal = user(2L, "normal", 0);
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission());
when(userMapper.selectList(any())).thenReturn(List.of(operator, explicitAdmin, normal));
int grantedCount = service.updateImageVideoDataPermissionUsers(operator, List.of(2L));
assertThat(grantedCount).isEqualTo(1);
verify(permissionMapper).deleteByMap(Map.of("column_id", 75L));
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(inserted.capture());
assertThat(inserted.getValue().getUserId()).isEqualTo(2L);
assertThat(inserted.getValue().getColumnId()).isEqualTo(75L);
}
private PermissionMenuCreateRequest createRequest(Long parentId, String menuType) {
PermissionMenuCreateRequest request = new PermissionMenuCreateRequest();
request.setName("child");
request.setColumnKey("child");
request.setMenuType(menuType);
request.setRoutePath("child");
request.setParentId(parentId);
return request;
}
private PermissionMenuEntity menu(Long id, Long parentId, String menuType, int sortOrder) {
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setId(id);
entity.setParentId(parentId);
entity.setMenuType(menuType);
entity.setColumnKey("menu-" + id);
entity.setRoutePath("route-" + id);
entity.setSortOrder(sortOrder);
return entity;
}
private UserColumnPermissionEntity grant(Long userId, Long columnId) {
UserColumnPermissionEntity entity = new UserColumnPermissionEntity();
entity.setUserId(userId);
entity.setColumnId(columnId);
return entity;
}
private AdminUserEntity user(Long id, String role, int isAdmin) {
AdminUserEntity entity = new AdminUserEntity();
entity.setId(id);
entity.setRole(role);
entity.setIsAdmin(isAdmin);
return entity;
}
private PermissionMenuEntity imageVideoPermission() {
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setId(75L);
entity.setColumnKey("admin_image_video_tasks");
return entity;
}
private PermissionMenuEntity imageVideoDataPermission() {
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setId(75L);
entity.setColumnKey("admin_image_video_task_data");
return entity;
}
}
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.publish.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -19,10 +20,15 @@ import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import org.apache.ibatis.builder.MapperBuilderAssistant;
@@ -32,6 +38,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
@@ -39,7 +46,12 @@ import org.springframework.transaction.support.TransactionTemplate;
import java.time.LocalDateTime;
import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -68,6 +80,8 @@ class PublishTaskServiceTest {
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class);
TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@Mock private LocalFileStorageService localFileStorageService;
@@ -77,17 +91,26 @@ class PublishTaskServiceTest {
@Mock private PublishItemMapper publishItemMapper;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private ObjectMapper objectMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private TransactionTemplate transactionTemplate;
@Mock private InstanceMetadata instanceMetadata;
@InjectMocks private PublishTaskService service;
private final List<TaskChunkEntity> storedChunks = new ArrayList<>();
private final List<TaskScopeStateEntity> storedScopes = new ArrayList<>();
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
private int nextPayloadId;
@BeforeEach
void executeTransactionsInline() {
configureChunkStorage();
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(transactionTemplate.execute(any())).thenAnswer(invocation -> {
TransactionCallback<?> callback = invocation.getArgument(0);
@@ -159,6 +182,62 @@ class PublishTaskServiceTest {
verifyNoInteractions(publishItemMapper);
}
@Test
void failedTaskOwnedByAnotherInstanceCanBeDeletedLocally() throws Exception {
long taskId = 110L;
FileTaskEntity task = task(taskId, 7L, "FAILED");
task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}");
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(fileResultMapper.selectList(any())).thenReturn(List.of());
service.deleteTask(taskId, 7L);
verify(taskFileJobService).deleteTaskJobs(taskId, PublishTaskService.MODULE_TYPE);
verify(publishItemMapper).delete(any());
verify(publishFileMapper).delete(any());
verify(fileResultMapper).delete(any());
verify(fileTaskMapper).deleteById(taskId);
verify(objectMapper, never()).readTree(any(String.class));
}
@Test
void runningTaskOwnedByAnotherInstanceStillRequiresRouting() throws Exception {
long taskId = 111L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}");
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(objectMapper.readTree(task.getResultJson())).thenReturn(
new ObjectMapper().readTree(task.getResultJson()));
TaskOwnerMismatchException error = assertThrows(TaskOwnerMismatchException.class,
() -> service.deleteTask(taskId, 7L));
assertEquals(taskId, error.getTaskId());
assertEquals("instance-b", error.getOwnerInstanceId());
verifyNoInteractions(taskFileJobService, publishItemMapper, publishFileMapper, fileResultMapper);
verify(fileTaskMapper, never()).deleteById(taskId);
}
@Test
void historyForFailedTaskOwnedByAnotherInstanceCanBeDeletedLocally() throws Exception {
long taskId = 112L;
long resultId = 212L;
FileTaskEntity task = task(taskId, 7L, "FAILED");
task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}");
FileResultEntity result = new FileResultEntity();
result.setId(resultId);
result.setTaskId(taskId);
result.setModuleType(PublishTaskService.MODULE_TYPE);
when(fileResultMapper.selectById(resultId)).thenReturn(result);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
service.deleteHistory(resultId, 7L);
verify(fileTaskMapper).deleteById(taskId);
verify(objectMapper, never()).readTree(any(String.class));
}
@Test
void resultCallbackRejectsAnotherUserBeforeReadingOrReplacingRows() {
long taskId = 101L;
@@ -258,6 +337,242 @@ class PublishTaskServiceTest {
verify(lock).close();
}
@Test
void resultCallbackMergesOutOfOrderChunksOnlyAfterAllChunksArrive() {
long taskId = 113L;
long fileId = 213L;
long resultId = 313L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "分片.xlsx");
FileResultEntity result = result(taskId, resultId);
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result);
when(publishItemMapper.selectCount(any())).thenReturn(2L);
service.submitResult(taskId, chunkResultRequest(7L, fileId, 2, 2, List.of(row("2"))));
assertEquals("RUNNING", file.getStatus());
assertEquals(1, storedChunks.size());
assertEquals(1, storedScopes.getFirst().getReceivedChunkCount());
assertEquals(0, storedScopes.getFirst().getCompleted());
verify(publishItemMapper, never()).delete(any());
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
assertEquals("SUCCESS", file.getStatus());
assertEquals(2, storedChunks.size());
assertEquals(2, storedScopes.getFirst().getReceivedChunkCount());
assertEquals(1, storedScopes.getFirst().getCompleted());
org.mockito.ArgumentCaptor<List<PublishItemEntity>> rowsCaptor =
org.mockito.ArgumentCaptor.forClass(List.class);
verify(publishItemMapper).insertBatch(rowsCaptor.capture());
assertEquals(List.of("1", "2"), rowsCaptor.getValue().stream()
.map(PublishItemEntity::getSourceId)
.toList());
verify(taskFileJobService).enqueueAssembleResult(
taskId, PublishTaskService.MODULE_TYPE, resultId,
"task:" + taskId + ":owner:instance-a");
}
@Test
void identicalResultChunkRetryIsIdempotent() {
long taskId = 114L;
long fileId = 214L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "重试.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
PublishSubmitResultRequest request = chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")));
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 314L));
service.submitResult(taskId, request);
service.submitResult(taskId, request);
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
verify(publishItemMapper, never()).delete(any());
}
@Test
void resultChunkRetryWithDifferentContentIsRejected() {
long taskId = 115L;
long fileId = 215L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "冲突.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 315L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("different")))));
assertTrue(error.getMessage().contains("不同内容"));
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
}
@Test
void resultChunksRejectChangedChunkTotal() {
long taskId = 116L;
long fileId = 216L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "总片数冲突.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 316L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 2, 3, List.of(row("2")))));
assertTrue(error.getMessage().contains("chunk_total"));
assertEquals(1, storedChunks.size());
}
@Test
void resultChunkRejectsCallbackWhenRustfsIsUnavailable() {
long taskId = 117L;
long fileId = 217L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "RustFS未配置.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(false);
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))));
assertTrue(error.getMessage().contains("RustFS 未配置"));
assertEquals(0, storedChunks.size());
verify(publishItemMapper, never()).delete(any());
}
@Test
void resultChunkRejectsLocalFallbackPointer() {
long taskId = 118L;
long fileId = 218L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "本地回退.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
String localPointer = "local:instance-a/publish/chunk-1";
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(transientPayloadStorageService.storeChunkPayloadVersioned(any(), any(), any(), any(), any()))
.thenReturn(localPointer);
when(transientPayloadStorageService.extractPointer(localPointer)).thenReturn(localPointer);
BusinessException error = assertThrows(BusinessException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))));
assertTrue(error.getMessage().contains("必须写入 RustFS"));
assertEquals(0, storedChunks.size());
verify(transientPayloadStorageService).deletePayloadIfPresent(localPointer);
}
@Test
void resultChunkDeletesRustfsPayloadWhenMetadataInsertFails() {
long taskId = 120L;
long fileId = 220L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "元数据失败.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(taskChunkMapper.insert(any(TaskChunkEntity.class)))
.thenThrow(new IllegalStateException("database unavailable"));
assertThrows(IllegalStateException.class, () -> service.submitResult(
taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")))));
assertTrue(rustfsPayloads.isEmpty());
assertTrue(storedChunks.isEmpty());
verify(transientPayloadStorageService).deletePayloadIfPresent(any());
}
@Test
void deletingTaskRemovesPublishChunksAndRustfsPayloads() {
long taskId = 119L;
long fileId = 219L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "待删除.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 319L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
String pointer = storedChunks.getFirst().getPayloadJson();
assertEquals(1, storedChunks.size());
assertEquals(1, storedScopes.size());
assertEquals(1, rustfsPayloads.size());
when(fileResultMapper.selectList(any())).thenReturn(List.of());
service.deleteTask(taskId, 7L);
assertTrue(storedChunks.isEmpty());
assertTrue(storedScopes.isEmpty());
assertTrue(rustfsPayloads.isEmpty());
verify(transientPayloadStorageService).deletePayloadIfPresent(pointer);
verify(fileTaskMapper).deleteById(taskId);
}
@Test
void allFailedTaskRemovesPreviouslyUploadedChunksAfterCommit() {
long taskId = 121L;
long fileId = 221L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "执行失败.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result(taskId, 321L));
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
assertEquals(1, storedChunks.size());
service.submitResult(taskId, errorResultRequest(7L, fileId, "Python 执行失败"));
assertEquals("FAILED", task.getStatus());
assertEquals("FAILED", file.getStatus());
assertTrue(storedChunks.isEmpty());
assertTrue(storedScopes.isEmpty());
assertTrue(rustfsPayloads.isEmpty());
}
@Test
void activateFileRejectsSecondRunningFileWhileHoldingTaskLock() {
long taskId = 104L;
@@ -346,6 +661,53 @@ class PublishTaskServiceTest {
verify(lock).close();
}
@Test
void staleTaskRecoversReceivedChunksAndEnqueuesAssembly() {
long taskId = 118L;
long fileId = 218L;
long resultId = 318L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
task.setSourceFileCount(1);
task.setUpdatedAt(LocalDateTime.now().minusHours(1));
PublishFileEntity file = file(taskId, fileId, "RUNNING", "部分结果.xlsx");
file.setTotalRows(2);
FileResultEntity result = result(taskId, resultId);
TaskDistributedLockService.LockHandle submitLock = mock(TaskDistributedLockService.LockHandle.class);
TaskDistributedLockService.LockHandle staleLock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(submitLock);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId, 0L)).thenReturn(staleLock);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result);
when(taskFileJobService.countUnfinishedAssembleJobs(taskId, PublishTaskService.MODULE_TYPE))
.thenReturn(0L);
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
task.setUpdatedAt(LocalDateTime.now().minusHours(1));
service.failStaleTasks();
assertEquals("SUCCESS", file.getStatus());
assertEquals(1, file.getTotalRows());
assertEquals(1, file.getProcessedRows());
assertTrue(file.getErrorMessage().contains("已保留 1 行"));
assertEquals(1, task.getSuccessFileCount());
assertEquals(0, task.getFailedFileCount());
org.mockito.ArgumentCaptor<List<PublishItemEntity>> rowsCaptor =
org.mockito.ArgumentCaptor.forClass(List.class);
verify(publishItemMapper).insertBatch(rowsCaptor.capture());
assertEquals(List.of("1"), rowsCaptor.getValue().stream()
.map(PublishItemEntity::getSourceId)
.toList());
verify(taskFileJobService).enqueueAssembleResult(
taskId, PublishTaskService.MODULE_TYPE, resultId,
"task:" + taskId + ":owner:instance-a");
verify(submitLock).close();
verify(staleLock).close();
}
@Test
void staleTaskWithoutSuccessfulFilesFailsTaskAndResult() {
long taskId = 107L;
@@ -386,6 +748,140 @@ class PublishTaskServiceTest {
verify(lock).close();
}
private void configureChunkStorage() {
storedChunks.clear();
storedScopes.clear();
rustfsPayloads.clear();
nextPayloadId = 0;
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
any(), any(), any(), any(), any())).thenAnswer(invocation -> {
String pointer = "rustfs:test/publish/chunk-" + (++nextPayloadId);
rustfsPayloads.put(pointer, invocation.getArgument(4));
return pointer;
});
lenient().when(transientPayloadStorageService.extractPointer(any())).thenAnswer(invocation -> {
String value = invocation.getArgument(0);
return value != null && value.startsWith("rustfs:") ? value : null;
});
lenient().when(transientPayloadStorageService.resolvePayload(any(), any())).thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
String payload = rustfsPayloads.get(pointer);
if (payload == null) {
throw new IllegalStateException("missing test RustFS payload: " + pointer);
}
return payload;
});
lenient().doAnswer(invocation -> {
rustfsPayloads.remove(invocation.getArgument(0));
return null;
}).when(transientPayloadStorageService).deletePayloadIfPresent(any());
lenient().when(taskChunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> {
TaskChunkEntity chunk = invocation.getArgument(0);
chunk.setId((long) storedChunks.size() + 1L);
storedChunks.add(chunk);
return 1;
});
lenient().when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
Integer chunkIndex = queryInteger(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, chunkIndex))
.findFirst()
.orElse(null);
});
lenient().when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.count();
});
lenient().when(taskChunkMapper.selectList(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex))
.toList();
});
lenient().when(taskChunkMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedChunks.size();
storedChunks.removeIf(chunk -> taskId == null || Objects.equals(taskId, chunk.getTaskId()));
return before - storedChunks.size();
});
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
scope.setId((long) storedScopes.size() + 1L);
storedScopes.add(scope);
return 1;
});
lenient().when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedScopes.stream()
.filter(scope -> Objects.equals(taskId, scope.getTaskId()))
.filter(scope -> Objects.equals(scopeHash, scope.getScopeHash()))
.findFirst()
.orElse(null);
});
lenient().when(taskScopeStateMapper.updateById(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedScopes.size();
storedScopes.removeIf(scope -> taskId == null || Objects.equals(taskId, scope.getTaskId()));
return before - storedScopes.size();
});
}
private boolean matchesChunk(TaskChunkEntity chunk,
Long taskId,
String scopeHash,
Integer chunkIndex) {
return (taskId == null || Objects.equals(taskId, chunk.getTaskId()))
&& (scopeHash == null || Objects.equals(scopeHash, chunk.getScopeHash()))
&& (chunkIndex == null || Objects.equals(chunkIndex, chunk.getChunkIndex()));
}
private Long queryLong(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Long.class::isInstance)
.map(Long.class::cast)
.findFirst()
.orElse(null);
}
private Integer queryInteger(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Integer.class::isInstance)
.map(Integer.class::cast)
.findFirst()
.orElse(null);
}
private String queryScopeHash(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.filter(value -> value.length() == 64)
.findFirst()
.orElse(null);
}
private FileTaskEntity task(long taskId, long userId, String status) {
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
@@ -407,11 +903,30 @@ class PublishTaskServiceTest {
return file;
}
private FileResultEntity result(long taskId, long resultId) {
FileResultEntity result = new FileResultEntity();
result.setId(resultId);
result.setTaskId(taskId);
result.setModuleType(PublishTaskService.MODULE_TYPE);
result.setSuccess(0);
return result;
}
private PublishSubmitResultRequest resultRequest(long userId,
long fileId,
List<PublishRowDto> rows) {
long fileId,
List<PublishRowDto> rows) {
return chunkResultRequest(userId, fileId, null, null, rows);
}
private PublishSubmitResultRequest chunkResultRequest(long userId,
long fileId,
Integer chunkIndex,
Integer chunkTotal,
List<PublishRowDto> rows) {
PublishResultFileDto file = new PublishResultFileDto();
file.setFileId(fileId);
file.setChunkIndex(chunkIndex);
file.setChunkTotal(chunkTotal);
file.setRows(rows);
PublishSubmitResultRequest request = new PublishSubmitResultRequest();
request.setUserId(userId);
@@ -419,6 +934,18 @@ class PublishTaskServiceTest {
return request;
}
private PublishSubmitResultRequest errorResultRequest(long userId,
long fileId,
String error) {
PublishResultFileDto file = new PublishResultFileDto();
file.setFileId(fileId);
file.setError(error);
PublishSubmitResultRequest request = new PublishSubmitResultRequest();
request.setUserId(userId);
request.setFiles(List.of(file));
return request;
}
private PublishRowDto row(String sourceId) {
PublishRowDto row = new PublishRowDto();
row.setSourceId(sourceId);
@@ -0,0 +1,56 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
class ShopDataCrawlExcelAssemblyServiceTest {
@TempDir Path tempDir;
@Test
void writesRowsIntoStrictTemplateWithoutChangingSheetOrderOrHeaders() throws Exception {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate("2026-07-25");
row.setAsin("B012345678");
row.setInventorySales("11");
row.setSalesRank("22");
row.setPageViews("33");
row.setUnitsSold("44");
row.setPrice("12.50");
row.setRecommendedOffer("12.00");
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
country.setCountry("UK");
country.setItems(List.of(row));
ShopDataCrawlResultItemVo item = new ShopDataCrawlResultItemVo();
item.setSuccess(true);
item.setCountryResults(List.of(country));
File output = tempDir.resolve("result.xlsx").toFile();
new ShopDataCrawlExcelAssemblyService().writeWorkbook(output, List.of(item));
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS,
java.util.stream.IntStream.range(0, workbook.getNumberOfSheets())
.mapToObj(i -> workbook.getSheetAt(i).getSheetName()).toList());
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
for (int columnIndex = 0; columnIndex < ShopDataCrawlExcelAssemblyService.HEADERS.size(); columnIndex++) {
assertEquals(ShopDataCrawlExcelAssemblyService.HEADERS.get(columnIndex),
workbook.getSheetAt(i).getRow(0).getCell(columnIndex).getStringCellValue());
}
}
assertEquals("2026-07-25", workbook.getSheet("英国").getRow(1).getCell(0).getStringCellValue());
assertEquals("B012345678", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue());
assertEquals(0, workbook.getSheet("德国").getLastRowNum());
}
}
}
@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.modules.shopkey.service.ShopManageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ShopDataCrawlResolveServiceTest {
@Mock com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper candidateMapper;
@Mock com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper countryPrefMapper;
@Mock com.fasterxml.jackson.databind.ObjectMapper objectMapper;
@Mock ZiniaoShopSwitchService ziniaoShopSwitchService;
@Mock ShopManageService shopManageService;
@InjectMocks ShopDataCrawlResolveService service;
@Test
void requiresManagedShopBeforeLookingUpZiniaoIndex() {
ZiniaoShopMatchResultVo match = new ZiniaoShopMatchResultVo();
match.setMatched(true);
when(ziniaoShopSwitchService.normalizeShopName(" Demo ")).thenReturn("Demo");
when(ziniaoShopSwitchService.findIndexedStoreByName("Demo", false)).thenReturn(match);
assertSame(match, service.requireMatchedShop(" Demo "));
var order = inOrder(shopManageService, ziniaoShopSwitchService);
order.verify(ziniaoShopSwitchService).normalizeShopName(" Demo ");
order.verify(shopManageService).requireShopByName("Demo");
order.verify(ziniaoShopSwitchService).findIndexedStoreByName("Demo", false);
}
}
@@ -0,0 +1,539 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ShopDataCrawlTaskServiceChunkTest {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
private static final String SHOP_NAME = "Demo Shop";
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private ShopDataCrawlResolveService shopDataCrawlResolveService;
@Mock private ShopDataCrawlExcelAssemblyService excelAssemblyService;
@Mock private ShopDataCrawlTaskCacheService taskCacheService;
@Mock private OssStorageService ossStorageService;
@Mock private ZiniaoShopSwitchService ziniaoShopSwitchService;
@Mock private TaskPressureProperties taskPressureProperties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskResultItemService taskResultItemService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private InstanceMetadata instanceMetadata;
@Spy private ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
private ShopDataCrawlTaskService service;
private final List<TaskChunkEntity> storedChunks = new ArrayList<>();
private final List<TaskScopeStateEntity> storedScopes = new ArrayList<>();
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
private FileTaskEntity task;
private FileResultEntity result;
private int nextPayloadId;
@BeforeEach
void configureStorage() {
service = new ShopDataCrawlTaskService(
fileTaskMapper,
fileResultMapper,
shopDataCrawlResolveService,
excelAssemblyService,
taskCacheService,
ossStorageService,
ziniaoShopSwitchService,
objectMapper,
taskPressureProperties,
taskFileJobService,
taskResultItemService,
taskProgressSnapshotService,
taskDistributedLockService,
taskChunkMapper,
taskScopeStateMapper,
transientPayloadStorageService,
instanceMetadata);
storedChunks.clear();
storedScopes.clear();
rustfsPayloads.clear();
nextPayloadId = 0;
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(eq(MODULE_TYPE), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(taskCacheService.getTaskCacheBatch(any())).thenReturn(Map.of());
lenient().when(taskCacheService.getAllShopMergedPayload(anyLong())).thenReturn(Map.of());
lenient().when(taskResultItemService.listResultSnapshots(anyLong(), eq(MODULE_TYPE), any())).thenReturn(List.of());
lenient().when(taskFileJobService.findAssembleJob(anyLong(), eq(MODULE_TYPE), anyLong())).thenReturn(null);
lenient().when(excelAssemblyService.countRows(any())).thenReturn(1);
lenient().when(fileTaskMapper.selectById(anyLong())).thenAnswer(invocation -> {
Long taskId = invocation.getArgument(0);
return task != null && Objects.equals(taskId, task.getId()) ? task : null;
});
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
lenient().when(fileResultMapper.selectById(anyLong())).thenAnswer(invocation -> {
Long resultId = invocation.getArgument(0);
return result != null && Objects.equals(resultId, result.getId()) ? result : null;
});
lenient().when(fileResultMapper.selectList(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<FileResultEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
return result != null && Objects.equals(taskId, result.getTaskId()) ? List.of(result) : List.of();
});
lenient().when(fileResultMapper.updateById(any(FileResultEntity.class))).thenReturn(1);
lenient().when(ziniaoShopSwitchService.normalizeShopName(nullable(String.class))).thenAnswer(invocation -> {
String value = invocation.getArgument(0);
return value == null ? "" : value.trim();
});
configureTransientPayloadStorage();
configureChunkMapper();
configureScopeMapper();
}
@Test
void completesChunkedResultWhenSecondChunkArrivesBeforeFirstChunk() {
givenRunningTask(101L, 201L);
service.submitResult(task.getId(), request(chunk(2, 2, "UK", row("2026-07-26", "B002"))));
assertEquals(-1, result.getSuccess());
assertEquals(1, storedChunks.size());
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), eq(MODULE_TYPE), anyLong(), anyString());
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
assertEquals(1, result.getSuccess());
assertEquals(2, storedChunks.size());
assertTrue(task.getResultJson().indexOf("B001") < task.getResultJson().indexOf("B002"));
verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(),
"task:" + task.getId() + ":owner:instance-a");
}
@Test
void identicalResultChunkRetryIsIdempotent() {
givenRunningTask(102L, 202L);
ShopDataCrawlSubmitResultRequest request = request(chunk(1, 2, "DE", row("2026-07-25", "B001")));
service.submitResult(task.getId(), request);
service.submitResult(task.getId(), request);
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
assertEquals(-1, result.getSuccess());
}
@Test
void resultChunkRetryWithDifferentContentIsRejected() {
givenRunningTask(103L, 203L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B099")))));
assertTrue(error.getMessage().contains("不同内容"));
assertEquals(1, storedChunks.size());
assertEquals(1, rustfsPayloads.size());
}
@Test
void resultChunksRejectChangedChunkTotal() {
givenRunningTask(104L, 204L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(2, 3, "UK", row("2026-07-26", "B002")))));
assertTrue(error.getMessage().contains("chunk_total"));
assertEquals(1, storedChunks.size());
}
@Test
void missingChunkDoesNotGenerateResultFile() {
givenRunningTask(105L, 205L);
service.submitResult(task.getId(), request(chunk(1, 3, "DE", row("2026-07-25", "B001"))));
service.submitResult(task.getId(), request(chunk(3, 3, "UK", row("2026-07-27", "B003"))));
assertEquals(-1, result.getSuccess());
assertEquals("RUNNING", task.getStatus());
assertEquals(2, storedChunks.size());
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), eq(MODULE_TYPE), anyLong(), anyString());
}
@Test
void staleCompensationRecoversReceivedResultChunks() {
givenRunningTask(110L, 210L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
boolean finalized = service.tryFinalizeTask(task.getId(), true);
assertTrue(finalized);
assertEquals(1, result.getSuccess());
assertTrue(result.getErrorMessage().contains("已保留"));
assertTrue(task.getResultJson().contains("B001"));
assertTrue(task.getResultJson().contains("已保留"));
verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(),
"task:" + task.getId() + ":owner:instance-a");
}
@Test
void staleCompensationRecoversLegacyMergedPayload() {
givenRunningTask(111L, 211L);
ShopDataCrawlShopPayloadDto cached = legacyChunk(false, "UK", row("2026-07-26", "B002"));
when(taskCacheService.getAllShopMergedPayload(task.getId())).thenReturn(Map.of(SHOP_NAME, cached));
boolean finalized = service.tryFinalizeTask(task.getId(), true);
assertTrue(finalized);
assertEquals(1, result.getSuccess());
assertTrue(result.getErrorMessage().contains("已保留"));
assertTrue(task.getResultJson().contains("B002"));
verify(taskCacheService).removeShopMergedPayload(task.getId(), SHOP_NAME);
verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(),
"task:" + task.getId() + ":owner:instance-a");
}
@Test
void resultChunkRejectsCallbackWhenRustfsIsUnavailable() {
givenRunningTask(106L, 206L);
when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(false);
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(1, 1, "DE", row("2026-07-25", "B001")))));
assertTrue(error.getMessage().contains("RustFS"));
assertEquals(0, storedChunks.size());
}
@Test
void resultChunkRejectsLocalFallbackPointer() {
givenRunningTask(107L, 207L);
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), anyLong(), anyString(), any(), anyString()))
.thenReturn("local:instance-a/chunk-1");
when(transientPayloadStorageService.extractPointer("local:instance-a/chunk-1"))
.thenReturn("local:instance-a/chunk-1");
BusinessException error = assertThrows(BusinessException.class, () ->
service.submitResult(task.getId(), request(chunk(1, 1, "DE", row("2026-07-25", "B001")))));
assertTrue(error.getMessage().contains("RustFS"));
assertEquals(0, storedChunks.size());
verify(transientPayloadStorageService).deletePayloadIfPresent("local:instance-a/chunk-1");
}
@Test
void legacyShopDoneFalseThenTrueUsesMergedPayloadCache() {
givenRunningTask(108L, 208L);
ShopDataCrawlShopPayloadDto cached = legacyChunk(false, "DE", row("2026-07-25", "B001"));
when(taskCacheService.getShopMergedPayload(task.getId(), SHOP_NAME)).thenReturn(null, cached);
service.submitResult(task.getId(), request(legacyChunk(false, "DE", row("2026-07-25", "B001"))));
assertEquals(-1, result.getSuccess());
verify(taskCacheService).saveShopMergedPayload(eq(task.getId()), eq(SHOP_NAME), any());
service.submitResult(task.getId(), request(legacyChunk(true, "UK", row("2026-07-26", "B002"))));
assertEquals(1, result.getSuccess());
assertTrue(task.getResultJson().contains("B001"));
assertTrue(task.getResultJson().contains("B002"));
assertEquals(0, storedChunks.size());
verify(taskCacheService).removeShopMergedPayload(task.getId(), SHOP_NAME);
}
@Test
void cleanupResultFileJobDeletesTransientChunkObjectsAndRows() {
givenRunningTask(109L, 209L);
service.submitResult(task.getId(), request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
String storedPayload = storedChunks.get(0).getPayloadJson();
TaskFileJobEntity job = new TaskFileJobEntity();
job.setTaskId(task.getId());
job.setModuleType(MODULE_TYPE);
job.setResultId(result.getId());
service.cleanupResultFileJob(job);
assertFalse(rustfsPayloads.containsKey(storedPayload));
assertEquals(0, storedChunks.size());
assertEquals(0, storedScopes.size());
verify(taskChunkMapper).delete(any());
verify(taskScopeStateMapper).delete(any());
}
private void configureTransientPayloadStorage() {
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
anyString(), anyLong(), anyString(), any(), anyString())).thenAnswer(invocation -> {
String pointer = "rustfs:payload-" + (++nextPayloadId);
rustfsPayloads.put(pointer, invocation.getArgument(4));
return pointer;
});
lenient().when(transientPayloadStorageService.extractPointer(any())).thenAnswer(invocation -> {
String value = invocation.getArgument(0);
if (value == null) {
return null;
}
return value.startsWith("rustfs:") || value.startsWith("local:") || value.startsWith("oss:")
? value : null;
});
lenient().when(transientPayloadStorageService.resolvePayload(any(), any())).thenAnswer(invocation -> {
String pointer = invocation.getArgument(0);
String payload = rustfsPayloads.get(pointer);
if (payload == null) {
throw new IllegalStateException("missing test RustFS payload: " + pointer);
}
return payload;
});
lenient().doAnswer(invocation -> {
rustfsPayloads.remove(invocation.getArgument(0));
return null;
}).when(transientPayloadStorageService).deletePayloadIfPresent(any());
}
private void configureChunkMapper() {
lenient().when(taskChunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> {
TaskChunkEntity chunk = invocation.getArgument(0);
chunk.setId((long) storedChunks.size() + 1L);
storedChunks.add(chunk);
return 1;
});
lenient().when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
Integer chunkIndex = queryInteger(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, chunkIndex))
.findFirst()
.orElse(null);
});
lenient().when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.count();
});
lenient().when(taskChunkMapper.selectList(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedChunks.stream()
.filter(chunk -> matchesChunk(chunk, taskId, scopeHash, null))
.sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex))
.toList();
});
lenient().when(taskChunkMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedChunks.size();
storedChunks.removeIf(chunk -> taskId == null || Objects.equals(taskId, chunk.getTaskId()));
return before - storedChunks.size();
});
}
private void configureScopeMapper() {
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
scope.setId((long) storedScopes.size() + 1L);
storedScopes.add(scope);
return 1;
});
lenient().when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
String scopeHash = queryScopeHash(query);
return storedScopes.stream()
.filter(scope -> Objects.equals(taskId, scope.getTaskId()))
.filter(scope -> scopeHash == null || Objects.equals(scopeHash, scope.getScopeHash()))
.findFirst()
.orElse(null);
});
lenient().when(taskScopeStateMapper.updateById(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.delete(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskScopeStateEntity> query = invocation.getArgument(0);
Long taskId = queryLong(query);
int before = storedScopes.size();
storedScopes.removeIf(scope -> taskId == null || Objects.equals(taskId, scope.getTaskId()));
return before - storedScopes.size();
});
}
private boolean matchesChunk(TaskChunkEntity chunk, Long taskId, String scopeHash, Integer chunkIndex) {
return (taskId == null || Objects.equals(taskId, chunk.getTaskId()))
&& (scopeHash == null || Objects.equals(scopeHash, chunk.getScopeHash()))
&& (chunkIndex == null || Objects.equals(chunkIndex, chunk.getChunkIndex()));
}
private Long queryLong(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Long.class::isInstance)
.map(Long.class::cast)
.findFirst()
.orElse(null);
}
private Integer queryInteger(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Integer.class::isInstance)
.map(Integer.class::cast)
.findFirst()
.orElse(null);
}
private String queryScopeHash(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.filter(value -> value.length() == 64)
.findFirst()
.orElse(null);
}
private void givenRunningTask(long taskId, long resultId) {
task = new FileTaskEntity();
task.setId(taskId);
task.setUserId(7L);
task.setModuleType(MODULE_TYPE);
task.setStatus("RUNNING");
task.setRequestJson("{\"ownerInstanceId\":\"instance-a\"}");
task.setResultJson("[]");
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
result = new FileResultEntity();
result.setId(resultId);
result.setTaskId(taskId);
result.setUserId(7L);
result.setModuleType(MODULE_TYPE);
result.setSourceFilename(SHOP_NAME);
result.setSourceFileUrl("shop-1");
result.setSuccess(-1);
result.setCreatedAt(LocalDateTime.now());
}
private ShopDataCrawlSubmitResultRequest request(ShopDataCrawlShopPayloadDto payload) {
ShopDataCrawlSubmitResultRequest request = new ShopDataCrawlSubmitResultRequest();
request.setShops(List.of(payload));
return request;
}
private ShopDataCrawlShopPayloadDto chunk(int chunkIndex,
int chunkTotal,
String country,
ShopDataCrawlRowDto row) {
ShopDataCrawlShopPayloadDto payload = legacyChunk(false, country, row);
payload.setChunkIndex(chunkIndex);
payload.setChunkTotal(chunkTotal);
return payload;
}
private ShopDataCrawlShopPayloadDto legacyChunk(boolean shopDone,
String country,
ShopDataCrawlRowDto row) {
ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto();
payload.setShopName(SHOP_NAME);
payload.setCountryResults(List.of(country(country, row)));
payload.setShopDone(shopDone);
return payload;
}
private ShopDataCrawlCountryResultDto country(String country, ShopDataCrawlRowDto row) {
ShopDataCrawlCountryResultDto result = new ShopDataCrawlCountryResultDto();
result.setCountry(country);
result.setItems(List.of(row));
return result;
}
private ShopDataCrawlRowDto row(String date, String asin) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setInventorySales("10");
row.setSalesRank("20");
row.setPageViews("30");
row.setUnitsSold("40");
row.setPrice("50");
row.setRecommendedOffer("60");
return row;
}
}
@@ -0,0 +1,99 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ShopDataCrawlTaskServiceOwnerTest {
@Mock InstanceMetadata instanceMetadata;
@Mock com.nanri.aiimage.modules.task.mapper.FileTaskMapper fileTaskMapper;
@Mock com.nanri.aiimage.modules.task.mapper.FileResultMapper fileResultMapper;
@Mock ShopDataCrawlResolveService resolveService;
@Mock ShopDataCrawlExcelAssemblyService excelAssemblyService;
@Mock ShopDataCrawlTaskCacheService cacheService;
@Mock com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService ziniaoShopSwitchService;
@Mock com.nanri.aiimage.config.TaskPressureProperties taskPressureProperties;
@Mock com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@Mock com.nanri.aiimage.modules.task.service.TaskResultItemService taskResultItemService;
@Mock com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock com.nanri.aiimage.modules.task.mapper.TaskChunkMapper taskChunkMapper;
@Mock com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper taskScopeStateMapper;
@Mock com.nanri.aiimage.modules.task.service.TransientPayloadStorageService transientPayloadStorageService;
@Spy private final ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks ShopDataCrawlTaskService service;
@Test
void rejectsTaskOwnedByAnotherInstance() {
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
FileTaskEntity task = task("instance-b");
assertThrows(TaskOwnerMismatchException.class,
() -> service.ensureTaskOwnedByCurrentInstance(task, "callback"));
}
@Test
void acceptsTaskOwnedByCurrentInstance() {
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
assertDoesNotThrow(() -> service.ensureTaskOwnedByCurrentInstance(task("instance-a"), "callback"));
}
@Test
void callbackWithoutShopNameBindsOnlyTaskShopAndWrongNameFails() {
var payload = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto();
when(ziniaoShopSwitchService.normalizeShopName(null)).thenReturn("");
assertEquals(payload, service.normalizePayloadByShop(java.util.List.of(payload), "Demo").get("Demo"));
var wrong = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto();
wrong.setShopName("Wrong");
when(ziniaoShopSwitchService.normalizeShopName("Wrong")).thenReturn("Wrong");
assertThrows(com.nanri.aiimage.common.exception.BusinessException.class,
() -> service.normalizePayloadByShop(java.util.List.of(wrong), "Demo"));
}
@Test
void partialPayloadIsNotCompleteUntilShopDoneIsTrue() {
var payload = new com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto();
assertFalse(service.isShopPayloadComplete(payload));
payload.setShopDone(false);
assertFalse(service.isShopPayloadComplete(payload));
payload.setShopDone(true);
assertTrue(service.isShopPayloadComplete(payload));
}
@Test
@SuppressWarnings("unchecked")
void deletesResultObjectOnlyAfterLastDatabaseReferenceIsGone() {
when(fileResultMapper.selectCount(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
.thenReturn(1L, 0L);
service.deleteResultObjectIfUnreferenced("result/shared.xlsx");
verify(ossStorageService, never()).deleteObject("result/shared.xlsx");
service.deleteResultObjectIfUnreferenced("result/shared.xlsx");
verify(ossStorageService).deleteObject("result/shared.xlsx");
}
private FileTaskEntity task(String owner) {
FileTaskEntity task = new FileTaskEntity();
task.setId(1L);
task.setRequestJson("{\"ownerInstanceId\":\"" + owner + "\"}");
return task;
}
}
@@ -7,6 +7,8 @@ import java.lang.reflect.Field;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SimilarAsinTaskServiceTest {
@@ -50,6 +52,14 @@ class SimilarAsinTaskServiceTest {
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
}
@Test
void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() {
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3));
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("FAILED", 3, 3));
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("RUNNING", 3, 3));
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3));
}
private int staticIntField(String name) throws Exception {
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
field.setAccessible(true);
@@ -1,5 +1,7 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
@@ -24,7 +26,33 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
class SimilarAsinImageEmbedderTest {
// properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底。
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null);
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService());
private static OssStorageService createOssStorageService() {
OssProperties properties = new OssProperties();
properties.setEndpoint("https://oss.aishufu.top");
properties.setPublicEndpoint("https://oss.aishufu.top");
properties.setBucket("nanri-ai-images");
properties.setImageVideoBucket("shufu-video");
properties.setDigitalHumanBucket("nanri-ai-digital-human");
properties.setAccessKeyId("test-access-key");
properties.setAccessKeySecret("test-secret-key");
return new OssStorageService(properties);
}
@Test
void normalizesLegacyMinioUrlBeforeHttpsValidation() {
String normalized = embedder.normalizeAndValidateDownloadUrl(
"http://47.110.241.161:9000/nanri-ai-images/supply_images/main.jpg");
assertEquals("https://oss.aishufu.top/nanri-ai-images/supply_images/main.jpg", normalized);
}
@Test
void leavesUnmanagedHttpUrlBlocked() {
assertThrows(SimilarAsinImageEmbedder.UnsupportedUrlException.class,
() -> embedder.normalizeAndValidateDownloadUrl("http://example.com/main.jpg"));
}
@Test
void resizeImageProducesThumbnailUnderHardCap() throws Exception {
@@ -13,6 +13,8 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
@@ -55,6 +57,8 @@ class TaskHeartbeatServiceTest {
@Mock private ShopMatchTaskCacheService shopMatchTaskCacheService;
@Mock private PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
@Mock private QueryAsinTaskCacheService queryAsinTaskCacheService;
@Mock private ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService;
@Mock private ShopDataCrawlTaskService shopDataCrawlTaskService;
@Mock private WithdrawTaskCacheService withdrawTaskCacheService;
@Mock private AppearancePatentTaskCacheService appearancePatentTaskCacheService;
@Mock private SimilarAsinTaskCacheService similarAsinTaskCacheService;
@@ -91,4 +95,27 @@ class TaskHeartbeatServiceTest {
routingBeforeUpdate.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
verify(publishTaskService).touchHeartbeat(taskId, request);
}
@Test
@SuppressWarnings("unchecked")
void shopDataCrawlHeartbeatChecksOwnerBeforeDatabaseUpdate() {
long taskId = 20143L;
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
task.setModuleType("SHOP_DATA_CRAWL");
task.setStatus("RUNNING");
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest());
assertTrue(result.isAlive());
InOrder order = inOrder(shopDataCrawlTaskService, fileTaskMapper);
order.verify(shopDataCrawlTaskService)
.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat");
order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
verify(shopDataCrawlTaskCacheService).touchTaskHeartbeat(taskId);
verify(shopDataCrawlTaskCacheService).saveTaskCache(task);
}
}
@@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
@@ -24,6 +25,7 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
@@ -43,6 +45,7 @@ class TaskResultFileJobWorkerTest {
@Mock private ProductRiskTaskService productRiskTaskService;
@Mock private PublishTaskService publishTaskService;
@Mock private QueryAsinTaskService queryAsinTaskService;
@Mock private ShopDataCrawlTaskService shopDataCrawlTaskService;
@Mock private WithdrawTaskService withdrawTaskService;
@Mock private PatrolDeleteTaskService patrolDeleteTaskService;
@Mock private AppearancePatentTaskService appearancePatentTaskService;
@@ -109,8 +112,10 @@ class TaskResultFileJobWorkerTest {
worker.process(job);
verify(publishTaskService).processResultFileJob(job);
verify(taskFileJobService).markSuccess(job, "result/publish/20141.xlsx");
InOrder order = inOrder(publishTaskService, taskFileJobService);
order.verify(publishTaskService).processResultFileJob(job);
order.verify(taskFileJobService).markSuccess(job, "result/publish/20141.xlsx");
order.verify(publishTaskService).cleanupResultFileJob(job);
verifyNoInteractions(taskResultPayloadService);
}
@@ -128,4 +133,56 @@ class TaskResultFileJobWorkerTest {
verifyNoInteractions(taskFileJobService, taskDistributedLockService, publishTaskService);
}
@Test
void shopDataCrawlOwnerJobDelegatesAndCleansPayload() {
long jobId = 13643L;
long taskId = 20143L;
long resultId = 22931L;
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(jobId);
job.setTaskId(taskId);
job.setResultId(resultId);
job.setModuleType("SHOP_DATA_CRAWL");
job.setScopeKey("task:20143:owner:instance-a");
FileResultEntity result = new FileResultEntity();
result.setResultFileUrl("result/shop-data-crawl/20143.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", taskId,
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
when(fileResultMapper.selectById(resultId)).thenReturn(result);
worker.process(job);
verify(shopDataCrawlTaskService).processResultFileJob(job);
verify(taskFileJobService).markSuccess(job, "result/shop-data-crawl/20143.xlsx");
verify(taskResultPayloadService).deleteLatest(taskId, "SHOP_DATA_CRAWL", job.getScopeKey());
verify(shopDataCrawlTaskService).cleanupResultFileJob(job);
}
@Test
void exhaustedShopDataCrawlFileJobFailsOwningTask() {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(13644L);
job.setTaskId(20144L);
job.setResultId(22932L);
job.setModuleType("SHOP_DATA_CRAWL");
job.setScopeKey("task:20144:owner:instance-a");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(taskFileJobService.markRunning(job.getId())).thenReturn(true);
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", job.getTaskId(),
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
doThrow(new IllegalStateException("upload failed"))
.when(shopDataCrawlTaskService).processResultFileJob(job);
when(taskFileJobService.isRetryExhausted(job.getId())).thenReturn(true);
worker.process(job);
verify(taskFileJobService).markFailed(job, "upload failed");
verify(shopDataCrawlTaskService).handleResultFileJobFailure(job, "upload failed");
}
}