diff --git a/.gitignore b/.gitignore index b324bde..ae2c06c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,18 @@ frontend-vue/npm-debug.log* frontend-vue/yarn-debug.log* frontend-vue/yarn-error.log* frontend-vue/pnpm-debug.log* + +# backend-java build output +backend-java/target/ + +# backend-java runtime data +backend-java/data/ +backend-java/*.log +backend-java/logs/ + +# backend-java local config +backend-java/src/main/resources/application-local.yml + +# backend-java IDE files +backend-java/.idea/ +backend-java/*.iml diff --git a/backend-java/pom.xml b/backend-java/pom.xml new file mode 100644 index 0000000..7f2ff65 --- /dev/null +++ b/backend-java/pom.xml @@ -0,0 +1,131 @@ + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 3.3.9 + + + + com.nanri + aiimage-backend + 0.0.1-SNAPSHOT + aiimage-backend + AI image backend service + + + 21 + 2.6.0 + 4.5.0 + 3.5.7 + 1.5.5.Final + 4.0.3 + 3.17.4 + 5.8.36 + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-security + + + com.baomidou + mybatis-plus-spring-boot3-starter + ${mybatis-plus.version} + + + com.mysql + mysql-connector-j + runtime + + + com.alibaba + easyexcel + ${easyexcel.version} + + + com.aliyun.oss + aliyun-sdk-oss + ${aliyun.oss.version} + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + ${springdoc.version} + + + com.github.xiaoymin + knife4j-openapi3-jakarta-spring-boot-starter + ${knife4j.version} + + + cn.hutool + hutool-all + ${hutool.version} + + + org.projectlombok + lombok + true + + + org.mapstruct + mapstruct + ${mapstruct.version} + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + + org.projectlombok + lombok + 1.18.36 + + + org.mapstruct + mapstruct-processor + ${mapstruct.version} + + + + + + + diff --git a/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java b/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java new file mode 100644 index 0000000..a85d792 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java @@ -0,0 +1,12 @@ +package com.nanri.aiimage; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class AiImageApplication { + + public static void main(String[] args) { + SpringApplication.run(AiImageApplication.class, args); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/api/ApiResponse.java b/backend-java/src/main/java/com/nanri/aiimage/common/api/ApiResponse.java new file mode 100644 index 0000000..a2db9aa --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/api/ApiResponse.java @@ -0,0 +1,34 @@ +package com.nanri.aiimage.common.api; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Schema(description = "统一响应体") +public class ApiResponse { + + @Schema(description = "是否成功") + private boolean success; + + @Schema(description = "提示信息") + private String message; + + @Schema(description = "响应数据") + private T data; + + public static ApiResponse success(T data) { + return new ApiResponse<>(true, "操作成功", data); + } + + public static ApiResponse success(String message, T data) { + return new ApiResponse<>(true, message, data); + } + + public static ApiResponse fail(String message) { + return new ApiResponse<>(false, message, null); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/exception/BusinessException.java b/backend-java/src/main/java/com/nanri/aiimage/common/exception/BusinessException.java new file mode 100644 index 0000000..0240697 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/exception/BusinessException.java @@ -0,0 +1,8 @@ +package com.nanri.aiimage.common.exception; + +public class BusinessException extends RuntimeException { + + public BusinessException(String message) { + super(message); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java b/backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java new file mode 100644 index 0000000..7b1e109 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java @@ -0,0 +1,37 @@ +package com.nanri.aiimage.common.exception; + +import com.nanri.aiimage.common.api.ApiResponse; +import jakarta.validation.ConstraintViolationException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +@Slf4j +@RestControllerAdvice +public class GlobalExceptionHandler { + + @ExceptionHandler(BusinessException.class) + public ApiResponse handleBusinessException(BusinessException ex) { + return ApiResponse.fail(ex.getMessage()); + } + + @ExceptionHandler(MethodArgumentNotValidException.class) + public ApiResponse handleMethodArgumentNotValidException(MethodArgumentNotValidException ex) { + String message = ex.getBindingResult().getFieldError() != null + ? ex.getBindingResult().getFieldError().getDefaultMessage() + : "参数校验失败"; + return ApiResponse.fail(message); + } + + @ExceptionHandler(ConstraintViolationException.class) + public ApiResponse handleConstraintViolationException(ConstraintViolationException ex) { + return ApiResponse.fail(ex.getMessage()); + } + + @ExceptionHandler(Exception.class) + public ApiResponse handleException(Exception ex) { + log.error("Unhandled exception", ex); + return ApiResponse.fail("服务异常：" + ex.getMessage()); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/OpenApiConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/OpenApiConfig.java new file mode 100644 index 0000000..6c6adf8 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/OpenApiConfig.java @@ -0,0 +1,44 @@ +package com.nanri.aiimage.config; + +import io.swagger.v3.oas.models.ExternalDocumentation; +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Contact; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.info.License; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class OpenApiConfig { + + @Bean + public OpenAPI openAPI() { + return new OpenAPI() + .info(new Info() + .title("AI Image Backend API") + .description(""" + 文件处理后端接口文档。 + + 当前迁移范围：仅迁移数据去重（dedupe）、格式转换（convert）、数据拆分（split）三块处理逻辑到 Java； + Python 桌面端壳、pywebview 调用方式、前端页面交互保持不变。 + + 建议联调顺序： + 1. 先启动 Java 后端； + 2. 在 Python 桌面端中选择文件并执行 dedupe/convert/split； + 3. Python 壳会先调用 /api/files/upload 上传临时文件； + 4. 再调用对应的 /api/dedupe/run、/api/convert/run、/api/split/run； + 5. Java 生成结果文件后，Python 壳再通过下载接口取回并保存到用户本地目录。 + + 本地启动说明： + - 默认配置读取 application.yml； + - 建议复制 application-local.example.yml 为本地配置，并在 IDE 中通过环境变量覆盖数据库与 OSS 参数； + - Knife4j 地址：/doc.html + """) + .version("v0.0.1") + .contact(new Contact().name("Nanri AI")) + .license(new License().name("Internal Use"))) + .externalDocs(new ExternalDocumentation() + .description("Knife4j 文档") + .url("/doc.html")); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java new file mode 100644 index 0000000..2e00956 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "aiimage.oss") +public class OssProperties { + private String region; + private String endpoint; + private String bucket; + private String accessKeyId; + private String accessKeySecret; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java new file mode 100644 index 0000000..2b77762 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.config; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableConfigurationProperties({OssProperties.class, StorageProperties.class}) +public class PropertiesConfig { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/SecurityConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/SecurityConfig.java new file mode 100644 index 0000000..968fadf --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/SecurityConfig.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.Customizer; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.web.SecurityFilterChain; + +@Configuration +public class SecurityConfig { + + @Bean + public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception { + return http + .csrf(csrf -> csrf.disable()) + .authorizeHttpRequests(auth -> auth.anyRequest().permitAll()) + .httpBasic(Customizer.withDefaults()) + .build(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java new file mode 100644 index 0000000..da584bf --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java @@ -0,0 +1,10 @@ +package com.nanri.aiimage.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "aiimage.storage") +public class StorageProperties { + private String localTempDir; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java new file mode 100644 index 0000000..49ed3c7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java @@ -0,0 +1,84 @@ +package com.nanri.aiimage.modules.convert.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest; +import com.nanri.aiimage.modules.convert.model.vo.ConvertHistoryVo; +import com.nanri.aiimage.modules.convert.model.vo.ConvertRunVo; +import com.nanri.aiimage.modules.convert.service.ConvertRunService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.File; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/convert") +@Tag(name = "格式转换执行", description = "负责根据已上传文件和模板执行格式转换，并返回结果下载地址。当前先打通接口链路，下一步再把旧 Python 的模板生成逻辑完整迁移到 Java。") +public class ConvertRunController { + + private final ConvertRunService convertRunService; + + @PostMapping("/run") + @Operation( + summary = "执行格式转换", + description = """ + 使用上传接口返回的 fileKey 和选中的模板执行格式转换。 + + 当前已按旧 Python 桌面端规则迁移以下逻辑： + - required_source_columns 缺失校验； + - preamble_lines 输出； + - header_columns 输出； + - blank_header_rows_after_schema 空白行补齐； + - field_mapping 字段映射； + - defaults 默认值填充； + - ASIN 为空的行跳过； + - sku 按 时间戳-序号 生成。 + + 输出文件名遵循模板配置的 outputFilename，若临时目录中重名则自动追加序号。 + """) + public ApiResponse run(@Valid @RequestBody ConvertRunRequest request) { + return ApiResponse.success(convertRunService.run(request)); + } + + @GetMapping("/history") + @Operation(summary = "查询格式转换历史", description = "查询最近的格式转换成功记录，用于页面右侧历史下载列表展示。") + public ApiResponse history() { + ConvertHistoryVo vo = new ConvertHistoryVo(); + vo.setItems(convertRunService.listHistory()); + return ApiResponse.success(vo); + } + + @DeleteMapping("/history/{resultId}") + @Operation(summary = "删除格式转换历史", description = "删除一条格式转换历史记录。") + public ApiResponse deleteHistory(@PathVariable Long resultId) { + convertRunService.deleteHistory(resultId); + return ApiResponse.success(null); + } + + @GetMapping("/results/{resultId}/download") + @Operation(summary = "下载格式转换结果", description = "根据结果记录 ID 下载已生成的转换结果文件。桌面端前端可继续通过 save_file_from_url 选择保存位置。") + public ResponseEntity download(@PathVariable Long resultId) { + File resultFile = convertRunService.getResultFile(resultId); + String encodedFilename = URLEncoder.encode(resultFile.getName(), StandardCharsets.UTF_8).replaceAll("\\+", "%20"); + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFilename) + .body(new FileSystemResource(resultFile)); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertTemplateController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertTemplateController.java new file mode 100644 index 0000000..370dc97 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertTemplateController.java @@ -0,0 +1,53 @@ +package com.nanri.aiimage.modules.convert.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.convert.model.dto.ConvertTemplateImportRequest; +import com.nanri.aiimage.modules.convert.model.vo.ConvertTemplateVo; +import com.nanri.aiimage.modules.convert.service.ConvertTemplateService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/convert/templates") +@Tag(name = "格式转换模板", description = "用于维护和查询格式转换模板，支持查询、导入、设默认、删除。") +public class ConvertTemplateController { + + private final ConvertTemplateService convertTemplateService; + + @GetMapping + @Operation(summary = "查询启用模板", description = "返回当前启用的格式转换模板列表，供前端‘格式转换’模块展示。") + public ApiResponse> listTemplates() { + return ApiResponse.success(convertTemplateService.listEnabledTemplates()); + } + + @PostMapping("/import") + @Operation(summary = "导入模板", description = "导入自定义 txt 模板，成功后返回新模板并可立即刷新模板列表。") + public ApiResponse importTemplate(@RequestBody ConvertTemplateImportRequest request) { + return ApiResponse.success(convertTemplateService.importTemplate(request)); + } + + @PostMapping("/{templateCode}/default") + @Operation(summary = "设为默认模板", description = "将指定模板设为默认模板。") + public ApiResponse setDefault(@PathVariable String templateCode) { + convertTemplateService.setDefaultTemplate(templateCode); + return ApiResponse.success(null); + } + + @DeleteMapping("/{templateCode}") + @Operation(summary = "删除模板", description = "删除指定模板。内置模板不允许删除。") + public ApiResponse delete(@PathVariable String templateCode) { + convertTemplateService.deleteTemplate(templateCode); + return ApiResponse.success(null); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/mapper/ConvertTemplateMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/mapper/ConvertTemplateMapper.java new file mode 100644 index 0000000..f47665c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/mapper/ConvertTemplateMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.convert.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.convert.model.entity.ConvertTemplateEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface ConvertTemplateMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java new file mode 100644 index 0000000..5aecfa2 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.convert.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "格式转换执行请求") +public class ConvertRunRequest { + + @Valid + @NotEmpty(message = "请先上传待转换文件") + @Schema(description = "已上传的源文件列表") + private List files; + + @NotBlank(message = "请选择模板") + @Schema(description = "模板编码，例如 uk_offer") + private String templateId; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertTemplateImportRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertTemplateImportRequest.java new file mode 100644 index 0000000..e07fb2f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertTemplateImportRequest.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.convert.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "格式转换模板导入请求") +public class ConvertTemplateImportRequest { + + @Schema(description = "模板显示名称") + private String templateName; + + @Schema(description = "模板文本内容") + private String templateContent; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java new file mode 100644 index 0000000..f4205c6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java @@ -0,0 +1,17 @@ +package com.nanri.aiimage.modules.convert.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +@Data +@Schema(description = "已上传源文件") +public class UploadedSourceFileDto { + + @NotBlank(message = "fileKey 不能为空") + @Schema(description = "上传接口返回的临时文件键") + private String fileKey; + + @Schema(description = "原始文件名") + private String originalFilename; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/entity/ConvertTemplateEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/entity/ConvertTemplateEntity.java new file mode 100644 index 0000000..0cb2be5 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/entity/ConvertTemplateEntity.java @@ -0,0 +1,30 @@ +package com.nanri.aiimage.modules.convert.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_convert_template") +public class ConvertTemplateEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private String templateCode; + private String templateName; + private String outputFilename; + private String requiredSourceColumnsJson; + private String headerColumnsJson; + private String fieldMappingJson; + private String defaultsJson; + private String preambleLinesJson; + private Integer blankHeaderRowsAfterSchema; + private Integer isDefault; + private Integer builtIn; + private Integer enabled; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertHistoryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertHistoryVo.java new file mode 100644 index 0000000..caf338e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertHistoryVo.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.convert.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "格式转换历史列表") +public class ConvertHistoryVo { + + @Schema(description = "历史记录") + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertResultItemVo.java new file mode 100644 index 0000000..a6f0393 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertResultItemVo.java @@ -0,0 +1,27 @@ +package com.nanri.aiimage.modules.convert.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "格式转换结果项") +public class ConvertResultItemVo { + + @Schema(description = "结果记录ID") + private Long resultId; + + @Schema(description = "源文件名") + private String sourceFilename; + + @Schema(description = "结果文件名") + private String outputFilename; + + @Schema(description = "是否成功") + private boolean success; + + @Schema(description = "错误信息") + private String error; + + @Schema(description = "下载地址") + private String downloadUrl; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertRunVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertRunVo.java new file mode 100644 index 0000000..4215265 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertRunVo.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.convert.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "格式转换执行结果") +public class ConvertRunVo { + + @Schema(description = "处理文件总数") + private Integer total; + + @Schema(description = "成功文件数") + private Integer successCount; + + @Schema(description = "失败文件数") + private Integer failedCount; + + @Schema(description = "结果列表") + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertTemplateVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertTemplateVo.java new file mode 100644 index 0000000..153fd41 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/vo/ConvertTemplateVo.java @@ -0,0 +1,27 @@ +package com.nanri.aiimage.modules.convert.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "格式转换模板") +public class ConvertTemplateVo { + + @Schema(description = "模板编码") + private String id; + + @Schema(description = "模板编码") + private String templateCode; + + @Schema(description = "模板名称") + private String templateName; + + @Schema(description = "输出文件名") + private String outputFilename; + + @Schema(description = "是否默认模板") + private Boolean isDefault; + + @Schema(description = "是否内置模板") + private Boolean builtIn; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java new file mode 100644 index 0000000..64e3d97 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java @@ -0,0 +1,317 @@ +package com.nanri.aiimage.modules.convert.service; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.json.JSONUtil; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest; +import com.nanri.aiimage.modules.convert.model.dto.UploadedSourceFileDto; +import com.nanri.aiimage.modules.convert.model.entity.ConvertTemplateEntity; +import com.nanri.aiimage.modules.convert.model.vo.ConvertResultItemVo; +import com.nanri.aiimage.modules.convert.model.vo.ConvertRunVo; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import lombok.RequiredArgsConstructor; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; + +@Service +@RequiredArgsConstructor +public class ConvertRunService { + + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final StorageProperties storageProperties; + private final ConvertTemplateService convertTemplateService; + private final ObjectMapper objectMapper; + private final OssStorageService ossStorageService; + + public ConvertRunVo run(ConvertRunRequest request) { + ConvertTemplateEntity template = convertTemplateService.getByCode(request.getTemplateId()); + + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo("CONVERT-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType("CONVERT"); + task.setTaskMode("IMMEDIATE"); + task.setStatus("SUCCESS"); + task.setSourceFileCount(request.getFiles().size()); + task.setCreatedBy("system"); + task.setRequestJson(JSONUtil.toJsonStr(request)); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.insert(task); + + List items = new ArrayList<>(); + int successCount = 0; + int failedCount = 0; + + for (UploadedSourceFileDto sourceFile : request.getFiles()) { + ConvertResultItemVo item = new ConvertResultItemVo(); + item.setSourceFilename(sourceFile.getOriginalFilename()); + try { + File inputFile = findLocalSourceFile(sourceFile.getFileKey()); + if (inputFile == null || !inputFile.exists()) { + throw new BusinessException("上传文件不存在，请重新上传"); + } + + String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); + String outputFilename = template.getOutputFilename() == null || template.getOutputFilename().isBlank() + ? "结果.txt" + : template.getOutputFilename(); + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result")); + File outputFile = buildNamedOutputFile(outputDir, outputFilename); + + writeTxtByLegacyRules(inputFile, outputFile, template); + + String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "CONVERT"); + String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + + item.setSuccess(true); + item.setOutputFilename(outputFilename); + item.setDownloadUrl(downloadUrl); + successCount++; + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("CONVERT"); + resultEntity.setSourceFilename(inputName); + resultEntity.setResultFilename(outputFilename); + resultEntity.setResultFileUrl(downloadUrl); + resultEntity.setResultFileSize(outputFile.length()); + resultEntity.setResultContentType("text/plain"); + resultEntity.setSuccess(1); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + item.setResultId(resultEntity.getId()); + } catch (Exception ex) { + item.setSuccess(false); + item.setError(ex.getMessage()); + failedCount++; + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("CONVERT"); + resultEntity.setSourceFilename(sourceFile.getOriginalFilename()); + resultEntity.setSuccess(0); + resultEntity.setErrorMessage(ex.getMessage()); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + } + items.add(item); + } + + task.setSuccessFileCount(successCount); + task.setFailedFileCount(failedCount); + task.setResultJson(JSONUtil.toJsonStr(items)); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + + ConvertRunVo vo = new ConvertRunVo(); + vo.setTotal(request.getFiles().size()); + vo.setSuccessCount(successCount); + vo.setFailedCount(failedCount); + vo.setItems(items); + return vo; + } + + public List listHistory() { + return fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, "CONVERT") + .eq(FileResultEntity::getSuccess, 1) + .orderByDesc(FileResultEntity::getCreatedAt) + .last("limit 50")) + .stream() + .map(entity -> { + ConvertResultItemVo vo = new ConvertResultItemVo(); + vo.setResultId(entity.getId()); + vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", "")); + vo.setOutputFilename(entity.getResultFilename()); + vo.setDownloadUrl(entity.getResultFileUrl()); + vo.setSuccess(true); + return vo; + }) + .toList(); + } + + public void deleteHistory(Long resultId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || !"CONVERT".equals(entity.getModuleType())) { + throw new BusinessException("记录不存在"); + } + fileResultMapper.deleteById(resultId); + } + + public File getResultFile(Long resultId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || entity.getResultFileUrl() == null || !"CONVERT".equals(entity.getModuleType())) { + throw new BusinessException("结果文件不存在"); + } + File file = new File(entity.getResultFileUrl()); + if (!file.exists()) { + throw new BusinessException("结果文件不存在"); + } + return file; + } + + private void writeTxtByLegacyRules(File inputFile, File outputFile, ConvertTemplateEntity templateEntity) throws IOException { + List requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson()); + List headerColumns = readStringList(templateEntity.getHeaderColumnsJson()); + Map fieldMapping = readStringMap(templateEntity.getFieldMappingJson()); + Map defaults = readStringMap(templateEntity.getDefaultsJson()); + List preambleLines = readStringList(templateEntity.getPreambleLinesJson()); + int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null ? 0 : templateEntity.getBlankHeaderRowsAfterSchema(); + + DataFormatter formatter = new DataFormatter(); + try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + throw new BusinessException("Excel 表头为空"); + } + + Map headerMap = new HashMap<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i))); + if (!value.isBlank() && !headerMap.containsKey(value)) { + headerMap.put(value, i); + } + } + + List missing = requiredColumns.stream().filter(column -> !headerMap.containsKey(column)).toList(); + if (!missing.isEmpty()) { + throw new BusinessException("缺少列：" + String.join("、", missing)); + } + + List rows = new ArrayList<>(); + rows.addAll(preambleLines); + rows.add(String.join("\t", headerColumns)); + for (int i = 0; i < blankHeaderRowsAfterSchema; i++) { + rows.add("\t".repeat(Math.max(0, headerColumns.size() - 1))); + } + + long batchId = System.currentTimeMillis(); + int rowIndex = 1; + int asinIndex = headerMap.getOrDefault("ASIN", -1); + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + continue; + } + String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : ""; + if (asin.isBlank()) { + continue; + } + + List lineValues = new ArrayList<>(); + for (String column : headerColumns) { + if ("sku".equals(column)) { + lineValues.add(batchId + "-" + rowIndex); + continue; + } + if (fieldMapping.containsKey(column)) { + String sourceColumn = fieldMapping.get(column); + Integer sourceIndex = headerMap.get(sourceColumn); + String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); + lineValues.add(value); + continue; + } + if (defaults.containsKey(column)) { + lineValues.add(defaults.get(column)); + continue; + } + lineValues.add(""); + } + rows.add(String.join("\t", lineValues)); + rowIndex++; + } + FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8); + } + } + + private List readStringList(String json) { + try { + if (json == null || json.isBlank()) { + return new ArrayList<>(); + } + return objectMapper.readValue(json, new TypeReference>() {}); + } catch (Exception ex) { + throw new BusinessException("模板配置解析失败"); + } + } + + private Map readStringMap(String json) { + try { + if (json == null || json.isBlank()) { + return new HashMap<>(); + } + return objectMapper.readValue(json, new TypeReference>() {}); + } catch (Exception ex) { + throw new BusinessException("模板配置解析失败"); + } + } + + private File findLocalSourceFile(String fileKey) { + File baseDir = FileUtil.file(storageProperties.getLocalTempDir()); + if (!baseDir.exists()) { + return null; + } + List matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); + return matchedFiles.isEmpty() ? null : matchedFiles.getFirst(); + } + + private File buildNamedOutputFile(File outputDir, String filename) { + File candidate = FileUtil.file(outputDir, filename); + if (!candidate.exists()) { + return candidate; + } + String mainName = FileUtil.mainName(filename); + String extName = FileUtil.extName(filename); + int index = 2; + while (true) { + String nextFilename = extName == null || extName.isBlank() + ? mainName + "_" + index + : mainName + "_" + index + "." + extName; + File nextCandidate = FileUtil.file(outputDir, nextFilename); + if (!nextCandidate.exists()) { + return nextCandidate; + } + index++; + } + } + + private String normalizeCellText(String value) { + if (value == null) { + return ""; + } + return value.replace("\ufeff", "") + .replace("\u3000", " ") + .replace("\r\n", " ") + .replace("\r", " ") + .replace("\n", " ") + .replace("\t", " ") + .trim() + .replaceAll("\\s+", " "); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertTemplateService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertTemplateService.java new file mode 100644 index 0000000..778e720 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertTemplateService.java @@ -0,0 +1,99 @@ +package com.nanri.aiimage.modules.convert.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.convert.mapper.ConvertTemplateMapper; +import com.nanri.aiimage.modules.convert.model.dto.ConvertTemplateImportRequest; +import com.nanri.aiimage.modules.convert.model.entity.ConvertTemplateEntity; +import com.nanri.aiimage.modules.convert.model.vo.ConvertTemplateVo; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class ConvertTemplateService { + + private final ConvertTemplateMapper convertTemplateMapper; + + public List listEnabledTemplates() { + return convertTemplateMapper.selectList(new LambdaQueryWrapper() + .eq(ConvertTemplateEntity::getEnabled, 1) + .orderByDesc(ConvertTemplateEntity::getIsDefault) + .orderByAsc(ConvertTemplateEntity::getId)) + .stream() + .map(this::toVo) + .toList(); + } + + public ConvertTemplateEntity getByCode(String templateCode) { + ConvertTemplateEntity entity = convertTemplateMapper.selectOne(new LambdaQueryWrapper() + .eq(ConvertTemplateEntity::getTemplateCode, templateCode) + .eq(ConvertTemplateEntity::getEnabled, 1) + .last("limit 1")); + if (entity == null) { + throw new BusinessException("模板不存在或已禁用"); + } + return entity; + } + + @Transactional + public ConvertTemplateVo importTemplate(ConvertTemplateImportRequest request) { + if (request.getTemplateContent() == null || request.getTemplateContent().isBlank()) { + throw new BusinessException("模板内容不能为空"); + } + ConvertTemplateEntity entity = new ConvertTemplateEntity(); + entity.setTemplateCode("user_" + System.currentTimeMillis()); + entity.setTemplateName((request.getTemplateName() == null || request.getTemplateName().isBlank()) ? "自定义 Offer 模板" : request.getTemplateName().trim()); + entity.setOutputFilename(entity.getTemplateName().endsWith(".txt") ? entity.getTemplateName() : entity.getTemplateName() + ".txt"); + entity.setRequiredSourceColumnsJson("[\"ASIN\",\"价格\"]"); + entity.setHeaderColumnsJson("[\"sku\",\"price\",\"quantity\",\"product-id\",\"product-id-type\",\"condition-type\",\"condition-note\",\"ASIN-hint\",\"title\",\"product-tax-code\",\"operation-type\",\"sale-price\",\"sale-start-date\",\"sale-end-date\",\"leadtime-to-ship\",\"launch-date\",\"is-giftwrap-available\",\"is-gift-message-available\"]"); + entity.setFieldMappingJson("{\"price\":\"价格\",\"product-id\":\"ASIN\"}"); + entity.setDefaultsJson("{\"quantity\":\"100\",\"product-id-type\":\"ASIN\",\"condition-type\":\"new\",\"leadtime-to-ship\":\"4\"}"); + entity.setPreambleLinesJson("[\"TemplateType=Offer\\tVersion=1.4\"]"); + entity.setBlankHeaderRowsAfterSchema(1); + entity.setIsDefault(0); + entity.setBuiltIn(0); + entity.setEnabled(1); + entity.setCreatedAt(LocalDateTime.now()); + entity.setUpdatedAt(LocalDateTime.now()); + convertTemplateMapper.insert(entity); + return toVo(entity); + } + + @Transactional + public void setDefaultTemplate(String templateCode) { + ConvertTemplateEntity entity = getByCode(templateCode); + convertTemplateMapper.selectList(new LambdaQueryWrapper().eq(ConvertTemplateEntity::getEnabled, 1)) + .forEach(item -> { + item.setIsDefault(item.getId().equals(entity.getId()) ? 1 : 0); + item.setUpdatedAt(LocalDateTime.now()); + convertTemplateMapper.updateById(item); + }); + } + + @Transactional + public void deleteTemplate(String templateCode) { + ConvertTemplateEntity entity = getByCode(templateCode); + if (Integer.valueOf(1).equals(entity.getBuiltIn())) { + throw new BusinessException("内置模板不允许删除"); + } + entity.setEnabled(0); + entity.setUpdatedAt(LocalDateTime.now()); + convertTemplateMapper.updateById(entity); + } + + private ConvertTemplateVo toVo(ConvertTemplateEntity entity) { + ConvertTemplateVo vo = new ConvertTemplateVo(); + vo.setId(entity.getTemplateCode()); + vo.setTemplateCode(entity.getTemplateCode()); + vo.setTemplateName(entity.getTemplateName()); + vo.setOutputFilename(entity.getOutputFilename()); + vo.setIsDefault(Integer.valueOf(1).equals(entity.getIsDefault())); + vo.setBuiltIn(Integer.valueOf(1).equals(entity.getBuiltIn())); + return vo; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java new file mode 100644 index 0000000..6cf2987 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java @@ -0,0 +1,74 @@ +package com.nanri.aiimage.modules.dedupe.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeHistoryVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo; +import com.nanri.aiimage.modules.dedupe.service.DedupeRunService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/dedupe") +@Tag(name = "数据去重", description = "按照旧 Python 桌面端规则执行列保留和 ID 过滤。桌面端交互不变，只将处理逻辑迁到 Java。") +public class DedupeRunController { + + private final DedupeRunService dedupeRunService; + + @PostMapping("/run") + @Operation( + summary = "执行数据去重", + description = """ + 根据上传文件、保留列、ID 保留规则执行数据去重，生成清洗后的 Excel 结果文件。 + + 规则与旧 Python 桌面端保持一致： + - 表头会先做文本清洗； + - 遇到“缩略图地址8”停止继续识别表头； + - 仅保留 selectedColumns 中指定的列； + - keepIntegerIds=true 时保留纯数字 ID； + - keepUnderscoreIds=true 时保留类似 1_1 的 ID； + - 输出文件命名遵循 原文件名_cleaned.xlsx，若重名自动追加序号。 + """) + public ApiResponse run(@Valid @RequestBody DedupeRunRequest request) { + return ApiResponse.success(dedupeRunService.run(request)); + } + + @GetMapping("/history") + @Operation(summary = "查询去重历史", description = "查询最近的去重成功记录，用于页面右侧历史下载列表展示。") + public ApiResponse history() { + DedupeHistoryVo vo = new DedupeHistoryVo(); + vo.setItems(dedupeRunService.listHistory()); + return ApiResponse.success(vo); + } + + @DeleteMapping("/history/{resultId}") + @Operation(summary = "删除去重历史", description = "删除一条去重历史记录。") + public ApiResponse deleteHistory(@PathVariable Long resultId) { + dedupeRunService.deleteHistory(resultId); + return ApiResponse.success(null); + } + + @GetMapping("/results/{resultId}/download") + @Operation(summary = "下载数据去重结果", description = "根据结果记录 ID 下载清洗后的 Excel 文件。") + public ResponseEntity download(@PathVariable Long resultId) { + Resource resource = dedupeRunService.getResultFile(resultId); + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment") + .body(resource); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java new file mode 100644 index 0000000..63baff1 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.dedupe.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "数据去重执行请求") +public class DedupeRunRequest { + + @Valid + @NotEmpty(message = "请先上传待处理文件") + @Schema(description = "已上传源文件列表") + private List files; + + @NotEmpty(message = "请至少选择一列") + @Schema(description = "保留列") + private List selectedColumns; + + @Schema(description = "是否保留纯数字 ID") + private boolean keepIntegerIds = true; + + @Schema(description = "是否保留带下划线 ID") + private boolean keepUnderscoreIds = true; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java new file mode 100644 index 0000000..48d1720 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java @@ -0,0 +1,17 @@ +package com.nanri.aiimage.modules.dedupe.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +@Data +@Schema(description = "数据去重源文件") +public class DedupeSourceFileDto { + + @NotBlank(message = "fileKey 不能为空") + @Schema(description = "上传接口返回的临时文件键") + private String fileKey; + + @Schema(description = "原始文件名") + private String originalFilename; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeHistoryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeHistoryVo.java new file mode 100644 index 0000000..6c1de57 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeHistoryVo.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.dedupe.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "数据去重历史列表") +public class DedupeHistoryVo { + + @Schema(description = "历史记录") + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeResultItemVo.java new file mode 100644 index 0000000..bb1c4dd --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeResultItemVo.java @@ -0,0 +1,27 @@ +package com.nanri.aiimage.modules.dedupe.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "数据去重结果项") +public class DedupeResultItemVo { + + @Schema(description = "结果记录ID") + private Long resultId; + + @Schema(description = "源文件名") + private String sourceFilename; + + @Schema(description = "结果文件名") + private String outputFilename; + + @Schema(description = "是否成功") + private boolean success; + + @Schema(description = "错误信息") + private String error; + + @Schema(description = "下载地址") + private String downloadUrl; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeRunVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeRunVo.java new file mode 100644 index 0000000..afd3191 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeRunVo.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.dedupe.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "数据去重执行结果") +public class DedupeRunVo { + + @Schema(description = "处理文件总数") + private Integer total; + + @Schema(description = "成功文件数") + private Integer successCount; + + @Schema(description = "失败文件数") + private Integer failedCount; + + @Schema(description = "结果列表") + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java new file mode 100644 index 0000000..f2d8755 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -0,0 +1,303 @@ +package com.nanri.aiimage.modules.dedupe.service; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.json.JSONUtil; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeResultItemVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import lombok.RequiredArgsConstructor; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; + +@Service +@RequiredArgsConstructor +public class DedupeRunService { + + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final StorageProperties storageProperties; + private final OssStorageService ossStorageService; + + public DedupeRunVo run(DedupeRunRequest request) { + if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds()) { + throw new BusinessException("请至少选择一种 ID 保留规则"); + } + + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType("DEDUPE"); + task.setTaskMode("IMMEDIATE"); + task.setStatus("SUCCESS"); + task.setSourceFileCount(request.getFiles().size()); + task.setCreatedBy("system"); + task.setRequestJson(JSONUtil.toJsonStr(request)); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.insert(task); + + List items = new ArrayList<>(); + int successCount = 0; + int failedCount = 0; + + for (DedupeSourceFileDto sourceFile : request.getFiles()) { + DedupeResultItemVo item = new DedupeResultItemVo(); + item.setSourceFilename(sourceFile.getOriginalFilename()); + try { + File inputFile = findLocalSourceFile(sourceFile.getFileKey()); + if (inputFile == null || !inputFile.exists()) { + throw new BusinessException("上传文件不存在，请重新上传"); + } + + String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); + String outputFilename = buildOutputFilename(inputName); + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); + File outputFile = buildNamedOutputFile(outputDir, outputFilename); + + cleanExcelByLegacyRules(inputFile, outputFile, request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds()); + + String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE"); + String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + + item.setSuccess(true); + item.setOutputFilename(outputFile.getName()); + item.setDownloadUrl(downloadUrl); + successCount++; + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("DEDUPE"); + resultEntity.setSourceFilename(inputName); + resultEntity.setResultFilename(outputFile.getName()); + resultEntity.setResultFileUrl(downloadUrl); + resultEntity.setResultFileSize(outputFile.length()); + resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + resultEntity.setSuccess(1); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + } catch (Exception ex) { + item.setSuccess(false); + item.setError(ex.getMessage()); + failedCount++; + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("DEDUPE"); + resultEntity.setSourceFilename(sourceFile.getOriginalFilename()); + resultEntity.setSuccess(0); + resultEntity.setErrorMessage(ex.getMessage()); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + } + items.add(item); + } + + task.setSuccessFileCount(successCount); + task.setFailedFileCount(failedCount); + task.setResultJson(JSONUtil.toJsonStr(items)); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + + DedupeRunVo vo = new DedupeRunVo(); + vo.setTotal(request.getFiles().size()); + vo.setSuccessCount(successCount); + vo.setFailedCount(failedCount); + vo.setItems(items); + return vo; + } + + public List listHistory() { + return fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, "DEDUPE") + .eq(FileResultEntity::getSuccess, 1) + .orderByDesc(FileResultEntity::getCreatedAt) + .last("limit 50")) + .stream() + .map(entity -> { + DedupeResultItemVo vo = new DedupeResultItemVo(); + vo.setResultId(entity.getId()); + vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", "")); + vo.setOutputFilename(entity.getResultFilename()); + vo.setDownloadUrl(entity.getResultFileUrl()); + vo.setSuccess(true); + return vo; + }) + .toList(); + } + + public void deleteHistory(Long resultId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || !"DEDUPE".equals(entity.getModuleType())) { + throw new BusinessException("记录不存在"); + } + fileResultMapper.deleteById(resultId); + } + + public Resource getResultFile(Long resultId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType())) { + throw new BusinessException("结果文件不存在"); + } + File file = new File(entity.getResultFileUrl()); + if (!file.exists()) { + throw new BusinessException("结果文件不存在"); + } + return new FileSystemResource(file); + } + + private void cleanExcelByLegacyRules(File inputFile, File outputFile, List selectedColumns, + boolean keepIntegerIds, boolean keepUnderscoreIds) throws Exception { + DataFormatter formatter = new DataFormatter(); + try (FileInputStream fis = new FileInputStream(inputFile); + Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis); + XSSFWorkbook outputWorkbook = new XSSFWorkbook()) { + Sheet sheet = workbook.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + throw new BusinessException("Excel 表头为空"); + } + + Map headerMap = new HashMap<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell)); + if (value.isBlank()) { + continue; + } + value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty() + ? value + : value.split("idASIN国家状态价格变体数量", 2)[0].trim(); + if (headerMap.containsKey(value)) { + continue; + } + headerMap.put(value, i); + if ("缩略图地址8".equals(value)) { + break; + } + } + + List missing = selectedColumns.stream().filter(column -> !headerMap.containsKey(column)).toList(); + if (!missing.isEmpty()) { + throw new BusinessException("缺少列：" + String.join("、", missing)); + } + + Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName()); + Row outputHeaderRow = outputSheet.createRow(0); + for (int i = 0; i < selectedColumns.size(); i++) { + outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i)); + } + + Integer idColumnIndex = headerMap.get("id"); + int outputRowIndex = 1; + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + continue; + } + if (idColumnIndex != null) { + String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))); + if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds)) { + continue; + } + } + + Row outputRow = outputSheet.createRow(outputRowIndex++); + for (int i = 0; i < selectedColumns.size(); i++) { + Integer sourceIndex = headerMap.get(selectedColumns.get(i)); + String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); + outputRow.createCell(i).setCellValue(value); + } + } + + try (FileOutputStream fos = new FileOutputStream(outputFile)) { + outputWorkbook.write(fos); + } + } + } + + private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds) { + if (text == null || text.isBlank()) { + return false; + } + if (keepIntegerIds && text.matches("\\d+")) { + return true; + } + return keepUnderscoreIds && text.matches("\\d+_\\d+"); + } + + private File findLocalSourceFile(String fileKey) { + File baseDir = FileUtil.file(storageProperties.getLocalTempDir()); + if (!baseDir.exists()) { + return null; + } + List matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); + return matchedFiles.isEmpty() ? null : matchedFiles.getFirst(); + } + + private File buildNamedOutputFile(File outputDir, String filename) { + File candidate = FileUtil.file(outputDir, filename); + if (!candidate.exists()) { + return candidate; + } + String mainName = FileUtil.mainName(filename); + String extName = FileUtil.extName(filename); + int index = 2; + while (true) { + String nextFilename = extName == null || extName.isBlank() + ? mainName + "_" + index + : mainName + "_" + index + "." + extName; + File nextCandidate = FileUtil.file(outputDir, nextFilename); + if (!nextCandidate.exists()) { + return nextCandidate; + } + index++; + } + } + + private String buildOutputFilename(String originalFilename) { + String mainName = originalFilename == null || originalFilename.isBlank() ? "source" : FileUtil.mainName(originalFilename); + String extName = originalFilename == null || originalFilename.isBlank() ? "xlsx" : FileUtil.extName(originalFilename); + return mainName + "_cleaned." + (extName == null || extName.isBlank() ? "xlsx" : extName); + } + + private String normalizeCellText(String value) { + if (value == null) { + return ""; + } + return value.replace("\ufeff", "") + .replace("\u3000", " ") + .replace("\r\n", " ") + .replace("\r", " ") + .replace("\n", " ") + .replace("\t", " ") + .trim() + .replaceAll("\\s+", " "); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java new file mode 100644 index 0000000..19d14a0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java @@ -0,0 +1,37 @@ +package com.nanri.aiimage.modules.file.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.file.model.vo.UploadFileVo; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/files") +@Tag(name = "文件上传", description = "用于前端将本地文件上传到后端临时目录，后续任务接口基于 fileKey 进行处理") +public class FileUploadController { + + private final LocalFileStorageService localFileStorageService; + + @PostMapping("/upload") + @Operation( + summary = "上传临时文件", + description = """ + 上传本地 Excel 文件到后端临时目录，返回 fileKey。 + + 该接口不是给浏览器页面直接使用的最终业务接口，而是给 Python 桌面端壳做文件中转： + - 桌面端先选本地文件； + - 再调用本接口上传到 Java 临时目录； + - 然后把 fileKey 传给 dedupe/convert/split 执行接口； + - 处理完成后，桌面端再调用下载接口把结果文件保存回用户本地目录。 + """) + public ApiResponse upload(MultipartFile file) throws Exception { + return ApiResponse.success(localFileStorageService.saveTempFile(file)); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java new file mode 100644 index 0000000..de83fe0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.modules.file.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "上传文件返回信息") +public class UploadFileVo { + + @Schema(description = "临时文件键") + private String fileKey; + + @Schema(description = "原始文件名") + private String originalFilename; + + @Schema(description = "本地暂存路径") + private String localPath; + + @Schema(description = "文件大小") + private Long size; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java new file mode 100644 index 0000000..6236bff --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java @@ -0,0 +1,39 @@ +package com.nanri.aiimage.modules.file.service; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.IdUtil; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.file.model.vo.UploadFileVo; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.io.IOException; + +@Service +@RequiredArgsConstructor +public class LocalFileStorageService { + + private final StorageProperties storageProperties; + + public UploadFileVo saveTempFile(MultipartFile file) throws IOException { + File tempDir = FileUtil.file(storageProperties.getLocalTempDir()); + File parentDir = tempDir.getParentFile(); + if (parentDir != null) { + FileUtil.mkdir(parentDir); + } + FileUtil.mkdir(tempDir); + String fileKey = IdUtil.fastSimpleUUID(); + String extName = FileUtil.extName(file.getOriginalFilename()); + File target = FileUtil.file(tempDir, fileKey + (extName.isEmpty() ? "" : "." + extName)); + file.transferTo(target); + + UploadFileVo vo = new UploadFileVo(); + vo.setFileKey(fileKey); + vo.setOriginalFilename(file.getOriginalFilename()); + vo.setLocalPath(target.getAbsolutePath()); + vo.setSize(file.getSize()); + return vo; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java new file mode 100644 index 0000000..bc75374 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -0,0 +1,49 @@ +package com.nanri.aiimage.modules.file.service.oss; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.nanri.aiimage.config.OssProperties; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.net.URL; +import java.util.Date; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +public class OssStorageService { + + private final OssProperties ossProperties; + + public String uploadResultFile(File file, String moduleType) { + String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName()); + OSS ossClient = new OSSClientBuilder().build( + "https://" + ossProperties.getEndpoint(), + ossProperties.getAccessKeyId(), + ossProperties.getAccessKeySecret() + ); + try { + ossClient.putObject(ossProperties.getBucket(), objectKey, file); + return objectKey; + } finally { + ossClient.shutdown(); + } + } + + public String generateDownloadUrl(String objectKey) { + OSS ossClient = new OSSClientBuilder().build( + "https://" + ossProperties.getEndpoint(), + ossProperties.getAccessKeyId(), + ossProperties.getAccessKeySecret() + ); + try { + Date expiration = new Date(System.currentTimeMillis() + 3600_000L); + URL url = ossClient.generatePresignedUrl(ossProperties.getBucket(), objectKey, expiration); + return url.toString(); + } finally { + ossClient.shutdown(); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java new file mode 100644 index 0000000..2057e1a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java @@ -0,0 +1,75 @@ +package com.nanri.aiimage.modules.split.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest; +import com.nanri.aiimage.modules.split.model.vo.SplitHistoryVo; +import com.nanri.aiimage.modules.split.model.vo.SplitRunVo; +import com.nanri.aiimage.modules.split.service.SplitRunService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/split") +@Tag(name = "数据拆分", description = "按照旧 Python 桌面端规则执行数据拆分。桌面端交互不变，只将处理逻辑迁到 Java。") +public class SplitRunController { + + private final SplitRunService splitRunService; + + @PostMapping("/run") + @Operation( + summary = "执行数据拆分", + description = """ + 根据上传文件、保留列、拆分模式和参数执行 Excel 拆分。 + + 规则与旧 Python 桌面端保持一致： + - 表头做文本清洗并兼容历史脏串； + - 遇到“缩略图地址8”停止继续识别表头； + - 仅保留 selectedColumns 中指定的列； + - splitMode=rows_per_file 时按固定条数切分； + - splitMode=parts 时按总条数尽量均分； + - 输出文件命名遵循 原文件名_split_序号.xlsx； + - 返回结果中 rowCount 表示该结果文件包含的数据行数。 + """) + public ApiResponse run(@Valid @RequestBody SplitRunRequest request) { + return ApiResponse.success(splitRunService.run(request)); + } + + @GetMapping("/history") + @Operation(summary = "查询数据拆分历史", description = "查询最近的数据拆分成功记录，用于页面右侧历史下载列表展示。") + public ApiResponse history() { + SplitHistoryVo vo = new SplitHistoryVo(); + vo.setItems(splitRunService.listHistory()); + return ApiResponse.success(vo); + } + + @DeleteMapping("/history/{resultId}") + @Operation(summary = "删除数据拆分历史", description = "删除一条数据拆分历史记录。") + public ApiResponse deleteHistory(@PathVariable Long resultId) { + splitRunService.deleteHistory(resultId); + return ApiResponse.success(null); + } + + @GetMapping("/results/{resultId}/download") + @Operation(summary = "下载数据拆分结果", description = "根据结果记录 ID 下载拆分后的 Excel 文件。") + public ResponseEntity download(@PathVariable Long resultId) { + Resource resource = splitRunService.getResultFile(resultId); + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment") + .body(resource); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java new file mode 100644 index 0000000..de8fefb --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java @@ -0,0 +1,33 @@ +package com.nanri.aiimage.modules.split.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "数据拆分执行请求") +public class SplitRunRequest { + + @Valid + @NotEmpty(message = "请先上传待拆分文件") + @Schema(description = "已上传源文件列表") + private List files; + + @NotEmpty(message = "请至少选择一列") + @Schema(description = "保留列") + private List selectedColumns; + + @NotBlank(message = "拆分模式不能为空") + @Schema(description = "拆分模式：rows_per_file / parts") + private String splitMode; + + @Schema(description = "按每份条数拆分时的条数") + private Integer rowsPerFile; + + @Schema(description = "按份数拆分时的份数") + private Integer parts; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java new file mode 100644 index 0000000..2316ba4 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java @@ -0,0 +1,17 @@ +package com.nanri.aiimage.modules.split.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +@Data +@Schema(description = "数据拆分源文件") +public class SplitSourceFileDto { + + @NotBlank(message = "fileKey 不能为空") + @Schema(description = "上传接口返回的临时文件键") + private String fileKey; + + @Schema(description = "原始文件名") + private String originalFilename; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitHistoryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitHistoryVo.java new file mode 100644 index 0000000..158808f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitHistoryVo.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.split.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "数据拆分历史列表") +public class SplitHistoryVo { + + @Schema(description = "历史记录") + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java new file mode 100644 index 0000000..95e1c3d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java @@ -0,0 +1,30 @@ +package com.nanri.aiimage.modules.split.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "数据拆分结果项") +public class SplitResultItemVo { + + @Schema(description = "结果记录ID") + private Long resultId; + + @Schema(description = "源文件名") + private String sourceFilename; + + @Schema(description = "结果文件名") + private String outputFilename; + + @Schema(description = "是否成功") + private boolean success; + + @Schema(description = "错误信息") + private String error; + + @Schema(description = "结果行数") + private Integer rowCount; + + @Schema(description = "下载地址") + private String downloadUrl; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitRunVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitRunVo.java new file mode 100644 index 0000000..613f30a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitRunVo.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.split.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "数据拆分执行结果") +public class SplitRunVo { + + @Schema(description = "处理文件总数") + private Integer total; + + @Schema(description = "成功文件数") + private Integer successCount; + + @Schema(description = "失败文件数") + private Integer failedCount; + + @Schema(description = "结果列表") + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java new file mode 100644 index 0000000..555b546 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java @@ -0,0 +1,333 @@ +package com.nanri.aiimage.modules.split.service; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.json.JSONUtil; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest; +import com.nanri.aiimage.modules.split.model.dto.SplitSourceFileDto; +import com.nanri.aiimage.modules.split.model.vo.SplitResultItemVo; +import com.nanri.aiimage.modules.split.model.vo.SplitRunVo; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import lombok.RequiredArgsConstructor; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; + +@Service +@RequiredArgsConstructor +public class SplitRunService { + + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final StorageProperties storageProperties; + private final OssStorageService ossStorageService; + + public SplitRunVo run(SplitRunRequest request) { + validateRequest(request); + + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo("SPLIT-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType("SPLIT"); + task.setTaskMode("IMMEDIATE"); + task.setStatus("SUCCESS"); + task.setSourceFileCount(request.getFiles().size()); + task.setCreatedBy("system"); + task.setRequestJson(JSONUtil.toJsonStr(request)); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.insert(task); + + List items = new ArrayList<>(); + int successCount = 0; + int failedCount = 0; + + for (SplitSourceFileDto sourceFile : request.getFiles()) { + try { + File inputFile = findLocalSourceFile(sourceFile.getFileKey()); + if (inputFile == null || !inputFile.exists()) { + throw new BusinessException("上传文件不存在，请重新上传"); + } + String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); + List chunkResults = splitExcelByLegacyRules(inputFile, inputName, request); + for (SplitChunkResult chunkResult : chunkResults) { + SplitResultItemVo item = new SplitResultItemVo(); + item.setSourceFilename(inputName); + item.setOutputFilename(chunkResult.outputFile().getName()); + item.setSuccess(true); + item.setRowCount(chunkResult.rowCount()); + + String ossObjectKey = ossStorageService.uploadResultFile(chunkResult.outputFile(), "SPLIT"); + String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("SPLIT"); + resultEntity.setSourceFilename(inputName); + resultEntity.setResultFilename(chunkResult.outputFile().getName()); + resultEntity.setResultFileUrl(downloadUrl); + resultEntity.setResultFileSize(chunkResult.outputFile().length()); + resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + resultEntity.setRowCount(chunkResult.rowCount()); + resultEntity.setSuccess(1); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + item.setDownloadUrl(downloadUrl); + item.setResultId(resultEntity.getId()); + items.add(item); + } + successCount++; + } catch (Exception ex) { + SplitResultItemVo item = new SplitResultItemVo(); + item.setSourceFilename(sourceFile.getOriginalFilename()); + item.setSuccess(false); + item.setError(ex.getMessage()); + items.add(item); + failedCount++; + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("SPLIT"); + resultEntity.setSourceFilename(sourceFile.getOriginalFilename()); + resultEntity.setSuccess(0); + resultEntity.setErrorMessage(ex.getMessage()); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + } + } + + task.setSuccessFileCount(successCount); + task.setFailedFileCount(failedCount); + task.setResultJson(JSONUtil.toJsonStr(items)); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + + SplitRunVo vo = new SplitRunVo(); + vo.setTotal(request.getFiles().size()); + vo.setSuccessCount(successCount); + vo.setFailedCount(failedCount); + vo.setItems(items); + return vo; + } + + public List listHistory() { + return fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, "SPLIT") + .eq(FileResultEntity::getSuccess, 1) + .orderByDesc(FileResultEntity::getCreatedAt) + .last("limit 100")) + .stream() + .map(entity -> { + SplitResultItemVo vo = new SplitResultItemVo(); + vo.setResultId(entity.getId()); + vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", "")); + vo.setOutputFilename(entity.getResultFilename()); + vo.setDownloadUrl(entity.getResultFileUrl()); + vo.setRowCount(entity.getRowCount()); + vo.setSuccess(true); + return vo; + }) + .toList(); + } + + public void deleteHistory(Long resultId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || !"SPLIT".equals(entity.getModuleType())) { + throw new BusinessException("记录不存在"); + } + fileResultMapper.deleteById(resultId); + } + + public Resource getResultFile(Long resultId) { + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null || entity.getResultFileUrl() == null || !"SPLIT".equals(entity.getModuleType())) { + throw new BusinessException("结果文件不存在"); + } + File file = new File(entity.getResultFileUrl()); + if (!file.exists()) { + throw new BusinessException("结果文件不存在"); + } + return new FileSystemResource(file); + } + + private List splitExcelByLegacyRules(File inputFile, String originalFilename, SplitRunRequest request) throws Exception { + DataFormatter formatter = new DataFormatter(); + try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + throw new BusinessException("Excel 表头为空"); + } + + Map headerMap = new HashMap<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell)); + if (value.isBlank()) { + continue; + } + value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty() + ? value + : value.split("idASIN国家状态价格变体数量", 2)[0].trim(); + if (headerMap.containsKey(value)) { + continue; + } + headerMap.put(value, i); + if ("缩略图地址8".equals(value)) { + break; + } + } + + List missing = request.getSelectedColumns().stream().filter(column -> !headerMap.containsKey(column)).toList(); + if (!missing.isEmpty()) { + throw new BusinessException("缺少列：" + String.join("、", missing)); + } + + List> rowsData = new ArrayList<>(); + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + continue; + } + List projected = new ArrayList<>(); + for (String column : request.getSelectedColumns()) { + Integer sourceIndex = headerMap.get(column); + String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); + projected.add(value); + } + rowsData.add(projected); + } + + int totalRows = rowsData.size(); + if (totalRows == 0) { + throw new BusinessException("没有可拆分的数据行"); + } + + List>> chunks = new ArrayList<>(); + if ("rows_per_file".equals(request.getSplitMode())) { + int chunkSize = request.getRowsPerFile(); + for (int i = 0; i < totalRows; i += chunkSize) { + chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows))); + } + } else { + int actualParts = Math.min(request.getParts(), totalRows); + int base = totalRows / actualParts; + int remainder = totalRows % actualParts; + int start = 0; + for (int i = 0; i < actualParts; i++) { + int size = base + (i < remainder ? 1 : 0); + int end = start + size; + chunks.add(rowsData.subList(start, end)); + start = end; + } + } + + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result")); + String sourceStem = FileUtil.mainName(originalFilename); + List results = new ArrayList<>(); + for (int i = 0; i < chunks.size(); i++) { + File outputFile = buildNamedOutputFile(outputDir, sourceStem + "_split_" + (i + 1) + ".xlsx"); + try (XSSFWorkbook outputWorkbook = new XSSFWorkbook()) { + Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName()); + Row outputHeaderRow = outputSheet.createRow(0); + for (int colIndex = 0; colIndex < request.getSelectedColumns().size(); colIndex++) { + outputHeaderRow.createCell(colIndex).setCellValue(request.getSelectedColumns().get(colIndex)); + } + int outputRowIndex = 1; + for (List rowValues : chunks.get(i)) { + Row outputRow = outputSheet.createRow(outputRowIndex++); + for (int colIndex = 0; colIndex < rowValues.size(); colIndex++) { + outputRow.createCell(colIndex).setCellValue(rowValues.get(colIndex)); + } + } + try (FileOutputStream fos = new FileOutputStream(outputFile)) { + outputWorkbook.write(fos); + } + } + results.add(new SplitChunkResult(outputFile, chunks.get(i).size())); + } + return results; + } + } + + private void validateRequest(SplitRunRequest request) { + if (!List.of("rows_per_file", "parts").contains(request.getSplitMode())) { + throw new BusinessException("拆分模式不合法"); + } + if ("rows_per_file".equals(request.getSplitMode()) && (request.getRowsPerFile() == null || request.getRowsPerFile() <= 0)) { + throw new BusinessException("每份条数不合法"); + } + if ("parts".equals(request.getSplitMode()) && (request.getParts() == null || request.getParts() <= 0)) { + throw new BusinessException("拆分份数不合法"); + } + } + + private File findLocalSourceFile(String fileKey) { + File baseDir = FileUtil.file(storageProperties.getLocalTempDir()); + if (!baseDir.exists()) { + return null; + } + List matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); + return matchedFiles.isEmpty() ? null : matchedFiles.getFirst(); + } + + private File buildNamedOutputFile(File outputDir, String filename) { + File candidate = FileUtil.file(outputDir, filename); + if (!candidate.exists()) { + return candidate; + } + String mainName = FileUtil.mainName(filename); + String extName = FileUtil.extName(filename); + int index = 2; + while (true) { + String nextFilename = extName == null || extName.isBlank() + ? mainName + "_" + index + : mainName + "_" + index + "." + extName; + File nextCandidate = FileUtil.file(outputDir, nextFilename); + if (!nextCandidate.exists()) { + return nextCandidate; + } + index++; + } + } + + private String normalizeCellText(String value) { + if (value == null) { + return ""; + } + return value.replace("\ufeff", "") + .replace("\u3000", " ") + .replace("\r\n", " ") + .replace("\r", " ") + .replace("\n", " ") + .replace("\t", " ") + .trim() + .replaceAll("\\s+", " "); + } + + private record SplitChunkResult(File outputFile, int rowCount) { + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileResultMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileResultMapper.java new file mode 100644 index 0000000..8df360d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileResultMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface FileResultMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapper.java new file mode 100644 index 0000000..7c3c133 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface FileTaskMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/FileResultEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/FileResultEntity.java new file mode 100644 index 0000000..e6c8359 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/FileResultEntity.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_file_result") +public class FileResultEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + private String moduleType; + private String sourceFilename; + private String sourceFileUrl; + private String resultFilename; + private String resultFileUrl; + private Long resultFileSize; + private String resultContentType; + private Integer rowCount; + private Integer success; + private String errorMessage; + private LocalDateTime createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/FileTaskEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/FileTaskEntity.java new file mode 100644 index 0000000..ae3fdc1 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/FileTaskEntity.java @@ -0,0 +1,30 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_file_task") +public class FileTaskEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private String taskNo; + private String moduleType; + private String taskMode; + private String status; + private Integer sourceFileCount; + private Integer successFileCount; + private Integer failedFileCount; + private String requestJson; + private String resultJson; + private String errorMessage; + private String createdBy; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; + private LocalDateTime finishedAt; +} diff --git a/backend-java/src/main/resources/application-local.example.yml b/backend-java/src/main/resources/application-local.example.yml new file mode 100644 index 0000000..274c75c --- /dev/null +++ b/backend-java/src/main/resources/application-local.example.yml @@ -0,0 +1,16 @@ +# 这个文件只是给你对照看的“环境变量配置示例”，不是 Spring Boot 自动加载的配置文件。 +# 你现在最简单的做法：把下面这些键值复制到 IDEA 的 Run/Debug Configuration -> Environment variables。 + +AIIMAGE_SERVER_PORT=18080 + +AIIMAGE_DB_URL=jdbc:mysql://159.75.121.33:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false +AIIMAGE_DB_USERNAME=AIimage +AIIMAGE_DB_PASSWORD=WTFrb5y6hNLz6hNy + +AIIMAGE_OSS_REGION=cn-hangzhou +AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com +AIIMAGE_OSS_BUCKET=nanri-ai-images +AIIMAGE_OSS_ACCESS_KEY_ID=LTAI5tNpyvzMNz9f2dHarsm8 +AIIMAGE_OSS_ACCESS_KEY_SECRET=bQSZnFH455i8tzyOgeahJmUzwmhynz + +AIIMAGE_STORAGE_LOCAL_TEMP_DIR=F:/project/crawler-plugin/backend-java/data/tmp diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml new file mode 100644 index 0000000..362a417 --- /dev/null +++ b/backend-java/src/main/resources/application.yml @@ -0,0 +1,49 @@ +server: + port: ${AIIMAGE_SERVER_PORT:18080} + +spring: + application: + name: aiimage-backend + config: + import: optional:classpath:application-local.yml + servlet: + multipart: + max-file-size: 200MB + max-request-size: 500MB + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: ${AIIMAGE_DB_URL:jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false} + username: ${AIIMAGE_DB_USERNAME:root} + password: ${AIIMAGE_DB_PASSWORD:change-me} + +management: + health: + db: + enabled: false + +mybatis-plus: + mapper-locations: classpath*:mapper/**/*.xml + configuration: + map-underscore-to-camel-case: true + log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + +springdoc: + api-docs: + enabled: true + swagger-ui: + path: /swagger-ui.html + +knife4j: + enable: true + setting: + language: zh_cn + +aiimage: + oss: + region: ${AIIMAGE_OSS_REGION:cn-hangzhou} + endpoint: ${AIIMAGE_OSS_ENDPOINT:oss-cn-hangzhou.aliyuncs.com} + bucket: ${AIIMAGE_OSS_BUCKET:change-me} + access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:change-me} + access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me} + storage: + local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp} diff --git a/backend-java/src/main/resources/db/V1__init.sql b/backend-java/src/main/resources/db/V1__init.sql new file mode 100644 index 0000000..5b3efb0 --- /dev/null +++ b/backend-java/src/main/resources/db/V1__init.sql @@ -0,0 +1,76 @@ +CREATE TABLE IF NOT EXISTS biz_file_task ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键', + task_no VARCHAR(64) NOT NULL COMMENT '任务编号', + module_type VARCHAR(32) NOT NULL COMMENT '模块类型：DEDUPE/CONVERT/SPLIT', + task_mode VARCHAR(32) DEFAULT 'IMMEDIATE' COMMENT '任务模式：IMMEDIATE/TASK', + status VARCHAR(32) NOT NULL COMMENT '任务状态：PENDING/RUNNING/SUCCESS/FAILED/CANCELLED', + source_file_count INT DEFAULT 0 COMMENT '源文件数量', + success_file_count INT DEFAULT 0 COMMENT '成功文件数量', + failed_file_count INT DEFAULT 0 COMMENT '失败文件数量', + request_json JSON NULL COMMENT '请求参数快照', + result_json JSON NULL COMMENT '结果快照', + error_message VARCHAR(1000) NULL COMMENT '错误信息', + created_by VARCHAR(64) DEFAULT 'system' COMMENT '创建人', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + finished_at DATETIME NULL COMMENT '完成时间', + UNIQUE KEY uk_task_no (task_no), + KEY idx_module_status (module_type, status), + KEY idx_created_at (created_at) +) COMMENT='文件处理任务表'; + +CREATE TABLE IF NOT EXISTS biz_file_result ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键', + task_id BIGINT NOT NULL COMMENT '任务ID', + module_type VARCHAR(32) NOT NULL COMMENT '模块类型：DEDUPE/CONVERT/SPLIT', + source_filename VARCHAR(255) NULL COMMENT '源文件名', + source_file_url VARCHAR(1000) NULL COMMENT '源文件地址', + result_filename VARCHAR(255) NULL COMMENT '结果文件名', + result_file_url VARCHAR(1000) NULL COMMENT '结果文件地址', + result_file_size BIGINT NULL COMMENT '结果文件大小', + result_content_type VARCHAR(128) NULL COMMENT '结果文件类型', + row_count INT NULL COMMENT '结果行数', + success TINYINT(1) NOT NULL DEFAULT 1 COMMENT '是否成功', + error_message VARCHAR(1000) NULL COMMENT '错误信息', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + KEY idx_task_id (task_id), + KEY idx_module_type (module_type) +) COMMENT='文件处理结果表'; + +CREATE TABLE IF NOT EXISTS biz_convert_template ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键', + template_code VARCHAR(64) NOT NULL COMMENT '模板编码', + template_name VARCHAR(128) NOT NULL COMMENT '模板名称', + output_filename VARCHAR(255) NOT NULL COMMENT '输出文件名', + required_source_columns_json JSON NOT NULL COMMENT '源文件必需列', + header_columns_json JSON NOT NULL COMMENT '输出表头列', + field_mapping_json JSON NOT NULL COMMENT '源字段映射', + defaults_json JSON NULL COMMENT '默认值配置', + preamble_lines_json JSON NULL COMMENT '前导说明行', + blank_header_rows_after_schema INT DEFAULT 0 COMMENT '表头后空白行数', + is_default TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否默认模板', + built_in TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否内置模板', + enabled TINYINT(1) NOT NULL DEFAULT 1 COMMENT '是否启用', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + UNIQUE KEY uk_template_code (template_code) +) COMMENT='格式转换模板表'; + +INSERT INTO biz_convert_template ( + template_code, template_name, output_filename, + required_source_columns_json, header_columns_json, field_mapping_json, + defaults_json, preamble_lines_json, blank_header_rows_after_schema, + is_default, built_in, enabled +) +SELECT + 'uk_offer', '英国模板', '英国.txt', + JSON_ARRAY('ASIN', '价格'), + JSON_ARRAY('sku', 'price', 'quantity', 'product-id', 'product-id-type', 'condition-type', 'condition-note', 'ASIN-hint', 'title', 'product-tax-code', 'operation-type', 'sale-price', 'sale-start-date', 'sale-end-date', 'leadtime-to-ship', 'launch-date', 'is-giftwrap-available', 'is-gift-message-available'), + JSON_OBJECT('price', '价格', 'product-id', 'ASIN'), + JSON_OBJECT('quantity', '100', 'product-id-type', 'ASIN', 'condition-type', 'new', 'leadtime-to-ship', '4'), + JSON_ARRAY('TemplateType=Offer\tVersion=1.4'), + 1, + 1, 1, 1 +WHERE NOT EXISTS ( + SELECT 1 FROM biz_convert_template WHERE template_code = 'uk_offer' +); diff --git a/backend-java/src/main/resources/db/V2__sync_uk_offer_template.sql b/backend-java/src/main/resources/db/V2__sync_uk_offer_template.sql new file mode 100644 index 0000000..b8395a8 --- /dev/null +++ b/backend-java/src/main/resources/db/V2__sync_uk_offer_template.sql @@ -0,0 +1,17 @@ +UPDATE biz_convert_template +SET required_source_columns_json = JSON_ARRAY('ASIN', '价格'), + header_columns_json = JSON_ARRAY( + 'sku', 'price', 'quantity', 'product-id', 'product-id-type', 'condition-type', 'condition-note', + 'ASIN-hint', 'title', 'product-tax-code', 'operation-type', 'sale-price', 'sale-start-date', + 'sale-end-date', 'leadtime-to-ship', 'launch-date', 'is-giftwrap-available', 'is-gift-message-available' + ), + field_mapping_json = JSON_OBJECT('price', '价格', 'product-id', 'ASIN'), + defaults_json = JSON_OBJECT('quantity', '100', 'product-id-type', 'ASIN', 'condition-type', 'new', 'leadtime-to-ship', '4'), + preamble_lines_json = JSON_ARRAY('TemplateType=Offer\tVersion=1.4'), + blank_header_rows_after_schema = 1, + output_filename = '英国.txt', + template_name = '英国模板', + built_in = 1, + enabled = 1, + updated_at = CURRENT_TIMESTAMP +WHERE template_code = 'uk_offer'; diff --git a/desktop/__pycache__/app.cpython-312.pyc b/desktop/__pycache__/app.cpython-312.pyc index ba952b6..6ec347a 100644 Binary files a/desktop/__pycache__/app.cpython-312.pyc and b/desktop/__pycache__/app.cpython-312.pyc differ diff --git a/desktop/app.py b/desktop/app.py index e2b1c52..b05d9d8 100644 --- a/desktop/app.py +++ b/desktop/app.py @@ -250,8 +250,8 @@ class DesktopConfig: class DesktopApi: - def __init__(self, app_base_url: str) -> None: - self.app_base_url = app_base_url.rstrip("/") + def __init__(self, backend_base_url: str) -> None: + self.app_base_url = backend_base_url.rstrip("/") self.session = requests.Session() def select_brand_xlsx_files(self) -> list[str]: @@ -347,17 +347,38 @@ class DesktopApi: return {"success": False, "error": str(exc)} def list_txt_templates(self) -> list[dict]: - templates, _default_template_id = build_effective_templates() - return [ - { - "id": template["id"], - "label": template["label"], - "output_filename": template["output_filename"], - "is_default": template.get("is_default", False), - "built_in": template.get("built_in", False), - } - for template in templates.values() - ] + try: + response = self.session.get( + resolve_url(self.app_base_url, "/api/convert/templates"), + timeout=60, + ) + response.raise_for_status() + payload = response.json() + if not payload.get("success"): + return [] + items = payload.get("data") or [] + return [ + { + "id": str(item.get("templateCode") or item.get("id") or ""), + "label": item.get("templateName") or "", + "output_filename": item.get("outputFilename") or "", + "is_default": bool(item.get("isDefault")), + "built_in": bool(item.get("builtIn")), + } + for item in items + ] + except Exception: + templates, _default_template_id = build_effective_templates() + return [ + { + "id": template["id"], + "label": template["label"], + "output_filename": template["output_filename"], + "is_default": template.get("is_default", False), + "built_in": template.get("built_in", False), + } + for template in templates.values() + ] def import_txt_template(self, name: str | None = None) -> dict: file_path = run_file_dialog( @@ -370,39 +391,60 @@ class DesktopApi: return {"success": False, "error": "用户取消"} try: - template_text = Path(file_path).read_text(encoding="utf-8") - except UnicodeDecodeError: - template_text = Path(file_path).read_text(encoding="utf-8-sig") + try: + template_text = Path(file_path).read_text(encoding="utf-8") + except UnicodeDecodeError: + template_text = Path(file_path).read_text(encoding="utf-8-sig") + + response = self.session.post( + resolve_url(self.app_base_url, "/api/convert/templates/import"), + json={ + "templateName": (name or "").strip() or Path(file_path).stem, + "templateContent": template_text, + }, + timeout=120, + ) + response.raise_for_status() + payload = response.json() + if not payload.get("success") or not payload.get("data"): + return {"success": False, "error": payload.get("message") or "导入失败"} + item = payload.get("data") or {} + return { + "success": True, + "template": { + "id": str(item.get("templateCode") or item.get("id") or ""), + "label": item.get("templateName") or "", + "output_filename": item.get("outputFilename") or "", + "is_default": bool(item.get("isDefault")), + "built_in": bool(item.get("builtIn")), + }, + } except Exception as exc: # noqa: BLE001 return {"success": False, "error": str(exc)} - template = recognize_txt_template(template_text, (name or "").strip() or None) - if not template: - return {"success": False, "error": "无法识别模板"} - - user_templates, default_template_id = load_user_templates() - user_templates[template["id"]] = template - save_user_templates(user_templates, default_template_id) - - return { - "success": True, - "template": { - "id": template["id"], - "label": template["label"], - "output_filename": template["output_filename"], - "is_default": False, - "built_in": False, - }, - } - def set_default_txt_template(self, template_id: str) -> dict: - templates, _default_template_id = build_effective_templates() - if template_id not in templates: - return {"success": False, "error": "模板不存在"} + try: + response = self.session.post( + resolve_url(self.app_base_url, f"/api/convert/templates/{template_id}/default"), + timeout=60, + ) + response.raise_for_status() + payload = response.json() + return {"success": bool(payload.get("success")), "error": payload.get("message") if not payload.get("success") else None} + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} - user_templates, _ = load_user_templates() - save_user_templates(user_templates, template_id) - return {"success": True} + def delete_txt_template(self, template_id: str) -> dict: + try: + response = self.session.delete( + resolve_url(self.app_base_url, f"/api/convert/templates/{template_id}"), + timeout=60, + ) + response.raise_for_status() + payload = response.json() + return {"success": bool(payload.get("success")), "error": payload.get("message") if not payload.get("success") else None} + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} def split_excel_files(self, payload: dict) -> dict: paths = payload.get("paths") or [] @@ -410,7 +452,6 @@ class DesktopApi: split_mode = payload.get("split_mode") or "rows_per_file" rows_per_file = payload.get("rows_per_file") parts = payload.get("parts") - output_dir = payload.get("output_dir") or "" if not isinstance(paths, list) or not paths: return {"success": False, "error": "未选择待拆分文件"} @@ -422,128 +463,171 @@ class DesktopApi: return {"success": False, "error": "每份条数不合法"} if split_mode == "parts" and (not isinstance(parts, int) or parts <= 0): return {"success": False, "error": "拆分份数不合法"} - if not output_dir: - return {"success": False, "error": "未选择输出目录"} - output_dir_path = Path(output_dir) - if not output_dir_path.exists() or not output_dir_path.is_dir(): - return {"success": False, "error": "输出目录不存在"} - - items: list[dict] = [] - success_count = 0 - failed_count = 0 - - for path in paths: - try: - workbook = load_workbook(path) - sheet = workbook[workbook.sheetnames[0]] - header_cells = next(sheet.iter_rows(min_row=1, max_row=1)) - header_map: dict[str, int] = {} - for index, cell in enumerate(header_cells): - value = normalize_cell_text(cell.value) - if not value: - continue - value = value.split("idASIN国家状态价格变体数量", 1)[0].strip() or value - if value in header_map: - continue - header_map[value] = index - if value == "缩略图地址8": - break - - missing = [column for column in selected_columns if column not in header_map] - if missing: - workbook.close() - items.append({ - "source_path": path, - "success": False, - "error": f"缺少列：{'、'.join(missing)}", - }) - failed_count += 1 - continue - - rows_data: list[list[str]] = [] - for row in sheet.iter_rows(min_row=2, values_only=True): - projected = [ - normalize_cell_text(row[header_map[column]]) if header_map[column] < len(row) else "" - for column in selected_columns - ] - rows_data.append(projected) - - total_rows = len(rows_data) - if total_rows == 0: - workbook.close() - items.append({ - "source_path": path, - "success": False, - "error": "没有可拆分的数据行", - }) - failed_count += 1 - continue - - if split_mode == "rows_per_file": - chunk_size = rows_per_file - chunks = [rows_data[i : i + chunk_size] for i in range(0, total_rows, chunk_size)] - else: - actual_parts = min(parts, total_rows) - base = total_rows // actual_parts - remainder = total_rows % actual_parts - chunks = [] - start = 0 - for idx in range(actual_parts): - size = base + (1 if idx < remainder else 0) - end = start + size - chunks.append(rows_data[start:end]) - start = end - - from openpyxl import Workbook - - source = Path(path) - for index, chunk in enumerate(chunks, start=1): - output_workbook = Workbook() - output_sheet = output_workbook.active - output_sheet.title = sheet.title - output_sheet.append(selected_columns) - for row_values in chunk: - output_sheet.append(row_values) - - output_path = build_named_output_path(output_dir, f"{source.stem}_split_{index}.xlsx") - output_workbook.save(output_path) - output_workbook.close() - - items.append({ - "source_path": path, - "output_path": output_path, - "success": True, - "row_count": len(chunk), - }) - - workbook.close() - success_count += 1 - except Exception as exc: # noqa: BLE001 - items.append({ - "source_path": path, - "success": False, - "error": str(exc), + try: + uploaded_files = [] + for path in paths: + with open(path, "rb") as file_obj: + response = self.session.post( + resolve_url(self.app_base_url, "/api/files/upload"), + files={"file": (Path(path).name, file_obj)}, + timeout=180, + ) + response.raise_for_status() + upload_payload = response.json() + if not upload_payload.get("success") or not upload_payload.get("data"): + return {"success": False, "error": upload_payload.get("message") or "上传文件失败"} + upload_data = upload_payload.get("data") or {} + uploaded_files.append({ + "fileKey": upload_data.get("fileKey"), + "originalFilename": upload_data.get("originalFilename") or Path(path).name, }) - failed_count += 1 - return { - "success": True, - "summary": { - "total": len(paths), - "success_count": success_count, - "failed_count": failed_count, - "output_dir": str(output_dir_path), - }, - "items": items, - } + run_response = self.session.post( + resolve_url(self.app_base_url, "/api/split/run"), + json={ + "files": uploaded_files, + "selectedColumns": selected_columns, + "splitMode": split_mode, + "rowsPerFile": rows_per_file, + "parts": parts, + }, + timeout=600, + ) + run_response.raise_for_status() + run_payload = run_response.json() + if not run_payload.get("success") or not run_payload.get("data"): + return {"success": False, "error": run_payload.get("message") or "数据拆分失败"} + + data = run_payload.get("data") or {} + result_items = [] + for item in data.get("items") or []: + if item.get("success") and item.get("downloadUrl"): + result_items.append({ + "result_id": item.get("resultId"), + "source_path": item.get("sourceFilename") or "", + "output_path": item.get("downloadUrl") or "", + "success": True, + "row_count": item.get("rowCount"), + }) + else: + result_items.append({ + "source_path": item.get("sourceFilename") or "", + "success": False, + "error": item.get("error") or "数据拆分失败", + }) + + return { + "success": True, + "summary": { + "total": data.get("total") or len(paths), + "success_count": data.get("successCount") or 0, + "failed_count": data.get("failedCount") or 0, + }, + "items": result_items, + } + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} + + def delete_history_item(self, module_type: str, result_id: int) -> dict: + try: + response = self.session.delete( + resolve_url(self.app_base_url, f"/api/{module_type}/history/{result_id}"), + timeout=60, + ) + response.raise_for_status() + payload = response.json() + return {"success": bool(payload.get("success")), "error": payload.get("message") if not payload.get("success") else None} + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} + + def get_convert_history(self) -> dict: + try: + response = self.session.get( + resolve_url(self.app_base_url, "/api/convert/history"), + timeout=60, + ) + response.raise_for_status() + payload = response.json() + if not payload.get("success") or not payload.get("data"): + return {"success": False, "error": payload.get("message") or "获取历史失败"} + + items = payload.get("data", {}).get("items") or [] + return { + "success": True, + "items": [ + { + "result_id": item.get("resultId"), + "source_path": item.get("sourceFilename") or "", + "output_path": item.get("downloadUrl") or "", + "success": bool(item.get("success", True)), + } + for item in items + ], + } + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} + + def get_split_history(self) -> dict: + try: + response = self.session.get( + resolve_url(self.app_base_url, "/api/split/history"), + timeout=60, + ) + response.raise_for_status() + payload = response.json() + if not payload.get("success") or not payload.get("data"): + return {"success": False, "error": payload.get("message") or "获取历史失败"} + + items = payload.get("data", {}).get("items") or [] + return { + "success": True, + "items": [ + { + "result_id": item.get("resultId"), + "source_path": item.get("sourceFilename") or "", + "output_path": item.get("downloadUrl") or "", + "success": bool(item.get("success", True)), + "row_count": item.get("rowCount"), + } + for item in items + ], + } + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} + + def get_dedupe_history(self) -> dict: + try: + response = self.session.get( + resolve_url(self.app_base_url, "/api/dedupe/history"), + timeout=60, + ) + response.raise_for_status() + payload = response.json() + if not payload.get("success") or not payload.get("data"): + return {"success": False, "error": payload.get("message") or "获取历史失败"} + + items = payload.get("data", {}).get("items") or [] + return { + "success": True, + "items": [ + { + "result_id": item.get("resultId"), + "source_path": item.get("sourceFilename") or "", + "output_path": item.get("downloadUrl") or "", + "success": bool(item.get("success", True)), + } + for item in items + ], + } + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} def clean_excel_files(self, payload: dict) -> dict: paths = payload.get("paths") or [] selected_columns = payload.get("selected_columns") or [] keep_integer_ids = bool(payload.get("keep_integer_ids", True)) keep_underscore_ids = bool(payload.get("keep_underscore_ids", True)) - output_dir = payload.get("output_dir") or "" if not isinstance(paths, list) or not paths: return {"success": False, "error": "未选择待处理文件"} @@ -551,213 +635,140 @@ class DesktopApi: return {"success": False, "error": "请至少选择一列"} if not keep_integer_ids and not keep_underscore_ids: return {"success": False, "error": "请至少选择一种 ID 保留规则"} - if not output_dir: - return {"success": False, "error": "未选择输出目录"} - output_dir_path = Path(output_dir) - if not output_dir_path.exists() or not output_dir_path.is_dir(): - return {"success": False, "error": "输出目录不存在"} + try: + uploaded_files = [] + for path in paths: + with open(path, "rb") as file_obj: + response = self.session.post( + resolve_url(self.app_base_url, "/api/files/upload"), + files={"file": (Path(path).name, file_obj)}, + timeout=180, + ) + response.raise_for_status() + upload_payload = response.json() + if not upload_payload.get("success") or not upload_payload.get("data"): + return {"success": False, "error": upload_payload.get("message") or "上传文件失败"} + upload_data = upload_payload.get("data") or {} + uploaded_files.append({ + "fileKey": upload_data.get("fileKey"), + "originalFilename": upload_data.get("originalFilename") or Path(path).name, + }) - items: list[dict] = [] - success_count = 0 - failed_count = 0 + run_response = self.session.post( + resolve_url(self.app_base_url, "/api/dedupe/run"), + json={ + "files": uploaded_files, + "selectedColumns": selected_columns, + "keepIntegerIds": keep_integer_ids, + "keepUnderscoreIds": keep_underscore_ids, + }, + timeout=600, + ) + run_response.raise_for_status() + run_payload = run_response.json() + if not run_payload.get("success") or not run_payload.get("data"): + return {"success": False, "error": run_payload.get("message") or "去重失败"} - for path in paths: - try: - workbook = load_workbook(path) - sheet = workbook[workbook.sheetnames[0]] - header_cells = next(sheet.iter_rows(min_row=1, max_row=1)) - headers: list[str] = [] - header_map: dict[str, int] = {} - for index, cell in enumerate(header_cells): - value = normalize_cell_text(cell.value) - if not value: - continue - value = value.split("idASIN国家状态价格变体数量", 1)[0].strip() or value - if value in header_map: - continue - header_map[value] = index - headers.append(value) - if value == "缩略图地址8": - break - - missing = [column for column in selected_columns if column not in header_map] - if missing: - workbook.close() - items.append({ - "source_path": path, - "success": False, - "error": f"缺少列：{'、'.join(missing)}", + data = run_payload.get("data") or {} + result_items = [] + for item in data.get("items") or []: + if item.get("success") and item.get("downloadUrl"): + result_items.append({ + "source_path": item.get("sourceFilename") or "", + "output_path": item.get("downloadUrl") or "", + "success": True, + }) + else: + result_items.append({ + "source_path": item.get("sourceFilename") or "", + "success": False, + "error": item.get("error") or "去重失败", }) - failed_count += 1 - continue - from openpyxl import Workbook - - output_workbook = Workbook() - output_sheet = output_workbook.active - output_sheet.title = sheet.title - output_sheet.append(selected_columns) - - id_column_index = header_map.get("id") - - for row in sheet.iter_rows(min_row=2, values_only=True): - original_values = list(row) - if id_column_index is not None: - id_value = original_values[id_column_index] if id_column_index < len(original_values) else None - if not should_keep_id(id_value, keep_integer_ids, keep_underscore_ids): - continue - - cleaned_values = [ - normalize_cell_text(original_values[header_map[column]]) - if header_map[column] < len(original_values) - else "" - for column in selected_columns - ] - output_sheet.append(cleaned_values) - - output_path = build_output_path(output_dir, path) - output_workbook.save(output_path) - output_workbook.close() - workbook.close() - - items.append({ - "source_path": path, - "output_path": output_path, - "success": True, - }) - success_count += 1 - except Exception as exc: # noqa: BLE001 - items.append({ - "source_path": path, - "success": False, - "error": str(exc), - }) - failed_count += 1 - - return { - "success": True, - "summary": { - "total": len(paths), - "success_count": success_count, - "failed_count": failed_count, - "output_dir": str(output_dir_path), - }, - "items": items, - } + return { + "success": True, + "summary": { + "total": data.get("total") or len(paths), + "success_count": data.get("successCount") or 0, + "failed_count": data.get("failedCount") or 0, + }, + "items": result_items, + } + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} def convert_excel_to_uk_txt(self, payload: dict) -> dict: paths = payload.get("paths") or [] - output_dir = payload.get("output_dir") or "" template_id = payload.get("template_id") or "uk_offer" if not isinstance(paths, list) or not paths: return {"success": False, "error": "未选择待转换文件"} - if not output_dir: - return {"success": False, "error": "未选择输出目录"} - templates, _default_template_id = build_effective_templates() - template = templates.get(template_id) - if not template: - return {"success": False, "error": "模板不存在"} + try: + templates = self.list_txt_templates() + selected_template = next((item for item in templates if item.get("id") == template_id), None) + if not selected_template: + return {"success": False, "error": "模板不存在"} - output_dir_path = Path(output_dir) - if not output_dir_path.exists() or not output_dir_path.is_dir(): - return {"success": False, "error": "输出目录不存在"} + uploaded_files = [] + for path in paths: + with open(path, "rb") as file_obj: + response = self.session.post( + resolve_url(self.app_base_url, "/api/files/upload"), + files={"file": (Path(path).name, file_obj)}, + timeout=180, + ) + response.raise_for_status() + upload_payload = response.json() + if not upload_payload.get("success") or not upload_payload.get("data"): + return {"success": False, "error": upload_payload.get("message") or "上传文件失败"} + upload_data = upload_payload.get("data") or {} + uploaded_files.append({ + "fileKey": upload_data.get("fileKey"), + "originalFilename": upload_data.get("originalFilename") or Path(path).name, + }) - items: list[dict] = [] - success_count = 0 - failed_count = 0 + run_response = self.session.post( + resolve_url(self.app_base_url, "/api/convert/run"), + json={ + "files": uploaded_files, + "templateId": str(template_id), + }, + timeout=600, + ) + run_response.raise_for_status() + run_payload = run_response.json() + if not run_payload.get("success") or not run_payload.get("data"): + return {"success": False, "error": run_payload.get("message") or "格式转换失败"} - for path in paths: - try: - workbook = load_workbook(path, read_only=True, data_only=True) - sheet = workbook[workbook.sheetnames[0]] - header_row = next(sheet.iter_rows(min_row=1, max_row=1, values_only=True), None) - if not header_row: - workbook.close() - items.append({"source_path": path, "success": False, "error": 'Excel 表头为空'}) - failed_count += 1 - continue - - header_map: dict[str, int] = {} - for index, cell in enumerate(header_row): - value = normalize_cell_text(cell) - if not value or value in header_map: - continue - header_map[value] = index - - required = template["required_source_columns"] - missing = [column for column in required if column not in header_map] - if missing: - workbook.close() - items.append({ - "source_path": path, - "success": False, - "error": f"缺少列：{'、'.join(missing)}", + data = run_payload.get("data") or {} + result_items = [] + for item in data.get("items") or []: + if item.get("success") and item.get("downloadUrl"): + result_items.append({ + "source_path": item.get("sourceFilename") or "", + "output_path": item.get("downloadUrl") or "", + "success": True, + }) + else: + result_items.append({ + "source_path": item.get("sourceFilename") or "", + "success": False, + "error": item.get("error") or "格式转换失败", }) - failed_count += 1 - continue - rows: list[str] = [] - rows.extend(template["preamble_lines"]) - rows.append('\t'.join(template["header_columns"])) - for _ in range(template.get("blank_header_rows_after_schema", 0)): - rows.append('\t' * (len(template["header_columns"]) - 1)) - - batch_id = generate_snowflake_like_id() - row_index = 1 - for row in sheet.iter_rows(min_row=2, values_only=True): - line_values: list[str] = [] - asin = normalize_cell_text(row[header_map['ASIN']] if header_map['ASIN'] < len(row) else '') - if not asin: - continue - - for column in template["header_columns"]: - if column == 'sku': - line_values.append(f'{batch_id}-{row_index}') - continue - if column in template["field_mapping"]: - source_column = template["field_mapping"][column] - value = normalize_cell_text(row[header_map[source_column]] if header_map[source_column] < len(row) else '') - line_values.append(value) - continue - if column in template["defaults"]: - line_values.append(str(template["defaults"][column])) - continue - line_values.append('') - - rows.append('\t'.join(line_values)) - row_index += 1 - - output_path = build_named_output_path(output_dir, template["output_filename"]) - Path(output_path).write_text('\n'.join(rows), encoding='utf-8') - workbook.close() - - items.append({ - "source_path": path, - "output_path": output_path, - "success": True, - }) - success_count += 1 - except Exception as exc: # noqa: BLE001 - items.append({ - "source_path": path, - "success": False, - "error": str(exc), - }) - failed_count += 1 - - return { - "success": True, - "summary": { - "total": len(paths), - "success_count": success_count, - "failed_count": failed_count, - "output_dir": str(output_dir_path), - }, - "items": items, - } + return { + "success": True, + "summary": { + "total": data.get("total") or len(paths), + "success_count": data.get("successCount") or 0, + "failed_count": data.get("failedCount") or 0, + }, + "items": result_items, + } + except Exception as exc: # noqa: BLE001 + return {"success": False, "error": str(exc)} def open_path(self, path: str) -> dict: try: @@ -939,7 +950,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--port", type=int, default=0) parser.add_argument("--width", type=int, default=1440) parser.add_argument("--height", type=int, default=960) - parser.add_argument("--title", default="南日AI") + parser.add_argument("--title", default="数富AI") return parser @@ -985,7 +996,7 @@ def main() -> None: ensure_dev_server_ready(config.frontend_url) app_url = config.frontend_url - api = DesktopApi(app_url) + api = DesktopApi(config.backend_url) window = webview.create_window( title=config.title, url=app_url, diff --git a/desktop/run_desktop_dev.ps1 b/desktop/run_desktop_dev.ps1 index 9610368..eda8ff8 100644 --- a/desktop/run_desktop_dev.ps1 +++ b/desktop/run_desktop_dev.ps1 @@ -64,4 +64,4 @@ Write-Host "Ensuring desktop dependencies..." Invoke-Python -PythonCommand $pythonCmd -Arguments @("-m", "pip", "install", "-r", ".\desktop\requirements.txt") Write-Host "Starting desktop app..." -Invoke-Python -PythonCommand $pythonCmd -Arguments @("-m", "desktop.app", "--mode", "dev", "--frontend-url", "http://127.0.0.1:5173", "--backend-url", "http://127.0.0.1:8000") +Invoke-Python -PythonCommand $pythonCmd -Arguments @("-m", "desktop.app", "--mode", "dev", "--frontend-url", "http://127.0.0.1:5173", "--backend-url", "http://127.0.0.1:18080") diff --git a/desktop/txt_templates.json b/desktop/txt_templates.json index 8873222..da5deed 100644 --- a/desktop/txt_templates.json +++ b/desktop/txt_templates.json @@ -90,6 +90,51 @@ "price": "价格", "product-id": "ASIN" } + }, + "user_1773982097249": { + "id": "user_1773982097249", + "label": "默认", + "output_filename": "默认.txt", + "is_default": false, + "built_in": false, + "preamble_lines": [ + "TemplateType=Offer\tVersion=1.4\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" + ], + "header_columns": [ + "sku", + "price", + "quantity", + "product-id", + "product-id-type", + "condition-type", + "condition-note", + "ASIN-hint", + "title", + "product-tax-code", + "operation-type", + "sale-price", + "sale-start-date", + "sale-end-date", + "leadtime-to-ship", + "launch-date", + "is-giftwrap-available", + "is-gift-message-available" + ], + "blank_header_rows_after_schema": 1, + "required_source_columns": [ + "ASIN", + "价格" + ], + "defaults": { + "quantity": "100", + "product-id-type": "ASIN", + "condition-type": "new", + "leadtime-to-ship": "4" + }, + "field_mapping": { + "price": "价格", + "product-id": "ASIN" + } } } } \ No newline at end of file diff --git a/frontend-vue/index.html b/frontend-vue/index.html index ab51280..707eae4 100644 --- a/frontend-vue/index.html +++ b/frontend-vue/index.html @@ -3,7 +3,7 @@ - 南日AI + 数富AI
diff --git a/frontend-vue/src/pages/brand/components/BrandConvertTab.vue b/frontend-vue/src/pages/brand/components/BrandConvertTab.vue index cfc2de6..8bbda4b 100644 --- a/frontend-vue/src/pages/brand/components/BrandConvertTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandConvertTab.vue @@ -41,13 +41,23 @@ :class="{ active: convertTemplateId === template.id }" > -
- - {{ template.label }} - 默认 - 内置 - -
输出文件：{{ template.output_filename }}
+
+
+ + {{ template.label }} + 默认 + 内置 + +
输出文件：{{ template.output_filename }}
+
+
@@ -81,7 +91,7 @@ {{ convertRunning ? '转换中...' : '开始转换' }} - {{ convertRunning ? '正在生成英国.txt，请稍候…' : '选择文件后即可执行格式转换' }} + {{ convertRunning ? '正在生成并上传结果，请稍候…' : '选择文件后即可执行格式转换，结果会显示在右侧供下载' }} @@ -108,9 +118,6 @@
生成的 txt 列表 -
@@ -120,12 +127,12 @@
  • - {{ shorten(item.source_path, 42) }} -
    输出文件：{{ item.output_path }}
    + {{ item.source_path || '-' }} +
    结果下载地址已生成
    错误信息：{{ item.error }}
    @@ -137,9 +144,17 @@ v-if="item.success && item.output_path" type="button" class="download" - @click="openConvertResult(item)" + @click="downloadConvertResult(item)" > - 打开文件 + 下载文件 + +
@@ -175,7 +190,6 @@ function shorten(value: string, maxLength: number) { } function resetConvertResults() { - convertResultItems.value = [] convertSummary.value = { total: 0, success_count: 0, failed_count: 0 } convertOutputDir.value = '' } @@ -217,6 +231,26 @@ async function importConvertTemplate() { ElMessage.success('模板导入成功') } +async function deleteConvertTemplate(templateId: string) { + const api = getPywebviewApi() + if (!api?.delete_txt_template) { + ElMessage.warning('当前环境不支持删除模板') + return + } + + const result = await api.delete_txt_template(templateId) + if (!result.success) { + ElMessage.error(result.error || '删除模板失败') + return + } + + if (convertTemplateId.value === templateId) { + convertTemplateId.value = '' + } + await loadConvertTemplates() + ElMessage.success('模板已删除') +} + async function setConvertDefaultTemplate() { const api = getPywebviewApi() if (!api?.set_default_txt_template || !convertTemplateId.value) { @@ -279,7 +313,7 @@ async function selectConvertFolder() { async function submitConvertRun() { const api = getPywebviewApi() - if (!api?.convert_excel_to_uk_txt || !api?.select_folder) { + if (!api?.convert_excel_to_uk_txt) { ElMessage.warning('当前环境不支持格式转换，请在桌面端打开') return } @@ -289,13 +323,9 @@ async function submitConvertRun() { } try { - const outputDir = await api.select_folder() - if (!outputDir) return - convertRunning.value = true const result = await api.convert_excel_to_uk_txt({ paths: convertSelectedPaths.value, - output_dir: outputDir, template_id: convertTemplateId.value, }) @@ -304,13 +334,13 @@ async function submitConvertRun() { return } - convertOutputDir.value = result.summary?.output_dir || outputDir + convertOutputDir.value = '' convertSummary.value = { total: result.summary?.total || 0, success_count: result.summary?.success_count || 0, failed_count: result.summary?.failed_count || 0, } - convertResultItems.value = result.items || [] + await loadConvertHistory() ElMessage.success('格式转换完成') } catch (error) { ElMessage.error(error instanceof Error ? error.message : '格式转换失败') @@ -319,34 +349,68 @@ async function submitConvertRun() { } } -async function openConvertOutputDir() { +async function deleteConvertHistory(resultId: number) { const api = getPywebviewApi() - if (!api?.open_path || !convertOutputDir.value) { - ElMessage.warning('当前环境不支持打开输出目录') + if (!api?.delete_history_item) { + ElMessage.warning('当前环境不支持删除历史') return } - const result = await api.open_path(convertOutputDir.value) + const result = await api.delete_history_item('convert', resultId) if (!result.success) { - ElMessage.error(result.error || '打开目录失败') + ElMessage.error(result.error || '删除失败') + return } + await loadConvertHistory() + ElMessage.success('已删除') } -async function openConvertResult(item: ConvertTxtResultItem) { +async function downloadConvertResult(item: ConvertTxtResultItem) { const api = getPywebviewApi() - if (!api?.open_path || !item.output_path) { - ElMessage.warning('当前环境不支持打开结果文件') + if (!item.output_path) { + ElMessage.warning('当前结果没有下载地址') return } - const result = await api.open_path(item.output_path) - if (!result.success) { - ElMessage.error(result.error || '打开文件失败') + const today = new Date() + const yyyy = String(today.getFullYear()) + const mm = String(today.getMonth() + 1).padStart(2, '0') + const dd = String(today.getDate()).padStart(2, '0') + const filename = `${yyyy}${mm}${dd}.txt` + if (api?.save_file_from_url) { + const result = await api.save_file_from_url(item.output_path, filename) + if (result.success) { + ElMessage.success(`已保存：${result.path || filename}`) + } else if (result.error && result.error !== '用户取消') { + ElMessage.error(result.error) + } + return + } + + window.open(item.output_path, '_blank') +} + +async function loadConvertHistory() { + const api = getPywebviewApi() + if (!api?.get_convert_history) return + + try { + const response = await api.get_convert_history() + if (!response.success || !response.items) return + convertResultItems.value = response.items + convertSummary.value = { + total: response.items.length, + success_count: response.items.filter((item) => item.success).length, + failed_count: response.items.filter((item) => !item.success).length, + } + } catch { + // ignore history load errors } } onMounted(() => { loadConvertTemplates().catch(() => undefined) + loadConvertHistory().catch(() => undefined) }) @@ -374,7 +438,10 @@ onMounted(() => { .template-tag { margin-left: 8px; padding: 2px 8px; border-radius: 999px; font-size: 11px; color: #8fd0ff; background: rgba(52, 152, 219, 0.14); } .template-tag.muted { color: #b8b8b8; background: rgba(255, 255, 255, 0.08); } .compact-row { margin-top: 4px; } -.radio-item { display: flex; align-items: flex-start; gap: 10px; cursor: pointer; padding: 10px 12px; border-radius: 8px; background: #252525; border: 1px solid #2a2a2a; margin-bottom: 8px; transition: all 0.2s ease; } + .radio-item { display: flex; align-items: flex-start; gap: 10px; cursor: pointer; padding: 10px 12px; border-radius: 8px; background: #252525; border: 1px solid #2a2a2a; margin-bottom: 8px; transition: all 0.2s ease; } +.template-item-content { display: flex; align-items: center; justify-content: space-between; gap: 12px; width: 100%; } +.template-delete-btn { padding: 6px 10px; border: 1px solid #5c2f2f; border-radius: 6px; background: rgba(231, 76, 60, 0.12); color: #ff8f8f; font-size: 12px; cursor: pointer; } +.template-delete-btn:hover { background: rgba(231, 76, 60, 0.2); } .radio-item:hover, .radio-item.active { background: #2a2a2a; border-color: #3a3a3a; } .radio-item input { margin-top: 3px; } .label { font-weight: 500; color: #e0e0e0; font-size: 13px; } diff --git a/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue b/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue index 5c35ef3..a6d83c8 100644 --- a/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue @@ -79,7 +79,7 @@ {{ cleanRunning ? '清洗中...' : '开始清洗' }} - {{ cleanRunning ? '正在处理文件，请稍候…' : '选择文件、列和 ID 规则后即可执行本地清洗' }} + {{ cleanRunning ? '正在处理文件并上传结果，请稍候…' : '选择文件、列和 ID 规则后即可执行清洗，结果会显示在右侧供下载' }}
@@ -106,9 +106,6 @@
处理后的 Excel 列表 -
@@ -118,8 +115,8 @@
  • - {{ shorten(item.source_path, 42) }} -
    输出文件：{{ item.output_path }}
    + {{ item.source_path || '-' }} +
    结果下载地址已生成
    错误信息：{{ item.error }}
    @@ -131,9 +128,17 @@ v-if="item.success && item.output_path" type="button" class="download" - @click="openCleanResult(item)" + @click="downloadCleanResult(item)" > - 打开文件 + 下载文件 + +
@@ -145,7 +150,7 @@ diff --git a/frontend-vue/src/pages/brand/components/BrandSplitTab.vue b/frontend-vue/src/pages/brand/components/BrandSplitTab.vue index 93c3382..b1debca 100644 --- a/frontend-vue/src/pages/brand/components/BrandSplitTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandSplitTab.vue @@ -90,7 +90,7 @@ {{ splitRunning ? '拆分中...' : '开始拆分' }} - {{ splitRunning ? '正在拆分文件，请稍候…' : '选择文件、保留列和拆分规则后即可执行本地拆分' }} + {{ splitRunning ? '正在拆分并上传结果，请稍候…' : '选择文件、保留列和拆分规则后即可执行拆分，结果会显示在右侧供下载' }}
@@ -116,9 +116,6 @@
拆分后的 Excel 列表 -
@@ -128,8 +125,8 @@
  • - {{ shorten(item.source_path, 42) }} -
    输出文件：{{ item.output_path }}
    + {{ item.source_path || '-' }} +
    结果下载地址已生成
    包含 {{ item.row_count }} 条数据
    错误信息：{{ item.error }}
    @@ -142,9 +139,17 @@ v-if="item.success && item.output_path" type="button" class="download" - @click="openSplitResult(item)" + @click="downloadSplitResult(item)" > - 打开文件 + 下载文件 + +
@@ -156,7 +161,7 @@ diff --git a/frontend-vue/src/pages/brand/index.vue b/frontend-vue/src/pages/brand/index.vue index aa23520..3e3f025 100644 --- a/frontend-vue/src/pages/brand/index.vue +++ b/frontend-vue/src/pages/brand/index.vue @@ -3,42 +3,26 @@
- 南日AI-亚马逊 + 数富AI-亚马逊 返回首页
- - - -
@@ -55,13 +39,8 @@
- +