完成新需求
This commit is contained in:
super
@@ -0,0 +1,12 @@
|
||||
package com.nanri.aiimage;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class AiImageApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(AiImageApplication.class, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.nanri.aiimage.common.api;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Schema(description = "统一响应体")
|
||||
public class ApiResponse<T> {
|
||||
|
||||
@Schema(description = "是否成功")
|
||||
private boolean success;
|
||||
|
||||
@Schema(description = "提示信息")
|
||||
private String message;
|
||||
|
||||
@Schema(description = "响应数据")
|
||||
private T data;
|
||||
|
||||
public static <T> ApiResponse<T> success(T data) {
|
||||
return new ApiResponse<>(true, "操作成功", data);
|
||||
}
|
||||
|
||||
public static <T> ApiResponse<T> success(String message, T data) {
|
||||
return new ApiResponse<>(true, message, data);
|
||||
}
|
||||
|
||||
public static <T> ApiResponse<T> fail(String message) {
|
||||
return new ApiResponse<>(false, message, null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.nanri.aiimage.common.exception;
|
||||
|
||||
public class BusinessException extends RuntimeException {
|
||||
|
||||
public BusinessException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.nanri.aiimage.common.exception;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.MethodArgumentNotValidException;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
|
||||
@Slf4j
|
||||
@RestControllerAdvice
|
||||
public class GlobalExceptionHandler {
|
||||
|
||||
@ExceptionHandler(BusinessException.class)
|
||||
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
||||
return ApiResponse.fail(ex.getMessage());
|
||||
}
|
||||
|
||||
@ExceptionHandler(MethodArgumentNotValidException.class)
|
||||
public ApiResponse<Void> handleMethodArgumentNotValidException(MethodArgumentNotValidException ex) {
|
||||
String message = ex.getBindingResult().getFieldError() != null
|
||||
? ex.getBindingResult().getFieldError().getDefaultMessage()
|
||||
: "参数校验失败";
|
||||
return ApiResponse.fail(message);
|
||||
}
|
||||
|
||||
@ExceptionHandler(ConstraintViolationException.class)
|
||||
public ApiResponse<Void> handleConstraintViolationException(ConstraintViolationException ex) {
|
||||
return ApiResponse.fail(ex.getMessage());
|
||||
}
|
||||
|
||||
@ExceptionHandler(Exception.class)
|
||||
public ApiResponse<Void> handleException(Exception ex) {
|
||||
log.error("Unhandled exception", ex);
|
||||
return ApiResponse.fail("服务异常：" + ex.getMessage());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import io.swagger.v3.oas.models.ExternalDocumentation;
|
||||
import io.swagger.v3.oas.models.OpenAPI;
|
||||
import io.swagger.v3.oas.models.info.Contact;
|
||||
import io.swagger.v3.oas.models.info.Info;
|
||||
import io.swagger.v3.oas.models.info.License;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class OpenApiConfig {
|
||||
|
||||
@Bean
|
||||
public OpenAPI openAPI() {
|
||||
return new OpenAPI()
|
||||
.info(new Info()
|
||||
.title("AI Image Backend API")
|
||||
.description("""
|
||||
文件处理后端接口文档。
|
||||
|
||||
当前迁移范围：仅迁移数据去重（dedupe）、格式转换（convert）、数据拆分（split）三块处理逻辑到 Java；
|
||||
Python 桌面端壳、pywebview 调用方式、前端页面交互保持不变。
|
||||
|
||||
建议联调顺序：
|
||||
1. 先启动 Java 后端；
|
||||
2. 在 Python 桌面端中选择文件并执行 dedupe/convert/split；
|
||||
3. Python 壳会先调用 /api/files/upload 上传临时文件；
|
||||
4. 再调用对应的 /api/dedupe/run、/api/convert/run、/api/split/run；
|
||||
5. Java 生成结果文件后，Python 壳再通过下载接口取回并保存到用户本地目录。
|
||||
|
||||
本地启动说明：
|
||||
- 默认配置读取 application.yml；
|
||||
- 建议复制 application-local.example.yml 为本地配置，并在 IDE 中通过环境变量覆盖数据库与 OSS 参数；
|
||||
- Knife4j 地址：/doc.html
|
||||
""")
|
||||
.version("v0.0.1")
|
||||
.contact(new Contact().name("Nanri AI"))
|
||||
.license(new License().name("Internal Use")))
|
||||
.externalDocs(new ExternalDocumentation()
|
||||
.description("Knife4j 文档")
|
||||
.url("/doc.html"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "aiimage.oss")
|
||||
public class OssProperties {
|
||||
private String region;
|
||||
private String endpoint;
|
||||
private String bucket;
|
||||
private String accessKeyId;
|
||||
private String accessKeySecret;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({OssProperties.class, StorageProperties.class})
|
||||
public class PropertiesConfig {
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.security.config.Customizer;
|
||||
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
||||
import org.springframework.security.web.SecurityFilterChain;
|
||||
|
||||
@Configuration
|
||||
public class SecurityConfig {
|
||||
|
||||
@Bean
|
||||
public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception {
|
||||
return http
|
||||
.csrf(csrf -> csrf.disable())
|
||||
.authorizeHttpRequests(auth -> auth.anyRequest().permitAll())
|
||||
.httpBasic(Customizer.withDefaults())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "aiimage.storage")
|
||||
public class StorageProperties {
|
||||
private String localTempDir;
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.nanri.aiimage.modules.convert.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest;
|
||||
import com.nanri.aiimage.modules.convert.model.vo.ConvertHistoryVo;
|
||||
import com.nanri.aiimage.modules.convert.model.vo.ConvertRunVo;
|
||||
import com.nanri.aiimage.modules.convert.service.ConvertRunService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/convert")
|
||||
@Tag(name = "格式转换执行", description = "负责根据已上传文件和模板执行格式转换，并返回结果下载地址。当前先打通接口链路，下一步再把旧 Python 的模板生成逻辑完整迁移到 Java。")
|
||||
public class ConvertRunController {
|
||||
|
||||
private final ConvertRunService convertRunService;
|
||||
|
||||
@PostMapping("/run")
|
||||
@Operation(
|
||||
summary = "执行格式转换",
|
||||
description = """
|
||||
使用上传接口返回的 fileKey 和选中的模板执行格式转换。
|
||||
|
||||
当前已按旧 Python 桌面端规则迁移以下逻辑：
|
||||
- required_source_columns 缺失校验；
|
||||
- preamble_lines 输出；
|
||||
- header_columns 输出；
|
||||
- blank_header_rows_after_schema 空白行补齐；
|
||||
- field_mapping 字段映射；
|
||||
- defaults 默认值填充；
|
||||
- ASIN 为空的行跳过；
|
||||
- sku 按 时间戳-序号 生成。
|
||||
|
||||
输出文件名遵循模板配置的 outputFilename，若临时目录中重名则自动追加序号。
|
||||
""")
|
||||
public ApiResponse<ConvertRunVo> run(@Valid @RequestBody ConvertRunRequest request) {
|
||||
return ApiResponse.success(convertRunService.run(request));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
@Operation(summary = "查询格式转换历史", description = "查询最近的格式转换成功记录，用于页面右侧历史下载列表展示。")
|
||||
public ApiResponse<ConvertHistoryVo> history() {
|
||||
ConvertHistoryVo vo = new ConvertHistoryVo();
|
||||
vo.setItems(convertRunService.listHistory());
|
||||
return ApiResponse.success(vo);
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除格式转换历史", description = "删除一条格式转换历史记录。")
|
||||
public ApiResponse<Void> deleteHistory(@PathVariable Long resultId) {
|
||||
convertRunService.deleteHistory(resultId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/results/{resultId}/download")
|
||||
@Operation(summary = "下载格式转换结果", description = "根据结果记录 ID 下载已生成的转换结果文件。桌面端前端可继续通过 save_file_from_url 选择保存位置。")
|
||||
public ResponseEntity<Resource> download(@PathVariable Long resultId) {
|
||||
File resultFile = convertRunService.getResultFile(resultId);
|
||||
String encodedFilename = URLEncoder.encode(resultFile.getName(), StandardCharsets.UTF_8).replaceAll("\\+", "%20");
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFilename)
|
||||
.body(new FileSystemResource(resultFile));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.nanri.aiimage.modules.convert.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.convert.model.dto.ConvertTemplateImportRequest;
|
||||
import com.nanri.aiimage.modules.convert.model.vo.ConvertTemplateVo;
|
||||
import com.nanri.aiimage.modules.convert.service.ConvertTemplateService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/convert/templates")
|
||||
@Tag(name = "格式转换模板", description = "用于维护和查询格式转换模板，支持查询、导入、设默认、删除。")
|
||||
public class ConvertTemplateController {
|
||||
|
||||
private final ConvertTemplateService convertTemplateService;
|
||||
|
||||
@GetMapping
|
||||
@Operation(summary = "查询启用模板", description = "返回当前启用的格式转换模板列表，供前端‘格式转换’模块展示。")
|
||||
public ApiResponse<List<ConvertTemplateVo>> listTemplates() {
|
||||
return ApiResponse.success(convertTemplateService.listEnabledTemplates());
|
||||
}
|
||||
|
||||
@PostMapping("/import")
|
||||
@Operation(summary = "导入模板", description = "导入自定义 txt 模板，成功后返回新模板并可立即刷新模板列表。")
|
||||
public ApiResponse<ConvertTemplateVo> importTemplate(@RequestBody ConvertTemplateImportRequest request) {
|
||||
return ApiResponse.success(convertTemplateService.importTemplate(request));
|
||||
}
|
||||
|
||||
@PostMapping("/{templateCode}/default")
|
||||
@Operation(summary = "设为默认模板", description = "将指定模板设为默认模板。")
|
||||
public ApiResponse<Void> setDefault(@PathVariable String templateCode) {
|
||||
convertTemplateService.setDefaultTemplate(templateCode);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@DeleteMapping("/{templateCode}")
|
||||
@Operation(summary = "删除模板", description = "删除指定模板。内置模板不允许删除。")
|
||||
public ApiResponse<Void> delete(@PathVariable String templateCode) {
|
||||
convertTemplateService.deleteTemplate(templateCode);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.convert.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.convert.model.entity.ConvertTemplateEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface ConvertTemplateMapper extends BaseMapper<ConvertTemplateEntity> {
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.convert.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "格式转换执行请求")
|
||||
public class ConvertRunRequest {
|
||||
|
||||
@Valid
|
||||
@NotEmpty(message = "请先上传待转换文件")
|
||||
@Schema(description = "已上传的源文件列表")
|
||||
private List<UploadedSourceFileDto> files;
|
||||
|
||||
@NotBlank(message = "请选择模板")
|
||||
@Schema(description = "模板编码，例如 uk_offer")
|
||||
private String templateId;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.nanri.aiimage.modules.convert.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "格式转换模板导入请求")
|
||||
public class ConvertTemplateImportRequest {
|
||||
|
||||
@Schema(description = "模板显示名称")
|
||||
private String templateName;
|
||||
|
||||
@Schema(description = "模板文本内容")
|
||||
private String templateContent;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.convert.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "已上传源文件")
|
||||
public class UploadedSourceFileDto {
|
||||
|
||||
@NotBlank(message = "fileKey 不能为空")
|
||||
@Schema(description = "上传接口返回的临时文件键")
|
||||
private String fileKey;
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.nanri.aiimage.modules.convert.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_convert_template")
|
||||
public class ConvertTemplateEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String templateCode;
|
||||
private String templateName;
|
||||
private String outputFilename;
|
||||
private String requiredSourceColumnsJson;
|
||||
private String headerColumnsJson;
|
||||
private String fieldMappingJson;
|
||||
private String defaultsJson;
|
||||
private String preambleLinesJson;
|
||||
private Integer blankHeaderRowsAfterSchema;
|
||||
private Integer isDefault;
|
||||
private Integer builtIn;
|
||||
private Integer enabled;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.nanri.aiimage.modules.convert.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "格式转换历史列表")
|
||||
public class ConvertHistoryVo {
|
||||
|
||||
@Schema(description = "历史记录")
|
||||
private List<ConvertResultItemVo> items;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.convert.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "格式转换结果项")
|
||||
public class ConvertResultItemVo {
|
||||
|
||||
@Schema(description = "结果记录ID")
|
||||
private Long resultId;
|
||||
|
||||
@Schema(description = "源文件名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "结果文件名")
|
||||
private String outputFilename;
|
||||
|
||||
@Schema(description = "是否成功")
|
||||
private boolean success;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "下载地址")
|
||||
private String downloadUrl;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.convert.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "格式转换执行结果")
|
||||
public class ConvertRunVo {
|
||||
|
||||
@Schema(description = "处理文件总数")
|
||||
private Integer total;
|
||||
|
||||
@Schema(description = "成功文件数")
|
||||
private Integer successCount;
|
||||
|
||||
@Schema(description = "失败文件数")
|
||||
private Integer failedCount;
|
||||
|
||||
@Schema(description = "结果列表")
|
||||
private List<ConvertResultItemVo> items;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.convert.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "格式转换模板")
|
||||
public class ConvertTemplateVo {
|
||||
|
||||
@Schema(description = "模板编码")
|
||||
private String id;
|
||||
|
||||
@Schema(description = "模板编码")
|
||||
private String templateCode;
|
||||
|
||||
@Schema(description = "模板名称")
|
||||
private String templateName;
|
||||
|
||||
@Schema(description = "输出文件名")
|
||||
private String outputFilename;
|
||||
|
||||
@Schema(description = "是否默认模板")
|
||||
private Boolean isDefault;
|
||||
|
||||
@Schema(description = "是否内置模板")
|
||||
private Boolean builtIn;
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package com.nanri.aiimage.modules.convert.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest;
|
||||
import com.nanri.aiimage.modules.convert.model.dto.UploadedSourceFileDto;
|
||||
import com.nanri.aiimage.modules.convert.model.entity.ConvertTemplateEntity;
|
||||
import com.nanri.aiimage.modules.convert.model.vo.ConvertResultItemVo;
|
||||
import com.nanri.aiimage.modules.convert.model.vo.ConvertRunVo;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ConvertRunService {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final StorageProperties storageProperties;
|
||||
private final ConvertTemplateService convertTemplateService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final OssStorageService ossStorageService;
|
||||
|
||||
public ConvertRunVo run(ConvertRunRequest request) {
|
||||
ConvertTemplateEntity template = convertTemplateService.getByCode(request.getTemplateId());
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo("CONVERT-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType("CONVERT");
|
||||
task.setTaskMode("IMMEDIATE");
|
||||
task.setStatus("SUCCESS");
|
||||
task.setSourceFileCount(request.getFiles().size());
|
||||
task.setCreatedBy("system");
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
List<ConvertResultItemVo> items = new ArrayList<>();
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
|
||||
for (UploadedSourceFileDto sourceFile : request.getFiles()) {
|
||||
ConvertResultItemVo item = new ConvertResultItemVo();
|
||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
try {
|
||||
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("上传文件不存在，请重新上传");
|
||||
}
|
||||
|
||||
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
|
||||
String outputFilename = template.getOutputFilename() == null || template.getOutputFilename().isBlank()
|
||||
? "结果.txt"
|
||||
: template.getOutputFilename();
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
|
||||
writeTxtByLegacyRules(inputFile, outputFile, template);
|
||||
|
||||
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "CONVERT");
|
||||
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
|
||||
|
||||
item.setSuccess(true);
|
||||
item.setOutputFilename(outputFilename);
|
||||
item.setDownloadUrl(downloadUrl);
|
||||
successCount++;
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("CONVERT");
|
||||
resultEntity.setSourceFilename(inputName);
|
||||
resultEntity.setResultFilename(outputFilename);
|
||||
resultEntity.setResultFileUrl(downloadUrl);
|
||||
resultEntity.setResultFileSize(outputFile.length());
|
||||
resultEntity.setResultContentType("text/plain");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
item.setResultId(resultEntity.getId());
|
||||
} catch (Exception ex) {
|
||||
item.setSuccess(false);
|
||||
item.setError(ex.getMessage());
|
||||
failedCount++;
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("CONVERT");
|
||||
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
resultEntity.setSuccess(0);
|
||||
resultEntity.setErrorMessage(ex.getMessage());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
}
|
||||
items.add(item);
|
||||
}
|
||||
|
||||
task.setSuccessFileCount(successCount);
|
||||
task.setFailedFileCount(failedCount);
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
ConvertRunVo vo = new ConvertRunVo();
|
||||
vo.setTotal(request.getFiles().size());
|
||||
vo.setSuccessCount(successCount);
|
||||
vo.setFailedCount(failedCount);
|
||||
vo.setItems(items);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public List<ConvertResultItemVo> listHistory() {
|
||||
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, "CONVERT")
|
||||
.eq(FileResultEntity::getSuccess, 1)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit 50"))
|
||||
.stream()
|
||||
.map(entity -> {
|
||||
ConvertResultItemVo vo = new ConvertResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl());
|
||||
vo.setSuccess(true);
|
||||
return vo;
|
||||
})
|
||||
.toList();
|
||||
}
|
||||
|
||||
public void deleteHistory(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !"CONVERT".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
}
|
||||
|
||||
public File getResultFile(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || entity.getResultFileUrl() == null || !"CONVERT".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
File file = new File(entity.getResultFileUrl());
|
||||
if (!file.exists()) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private void writeTxtByLegacyRules(File inputFile, File outputFile, ConvertTemplateEntity templateEntity) throws IOException {
|
||||
List<String> requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson());
|
||||
List<String> headerColumns = readStringList(templateEntity.getHeaderColumnsJson());
|
||||
Map<String, String> fieldMapping = readStringMap(templateEntity.getFieldMappingJson());
|
||||
Map<String, String> defaults = readStringMap(templateEntity.getDefaultsJson());
|
||||
List<String> preambleLines = readStringList(templateEntity.getPreambleLinesJson());
|
||||
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null ? 0 : templateEntity.getBlankHeaderRowsAfterSchema();
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
|
||||
if (!value.isBlank() && !headerMap.containsKey(value)) {
|
||||
headerMap.put(value, i);
|
||||
}
|
||||
}
|
||||
|
||||
List<String> missing = requiredColumns.stream().filter(column -> !headerMap.containsKey(column)).toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("缺少列：" + String.join("、", missing));
|
||||
}
|
||||
|
||||
List<String> rows = new ArrayList<>();
|
||||
rows.addAll(preambleLines);
|
||||
rows.add(String.join("\t", headerColumns));
|
||||
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
|
||||
rows.add("\t".repeat(Math.max(0, headerColumns.size() - 1)));
|
||||
}
|
||||
|
||||
long batchId = System.currentTimeMillis();
|
||||
int rowIndex = 1;
|
||||
int asinIndex = headerMap.getOrDefault("ASIN", -1);
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : "";
|
||||
if (asin.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> lineValues = new ArrayList<>();
|
||||
for (String column : headerColumns) {
|
||||
if ("sku".equals(column)) {
|
||||
lineValues.add(batchId + "-" + rowIndex);
|
||||
continue;
|
||||
}
|
||||
if (fieldMapping.containsKey(column)) {
|
||||
String sourceColumn = fieldMapping.get(column);
|
||||
Integer sourceIndex = headerMap.get(sourceColumn);
|
||||
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
|
||||
lineValues.add(value);
|
||||
continue;
|
||||
}
|
||||
if (defaults.containsKey(column)) {
|
||||
lineValues.add(defaults.get(column));
|
||||
continue;
|
||||
}
|
||||
lineValues.add("");
|
||||
}
|
||||
rows.add(String.join("\t", lineValues));
|
||||
rowIndex++;
|
||||
}
|
||||
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> readStringList(String json) {
|
||||
try {
|
||||
if (json == null || json.isBlank()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
return objectMapper.readValue(json, new TypeReference<List<String>>() {});
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("模板配置解析失败");
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> readStringMap(String json) {
|
||||
try {
|
||||
if (json == null || json.isBlank()) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {});
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("模板配置解析失败");
|
||||
}
|
||||
}
|
||||
|
||||
private File findLocalSourceFile(String fileKey) {
|
||||
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
|
||||
if (!baseDir.exists()) {
|
||||
return null;
|
||||
}
|
||||
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
|
||||
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
|
||||
}
|
||||
|
||||
private File buildNamedOutputFile(File outputDir, String filename) {
|
||||
File candidate = FileUtil.file(outputDir, filename);
|
||||
if (!candidate.exists()) {
|
||||
return candidate;
|
||||
}
|
||||
String mainName = FileUtil.mainName(filename);
|
||||
String extName = FileUtil.extName(filename);
|
||||
int index = 2;
|
||||
while (true) {
|
||||
String nextFilename = extName == null || extName.isBlank()
|
||||
? mainName + "_" + index
|
||||
: mainName + "_" + index + "." + extName;
|
||||
File nextCandidate = FileUtil.file(outputDir, nextFilename);
|
||||
if (!nextCandidate.exists()) {
|
||||
return nextCandidate;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeCellText(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.replace("\ufeff", "")
|
||||
.replace("\u3000", " ")
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim()
|
||||
.replaceAll("\\s+", " ");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package com.nanri.aiimage.modules.convert.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.convert.mapper.ConvertTemplateMapper;
|
||||
import com.nanri.aiimage.modules.convert.model.dto.ConvertTemplateImportRequest;
|
||||
import com.nanri.aiimage.modules.convert.model.entity.ConvertTemplateEntity;
|
||||
import com.nanri.aiimage.modules.convert.model.vo.ConvertTemplateVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ConvertTemplateService {
|
||||
|
||||
private final ConvertTemplateMapper convertTemplateMapper;
|
||||
|
||||
public List<ConvertTemplateVo> listEnabledTemplates() {
|
||||
return convertTemplateMapper.selectList(new LambdaQueryWrapper<ConvertTemplateEntity>()
|
||||
.eq(ConvertTemplateEntity::getEnabled, 1)
|
||||
.orderByDesc(ConvertTemplateEntity::getIsDefault)
|
||||
.orderByAsc(ConvertTemplateEntity::getId))
|
||||
.stream()
|
||||
.map(this::toVo)
|
||||
.toList();
|
||||
}
|
||||
|
||||
public ConvertTemplateEntity getByCode(String templateCode) {
|
||||
ConvertTemplateEntity entity = convertTemplateMapper.selectOne(new LambdaQueryWrapper<ConvertTemplateEntity>()
|
||||
.eq(ConvertTemplateEntity::getTemplateCode, templateCode)
|
||||
.eq(ConvertTemplateEntity::getEnabled, 1)
|
||||
.last("limit 1"));
|
||||
if (entity == null) {
|
||||
throw new BusinessException("模板不存在或已禁用");
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public ConvertTemplateVo importTemplate(ConvertTemplateImportRequest request) {
|
||||
if (request.getTemplateContent() == null || request.getTemplateContent().isBlank()) {
|
||||
throw new BusinessException("模板内容不能为空");
|
||||
}
|
||||
ConvertTemplateEntity entity = new ConvertTemplateEntity();
|
||||
entity.setTemplateCode("user_" + System.currentTimeMillis());
|
||||
entity.setTemplateName((request.getTemplateName() == null || request.getTemplateName().isBlank()) ? "自定义 Offer 模板" : request.getTemplateName().trim());
|
||||
entity.setOutputFilename(entity.getTemplateName().endsWith(".txt") ? entity.getTemplateName() : entity.getTemplateName() + ".txt");
|
||||
entity.setRequiredSourceColumnsJson("[\"ASIN\",\"价格\"]");
|
||||
entity.setHeaderColumnsJson("[\"sku\",\"price\",\"quantity\",\"product-id\",\"product-id-type\",\"condition-type\",\"condition-note\",\"ASIN-hint\",\"title\",\"product-tax-code\",\"operation-type\",\"sale-price\",\"sale-start-date\",\"sale-end-date\",\"leadtime-to-ship\",\"launch-date\",\"is-giftwrap-available\",\"is-gift-message-available\"]");
|
||||
entity.setFieldMappingJson("{\"price\":\"价格\",\"product-id\":\"ASIN\"}");
|
||||
entity.setDefaultsJson("{\"quantity\":\"100\",\"product-id-type\":\"ASIN\",\"condition-type\":\"new\",\"leadtime-to-ship\":\"4\"}");
|
||||
entity.setPreambleLinesJson("[\"TemplateType=Offer\\tVersion=1.4\"]");
|
||||
entity.setBlankHeaderRowsAfterSchema(1);
|
||||
entity.setIsDefault(0);
|
||||
entity.setBuiltIn(0);
|
||||
entity.setEnabled(1);
|
||||
entity.setCreatedAt(LocalDateTime.now());
|
||||
entity.setUpdatedAt(LocalDateTime.now());
|
||||
convertTemplateMapper.insert(entity);
|
||||
return toVo(entity);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void setDefaultTemplate(String templateCode) {
|
||||
ConvertTemplateEntity entity = getByCode(templateCode);
|
||||
convertTemplateMapper.selectList(new LambdaQueryWrapper<ConvertTemplateEntity>().eq(ConvertTemplateEntity::getEnabled, 1))
|
||||
.forEach(item -> {
|
||||
item.setIsDefault(item.getId().equals(entity.getId()) ? 1 : 0);
|
||||
item.setUpdatedAt(LocalDateTime.now());
|
||||
convertTemplateMapper.updateById(item);
|
||||
});
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTemplate(String templateCode) {
|
||||
ConvertTemplateEntity entity = getByCode(templateCode);
|
||||
if (Integer.valueOf(1).equals(entity.getBuiltIn())) {
|
||||
throw new BusinessException("内置模板不允许删除");
|
||||
}
|
||||
entity.setEnabled(0);
|
||||
entity.setUpdatedAt(LocalDateTime.now());
|
||||
convertTemplateMapper.updateById(entity);
|
||||
}
|
||||
|
||||
private ConvertTemplateVo toVo(ConvertTemplateEntity entity) {
|
||||
ConvertTemplateVo vo = new ConvertTemplateVo();
|
||||
vo.setId(entity.getTemplateCode());
|
||||
vo.setTemplateCode(entity.getTemplateCode());
|
||||
vo.setTemplateName(entity.getTemplateName());
|
||||
vo.setOutputFilename(entity.getOutputFilename());
|
||||
vo.setIsDefault(Integer.valueOf(1).equals(entity.getIsDefault()));
|
||||
vo.setBuiltIn(Integer.valueOf(1).equals(entity.getBuiltIn()));
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package com.nanri.aiimage.modules.dedupe.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeHistoryVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeRunService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/dedupe")
|
||||
@Tag(name = "数据去重", description = "按照旧 Python 桌面端规则执行列保留和 ID 过滤。桌面端交互不变，只将处理逻辑迁到 Java。")
|
||||
public class DedupeRunController {
|
||||
|
||||
private final DedupeRunService dedupeRunService;
|
||||
|
||||
@PostMapping("/run")
|
||||
@Operation(
|
||||
summary = "执行数据去重",
|
||||
description = """
|
||||
根据上传文件、保留列、ID 保留规则执行数据去重，生成清洗后的 Excel 结果文件。
|
||||
|
||||
规则与旧 Python 桌面端保持一致：
|
||||
- 表头会先做文本清洗；
|
||||
- 遇到“缩略图地址8”停止继续识别表头；
|
||||
- 仅保留 selectedColumns 中指定的列；
|
||||
- keepIntegerIds=true 时保留纯数字 ID；
|
||||
- keepUnderscoreIds=true 时保留类似 1_1 的 ID；
|
||||
- 输出文件命名遵循 原文件名_cleaned.xlsx，若重名自动追加序号。
|
||||
""")
|
||||
public ApiResponse<DedupeRunVo> run(@Valid @RequestBody DedupeRunRequest request) {
|
||||
return ApiResponse.success(dedupeRunService.run(request));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
@Operation(summary = "查询去重历史", description = "查询最近的去重成功记录，用于页面右侧历史下载列表展示。")
|
||||
public ApiResponse<DedupeHistoryVo> history() {
|
||||
DedupeHistoryVo vo = new DedupeHistoryVo();
|
||||
vo.setItems(dedupeRunService.listHistory());
|
||||
return ApiResponse.success(vo);
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除去重历史", description = "删除一条去重历史记录。")
|
||||
public ApiResponse<Void> deleteHistory(@PathVariable Long resultId) {
|
||||
dedupeRunService.deleteHistory(resultId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/results/{resultId}/download")
|
||||
@Operation(summary = "下载数据去重结果", description = "根据结果记录 ID 下载清洗后的 Excel 文件。")
|
||||
public ResponseEntity<Resource> download(@PathVariable Long resultId) {
|
||||
Resource resource = dedupeRunService.getResultFile(resultId);
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment")
|
||||
.body(resource);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重执行请求")
|
||||
public class DedupeRunRequest {
|
||||
|
||||
@Valid
|
||||
@NotEmpty(message = "请先上传待处理文件")
|
||||
@Schema(description = "已上传源文件列表")
|
||||
private List<DedupeSourceFileDto> files;
|
||||
|
||||
@NotEmpty(message = "请至少选择一列")
|
||||
@Schema(description = "保留列")
|
||||
private List<String> selectedColumns;
|
||||
|
||||
@Schema(description = "是否保留纯数字 ID")
|
||||
private boolean keepIntegerIds = true;
|
||||
|
||||
@Schema(description = "是否保留带下划线 ID")
|
||||
private boolean keepUnderscoreIds = true;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重源文件")
|
||||
public class DedupeSourceFileDto {
|
||||
|
||||
@NotBlank(message = "fileKey 不能为空")
|
||||
@Schema(description = "上传接口返回的临时文件键")
|
||||
private String fileKey;
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重历史列表")
|
||||
public class DedupeHistoryVo {
|
||||
|
||||
@Schema(description = "历史记录")
|
||||
private List<DedupeResultItemVo> items;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重结果项")
|
||||
public class DedupeResultItemVo {
|
||||
|
||||
@Schema(description = "结果记录ID")
|
||||
private Long resultId;
|
||||
|
||||
@Schema(description = "源文件名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "结果文件名")
|
||||
private String outputFilename;
|
||||
|
||||
@Schema(description = "是否成功")
|
||||
private boolean success;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "下载地址")
|
||||
private String downloadUrl;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.dedupe.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据去重执行结果")
|
||||
public class DedupeRunVo {
|
||||
|
||||
@Schema(description = "处理文件总数")
|
||||
private Integer total;
|
||||
|
||||
@Schema(description = "成功文件数")
|
||||
private Integer successCount;
|
||||
|
||||
@Schema(description = "失败文件数")
|
||||
private Integer failedCount;
|
||||
|
||||
@Schema(description = "结果列表")
|
||||
private List<DedupeResultItemVo> items;
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package com.nanri.aiimage.modules.dedupe.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeResultItemVo;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DedupeRunService {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final StorageProperties storageProperties;
|
||||
private final OssStorageService ossStorageService;
|
||||
|
||||
public DedupeRunVo run(DedupeRunRequest request) {
|
||||
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds()) {
|
||||
throw new BusinessException("请至少选择一种 ID 保留规则");
|
||||
}
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo("DEDUPE-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType("DEDUPE");
|
||||
task.setTaskMode("IMMEDIATE");
|
||||
task.setStatus("SUCCESS");
|
||||
task.setSourceFileCount(request.getFiles().size());
|
||||
task.setCreatedBy("system");
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
List<DedupeResultItemVo> items = new ArrayList<>();
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
try {
|
||||
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("上传文件不存在，请重新上传");
|
||||
}
|
||||
|
||||
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
|
||||
String outputFilename = buildOutputFilename(inputName);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
|
||||
cleanExcelByLegacyRules(inputFile, outputFile, request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds());
|
||||
|
||||
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE");
|
||||
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
|
||||
|
||||
item.setSuccess(true);
|
||||
item.setOutputFilename(outputFile.getName());
|
||||
item.setDownloadUrl(downloadUrl);
|
||||
successCount++;
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(inputName);
|
||||
resultEntity.setResultFilename(outputFile.getName());
|
||||
resultEntity.setResultFileUrl(downloadUrl);
|
||||
resultEntity.setResultFileSize(outputFile.length());
|
||||
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
} catch (Exception ex) {
|
||||
item.setSuccess(false);
|
||||
item.setError(ex.getMessage());
|
||||
failedCount++;
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("DEDUPE");
|
||||
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
resultEntity.setSuccess(0);
|
||||
resultEntity.setErrorMessage(ex.getMessage());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
}
|
||||
items.add(item);
|
||||
}
|
||||
|
||||
task.setSuccessFileCount(successCount);
|
||||
task.setFailedFileCount(failedCount);
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
DedupeRunVo vo = new DedupeRunVo();
|
||||
vo.setTotal(request.getFiles().size());
|
||||
vo.setSuccessCount(successCount);
|
||||
vo.setFailedCount(failedCount);
|
||||
vo.setItems(items);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public List<DedupeResultItemVo> listHistory() {
|
||||
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, "DEDUPE")
|
||||
.eq(FileResultEntity::getSuccess, 1)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit 50"))
|
||||
.stream()
|
||||
.map(entity -> {
|
||||
DedupeResultItemVo vo = new DedupeResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl());
|
||||
vo.setSuccess(true);
|
||||
return vo;
|
||||
})
|
||||
.toList();
|
||||
}
|
||||
|
||||
public void deleteHistory(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !"DEDUPE".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
}
|
||||
|
||||
public Resource getResultFile(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || entity.getResultFileUrl() == null || !"DEDUPE".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
File file = new File(entity.getResultFileUrl());
|
||||
if (!file.exists()) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
return new FileSystemResource(file);
|
||||
}
|
||||
|
||||
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds) throws Exception {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
|
||||
XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
|
||||
? value
|
||||
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
|
||||
if (headerMap.containsKey(value)) {
|
||||
continue;
|
||||
}
|
||||
headerMap.put(value, i);
|
||||
if ("缩略图地址8".equals(value)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
List<String> missing = selectedColumns.stream().filter(column -> !headerMap.containsKey(column)).toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("缺少列：" + String.join("、", missing));
|
||||
}
|
||||
|
||||
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
|
||||
Row outputHeaderRow = outputSheet.createRow(0);
|
||||
for (int i = 0; i < selectedColumns.size(); i++) {
|
||||
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
|
||||
}
|
||||
|
||||
Integer idColumnIndex = headerMap.get("id");
|
||||
int outputRowIndex = 1;
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
if (idColumnIndex != null) {
|
||||
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
|
||||
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Row outputRow = outputSheet.createRow(outputRowIndex++);
|
||||
for (int i = 0; i < selectedColumns.size(); i++) {
|
||||
Integer sourceIndex = headerMap.get(selectedColumns.get(i));
|
||||
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
|
||||
outputRow.createCell(i).setCellValue(value);
|
||||
}
|
||||
}
|
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
outputWorkbook.write(fos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds) {
|
||||
if (text == null || text.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
if (keepIntegerIds && text.matches("\\d+")) {
|
||||
return true;
|
||||
}
|
||||
return keepUnderscoreIds && text.matches("\\d+_\\d+");
|
||||
}
|
||||
|
||||
private File findLocalSourceFile(String fileKey) {
|
||||
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
|
||||
if (!baseDir.exists()) {
|
||||
return null;
|
||||
}
|
||||
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
|
||||
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
|
||||
}
|
||||
|
||||
private File buildNamedOutputFile(File outputDir, String filename) {
|
||||
File candidate = FileUtil.file(outputDir, filename);
|
||||
if (!candidate.exists()) {
|
||||
return candidate;
|
||||
}
|
||||
String mainName = FileUtil.mainName(filename);
|
||||
String extName = FileUtil.extName(filename);
|
||||
int index = 2;
|
||||
while (true) {
|
||||
String nextFilename = extName == null || extName.isBlank()
|
||||
? mainName + "_" + index
|
||||
: mainName + "_" + index + "." + extName;
|
||||
File nextCandidate = FileUtil.file(outputDir, nextFilename);
|
||||
if (!nextCandidate.exists()) {
|
||||
return nextCandidate;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
}
|
||||
|
||||
private String buildOutputFilename(String originalFilename) {
|
||||
String mainName = originalFilename == null || originalFilename.isBlank() ? "source" : FileUtil.mainName(originalFilename);
|
||||
String extName = originalFilename == null || originalFilename.isBlank() ? "xlsx" : FileUtil.extName(originalFilename);
|
||||
return mainName + "_cleaned." + (extName == null || extName.isBlank() ? "xlsx" : extName);
|
||||
}
|
||||
|
||||
private String normalizeCellText(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.replace("\ufeff", "")
|
||||
.replace("\u3000", " ")
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim()
|
||||
.replaceAll("\\s+", " ");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.nanri.aiimage.modules.file.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.file.model.vo.UploadFileVo;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/files")
|
||||
@Tag(name = "文件上传", description = "用于前端将本地文件上传到后端临时目录，后续任务接口基于 fileKey 进行处理")
|
||||
public class FileUploadController {
|
||||
|
||||
private final LocalFileStorageService localFileStorageService;
|
||||
|
||||
@PostMapping("/upload")
|
||||
@Operation(
|
||||
summary = "上传临时文件",
|
||||
description = """
|
||||
上传本地 Excel 文件到后端临时目录，返回 fileKey。
|
||||
|
||||
该接口不是给浏览器页面直接使用的最终业务接口，而是给 Python 桌面端壳做文件中转：
|
||||
- 桌面端先选本地文件；
|
||||
- 再调用本接口上传到 Java 临时目录；
|
||||
- 然后把 fileKey 传给 dedupe/convert/split 执行接口；
|
||||
- 处理完成后，桌面端再调用下载接口把结果文件保存回用户本地目录。
|
||||
""")
|
||||
public ApiResponse<UploadFileVo> upload(MultipartFile file) throws Exception {
|
||||
return ApiResponse.success(localFileStorageService.saveTempFile(file));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.nanri.aiimage.modules.file.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "上传文件返回信息")
|
||||
public class UploadFileVo {
|
||||
|
||||
@Schema(description = "临时文件键")
|
||||
private String fileKey;
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
|
||||
@Schema(description = "本地暂存路径")
|
||||
private String localPath;
|
||||
|
||||
@Schema(description = "文件大小")
|
||||
private Long size;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.nanri.aiimage.modules.file.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.model.vo.UploadFileVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class LocalFileStorageService {
|
||||
|
||||
private final StorageProperties storageProperties;
|
||||
|
||||
public UploadFileVo saveTempFile(MultipartFile file) throws IOException {
|
||||
File tempDir = FileUtil.file(storageProperties.getLocalTempDir());
|
||||
File parentDir = tempDir.getParentFile();
|
||||
if (parentDir != null) {
|
||||
FileUtil.mkdir(parentDir);
|
||||
}
|
||||
FileUtil.mkdir(tempDir);
|
||||
String fileKey = IdUtil.fastSimpleUUID();
|
||||
String extName = FileUtil.extName(file.getOriginalFilename());
|
||||
File target = FileUtil.file(tempDir, fileKey + (extName.isEmpty() ? "" : "." + extName));
|
||||
file.transferTo(target);
|
||||
|
||||
UploadFileVo vo = new UploadFileVo();
|
||||
vo.setFileKey(fileKey);
|
||||
vo.setOriginalFilename(file.getOriginalFilename());
|
||||
vo.setLocalPath(target.getAbsolutePath());
|
||||
vo.setSize(file.getSize());
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.nanri.aiimage.modules.file.service.oss;
|
||||
|
||||
import com.aliyun.oss.OSS;
|
||||
import com.aliyun.oss.OSSClientBuilder;
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URL;
|
||||
import java.util.Date;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class OssStorageService {
|
||||
|
||||
private final OssProperties ossProperties;
|
||||
|
||||
public String uploadResultFile(File file, String moduleType) {
|
||||
String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName());
|
||||
OSS ossClient = new OSSClientBuilder().build(
|
||||
"https://" + ossProperties.getEndpoint(),
|
||||
ossProperties.getAccessKeyId(),
|
||||
ossProperties.getAccessKeySecret()
|
||||
);
|
||||
try {
|
||||
ossClient.putObject(ossProperties.getBucket(), objectKey, file);
|
||||
return objectKey;
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
public String generateDownloadUrl(String objectKey) {
|
||||
OSS ossClient = new OSSClientBuilder().build(
|
||||
"https://" + ossProperties.getEndpoint(),
|
||||
ossProperties.getAccessKeyId(),
|
||||
ossProperties.getAccessKeySecret()
|
||||
);
|
||||
try {
|
||||
Date expiration = new Date(System.currentTimeMillis() + 3600_000L);
|
||||
URL url = ossClient.generatePresignedUrl(ossProperties.getBucket(), objectKey, expiration);
|
||||
return url.toString();
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.nanri.aiimage.modules.split.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest;
|
||||
import com.nanri.aiimage.modules.split.model.vo.SplitHistoryVo;
|
||||
import com.nanri.aiimage.modules.split.model.vo.SplitRunVo;
|
||||
import com.nanri.aiimage.modules.split.service.SplitRunService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/split")
|
||||
@Tag(name = "数据拆分", description = "按照旧 Python 桌面端规则执行数据拆分。桌面端交互不变，只将处理逻辑迁到 Java。")
|
||||
public class SplitRunController {
|
||||
|
||||
private final SplitRunService splitRunService;
|
||||
|
||||
@PostMapping("/run")
|
||||
@Operation(
|
||||
summary = "执行数据拆分",
|
||||
description = """
|
||||
根据上传文件、保留列、拆分模式和参数执行 Excel 拆分。
|
||||
|
||||
规则与旧 Python 桌面端保持一致：
|
||||
- 表头做文本清洗并兼容历史脏串；
|
||||
- 遇到“缩略图地址8”停止继续识别表头；
|
||||
- 仅保留 selectedColumns 中指定的列；
|
||||
- splitMode=rows_per_file 时按固定条数切分；
|
||||
- splitMode=parts 时按总条数尽量均分；
|
||||
- 输出文件命名遵循 原文件名_split_序号.xlsx；
|
||||
- 返回结果中 rowCount 表示该结果文件包含的数据行数。
|
||||
""")
|
||||
public ApiResponse<SplitRunVo> run(@Valid @RequestBody SplitRunRequest request) {
|
||||
return ApiResponse.success(splitRunService.run(request));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
@Operation(summary = "查询数据拆分历史", description = "查询最近的数据拆分成功记录，用于页面右侧历史下载列表展示。")
|
||||
public ApiResponse<SplitHistoryVo> history() {
|
||||
SplitHistoryVo vo = new SplitHistoryVo();
|
||||
vo.setItems(splitRunService.listHistory());
|
||||
return ApiResponse.success(vo);
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除数据拆分历史", description = "删除一条数据拆分历史记录。")
|
||||
public ApiResponse<Void> deleteHistory(@PathVariable Long resultId) {
|
||||
splitRunService.deleteHistory(resultId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/results/{resultId}/download")
|
||||
@Operation(summary = "下载数据拆分结果", description = "根据结果记录 ID 下载拆分后的 Excel 文件。")
|
||||
public ResponseEntity<Resource> download(@PathVariable Long resultId) {
|
||||
Resource resource = splitRunService.getResultFile(resultId);
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment")
|
||||
.body(resource);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.nanri.aiimage.modules.split.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据拆分执行请求")
|
||||
public class SplitRunRequest {
|
||||
|
||||
@Valid
|
||||
@NotEmpty(message = "请先上传待拆分文件")
|
||||
@Schema(description = "已上传源文件列表")
|
||||
private List<SplitSourceFileDto> files;
|
||||
|
||||
@NotEmpty(message = "请至少选择一列")
|
||||
@Schema(description = "保留列")
|
||||
private List<String> selectedColumns;
|
||||
|
||||
@NotBlank(message = "拆分模式不能为空")
|
||||
@Schema(description = "拆分模式：rows_per_file / parts")
|
||||
private String splitMode;
|
||||
|
||||
@Schema(description = "按每份条数拆分时的条数")
|
||||
private Integer rowsPerFile;
|
||||
|
||||
@Schema(description = "按份数拆分时的份数")
|
||||
private Integer parts;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.split.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据拆分源文件")
|
||||
public class SplitSourceFileDto {
|
||||
|
||||
@NotBlank(message = "fileKey 不能为空")
|
||||
@Schema(description = "上传接口返回的临时文件键")
|
||||
private String fileKey;
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.nanri.aiimage.modules.split.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据拆分历史列表")
|
||||
public class SplitHistoryVo {
|
||||
|
||||
@Schema(description = "历史记录")
|
||||
private List<SplitResultItemVo> items;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.nanri.aiimage.modules.split.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据拆分结果项")
|
||||
public class SplitResultItemVo {
|
||||
|
||||
@Schema(description = "结果记录ID")
|
||||
private Long resultId;
|
||||
|
||||
@Schema(description = "源文件名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "结果文件名")
|
||||
private String outputFilename;
|
||||
|
||||
@Schema(description = "是否成功")
|
||||
private boolean success;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "结果行数")
|
||||
private Integer rowCount;
|
||||
|
||||
@Schema(description = "下载地址")
|
||||
private String downloadUrl;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.split.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "数据拆分执行结果")
|
||||
public class SplitRunVo {
|
||||
|
||||
@Schema(description = "处理文件总数")
|
||||
private Integer total;
|
||||
|
||||
@Schema(description = "成功文件数")
|
||||
private Integer successCount;
|
||||
|
||||
@Schema(description = "失败文件数")
|
||||
private Integer failedCount;
|
||||
|
||||
@Schema(description = "结果列表")
|
||||
private List<SplitResultItemVo> items;
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
package com.nanri.aiimage.modules.split.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest;
|
||||
import com.nanri.aiimage.modules.split.model.dto.SplitSourceFileDto;
|
||||
import com.nanri.aiimage.modules.split.model.vo.SplitResultItemVo;
|
||||
import com.nanri.aiimage.modules.split.model.vo.SplitRunVo;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SplitRunService {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final StorageProperties storageProperties;
|
||||
private final OssStorageService ossStorageService;
|
||||
|
||||
public SplitRunVo run(SplitRunRequest request) {
|
||||
validateRequest(request);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo("SPLIT-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType("SPLIT");
|
||||
task.setTaskMode("IMMEDIATE");
|
||||
task.setStatus("SUCCESS");
|
||||
task.setSourceFileCount(request.getFiles().size());
|
||||
task.setCreatedBy("system");
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
List<SplitResultItemVo> items = new ArrayList<>();
|
||||
int successCount = 0;
|
||||
int failedCount = 0;
|
||||
|
||||
for (SplitSourceFileDto sourceFile : request.getFiles()) {
|
||||
try {
|
||||
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("上传文件不存在，请重新上传");
|
||||
}
|
||||
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
|
||||
List<SplitChunkResult> chunkResults = splitExcelByLegacyRules(inputFile, inputName, request);
|
||||
for (SplitChunkResult chunkResult : chunkResults) {
|
||||
SplitResultItemVo item = new SplitResultItemVo();
|
||||
item.setSourceFilename(inputName);
|
||||
item.setOutputFilename(chunkResult.outputFile().getName());
|
||||
item.setSuccess(true);
|
||||
item.setRowCount(chunkResult.rowCount());
|
||||
|
||||
String ossObjectKey = ossStorageService.uploadResultFile(chunkResult.outputFile(), "SPLIT");
|
||||
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("SPLIT");
|
||||
resultEntity.setSourceFilename(inputName);
|
||||
resultEntity.setResultFilename(chunkResult.outputFile().getName());
|
||||
resultEntity.setResultFileUrl(downloadUrl);
|
||||
resultEntity.setResultFileSize(chunkResult.outputFile().length());
|
||||
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
resultEntity.setRowCount(chunkResult.rowCount());
|
||||
resultEntity.setSuccess(1);
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
item.setDownloadUrl(downloadUrl);
|
||||
item.setResultId(resultEntity.getId());
|
||||
items.add(item);
|
||||
}
|
||||
successCount++;
|
||||
} catch (Exception ex) {
|
||||
SplitResultItemVo item = new SplitResultItemVo();
|
||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
item.setSuccess(false);
|
||||
item.setError(ex.getMessage());
|
||||
items.add(item);
|
||||
failedCount++;
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
resultEntity.setModuleType("SPLIT");
|
||||
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
resultEntity.setSuccess(0);
|
||||
resultEntity.setErrorMessage(ex.getMessage());
|
||||
resultEntity.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(resultEntity);
|
||||
}
|
||||
}
|
||||
|
||||
task.setSuccessFileCount(successCount);
|
||||
task.setFailedFileCount(failedCount);
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
SplitRunVo vo = new SplitRunVo();
|
||||
vo.setTotal(request.getFiles().size());
|
||||
vo.setSuccessCount(successCount);
|
||||
vo.setFailedCount(failedCount);
|
||||
vo.setItems(items);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public List<SplitResultItemVo> listHistory() {
|
||||
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, "SPLIT")
|
||||
.eq(FileResultEntity::getSuccess, 1)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit 100"))
|
||||
.stream()
|
||||
.map(entity -> {
|
||||
SplitResultItemVo vo = new SplitResultItemVo();
|
||||
vo.setResultId(entity.getId());
|
||||
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
|
||||
vo.setOutputFilename(entity.getResultFilename());
|
||||
vo.setDownloadUrl(entity.getResultFileUrl());
|
||||
vo.setRowCount(entity.getRowCount());
|
||||
vo.setSuccess(true);
|
||||
return vo;
|
||||
})
|
||||
.toList();
|
||||
}
|
||||
|
||||
public void deleteHistory(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !"SPLIT".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
}
|
||||
|
||||
public Resource getResultFile(Long resultId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || entity.getResultFileUrl() == null || !"SPLIT".equals(entity.getModuleType())) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
File file = new File(entity.getResultFileUrl());
|
||||
if (!file.exists()) {
|
||||
throw new BusinessException("结果文件不存在");
|
||||
}
|
||||
return new FileSystemResource(file);
|
||||
}
|
||||
|
||||
private List<SplitChunkResult> splitExcelByLegacyRules(File inputFile, String originalFilename, SplitRunRequest request) throws Exception {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
|
||||
? value
|
||||
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
|
||||
if (headerMap.containsKey(value)) {
|
||||
continue;
|
||||
}
|
||||
headerMap.put(value, i);
|
||||
if ("缩略图地址8".equals(value)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
List<String> missing = request.getSelectedColumns().stream().filter(column -> !headerMap.containsKey(column)).toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("缺少列：" + String.join("、", missing));
|
||||
}
|
||||
|
||||
List<List<String>> rowsData = new ArrayList<>();
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
List<String> projected = new ArrayList<>();
|
||||
for (String column : request.getSelectedColumns()) {
|
||||
Integer sourceIndex = headerMap.get(column);
|
||||
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
|
||||
projected.add(value);
|
||||
}
|
||||
rowsData.add(projected);
|
||||
}
|
||||
|
||||
int totalRows = rowsData.size();
|
||||
if (totalRows == 0) {
|
||||
throw new BusinessException("没有可拆分的数据行");
|
||||
}
|
||||
|
||||
List<List<List<String>>> chunks = new ArrayList<>();
|
||||
if ("rows_per_file".equals(request.getSplitMode())) {
|
||||
int chunkSize = request.getRowsPerFile();
|
||||
for (int i = 0; i < totalRows; i += chunkSize) {
|
||||
chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows)));
|
||||
}
|
||||
} else {
|
||||
int actualParts = Math.min(request.getParts(), totalRows);
|
||||
int base = totalRows / actualParts;
|
||||
int remainder = totalRows % actualParts;
|
||||
int start = 0;
|
||||
for (int i = 0; i < actualParts; i++) {
|
||||
int size = base + (i < remainder ? 1 : 0);
|
||||
int end = start + size;
|
||||
chunks.add(rowsData.subList(start, end));
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result"));
|
||||
String sourceStem = FileUtil.mainName(originalFilename);
|
||||
List<SplitChunkResult> results = new ArrayList<>();
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
File outputFile = buildNamedOutputFile(outputDir, sourceStem + "_split_" + (i + 1) + ".xlsx");
|
||||
try (XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
|
||||
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
|
||||
Row outputHeaderRow = outputSheet.createRow(0);
|
||||
for (int colIndex = 0; colIndex < request.getSelectedColumns().size(); colIndex++) {
|
||||
outputHeaderRow.createCell(colIndex).setCellValue(request.getSelectedColumns().get(colIndex));
|
||||
}
|
||||
int outputRowIndex = 1;
|
||||
for (List<String> rowValues : chunks.get(i)) {
|
||||
Row outputRow = outputSheet.createRow(outputRowIndex++);
|
||||
for (int colIndex = 0; colIndex < rowValues.size(); colIndex++) {
|
||||
outputRow.createCell(colIndex).setCellValue(rowValues.get(colIndex));
|
||||
}
|
||||
}
|
||||
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
outputWorkbook.write(fos);
|
||||
}
|
||||
}
|
||||
results.add(new SplitChunkResult(outputFile, chunks.get(i).size()));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
private void validateRequest(SplitRunRequest request) {
|
||||
if (!List.of("rows_per_file", "parts").contains(request.getSplitMode())) {
|
||||
throw new BusinessException("拆分模式不合法");
|
||||
}
|
||||
if ("rows_per_file".equals(request.getSplitMode()) && (request.getRowsPerFile() == null || request.getRowsPerFile() <= 0)) {
|
||||
throw new BusinessException("每份条数不合法");
|
||||
}
|
||||
if ("parts".equals(request.getSplitMode()) && (request.getParts() == null || request.getParts() <= 0)) {
|
||||
throw new BusinessException("拆分份数不合法");
|
||||
}
|
||||
}
|
||||
|
||||
private File findLocalSourceFile(String fileKey) {
|
||||
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
|
||||
if (!baseDir.exists()) {
|
||||
return null;
|
||||
}
|
||||
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
|
||||
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
|
||||
}
|
||||
|
||||
private File buildNamedOutputFile(File outputDir, String filename) {
|
||||
File candidate = FileUtil.file(outputDir, filename);
|
||||
if (!candidate.exists()) {
|
||||
return candidate;
|
||||
}
|
||||
String mainName = FileUtil.mainName(filename);
|
||||
String extName = FileUtil.extName(filename);
|
||||
int index = 2;
|
||||
while (true) {
|
||||
String nextFilename = extName == null || extName.isBlank()
|
||||
? mainName + "_" + index
|
||||
: mainName + "_" + index + "." + extName;
|
||||
File nextCandidate = FileUtil.file(outputDir, nextFilename);
|
||||
if (!nextCandidate.exists()) {
|
||||
return nextCandidate;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeCellText(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.replace("\ufeff", "")
|
||||
.replace("\u3000", " ")
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim()
|
||||
.replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private record SplitChunkResult(File outputFile, int rowCount) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.task.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface FileResultMapper extends BaseMapper<FileResultEntity> {
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.task.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface FileTaskMapper extends BaseMapper<FileTaskEntity> {
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package com.nanri.aiimage.modules.task.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_file_result")
|
||||
public class FileResultEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long taskId;
|
||||
private String moduleType;
|
||||
private String sourceFilename;
|
||||
private String sourceFileUrl;
|
||||
private String resultFilename;
|
||||
private String resultFileUrl;
|
||||
private Long resultFileSize;
|
||||
private String resultContentType;
|
||||
private Integer rowCount;
|
||||
private Integer success;
|
||||
private String errorMessage;
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.nanri.aiimage.modules.task.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_file_task")
|
||||
public class FileTaskEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String taskNo;
|
||||
private String moduleType;
|
||||
private String taskMode;
|
||||
private String status;
|
||||
private Integer sourceFileCount;
|
||||
private Integer successFileCount;
|
||||
private Integer failedFileCount;
|
||||
private String requestJson;
|
||||
private String resultJson;
|
||||
private String errorMessage;
|
||||
private String createdBy;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
private LocalDateTime finishedAt;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
# 这个文件只是给你对照看的“环境变量配置示例”，不是 Spring Boot 自动加载的配置文件。
|
||||
# 你现在最简单的做法：把下面这些键值复制到 IDEA 的 Run/Debug Configuration -> Environment variables。
|
||||
|
||||
AIIMAGE_SERVER_PORT=18080
|
||||
|
||||
AIIMAGE_DB_URL=jdbc:mysql://159.75.121.33:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
|
||||
AIIMAGE_DB_USERNAME=AIimage
|
||||
AIIMAGE_DB_PASSWORD=WTFrb5y6hNLz6hNy
|
||||
|
||||
AIIMAGE_OSS_REGION=cn-hangzhou
|
||||
AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
|
||||
AIIMAGE_OSS_BUCKET=nanri-ai-images
|
||||
AIIMAGE_OSS_ACCESS_KEY_ID=LTAI5tNpyvzMNz9f2dHarsm8
|
||||
AIIMAGE_OSS_ACCESS_KEY_SECRET=bQSZnFH455i8tzyOgeahJmUzwmhynz
|
||||
|
||||
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=F:/project/crawler-plugin/backend-java/data/tmp
|
||||
49
backend-java/src/main/resources/application.yml
Normal file
49
backend-java/src/main/resources/application.yml
Normal file
@@ -0,0 +1,49 @@
|
||||
server:
|
||||
port: ${AIIMAGE_SERVER_PORT:18080}
|
||||
|
||||
spring:
|
||||
application:
|
||||
name: aiimage-backend
|
||||
config:
|
||||
import: optional:classpath:application-local.yml
|
||||
servlet:
|
||||
multipart:
|
||||
max-file-size: 200MB
|
||||
max-request-size: 500MB
|
||||
datasource:
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
url: ${AIIMAGE_DB_URL:jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false}
|
||||
username: ${AIIMAGE_DB_USERNAME:root}
|
||||
password: ${AIIMAGE_DB_PASSWORD:change-me}
|
||||
|
||||
management:
|
||||
health:
|
||||
db:
|
||||
enabled: false
|
||||
|
||||
mybatis-plus:
|
||||
mapper-locations: classpath*:mapper/**/*.xml
|
||||
configuration:
|
||||
map-underscore-to-camel-case: true
|
||||
log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
|
||||
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: true
|
||||
swagger-ui:
|
||||
path: /swagger-ui.html
|
||||
|
||||
knife4j:
|
||||
enable: true
|
||||
setting:
|
||||
language: zh_cn
|
||||
|
||||
aiimage:
|
||||
oss:
|
||||
region: ${AIIMAGE_OSS_REGION:cn-hangzhou}
|
||||
endpoint: ${AIIMAGE_OSS_ENDPOINT:oss-cn-hangzhou.aliyuncs.com}
|
||||
bucket: ${AIIMAGE_OSS_BUCKET:change-me}
|
||||
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:change-me}
|
||||
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me}
|
||||
storage:
|
||||
local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}
|
||||
76
backend-java/src/main/resources/db/V1__init.sql
Normal file
76
backend-java/src/main/resources/db/V1__init.sql
Normal file
@@ -0,0 +1,76 @@
|
||||
CREATE TABLE IF NOT EXISTS biz_file_task (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键',
|
||||
task_no VARCHAR(64) NOT NULL COMMENT '任务编号',
|
||||
module_type VARCHAR(32) NOT NULL COMMENT '模块类型：DEDUPE/CONVERT/SPLIT',
|
||||
task_mode VARCHAR(32) DEFAULT 'IMMEDIATE' COMMENT '任务模式：IMMEDIATE/TASK',
|
||||
status VARCHAR(32) NOT NULL COMMENT '任务状态：PENDING/RUNNING/SUCCESS/FAILED/CANCELLED',
|
||||
source_file_count INT DEFAULT 0 COMMENT '源文件数量',
|
||||
success_file_count INT DEFAULT 0 COMMENT '成功文件数量',
|
||||
failed_file_count INT DEFAULT 0 COMMENT '失败文件数量',
|
||||
request_json JSON NULL COMMENT '请求参数快照',
|
||||
result_json JSON NULL COMMENT '结果快照',
|
||||
error_message VARCHAR(1000) NULL COMMENT '错误信息',
|
||||
created_by VARCHAR(64) DEFAULT 'system' COMMENT '创建人',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
finished_at DATETIME NULL COMMENT '完成时间',
|
||||
UNIQUE KEY uk_task_no (task_no),
|
||||
KEY idx_module_status (module_type, status),
|
||||
KEY idx_created_at (created_at)
|
||||
) COMMENT='文件处理任务表';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS biz_file_result (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键',
|
||||
task_id BIGINT NOT NULL COMMENT '任务ID',
|
||||
module_type VARCHAR(32) NOT NULL COMMENT '模块类型：DEDUPE/CONVERT/SPLIT',
|
||||
source_filename VARCHAR(255) NULL COMMENT '源文件名',
|
||||
source_file_url VARCHAR(1000) NULL COMMENT '源文件地址',
|
||||
result_filename VARCHAR(255) NULL COMMENT '结果文件名',
|
||||
result_file_url VARCHAR(1000) NULL COMMENT '结果文件地址',
|
||||
result_file_size BIGINT NULL COMMENT '结果文件大小',
|
||||
result_content_type VARCHAR(128) NULL COMMENT '结果文件类型',
|
||||
row_count INT NULL COMMENT '结果行数',
|
||||
success TINYINT(1) NOT NULL DEFAULT 1 COMMENT '是否成功',
|
||||
error_message VARCHAR(1000) NULL COMMENT '错误信息',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
KEY idx_task_id (task_id),
|
||||
KEY idx_module_type (module_type)
|
||||
) COMMENT='文件处理结果表';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS biz_convert_template (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键',
|
||||
template_code VARCHAR(64) NOT NULL COMMENT '模板编码',
|
||||
template_name VARCHAR(128) NOT NULL COMMENT '模板名称',
|
||||
output_filename VARCHAR(255) NOT NULL COMMENT '输出文件名',
|
||||
required_source_columns_json JSON NOT NULL COMMENT '源文件必需列',
|
||||
header_columns_json JSON NOT NULL COMMENT '输出表头列',
|
||||
field_mapping_json JSON NOT NULL COMMENT '源字段映射',
|
||||
defaults_json JSON NULL COMMENT '默认值配置',
|
||||
preamble_lines_json JSON NULL COMMENT '前导说明行',
|
||||
blank_header_rows_after_schema INT DEFAULT 0 COMMENT '表头后空白行数',
|
||||
is_default TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否默认模板',
|
||||
built_in TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否内置模板',
|
||||
enabled TINYINT(1) NOT NULL DEFAULT 1 COMMENT '是否启用',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
UNIQUE KEY uk_template_code (template_code)
|
||||
) COMMENT='格式转换模板表';
|
||||
|
||||
INSERT INTO biz_convert_template (
|
||||
template_code, template_name, output_filename,
|
||||
required_source_columns_json, header_columns_json, field_mapping_json,
|
||||
defaults_json, preamble_lines_json, blank_header_rows_after_schema,
|
||||
is_default, built_in, enabled
|
||||
)
|
||||
SELECT
|
||||
'uk_offer', '英国模板', '英国.txt',
|
||||
JSON_ARRAY('ASIN', '价格'),
|
||||
JSON_ARRAY('sku', 'price', 'quantity', 'product-id', 'product-id-type', 'condition-type', 'condition-note', 'ASIN-hint', 'title', 'product-tax-code', 'operation-type', 'sale-price', 'sale-start-date', 'sale-end-date', 'leadtime-to-ship', 'launch-date', 'is-giftwrap-available', 'is-gift-message-available'),
|
||||
JSON_OBJECT('price', '价格', 'product-id', 'ASIN'),
|
||||
JSON_OBJECT('quantity', '100', 'product-id-type', 'ASIN', 'condition-type', 'new', 'leadtime-to-ship', '4'),
|
||||
JSON_ARRAY('TemplateType=Offer\tVersion=1.4'),
|
||||
1,
|
||||
1, 1, 1
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM biz_convert_template WHERE template_code = 'uk_offer'
|
||||
);
|
||||
@@ -0,0 +1,17 @@
|
||||
UPDATE biz_convert_template
|
||||
SET required_source_columns_json = JSON_ARRAY('ASIN', '价格'),
|
||||
header_columns_json = JSON_ARRAY(
|
||||
'sku', 'price', 'quantity', 'product-id', 'product-id-type', 'condition-type', 'condition-note',
|
||||
'ASIN-hint', 'title', 'product-tax-code', 'operation-type', 'sale-price', 'sale-start-date',
|
||||
'sale-end-date', 'leadtime-to-ship', 'launch-date', 'is-giftwrap-available', 'is-gift-message-available'
|
||||
),
|
||||
field_mapping_json = JSON_OBJECT('price', '价格', 'product-id', 'ASIN'),
|
||||
defaults_json = JSON_OBJECT('quantity', '100', 'product-id-type', 'ASIN', 'condition-type', 'new', 'leadtime-to-ship', '4'),
|
||||
preamble_lines_json = JSON_ARRAY('TemplateType=Offer\tVersion=1.4'),
|
||||
blank_header_rows_after_schema = 1,
|
||||
output_filename = '英国.txt',
|
||||
template_name = '英国模板',
|
||||
built_in = 1,
|
||||
enabled = 1,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE template_code = 'uk_offer';
|
||||
Reference in New Issue
Block a user