完成新需求

This commit is contained in:
super
2026-03-20 14:26:34 +08:00
parent 43d508f527
commit 92d3fc965b
71 changed files with 3265 additions and 540 deletions
@@ -0,0 +1,75 @@
package com.nanri.aiimage.modules.split.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest;
import com.nanri.aiimage.modules.split.model.vo.SplitHistoryVo;
import com.nanri.aiimage.modules.split.model.vo.SplitRunVo;
import com.nanri.aiimage.modules.split.service.SplitRunService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/split")
@Tag(name = "数据拆分", description = "按照旧 Python 桌面端规则执行数据拆分。桌面端交互不变,只将处理逻辑迁到 Java。")
public class SplitRunController {
private final SplitRunService splitRunService;
@PostMapping("/run")
@Operation(
summary = "执行数据拆分",
description = """
根据上传文件、保留列、拆分模式和参数执行 Excel 拆分。
规则与旧 Python 桌面端保持一致:
- 表头做文本清洗并兼容历史脏串;
- 遇到“缩略图地址8”停止继续识别表头;
- 仅保留 selectedColumns 中指定的列;
- splitMode=rows_per_file 时按固定条数切分;
- splitMode=parts 时按总条数尽量均分;
- 输出文件命名遵循 原文件名_split_序号.xlsx;
- 返回结果中 rowCount 表示该结果文件包含的数据行数。
""")
public ApiResponse<SplitRunVo> run(@Valid @RequestBody SplitRunRequest request) {
return ApiResponse.success(splitRunService.run(request));
}
@GetMapping("/history")
@Operation(summary = "查询数据拆分历史", description = "查询最近的数据拆分成功记录,用于页面右侧历史下载列表展示。")
public ApiResponse<SplitHistoryVo> history() {
SplitHistoryVo vo = new SplitHistoryVo();
vo.setItems(splitRunService.listHistory());
return ApiResponse.success(vo);
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除数据拆分历史", description = "删除一条数据拆分历史记录。")
public ApiResponse<Void> deleteHistory(@PathVariable Long resultId) {
splitRunService.deleteHistory(resultId);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载数据拆分结果", description = "根据结果记录 ID 下载拆分后的 Excel 文件。")
public ResponseEntity<Resource> download(@PathVariable Long resultId) {
Resource resource = splitRunService.getResultFile(resultId);
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment")
.body(resource);
}
}
@@ -0,0 +1,33 @@
package com.nanri.aiimage.modules.split.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "数据拆分执行请求")
public class SplitRunRequest {
@Valid
@NotEmpty(message = "请先上传待拆分文件")
@Schema(description = "已上传源文件列表")
private List<SplitSourceFileDto> files;
@NotEmpty(message = "请至少选择一列")
@Schema(description = "保留列")
private List<String> selectedColumns;
@NotBlank(message = "拆分模式不能为空")
@Schema(description = "拆分模式:rows_per_file / parts")
private String splitMode;
@Schema(description = "按每份条数拆分时的条数")
private Integer rowsPerFile;
@Schema(description = "按份数拆分时的份数")
private Integer parts;
}
@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.split.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@Data
@Schema(description = "数据拆分源文件")
public class SplitSourceFileDto {
@NotBlank(message = "fileKey 不能为空")
@Schema(description = "上传接口返回的临时文件键")
private String fileKey;
@Schema(description = "原始文件名")
private String originalFilename;
}
@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.split.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "数据拆分历史列表")
public class SplitHistoryVo {
@Schema(description = "历史记录")
private List<SplitResultItemVo> items;
}
@@ -0,0 +1,30 @@
package com.nanri.aiimage.modules.split.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "数据拆分结果项")
public class SplitResultItemVo {
@Schema(description = "结果记录ID")
private Long resultId;
@Schema(description = "源文件名")
private String sourceFilename;
@Schema(description = "结果文件名")
private String outputFilename;
@Schema(description = "是否成功")
private boolean success;
@Schema(description = "错误信息")
private String error;
@Schema(description = "结果行数")
private Integer rowCount;
@Schema(description = "下载地址")
private String downloadUrl;
}
@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.split.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "数据拆分执行结果")
public class SplitRunVo {
@Schema(description = "处理文件总数")
private Integer total;
@Schema(description = "成功文件数")
private Integer successCount;
@Schema(description = "失败文件数")
private Integer failedCount;
@Schema(description = "结果列表")
private List<SplitResultItemVo> items;
}
@@ -0,0 +1,333 @@
package com.nanri.aiimage.modules.split.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest;
import com.nanri.aiimage.modules.split.model.dto.SplitSourceFileDto;
import com.nanri.aiimage.modules.split.model.vo.SplitResultItemVo;
import com.nanri.aiimage.modules.split.model.vo.SplitRunVo;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@Service
@RequiredArgsConstructor
public class SplitRunService {
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties;
private final OssStorageService ossStorageService;
public SplitRunVo run(SplitRunRequest request) {
validateRequest(request);
FileTaskEntity task = new FileTaskEntity();
task.setTaskNo("SPLIT-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType("SPLIT");
task.setTaskMode("IMMEDIATE");
task.setStatus("SUCCESS");
task.setSourceFileCount(request.getFiles().size());
task.setCreatedBy("system");
task.setRequestJson(JSONUtil.toJsonStr(request));
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
List<SplitResultItemVo> items = new ArrayList<>();
int successCount = 0;
int failedCount = 0;
for (SplitSourceFileDto sourceFile : request.getFiles()) {
try {
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("上传文件不存在,请重新上传");
}
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
List<SplitChunkResult> chunkResults = splitExcelByLegacyRules(inputFile, inputName, request);
for (SplitChunkResult chunkResult : chunkResults) {
SplitResultItemVo item = new SplitResultItemVo();
item.setSourceFilename(inputName);
item.setOutputFilename(chunkResult.outputFile().getName());
item.setSuccess(true);
item.setRowCount(chunkResult.rowCount());
String ossObjectKey = ossStorageService.uploadResultFile(chunkResult.outputFile(), "SPLIT");
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("SPLIT");
resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(chunkResult.outputFile().getName());
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(chunkResult.outputFile().length());
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
resultEntity.setRowCount(chunkResult.rowCount());
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setDownloadUrl(downloadUrl);
item.setResultId(resultEntity.getId());
items.add(item);
}
successCount++;
} catch (Exception ex) {
SplitResultItemVo item = new SplitResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
item.setSuccess(false);
item.setError(ex.getMessage());
items.add(item);
failedCount++;
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("SPLIT");
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
resultEntity.setSuccess(0);
resultEntity.setErrorMessage(ex.getMessage());
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
}
}
task.setSuccessFileCount(successCount);
task.setFailedFileCount(failedCount);
task.setResultJson(JSONUtil.toJsonStr(items));
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
SplitRunVo vo = new SplitRunVo();
vo.setTotal(request.getFiles().size());
vo.setSuccessCount(successCount);
vo.setFailedCount(failedCount);
vo.setItems(items);
return vo;
}
public List<SplitResultItemVo> listHistory() {
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, "SPLIT")
.eq(FileResultEntity::getSuccess, 1)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100"))
.stream()
.map(entity -> {
SplitResultItemVo vo = new SplitResultItemVo();
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl());
vo.setRowCount(entity.getRowCount());
vo.setSuccess(true);
return vo;
})
.toList();
}
public void deleteHistory(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !"SPLIT".equals(entity.getModuleType())) {
throw new BusinessException("记录不存在");
}
fileResultMapper.deleteById(resultId);
}
public Resource getResultFile(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !"SPLIT".equals(entity.getModuleType())) {
throw new BusinessException("结果文件不存在");
}
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("结果文件不存在");
}
return new FileSystemResource(file);
}
private List<SplitChunkResult> splitExcelByLegacyRules(File inputFile, String originalFilename, SplitRunRequest request) throws Exception {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank()) {
continue;
}
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
? value
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if ("缩略图地址8".equals(value)) {
break;
}
}
List<String> missing = request.getSelectedColumns().stream().filter(column -> !headerMap.containsKey(column)).toList();
if (!missing.isEmpty()) {
throw new BusinessException("缺少列:" + String.join("、", missing));
}
List<List<String>> rowsData = new ArrayList<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
List<String> projected = new ArrayList<>();
for (String column : request.getSelectedColumns()) {
Integer sourceIndex = headerMap.get(column);
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
projected.add(value);
}
rowsData.add(projected);
}
int totalRows = rowsData.size();
if (totalRows == 0) {
throw new BusinessException("没有可拆分的数据行");
}
List<List<List<String>>> chunks = new ArrayList<>();
if ("rows_per_file".equals(request.getSplitMode())) {
int chunkSize = request.getRowsPerFile();
for (int i = 0; i < totalRows; i += chunkSize) {
chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows)));
}
} else {
int actualParts = Math.min(request.getParts(), totalRows);
int base = totalRows / actualParts;
int remainder = totalRows % actualParts;
int start = 0;
for (int i = 0; i < actualParts; i++) {
int size = base + (i < remainder ? 1 : 0);
int end = start + size;
chunks.add(rowsData.subList(start, end));
start = end;
}
}
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result"));
String sourceStem = FileUtil.mainName(originalFilename);
List<SplitChunkResult> results = new ArrayList<>();
for (int i = 0; i < chunks.size(); i++) {
File outputFile = buildNamedOutputFile(outputDir, sourceStem + "_split_" + (i + 1) + ".xlsx");
try (XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
Row outputHeaderRow = outputSheet.createRow(0);
for (int colIndex = 0; colIndex < request.getSelectedColumns().size(); colIndex++) {
outputHeaderRow.createCell(colIndex).setCellValue(request.getSelectedColumns().get(colIndex));
}
int outputRowIndex = 1;
for (List<String> rowValues : chunks.get(i)) {
Row outputRow = outputSheet.createRow(outputRowIndex++);
for (int colIndex = 0; colIndex < rowValues.size(); colIndex++) {
outputRow.createCell(colIndex).setCellValue(rowValues.get(colIndex));
}
}
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
outputWorkbook.write(fos);
}
}
results.add(new SplitChunkResult(outputFile, chunks.get(i).size()));
}
return results;
}
}
private void validateRequest(SplitRunRequest request) {
if (!List.of("rows_per_file", "parts").contains(request.getSplitMode())) {
throw new BusinessException("拆分模式不合法");
}
if ("rows_per_file".equals(request.getSplitMode()) && (request.getRowsPerFile() == null || request.getRowsPerFile() <= 0)) {
throw new BusinessException("每份条数不合法");
}
if ("parts".equals(request.getSplitMode()) && (request.getParts() == null || request.getParts() <= 0)) {
throw new BusinessException("拆分份数不合法");
}
}
private File findLocalSourceFile(String fileKey) {
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
if (!baseDir.exists()) {
return null;
}
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
}
private File buildNamedOutputFile(File outputDir, String filename) {
File candidate = FileUtil.file(outputDir, filename);
if (!candidate.exists()) {
return candidate;
}
String mainName = FileUtil.mainName(filename);
String extName = FileUtil.extName(filename);
int index = 2;
while (true) {
String nextFilename = extName == null || extName.isBlank()
? mainName + "_" + index
: mainName + "_" + index + "." + extName;
File nextCandidate = FileUtil.file(outputDir, nextFilename);
if (!nextCandidate.exists()) {
return nextCandidate;
}
index++;
}
}
private String normalizeCellText(String value) {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")
.replace("\t", " ")
.trim()
.replaceAll("\\s+", " ");
}
private record SplitChunkResult(File outputFile, int rowCount) {
}
}