super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新新增三个模块

Browse Source
This commit is contained in:
super
2026-03-22 11:20:09 +08:00
parent c80e1889ef
commit 497d65d41d
96 changed files with 5393 additions and 588 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java
View File

@@ -20,4 +20,7 @@ public class ConvertRunRequest {
@NotBlank(message = "请选择模板")
@Schema(description = "模板编码，例如 uk_offer")
private String templateId;
@Schema(description = "文件夹上传场景下的总压缩包名称（不含扩展名）")
private String archiveName;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java
View File

@@ -14,4 +14,7 @@ public class UploadedSourceFileDto {
@Schema(description = "原始文件名")
private String originalFilename;
@Schema(description = "相对上传目录路径")
private String relativePath;
}

253
backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java
View File

@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.convert.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
@@ -22,24 +23,38 @@ import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
public class ConvertRunService {
private static final String MODULE_TYPE = "CONVERT";
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
"\u82f1\u56fd.txt",
"\u6cd5\u56fd.txt",
"\u5fb7\u56fd.txt",
"\u897f\u73ed\u7259.txt",
"\u610f\u5927\u5229.txt"
);
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties;
@@ -51,8 +66,8 @@ public class ConvertRunService {
ConvertTemplateEntity template = convertTemplateService.getByCode(request.getTemplateId());
FileTaskEntity task = new FileTaskEntity();
task.setTaskNo("CONVERT-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType("CONVERT");
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType(MODULE_TYPE);
task.setTaskMode("IMMEDIATE");
task.setStatus("SUCCESS");
task.setSourceFileCount(request.getFiles().size());
@@ -63,63 +78,107 @@ public class ConvertRunService {
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
List<ConvertResultItemVo> items = new ArrayList<>();
int successCount = 0;
int failedCount = 0;
List<ConvertArchiveEntry> archiveEntries = new ArrayList<>();
for (UploadedSourceFileDto sourceFile : request.getFiles()) {
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
try {
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("上传文件不存在，请重新上传");
throw new BusinessException("Uploaded source file does not exist, please upload again.");
}
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
String outputFilename = template.getOutputFilename() == null || template.getOutputFilename().isBlank()
? "结果.txt"
: template.getOutputFilename();
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
String inputName = sourceFile.getOriginalFilename() == null
? inputFile.getName()
: sourceFile.getOriginalFilename();
List<GeneratedConvertFile> generatedFiles = generateOutputFiles(inputFile, template);
if (folderMode) {
for (GeneratedConvertFile generatedFile : generatedFiles) {
archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile));
}
successCount++;
continue;
}
writeTxtByLegacyRules(inputFile, outputFile, template);
boolean useZipPackage = request.getFiles().size() == 1 && generatedFiles.size() > 1;
File packagedResultFile = useZipPackage
? packageGeneratedFilesAsZip(inputName, generatedFiles)
: generatedFiles.getFirst().file();
String packagedFilename = useZipPackage
? FileUtil.mainName(inputName) + ".zip"
: generatedFiles.getFirst().filename();
String contentType = useZipPackage ? "application/zip" : "text/plain";
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "CONVERT");
String ossObjectKey = ossStorageService.uploadResultFile(packagedResultFile, MODULE_TYPE);
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(inputName);
item.setSuccess(true);
item.setOutputFilename(outputFilename);
item.setOutputFilename(packagedFilename);
item.setDownloadUrl(downloadUrl);
successCount++;
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("CONVERT");
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(outputFilename);
resultEntity.setResultFilename(packagedFilename);
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType("text/plain");
resultEntity.setResultFileSize(packagedResultFile.length());
resultEntity.setResultContentType(contentType);
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setResultId(resultEntity.getId());
items.add(item);
successCount++;
} catch (Exception ex) {
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
item.setSuccess(false);
item.setError(ex.getMessage());
items.add(item);
failedCount++;
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("CONVERT");
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
resultEntity.setSuccess(0);
resultEntity.setErrorMessage(ex.getMessage());
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
}
items.add(item);
}
if (folderMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderConvertResultsAsZip(request.getArchiveName(), archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(request.getArchiveName());
item.setOutputFilename(zipFile.getName());
item.setSuccess(true);
item.setDownloadUrl(downloadUrl);
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(request.getArchiveName());
resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(zipFile.length());
resultEntity.setResultContentType("application/zip");
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setResultId(resultEntity.getId());
items.add(0, item);
}
task.setSuccessFileCount(successCount);
@@ -138,18 +197,18 @@ public class ConvertRunService {
public List<ConvertResultItemVo> listHistory() {
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, "CONVERT")
.eq(FileResultEntity::getSuccess, 1)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 50"))
.last("limit 200"))
.stream()
.map(entity -> {
ConvertResultItemVo vo = new ConvertResultItemVo();
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl());
vo.setSuccess(true);
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
return vo;
})
.toList();
@@ -157,41 +216,71 @@ public class ConvertRunService {
public void deleteHistory(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !"CONVERT".equals(entity.getModuleType())) {
throw new BusinessException("记录不存在");
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("Convert result record does not exist.");
}
fileResultMapper.deleteById(resultId);
}
public File getResultFile(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !"CONVERT".equals(entity.getModuleType())) {
throw new BusinessException("结果文件不存在");
if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("Convert result file does not exist.");
}
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("结果文件不存在");
throw new BusinessException("Convert result file does not exist.");
}
return file;
}
private void writeTxtByLegacyRules(File inputFile, File outputFile, ConvertTemplateEntity templateEntity) throws IOException {
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
List<String> outputFilenames = resolveOutputFilenames(templateEntity);
List<String> rows = buildTxtRows(inputFile, templateEntity);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
for (String outputFilename : outputFilenames) {
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
}
return generatedFiles;
}
private List<String> resolveOutputFilenames(ConvertTemplateEntity templateEntity) {
if (TEMPLATE_CODE_FIVE_COUNTRIES.equals(templateEntity.getTemplateCode())) {
return FIVE_COUNTRY_OUTPUT_FILES;
}
String outputFilename = templateEntity.getOutputFilename();
if (outputFilename == null || outputFilename.isBlank()) {
return List.of("result.txt");
}
return List.of(outputFilename);
}
private List<String> buildTxtRows(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
List<String> requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson());
List<String> headerColumns = readStringList(templateEntity.getHeaderColumnsJson());
Map<String, String> fieldMapping = readStringMap(templateEntity.getFieldMappingJson());
Map<String, String> defaults = readStringMap(templateEntity.getDefaultsJson());
List<String> preambleLines = readStringList(templateEntity.getPreambleLinesJson());
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null ? 0 : templateEntity.getBlankHeaderRowsAfterSchema();
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null
? 0
: templateEntity.getBlankHeaderRowsAfterSchema();
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
throw new BusinessException("Excel header row is empty.");
}
Map<String, Integer> headerMap = new HashMap<>();
Map<String, Integer> headerMap = new LinkedHashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
if (!value.isBlank() && !headerMap.containsKey(value)) {
@@ -199,9 +288,11 @@ public class ConvertRunService {
}
}
List<String> missing = requiredColumns.stream().filter(column -> !headerMap.containsKey(column)).toList();
List<String> missing = requiredColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("缺少列：" + String.join("、", missing));
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
List<String> rows = new ArrayList<>();
@@ -219,6 +310,7 @@ public class ConvertRunService {
if (row == null) {
continue;
}
String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : "";
if (asin.isBlank()) {
continue;
@@ -233,7 +325,9 @@ public class ConvertRunService {
if (fieldMapping.containsKey(column)) {
String sourceColumn = fieldMapping.get(column);
Integer sourceIndex = headerMap.get(sourceColumn);
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
String value = sourceIndex == null
? ""
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
lineValues.add(value);
continue;
}
@@ -243,10 +337,11 @@ public class ConvertRunService {
}
lineValues.add("");
}
rows.add(String.join("\t", lineValues));
rowIndex++;
}
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
return rows;
}
}
@@ -255,20 +350,70 @@ public class ConvertRunService {
if (json == null || json.isBlank()) {
return new ArrayList<>();
}
return objectMapper.readValue(json, new TypeReference<List<String>>() {});
return objectMapper.readValue(json, new TypeReference<List<String>>() {
});
} catch (Exception ex) {
throw new BusinessException("模板配置解析失败");
throw new BusinessException("Template config parsing failed.");
}
}
private File packageGeneratedFilesAsZip(String inputName, List<GeneratedConvertFile> generatedFiles) {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
File zipFile = buildNamedOutputFile(outputDir, FileUtil.mainName(inputName) + ".zip");
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
for (GeneratedConvertFile generatedFile : generatedFiles) {
ZipEntry entry = new ZipEntry(generatedFile.filename());
zos.putNextEntry(entry);
Files.copy(generatedFile.file().toPath(), zos);
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("Convert result packaging failed: " + ex.getMessage());
}
return zipFile;
}
private File packageFolderConvertResultsAsZip(String archiveName, List<ConvertArchiveEntry> archiveEntries) {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip");
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
for (ConvertArchiveEntry entry : archiveEntries) {
String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.generatedFile().filename(), true);
zos.putNextEntry(new ZipEntry(zipEntryName));
Files.copy(entry.generatedFile().file().toPath(), zos);
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("Convert folder result packaging failed: " + ex.getMessage());
}
return zipFile;
}
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/');
String relativeDir = normalizedRelativePath.contains("/")
? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/'))
: "";
List<String> parts = new ArrayList<>();
if (!relativeDir.isBlank()) {
parts.add(relativeDir);
}
if (includeSourceStemFolder) {
parts.add(FileUtil.mainName(inputName));
}
parts.add(outputFilename);
return String.join("/", parts);
}
private Map<String, String> readStringMap(String json) {
try {
if (json == null || json.isBlank()) {
return new HashMap<>();
}
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {});
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {
});
} catch (Exception ex) {
throw new BusinessException("模板配置解析失败");
throw new BusinessException("Template config parsing failed.");
}
}
@@ -277,8 +422,11 @@ public class ConvertRunService {
if (!baseDir.exists()) {
return null;
}
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
List<File> matchedFiles = FileUtil.loopFiles(
baseDir,
pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)
);
return matchedFiles.isEmpty() ? null : matchedFiles.get(0);
}
private File buildNamedOutputFile(File outputDir, String filename) {
@@ -286,6 +434,7 @@ public class ConvertRunService {
if (!candidate.exists()) {
return candidate;
}
String mainName = FileUtil.mainName(filename);
String extName = FileUtil.extName(filename);
int index = 2;
@@ -314,4 +463,10 @@ public class ConvertRunService {
.trim()
.replaceAll("\\s+", " ");
}
private record GeneratedConvertFile(String filename, File file) {
}
private record ConvertArchiveEntry(String relativePath, String inputName, GeneratedConvertFile generatedFile) {
}
}