super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成匹配、跟价、权限部分

Browse Source
This commit is contained in:
super
2026-04-17 12:52:16 +08:00
parent ecc8b6ce6c
commit d5764f3b50
63 changed files with 8858 additions and 1473 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java
+75 -57
View File
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest;
import com.nanri.aiimage.modules.convert.model.dto.UploadedSourceFileDto;
@@ -19,15 +20,10 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.stereotype.Service;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -242,14 +238,30 @@ public class ConvertRunService {
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
List<String> outputFilenames = resolveOutputFilenames(templateEntity);
List<String> rows = buildTxtRows(inputFile, templateEntity);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
Map<String, BufferedWriter> writers = new LinkedHashMap<>();
for (String outputFilename : outputFilenames) {
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
writers.put(outputFilename, Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8));
}
try {
streamTxtRowsToOutputs(inputFile, templateEntity, writers);
} finally {
IOException closeException = null;
for (BufferedWriter writer : writers.values()) {
try {
writer.close();
} catch (IOException ex) {
if (closeException == null) {
closeException = ex;
}
}
}
if (closeException != null) {
throw closeException;
}
}
return generatedFiles;
}
@@ -266,7 +278,9 @@ public class ConvertRunService {
return List.of(outputFilename);
}
private List<String> buildTxtRows(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
private void streamTxtRowsToOutputs(File inputFile,
ConvertTemplateEntity templateEntity,
Map<String, BufferedWriter> writers) throws IOException {
List<String> requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson());
List<String> headerColumns = readStringList(templateEntity.getHeaderColumnsJson());
Map<String, String> fieldMapping = readStringMap(templateEntity.getFieldMappingJson());
@@ -275,65 +289,60 @@ public class ConvertRunService {
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null
? 0
: templateEntity.getBlankHeaderRowsAfterSchema();
long batchId = System.currentTimeMillis();
int[] rowIndex = new int[]{1};
boolean[] headerRead = new boolean[]{false};
Map<String, Integer>[] headerMapHolder = new Map[]{null};
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel header row is empty.");
}
Map<String, Integer> headerMap = new LinkedHashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
if (!value.isBlank() && !headerMap.containsKey(value)) {
headerMap.put(value, i);
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
Map<String, Integer> resolvedHeaderMap = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
String value = normalizeCellText(entry.getValue());
if (!value.isBlank() && !resolvedHeaderMap.containsKey(value)) {
resolvedHeaderMap.put(value, entry.getKey());
}
}
List<String> missing = requiredColumns.stream()
.filter(column -> !resolvedHeaderMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
headerRead[0] = true;
headerMapHolder[0] = resolvedHeaderMap;
for (String line : preambleLines) {
writeLineToAll(writers, line);
}
writeLineToAll(writers, String.join("\t", headerColumns));
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
writeLineToAll(writers, "\t".repeat(Math.max(0, headerColumns.size() - 1)));
}
}
List<String> missing = requiredColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
List<String> rows = new ArrayList<>();
rows.addAll(preambleLines);
rows.add(String.join("\t", headerColumns));
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
rows.add("\t".repeat(Math.max(0, headerColumns.size() - 1)));
}
long batchId = System.currentTimeMillis();
int rowIndex = 1;
int asinIndex = headerMap.getOrDefault("ASIN", -1);
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
@Override
public void onRow(String sheetName, Integer sheetNo, int excelRowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception {
Map<String, Integer> resolvedHeaderMap = headerMapHolder[0];
if (resolvedHeaderMap == null) {
throw new BusinessException("Excel header row is empty.");
}
String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : "";
int asinIndex = resolvedHeaderMap.getOrDefault("ASIN", -1);
String asin = asinIndex >= 0 ? normalizeCellText(rowMap.get(asinIndex)) : "";
if (asin.isBlank()) {
continue;
return;
}
List<String> lineValues = new ArrayList<>();
for (String column : headerColumns) {
if ("sku".equals(column)) {
lineValues.add(batchId + "-" + rowIndex);
lineValues.add(batchId + "-" + rowIndex[0]);
continue;
}
if (fieldMapping.containsKey(column)) {
String sourceColumn = fieldMapping.get(column);
Integer sourceIndex = headerMap.get(sourceColumn);
String value = sourceIndex == null
? ""
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
lineValues.add(value);
Integer sourceIndex = resolvedHeaderMap.get(sourceColumn);
lineValues.add(sourceIndex == null ? "" : normalizeCellText(rowMap.get(sourceIndex)));
continue;
}
if (defaults.containsKey(column)) {
@@ -343,10 +352,19 @@ public class ConvertRunService {
lineValues.add("");
}
rows.add(String.join("\t", lineValues));
rowIndex++;
writeLineToAll(writers, String.join("\t", lineValues));
rowIndex[0]++;
}
return rows;
});
if (!headerRead[0]) {
throw new BusinessException("Excel header row is empty.");
}
}
private void writeLineToAll(Map<String, BufferedWriter> writers, String line) throws IOException {
for (BufferedWriter writer : writers.values()) {
writer.write(line);
writer.newLine();
}
}