完成匹配、跟价、权限部分
This commit is contained in:
+157
-88
@@ -5,6 +5,7 @@ import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest;
|
||||
@@ -17,20 +18,14 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
@@ -217,84 +212,119 @@ public class SplitRunService {
|
||||
String originalFilename,
|
||||
SplitRunRequest request
|
||||
) throws Exception {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel header row is empty.");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = buildHeaderMap(headerRow, formatter);
|
||||
List<String> missing = request.getSelectedColumns().stream()
|
||||
.filter(column -> !headerMap.containsKey(column))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
|
||||
}
|
||||
|
||||
List<List<String>> rowsData = new ArrayList<>();
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
List<String> projected = new ArrayList<>();
|
||||
for (String column : request.getSelectedColumns()) {
|
||||
Integer sourceIndex = headerMap.get(column);
|
||||
String value = sourceIndex == null
|
||||
? ""
|
||||
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
|
||||
projected.add(value);
|
||||
}
|
||||
rowsData.add(projected);
|
||||
}
|
||||
|
||||
if (rowsData.isEmpty()) {
|
||||
throw new BusinessException("No data rows available for splitting.");
|
||||
}
|
||||
|
||||
List<List<List<String>>> chunks = buildChunks(rowsData, request);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result"));
|
||||
String sourceStem = FileUtil.mainName(originalFilename);
|
||||
List<SplitChunkResult> results = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
File outputFile = buildNamedOutputFile(outputDir, sourceStem + "_split_" + (i + 1) + ".xlsx");
|
||||
try (XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
|
||||
Sheet outputSheet = outputWorkbook.createSheet(sheet.getSheetName());
|
||||
Row outputHeaderRow = outputSheet.createRow(0);
|
||||
for (int colIndex = 0; colIndex < request.getSelectedColumns().size(); colIndex++) {
|
||||
outputHeaderRow.createCell(colIndex).setCellValue(request.getSelectedColumns().get(colIndex));
|
||||
}
|
||||
int outputRowIndex = 1;
|
||||
for (List<String> rowValues : chunks.get(i)) {
|
||||
Row outputRow = outputSheet.createRow(outputRowIndex++);
|
||||
for (int colIndex = 0; colIndex < rowValues.size(); colIndex++) {
|
||||
outputRow.createCell(colIndex).setCellValue(rowValues.get(colIndex));
|
||||
}
|
||||
}
|
||||
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
outputWorkbook.write(fos);
|
||||
}
|
||||
}
|
||||
results.add(new SplitChunkResult(outputFile, chunks.get(i).size()));
|
||||
}
|
||||
|
||||
return results;
|
||||
CountResult countResult = countSplitRows(inputFile, request.getSelectedColumns());
|
||||
if (countResult.rowCount() <= 0) {
|
||||
throw new BusinessException("No data rows available for splitting.");
|
||||
}
|
||||
|
||||
List<Integer> chunkSizes = buildChunkSizes(countResult.rowCount(), request);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result"));
|
||||
String sourceStem = FileUtil.mainName(originalFilename);
|
||||
List<SplitChunkResult> results = new ArrayList<>();
|
||||
List<ChunkWriter> writers = new ArrayList<>();
|
||||
try {
|
||||
for (int i = 0; i < chunkSizes.size(); i++) {
|
||||
File outputFile = buildNamedOutputFile(outputDir, sourceStem + "_split_" + (i + 1) + ".xlsx");
|
||||
ChunkWriter writer = new ChunkWriter(outputFile, countResult.sheetName(), request.getSelectedColumns(), chunkSizes.get(i));
|
||||
writers.add(writer);
|
||||
results.add(new SplitChunkResult(outputFile, chunkSizes.get(i)));
|
||||
}
|
||||
|
||||
Map<String, Integer>[] headerMapHolder = new Map[]{null};
|
||||
int[] chunkIndex = new int[]{0};
|
||||
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
Map<String, Integer> resolvedHeaderMap = buildHeaderMap(headerMap);
|
||||
List<String> missing = request.getSelectedColumns().stream()
|
||||
.filter(column -> !resolvedHeaderMap.containsKey(column))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
|
||||
}
|
||||
headerMapHolder[0] = resolvedHeaderMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception {
|
||||
if (chunkIndex[0] >= writers.size()) {
|
||||
return;
|
||||
}
|
||||
Map<String, Integer> resolvedHeaderMap = headerMapHolder[0];
|
||||
if (resolvedHeaderMap == null) {
|
||||
throw new BusinessException("Excel header row is empty.");
|
||||
}
|
||||
List<String> projected = new ArrayList<>();
|
||||
for (String column : request.getSelectedColumns()) {
|
||||
Integer sourceIndex = resolvedHeaderMap.get(column);
|
||||
projected.add(sourceIndex == null ? "" : normalizeCellText(rowMap.get(sourceIndex)));
|
||||
}
|
||||
ChunkWriter writer = writers.get(chunkIndex[0]);
|
||||
writer.writeRow(projected);
|
||||
if (writer.isFull()) {
|
||||
chunkIndex[0]++;
|
||||
}
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
IOException closeException = null;
|
||||
for (ChunkWriter writer : writers) {
|
||||
try {
|
||||
writer.close();
|
||||
} catch (IOException ex) {
|
||||
if (closeException == null) {
|
||||
closeException = ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (closeException != null) {
|
||||
throw closeException;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Row headerRow, DataFormatter formatter) {
|
||||
private CountResult countSplitRows(File inputFile, List<String> selectedColumns) throws IOException {
|
||||
int[] rowCount = new int[]{0};
|
||||
String[] sheetName = new String[]{"Sheet1"};
|
||||
boolean[] headerRead = new boolean[]{false};
|
||||
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String currentSheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
Map<String, Integer> resolvedHeaderMap = buildHeaderMap(headerMap);
|
||||
List<String> missing = selectedColumns.stream()
|
||||
.filter(column -> !resolvedHeaderMap.containsKey(column))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
|
||||
}
|
||||
sheetName[0] = currentSheetName == null || currentSheetName.isBlank() ? "Sheet1" : currentSheetName;
|
||||
headerRead[0] = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String currentSheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
if (!headerRead[0]) {
|
||||
throw new BusinessException("Excel header row is empty.");
|
||||
}
|
||||
rowCount[0]++;
|
||||
}
|
||||
});
|
||||
if (!headerRead[0]) {
|
||||
throw new BusinessException("Excel header row is empty.");
|
||||
}
|
||||
return new CountResult(rowCount[0], sheetName[0]);
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Map<Integer, String> headerRow) {
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeHeaderName(cell == null ? null : formatter.formatCellValue(cell));
|
||||
for (Map.Entry<Integer, String> entry : headerRow.entrySet()) {
|
||||
String value = normalizeHeaderName(entry.getValue());
|
||||
if (value.isBlank() || headerMap.containsKey(value)) {
|
||||
continue;
|
||||
}
|
||||
headerMap.put(value, i);
|
||||
headerMap.put(value, entry.getKey());
|
||||
if (HEADER_STOP_COLUMN.equals(value)) {
|
||||
break;
|
||||
}
|
||||
@@ -302,14 +332,13 @@ public class SplitRunService {
|
||||
return headerMap;
|
||||
}
|
||||
|
||||
private List<List<List<String>>> buildChunks(List<List<String>> rowsData, SplitRunRequest request) {
|
||||
int totalRows = rowsData.size();
|
||||
List<List<List<String>>> chunks = new ArrayList<>();
|
||||
private List<Integer> buildChunkSizes(int totalRows, SplitRunRequest request) {
|
||||
List<Integer> chunks = new ArrayList<>();
|
||||
|
||||
if (SPLIT_MODE_ROWS_PER_FILE.equals(request.getSplitMode())) {
|
||||
int chunkSize = request.getRowsPerFile();
|
||||
for (int i = 0; i < totalRows; i += chunkSize) {
|
||||
chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows)));
|
||||
chunks.add(Math.min(chunkSize, totalRows - i));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
@@ -317,12 +346,8 @@ public class SplitRunService {
|
||||
int actualParts = Math.min(request.getParts(), totalRows);
|
||||
int base = totalRows / actualParts;
|
||||
int remainder = totalRows % actualParts;
|
||||
int start = 0;
|
||||
for (int i = 0; i < actualParts; i++) {
|
||||
int size = base + (i < remainder ? 1 : 0);
|
||||
int end = start + size;
|
||||
chunks.add(rowsData.subList(start, end));
|
||||
start = end;
|
||||
chunks.add(base + (i < remainder ? 1 : 0));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
@@ -448,4 +473,48 @@ public class SplitRunService {
|
||||
|
||||
private record TaskSplitChunkResult(String relativePath, String sourceFilename, File outputFile, int rowCount) {
|
||||
}
|
||||
|
||||
private record CountResult(int rowCount, String sheetName) {
|
||||
}
|
||||
|
||||
private static final class ChunkWriter implements AutoCloseable {
|
||||
private final File outputFile;
|
||||
private final SXSSFWorkbook workbook;
|
||||
private final org.apache.poi.ss.usermodel.Sheet sheet;
|
||||
private final int rowLimit;
|
||||
private int writtenRows;
|
||||
|
||||
private ChunkWriter(File outputFile, String sheetName, List<String> columns, int rowLimit) {
|
||||
this.outputFile = outputFile;
|
||||
this.rowLimit = rowLimit;
|
||||
this.workbook = new SXSSFWorkbook(200);
|
||||
this.sheet = workbook.createSheet(sheetName == null || sheetName.isBlank() ? "Sheet1" : sheetName);
|
||||
org.apache.poi.ss.usermodel.Row headerRow = sheet.createRow(0);
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
headerRow.createCell(i).setCellValue(columns.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
private void writeRow(List<String> values) {
|
||||
org.apache.poi.ss.usermodel.Row outputRow = sheet.createRow(writtenRows + 1);
|
||||
for (int colIndex = 0; colIndex < values.size(); colIndex++) {
|
||||
outputRow.createCell(colIndex).setCellValue(values.get(colIndex));
|
||||
}
|
||||
writtenRows++;
|
||||
}
|
||||
|
||||
private boolean isFull() {
|
||||
return writtenRows >= rowLimit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
workbook.write(fos);
|
||||
} finally {
|
||||
workbook.dispose();
|
||||
workbook.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user