refactor(phase-7): Excel 栈收口到 POI 并移除 EasyExcel

xlsx 改走 OPCPackage/XSSFReader 事件模型(流式,不整表入内存),
xls 走 HSSF;对外 readFirstSheet/readAllSheets 契约与行回调不变。
解析失败统一收成 IOException,行处理异常按 BusinessException/RuntimeException 原样透传。
pom 删 EasyExcel 4.0.3,只保留 poi-ooxml 5.4.1,消除双 POI 版本劈叉。
This commit is contained in:
2026-10-24 22:31:51 +08:00
parent 969f86c612
commit bb4cf3cdd2
4 changed files with 327 additions and 95 deletions
-6
View File
@@ -19,7 +19,6 @@
<java.version>21</java.version> <java.version>21</java.version>
<knife4j.version>4.5.0</knife4j.version> <knife4j.version>4.5.0</knife4j.version>
<mybatis-plus.version>3.5.7</mybatis-plus.version> <mybatis-plus.version>3.5.7</mybatis-plus.version>
<easyexcel.version>4.0.3</easyexcel.version>
<hutool.version>5.8.36</hutool.version> <hutool.version>5.8.36</hutool.version>
<rocketmq-spring.version>2.3.5</rocketmq-spring.version> <rocketmq-spring.version>2.3.5</rocketmq-spring.version>
<minio.version>8.6.0</minio.version> <minio.version>8.6.0</minio.version>
@@ -83,11 +82,6 @@
<groupId>org.flywaydb</groupId> <groupId>org.flywaydb</groupId>
<artifactId>flyway-mysql</artifactId> <artifactId>flyway-mysql</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easyexcel.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.apache.poi</groupId> <groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId> <artifactId>poi-ooxml</artifactId>
@@ -1,16 +1,43 @@
package com.nanri.aiimage.common.excel; package com.nanri.aiimage.common.excel;
import com.alibaba.excel.EasyExcel; import org.apache.poi.hssf.usermodel.HSSFSheet;
import com.alibaba.excel.context.AnalysisContext; import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import com.alibaba.excel.event.AnalysisEventListener; import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.util.XMLHelper;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler;
import org.apache.poi.xssf.model.SharedStrings;
import org.apache.poi.xssf.model.StylesTable;
import org.apache.poi.xssf.usermodel.XSSFComment;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLFilterImpl;
import javax.xml.parsers.ParserConfigurationException;
import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
/**
* Excel 流式读取:xlsx 走 POI 事件模型(不把整表读成 DOM),xls 走 HSSF
* (BIFF8 单表上限 65,536 行,DOM 占用有界)。
*
* <p>历史上这里用 EasyExcel;它与 poi-ooxml 各带一套 POI 版本,收口到 POI 后
* 只保留一个 Excel 栈。对外契约不变:首 sheet / 全 sheet、表头与数据行回调、
* 行处理异常原样透传、解析异常统一收成 IOException。
*/
public final class ExcelStreamReader { public final class ExcelStreamReader {
private ExcelStreamReader() { private ExcelStreamReader() {
@@ -36,22 +63,147 @@ public final class ExcelStreamReader {
runRead(inputStream, true, handler); runRead(inputStream, true, handler);
} }
private static void runRead(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) throws IOException { private static void runRead(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler)
StreamingListener listener = new StreamingListener(handler); throws IOException {
try { byte[] prefix = inputStream.readNBytes(8);
if (readAllSheets) { InputStream replay = new SequenceInputStream(new ByteArrayInputStream(prefix), inputStream);
EasyExcel.read(inputStream, listener).headRowNumber(1).doReadAll(); if (isOoxml(prefix)) {
} else { readOoxml(replay, readAllSheets, handler);
EasyExcel.read(inputStream, listener).sheet(0).headRowNumber(1).doRead(); } else {
} readHssf(replay, readAllSheets, handler);
} catch (RuntimeException ex) {
// EasyExcel/POI 的解析异常是 RuntimeException,原样冒到接口层会变成 HTTP 500;
// 本工具的契约是 IOException,调用方已按 IOException 给用户回 400。
throw toContractException(listener.handlerFailure(), ex);
} }
} }
private static IOException toContractException(Exception handlerFailure, RuntimeException readFailure) throws IOException { private static boolean isOoxml(byte[] prefix) {
// OOXML 是 zip(PK\x03\x04),xls 是 OLE2(D0 CF 11 E0 A1 B1 1A E1)
return prefix.length >= 4
&& prefix[0] == 'P' && prefix[1] == 'K' && prefix[2] == 3 && prefix[3] == 4;
}
private static void readOoxml(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler)
throws IOException {
FailureRecorder recorder = new FailureRecorder();
try (OPCPackage pkg = OPCPackage.open(inputStream)) {
XSSFReader reader = new XSSFReader(pkg);
StylesTable styles = reader.getStylesTable();
SharedStrings sharedStrings = reader.getSharedStringsTable();
DataFormatter formatter = new DataFormatter();
Iterator<InputStream> sheets = reader.getSheetsData();
XSSFReader.SheetIterator sheetIterator = (XSSFReader.SheetIterator) sheets;
int sheetNo = 0;
while (sheetIterator.hasNext()) {
try (InputStream sheet = sheetIterator.next()) {
String sheetName = sheetIterator.getSheetName();
parseOoxmlSheet(sheet, sheetName, sheetNo, styles, sharedStrings, formatter, handler, recorder);
}
sheetNo++;
if (!readAllSheets) {
break;
}
}
} catch (OpenXML4JException ex) {
throw toContractException(recorder.failure(), ex);
} catch (SAXException ex) {
if (ex.getCause() instanceof RuntimeException runtimeException) {
throw toContractException(recorder.failure(), runtimeException);
}
throw toContractException(recorder.failure(), ex);
} catch (RuntimeException ex) {
throw toContractException(recorder.failure(), ex);
}
}
private static void parseOoxmlSheet(InputStream sheet, String sheetName, int sheetNo,
StylesTable styles, SharedStrings sharedStrings,
DataFormatter formatter, SheetRowHandler handler,
FailureRecorder recorder) throws IOException, SAXException {
XMLReader parser;
try {
parser = XMLHelper.newXMLReader();
} catch (ParserConfigurationException ex) {
throw new IOException("Excel 解析器初始化失败:" + describe(ex), ex);
}
SheetCollector collector = new SheetCollector(sheetName, sheetNo, handler, recorder);
XSSFSheetXMLHandler sheetHandler =
new XSSFSheetXMLHandler(styles, sharedStrings, collector, formatter, false);
DimensionFilter filter = new DimensionFilter(parser, maxRows -> {
try {
handler.onSheetTotal(sheetName, sheetNo, maxRows);
} catch (Exception ex) {
throw recorder.record(ex);
}
});
filter.setContentHandler(sheetHandler);
parser.setContentHandler(filter);
parser.parse(new InputSource(sheet));
}
private static void readHssf(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler)
throws IOException {
FailureRecorder recorder = new FailureRecorder();
DataFormatter formatter = new DataFormatter();
try (HSSFWorkbook workbook = new HSSFWorkbook(inputStream)) {
int sheetCount = workbook.getNumberOfSheets();
for (int sheetNo = 0; sheetNo < sheetCount; sheetNo++) {
if (!readAllSheets && sheetNo > 0) {
break;
}
HSSFSheet sheet = workbook.getSheetAt(sheetNo);
final String sheetName = sheet.getSheetName();
final int currentSheetNo = sheetNo;
final int lastRow = sheet.getLastRowNum();
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
final Map<Integer, String> header = rowToMap(sheet.getRow(0), formatter);
if (!header.isEmpty()) {
callRecorder(recorder, () -> handler.onHeader(sheetName, currentSheetNo, header));
}
for (int rowIndex = 1; rowIndex <= lastRow; rowIndex++) {
Row row = sheet.getRow(rowIndex);
if (row == null) {
continue;
}
final Map<Integer, String> values = rowToMap(row, formatter);
if (values.isEmpty()) {
continue;
}
final int finalRowIndex = rowIndex;
callRecorder(recorder,
() -> handler.onRow(sheetName, currentSheetNo, finalRowIndex, header, values));
}
}
} catch (RuntimeException ex) {
throw toContractException(recorder.failure(), ex);
}
}
private static void callRecorder(FailureRecorder recorder, ThrowingRunnable action) {
try {
action.run();
} catch (Exception ex) {
throw recorder.record(ex);
}
}
private static Map<Integer, String> rowToMap(Row row, DataFormatter formatter) {
Map<Integer, String> values = new LinkedHashMap<>();
if (row == null) {
return values;
}
short first = row.getFirstCellNum();
short last = row.getLastCellNum();
if (first < 0 || last < 0) {
return values;
}
for (int column = first; column < last; column++) {
if (row.getCell(column) == null) {
continue;
}
values.put(column, formatter.formatCellValue(row.getCell(column)));
}
return values;
}
private static IOException toContractException(Exception handlerFailure, Exception readFailure) throws IOException {
if (handlerFailure != null) { if (handlerFailure != null) {
// 调用方自己抛的不算"文件坏了":运行时异常原样透传,保住 BusinessException 的 400 语义。 // 调用方自己抛的不算"文件坏了":运行时异常原样透传,保住 BusinessException 的 400 语义。
if (handlerFailure instanceof IOException ioException) { if (handlerFailure instanceof IOException ioException) {
@@ -83,33 +235,26 @@ public final class ExcelStreamReader {
} }
/** /**
* 表头之后回调本次 sheet 的近似总行数(EasyExcel 基于 sheet 尺寸,可能为 null)。 * 表头之后回调本次 sheet 的近似总行数(POI 维度口径,含表头;维度缺失时为 null)。
* 流式解析无法在读完前得到精确行数,需要展示进度/做前置上限校验的调用方可用它近似。 * 流式解析无法在读完前得到精确行数,需要展示进度/做前置上限校验的调用方可用它近似。
*/ */
default void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) throws Exception { default void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) throws Exception {
} }
void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception; void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap,
Map<Integer, String> rowMap) throws Exception;
} }
private static final class StreamingListener extends AnalysisEventListener<Map<Integer, String>> { private interface ThrowingRunnable {
void run() throws Exception;
}
private final SheetRowHandler handler; private static final class FailureRecorder {
private Map<Integer, String> currentHeaderMap = Map.of(); private Exception failure;
private Exception handlerFailure;
private StreamingListener(SheetRowHandler handler) { private RuntimeException record(Exception ex) {
this.handler = handler; if (failure == null) {
} failure = ex;
private Exception handlerFailure() {
return handlerFailure;
}
/** 记下调用方抛出的异常,好让 runRead 分得清「文件解析坏了」和「行处理被调用方拒了」。 */
private RuntimeException recordHandlerFailure(Exception ex) {
if (handlerFailure == null) {
handlerFailure = ex;
} }
if (ex instanceof RuntimeException runtimeException) { if (ex instanceof RuntimeException runtimeException) {
return runtimeException; return runtimeException;
@@ -117,59 +262,93 @@ public final class ExcelStreamReader {
return new WrappedRuntimeException(ex); return new WrappedRuntimeException(ex);
} }
private Exception failure() {
return failure;
}
}
private static final class SheetCollector implements XSSFSheetXMLHandler.SheetContentsHandler {
private final String sheetName;
private final int sheetNo;
private final SheetRowHandler handler;
private final FailureRecorder recorder;
private Map<Integer, String> header = Map.of();
private Map<Integer, String> row = new LinkedHashMap<>();
private SheetCollector(String sheetName, int sheetNo, SheetRowHandler handler, FailureRecorder recorder) {
this.sheetName = sheetName;
this.sheetNo = sheetNo;
this.handler = handler;
this.recorder = recorder;
}
@Override @Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) { public void startRow(int rowNum) {
Map<Integer, String> normalizedHeadMap = new LinkedHashMap<>(); row = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : headMap.entrySet()) { }
normalizedHeadMap.put(entry.getKey(), entry.getValue());
@Override
public void endRow(int rowNum) {
if (row.isEmpty()) {
return;
} }
currentHeaderMap = normalizedHeadMap; if (rowNum == 0) {
try { header = new LinkedHashMap<>(row);
handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap); callRecorder(recorder, () -> handler.onHeader(sheetName, sheetNo, header));
handler.onSheetTotal(sheetName(context), sheetNo(context), approximateTotalRows(context)); } else {
} catch (Exception ex) { Map<Integer, String> values = new LinkedHashMap<>(row);
throw recordHandlerFailure(ex); int rowIndex = rowNum;
callRecorder(recorder, () -> handler.onRow(sheetName, sheetNo, rowIndex, header, values));
} }
} }
@Override @Override
public void invoke(Map<Integer, String> data, AnalysisContext context) { public void cell(String cellReference, String formattedValue, XSSFComment comment) {
Map<Integer, String> normalizedRow = new LinkedHashMap<>(); if (cellReference == null) {
for (Map.Entry<Integer, String> entry : data.entrySet()) { return;
normalizedRow.put(entry.getKey(), entry.getValue());
}
try {
handler.onRow(
sheetName(context),
sheetNo(context),
context.readRowHolder().getRowIndex(),
currentHeaderMap,
normalizedRow
);
} catch (Exception ex) {
throw recordHandlerFailure(ex);
} }
int column = new CellReference(cellReference).getCol();
row.put(column, formattedValue == null ? "" : formattedValue);
} }
@Override @Override
public void doAfterAllAnalysed(AnalysisContext context) { public void headerFooter(String text, boolean isHeader, String tagName) {
}
}
/** 在 sheetData 之前截获 dimension,给 onSheetTotal 一个近似总行数(含表头)。 */
private static final class DimensionFilter extends XMLFilterImpl {
private final java.util.function.IntConsumer onDimension;
private boolean captured;
private DimensionFilter(XMLReader parent, java.util.function.IntConsumer onDimension) {
super(parent);
this.onDimension = onDimension;
} }
private Integer approximateTotalRows(AnalysisContext context) { @Override
try { public void startElement(String uri, String localName, String qName, Attributes attributes)
return context.readSheetHolder() == null ? null throws SAXException {
: context.readSheetHolder().getApproximateTotalRowNumber(); if (!captured && ("dimension".equals(localName) || "dimension".equals(qName))) {
} catch (Exception ex) { captured = true;
Integer maxRows = maxRows(attributes.getValue("ref"));
if (maxRows != null) {
onDimension.accept(maxRows);
}
}
super.startElement(uri, localName, qName, attributes);
}
private Integer maxRows(String ref) {
if (ref == null || ref.isBlank()) {
return null;
}
String last = ref.contains(":") ? ref.substring(ref.indexOf(':') + 1) : ref;
try {
return new CellReference(last).getRow() + 1;
} catch (RuntimeException ex) {
return null; return null;
} }
}
private String sheetName(AnalysisContext context) {
return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName();
}
private Integer sheetNo(AnalysisContext context) {
return context.readSheetHolder() == null ? null : context.readSheetHolder().getSheetNo();
} }
} }
@@ -1,16 +1,23 @@
package com.nanri.aiimage.common.excel; package com.nanri.aiimage.common.excel;
import com.alibaba.excel.EasyExcel;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.io.TempDir;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
@@ -97,11 +104,42 @@ class ExcelStreamReaderParseFailureTest {
assertEquals("Nike", seen.toString()); assertEquals("Nike", seen.toString());
} }
private Path writeWorkbook() { /** xls 分支单独钉:BIFF8 走 HSSFWorkbook,不能因为 xlsx 改事件模型就把旧格式丢了。 */
@Test
void hssfWorkbookStillReadsRows() throws Exception {
Path file = tempDir.resolve("ok.xls");
try (HSSFWorkbook workbook = new HSSFWorkbook()) {
Sheet sheet = workbook.createSheet("Sheet1");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("品牌");
header.createCell(1).setCellValue("ASIN");
Row data = sheet.createRow(1);
data.createCell(0).setCellValue("Nike");
data.createCell(1).setCellValue("B01ABC");
try (OutputStream out = Files.newOutputStream(file)) {
workbook.write(out);
}
}
StringBuilder seen = new StringBuilder();
ExcelStreamReader.readFirstSheet(file.toFile(), (sheetName, sheetNo, rowIndex, headerMap, rowMap) ->
seen.append(rowMap.get(0)));
assertEquals("Nike", seen.toString());
}
private Path writeWorkbook() throws IOException {
Path file = tempDir.resolve("ok.xlsx"); Path file = tempDir.resolve("ok.xlsx");
List<List<String>> head = List.of(List.of("品牌"), List.of("ASIN")); try (XSSFWorkbook workbook = new XSSFWorkbook()) {
List<List<String>> data = List.of(List.of("Nike", "B01ABC")); Sheet sheet = workbook.createSheet("Sheet1");
EasyExcel.write(file.toFile()).head(head).sheet("Sheet1").doWrite(data); Row header = sheet.createRow(0);
header.createCell(0).setCellValue("品牌");
header.createCell(1).setCellValue("ASIN");
Row data = sheet.createRow(1);
data.createCell(0).setCellValue("Nike");
data.createCell(1).setCellValue("B01ABC");
try (OutputStream out = Files.newOutputStream(file)) {
workbook.write(out);
}
}
return file; return file;
} }
@@ -126,17 +164,28 @@ class ExcelStreamReaderParseFailureTest {
/** 直接喂同一个坏包,取 POI 自己报出来的真因文本,用来断言它没被收口吞掉。 */ /** 直接喂同一个坏包,取 POI 自己报出来的真因文本,用来断言它没被收口吞掉。 */
private String rawParseFailureMessageOf(Path badFile) throws IOException { private String rawParseFailureMessageOf(Path badFile) throws IOException {
String message = null; Throwable failure = null;
try (var input = Files.newInputStream(badFile)) { try (InputStream input = Files.newInputStream(badFile);
EasyExcel.read(input).sheet(0).headRowNumber(1).doRead(); OPCPackage pkg = OPCPackage.open(input)) {
throw new AssertionError("构造的坏包必须让 EasyExcel 抛异常,否则这条用例证明不了收口"); XSSFReader reader = new XSSFReader(pkg);
} catch (RuntimeException ex) { reader.getSharedStringsTable();
Throwable deepest = ex; Iterator<InputStream> sheets = reader.getSheetsData();
while (deepest.getCause() != null) { if (sheets.hasNext()) {
deepest = deepest.getCause(); try (InputStream sheet = sheets.next()) {
sheet.transferTo(OutputStream.nullOutputStream());
}
} }
message = deepest.getMessage(); throw new AssertionError("构造的坏包必须让 POI 抛异常,否则这条用例证明不了收口");
} catch (AssertionError error) {
throw error;
} catch (Exception ex) {
failure = ex;
} }
Throwable deepest = failure;
while (deepest.getCause() != null) {
deepest = deepest.getCause();
}
String message = deepest.getMessage();
assertTrue(message != null && message.length() > 20, assertTrue(message != null && message.length() > 20,
"底层真因文本应当有信息量,实际=" + message); "底层真因文本应当有信息量,实际=" + message);
return message; return message;
@@ -94,8 +94,18 @@ class CatchSilenceRatchetTest {
* {@code ba20f5ed feat(workbench-client)} 那枚新增 catch,不属本批、也未由本批记名**, * {@code ba20f5ed feat(workbench-client)} 那枚新增 catch,不属本批、也未由本批记名**,
* 该由那一笔自己把出处补进这份 diff。每一处本批的新增都在这里点名。 * 该由那一笔自己把出处补进这份 diff。每一处本批的新增都在这里点名。
* 不重抛/只 log/空体三格上限一个字没动。 * 不重抛/只 log/空体三格上限一个字没动。
*
* <p>10-24 Phase 7 EasyExcel→POI 收口再 +4,逐处点名(全部是 RETHROW 形状,气味三格未动):
* <ul>
* <li>{@code ExcelStreamReader.readOoxml} 三枚:{@code OpenXML4JException}(坏包)、
* {@code SAXException}(sheet XML 坏了)、{@code RuntimeException}(POI 解析期异常)
* 全部经 {@code toContractException} 转成 IOException 或原样重抛,不吞;</li>
* <li>{@code ExcelStreamReader.readHssf} 一枚:{@code RuntimeException} 同上收口。</li>
* </ul>
* 这 4 处是替换 EasyExcel 监听器后的等价出口(原实现的 catch 只在 {@code runRead} 一处,
* 但监听器内部另有 POI 的异常分诊);实测 total 1008。
*/ */
static final int STRICT_TOTAL = 1004; static final int STRICT_TOTAL = 1008;
/** /**
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。 * 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。 * 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。