From bb4cf3cdd2e7dff1bcc589bf47e5c61e1eb0b856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 24 Oct 2026 22:31:51 +0800 Subject: [PATCH] =?UTF-8?q?refactor(phase-7):=20Excel=20=E6=A0=88=E6=94=B6?= =?UTF-8?q?=E5=8F=A3=E5=88=B0=20POI=20=E5=B9=B6=E7=A7=BB=E9=99=A4=20EasyEx?= =?UTF-8?q?cel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xlsx 改走 OPCPackage/XSSFReader 事件模型(流式,不整表入内存), xls 走 HSSF;对外 readFirstSheet/readAllSheets 契约与行回调不变。 解析失败统一收成 IOException,行处理异常按 BusinessException/RuntimeException 原样透传。 pom 删 EasyExcel 4.0.3,只保留 poi-ooxml 5.4.1,消除双 POI 版本劈叉。 --- backend-java/pom.xml | 6 - .../common/excel/ExcelStreamReader.java | 325 ++++++++++++++---- .../ExcelStreamReaderParseFailureTest.java | 79 ++++- .../config/CatchSilenceRatchetTest.java | 12 +- 4 files changed, 327 insertions(+), 95 deletions(-) diff --git a/backend-java/pom.xml b/backend-java/pom.xml index f136edfed..85ad9f939 100644 --- a/backend-java/pom.xml +++ b/backend-java/pom.xml @@ -19,7 +19,6 @@ 21 4.5.0 3.5.7 - 4.0.3 5.8.36 2.3.5 8.6.0 @@ -83,11 +82,6 @@ org.flywaydb flyway-mysql - - com.alibaba - easyexcel - ${easyexcel.version} - org.apache.poi poi-ooxml diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/excel/ExcelStreamReader.java b/backend-java/src/main/java/com/nanri/aiimage/common/excel/ExcelStreamReader.java index 48ba73ec0..3aa598ef8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/common/excel/ExcelStreamReader.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/excel/ExcelStreamReader.java @@ -1,16 +1,43 @@ package com.nanri.aiimage.common.excel; -import com.alibaba.excel.EasyExcel; -import com.alibaba.excel.context.AnalysisContext; -import com.alibaba.excel.event.AnalysisEventListener; +import org.apache.poi.hssf.usermodel.HSSFSheet; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.openxml4j.exceptions.OpenXML4JException; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.util.CellReference; +import org.apache.poi.util.XMLHelper; +import org.apache.poi.xssf.eventusermodel.XSSFReader; +import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler; +import org.apache.poi.xssf.model.SharedStrings; +import org.apache.poi.xssf.model.StylesTable; +import org.apache.poi.xssf.usermodel.XSSFComment; +import org.xml.sax.Attributes; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.XMLFilterImpl; +import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +/** + * Excel 流式读取:xlsx 走 POI 事件模型(不把整表读成 DOM),xls 走 HSSF + * (BIFF8 单表上限 65,536 行,DOM 占用有界)。 + * + *

历史上这里用 EasyExcel;它与 poi-ooxml 各带一套 POI 版本,收口到 POI 后 + * 只保留一个 Excel 栈。对外契约不变:首 sheet / 全 sheet、表头与数据行回调、 + * 行处理异常原样透传、解析异常统一收成 IOException。 + */ public final class ExcelStreamReader { private ExcelStreamReader() { @@ -36,22 +63,147 @@ public final class ExcelStreamReader { runRead(inputStream, true, handler); } - private static void runRead(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) throws IOException { - StreamingListener listener = new StreamingListener(handler); - try { - if (readAllSheets) { - EasyExcel.read(inputStream, listener).headRowNumber(1).doReadAll(); - } else { - EasyExcel.read(inputStream, listener).sheet(0).headRowNumber(1).doRead(); - } - } catch (RuntimeException ex) { - // EasyExcel/POI 的解析异常是 RuntimeException,原样冒到接口层会变成 HTTP 500; - // 本工具的契约是 IOException,调用方已按 IOException 给用户回 400。 - throw toContractException(listener.handlerFailure(), ex); + private static void runRead(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) + throws IOException { + byte[] prefix = inputStream.readNBytes(8); + InputStream replay = new SequenceInputStream(new ByteArrayInputStream(prefix), inputStream); + if (isOoxml(prefix)) { + readOoxml(replay, readAllSheets, handler); + } else { + readHssf(replay, readAllSheets, handler); } } - private static IOException toContractException(Exception handlerFailure, RuntimeException readFailure) throws IOException { + private static boolean isOoxml(byte[] prefix) { + // OOXML 是 zip(PK\x03\x04),xls 是 OLE2(D0 CF 11 E0 A1 B1 1A E1) + return prefix.length >= 4 + && prefix[0] == 'P' && prefix[1] == 'K' && prefix[2] == 3 && prefix[3] == 4; + } + + private static void readOoxml(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) + throws IOException { + FailureRecorder recorder = new FailureRecorder(); + try (OPCPackage pkg = OPCPackage.open(inputStream)) { + XSSFReader reader = new XSSFReader(pkg); + StylesTable styles = reader.getStylesTable(); + SharedStrings sharedStrings = reader.getSharedStringsTable(); + DataFormatter formatter = new DataFormatter(); + Iterator sheets = reader.getSheetsData(); + XSSFReader.SheetIterator sheetIterator = (XSSFReader.SheetIterator) sheets; + int sheetNo = 0; + while (sheetIterator.hasNext()) { + try (InputStream sheet = sheetIterator.next()) { + String sheetName = sheetIterator.getSheetName(); + parseOoxmlSheet(sheet, sheetName, sheetNo, styles, sharedStrings, formatter, handler, recorder); + } + sheetNo++; + if (!readAllSheets) { + break; + } + } + } catch (OpenXML4JException ex) { + throw toContractException(recorder.failure(), ex); + } catch (SAXException ex) { + if (ex.getCause() instanceof RuntimeException runtimeException) { + throw toContractException(recorder.failure(), runtimeException); + } + throw toContractException(recorder.failure(), ex); + } catch (RuntimeException ex) { + throw toContractException(recorder.failure(), ex); + } + } + + private static void parseOoxmlSheet(InputStream sheet, String sheetName, int sheetNo, + StylesTable styles, SharedStrings sharedStrings, + DataFormatter formatter, SheetRowHandler handler, + FailureRecorder recorder) throws IOException, SAXException { + XMLReader parser; + try { + parser = XMLHelper.newXMLReader(); + } catch (ParserConfigurationException ex) { + throw new IOException("Excel 解析器初始化失败:" + describe(ex), ex); + } + SheetCollector collector = new SheetCollector(sheetName, sheetNo, handler, recorder); + XSSFSheetXMLHandler sheetHandler = + new XSSFSheetXMLHandler(styles, sharedStrings, collector, formatter, false); + DimensionFilter filter = new DimensionFilter(parser, maxRows -> { + try { + handler.onSheetTotal(sheetName, sheetNo, maxRows); + } catch (Exception ex) { + throw recorder.record(ex); + } + }); + filter.setContentHandler(sheetHandler); + parser.setContentHandler(filter); + parser.parse(new InputSource(sheet)); + } + + private static void readHssf(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) + throws IOException { + FailureRecorder recorder = new FailureRecorder(); + DataFormatter formatter = new DataFormatter(); + try (HSSFWorkbook workbook = new HSSFWorkbook(inputStream)) { + int sheetCount = workbook.getNumberOfSheets(); + for (int sheetNo = 0; sheetNo < sheetCount; sheetNo++) { + if (!readAllSheets && sheetNo > 0) { + break; + } + HSSFSheet sheet = workbook.getSheetAt(sheetNo); + final String sheetName = sheet.getSheetName(); + final int currentSheetNo = sheetNo; + final int lastRow = sheet.getLastRowNum(); + callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1)); + final Map header = rowToMap(sheet.getRow(0), formatter); + if (!header.isEmpty()) { + callRecorder(recorder, () -> handler.onHeader(sheetName, currentSheetNo, header)); + } + for (int rowIndex = 1; rowIndex <= lastRow; rowIndex++) { + Row row = sheet.getRow(rowIndex); + if (row == null) { + continue; + } + final Map values = rowToMap(row, formatter); + if (values.isEmpty()) { + continue; + } + final int finalRowIndex = rowIndex; + callRecorder(recorder, + () -> handler.onRow(sheetName, currentSheetNo, finalRowIndex, header, values)); + } + } + } catch (RuntimeException ex) { + throw toContractException(recorder.failure(), ex); + } + } + + private static void callRecorder(FailureRecorder recorder, ThrowingRunnable action) { + try { + action.run(); + } catch (Exception ex) { + throw recorder.record(ex); + } + } + + private static Map rowToMap(Row row, DataFormatter formatter) { + Map values = new LinkedHashMap<>(); + if (row == null) { + return values; + } + short first = row.getFirstCellNum(); + short last = row.getLastCellNum(); + if (first < 0 || last < 0) { + return values; + } + for (int column = first; column < last; column++) { + if (row.getCell(column) == null) { + continue; + } + values.put(column, formatter.formatCellValue(row.getCell(column))); + } + return values; + } + + private static IOException toContractException(Exception handlerFailure, Exception readFailure) throws IOException { if (handlerFailure != null) { // 调用方自己抛的不算"文件坏了":运行时异常原样透传,保住 BusinessException 的 400 语义。 if (handlerFailure instanceof IOException ioException) { @@ -83,33 +235,26 @@ public final class ExcelStreamReader { } /** - * 表头之后回调本次 sheet 的近似总行数(EasyExcel 基于 sheet 尺寸,可能为 null)。 + * 表头之后回调本次 sheet 的近似总行数(POI 维度口径,含表头;维度缺失时为 null)。 * 流式解析无法在读完前得到精确行数,需要展示进度/做前置上限校验的调用方可用它近似。 */ default void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) throws Exception { } - void onRow(String sheetName, Integer sheetNo, int rowIndex, Map headerMap, Map rowMap) throws Exception; + void onRow(String sheetName, Integer sheetNo, int rowIndex, Map headerMap, + Map rowMap) throws Exception; } - private static final class StreamingListener extends AnalysisEventListener> { + private interface ThrowingRunnable { + void run() throws Exception; + } - private final SheetRowHandler handler; - private Map currentHeaderMap = Map.of(); - private Exception handlerFailure; + private static final class FailureRecorder { + private Exception failure; - private StreamingListener(SheetRowHandler handler) { - this.handler = handler; - } - - private Exception handlerFailure() { - return handlerFailure; - } - - /** 记下调用方抛出的异常,好让 runRead 分得清「文件解析坏了」和「行处理被调用方拒了」。 */ - private RuntimeException recordHandlerFailure(Exception ex) { - if (handlerFailure == null) { - handlerFailure = ex; + private RuntimeException record(Exception ex) { + if (failure == null) { + failure = ex; } if (ex instanceof RuntimeException runtimeException) { return runtimeException; @@ -117,59 +262,93 @@ public final class ExcelStreamReader { return new WrappedRuntimeException(ex); } + private Exception failure() { + return failure; + } + } + + private static final class SheetCollector implements XSSFSheetXMLHandler.SheetContentsHandler { + private final String sheetName; + private final int sheetNo; + private final SheetRowHandler handler; + private final FailureRecorder recorder; + private Map header = Map.of(); + private Map row = new LinkedHashMap<>(); + + private SheetCollector(String sheetName, int sheetNo, SheetRowHandler handler, FailureRecorder recorder) { + this.sheetName = sheetName; + this.sheetNo = sheetNo; + this.handler = handler; + this.recorder = recorder; + } + @Override - public void invokeHeadMap(Map headMap, AnalysisContext context) { - Map normalizedHeadMap = new LinkedHashMap<>(); - for (Map.Entry entry : headMap.entrySet()) { - normalizedHeadMap.put(entry.getKey(), entry.getValue()); + public void startRow(int rowNum) { + row = new LinkedHashMap<>(); + } + + @Override + public void endRow(int rowNum) { + if (row.isEmpty()) { + return; } - currentHeaderMap = normalizedHeadMap; - try { - handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap); - handler.onSheetTotal(sheetName(context), sheetNo(context), approximateTotalRows(context)); - } catch (Exception ex) { - throw recordHandlerFailure(ex); + if (rowNum == 0) { + header = new LinkedHashMap<>(row); + callRecorder(recorder, () -> handler.onHeader(sheetName, sheetNo, header)); + } else { + Map values = new LinkedHashMap<>(row); + int rowIndex = rowNum; + callRecorder(recorder, () -> handler.onRow(sheetName, sheetNo, rowIndex, header, values)); } } @Override - public void invoke(Map data, AnalysisContext context) { - Map normalizedRow = new LinkedHashMap<>(); - for (Map.Entry entry : data.entrySet()) { - normalizedRow.put(entry.getKey(), entry.getValue()); - } - try { - handler.onRow( - sheetName(context), - sheetNo(context), - context.readRowHolder().getRowIndex(), - currentHeaderMap, - normalizedRow - ); - } catch (Exception ex) { - throw recordHandlerFailure(ex); + public void cell(String cellReference, String formattedValue, XSSFComment comment) { + if (cellReference == null) { + return; } + int column = new CellReference(cellReference).getCol(); + row.put(column, formattedValue == null ? "" : formattedValue); } @Override - public void doAfterAllAnalysed(AnalysisContext context) { + public void headerFooter(String text, boolean isHeader, String tagName) { + } + } + + /** 在 sheetData 之前截获 dimension,给 onSheetTotal 一个近似总行数(含表头)。 */ + private static final class DimensionFilter extends XMLFilterImpl { + private final java.util.function.IntConsumer onDimension; + private boolean captured; + + private DimensionFilter(XMLReader parent, java.util.function.IntConsumer onDimension) { + super(parent); + this.onDimension = onDimension; } - private Integer approximateTotalRows(AnalysisContext context) { - try { - return context.readSheetHolder() == null ? null - : context.readSheetHolder().getApproximateTotalRowNumber(); - } catch (Exception ex) { + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) + throws SAXException { + if (!captured && ("dimension".equals(localName) || "dimension".equals(qName))) { + captured = true; + Integer maxRows = maxRows(attributes.getValue("ref")); + if (maxRows != null) { + onDimension.accept(maxRows); + } + } + super.startElement(uri, localName, qName, attributes); + } + + private Integer maxRows(String ref) { + if (ref == null || ref.isBlank()) { + return null; + } + String last = ref.contains(":") ? ref.substring(ref.indexOf(':') + 1) : ref; + try { + return new CellReference(last).getRow() + 1; + } catch (RuntimeException ex) { return null; } - } - - private String sheetName(AnalysisContext context) { - return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName(); - } - - private Integer sheetNo(AnalysisContext context) { - return context.readSheetHolder() == null ? null : context.readSheetHolder().getSheetNo(); } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/excel/ExcelStreamReaderParseFailureTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/excel/ExcelStreamReaderParseFailureTest.java index 070012946..cf13bc39d 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/common/excel/ExcelStreamReaderParseFailureTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/common/excel/ExcelStreamReaderParseFailureTest.java @@ -1,16 +1,23 @@ package com.nanri.aiimage.common.excel; -import com.alibaba.excel.EasyExcel; import com.nanri.aiimage.common.exception.BusinessException; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.eventusermodel.XSSFReader; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; +import java.util.Iterator; import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -97,11 +104,42 @@ class ExcelStreamReaderParseFailureTest { assertEquals("Nike", seen.toString()); } - private Path writeWorkbook() { + /** xls 分支单独钉:BIFF8 走 HSSFWorkbook,不能因为 xlsx 改事件模型就把旧格式丢了。 */ + @Test + void hssfWorkbookStillReadsRows() throws Exception { + Path file = tempDir.resolve("ok.xls"); + try (HSSFWorkbook workbook = new HSSFWorkbook()) { + Sheet sheet = workbook.createSheet("Sheet1"); + Row header = sheet.createRow(0); + header.createCell(0).setCellValue("品牌"); + header.createCell(1).setCellValue("ASIN"); + Row data = sheet.createRow(1); + data.createCell(0).setCellValue("Nike"); + data.createCell(1).setCellValue("B01ABC"); + try (OutputStream out = Files.newOutputStream(file)) { + workbook.write(out); + } + } + StringBuilder seen = new StringBuilder(); + ExcelStreamReader.readFirstSheet(file.toFile(), (sheetName, sheetNo, rowIndex, headerMap, rowMap) -> + seen.append(rowMap.get(0))); + assertEquals("Nike", seen.toString()); + } + + private Path writeWorkbook() throws IOException { Path file = tempDir.resolve("ok.xlsx"); - List> head = List.of(List.of("品牌"), List.of("ASIN")); - List> data = List.of(List.of("Nike", "B01ABC")); - EasyExcel.write(file.toFile()).head(head).sheet("Sheet1").doWrite(data); + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("Sheet1"); + Row header = sheet.createRow(0); + header.createCell(0).setCellValue("品牌"); + header.createCell(1).setCellValue("ASIN"); + Row data = sheet.createRow(1); + data.createCell(0).setCellValue("Nike"); + data.createCell(1).setCellValue("B01ABC"); + try (OutputStream out = Files.newOutputStream(file)) { + workbook.write(out); + } + } return file; } @@ -126,17 +164,28 @@ class ExcelStreamReaderParseFailureTest { /** 直接喂同一个坏包,取 POI 自己报出来的真因文本,用来断言它没被收口吞掉。 */ private String rawParseFailureMessageOf(Path badFile) throws IOException { - String message = null; - try (var input = Files.newInputStream(badFile)) { - EasyExcel.read(input).sheet(0).headRowNumber(1).doRead(); - throw new AssertionError("构造的坏包必须让 EasyExcel 抛异常,否则这条用例证明不了收口"); - } catch (RuntimeException ex) { - Throwable deepest = ex; - while (deepest.getCause() != null) { - deepest = deepest.getCause(); + Throwable failure = null; + try (InputStream input = Files.newInputStream(badFile); + OPCPackage pkg = OPCPackage.open(input)) { + XSSFReader reader = new XSSFReader(pkg); + reader.getSharedStringsTable(); + Iterator sheets = reader.getSheetsData(); + if (sheets.hasNext()) { + try (InputStream sheet = sheets.next()) { + sheet.transferTo(OutputStream.nullOutputStream()); + } } - message = deepest.getMessage(); + throw new AssertionError("构造的坏包必须让 POI 抛异常,否则这条用例证明不了收口"); + } catch (AssertionError error) { + throw error; + } catch (Exception ex) { + failure = ex; } + Throwable deepest = failure; + while (deepest.getCause() != null) { + deepest = deepest.getCause(); + } + String message = deepest.getMessage(); assertTrue(message != null && message.length() > 20, "底层真因文本应当有信息量,实际=" + message); return message; diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java index 6e0783f92..2bd56e971 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java @@ -94,8 +94,18 @@ class CatchSilenceRatchetTest { * {@code ba20f5ed feat(workbench-client)} 那枚新增 catch,不属本批、也未由本批记名**, * 该由那一笔自己把出处补进这份 diff。每一处本批的新增都在这里点名。 * 不重抛/只 log/空体三格上限一个字没动。 + * + *

10-24 Phase 7 EasyExcel→POI 收口再 +4,逐处点名(全部是 RETHROW 形状,气味三格未动): + *

+ * 这 4 处是替换 EasyExcel 监听器后的等价出口(原实现的 catch 只在 {@code runRead} 一处, + * 但监听器内部另有 POI 的异常分诊);实测 total 1008。 */ - static final int STRICT_TOTAL = 1004; + static final int STRICT_TOTAL = 1008; /** * 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。 * 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。