fix(健壮性): Java OOM 三处 + 双TE 502 根因 + admin-vue 403 白屏

Java:
- similarasin Excel 解析改 EasyExcel 流式(原 WorkbookFactory 全量 DOM,大表 OOM)+ 魔数校验
- collectdata 结果组装改游标分批 + 导入改流式(原全量驻留内存)
- GlobalExceptionHandler 转发响应过滤逐跳头与实例标识头(双 Transfer-Encoding 导致 nginx 502 的根因)
admin-vue:
- 403(无后台权限,如工具号 token)与 401 同样跳登录页,修复后台白屏
- task-266 测试断言对齐 daily-files 端点演进
This commit is contained in:
2026-09-13 12:58:49 +08:00
parent 54666fb0e2
commit 995f710d41
8 changed files with 473 additions and 192 deletions
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
@@ -8,10 +9,14 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PushbackInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -37,9 +42,8 @@ public class SimilarAsinExcelParser {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (FileInputStream fis = new FileInputStream(input);
Workbook workbook = WorkbookFactory.create(fis)) {
return parseWorkbook(workbook, maxFieldLength);
try {
return readStreaming(new FileInputStream(input), maxFieldLength);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -52,8 +56,8 @@ public class SimilarAsinExcelParser {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (Workbook workbook = WorkbookFactory.create(input)) {
return parseWorkbook(workbook, DEFAULT_MAX_FIELD_LENGTH);
try {
return readStreaming(input, DEFAULT_MAX_FIELD_LENGTH);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -62,50 +66,152 @@ public class SimilarAsinExcelParser {
}
}
private ParsedSheet parseWorkbook(Workbook workbook, int maxFieldLength) {
DataFormatter formatter = new DataFormatter();
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
/**
* 流式解析(EasyExcel SAX),替代 POI WorkbookFactory 全量 DOM 加载:
* 大表(几十万行/50MB+)不再整表驻留堆内存,且首行前没有可返回的 headers 时报「Excel 表头为空」。
* 语义与原 POI 路径逐字段一致:cell 归一化、错误值转空、单字段截断、表头别名匹配、空行跳过。
*/
private ParsedSheet readStreaming(InputStream inputStream, int maxFieldLength) throws Exception {
// EasyExcel 会把非 zip 文本当 CSV 解析成功;原 WorkbookFactory 只认 xlsx/xls,
// 这里先做文件魔数校验,保持「垃圾文件→解析 Excel 失败」的语义并拒绝 CSV 误解析。
PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(inputStream, 8192), 8);
requireExcelMagic(pb);
SheetContext ctx = new SheetContext(maxFieldLength);
ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
if (headerMap == null || headerMap.isEmpty()) {
throw new BusinessException("Excel 表头为空");
}
ctx.initHeader(headerMap);
}
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
ctx.consumeRow(rowIndex, rowMap);
}
});
if (ctx.headers == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
return new ParsedSheet(ctx.headers, ctx.rows);
}
List<SimilarAsinExcelRow> rows = new ArrayList<>();
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
/** 单表解析上下文:表头就绪后逐行累积结果行。 */
private final class SheetContext {
private final int maxFieldLength;
private DataFormatter formatter = new DataFormatter();
private List<String> headers;
private Map<String, Integer> headerMap;
private int idCol;
private int asinCol;
private int countryCol;
private int skuCol;
private int priceCol;
private int urlCol;
private int titleCol;
private final List<SimilarAsinExcelRow> rows = new ArrayList<>();
SheetContext(int maxFieldLength) {
this.maxFieldLength = maxFieldLength;
}
void initHeader(Map<Integer, String> headerMapRaw) {
// EasyExcel 回调给出 列号 → 表头文本,与 POI Row 遍历等价(缺列一般为 null/空串)
Map<String, Integer> map = new LinkedHashMap<>();
List<String> headerNames = new ArrayList<>();
int lastCol = -1;
for (Map.Entry<Integer, String> e : headerMapRaw.entrySet()) {
if (e.getKey() != null && e.getKey() > lastCol) {
lastCol = e.getKey();
}
}
String id = cell(row, idCol, formatter, maxFieldLength);
String asin = cell(row, asinCol, formatter, maxFieldLength).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter, maxFieldLength);
for (int i = 0; i <= lastCol; i++) {
String val = normalize(headerMapRaw.getOrDefault(i, ""));
headerNames.add(val.isBlank() ? "列" + (i + 1) : val);
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
this.headerMap = map;
this.headers = headerNames;
this.idCol = findRequiredHeader(map, "id");
this.asinCol = findRequiredHeader(map, "asin");
this.countryCol = findRequiredHeader(map, "国家", "country");
this.skuCol = findOptionalHeaderExact(map, "sku", "seller sku", "seller_sku", "msku", "货号");
this.priceCol = findOptionalHeaderExact(map, "价格", "price");
this.urlCol = findOptionalHeaderExact(map,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
this.titleCol = findOptionalHeaderExact(map,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
this.formatter = new DataFormatter();
}
void consumeRow(int rowIndex, Map<Integer, String> rowMap) {
if (headers == null) {
return;
}
// EasyExcel rowIndex 从 0 起(0 为表头),POI 原实现行号同样 0 起并 +1 展示
String id = streamCell(rowMap, idCol, maxFieldLength);
String asin = streamCell(rowMap, asinCol, maxFieldLength).toUpperCase(Locale.ROOT);
String country = streamCell(rowMap, countryCol, maxFieldLength);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
return;
}
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), streamCell(rowMap, i, maxFieldLength));
}
rows.add(new SimilarAsinExcelRow(
i + 1,
rowIndex + 1,
id,
asin,
country,
skuCol >= 0 ? cell(row, skuCol, formatter, maxFieldLength) : "",
priceCol >= 0 ? cell(row, priceCol, formatter, maxFieldLength) : "",
urlCol >= 0 ? cell(row, urlCol, formatter, maxFieldLength) : "",
titleCol >= 0 ? cell(row, titleCol, formatter, maxFieldLength) : "",
readRowValues(row, headers, formatter, maxFieldLength)));
skuCol >= 0 ? streamCell(rowMap, skuCol, maxFieldLength) : "",
priceCol >= 0 ? streamCell(rowMap, priceCol, maxFieldLength) : "",
urlCol >= 0 ? streamCell(rowMap, urlCol, maxFieldLength) : "",
titleCol >= 0 ? streamCell(rowMap, titleCol, maxFieldLength) : "",
values));
}
private String streamCell(Map<Integer, String> rowMap, int col, int maxFieldLength) {
if (col < 0) {
return "";
}
String value = normalize(rowMap.getOrDefault(col, ""));
if (isSpreadsheetErrorValue(value)) {
return "";
}
if (value.length() > maxFieldLength) {
return value.substring(0, maxFieldLength);
}
return value;
}
}
/** 校验文件头魔数:xlsx=PK(zip)、xls=OLE2(CFD)。判非抛「解析 Excel 失败」;用 unread 回退已读字节。 */
private void requireExcelMagic(PushbackInputStream in) throws IOException {
byte[] head = new byte[8];
int n = 0;
while (n < head.length) {
int r = in.read(head, n, head.length - n);
if (r < 0) {
break;
}
n += r;
}
if (n > 0) {
in.unread(head, 0, n);
}
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
boolean isOle2 = n >= 8
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
if (!isZip && !isOle2) {
log.warn("[similar-asin] parse rejected non-excel magic head={}", Arrays.copyOf(head, Math.max(n, 0)));
throw new BusinessException("解析 Excel 失败");
}
return new ParsedSheet(headers, rows);
}
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {