fix(健壮性): Java OOM 三处 + 双TE 502 根因 + admin-vue 403 白屏
Java: - similarasin Excel 解析改 EasyExcel 流式(原 WorkbookFactory 全量 DOM,大表 OOM)+ 魔数校验 - collectdata 结果组装改游标分批 + 导入改流式(原全量驻留内存) - GlobalExceptionHandler 转发响应过滤逐跳头与实例标识头(双 Transfer-Encoding 导致 nginx 502 的根因) admin-vue: - 403(无后台权限,如工具号 token)与 401 同样跳登录页,修复后台白屏 - task-266 测试断言对齐 daily-files 端点演进
This commit is contained in:
+144
-38
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
@@ -8,10 +9,14 @@ import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PushbackInputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -37,9 +42,8 @@ public class SimilarAsinExcelParser {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (FileInputStream fis = new FileInputStream(input);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
return parseWorkbook(workbook, maxFieldLength);
|
||||
try {
|
||||
return readStreaming(new FileInputStream(input), maxFieldLength);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -52,8 +56,8 @@ public class SimilarAsinExcelParser {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (Workbook workbook = WorkbookFactory.create(input)) {
|
||||
return parseWorkbook(workbook, DEFAULT_MAX_FIELD_LENGTH);
|
||||
try {
|
||||
return readStreaming(input, DEFAULT_MAX_FIELD_LENGTH);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -62,50 +66,152 @@ public class SimilarAsinExcelParser {
|
||||
}
|
||||
}
|
||||
|
||||
private ParsedSheet parseWorkbook(Workbook workbook, int maxFieldLength) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) {
|
||||
/**
|
||||
* 流式解析(EasyExcel SAX),替代 POI WorkbookFactory 全量 DOM 加载:
|
||||
* 大表(几十万行/50MB+)不再整表驻留堆内存,且首行前没有可返回的 headers 时报「Excel 表头为空」。
|
||||
* 语义与原 POI 路径逐字段一致:cell 归一化、错误值转空、单字段截断、表头别名匹配、空行跳过。
|
||||
*/
|
||||
private ParsedSheet readStreaming(InputStream inputStream, int maxFieldLength) throws Exception {
|
||||
// EasyExcel 会把非 zip 文本当 CSV 解析成功;原 WorkbookFactory 只认 xlsx/xls,
|
||||
// 这里先做文件魔数校验,保持「垃圾文件→解析 Excel 失败」的语义并拒绝 CSV 误解析。
|
||||
PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(inputStream, 8192), 8);
|
||||
requireExcelMagic(pb);
|
||||
SheetContext ctx = new SheetContext(maxFieldLength);
|
||||
ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
if (headerMap == null || headerMap.isEmpty()) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
ctx.initHeader(headerMap);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
ctx.consumeRow(rowIndex, rowMap);
|
||||
}
|
||||
});
|
||||
if (ctx.headers == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
||||
List<String> headers = readHeaders(header, formatter);
|
||||
int idCol = findRequiredHeader(headerMap, "id");
|
||||
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
|
||||
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||
int urlCol = findOptionalHeaderExact(headerMap,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
int titleCol = findOptionalHeaderExact(headerMap,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
return new ParsedSheet(ctx.headers, ctx.rows);
|
||||
}
|
||||
|
||||
List<SimilarAsinExcelRow> rows = new ArrayList<>();
|
||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
if (row == null) {
|
||||
continue;
|
||||
/** 单表解析上下文:表头就绪后逐行累积结果行。 */
|
||||
private final class SheetContext {
|
||||
private final int maxFieldLength;
|
||||
private DataFormatter formatter = new DataFormatter();
|
||||
private List<String> headers;
|
||||
private Map<String, Integer> headerMap;
|
||||
private int idCol;
|
||||
private int asinCol;
|
||||
private int countryCol;
|
||||
private int skuCol;
|
||||
private int priceCol;
|
||||
private int urlCol;
|
||||
private int titleCol;
|
||||
private final List<SimilarAsinExcelRow> rows = new ArrayList<>();
|
||||
|
||||
SheetContext(int maxFieldLength) {
|
||||
this.maxFieldLength = maxFieldLength;
|
||||
}
|
||||
|
||||
void initHeader(Map<Integer, String> headerMapRaw) {
|
||||
// EasyExcel 回调给出 列号 → 表头文本,与 POI Row 遍历等价(缺列一般为 null/空串)
|
||||
Map<String, Integer> map = new LinkedHashMap<>();
|
||||
List<String> headerNames = new ArrayList<>();
|
||||
int lastCol = -1;
|
||||
for (Map.Entry<Integer, String> e : headerMapRaw.entrySet()) {
|
||||
if (e.getKey() != null && e.getKey() > lastCol) {
|
||||
lastCol = e.getKey();
|
||||
}
|
||||
}
|
||||
String id = cell(row, idCol, formatter, maxFieldLength);
|
||||
String asin = cell(row, asinCol, formatter, maxFieldLength).toUpperCase(Locale.ROOT);
|
||||
String country = cell(row, countryCol, formatter, maxFieldLength);
|
||||
for (int i = 0; i <= lastCol; i++) {
|
||||
String val = normalize(headerMapRaw.getOrDefault(i, ""));
|
||||
headerNames.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||
if (!val.isBlank()) {
|
||||
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||
}
|
||||
}
|
||||
this.headerMap = map;
|
||||
this.headers = headerNames;
|
||||
this.idCol = findRequiredHeader(map, "id");
|
||||
this.asinCol = findRequiredHeader(map, "asin");
|
||||
this.countryCol = findRequiredHeader(map, "国家", "country");
|
||||
this.skuCol = findOptionalHeaderExact(map, "sku", "seller sku", "seller_sku", "msku", "货号");
|
||||
this.priceCol = findOptionalHeaderExact(map, "价格", "price");
|
||||
this.urlCol = findOptionalHeaderExact(map,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
this.titleCol = findOptionalHeaderExact(map,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
this.formatter = new DataFormatter();
|
||||
}
|
||||
|
||||
void consumeRow(int rowIndex, Map<Integer, String> rowMap) {
|
||||
if (headers == null) {
|
||||
return;
|
||||
}
|
||||
// EasyExcel rowIndex 从 0 起(0 为表头),POI 原实现行号同样 0 起并 +1 展示
|
||||
String id = streamCell(rowMap, idCol, maxFieldLength);
|
||||
String asin = streamCell(rowMap, asinCol, maxFieldLength).toUpperCase(Locale.ROOT);
|
||||
String country = streamCell(rowMap, countryCol, maxFieldLength);
|
||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
values.put(headers.get(i), streamCell(rowMap, i, maxFieldLength));
|
||||
}
|
||||
rows.add(new SimilarAsinExcelRow(
|
||||
i + 1,
|
||||
rowIndex + 1,
|
||||
id,
|
||||
asin,
|
||||
country,
|
||||
skuCol >= 0 ? cell(row, skuCol, formatter, maxFieldLength) : "",
|
||||
priceCol >= 0 ? cell(row, priceCol, formatter, maxFieldLength) : "",
|
||||
urlCol >= 0 ? cell(row, urlCol, formatter, maxFieldLength) : "",
|
||||
titleCol >= 0 ? cell(row, titleCol, formatter, maxFieldLength) : "",
|
||||
readRowValues(row, headers, formatter, maxFieldLength)));
|
||||
skuCol >= 0 ? streamCell(rowMap, skuCol, maxFieldLength) : "",
|
||||
priceCol >= 0 ? streamCell(rowMap, priceCol, maxFieldLength) : "",
|
||||
urlCol >= 0 ? streamCell(rowMap, urlCol, maxFieldLength) : "",
|
||||
titleCol >= 0 ? streamCell(rowMap, titleCol, maxFieldLength) : "",
|
||||
values));
|
||||
}
|
||||
|
||||
private String streamCell(Map<Integer, String> rowMap, int col, int maxFieldLength) {
|
||||
if (col < 0) {
|
||||
return "";
|
||||
}
|
||||
String value = normalize(rowMap.getOrDefault(col, ""));
|
||||
if (isSpreadsheetErrorValue(value)) {
|
||||
return "";
|
||||
}
|
||||
if (value.length() > maxFieldLength) {
|
||||
return value.substring(0, maxFieldLength);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
/** 校验文件头魔数:xlsx=PK(zip)、xls=OLE2(CFD)。判非抛「解析 Excel 失败」;用 unread 回退已读字节。 */
|
||||
private void requireExcelMagic(PushbackInputStream in) throws IOException {
|
||||
byte[] head = new byte[8];
|
||||
int n = 0;
|
||||
while (n < head.length) {
|
||||
int r = in.read(head, n, head.length - n);
|
||||
if (r < 0) {
|
||||
break;
|
||||
}
|
||||
n += r;
|
||||
}
|
||||
if (n > 0) {
|
||||
in.unread(head, 0, n);
|
||||
}
|
||||
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
|
||||
boolean isOle2 = n >= 8
|
||||
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
|
||||
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
|
||||
if (!isZip && !isOle2) {
|
||||
log.warn("[similar-asin] parse rejected non-excel magic head={}", Arrays.copyOf(head, Math.max(n, 0)));
|
||||
throw new BusinessException("解析 Excel 失败");
|
||||
}
|
||||
return new ParsedSheet(headers, rows);
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||
|
||||
Reference in New Issue
Block a user