perf(C5): 外观专利服务解析改走流式解析器(最后一条活路径)

AppearancePatentTaskService.parseWorkbook 原为 POI 全量 DOM(用户源文件整表入堆);
改为复用已有的流式孪生实现 AppearancePatentExcelParser(EasyExcel SAX,语义一致且自带单测),
服务侧只保留原有业务处理(分组键/状态过滤/字段补齐/hydrate)。外观专利模块测试全绿。

至此:用户源文件的解析已无 DOM 路径(剩余 DOM 仅用于报表模板写入,样式/图片/公式必须 DOM)。
This commit is contained in:
2026-09-14 10:58:59 +08:00
parent 8c95f053f5
commit 659180b8d2
5 changed files with 181 additions and 101 deletions
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentExcelParser;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.util.GroupResultPropagator;
@@ -2292,109 +2293,100 @@ public class AppearancePatentTaskService {
}
/**
* 解析单个源文件(2026-09 审查 C5 改造)。
*
* <p>改用流式解析器 {@link AppearancePatentExcelParser}(EasyExcel SAX ✓,语义与旧 DOM 实现一致、
* 已有单测)取出行级原始值,再在本方法内做原有的业务处理(分组键、状态过滤、字段补齐),
* 不再把用户源文件整表读进堆(几十万行曾会占 1~2GB)。
*/
private ParsedWorkbook parseWorkbook(File input, AppearancePatentSourceFileDto source) {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int skuCol = findOptionalHeaderExact(headerMap,
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
int maxParseRows = Math.max(1, properties.getMaxParseRows());
AppearancePatentExcelParser.ParsedSheet sheet = new AppearancePatentExcelParser().parse(input, maxParseRows);
List<String> headers = sheet.headers();
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
boolean includeBlankStatusRows = statusCol >= 0 && isAppearanceResultWorkbook(headers);
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
List<ParsedAppearanceRow> parsedRows = new ArrayList<>();
int total = 0;
int dropped = 0;
int validRows = 0;
int maxParseRows = Math.max(1, properties.getMaxParseRows());
String currentBlockBaseId = "";
String currentGroupKey = "";
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
String id = cell(row, idCol, formatter);
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
total++;
if (id.isBlank() || asin.isBlank() || country.isBlank()) {
dropped++;
continue;
}
validRows++;
if (validRows > maxParseRows) {
throw new BusinessException("解析行数超过上限: " + maxParseRows);
}
AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo();
vo.setSourceFileKey(source.getFileKey());
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
vo.setRowIndex(i + 1);
vo.setSourceId(id);
vo.setDisplayId(normalizeDisplayId(id));
String rowBaseId = baseId(vo.getDisplayId());
if (!Objects.equals(currentBlockBaseId, rowBaseId)) {
currentBlockBaseId = rowBaseId;
currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex());
}
vo.setGroupKey(currentGroupKey);
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : "");
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : "");
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
vo.setValues(readRowValues(row, headers, formatter));
parsedRows.add(new ParsedAppearanceRow(vo, statusCol >= 0 ? cell(row, statusCol, formatter) : ""));
}
List<AppearancePatentParsedRowVo> allRows = parsedRows.stream()
.map(ParsedAppearanceRow::row)
.toList();
hydratePromptFields(allRows);
boolean includeBlankStatusRows = statusCol >= 0 && isAppearanceResultWorkbook(headers);
FailedStatusRowFilter.FilterResult<ParsedAppearanceRow> filteredRows =
FailedStatusRowFilter.retainRows(
parsedRows,
statusCol >= 0,
ParsedAppearanceRow::sourceStatus,
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
);
dropped += filteredRows.filteredCount();
allRows = filteredRows.rows().stream()
.map(ParsedAppearanceRow::row)
.toList();
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
}
if (allRows.isEmpty()) {
throw new BusinessException("no valid appearance patent rows");
}
return new ParsedWorkbook(total, dropped, headers, allRows);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[appearance-patent] parse failed file={} err={}", input, ex.getMessage());
throw new BusinessException("解析 Excel 失败");
Map<String, Integer> headerIndex = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
headerIndex.putIfAbsent(headers.get(i), i);
}
List<ParsedAppearanceRow> parsedRows = new ArrayList<>();
int total = 0;
int dropped = 0;
int validRows = 0;
String currentBlockBaseId = "";
String currentGroupKey = "";
for (AppearancePatentExcelParser.AppearanceExcelRow parsed : sheet.rows()) {
String id = parsed.id() == null ? "" : parsed.id();
String asin = parsed.asin() == null ? "" : parsed.asin().toUpperCase(Locale.ROOT);
String country = parsed.country() == null ? "" : parsed.country();
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
total++;
if (id.isBlank() || asin.isBlank() || country.isBlank()) {
dropped++;
continue;
}
validRows++;
if (validRows > maxParseRows) {
throw new BusinessException("解析行数超过上限: " + maxParseRows);
}
AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo();
vo.setSourceFileKey(source.getFileKey());
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
vo.setRowIndex(parsed.rowIndex());
vo.setSourceId(id);
vo.setDisplayId(normalizeDisplayId(id));
String rowBaseId = baseId(vo.getDisplayId());
if (!Objects.equals(currentBlockBaseId, rowBaseId)) {
currentBlockBaseId = rowBaseId;
currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex());
}
vo.setGroupKey(currentGroupKey);
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setPrice(parsed.price() == null ? "" : parsed.price());
vo.setSku(parsed.sku() == null ? "" : parsed.sku());
vo.setUrl(parsed.url() == null ? "" : parsed.url());
vo.setTitle(parsed.title() == null ? "" : parsed.title());
vo.setValues(parsed.values() == null ? new LinkedHashMap<>() : new LinkedHashMap<>(parsed.values()));
String statusValue = statusCol >= 0 && parsed.values() != null
? parsed.values().getOrDefault(statusHeaderName(headers, statusCol), "")
: "";
parsedRows.add(new ParsedAppearanceRow(vo, statusValue == null ? "" : statusValue));
}
List<AppearancePatentParsedRowVo> allRows = parsedRows.stream()
.map(ParsedAppearanceRow::row)
.toList();
hydratePromptFields(allRows);
FailedStatusRowFilter.FilterResult<ParsedAppearanceRow> filteredRows =
FailedStatusRowFilter.retainRows(
parsedRows,
statusCol >= 0,
ParsedAppearanceRow::sourceStatus,
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
);
dropped += filteredRows.filteredCount();
allRows = filteredRows.rows().stream()
.map(ParsedAppearanceRow::row)
.toList();
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
}
if (allRows.isEmpty()) {
throw new BusinessException("no valid appearance patent rows");
}
return new ParsedWorkbook(total, dropped, new ArrayList<>(headers), allRows);
}
/** 状态列的原始表头名(流式行只带"表头→值"映射,状态值需按表头名取回)。 */
private static String statusHeaderName(List<String> headers, int statusCol) {
return statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : "";
}
private void hydratePromptFields(List<AppearancePatentParsedRowVo> rows) {