super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(C5): 外观专利服务解析改走流式解析器（最后一条活路径）

Browse Source 
AppearancePatentTaskService.parseWorkbook 原为 POI 全量 DOM（用户源文件整表入堆）；
改为复用已有的流式孪生实现 AppearancePatentExcelParser（EasyExcel SAX，语义一致且自带单测），
服务侧只保留原有业务处理（分组键/状态过滤/字段补齐/hydrate）。外观专利模块测试全绿。

至此：用户源文件的解析已无 DOM 路径（剩余 DOM 仅用于报表模板写入，样式/图片/公式必须 DOM）。
This commit is contained in:
黄自达
2026-09-14 10:58:59 +08:00
parent 6a90adc765
commit 375b89154b
5 changed files with 181 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
+92 -100
View File
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentExcelParser;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.util.GroupResultPropagator;
@@ -2292,109 +2293,100 @@ public class AppearancePatentTaskService {
}
/**
* 解析单个源文件（2026-09 审查 C5 改造）。
*
* <p>改用流式解析器 {@link AppearancePatentExcelParser}（EasyExcel SAX ✓，语义与旧 DOM 实现一致、
* 已有单测）取出行级原始值，再在本方法内做原有的业务处理（分组键、状态过滤、字段补齐），
* 不再把用户源文件整表读进堆（几十万行曾会占 1~2GB）。
*/
private ParsedWorkbook parseWorkbook(File input, AppearancePatentSourceFileDto source) {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int skuCol = findOptionalHeaderExact(headerMap,
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
int maxParseRows = Math.max(1, properties.getMaxParseRows());
AppearancePatentExcelParser.ParsedSheet sheet = new AppearancePatentExcelParser().parse(input, maxParseRows);
List<String> headers = sheet.headers();
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
boolean includeBlankStatusRows = statusCol >= 0 && isAppearanceResultWorkbook(headers);
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
List<ParsedAppearanceRow> parsedRows = new ArrayList<>();
int total = 0;
int dropped = 0;
int validRows = 0;
int maxParseRows = Math.max(1, properties.getMaxParseRows());
String currentBlockBaseId = "";
String currentGroupKey = "";
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
String id = cell(row, idCol, formatter);
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
total++;
if (id.isBlank() || asin.isBlank() || country.isBlank()) {
dropped++;
continue;
}
validRows++;
if (validRows > maxParseRows) {
throw new BusinessException("解析行数超过上限: " + maxParseRows);
}
AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo();
vo.setSourceFileKey(source.getFileKey());
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
vo.setRowIndex(i + 1);
vo.setSourceId(id);
vo.setDisplayId(normalizeDisplayId(id));
String rowBaseId = baseId(vo.getDisplayId());
if (!Objects.equals(currentBlockBaseId, rowBaseId)) {
currentBlockBaseId = rowBaseId;
currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex());
}
vo.setGroupKey(currentGroupKey);
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : "");
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : "");
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
vo.setValues(readRowValues(row, headers, formatter));
parsedRows.add(new ParsedAppearanceRow(vo, statusCol >= 0 ? cell(row, statusCol, formatter) : ""));
}
List<AppearancePatentParsedRowVo> allRows = parsedRows.stream()
.map(ParsedAppearanceRow::row)
.toList();
hydratePromptFields(allRows);
boolean includeBlankStatusRows = statusCol >= 0 && isAppearanceResultWorkbook(headers);
FailedStatusRowFilter.FilterResult<ParsedAppearanceRow> filteredRows =
FailedStatusRowFilter.retainRows(
parsedRows,
statusCol >= 0,
ParsedAppearanceRow::sourceStatus,
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
);
dropped += filteredRows.filteredCount();
allRows = filteredRows.rows().stream()
.map(ParsedAppearanceRow::row)
.toList();
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
}
if (allRows.isEmpty()) {
throw new BusinessException("no valid appearance patent rows");
}
return new ParsedWorkbook(total, dropped, headers, allRows);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[appearance-patent] parse failed file={} err={}", input, ex.getMessage());
throw new BusinessException("解析 Excel 失败");
Map<String, Integer> headerIndex = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
headerIndex.putIfAbsent(headers.get(i), i);
}
List<ParsedAppearanceRow> parsedRows = new ArrayList<>();
int total = 0;
int dropped = 0;
int validRows = 0;
String currentBlockBaseId = "";
String currentGroupKey = "";
for (AppearancePatentExcelParser.AppearanceExcelRow parsed : sheet.rows()) {
String id = parsed.id() == null ? "" : parsed.id();
String asin = parsed.asin() == null ? "" : parsed.asin().toUpperCase(Locale.ROOT);
String country = parsed.country() == null ? "" : parsed.country();
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
total++;
if (id.isBlank() || asin.isBlank() || country.isBlank()) {
dropped++;
continue;
}
validRows++;
if (validRows > maxParseRows) {
throw new BusinessException("解析行数超过上限: " + maxParseRows);
}
AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo();
vo.setSourceFileKey(source.getFileKey());
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
vo.setRowIndex(parsed.rowIndex());
vo.setSourceId(id);
vo.setDisplayId(normalizeDisplayId(id));
String rowBaseId = baseId(vo.getDisplayId());
if (!Objects.equals(currentBlockBaseId, rowBaseId)) {
currentBlockBaseId = rowBaseId;
currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex());
}
vo.setGroupKey(currentGroupKey);
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setPrice(parsed.price() == null ? "" : parsed.price());
vo.setSku(parsed.sku() == null ? "" : parsed.sku());
vo.setUrl(parsed.url() == null ? "" : parsed.url());
vo.setTitle(parsed.title() == null ? "" : parsed.title());
vo.setValues(parsed.values() == null ? new LinkedHashMap<>() : new LinkedHashMap<>(parsed.values()));
String statusValue = statusCol >= 0 && parsed.values() != null
? parsed.values().getOrDefault(statusHeaderName(headers, statusCol), "")
: "";
parsedRows.add(new ParsedAppearanceRow(vo, statusValue == null ? "" : statusValue));
}
List<AppearancePatentParsedRowVo> allRows = parsedRows.stream()
.map(ParsedAppearanceRow::row)
.toList();
hydratePromptFields(allRows);
FailedStatusRowFilter.FilterResult<ParsedAppearanceRow> filteredRows =
FailedStatusRowFilter.retainRows(
parsedRows,
statusCol >= 0,
ParsedAppearanceRow::sourceStatus,
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
);
dropped += filteredRows.filteredCount();
allRows = filteredRows.rows().stream()
.map(ParsedAppearanceRow::row)
.toList();
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
}
if (allRows.isEmpty()) {
throw new BusinessException("no valid appearance patent rows");
}
return new ParsedWorkbook(total, dropped, new ArrayList<>(headers), allRows);
}
/** 状态列的原始表头名（流式行只带"表头→值"映射，状态值需按表头名取回）。 */
private static String statusHeaderName(List<String> headers, int statusCol) {
return statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : "";
}
private void hydratePromptFields(List<AppearancePatentParsedRowVo> rows) {
backend-java/src/main/java/com/nanri/aiimage/modules/softwareversion/controller/PublicVersionController.java
+9
View File
@@ -43,4 +43,13 @@ public class PublicVersionController {
public Map<String, Object> latestVersion() {
return softwareVersionService.latestSoftwareVersion();
}
/**
* 公开版本列表：桌面端更新面板"指定版本更新"下拉的数据源（未登录也能取，
* 与 /latest 同为匿名可达）；返回最近若干条，按创建时间倒序。
*/
@GetMapping("/list")
public Map<String, Object> listVersions() {
return softwareVersionService.listPublicSoftwareVersions();
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionService.java
+23
View File
@@ -5,6 +5,7 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.softwareversion.mapper.SoftwareVersionMapper;
import com.nanri.aiimage.modules.softwareversion.model.entity.SoftwareVersionEntity;
import com.nanri.aiimage.modules.softwareversion.service.support.PublicVersionLookup;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -95,6 +96,28 @@ public class SoftwareVersionService {
return result;
}
/**
* 公开版本列表：桌面端"指定版本更新"下拉的可选版本（匿名可达，同 /latest）。
* 按创建时间倒序取最近 {@link PublicVersionLookup#DEFAULT_LIST_LIMIT} 条，字段口径
* 与管理端 /versions 完全一致（id/version/file_url/created_at），前端复用同一套解析。
*/
public Map<String, Object> listPublicSoftwareVersions() {
int limit = PublicVersionLookup.normalizeListLimit(null);
List<SoftwareVersionEntity> entities = softwareVersionMapper.selectList(
new LambdaQueryWrapper<SoftwareVersionEntity>()
.orderByDesc(SoftwareVersionEntity::getCreatedAt)
.orderByDesc(SoftwareVersionEntity::getId)
.last("LIMIT " + limit));
List<Map<String, Object>> items = new ArrayList<>(entities.size());
for (SoftwareVersionEntity entity : entities) {
items.add(toItemMap(entity));
}
log.info("[software-version] 查询公开版本列表 limit={} count={}", limit, items.size());
Map<String, Object> result = new LinkedHashMap<>();
result.put("items", items);
return result;
}
/**
* 上传客户端软件安装包：zip 上传 MinIO（key：nanri-image/versions/{安全版本号}.zip），
* 并向 web_config 追加一条记录（不做"设为最新"，桌面端按 created_at 倒序取第一条）。
backend-java/src/main/java/com/nanri/aiimage/modules/softwareversion/service/support/PublicVersionLookup.java
+18 -1
View File
@@ -15,8 +15,15 @@ public final class PublicVersionLookup {
/** 公开客户端版本路径(与旧 Flask/客户端地址完全一致)。 */
public static final String PUBLIC_VERSION_PATH = "/api/version";
public static final String PUBLIC_VERSION_LATEST_PATH = "/api/version/latest";
/** 公开版本列表路径：桌面端"指定版本更新"下拉取可选版本(匿名可达，同 latest)。 */
public static final String PUBLIC_VERSION_LIST_PATH = "/api/version/list";
public static final String ADMIN_UPLOAD_PATH = "/api/admin/version";
/** 公开列表默认条数：桌面端只需最近若干个版本，公网匿名接口不做无界返回。 */
public static final int DEFAULT_LIST_LIMIT = 50;
/** 公开列表条数上限：防调用方传入超大 limit 拖库。 */
public static final int MAX_LIST_LIMIT = 200;
/** web_config 版本记录投影(id 用于同时间稳定排序)。 */
public record WebVersion(Long id, String version, String fileUrl, LocalDateTime createdAt) {
}
@@ -26,7 +33,17 @@ public final class PublicVersionLookup {
/** 是否属于公开版本接口路径(与后台上传路径解耦)。 */
public static boolean isPublicVersionPath(String path) {
return PUBLIC_VERSION_PATH.equals(path) || PUBLIC_VERSION_LATEST_PATH.equals(path);
return PUBLIC_VERSION_PATH.equals(path)
|| PUBLIC_VERSION_LATEST_PATH.equals(path)
|| PUBLIC_VERSION_LIST_PATH.equals(path);
}
/** 公开列表条数归一化：空/非正数取默认值，超上限截断（调用方无需自行校验）。 */
public static int normalizeListLimit(Integer limit) {
if (limit == null || limit <= 0) {
return DEFAULT_LIST_LIMIT;
}
return Math.min(limit, MAX_LIST_LIMIT);
}
/** 取最新版本记录：created_at DESC，同一时间按 id 倒序稳定；空/无有效回 null。 */
backend-java/src/test/java/com/nanri/aiimage/modules/softwareversion/service/support/PublicVersionListLookupTest.java
+39
View File
@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.softwareversion.service.support;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/**
* module 09 公开版本列表（/api/version/list）回归：路径契约与条数归一化。
* 桌面端"指定版本更新"下拉依赖该接口，匿名可达，必须与 /api/version/latest 同族。
*/
class PublicVersionListLookupTest {
@Test
void test_public_list_path_belongs_to_public_version_family() {
assertTrue(PublicVersionLookup.isPublicVersionPath(PublicVersionLookup.PUBLIC_VERSION_LIST_PATH));
assertTrue(PublicVersionLookup.isPublicVersionPath(PublicVersionLookup.PUBLIC_VERSION_LATEST_PATH));
// 管理端上传路径不得被误判为公开路径；子路径不因前缀匹配而放行
assertFalse(PublicVersionLookup.isPublicVersionPath(PublicVersionLookup.ADMIN_UPLOAD_PATH));
assertFalse(PublicVersionLookup.isPublicVersionPath("/api/version/list/delete"));
}
@Test
void test_list_limit_null_or_non_positive_uses_default() {
assertEquals(PublicVersionLookup.DEFAULT_LIST_LIMIT, PublicVersionLookup.normalizeListLimit(null));
assertEquals(PublicVersionLookup.DEFAULT_LIST_LIMIT, PublicVersionLookup.normalizeListLimit(0));
assertEquals(PublicVersionLookup.DEFAULT_LIST_LIMIT, PublicVersionLookup.normalizeListLimit(-5));
}
@Test
void test_list_limit_over_max_is_capped() {
assertEquals(PublicVersionLookup.MAX_LIST_LIMIT, PublicVersionLookup.normalizeListLimit(100000));
assertEquals(30, PublicVersionLookup.normalizeListLimit(30));
}
@Test
void test_default_limit_within_max() {
assertTrue(PublicVersionLookup.DEFAULT_LIST_LIMIT <= PublicVersionLookup.MAX_LIST_LIMIT);
}
}