refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权
模块边界(G5 / G7) - 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何 业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册 - 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、 CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块 - G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo - 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测) - 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写) 性能与容量(C5/C6/C7/D13) - C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、 BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader; 行数上限改为迭代中生效 - C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country - C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存 - D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与 TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志 - V123 删除 biz_file_result 两个被复合索引覆盖的单列索引 安全(A3/A4/A7 + 守卫名单) - A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外) - A3 上传接口加危险扩展名黑名单(可配置) - AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层) 容量(明细表保留期) - 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理 (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
+30
@@ -0,0 +1,30 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 外观专利检测 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final AppearancePatentTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "APPEARANCE_PATENT";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
}
|
||||
+140
-74
@@ -1,17 +1,17 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PushbackInputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -37,9 +37,8 @@ public class AppearancePatentExcelParser {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (FileInputStream fis = new FileInputStream(input);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
return parseWorkbook(workbook, maxRows);
|
||||
try (InputStream inputStream = new FileInputStream(input)) {
|
||||
return readStreaming(inputStream, maxRows);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -52,8 +51,8 @@ public class AppearancePatentExcelParser {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (Workbook workbook = WorkbookFactory.create(input)) {
|
||||
return parseWorkbook(workbook, DEFAULT_MAX_ROWS);
|
||||
try {
|
||||
return readStreaming(input, DEFAULT_MAX_ROWS);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -62,83 +61,154 @@ public class AppearancePatentExcelParser {
|
||||
}
|
||||
}
|
||||
|
||||
private ParsedSheet parseWorkbook(Workbook workbook, int maxRows) {
|
||||
int safeMaxRows = Math.max(1, maxRows);
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) {
|
||||
/**
|
||||
* 流式解析(EasyExcel SAX),替代 POI WorkbookFactory 全量 DOM 加载:
|
||||
* 大表不再整表驻留堆内存,行数上限在迭代过程中即时生效(超限抛错)。
|
||||
* 语义与原 POI 路径一致:cell 归一化、表头别名匹配、空行跳过、必填表头缺失抛错、无字段截断。
|
||||
* 注意不关闭传入的 InputStream(由调用方负责)。
|
||||
*/
|
||||
private ParsedSheet readStreaming(InputStream inputStream, int maxRows) throws Exception {
|
||||
// EasyExcel 会把非 zip 文本当 CSV 解析成功;原 WorkbookFactory 只认 xlsx/xls,
|
||||
// 这里先做文件魔数校验,保持「垃圾文件→解析 Excel 失败」的语义并拒绝 CSV 误解析。
|
||||
PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(inputStream, 8192), 8);
|
||||
requireExcelMagic(pb);
|
||||
SheetContext ctx = new SheetContext(Math.max(1, maxRows));
|
||||
ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
ctx.initHeader(headerMap);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
ctx.consumeRow(rowIndex, rowMap);
|
||||
}
|
||||
});
|
||||
if (ctx.headers == null) {
|
||||
// 空表 / 表头行整行为空(EasyExcel 不上报表头回调):与旧实现 header == null 一致
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
||||
List<String> headers = readHeaders(header, formatter);
|
||||
int idCol = findRequiredHeader(headerMap, "id");
|
||||
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||
int skuCol = findOptionalHeaderExact(headerMap,
|
||||
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
|
||||
int urlCol = findOptionalHeaderExact(headerMap,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
int titleCol = findOptionalHeaderExact(headerMap,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
return new ParsedSheet(ctx.headers, ctx.rows);
|
||||
}
|
||||
|
||||
List<AppearanceExcelRow> rows = new ArrayList<>();
|
||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
if (row == null) {
|
||||
continue;
|
||||
/** 校验文件头魔数:xlsx=PK(zip)、xls=OLE2(CFD)。判非抛「解析 Excel 失败」;用 unread 回退已读字节。 */
|
||||
private void requireExcelMagic(PushbackInputStream in) throws IOException {
|
||||
byte[] head = new byte[8];
|
||||
int n = 0;
|
||||
while (n < head.length) {
|
||||
int r = in.read(head, n, head.length - n);
|
||||
if (r < 0) {
|
||||
break;
|
||||
}
|
||||
String id = cell(row, idCol, formatter);
|
||||
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
|
||||
String country = cell(row, countryCol, formatter);
|
||||
n += r;
|
||||
}
|
||||
if (n > 0) {
|
||||
in.unread(head, 0, n);
|
||||
}
|
||||
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
|
||||
boolean isOle2 = n >= 8
|
||||
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
|
||||
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
|
||||
if (!isZip && !isOle2) {
|
||||
log.warn("[appearance-patent] parse rejected non-excel magic head={}", Arrays.copyOf(head, Math.max(n, 0)));
|
||||
throw new BusinessException("解析 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
/** 单表解析上下文:表头就绪后逐行累积结果行。 */
|
||||
private final class SheetContext {
|
||||
|
||||
private final int maxRows;
|
||||
private final List<AppearanceExcelRow> rows = new ArrayList<>();
|
||||
private List<String> headers;
|
||||
private int idCol;
|
||||
private int asinCol;
|
||||
private int countryCol;
|
||||
private int priceCol;
|
||||
private int skuCol;
|
||||
private int urlCol;
|
||||
private int titleCol;
|
||||
|
||||
SheetContext(int maxRows) {
|
||||
this.maxRows = maxRows;
|
||||
}
|
||||
|
||||
void initHeader(Map<Integer, String> rawHeaderMap) {
|
||||
// EasyExcel 回调给出 列号 → 表头文本,与旧 Row 遍历等价(缺列一般为 null/空串)
|
||||
int lastColumnCount = lastColumnCount(rawHeaderMap);
|
||||
Map<String, Integer> map = new LinkedHashMap<>();
|
||||
List<String> headerNames = new ArrayList<>();
|
||||
for (int i = 0; i < lastColumnCount; i++) {
|
||||
String val = normalize(rawHeaderMap.getOrDefault(i, ""));
|
||||
headerNames.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||
if (!val.isBlank()) {
|
||||
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||
}
|
||||
}
|
||||
this.headers = headerNames;
|
||||
this.idCol = findRequiredHeader(map, "id");
|
||||
this.asinCol = findRequiredHeader(map, "asin");
|
||||
this.countryCol = findRequiredHeader(map, "国家", "country");
|
||||
this.priceCol = findOptionalHeaderExact(map, "价格", "price");
|
||||
this.skuCol = findOptionalHeaderExact(map,
|
||||
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
|
||||
this.urlCol = findOptionalHeaderExact(map,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
this.titleCol = findOptionalHeaderExact(map,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
}
|
||||
|
||||
void consumeRow(int rowIndex, Map<Integer, String> rowMap) {
|
||||
if (headers == null) {
|
||||
return;
|
||||
}
|
||||
// EasyExcel rowIndex 从 0 起(0 为表头),POI 原实现行号同样 0 起并 +1 展示
|
||||
String id = streamCell(rowMap, idCol);
|
||||
String asin = streamCell(rowMap, asinCol).toUpperCase(Locale.ROOT);
|
||||
String country = streamCell(rowMap, countryCol);
|
||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
if (rows.size() >= safeMaxRows) {
|
||||
throw new BusinessException("解析行数超过上限: " + safeMaxRows);
|
||||
if (rows.size() >= maxRows) {
|
||||
throw new BusinessException("解析行数超过上限: " + maxRows);
|
||||
}
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
values.put(headers.get(i), streamCell(rowMap, i));
|
||||
}
|
||||
rows.add(new AppearanceExcelRow(
|
||||
i + 1,
|
||||
rowIndex + 1,
|
||||
id,
|
||||
asin,
|
||||
country,
|
||||
priceCol >= 0 ? cell(row, priceCol, formatter) : "",
|
||||
skuCol >= 0 ? cell(row, skuCol, formatter) : "",
|
||||
urlCol >= 0 ? cell(row, urlCol, formatter) : "",
|
||||
titleCol >= 0 ? cell(row, titleCol, formatter) : "",
|
||||
readRowValues(row, headers, formatter)));
|
||||
priceCol >= 0 ? streamCell(rowMap, priceCol) : "",
|
||||
skuCol >= 0 ? streamCell(rowMap, skuCol) : "",
|
||||
urlCol >= 0 ? streamCell(rowMap, urlCol) : "",
|
||||
titleCol >= 0 ? streamCell(rowMap, titleCol) : "",
|
||||
values));
|
||||
}
|
||||
return new ParsedSheet(headers, rows);
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||
Map<String, Integer> map = new LinkedHashMap<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
if (!val.isBlank()) {
|
||||
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||
private int lastColumnCount(Map<Integer, String> rowMap) {
|
||||
if (rowMap == null || rowMap.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int lastColumnCount = 0;
|
||||
for (Integer columnIndex : rowMap.keySet()) {
|
||||
if (columnIndex != null && columnIndex >= lastColumnCount) {
|
||||
lastColumnCount = columnIndex + 1;
|
||||
}
|
||||
}
|
||||
return lastColumnCount;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private List<String> readHeaders(Row header, DataFormatter formatter) {
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
headers.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||
private String streamCell(Map<Integer, String> rowMap, int col) {
|
||||
if (col < 0 || rowMap == null) {
|
||||
return "";
|
||||
}
|
||||
return normalize(rowMap.get(col));
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
values.put(headers.get(i), cell(row, i, formatter));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private int findRequiredHeader(Map<String, Integer> map, String... names) {
|
||||
@@ -177,10 +247,6 @@ public class AppearancePatentExcelParser {
|
||||
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
|
||||
}
|
||||
|
||||
private String cell(Row row, int col, DataFormatter formatter) {
|
||||
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
}
|
||||
|
||||
private String normalize(String val) {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user