refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权

模块边界(G5 / G7)
- 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何
  业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册
- 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、
  CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块
- G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo
- 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测)
- 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写)

性能与容量(C5/C6/C7/D13)
- C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、
  BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader;
  行数上限改为迭代中生效
- C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country
- C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存
- D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与
  TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志
- V123 删除 biz_file_result 两个被复合索引覆盖的单列索引

安全(A3/A4/A7 + 守卫名单)
- A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外)
- A3 上传接口加危险扩展名黑名单(可配置)
- AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层)

容量(明细表保留期)
- 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理
  (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
2026-09-14 06:46:27 +08:00
parent d87d76398e
commit f4ee7e16f0
86 changed files with 3191 additions and 952 deletions
@@ -0,0 +1,30 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
/**
* 外观专利检测 的任务心跳实现(2026-09 全维度审查 G5)。
*
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
* 消除 task → 业务模块的编译期依赖。
*/
@Service
@RequiredArgsConstructor
public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
private final AppearancePatentTaskCacheService cacheService;
@Override
public String moduleType() {
return "APPEARANCE_PATENT";
}
@Override
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
cacheService.touchTaskHeartbeat(taskId);
}
}
@@ -1,17 +1,17 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PushbackInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -37,9 +37,8 @@ public class AppearancePatentExcelParser {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (FileInputStream fis = new FileInputStream(input);
Workbook workbook = WorkbookFactory.create(fis)) {
return parseWorkbook(workbook, maxRows);
try (InputStream inputStream = new FileInputStream(input)) {
return readStreaming(inputStream, maxRows);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -52,8 +51,8 @@ public class AppearancePatentExcelParser {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (Workbook workbook = WorkbookFactory.create(input)) {
return parseWorkbook(workbook, DEFAULT_MAX_ROWS);
try {
return readStreaming(input, DEFAULT_MAX_ROWS);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -62,83 +61,154 @@ public class AppearancePatentExcelParser {
}
}
private ParsedSheet parseWorkbook(Workbook workbook, int maxRows) {
int safeMaxRows = Math.max(1, maxRows);
DataFormatter formatter = new DataFormatter();
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
/**
* 流式解析(EasyExcel SAX),替代 POI WorkbookFactory 全量 DOM 加载:
* 大表不再整表驻留堆内存,行数上限在迭代过程中即时生效(超限抛错)。
* 语义与原 POI 路径一致:cell 归一化、表头别名匹配、空行跳过、必填表头缺失抛错、无字段截断。
* 注意不关闭传入的 InputStream(由调用方负责)。
*/
private ParsedSheet readStreaming(InputStream inputStream, int maxRows) throws Exception {
// EasyExcel 会把非 zip 文本当 CSV 解析成功;原 WorkbookFactory 只认 xlsx/xls,
// 这里先做文件魔数校验,保持「垃圾文件→解析 Excel 失败」的语义并拒绝 CSV 误解析。
PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(inputStream, 8192), 8);
requireExcelMagic(pb);
SheetContext ctx = new SheetContext(Math.max(1, maxRows));
ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
ctx.initHeader(headerMap);
}
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
ctx.consumeRow(rowIndex, rowMap);
}
});
if (ctx.headers == null) {
// 空表 / 表头行整行为空(EasyExcel 不上报表头回调):与旧实现 header == null 一致
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int skuCol = findOptionalHeaderExact(headerMap,
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
return new ParsedSheet(ctx.headers, ctx.rows);
}
List<AppearanceExcelRow> rows = new ArrayList<>();
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
/** 校验文件头魔数:xlsx=PK(zip)、xls=OLE2(CFD)。判非抛「解析 Excel 失败」;用 unread 回退已读字节。 */
private void requireExcelMagic(PushbackInputStream in) throws IOException {
byte[] head = new byte[8];
int n = 0;
while (n < head.length) {
int r = in.read(head, n, head.length - n);
if (r < 0) {
break;
}
String id = cell(row, idCol, formatter);
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter);
n += r;
}
if (n > 0) {
in.unread(head, 0, n);
}
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
boolean isOle2 = n >= 8
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
if (!isZip && !isOle2) {
log.warn("[appearance-patent] parse rejected non-excel magic head={}", Arrays.copyOf(head, Math.max(n, 0)));
throw new BusinessException("解析 Excel 失败");
}
}
/** 单表解析上下文:表头就绪后逐行累积结果行。 */
private final class SheetContext {
private final int maxRows;
private final List<AppearanceExcelRow> rows = new ArrayList<>();
private List<String> headers;
private int idCol;
private int asinCol;
private int countryCol;
private int priceCol;
private int skuCol;
private int urlCol;
private int titleCol;
SheetContext(int maxRows) {
this.maxRows = maxRows;
}
void initHeader(Map<Integer, String> rawHeaderMap) {
// EasyExcel 回调给出 列号 → 表头文本,与旧 Row 遍历等价(缺列一般为 null/空串)
int lastColumnCount = lastColumnCount(rawHeaderMap);
Map<String, Integer> map = new LinkedHashMap<>();
List<String> headerNames = new ArrayList<>();
for (int i = 0; i < lastColumnCount; i++) {
String val = normalize(rawHeaderMap.getOrDefault(i, ""));
headerNames.add(val.isBlank() ? "列" + (i + 1) : val);
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
this.headers = headerNames;
this.idCol = findRequiredHeader(map, "id");
this.asinCol = findRequiredHeader(map, "asin");
this.countryCol = findRequiredHeader(map, "国家", "country");
this.priceCol = findOptionalHeaderExact(map, "价格", "price");
this.skuCol = findOptionalHeaderExact(map,
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
this.urlCol = findOptionalHeaderExact(map,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
this.titleCol = findOptionalHeaderExact(map,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
}
void consumeRow(int rowIndex, Map<Integer, String> rowMap) {
if (headers == null) {
return;
}
// EasyExcel rowIndex 从 0 起(0 为表头),POI 原实现行号同样 0 起并 +1 展示
String id = streamCell(rowMap, idCol);
String asin = streamCell(rowMap, asinCol).toUpperCase(Locale.ROOT);
String country = streamCell(rowMap, countryCol);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
return;
}
if (rows.size() >= safeMaxRows) {
throw new BusinessException("解析行数超过上限: " + safeMaxRows);
if (rows.size() >= maxRows) {
throw new BusinessException("解析行数超过上限: " + maxRows);
}
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), streamCell(rowMap, i));
}
rows.add(new AppearanceExcelRow(
i + 1,
rowIndex + 1,
id,
asin,
country,
priceCol >= 0 ? cell(row, priceCol, formatter) : "",
skuCol >= 0 ? cell(row, skuCol, formatter) : "",
urlCol >= 0 ? cell(row, urlCol, formatter) : "",
titleCol >= 0 ? cell(row, titleCol, formatter) : "",
readRowValues(row, headers, formatter)));
priceCol >= 0 ? streamCell(rowMap, priceCol) : "",
skuCol >= 0 ? streamCell(rowMap, skuCol) : "",
urlCol >= 0 ? streamCell(rowMap, urlCol) : "",
titleCol >= 0 ? streamCell(rowMap, titleCol) : "",
values));
}
return new ParsedSheet(headers, rows);
}
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
Map<String, Integer> map = new LinkedHashMap<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
private int lastColumnCount(Map<Integer, String> rowMap) {
if (rowMap == null || rowMap.isEmpty()) {
return 0;
}
int lastColumnCount = 0;
for (Integer columnIndex : rowMap.keySet()) {
if (columnIndex != null && columnIndex >= lastColumnCount) {
lastColumnCount = columnIndex + 1;
}
}
return lastColumnCount;
}
return map;
}
private List<String> readHeaders(Row header, DataFormatter formatter) {
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(val.isBlank() ? "列" + (i + 1) : val);
private String streamCell(Map<Integer, String> rowMap, int col) {
if (col < 0 || rowMap == null) {
return "";
}
return normalize(rowMap.get(col));
}
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter));
}
return values;
}
private int findRequiredHeader(Map<String, Integer> map, String... names) {
@@ -177,10 +247,6 @@ public class AppearancePatentExcelParser {
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
}
private String cell(Row row, int col, DataFormatter formatter) {
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
}
private String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}