refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权

模块边界(G5 / G7)
- 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何
  业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册
- 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、
  CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块
- G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo
- 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测)
- 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写)

性能与容量(C5/C6/C7/D13)
- C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、
  BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader;
  行数上限改为迭代中生效
- C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country
- C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存
- D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与
  TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志
- V123 删除 biz_file_result 两个被复合索引覆盖的单列索引

安全(A3/A4/A7 + 守卫名单)
- A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外)
- A3 上传接口加危险扩展名黑名单(可配置)
- AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层)

容量(明细表保留期)
- 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理
  (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
2026-09-14 06:46:27 +08:00
parent d87d76398e
commit f4ee7e16f0
86 changed files with 3191 additions and 952 deletions
@@ -3,34 +3,46 @@ package com.nanri.aiimage.modules.file.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.file.model.vo.ExcelInfoVo;
import com.nanri.aiimage.modules.file.model.vo.UploadFileVo;
import lombok.RequiredArgsConstructor;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
@Slf4j
public class LocalFileStorageService {
/** fileKey → 文件名 索引容量:超限淘汰最旧条目,避免索引自身无界增长。 */
static final int SOURCE_FILE_INDEX_CAPACITY = 1024;
/**
* 禁止上传的可执行/脚本类扩展名(2026-09 全维度审查 A3)。
* 上传接口是桌面端文件中转口(临时目录落盘,2GB/次),此前对类型无任何限制;
* 这里挡掉"落盘后一旦被其它组件误执行/误引用即可造成危害"的扩展名。
* 业务文件(xlsx/xls/csv/txt/json/zip/图片/视频)不受影响;
* 如需调整由运维在 application.yml 覆盖 aiimage.files.upload-blocked-extensions。
*/
private static final String DEFAULT_BLOCKED_EXTENSIONS =
"exe,bat,cmd,com,scr,msi,dll,sh,ps1,vbs,vbe,wsf,hta,jsp,jspx,php,jar,so,dylib";
@Value("${aiimage.files.upload-blocked-extensions:" + DEFAULT_BLOCKED_EXTENSIONS + "}")
private String blockedExtensions = DEFAULT_BLOCKED_EXTENSIONS;
private final StorageProperties storageProperties;
/** 临时目录标准路径:供调用方做"文件必须落在上传临时目录内"的穿越校验。 */
@@ -51,6 +63,7 @@ public class LocalFileStorageService {
FileUtil.mkdir(tempDir);
String fileKey = IdUtil.fastSimpleUUID();
String extName = FileUtil.extName(file.getOriginalFilename());
rejectBlockedExtension(file.getOriginalFilename(), extName);
String filename = fileKey + (extName.isEmpty() ? "" : "." + extName);
File target = FileUtil.file(tempDir, filename);
file.transferTo(target);
@@ -65,49 +78,125 @@ public class LocalFileStorageService {
return vo;
}
/** 扩展名黑名单校验:命中即拒绝落盘(大小写不敏感)。 */
private void rejectBlockedExtension(String originalFilename, String extName) {
if (extName == null || extName.isBlank()) {
return;
}
String normalized = extName.trim().toLowerCase(java.util.Locale.ROOT);
for (String blocked : blockedExtensions.split(",")) {
if (!blocked.isBlank() && blocked.trim().toLowerCase(java.util.Locale.ROOT).equals(normalized)) {
log.warn("[file-upload] 拒绝上传被禁扩展名 originalFilename={} ext={}", originalFilename, normalized);
throw new BusinessException("不支持上传该类型文件:." + normalized);
}
}
}
public ExcelInfoVo getExcelInfo(String fileKey) throws IOException {
File inputFile = findLocalSourceFile(fileKey);
if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("源文件不存在");
}
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
// 2026-09 全维度审查 C5:改流式解析(ExcelStreamReader → EasyExcel SAX)。
// 此前 WorkbookFactory.create 把整表读成 DOM,只为取表头与行数却让整表驻留堆内存。
// 返回值结构与字段含义不变:headers 同旧口径(去重、空表头跳过、命中「缩略图地址8」截断),
// totalRows 仍是"数据行数"(旧实现取 lastRowNum,等于物理末行行号-1)。
try {
ExcelInfoStreamingReader reader = new ExcelInfoStreamingReader();
ExcelStreamReader.readFirstSheet(inputFile, reader);
reader.assertHeaderSeen();
ExcelInfoVo vo = new ExcelInfoVo();
vo.setHeaders(reader.headers());
vo.setTotalRows(reader.totalRows());
return vo;
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
log.warn("[file-excel-info] 读取 Excel 信息失败 fileKey={} err={}", fileKey, e.getMessage(), e);
throw new BusinessException("读取 Excel 信息失败");
}
}
Map<String, Integer> headerMap = new HashMap<>();
List<String> headers = new ArrayList<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
/**
* 表头与行数的流式读取:内存不随文件行数增长。
* 表头行整行为空(EasyExcel 不上报表头回调)时视为「Excel 表头为空」——旧实现仅在 getRow(0)==null
* 时报同一错误,此处口径略严(不存在的表头与空表头同样处理)。
*/
private final class ExcelInfoStreamingReader implements ExcelStreamReader.SheetRowHandler {
private final List<String> headers = new ArrayList<>();
private final Set<String> seenHeaders = new LinkedHashSet<>();
private Integer approximateTotalRows;
private boolean headerSeen;
private boolean headerTruncated;
private int countedRows;
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
headerSeen = true;
int lastColumnCount = lastHeaderColumnCount(headerMap);
for (int i = 0; i < lastColumnCount && !headerTruncated; i++) {
String value = normalizeCellText(headerMap.get(i));
if (value.isBlank()) {
continue;
}
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
? value
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
if (headerMap.containsKey(value)) {
if (!seenHeaders.add(value)) {
continue;
}
headerMap.put(value, i);
headers.add(value);
if ("缩略图地址8".equals(value)) {
break;
headerTruncated = true;
}
}
}
ExcelInfoVo vo = new ExcelInfoVo();
vo.setHeaders(headers);
vo.setTotalRows(Math.max(sheet.getLastRowNum(), 0));
return vo;
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("读取 Excel 信息失败");
@Override
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
this.approximateTotalRows = approximateTotalRows;
}
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
countedRows++;
}
private void assertHeaderSeen() {
if (!headerSeen) {
throw new BusinessException("Excel 表头为空");
}
}
private List<String> headers() {
return headers;
}
/**
* onSheetTotal 的近似值 = 含表头的物理末行行号(POI lastRowNum+1 口径),换算回旧的
* "数据行数"口径;近似值缺失(dimension 标签缺失)或偏小时用遍历计数兜底。
*/
private int totalRows() {
int approximateDataRows = approximateTotalRows == null || approximateTotalRows <= 0
? 0
: approximateTotalRows - 1;
return Math.max(approximateDataRows, countedRows);
}
private int lastHeaderColumnCount(Map<Integer, String> headerMap) {
if (headerMap == null || headerMap.isEmpty()) {
return 0;
}
int lastColumnCount = 0;
for (Integer columnIndex : headerMap.keySet()) {
if (columnIndex != null && columnIndex >= lastColumnCount) {
lastColumnCount = columnIndex + 1;
}
}
return lastColumnCount;
}
}
@@ -1,11 +1,15 @@
package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.common.util.PendingDeleteJournal;
import com.nanri.aiimage.config.TransientStorageProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashSet;
@@ -26,6 +30,48 @@ public class RustfsDeleteRetryService {
private final ConcurrentLinkedQueue<RetryItem> queue = new ConcurrentLinkedQueue<>();
private final ConcurrentHashMap<String, RetryItem> pending = new ConcurrentHashMap<>();
private final Object admissionLock = new Object();
/**
* 待删对象的本地落盘日志:内存队列重启即丢,对应对象会一直残留在桶里直到生命周期过期。
* 入队时追加、每轮重试后按 pending 收敛重写、启动时回放。
*/
private volatile PendingDeleteJournal journal;
/**
* 启动回放:把上次进程残留的待删对象重新入队(容量准入、去重语义与在线入队一致),
* 随后由 {@link #retryPendingDeletes()} 的周期任务继续重试。
*/
@EventListener(ApplicationReadyEvent.class)
public void replayJournalOnStartup() {
if (!properties.isDeleteRetryEnabled()) {
return;
}
List<String> objectKeys = journal().readAll();
if (objectKeys.isEmpty()) {
return;
}
int replayed = 0;
for (String objectKey : objectKeys) {
enqueue(objectKey, null);
replayed++;
}
log.info("[rustfs] 启动回放待删对象 journalSize={} replayed={} pending={}",
objectKeys.size(), replayed, pending.size());
}
private PendingDeleteJournal journal() {
PendingDeleteJournal local = journal;
if (local != null) {
return local;
}
synchronized (admissionLock) {
if (journal == null) {
Path path = Path.of(System.getProperty("java.io.tmpdir"),
"aiimage-delete-journal", "rustfs-delete-retry.log");
journal = new PendingDeleteJournal(path, Math.max(1, properties.getDeleteRetryQueueCapacity()));
}
return journal;
}
}
public void enqueue(String objectKey, Throwable cause) {
if (!properties.isDeleteRetryEnabled() || objectKey == null || objectKey.isBlank()) {
@@ -97,6 +143,8 @@ public class RustfsDeleteRetryService {
}
retryNextInvocation.forEach(this::requeue);
if (processed > 0) {
// 收敛落盘:成功删除的对象从日志移除,仍失败/待重试的保留
journal().rewrite(pending.keySet());
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={}",
processed, success, failed, pending.size());
}
@@ -124,6 +172,8 @@ public class RustfsDeleteRetryService {
}
item = new RetryItem(objectKey);
pending.put(objectKey, item);
// 只在"新准入"时记账:重写以 pending 为准,重复失败无需重复追加
journal().record(objectKey);
return item;
}
}