refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权
模块边界(G5 / G7) - 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何 业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册 - 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、 CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块 - G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo - 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测) - 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写) 性能与容量(C5/C6/C7/D13) - C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、 BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader; 行数上限改为迭代中生效 - C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country - C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存 - D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与 TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志 - V123 删除 biz_file_result 两个被复合索引覆盖的单列索引 安全(A3/A4/A7 + 守卫名单) - A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外) - A3 上传接口加危险扩展名黑名单(可配置) - AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层) 容量(明细表保留期) - 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理 (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
+119
-30
@@ -3,34 +3,46 @@ package com.nanri.aiimage.modules.file.service;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.model.vo.ExcelInfoVo;
|
||||
import com.nanri.aiimage.modules.file.model.vo.UploadFileVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class LocalFileStorageService {
|
||||
|
||||
/** fileKey → 文件名 索引容量:超限淘汰最旧条目,避免索引自身无界增长。 */
|
||||
static final int SOURCE_FILE_INDEX_CAPACITY = 1024;
|
||||
|
||||
/**
|
||||
* 禁止上传的可执行/脚本类扩展名(2026-09 全维度审查 A3)。
|
||||
* 上传接口是桌面端文件中转口(临时目录落盘,2GB/次),此前对类型无任何限制;
|
||||
* 这里挡掉"落盘后一旦被其它组件误执行/误引用即可造成危害"的扩展名。
|
||||
* 业务文件(xlsx/xls/csv/txt/json/zip/图片/视频)不受影响;
|
||||
* 如需调整由运维在 application.yml 覆盖 aiimage.files.upload-blocked-extensions。
|
||||
*/
|
||||
private static final String DEFAULT_BLOCKED_EXTENSIONS =
|
||||
"exe,bat,cmd,com,scr,msi,dll,sh,ps1,vbs,vbe,wsf,hta,jsp,jspx,php,jar,so,dylib";
|
||||
|
||||
@Value("${aiimage.files.upload-blocked-extensions:" + DEFAULT_BLOCKED_EXTENSIONS + "}")
|
||||
private String blockedExtensions = DEFAULT_BLOCKED_EXTENSIONS;
|
||||
|
||||
private final StorageProperties storageProperties;
|
||||
|
||||
/** 临时目录标准路径:供调用方做"文件必须落在上传临时目录内"的穿越校验。 */
|
||||
@@ -51,6 +63,7 @@ public class LocalFileStorageService {
|
||||
FileUtil.mkdir(tempDir);
|
||||
String fileKey = IdUtil.fastSimpleUUID();
|
||||
String extName = FileUtil.extName(file.getOriginalFilename());
|
||||
rejectBlockedExtension(file.getOriginalFilename(), extName);
|
||||
String filename = fileKey + (extName.isEmpty() ? "" : "." + extName);
|
||||
File target = FileUtil.file(tempDir, filename);
|
||||
file.transferTo(target);
|
||||
@@ -65,49 +78,125 @@ public class LocalFileStorageService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 扩展名黑名单校验:命中即拒绝落盘(大小写不敏感)。 */
|
||||
private void rejectBlockedExtension(String originalFilename, String extName) {
|
||||
if (extName == null || extName.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String normalized = extName.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
for (String blocked : blockedExtensions.split(",")) {
|
||||
if (!blocked.isBlank() && blocked.trim().toLowerCase(java.util.Locale.ROOT).equals(normalized)) {
|
||||
log.warn("[file-upload] 拒绝上传被禁扩展名 originalFilename={} ext={}", originalFilename, normalized);
|
||||
throw new BusinessException("不支持上传该类型文件:." + normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ExcelInfoVo getExcelInfo(String fileKey) throws IOException {
|
||||
File inputFile = findLocalSourceFile(fileKey);
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("源文件不存在");
|
||||
}
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
// 2026-09 全维度审查 C5:改流式解析(ExcelStreamReader → EasyExcel SAX)。
|
||||
// 此前 WorkbookFactory.create 把整表读成 DOM,只为取表头与行数却让整表驻留堆内存。
|
||||
// 返回值结构与字段含义不变:headers 同旧口径(去重、空表头跳过、命中「缩略图地址8」截断),
|
||||
// totalRows 仍是"数据行数"(旧实现取 lastRowNum,等于物理末行行号-1)。
|
||||
try {
|
||||
ExcelInfoStreamingReader reader = new ExcelInfoStreamingReader();
|
||||
ExcelStreamReader.readFirstSheet(inputFile, reader);
|
||||
reader.assertHeaderSeen();
|
||||
ExcelInfoVo vo = new ExcelInfoVo();
|
||||
vo.setHeaders(reader.headers());
|
||||
vo.setTotalRows(reader.totalRows());
|
||||
return vo;
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.warn("[file-excel-info] 读取 Excel 信息失败 fileKey={} err={}", fileKey, e.getMessage(), e);
|
||||
throw new BusinessException("读取 Excel 信息失败");
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
/**
|
||||
* 表头与行数的流式读取:内存不随文件行数增长。
|
||||
* 表头行整行为空(EasyExcel 不上报表头回调)时视为「Excel 表头为空」——旧实现仅在 getRow(0)==null
|
||||
* 时报同一错误,此处口径略严(不存在的表头与空表头同样处理)。
|
||||
*/
|
||||
private final class ExcelInfoStreamingReader implements ExcelStreamReader.SheetRowHandler {
|
||||
|
||||
private final List<String> headers = new ArrayList<>();
|
||||
private final Set<String> seenHeaders = new LinkedHashSet<>();
|
||||
private Integer approximateTotalRows;
|
||||
private boolean headerSeen;
|
||||
private boolean headerTruncated;
|
||||
private int countedRows;
|
||||
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
headerSeen = true;
|
||||
int lastColumnCount = lastHeaderColumnCount(headerMap);
|
||||
for (int i = 0; i < lastColumnCount && !headerTruncated; i++) {
|
||||
String value = normalizeCellText(headerMap.get(i));
|
||||
if (value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
|
||||
? value
|
||||
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
|
||||
if (headerMap.containsKey(value)) {
|
||||
if (!seenHeaders.add(value)) {
|
||||
continue;
|
||||
}
|
||||
headerMap.put(value, i);
|
||||
headers.add(value);
|
||||
if ("缩略图地址8".equals(value)) {
|
||||
break;
|
||||
headerTruncated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ExcelInfoVo vo = new ExcelInfoVo();
|
||||
vo.setHeaders(headers);
|
||||
vo.setTotalRows(Math.max(sheet.getLastRowNum(), 0));
|
||||
return vo;
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("读取 Excel 信息失败");
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
this.approximateTotalRows = approximateTotalRows;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
countedRows++;
|
||||
}
|
||||
|
||||
private void assertHeaderSeen() {
|
||||
if (!headerSeen) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> headers() {
|
||||
return headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* onSheetTotal 的近似值 = 含表头的物理末行行号(POI lastRowNum+1 口径),换算回旧的
|
||||
* "数据行数"口径;近似值缺失(dimension 标签缺失)或偏小时用遍历计数兜底。
|
||||
*/
|
||||
private int totalRows() {
|
||||
int approximateDataRows = approximateTotalRows == null || approximateTotalRows <= 0
|
||||
? 0
|
||||
: approximateTotalRows - 1;
|
||||
return Math.max(approximateDataRows, countedRows);
|
||||
}
|
||||
|
||||
private int lastHeaderColumnCount(Map<Integer, String> headerMap) {
|
||||
if (headerMap == null || headerMap.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int lastColumnCount = 0;
|
||||
for (Integer columnIndex : headerMap.keySet()) {
|
||||
if (columnIndex != null && columnIndex >= lastColumnCount) {
|
||||
lastColumnCount = columnIndex + 1;
|
||||
}
|
||||
}
|
||||
return lastColumnCount;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+50
@@ -1,11 +1,15 @@
|
||||
package com.nanri.aiimage.modules.file.service.object;
|
||||
|
||||
import com.nanri.aiimage.common.util.PendingDeleteJournal;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
@@ -26,6 +30,48 @@ public class RustfsDeleteRetryService {
|
||||
private final ConcurrentLinkedQueue<RetryItem> queue = new ConcurrentLinkedQueue<>();
|
||||
private final ConcurrentHashMap<String, RetryItem> pending = new ConcurrentHashMap<>();
|
||||
private final Object admissionLock = new Object();
|
||||
/**
|
||||
* 待删对象的本地落盘日志:内存队列重启即丢,对应对象会一直残留在桶里直到生命周期过期。
|
||||
* 入队时追加、每轮重试后按 pending 收敛重写、启动时回放。
|
||||
*/
|
||||
private volatile PendingDeleteJournal journal;
|
||||
|
||||
/**
|
||||
* 启动回放:把上次进程残留的待删对象重新入队(容量准入、去重语义与在线入队一致),
|
||||
* 随后由 {@link #retryPendingDeletes()} 的周期任务继续重试。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void replayJournalOnStartup() {
|
||||
if (!properties.isDeleteRetryEnabled()) {
|
||||
return;
|
||||
}
|
||||
List<String> objectKeys = journal().readAll();
|
||||
if (objectKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int replayed = 0;
|
||||
for (String objectKey : objectKeys) {
|
||||
enqueue(objectKey, null);
|
||||
replayed++;
|
||||
}
|
||||
log.info("[rustfs] 启动回放待删对象 journalSize={} replayed={} pending={}",
|
||||
objectKeys.size(), replayed, pending.size());
|
||||
}
|
||||
|
||||
private PendingDeleteJournal journal() {
|
||||
PendingDeleteJournal local = journal;
|
||||
if (local != null) {
|
||||
return local;
|
||||
}
|
||||
synchronized (admissionLock) {
|
||||
if (journal == null) {
|
||||
Path path = Path.of(System.getProperty("java.io.tmpdir"),
|
||||
"aiimage-delete-journal", "rustfs-delete-retry.log");
|
||||
journal = new PendingDeleteJournal(path, Math.max(1, properties.getDeleteRetryQueueCapacity()));
|
||||
}
|
||||
return journal;
|
||||
}
|
||||
}
|
||||
|
||||
public void enqueue(String objectKey, Throwable cause) {
|
||||
if (!properties.isDeleteRetryEnabled() || objectKey == null || objectKey.isBlank()) {
|
||||
@@ -97,6 +143,8 @@ public class RustfsDeleteRetryService {
|
||||
}
|
||||
retryNextInvocation.forEach(this::requeue);
|
||||
if (processed > 0) {
|
||||
// 收敛落盘:成功删除的对象从日志移除,仍失败/待重试的保留
|
||||
journal().rewrite(pending.keySet());
|
||||
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={}",
|
||||
processed, success, failed, pending.size());
|
||||
}
|
||||
@@ -124,6 +172,8 @@ public class RustfsDeleteRetryService {
|
||||
}
|
||||
item = new RetryItem(objectKey);
|
||||
pending.put(objectKey, item);
|
||||
// 只在"新准入"时记账:重写以 pending 为准,重复失败无需重复追加
|
||||
journal().record(objectKey);
|
||||
return item;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user