refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权

模块边界(G5 / G7)
- 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何
  业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册
- 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、
  CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块
- G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo
- 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测)
- 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写)

性能与容量(C5/C6/C7/D13)
- C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、
  BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader;
  行数上限改为迭代中生效
- C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country
- C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存
- D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与
  TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志
- V123 删除 biz_file_result 两个被复合索引覆盖的单列索引

安全(A3/A4/A7 + 守卫名单)
- A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外)
- A3 上传接口加危险扩展名黑名单(可配置)
- AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层)

容量(明细表保留期)
- 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理
  (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
2026-09-14 06:46:27 +08:00
parent d87d76398e
commit f4ee7e16f0
86 changed files with 3191 additions and 952 deletions
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service;
import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
@@ -122,7 +123,10 @@ public class DedupeTotalDataService {
String safeCountry = country == null ? "" : country.trim();
AccessScope scope = resolveAccessScope(operatorId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
// 2026-09 全维度审查 C6:关键字由"包含"改为"前缀"匹配——后台该输入框标注的是
// 「输入 ASIN」(相邻的用户名框才是"模糊搜索"),前缀匹配才能命中 data_value 索引;
// 此前的 %kw% 前置通配使 idx 永不生效,93 万行表每翻页两次全表扫
.likeRight(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.apply(!safeCountry.isEmpty(),
"FIND_IN_SET({0}, IFNULL(country, '')) > 0",
@@ -667,101 +671,162 @@ public class DedupeTotalDataService {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
}
DataFormatter formatter = new DataFormatter();
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
// 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。
// 此前 WorkbookFactory.create 把整表读成 DOM(50 万行约 1~2GB 堆),且行数上限在
// 加载完之后才校验(等于没有防线)。现在内存只保留单批待插入值与文件内去重集合,
// 行数上限在迭代过程中即时生效。
ImportSession session = new ImportSession(progress, groupId, uploaderUserId, uploaderUsername);
try {
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
private Integer countryColumnIndex;
private boolean headerReady;
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
if (!value.isBlank() && !headerMap.containsKey(value)) {
headerMap.put(value, i);
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
session.headerIndex = buildHeaderIndex(headerMap);
if (session.headerIndex.get("ASIN") == null) {
throw new BusinessException("缺少 ASIN 列");
}
countryColumnIndex = session.headerIndex.get("国家");
headerReady = true;
}
}
Integer asinIndex = headerMap.get("ASIN");
if (asinIndex == null) {
throw new BusinessException("缺少 ASIN 列");
}
Integer countryIndex = headerMap.get("国家");
@Override
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) {
progress.setTotalRows(approximateTotalRows);
}
}
Set<String> seenInFile = new HashSet<>();
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
Map<String, String> pendingCountries = new HashMap<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
if (maxImportRows > 0 && totalRows > maxImportRows) {
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
if (!headerReady) {
// 无表头(空表):与旧 DOM 实现一致,不处理任何数据行
return;
}
session.acceptRow(rowMap, countryColumnIndex);
}
});
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
log.error("[dedupe-total-data] 导入 Excel 失败 filename={} processedRows={}",
filename, session.processedRows(), e);
throw new BusinessException("导入 Excel 失败", e);
}
return session.finish();
}
/** 表头名 → 列索引(首次出现优先、忽略空表头),与旧 DOM 实现口径一致。 */
private Map<String, Integer> buildHeaderIndex(Map<Integer, String> headerMap) {
Map<String, Integer> index = new HashMap<>();
if (headerMap == null) {
return index;
}
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
if (entry.getKey() == null) {
continue;
}
String value = normalizeExcelText(entry.getValue());
if (!value.isBlank() && !index.containsKey(value)) {
index.put(value, entry.getKey());
}
}
return index;
}
/** 流式解析下按列索引取行内文本。 */
private String cellText(Map<Integer, String> rowMap, Integer columnIndex) {
if (rowMap == null || columnIndex == null) {
return null;
}
return rowMap.get(columnIndex);
}
/** 流式导入的累加器:单批待插入值 + 计数与进度(内存只随批大小增长,不随文件行数增长)。 */
private final class ImportSession {
private final DedupeTotalDataImportProgressVo progress;
private final Long groupId;
private final Long uploaderUserId;
private final String uploaderUsername;
private final Set<String> seenInFile = new HashSet<>();
private final List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
private final Map<String, String> pendingCountries = new HashMap<>();
private Map<String, Integer> headerIndex = Map.of();
private int processedRows;
private int asinCount;
private int insertedCount;
private int skippedCount;
private ImportSession(DedupeTotalDataImportProgressVo progress, Long groupId,
Long uploaderUserId, String uploaderUsername) {
this.progress = progress;
this.groupId = groupId;
this.uploaderUserId = uploaderUserId;
this.uploaderUsername = uploaderUsername;
}
private void acceptRow(Map<Integer, String> rowMap, Integer countryColumnIndex) {
processedRows++;
if (maxImportRows > 0 && processedRows > maxImportRows) {
// 流式下无法前置知道总行数:超限即中断,已插入批次保留(与旧实现的"截断"语义不同,
// 但旧实现在加载后才校验,实际是 OOM 之后才失败)
throw new BusinessException("导入行数超过上限: " + maxImportRows);
}
int asinCount = 0;
int insertedCount = 0;
int skippedCount = 0;
if (progress != null) {
progress.setTotalRows(totalRows);
progress.setProcessedRows(0);
progress.setAsinCount(0);
progress.setInsertedCount(0);
progress.setSkippedCount(0);
progress.setErrorMessage(null);
Integer asinIndex = headerIndex.get("ASIN");
String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex));
if (asin.isBlank()) {
skippedCount++;
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
return;
}
asinCount++;
if (!seenInFile.add(asin)) {
skippedCount++;
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
return;
}
pendingValues.add(asin);
if (countryColumnIndex != null) {
pendingCountries.put(asin, parseCountries(cellText(rowMap, countryColumnIndex)));
}
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
flushBatch();
}
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
}
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
skippedCount++;
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
continue;
}
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) {
skippedCount++;
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
continue;
}
asinCount++;
if (!seenInFile.add(asin)) {
skippedCount++;
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
continue;
}
pendingValues.add(asin);
if (countryIndex != null) {
pendingCountries.put(asin, parseCountries(formatter.formatCellValue(row.getCell(countryIndex))));
}
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
ImportBatchResult batch = insertImportBatch(
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
pendingCountries.clear();
}
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
private void flushBatch() {
if (pendingValues.isEmpty()) {
return;
}
if (!pendingValues.isEmpty()) {
ImportBatchResult batch = insertImportBatch(
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
pendingCountries.clear();
}
updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount);
ImportBatchResult batch = insertImportBatch(
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
pendingCountries.clear();
}
private int processedRows() {
return processedRows;
}
private DedupeTotalDataImportVo finish() {
flushBatch();
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
if (progress != null && processedRows > progress.getTotalRows()) {
// 近似总行数缺失(或与实际物理行数不一致)时用实际处理行数兜底,避免展示成 x/0
progress.setTotalRows(processedRows);
}
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
vo.setTotalRows(totalRows);
vo.setTotalRows(processedRows);
vo.setAsinCount(asinCount);
vo.setInsertedCount(insertedCount);
vo.setSkippedCount(skippedCount);
return vo;
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("导入 Excel 失败");
}
}
@@ -850,112 +915,120 @@ public class DedupeTotalDataService {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
}
DataFormatter formatter = new DataFormatter();
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
DeleteSession session = new DeleteSession(progress, scope, groupId);
try {
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
private boolean headerReady;
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
if (!value.isBlank() && !headerMap.containsKey(value)) {
headerMap.put(value, i);
}
}
Integer asinIndex = headerMap.get("ASIN");
if (asinIndex == null) {
throw new BusinessException("缺少 ASIN 列");
}
Set<String> seenInFile = new HashSet<>();
List<String> pendingDeletes = new ArrayList<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
if (maxImportRows > 0 && totalRows > maxImportRows) {
throw new BusinessException("导入行数超过上限: " + maxImportRows);
}
int asinCount = 0;
int deletedCount = 0;
int skippedCount = 0;
if (progress != null) {
progress.setTotalRows(totalRows);
progress.setProcessedRows(0);
progress.setAsinCount(0);
progress.setInsertedCount(0);
progress.setSkippedCount(0);
progress.setErrorMessage(null);
}
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
session.headerIndex = buildHeaderIndex(headerMap);
if (session.headerIndex.get("ASIN") == null) {
throw new BusinessException("缺少 ASIN 列");
}
continue;
}
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
asinCount++;
String dataValue = asin;
if (!seenInFile.add(dataValue)) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
continue;
headerReady = true;
}
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
pendingDeletes.add(dataValue);
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
pendingDeletes = new ArrayList<>();
@Override
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) {
progress.setTotalRows(approximateTotalRows);
}
}
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
}
if (!pendingDeletes.isEmpty()) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
}
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
vo.setTotalRows(totalRows);
vo.setAsinCount(asinCount);
vo.setInsertedCount(deletedCount);
vo.setSkippedCount(skippedCount);
return vo;
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
if (!headerReady) {
return;
}
session.acceptRow(rowMap);
}
});
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败 filename={} processedRows={}",
filename, session.processedRows(), e);
throw new BusinessException("删除 Excel 匹配数据失败", e);
}
return session.finish();
}
/** 流式删除匹配的累加器:单批待删值 + 计数与进度。 */
private final class DeleteSession {
private final DedupeTotalDataImportProgressVo progress;
private final AccessScope scope;
private final Long groupId;
private final Set<String> seenInFile = new HashSet<>();
private final List<String> pendingDeletes = new ArrayList<>();
private Map<String, Integer> headerIndex = Map.of();
private int processedRows;
private int asinCount;
private int deletedCount;
private int skippedCount;
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) {
this.progress = progress;
this.scope = scope;
this.groupId = groupId;
}
private void acceptRow(Map<Integer, String> rowMap) {
processedRows++;
if (maxImportRows > 0 && processedRows > maxImportRows) {
throw new BusinessException("导入行数超过上限: " + maxImportRows);
}
Integer asinIndex = headerIndex.get("ASIN");
String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex));
if (asin.isBlank() || !seenInFile.add(asin)) {
skippedCount++;
updateProgress();
return;
}
asinCount++;
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
pendingDeletes.add(asin);
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
pendingDeletes.clear();
}
updateProgress();
}
private void updateProgress() {
if (progress == null) {
return;
}
progress.setProcessedRows(processedRows);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
private int processedRows() {
return processedRows;
}
private DedupeTotalDataImportVo finish() {
if (!pendingDeletes.isEmpty()) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
pendingDeletes.clear();
}
updateProgress();
if (progress != null && processedRows > progress.getTotalRows()) {
progress.setTotalRows(processedRows);
}
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
vo.setTotalRows(processedRows);
vo.setAsinCount(asinCount);
vo.setInsertedCount(deletedCount);
vo.setSkippedCount(skippedCount);
return vo;
}
}
@Transactional