refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权
模块边界(G5 / G7) - 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何 业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册 - 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、 CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块 - G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo - 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测) - 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写) 性能与容量(C5/C6/C7/D13) - C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、 BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader; 行数上限改为迭代中生效 - C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country - C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存 - D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与 TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志 - V123 删除 biz_file_result 两个被复合索引覆盖的单列索引 安全(A3/A4/A7 + 守卫名单) - A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外) - A3 上传接口加危险扩展名黑名单(可配置) - AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层) 容量(明细表保留期) - 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理 (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
+248
-175
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
|
||||
@@ -122,7 +123,10 @@ public class DedupeTotalDataService {
|
||||
String safeCountry = country == null ? "" : country.trim();
|
||||
AccessScope scope = resolveAccessScope(operatorId);
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
|
||||
// 2026-09 全维度审查 C6:关键字由"包含"改为"前缀"匹配——后台该输入框标注的是
|
||||
// 「输入 ASIN」(相邻的用户名框才是"模糊搜索"),前缀匹配才能命中 data_value 索引;
|
||||
// 此前的 %kw% 前置通配使 idx 永不生效,93 万行表每翻页两次全表扫
|
||||
.likeRight(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
|
||||
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
|
||||
.apply(!safeCountry.isEmpty(),
|
||||
"FIND_IN_SET({0}, IFNULL(country, '')) > 0",
|
||||
@@ -667,101 +671,162 @@ public class DedupeTotalDataService {
|
||||
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
||||
}
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
// 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。
|
||||
// 此前 WorkbookFactory.create 把整表读成 DOM(50 万行约 1~2GB 堆),且行数上限在
|
||||
// 加载完之后才校验(等于没有防线)。现在内存只保留单批待插入值与文件内去重集合,
|
||||
// 行数上限在迭代过程中即时生效。
|
||||
ImportSession session = new ImportSession(progress, groupId, uploaderUserId, uploaderUsername);
|
||||
try {
|
||||
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
|
||||
private Integer countryColumnIndex;
|
||||
private boolean headerReady;
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (!value.isBlank() && !headerMap.containsKey(value)) {
|
||||
headerMap.put(value, i);
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
session.headerIndex = buildHeaderIndex(headerMap);
|
||||
if (session.headerIndex.get("ASIN") == null) {
|
||||
throw new BusinessException("缺少 ASIN 列");
|
||||
}
|
||||
countryColumnIndex = session.headerIndex.get("国家");
|
||||
headerReady = true;
|
||||
}
|
||||
}
|
||||
|
||||
Integer asinIndex = headerMap.get("ASIN");
|
||||
if (asinIndex == null) {
|
||||
throw new BusinessException("缺少 ASIN 列");
|
||||
}
|
||||
Integer countryIndex = headerMap.get("国家");
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) {
|
||||
progress.setTotalRows(approximateTotalRows);
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> seenInFile = new HashSet<>();
|
||||
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
|
||||
Map<String, String> pendingCountries = new HashMap<>();
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
if (maxImportRows > 0 && totalRows > maxImportRows) {
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
if (!headerReady) {
|
||||
// 无表头(空表):与旧 DOM 实现一致,不处理任何数据行
|
||||
return;
|
||||
}
|
||||
session.acceptRow(rowMap, countryColumnIndex);
|
||||
}
|
||||
});
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.error("[dedupe-total-data] 导入 Excel 失败 filename={} processedRows={}",
|
||||
filename, session.processedRows(), e);
|
||||
throw new BusinessException("导入 Excel 失败", e);
|
||||
}
|
||||
return session.finish();
|
||||
}
|
||||
|
||||
/** 表头名 → 列索引(首次出现优先、忽略空表头),与旧 DOM 实现口径一致。 */
|
||||
private Map<String, Integer> buildHeaderIndex(Map<Integer, String> headerMap) {
|
||||
Map<String, Integer> index = new HashMap<>();
|
||||
if (headerMap == null) {
|
||||
return index;
|
||||
}
|
||||
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
|
||||
if (entry.getKey() == null) {
|
||||
continue;
|
||||
}
|
||||
String value = normalizeExcelText(entry.getValue());
|
||||
if (!value.isBlank() && !index.containsKey(value)) {
|
||||
index.put(value, entry.getKey());
|
||||
}
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/** 流式解析下按列索引取行内文本。 */
|
||||
private String cellText(Map<Integer, String> rowMap, Integer columnIndex) {
|
||||
if (rowMap == null || columnIndex == null) {
|
||||
return null;
|
||||
}
|
||||
return rowMap.get(columnIndex);
|
||||
}
|
||||
|
||||
/** 流式导入的累加器:单批待插入值 + 计数与进度(内存只随批大小增长,不随文件行数增长)。 */
|
||||
private final class ImportSession {
|
||||
|
||||
private final DedupeTotalDataImportProgressVo progress;
|
||||
private final Long groupId;
|
||||
private final Long uploaderUserId;
|
||||
private final String uploaderUsername;
|
||||
private final Set<String> seenInFile = new HashSet<>();
|
||||
private final List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
|
||||
private final Map<String, String> pendingCountries = new HashMap<>();
|
||||
private Map<String, Integer> headerIndex = Map.of();
|
||||
private int processedRows;
|
||||
private int asinCount;
|
||||
private int insertedCount;
|
||||
private int skippedCount;
|
||||
|
||||
private ImportSession(DedupeTotalDataImportProgressVo progress, Long groupId,
|
||||
Long uploaderUserId, String uploaderUsername) {
|
||||
this.progress = progress;
|
||||
this.groupId = groupId;
|
||||
this.uploaderUserId = uploaderUserId;
|
||||
this.uploaderUsername = uploaderUsername;
|
||||
}
|
||||
|
||||
private void acceptRow(Map<Integer, String> rowMap, Integer countryColumnIndex) {
|
||||
processedRows++;
|
||||
if (maxImportRows > 0 && processedRows > maxImportRows) {
|
||||
// 流式下无法前置知道总行数:超限即中断,已插入批次保留(与旧实现的"截断"语义不同,
|
||||
// 但旧实现在加载后才校验,实际是 OOM 之后才失败)
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
int asinCount = 0;
|
||||
int insertedCount = 0;
|
||||
int skippedCount = 0;
|
||||
if (progress != null) {
|
||||
progress.setTotalRows(totalRows);
|
||||
progress.setProcessedRows(0);
|
||||
progress.setAsinCount(0);
|
||||
progress.setInsertedCount(0);
|
||||
progress.setSkippedCount(0);
|
||||
progress.setErrorMessage(null);
|
||||
Integer asinIndex = headerIndex.get("ASIN");
|
||||
String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex));
|
||||
if (asin.isBlank()) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
return;
|
||||
}
|
||||
asinCount++;
|
||||
if (!seenInFile.add(asin)) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
return;
|
||||
}
|
||||
pendingValues.add(asin);
|
||||
if (countryColumnIndex != null) {
|
||||
pendingCountries.put(asin, parseCountries(cellText(rowMap, countryColumnIndex)));
|
||||
}
|
||||
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
|
||||
flushBatch();
|
||||
}
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
}
|
||||
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
}
|
||||
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
|
||||
if (asin.isBlank()) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
}
|
||||
asinCount++;
|
||||
if (!seenInFile.add(asin)) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
}
|
||||
pendingValues.add(asin);
|
||||
if (countryIndex != null) {
|
||||
pendingCountries.put(asin, parseCountries(formatter.formatCellValue(row.getCell(countryIndex))));
|
||||
}
|
||||
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
pendingCountries.clear();
|
||||
}
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
private void flushBatch() {
|
||||
if (pendingValues.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
if (!pendingValues.isEmpty()) {
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
pendingCountries.clear();
|
||||
}
|
||||
updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount);
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
pendingCountries.clear();
|
||||
}
|
||||
|
||||
private int processedRows() {
|
||||
return processedRows;
|
||||
}
|
||||
|
||||
private DedupeTotalDataImportVo finish() {
|
||||
flushBatch();
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
if (progress != null && processedRows > progress.getTotalRows()) {
|
||||
// 近似总行数缺失(或与实际物理行数不一致)时用实际处理行数兜底,避免展示成 x/0
|
||||
progress.setTotalRows(processedRows);
|
||||
}
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(totalRows);
|
||||
vo.setTotalRows(processedRows);
|
||||
vo.setAsinCount(asinCount);
|
||||
vo.setInsertedCount(insertedCount);
|
||||
vo.setSkippedCount(skippedCount);
|
||||
return vo;
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("导入 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -850,112 +915,120 @@ public class DedupeTotalDataService {
|
||||
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
||||
}
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
|
||||
DeleteSession session = new DeleteSession(progress, scope, groupId);
|
||||
try {
|
||||
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
|
||||
private boolean headerReady;
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (!value.isBlank() && !headerMap.containsKey(value)) {
|
||||
headerMap.put(value, i);
|
||||
}
|
||||
}
|
||||
|
||||
Integer asinIndex = headerMap.get("ASIN");
|
||||
if (asinIndex == null) {
|
||||
throw new BusinessException("缺少 ASIN 列");
|
||||
}
|
||||
|
||||
Set<String> seenInFile = new HashSet<>();
|
||||
List<String> pendingDeletes = new ArrayList<>();
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
if (maxImportRows > 0 && totalRows > maxImportRows) {
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
int asinCount = 0;
|
||||
int deletedCount = 0;
|
||||
int skippedCount = 0;
|
||||
if (progress != null) {
|
||||
progress.setTotalRows(totalRows);
|
||||
progress.setProcessedRows(0);
|
||||
progress.setAsinCount(0);
|
||||
progress.setInsertedCount(0);
|
||||
progress.setSkippedCount(0);
|
||||
progress.setErrorMessage(null);
|
||||
}
|
||||
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
session.headerIndex = buildHeaderIndex(headerMap);
|
||||
if (session.headerIndex.get("ASIN") == null) {
|
||||
throw new BusinessException("缺少 ASIN 列");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
|
||||
if (asin.isBlank()) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
asinCount++;
|
||||
String dataValue = asin;
|
||||
if (!seenInFile.add(dataValue)) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
continue;
|
||||
headerReady = true;
|
||||
}
|
||||
|
||||
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
|
||||
pendingDeletes.add(dataValue);
|
||||
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
pendingDeletes = new ArrayList<>();
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) {
|
||||
progress.setTotalRows(approximateTotalRows);
|
||||
}
|
||||
}
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
}
|
||||
if (!pendingDeletes.isEmpty()) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
}
|
||||
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(totalRows);
|
||||
vo.setAsinCount(asinCount);
|
||||
vo.setInsertedCount(deletedCount);
|
||||
vo.setSkippedCount(skippedCount);
|
||||
return vo;
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
if (!headerReady) {
|
||||
return;
|
||||
}
|
||||
session.acceptRow(rowMap);
|
||||
}
|
||||
});
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
|
||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
|
||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败 filename={} processedRows={}",
|
||||
filename, session.processedRows(), e);
|
||||
throw new BusinessException("删除 Excel 匹配数据失败", e);
|
||||
}
|
||||
return session.finish();
|
||||
}
|
||||
|
||||
/** 流式删除匹配的累加器:单批待删值 + 计数与进度。 */
|
||||
private final class DeleteSession {
|
||||
|
||||
private final DedupeTotalDataImportProgressVo progress;
|
||||
private final AccessScope scope;
|
||||
private final Long groupId;
|
||||
private final Set<String> seenInFile = new HashSet<>();
|
||||
private final List<String> pendingDeletes = new ArrayList<>();
|
||||
private Map<String, Integer> headerIndex = Map.of();
|
||||
private int processedRows;
|
||||
private int asinCount;
|
||||
private int deletedCount;
|
||||
private int skippedCount;
|
||||
|
||||
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) {
|
||||
this.progress = progress;
|
||||
this.scope = scope;
|
||||
this.groupId = groupId;
|
||||
}
|
||||
|
||||
private void acceptRow(Map<Integer, String> rowMap) {
|
||||
processedRows++;
|
||||
if (maxImportRows > 0 && processedRows > maxImportRows) {
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
Integer asinIndex = headerIndex.get("ASIN");
|
||||
String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex));
|
||||
if (asin.isBlank() || !seenInFile.add(asin)) {
|
||||
skippedCount++;
|
||||
updateProgress();
|
||||
return;
|
||||
}
|
||||
asinCount++;
|
||||
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
|
||||
pendingDeletes.add(asin);
|
||||
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
pendingDeletes.clear();
|
||||
}
|
||||
updateProgress();
|
||||
}
|
||||
|
||||
private void updateProgress() {
|
||||
if (progress == null) {
|
||||
return;
|
||||
}
|
||||
progress.setProcessedRows(processedRows);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
|
||||
private int processedRows() {
|
||||
return processedRows;
|
||||
}
|
||||
|
||||
private DedupeTotalDataImportVo finish() {
|
||||
if (!pendingDeletes.isEmpty()) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
pendingDeletes.clear();
|
||||
}
|
||||
updateProgress();
|
||||
if (progress != null && processedRows > progress.getTotalRows()) {
|
||||
progress.setTotalRows(processedRows);
|
||||
}
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(processedRows);
|
||||
vo.setAsinCount(asinCount);
|
||||
vo.setInsertedCount(deletedCount);
|
||||
vo.setSkippedCount(skippedCount);
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
|
||||
Reference in New Issue
Block a user