super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

总表更新

Browse Source
This commit is contained in:
super
2026-03-22 17:51:34 +08:00
parent 55ac81f0cc
commit ed91c2ae7d
11 changed files with 413 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java
View File

@@ -75,6 +75,23 @@ public class DedupeTotalDataController {
return ApiResponse.success(dedupeTotalDataService.getImportProgress(importId));
}
@PostMapping("/delete-import")
@Operation(summary = "删除导入总数据", description = "上传 xlsx 文件，读取 ASIN 列并批量删除已存在的数据去重总数据。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "启动删除成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataImportStartVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "文件不合法或缺少 ASIN 列")
})
public ApiResponse<DedupeTotalDataImportStartVo> deleteImportExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file) {
return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file));
}
@GetMapping("/delete-import/{importId}")
@Operation(summary = "查询删除导入进度", description = "根据删除导入任务 ID 查询当前进度。")
public ApiResponse<DedupeTotalDataImportProgressVo> deleteImportProgress(@PathVariable String importId) {
return ApiResponse.success(dedupeTotalDataService.getDeleteImportProgress(importId));
}
@PutMapping("/{id}")
@Operation(summary = "更新总数据", description = "按 ID 更新一条数据去重总数据。")
@ApiResponses({

48
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
View File

@@ -262,25 +262,9 @@ public class DedupeRunService {
}
Integer idColumnIndex = headerMap.get("id");
Map<String, Boolean> mainIdHasSubIdMap = new HashMap<>();
if (idColumnIndex != null) {
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
String mainId = extractMainId(idValue);
if (mainId.isEmpty()) {
continue;
}
if (!mainIdHasSubIdMap.containsKey(mainId)) {
mainIdHasSubIdMap.put(mainId, false);
}
if (idValue.matches("\\d+_\\d+")) {
mainIdHasSubIdMap.put(mainId, true);
}
}
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
int outputRowIndex = 1;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
@@ -290,7 +274,7 @@ public class DedupeRunService {
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, mainIdHasSubIdMap)) {
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
continue;
}
}
@@ -342,7 +326,7 @@ public class DedupeRunService {
}
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds, Map<String, Boolean> mainIdHasSubIdMap) {
boolean keepIntegerMainIdsWhenNoSubIds, Sheet sheet, int currentRowNum, int idColumnIndex, DataFormatter formatter) {
if (text == null || text.isBlank()) {
return false;
}
@@ -354,7 +338,29 @@ public class DedupeRunService {
}
if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) {
String mainId = extractMainId(text);
return !mainId.isEmpty() && !Boolean.TRUE.equals(mainIdHasSubIdMap.get(mainId));
if (mainId.isEmpty()) return false;
boolean hasSubIdInGroup = false;
for (int r = currentRowNum + 1; r <= sheet.getLastRowNum(); r++) {
Row nextRow = sheet.getRow(r);
if (nextRow == null) continue;
Cell cell = nextRow.getCell(idColumnIndex);
if (cell == null) continue;
String nextIdValue = normalizeCellText(formatter.formatCellValue(cell));
if (nextIdValue.isBlank()) continue;
String nextMainId = extractMainId(nextIdValue);
if (nextMainId.equals(mainId)) {
if (nextIdValue.matches("\\d+_\\d+")) {
hasSubIdInGroup = true;
break;
}
} else {
// Encountered a different main ID. The group for 'mainId' has ended.
break;
}
}
return !hasSubIdInGroup;
}
return false;
}

182
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
View File

@@ -38,6 +38,7 @@ public class DedupeTotalDataService {
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();
private final Map<String, DedupeTotalDataImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword) {
long safePage = Math.max(page, 1);
@@ -105,6 +106,62 @@ public class DedupeTotalDataService {
return progress;
}
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
String importId = IdUtil.fastSimpleUUID();
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("pending");
progress.setTotalRows(0);
progress.setProcessedRows(0);
progress.setAsinCount(0);
progress.setInsertedCount(0);
progress.setSkippedCount(0);
deleteImportProgressMap.put(importId, progress);
try {
byte[] fileBytes = file.getBytes();
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, fileBytes, filename));
} catch (Exception e) {
deleteImportProgressMap.remove(importId);
throw new BusinessException("读取上传文件失败");
}
DedupeTotalDataImportStartVo vo = new DedupeTotalDataImportStartVo();
vo.setImportId(importId);
return vo;
}
public DedupeTotalDataImportProgressVo getDeleteImportProgress(String importId) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
throw new BusinessException("删除任务不存在");
}
return progress;
}
private void runDeleteImportTask(String importId, byte[] fileBytes, String filename) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
return;
}
progress.setStatus("running");
try (InputStream inputStream = new ByteArrayInputStream(fileBytes)) {
DedupeTotalDataImportVo result = deleteFromExcelInternal(inputStream, filename, progress);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
progress.setSkippedCount(result.getSkippedCount());
progress.setProcessedRows(result.getTotalRows());
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "删除 Excel 匹配数据失败");
}
}
private void runImportTask(String importId, byte[] fileBytes, String filename) {
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
if (progress == null) {
@@ -139,6 +196,20 @@ public class DedupeTotalDataService {
}
}
@Transactional
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
try (InputStream inputStream = file.getInputStream()) {
return deleteFromExcelInternal(inputStream, file.getOriginalFilename(), null);
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("删除 Excel 匹配数据失败");
}
}
@Transactional
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename, DedupeTotalDataImportProgressVo progress) {
String lowerFilename = filename == null ? "" : filename.toLowerCase();
@@ -252,6 +323,112 @@ public class DedupeTotalDataService {
}
}
@Transactional
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename, DedupeTotalDataImportProgressVo progress) {
String lowerFilename = filename == null ? "" : filename.toLowerCase();
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
}
DataFormatter formatter = new DataFormatter();
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
if (!value.isBlank() && !headerMap.containsKey(value)) {
headerMap.put(value, i);
}
}
Integer asinIndex = headerMap.get("ASIN");
if (asinIndex == null) {
throw new BusinessException("缺少 ASIN 列");
}
Set<String> seenInFile = new HashSet<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
int asinCount = 0;
int deletedCount = 0;
int skippedCount = 0;
if (progress != null) {
progress.setTotalRows(totalRows);
progress.setProcessedRows(0);
progress.setAsinCount(0);
progress.setInsertedCount(0);
progress.setSkippedCount(0);
progress.setErrorMessage(null);
}
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
asinCount++;
String dataValue = normalizeDataValue(asin);
if (!seenInFile.add(dataValue)) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
int deletedThisRow = deleteByDataValue(dataValue);
if (deletedThisRow > 0) {
deletedCount += deletedThisRow;
} else {
skippedCount++;
}
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(deletedCount);
progress.setSkippedCount(skippedCount);
}
}
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
vo.setTotalRows(totalRows);
vo.setAsinCount(asinCount);
vo.setInsertedCount(deletedCount);
vo.setSkippedCount(skippedCount);
return vo;
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("删除 Excel 匹配数据失败");
}
}
@Transactional
public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) {
DedupeTotalDataEntity entity = getById(id);
@@ -301,6 +478,11 @@ public class DedupeTotalDataService {
return dedupeTotalDataMapper.selectOne(query) != null;
}
private int deleteByDataValue(String dataValue) {
return dedupeTotalDataMapper.delete(new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getDataValue, dataValue));
}
private String normalizeExcelText(String value) {
if (value == null) {
return "";