diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java index b451b9f..25c48d0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java @@ -75,6 +75,23 @@ public class DedupeTotalDataController { return ApiResponse.success(dedupeTotalDataService.getImportProgress(importId)); } + @PostMapping("/delete-import") + @Operation(summary = "删除导入总数据", description = "上传 xlsx 文件，读取 ASIN 列并批量删除已存在的数据去重总数据。") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "启动删除成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataImportStartVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "文件不合法或缺少 ASIN 列") + }) + public ApiResponse deleteImportExcel( + @Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file) { + return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file)); + } + + @GetMapping("/delete-import/{importId}") + @Operation(summary = "查询删除导入进度", description = "根据删除导入任务 ID 查询当前进度。") + public ApiResponse deleteImportProgress(@PathVariable String importId) { + return ApiResponse.success(dedupeTotalDataService.getDeleteImportProgress(importId)); + } + @PutMapping("/{id}") @Operation(summary = "更新总数据", description = "按 ID 更新一条数据去重总数据。") @ApiResponses({ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 8319c2c..1355d3f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -262,25 +262,9 @@ public class DedupeRunService { } Integer idColumnIndex = headerMap.get("id"); - Map mainIdHasSubIdMap = new HashMap<>(); if (idColumnIndex != null) { - for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { - continue; - } - String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))); - String mainId = extractMainId(idValue); - if (mainId.isEmpty()) { - continue; - } - if (!mainIdHasSubIdMap.containsKey(mainId)) { - mainIdHasSubIdMap.put(mainId, false); - } - if (idValue.matches("\\d+_\\d+")) { - mainIdHasSubIdMap.put(mainId, true); - } - } + // The preliminary global scan for mainIdHasSubIdMap was removed. + // We now determine subset existence contextually (per group) during the main loop. } int outputRowIndex = 1; for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { @@ -290,7 +274,7 @@ public class DedupeRunService { } if (idColumnIndex != null) { String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))); - if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, mainIdHasSubIdMap)) { + if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) { continue; } } @@ -342,7 +326,7 @@ public class DedupeRunService { } private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds, - boolean keepIntegerMainIdsWhenNoSubIds, Map mainIdHasSubIdMap) { + boolean keepIntegerMainIdsWhenNoSubIds, Sheet sheet, int currentRowNum, int idColumnIndex, DataFormatter formatter) { if (text == null || text.isBlank()) { return false; } @@ -354,7 +338,29 @@ public class DedupeRunService { } if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) { String mainId = extractMainId(text); - return !mainId.isEmpty() && !Boolean.TRUE.equals(mainIdHasSubIdMap.get(mainId)); + if (mainId.isEmpty()) return false; + + boolean hasSubIdInGroup = false; + for (int r = currentRowNum + 1; r <= sheet.getLastRowNum(); r++) { + Row nextRow = sheet.getRow(r); + if (nextRow == null) continue; + Cell cell = nextRow.getCell(idColumnIndex); + if (cell == null) continue; + String nextIdValue = normalizeCellText(formatter.formatCellValue(cell)); + if (nextIdValue.isBlank()) continue; + + String nextMainId = extractMainId(nextIdValue); + if (nextMainId.equals(mainId)) { + if (nextIdValue.matches("\\d+_\\d+")) { + hasSubIdInGroup = true; + break; + } + } else { + // Encountered a different main ID. The group for 'mainId' has ended. + break; + } + } + return !hasSubIdInGroup; } return false; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java index 4387087..80d2fd4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -38,6 +38,7 @@ public class DedupeTotalDataService { private final DedupeTotalDataMapper dedupeTotalDataMapper; private final Map importProgressMap = new ConcurrentHashMap<>(); + private final Map deleteImportProgressMap = new ConcurrentHashMap<>(); public DedupeTotalDataPageVo page(long page, long pageSize, String keyword) { long safePage = Math.max(page, 1); @@ -105,6 +106,62 @@ public class DedupeTotalDataService { return progress; } + public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file) { + if (file == null || file.isEmpty()) { + throw new BusinessException("请上传 xlsx 文件"); + } + String importId = IdUtil.fastSimpleUUID(); + DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo(); + progress.setStatus("pending"); + progress.setTotalRows(0); + progress.setProcessedRows(0); + progress.setAsinCount(0); + progress.setInsertedCount(0); + progress.setSkippedCount(0); + deleteImportProgressMap.put(importId, progress); + + try { + byte[] fileBytes = file.getBytes(); + String filename = file.getOriginalFilename(); + Thread.ofVirtual().start(() -> runDeleteImportTask(importId, fileBytes, filename)); + } catch (Exception e) { + deleteImportProgressMap.remove(importId); + throw new BusinessException("读取上传文件失败"); + } + + DedupeTotalDataImportStartVo vo = new DedupeTotalDataImportStartVo(); + vo.setImportId(importId); + return vo; + } + + public DedupeTotalDataImportProgressVo getDeleteImportProgress(String importId) { + DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId); + if (progress == null) { + throw new BusinessException("删除任务不存在"); + } + return progress; + } + + private void runDeleteImportTask(String importId, byte[] fileBytes, String filename) { + DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId); + if (progress == null) { + return; + } + progress.setStatus("running"); + try (InputStream inputStream = new ByteArrayInputStream(fileBytes)) { + DedupeTotalDataImportVo result = deleteFromExcelInternal(inputStream, filename, progress); + progress.setTotalRows(result.getTotalRows()); + progress.setAsinCount(result.getAsinCount()); + progress.setInsertedCount(result.getInsertedCount()); + progress.setSkippedCount(result.getSkippedCount()); + progress.setProcessedRows(result.getTotalRows()); + progress.setStatus("success"); + } catch (Exception e) { + progress.setStatus("failed"); + progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "删除 Excel 匹配数据失败"); + } + } + private void runImportTask(String importId, byte[] fileBytes, String filename) { DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId); if (progress == null) { @@ -139,6 +196,20 @@ public class DedupeTotalDataService { } } + @Transactional + public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file) { + if (file == null || file.isEmpty()) { + throw new BusinessException("请上传 xlsx 文件"); + } + try (InputStream inputStream = file.getInputStream()) { + return deleteFromExcelInternal(inputStream, file.getOriginalFilename(), null); + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + throw new BusinessException("删除 Excel 匹配数据失败"); + } + } + @Transactional private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename, DedupeTotalDataImportProgressVo progress) { String lowerFilename = filename == null ? "" : filename.toLowerCase(); @@ -252,6 +323,112 @@ public class DedupeTotalDataService { } } + @Transactional + private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename, DedupeTotalDataImportProgressVo progress) { + String lowerFilename = filename == null ? "" : filename.toLowerCase(); + if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) { + throw new BusinessException("仅支持 .xlsx 或 .xls 文件"); + } + + DataFormatter formatter = new DataFormatter(); + try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) { + Sheet sheet = workbook.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + throw new BusinessException("Excel 表头为空"); + } + + Map headerMap = new HashMap<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell)); + if (!value.isBlank() && !headerMap.containsKey(value)) { + headerMap.put(value, i); + } + } + + Integer asinIndex = headerMap.get("ASIN"); + if (asinIndex == null) { + throw new BusinessException("缺少 ASIN 列"); + } + + Set seenInFile = new HashSet<>(); + int totalRows = Math.max(sheet.getLastRowNum(), 0); + int asinCount = 0; + int deletedCount = 0; + int skippedCount = 0; + if (progress != null) { + progress.setTotalRows(totalRows); + progress.setProcessedRows(0); + progress.setAsinCount(0); + progress.setInsertedCount(0); + progress.setSkippedCount(0); + progress.setErrorMessage(null); + } + + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + skippedCount++; + if (progress != null) { + progress.setProcessedRows(rowNum); + progress.setAsinCount(asinCount); + progress.setInsertedCount(deletedCount); + progress.setSkippedCount(skippedCount); + } + continue; + } + String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex))); + if (asin.isBlank()) { + skippedCount++; + if (progress != null) { + progress.setProcessedRows(rowNum); + progress.setAsinCount(asinCount); + progress.setInsertedCount(deletedCount); + progress.setSkippedCount(skippedCount); + } + continue; + } + asinCount++; + String dataValue = normalizeDataValue(asin); + if (!seenInFile.add(dataValue)) { + skippedCount++; + if (progress != null) { + progress.setProcessedRows(rowNum); + progress.setAsinCount(asinCount); + progress.setInsertedCount(deletedCount); + progress.setSkippedCount(skippedCount); + } + continue; + } + + int deletedThisRow = deleteByDataValue(dataValue); + if (deletedThisRow > 0) { + deletedCount += deletedThisRow; + } else { + skippedCount++; + } + if (progress != null) { + progress.setProcessedRows(rowNum); + progress.setAsinCount(asinCount); + progress.setInsertedCount(deletedCount); + progress.setSkippedCount(skippedCount); + } + } + + DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo(); + vo.setTotalRows(totalRows); + vo.setAsinCount(asinCount); + vo.setInsertedCount(deletedCount); + vo.setSkippedCount(skippedCount); + return vo; + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + throw new BusinessException("删除 Excel 匹配数据失败"); + } + } + @Transactional public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) { DedupeTotalDataEntity entity = getById(id); @@ -301,6 +478,11 @@ public class DedupeTotalDataService { return dedupeTotalDataMapper.selectOne(query) != null; } + private int deleteByDataValue(String dataValue) { + return dedupeTotalDataMapper.delete(new LambdaQueryWrapper() + .eq(DedupeTotalDataEntity::getDataValue, dataValue)); + } + private String normalizeExcelText(String value) { if (value == null) { return ""; diff --git a/backend/__pycache__/config.cpython-312.pyc b/backend/__pycache__/config.cpython-312.pyc index 6e5d0ba..79187ce 100644 Binary files a/backend/__pycache__/config.cpython-312.pyc and b/backend/__pycache__/config.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc index 3fbdceb..4957fdd 100644 Binary files a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc and b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc differ diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index 3eec1d9..180ed78 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -679,6 +679,54 @@ def import_dedupe_total_data(): }) +@admin_api.route('/dedupe-total-data/delete-import/') +@admin_required +def dedupe_total_data_delete_import_progress(import_id): + result, error_response, status = _proxy_backend_java( + 'GET', + f'/api/admin/dedupe-total-data/delete-import/{import_id}', + ) + if error_response is not None: + return error_response, status + progress = result.get('data') or {} + return jsonify({ + 'success': True, + 'progress': { + 'status': progress.get('status') or 'pending', + 'total_rows': progress.get('totalRows') or 0, + 'processed_rows': progress.get('processedRows') or 0, + 'asin_count': progress.get('asinCount') or 0, + 'deleted_count': progress.get('insertedCount') or 0, + 'skipped_count': progress.get('skippedCount') or 0, + 'error_message': progress.get('errorMessage') or '', + }, + }) + + +@admin_api.route('/dedupe-total-data/delete-import', methods=['POST']) +@admin_required +def delete_import_dedupe_total_data(): + file_storage = request.files.get('file') + if not file_storage or file_storage.filename == '': + return jsonify({'success': False, 'error': '请选择 Excel 文件'}) + files = { + 'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream') + } + result, error_response, status = _proxy_backend_java( + 'POST', + '/api/admin/dedupe-total-data/delete-import', + files=files, + ) + if error_response is not None: + return error_response, status + summary = result.get('data') or {} + return jsonify({ + 'success': True, + 'msg': result.get('message') or '开始删除', + 'import_id': summary.get('importId') or '', + }) + + @admin_api.route('/dedupe-total-data', methods=['POST']) @admin_required def create_dedupe_total_data(): diff --git a/backend/config.py b/backend/config.py index 20612c2..6692838 100644 --- a/backend/config.py +++ b/backend/config.py @@ -23,6 +23,7 @@ bucket_path = "nanri-image/" file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/" import os +# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 0224ae3..55177a3 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -251,18 +251,37 @@
-

新增总数据

-
-
- - +
+
+

新增总数据

+
+
+ + +
+ +
+

+ +
+
+

删除匹配总数据

+
+
+ + +
+ +
+

+
- -
-

-
@@ -869,12 +888,19 @@ } document.getElementById('btnSearchDedupeTotalData').onclick = function() { loadDedupeTotalData(1); }; var dedupeImportPollTimer = null; + var dedupeDeleteImportPollTimer = null; function stopDedupeImportProgress() { if (dedupeImportPollTimer) { clearInterval(dedupeImportPollTimer); dedupeImportPollTimer = null; } } + function stopDedupeDeleteImportProgress() { + if (dedupeDeleteImportPollTimer) { + clearInterval(dedupeDeleteImportPollTimer); + dedupeDeleteImportPollTimer = null; + } + } function setDedupeImportProgress(percent, text) { var wrap = document.getElementById('dedupeTotalDataProgressWrap'); var fill = document.getElementById('dedupeTotalDataProgressFill'); @@ -883,6 +909,14 @@ fill.style.width = Math.max(0, Math.min(100, percent || 0)) + '%'; textEl.textContent = text || ''; } + function setDedupeDeleteImportProgress(percent, text) { + var wrap = document.getElementById('dedupeTotalDataDeleteProgressWrap'); + var fill = document.getElementById('dedupeTotalDataDeleteProgressFill'); + var textEl = document.getElementById('dedupeTotalDataDeleteProgressText'); + wrap.style.display = 'block'; + fill.style.width = Math.max(0, Math.min(100, percent || 0)) + '%'; + textEl.textContent = text || ''; + } function pollDedupeImport(importId) { stopDedupeImportProgress(); function tick() { @@ -920,6 +954,43 @@ tick(); dedupeImportPollTimer = setInterval(tick, 1000); } + function pollDedupeDeleteImport(importId) { + stopDedupeDeleteImportProgress(); + function tick() { + fetch('/api/admin/dedupe-total-data/delete-import/' + encodeURIComponent(importId)) + .then(function(r) { return r.json(); }) + .then(function(res) { + if (!res.success) { + stopDedupeDeleteImportProgress(); + document.getElementById('msgDeleteDedupeTotalData').textContent = res.error || '查询删除进度失败'; + document.getElementById('msgDeleteDedupeTotalData').className = 'msg err'; + return; + } + var progress = res.progress || {}; + var totalRows = progress.total_rows || 0; + var processedRows = progress.processed_rows || 0; + var percent = totalRows > 0 ? Math.round(processedRows * 100 / totalRows) : 0; + setDedupeDeleteImportProgress(percent, '处理中：已处理 ' + processedRows + ' / ' + totalRows + '，删除 ' + (progress.deleted_count || 0) + '，跳过 ' + (progress.skipped_count || 0)); + if (progress.status === 'success') { + stopDedupeDeleteImportProgress(); + document.getElementById('msgDeleteDedupeTotalData').textContent = '删除成功：总行数 ' + (progress.total_rows || 0) + '，ASIN 数量 ' + (progress.asin_count || 0) + '，删除 ' + (progress.deleted_count || 0) + '，跳过 ' + (progress.skipped_count || 0); + document.getElementById('msgDeleteDedupeTotalData').className = 'msg ok'; + loadDedupeTotalData(1); + } else if (progress.status === 'failed') { + stopDedupeDeleteImportProgress(); + document.getElementById('msgDeleteDedupeTotalData').textContent = progress.error_message || '删除失败'; + document.getElementById('msgDeleteDedupeTotalData').className = 'msg err'; + } + }) + .catch(function() { + stopDedupeDeleteImportProgress(); + document.getElementById('msgDeleteDedupeTotalData').textContent = '查询删除进度失败'; + document.getElementById('msgDeleteDedupeTotalData').className = 'msg err'; + }); + } + tick(); + dedupeDeleteImportPollTimer = setInterval(tick, 1000); + } document.getElementById('btnAddDedupeTotalData').onclick = function() { var fileInput = document.getElementById('dedupeTotalDataFile'); @@ -974,6 +1045,62 @@ }; xhr.send(formData); }; + document.getElementById('btnDeleteImportDedupeTotalData').onclick = function() { + var fileInput = document.getElementById('dedupeTotalDataDeleteFile'); + var msgEl = document.getElementById('msgDeleteDedupeTotalData'); + msgEl.textContent = ''; + msgEl.className = 'msg'; + stopDedupeDeleteImportProgress(); + document.getElementById('dedupeTotalDataDeleteProgressWrap').style.display = 'none'; + if (!fileInput.files || fileInput.files.length === 0) { + msgEl.textContent = '请选择 Excel 文件'; + msgEl.classList.add('err'); + return; + } + var file = fileInput.files[0]; + var lowerName = (file.name || '').toLowerCase(); + if (!(lowerName.endsWith('.xlsx') || lowerName.endsWith('.xls'))) { + msgEl.textContent = '仅支持 .xlsx 或 .xls 文件'; + msgEl.classList.add('err'); + return; + } + if (!confirm('确定按 Excel 中的 ASIN 批量删除匹配的总数据吗？')) { + return; + } + var formData = new FormData(); + formData.append('file', file); + var xhr = new XMLHttpRequest(); + xhr.open('POST', '/api/admin/dedupe-total-data/delete-import', true); + xhr.upload.onprogress = function(event) { + if (event.lengthComputable) { + var percent = Math.round(event.loaded * 100 / event.total); + setDedupeDeleteImportProgress(percent, '上传中：' + percent + '%'); + } + }; + xhr.onreadystatechange = function() { + if (xhr.readyState !== 4) return; + if (xhr.status < 200 || xhr.status >= 300) { + msgEl.textContent = '请求失败'; + msgEl.className = 'msg err'; + return; + } + var res; + try { res = JSON.parse(xhr.responseText || '{}'); } catch (e) { res = { success: false, error: '返回格式错误' }; } + if (!res.success) { + msgEl.textContent = res.error || '删除失败'; + msgEl.className = 'msg err'; + return; + } + setDedupeDeleteImportProgress(100, '上传完成，后端处理中...'); + pollDedupeDeleteImport(res.import_id); + fileInput.value = ''; + }; + xhr.onerror = function() { + msgEl.textContent = '请求失败'; + msgEl.className = 'msg err'; + }; + xhr.send(formData); + }; document.getElementById('btnSaveDedupeTotalData').onclick = function() { var itemId = document.getElementById('editDedupeTotalDataId').value; var value = (document.getElementById('editDedupeTotalDataValue').value || '').trim();