super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成后台管理开发

Browse Source
This commit is contained in:
super
2026-03-22 13:33:37 +08:00
parent 923dfd2495
commit 903c591181
8 changed files with 303 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java
View File

@@ -3,6 +3,8 @@ package com.nanri.aiimage.modules.dedupe.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportProgressVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportStartVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
@@ -59,12 +61,18 @@ public class DedupeTotalDataController {
@PostMapping("/import")
@Operation(summary = "导入总数据", description = "上传 xlsx 文件，读取 ASIN 列并批量导入数据去重总数据。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "导入成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataImportVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "启动导入成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataImportStartVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "文件不合法或缺少 ASIN 列")
})
public ApiResponse<DedupeTotalDataImportVo> importExcel(
public ApiResponse<DedupeTotalDataImportStartVo> importExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file) {
return ApiResponse.success("导入成功", dedupeTotalDataService.importFromExcel(file));
return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file));
}
@GetMapping("/import/{importId}")
@Operation(summary = "查询导入进度", description = "根据导入任务 ID 查询当前进度。")
public ApiResponse<DedupeTotalDataImportProgressVo> importProgress(@PathVariable String importId) {
return ApiResponse.success(dedupeTotalDataService.getImportProgress(importId));
}
@PutMapping("/{id}")

30
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataImportProgressVo.java Normal file
View File

@@ -0,0 +1,30 @@
package com.nanri.aiimage.modules.dedupe.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "数据去重总数据导入进度")
public class DedupeTotalDataImportProgressVo {
@Schema(description = "任务状态：pending/running/success/failed")
private String status;
@Schema(description = "总行数")
private Integer totalRows;
@Schema(description = "已处理行数")
private Integer processedRows;
@Schema(description = "读取到的 ASIN 数量")
private Integer asinCount;
@Schema(description = "实际新增数量")
private Integer insertedCount;
@Schema(description = "跳过数量")
private Integer skippedCount;
@Schema(description = "错误信息")
private String errorMessage;
}

12
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataImportStartVo.java Normal file
View File

@@ -0,0 +1,12 @@
package com.nanri.aiimage.modules.dedupe.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "数据去重总数据导入启动结果")
public class DedupeTotalDataImportStartVo {
@Schema(description = "导入任务ID")
private String importId;
}

118
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
View File

@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.dedupe.service;
import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
@@ -7,6 +8,8 @@ import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportProgressVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportStartVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
@@ -20,18 +23,21 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
public class DedupeTotalDataService {
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword) {
long safePage = Math.max(page, 1);
@@ -63,19 +69,85 @@ public class DedupeTotalDataService {
return toItemVo(getById(entity.getId()));
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
String importId = IdUtil.fastSimpleUUID();
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("pending");
progress.setTotalRows(0);
progress.setProcessedRows(0);
progress.setAsinCount(0);
progress.setInsertedCount(0);
progress.setSkippedCount(0);
importProgressMap.put(importId, progress);
try {
byte[] fileBytes = file.getBytes();
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runImportTask(importId, fileBytes, filename));
} catch (Exception e) {
importProgressMap.remove(importId);
throw new BusinessException("读取上传文件失败");
}
DedupeTotalDataImportStartVo vo = new DedupeTotalDataImportStartVo();
vo.setImportId(importId);
return vo;
}
public DedupeTotalDataImportProgressVo getImportProgress(String importId) {
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
if (progress == null) {
throw new BusinessException("导入任务不存在");
}
return progress;
}
private void runImportTask(String importId, byte[] fileBytes, String filename) {
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
if (progress == null) {
return;
}
progress.setStatus("running");
try (InputStream inputStream = new ByteArrayInputStream(fileBytes)) {
DedupeTotalDataImportVo result = importFromExcelInternal(inputStream, filename, progress);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
progress.setSkippedCount(result.getSkippedCount());
progress.setProcessedRows(result.getTotalRows());
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "导入 Excel 失败");
}
}
@Transactional
public DedupeTotalDataImportVo importFromExcel(MultipartFile file) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
String filename = file.getOriginalFilename() == null ? "" : file.getOriginalFilename().toLowerCase();
if (!(filename.endsWith(".xlsx") || filename.endsWith(".xls"))) {
try (InputStream inputStream = file.getInputStream()) {
return importFromExcelInternal(inputStream, file.getOriginalFilename(), null);
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("导入 Excel 失败");
}
}
@Transactional
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename, DedupeTotalDataImportProgressVo progress) {
String lowerFilename = filename == null ? "" : filename.toLowerCase();
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
}
DataFormatter formatter = new DataFormatter();
try (InputStream inputStream = file.getInputStream();
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
@@ -101,32 +173,70 @@ public class DedupeTotalDataService {
int asinCount = 0;
int insertedCount = 0;
int skippedCount = 0;
if (progress != null) {
progress.setTotalRows(totalRows);
progress.setProcessedRows(0);
progress.setAsinCount(0);
progress.setInsertedCount(0);
progress.setSkippedCount(0);
progress.setErrorMessage(null);
}
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
asinCount++;
String dataValue = normalizeDataValue(asin);
if (!seenInFile.add(dataValue)) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
if (existsDataValue(dataValue)) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(dataValue);
dedupeTotalDataMapper.insert(entity);
insertedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
}
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();