后台更新 服务优化

This commit is contained in:
supernijia
2026-08-11 14:50:21 +08:00
parent 641c70d451
commit 91cd02eeb4
24 changed files with 1623 additions and 502 deletions
@@ -38,6 +38,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -91,7 +92,7 @@ public class DedupeTotalDataController {
@GetMapping("/export")
@Operation(summary = "导出总数据", description = "按上传用户名和创建日期导出当前用户可访问的总数据。")
public ResponseEntity<byte[]> export(
public ResponseEntity<StreamingResponseBody> export(
@Parameter(description = "用户名模糊搜索关键字") @RequestParam(required = false) String username,
@Parameter(description = "开始日期(包含)")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
@@ -100,13 +101,13 @@ public class DedupeTotalDataController {
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
byte[] bytes = dedupeTotalDataService.export(username, startDate, endDate, groupId, operator.id());
String filename = "dedupe-total-data-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
StreamingResponseBody body = outputStream -> dedupeTotalDataService.writeExport(
outputStream, username, startDate, endDate, groupId, operator.id());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
.contentType(MediaType.parseMediaType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"))
.contentLength(bytes.length)
.body(bytes);
.body(body);
}
@PostMapping
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.dedupe.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
@@ -19,6 +20,23 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
List<String> selectExistingDataValues(@Param("values") List<String> values);
/**
* Import is deliberately a single SQL round trip per batch. The unique
* index remains the final concurrency guard when another request inserts
* the same value between the pre-check and this statement.
*/
@Insert("""
<script>
INSERT IGNORE INTO biz_dedupe_total_data
(data_value, group_id, uploader_user_id, uploader_username)
VALUES
<foreach collection="rows" item="row" separator=",">
(#{row.dataValue}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername})
</foreach>
</script>
""")
int insertBatchIgnore(@Param("rows") List<DedupeTotalDataEntity> rows);
class SqlProvider {
public String selectExistingDataValues(@Param("values") List<String> values) {
StringJoiner placeholders = new StringJoiner(", ");
@@ -36,10 +36,12 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -50,12 +52,15 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
@Service
@RequiredArgsConstructor
public class DedupeTotalDataService {
private static final int COMPARE_BATCH_SIZE = 5000;
private static final int IMPORT_BATCH_SIZE = 5000;
private static final int EXPORT_PAGE_SIZE = 2000;
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
private static final DateTimeFormatter EXPORT_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
@@ -130,21 +135,35 @@ public class DedupeTotalDataService {
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
.orderByDesc(DedupeTotalDataEntity::getId);
applyGroupScope(query, scope, groupId);
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(query);
return buildExportWorkbook(rows, loadGroupNames(rows));
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
return outputStream.toByteArray();
}
private byte[] buildExportWorkbook(List<DedupeTotalDataEntity> rows, Map<Long, String> groupNames) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
public void writeExport(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
Long operatorId) {
if (outputStream == null) {
throw new BusinessException("导出输出流不能为空");
}
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("invalid export date range");
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
}
private void buildExportWorkbook(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
AccessScope scope) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100)) {
Sheet sheet = workbook.createSheet("DedupeTotalData");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("ID");
@@ -152,16 +171,38 @@ public class DedupeTotalDataService {
header.createCell(2).setCellValue("用户名");
header.createCell(3).setCellValue("分组");
header.createCell(4).setCellValue("创建时间");
for (int index = 0; index < rows.size(); index++) {
DedupeTotalDataEntity entity = rows.get(index);
Row row = sheet.createRow(index + 1);
row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId()));
row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue());
row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
row.createCell(3).setCellValue(entity.getGroupId() == null
? ""
: groupNames.getOrDefault(entity.getGroupId(), ""));
row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt()));
Long lastId = null;
int rowIndex = 1;
while (true) {
LambdaQueryWrapper<DedupeTotalDataEntity> pageQuery = buildExportQuery(
username, startDate, endDate, groupId, scope);
if (lastId != null) {
pageQuery.lt(DedupeTotalDataEntity::getId, lastId);
}
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(
pageQuery.last("LIMIT " + EXPORT_PAGE_SIZE));
if (rows == null || rows.isEmpty()) {
break;
}
Map<Long, String> groupNames = loadGroupNames(rows);
for (DedupeTotalDataEntity entity : rows) {
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId()));
row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue());
row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
row.createCell(3).setCellValue(entity.getGroupId() == null
? ""
: groupNames.getOrDefault(entity.getGroupId(), ""));
row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt()));
}
DedupeTotalDataEntity lastRow = rows.getLast();
lastId = lastRow == null ? null : lastRow.getId();
if (lastId == null) {
break;
}
if (rows.size() < EXPORT_PAGE_SIZE) {
break;
}
}
sheet.setColumnWidth(0, 3600);
sheet.setColumnWidth(1, 5200);
@@ -170,12 +211,28 @@ public class DedupeTotalDataService {
sheet.setColumnWidth(4, 5600);
workbook.write(outputStream);
workbook.dispose();
return outputStream.toByteArray();
} catch (Exception ex) {
throw new BusinessException("导出数据去重总数据失败");
}
}
private LambdaQueryWrapper<DedupeTotalDataEntity> buildExportQuery(String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
AccessScope scope) {
String safeUsername = username == null ? "" : username.trim();
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
.orderByDesc(DedupeTotalDataEntity::getId);
applyGroupScope(query, scope, groupId);
return query;
}
private String formatExportTime(LocalDateTime value) {
return value == null ? "" : value.format(EXPORT_TIME_FORMATTER);
}
@@ -481,6 +538,7 @@ public class DedupeTotalDataService {
}
Set<String> seenInFile = new HashSet<>();
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
int totalRows = Math.max(sheet.getLastRowNum(), 0);
int asinCount = 0;
int insertedCount = 0;
@@ -498,77 +556,39 @@ public class DedupeTotalDataService {
Row row = sheet.getRow(rowNum);
if (row == null) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
continue;
}
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
continue;
}
asinCount++;
String dataValue = asin;
if (!seenInFile.add(dataValue)) {
if (!seenInFile.add(asin)) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
continue;
}
if (existsDataValue(dataValue)) {
skippedCount++;
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
continue;
}
final String pendingDataValue = dataValue;
Boolean inserted;
try {
inserted = newRequiresNewTemplate().execute(status -> {
if (existsDataValue(pendingDataValue)) {
return false;
}
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(pendingDataValue);
entity.setGroupId(groupId);
entity.setUploaderUserId(uploaderUserId);
entity.setUploaderUsername(uploaderUsername);
dedupeTotalDataMapper.insert(entity);
return true;
});
} catch (DuplicateKeyException ignored) {
inserted = false;
}
if (Boolean.TRUE.equals(inserted)) {
insertedCount++;
} else {
skippedCount++;
}
if (progress != null) {
progress.setProcessedRows(rowNum);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
pendingValues.add(asin);
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
ImportBatchResult batch = insertImportBatch(
pendingValues, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
}
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
}
if (!pendingValues.isEmpty()) {
ImportBatchResult batch = insertImportBatch(
pendingValues, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
}
updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount);
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
vo.setTotalRows(totalRows);
@@ -583,6 +603,81 @@ public class DedupeTotalDataService {
}
}
private ImportBatchResult insertImportBatch(List<String> values,
Long groupId,
Long uploaderUserId,
String uploaderUsername) {
if (values == null || values.isEmpty()) {
return new ImportBatchResult(0, 0);
}
Set<String> existingValues = findExistingComparableValues(values);
List<DedupeTotalDataEntity> rows = new ArrayList<>(values.size());
for (String value : values) {
if (value == null || value.isBlank() || existingValues.contains(value)) {
continue;
}
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(value);
entity.setGroupId(groupId);
entity.setUploaderUserId(uploaderUserId);
entity.setUploaderUsername(uploaderUsername);
rows.add(entity);
}
if (rows.isEmpty()) {
return new ImportBatchResult(0, values.size());
}
int inserted;
try {
inserted = executeInNewTransaction(() -> dedupeTotalDataMapper.insertBatchIgnore(rows));
} catch (DuplicateKeyException ex) {
// INSERT IGNORE is the normal path. Keep a correctness-preserving
// fallback for deployments whose SQL mode/driver still surfaces a
// duplicate-key error for a multi-row statement.
inserted = 0;
for (DedupeTotalDataEntity row : rows) {
try {
Boolean oneInserted = executeInNewTransaction(() -> {
if (existsDataValue(row.getDataValue())) {
return false;
}
return dedupeTotalDataMapper.insert(row) > 0;
});
if (Boolean.TRUE.equals(oneInserted)) {
inserted++;
}
} catch (DuplicateKeyException ignored) {
// Another importer won the unique-key race.
}
}
}
return new ImportBatchResult(inserted, values.size() - inserted);
}
private void updateImportProgress(DedupeTotalDataImportProgressVo progress,
int processedRows,
int asinCount,
int insertedCount,
int skippedCount) {
if (progress == null) {
return;
}
progress.setProcessedRows(processedRows);
progress.setAsinCount(asinCount);
progress.setInsertedCount(insertedCount);
progress.setSkippedCount(skippedCount);
}
private <T> T executeInNewTransaction(Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
return newRequiresNewTemplate().execute(status -> action.get());
}
private record ImportBatchResult(int insertedCount, int skippedCount) {
}
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
DedupeTotalDataImportProgressVo progress,
AccessScope scope, Long groupId) {