后台更新 服务优化
This commit is contained in:
+5
-4
@@ -38,6 +38,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -91,7 +92,7 @@ public class DedupeTotalDataController {
|
||||
|
||||
@GetMapping("/export")
|
||||
@Operation(summary = "导出总数据", description = "按上传用户名和创建日期导出当前用户可访问的总数据。")
|
||||
public ResponseEntity<byte[]> export(
|
||||
public ResponseEntity<StreamingResponseBody> export(
|
||||
@Parameter(description = "用户名模糊搜索关键字") @RequestParam(required = false) String username,
|
||||
@Parameter(description = "开始日期(包含)")
|
||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
|
||||
@@ -100,13 +101,13 @@ public class DedupeTotalDataController {
|
||||
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
|
||||
HttpServletRequest request) {
|
||||
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
||||
byte[] bytes = dedupeTotalDataService.export(username, startDate, endDate, groupId, operator.id());
|
||||
String filename = "dedupe-total-data-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
|
||||
StreamingResponseBody body = outputStream -> dedupeTotalDataService.writeExport(
|
||||
outputStream, username, startDate, endDate, groupId, operator.id());
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
|
||||
.contentType(MediaType.parseMediaType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"))
|
||||
.contentLength(bytes.length)
|
||||
.body(bytes);
|
||||
.body(body);
|
||||
}
|
||||
|
||||
@PostMapping
|
||||
|
||||
+18
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.dedupe.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
@@ -19,6 +20,23 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
|
||||
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
|
||||
List<String> selectExistingDataValues(@Param("values") List<String> values);
|
||||
|
||||
/**
|
||||
* Import is deliberately a single SQL round trip per batch. The unique
|
||||
* index remains the final concurrency guard when another request inserts
|
||||
* the same value between the pre-check and this statement.
|
||||
*/
|
||||
@Insert("""
|
||||
<script>
|
||||
INSERT IGNORE INTO biz_dedupe_total_data
|
||||
(data_value, group_id, uploader_user_id, uploader_username)
|
||||
VALUES
|
||||
<foreach collection="rows" item="row" separator=",">
|
||||
(#{row.dataValue}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername})
|
||||
</foreach>
|
||||
</script>
|
||||
""")
|
||||
int insertBatchIgnore(@Param("rows") List<DedupeTotalDataEntity> rows);
|
||||
|
||||
class SqlProvider {
|
||||
public String selectExistingDataValues(@Param("values") List<String> values) {
|
||||
StringJoiner placeholders = new StringJoiner(", ");
|
||||
|
||||
+177
-82
@@ -36,10 +36,12 @@ import org.springframework.web.server.ResponseStatusException;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -50,12 +52,15 @@ import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DedupeTotalDataService {
|
||||
|
||||
private static final int COMPARE_BATCH_SIZE = 5000;
|
||||
private static final int IMPORT_BATCH_SIZE = 5000;
|
||||
private static final int EXPORT_PAGE_SIZE = 2000;
|
||||
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
|
||||
private static final DateTimeFormatter EXPORT_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
@@ -130,21 +135,35 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
String safeUsername = username == null ? "" : username.trim();
|
||||
AccessScope scope = resolveAccessScope(operatorId);
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
|
||||
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
|
||||
startDate == null ? null : startDate.atStartOfDay())
|
||||
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
|
||||
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
|
||||
.orderByDesc(DedupeTotalDataEntity::getId);
|
||||
applyGroupScope(query, scope, groupId);
|
||||
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(query);
|
||||
return buildExportWorkbook(rows, loadGroupNames(rows));
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
|
||||
private byte[] buildExportWorkbook(List<DedupeTotalDataEntity> rows, Map<Long, String> groupNames) {
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100);
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
|
||||
public void writeExport(OutputStream outputStream,
|
||||
String username,
|
||||
LocalDate startDate,
|
||||
LocalDate endDate,
|
||||
Long groupId,
|
||||
Long operatorId) {
|
||||
if (outputStream == null) {
|
||||
throw new BusinessException("导出输出流不能为空");
|
||||
}
|
||||
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
|
||||
throw new BusinessException("invalid export date range");
|
||||
}
|
||||
String safeUsername = username == null ? "" : username.trim();
|
||||
AccessScope scope = resolveAccessScope(operatorId);
|
||||
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
|
||||
}
|
||||
|
||||
private void buildExportWorkbook(OutputStream outputStream,
|
||||
String username,
|
||||
LocalDate startDate,
|
||||
LocalDate endDate,
|
||||
Long groupId,
|
||||
AccessScope scope) {
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100)) {
|
||||
Sheet sheet = workbook.createSheet("DedupeTotalData");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("ID");
|
||||
@@ -152,16 +171,38 @@ public class DedupeTotalDataService {
|
||||
header.createCell(2).setCellValue("用户名");
|
||||
header.createCell(3).setCellValue("分组");
|
||||
header.createCell(4).setCellValue("创建时间");
|
||||
for (int index = 0; index < rows.size(); index++) {
|
||||
DedupeTotalDataEntity entity = rows.get(index);
|
||||
Row row = sheet.createRow(index + 1);
|
||||
row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId()));
|
||||
row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue());
|
||||
row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
|
||||
row.createCell(3).setCellValue(entity.getGroupId() == null
|
||||
? ""
|
||||
: groupNames.getOrDefault(entity.getGroupId(), ""));
|
||||
row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt()));
|
||||
Long lastId = null;
|
||||
int rowIndex = 1;
|
||||
while (true) {
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> pageQuery = buildExportQuery(
|
||||
username, startDate, endDate, groupId, scope);
|
||||
if (lastId != null) {
|
||||
pageQuery.lt(DedupeTotalDataEntity::getId, lastId);
|
||||
}
|
||||
List<DedupeTotalDataEntity> rows = dedupeTotalDataMapper.selectList(
|
||||
pageQuery.last("LIMIT " + EXPORT_PAGE_SIZE));
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
Map<Long, String> groupNames = loadGroupNames(rows);
|
||||
for (DedupeTotalDataEntity entity : rows) {
|
||||
Row row = sheet.createRow(rowIndex++);
|
||||
row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId()));
|
||||
row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue());
|
||||
row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
|
||||
row.createCell(3).setCellValue(entity.getGroupId() == null
|
||||
? ""
|
||||
: groupNames.getOrDefault(entity.getGroupId(), ""));
|
||||
row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt()));
|
||||
}
|
||||
DedupeTotalDataEntity lastRow = rows.getLast();
|
||||
lastId = lastRow == null ? null : lastRow.getId();
|
||||
if (lastId == null) {
|
||||
break;
|
||||
}
|
||||
if (rows.size() < EXPORT_PAGE_SIZE) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sheet.setColumnWidth(0, 3600);
|
||||
sheet.setColumnWidth(1, 5200);
|
||||
@@ -170,12 +211,28 @@ public class DedupeTotalDataService {
|
||||
sheet.setColumnWidth(4, 5600);
|
||||
workbook.write(outputStream);
|
||||
workbook.dispose();
|
||||
return outputStream.toByteArray();
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("导出数据去重总数据失败");
|
||||
}
|
||||
}
|
||||
|
||||
private LambdaQueryWrapper<DedupeTotalDataEntity> buildExportQuery(String username,
|
||||
LocalDate startDate,
|
||||
LocalDate endDate,
|
||||
Long groupId,
|
||||
AccessScope scope) {
|
||||
String safeUsername = username == null ? "" : username.trim();
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
|
||||
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
|
||||
startDate == null ? null : startDate.atStartOfDay())
|
||||
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
|
||||
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
|
||||
.orderByDesc(DedupeTotalDataEntity::getId);
|
||||
applyGroupScope(query, scope, groupId);
|
||||
return query;
|
||||
}
|
||||
|
||||
private String formatExportTime(LocalDateTime value) {
|
||||
return value == null ? "" : value.format(EXPORT_TIME_FORMATTER);
|
||||
}
|
||||
@@ -481,6 +538,7 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
|
||||
Set<String> seenInFile = new HashSet<>();
|
||||
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
int asinCount = 0;
|
||||
int insertedCount = 0;
|
||||
@@ -498,77 +556,39 @@ public class DedupeTotalDataService {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(insertedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
}
|
||||
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
|
||||
if (asin.isBlank()) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(insertedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
}
|
||||
asinCount++;
|
||||
String dataValue = asin;
|
||||
if (!seenInFile.add(dataValue)) {
|
||||
if (!seenInFile.add(asin)) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(insertedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
}
|
||||
if (existsDataValue(dataValue)) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(insertedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
final String pendingDataValue = dataValue;
|
||||
Boolean inserted;
|
||||
try {
|
||||
inserted = newRequiresNewTemplate().execute(status -> {
|
||||
if (existsDataValue(pendingDataValue)) {
|
||||
return false;
|
||||
}
|
||||
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
|
||||
entity.setDataValue(pendingDataValue);
|
||||
entity.setGroupId(groupId);
|
||||
entity.setUploaderUserId(uploaderUserId);
|
||||
entity.setUploaderUsername(uploaderUsername);
|
||||
dedupeTotalDataMapper.insert(entity);
|
||||
return true;
|
||||
});
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
inserted = false;
|
||||
}
|
||||
if (Boolean.TRUE.equals(inserted)) {
|
||||
insertedCount++;
|
||||
} else {
|
||||
skippedCount++;
|
||||
}
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(insertedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
pendingValues.add(asin);
|
||||
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
}
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
}
|
||||
if (!pendingValues.isEmpty()) {
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
}
|
||||
updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount);
|
||||
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(totalRows);
|
||||
@@ -583,6 +603,81 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
}
|
||||
|
||||
private ImportBatchResult insertImportBatch(List<String> values,
|
||||
Long groupId,
|
||||
Long uploaderUserId,
|
||||
String uploaderUsername) {
|
||||
if (values == null || values.isEmpty()) {
|
||||
return new ImportBatchResult(0, 0);
|
||||
}
|
||||
Set<String> existingValues = findExistingComparableValues(values);
|
||||
List<DedupeTotalDataEntity> rows = new ArrayList<>(values.size());
|
||||
for (String value : values) {
|
||||
if (value == null || value.isBlank() || existingValues.contains(value)) {
|
||||
continue;
|
||||
}
|
||||
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
|
||||
entity.setDataValue(value);
|
||||
entity.setGroupId(groupId);
|
||||
entity.setUploaderUserId(uploaderUserId);
|
||||
entity.setUploaderUsername(uploaderUsername);
|
||||
rows.add(entity);
|
||||
}
|
||||
if (rows.isEmpty()) {
|
||||
return new ImportBatchResult(0, values.size());
|
||||
}
|
||||
|
||||
int inserted;
|
||||
try {
|
||||
inserted = executeInNewTransaction(() -> dedupeTotalDataMapper.insertBatchIgnore(rows));
|
||||
} catch (DuplicateKeyException ex) {
|
||||
// INSERT IGNORE is the normal path. Keep a correctness-preserving
|
||||
// fallback for deployments whose SQL mode/driver still surfaces a
|
||||
// duplicate-key error for a multi-row statement.
|
||||
inserted = 0;
|
||||
for (DedupeTotalDataEntity row : rows) {
|
||||
try {
|
||||
Boolean oneInserted = executeInNewTransaction(() -> {
|
||||
if (existsDataValue(row.getDataValue())) {
|
||||
return false;
|
||||
}
|
||||
return dedupeTotalDataMapper.insert(row) > 0;
|
||||
});
|
||||
if (Boolean.TRUE.equals(oneInserted)) {
|
||||
inserted++;
|
||||
}
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
// Another importer won the unique-key race.
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ImportBatchResult(inserted, values.size() - inserted);
|
||||
}
|
||||
|
||||
private void updateImportProgress(DedupeTotalDataImportProgressVo progress,
|
||||
int processedRows,
|
||||
int asinCount,
|
||||
int insertedCount,
|
||||
int skippedCount) {
|
||||
if (progress == null) {
|
||||
return;
|
||||
}
|
||||
progress.setProcessedRows(processedRows);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(insertedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
|
||||
private <T> T executeInNewTransaction(Supplier<T> action) {
|
||||
if (transactionManager == null) {
|
||||
return action.get();
|
||||
}
|
||||
return newRequiresNewTemplate().execute(status -> action.get());
|
||||
}
|
||||
|
||||
private record ImportBatchResult(int insertedCount, int skippedCount) {
|
||||
}
|
||||
|
||||
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
|
||||
DedupeTotalDataImportProgressVo progress,
|
||||
AccessScope scope, Long groupId) {
|
||||
|
||||
Reference in New Issue
Block a user