fix(backend-java): 收口并发资源与恢复链路

This commit is contained in:
2026-09-05 14:33:07 +08:00
parent 5a00201931
commit 0bf39b04db
35 changed files with 917 additions and 154 deletions
@@ -5,6 +5,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
import java.util.List;
@@ -15,6 +16,7 @@ public class DedupeRunRequest {
@Valid
@NotEmpty(message = "请先上传待处理文件")
@Size(max = 200, message = "单次最多提交 200 个文件")
@Schema(description = "已上传源文件列表")
private List<DedupeSourceFileDto> files;
@@ -17,11 +17,13 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -35,6 +37,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.ZipEntry;
@@ -52,6 +57,8 @@ public class DedupeRunService {
private static final int MAX_PARALLEL_FILES = 4;
/** 单用户同时运行中的去重任务数上限 */
private static final int MAX_RUNNING_TASKS_PER_USER = 2;
/** 单次请求最多允许的源文件数,避免每个文件形成大量排队对象。 */
private static final int MAX_INPUT_FILES = 200;
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
private final FileTaskMapper fileTaskMapper;
@@ -67,21 +74,35 @@ public class DedupeRunService {
/** userId -> 运行中任务数,限制单用户并发任务 */
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
/** 共享有界文件执行器;不再为请求中的每个文件创建虚拟线程。 */
private final ExecutorService fileExecutor = Executors.newFixedThreadPool(
MAX_PARALLEL_FILES, runnable -> {
Thread thread = new Thread(runnable, "dedupe-file-worker");
thread.setDaemon(true);
return thread;
});
/**
* 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。
*/
public DedupeRunProgressVo submitRun(DedupeRunRequest request) {
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("用户 ID 不合法");
}
if (request.getFiles() == null || request.getFiles().isEmpty()) {
throw new BusinessException("请先上传待处理文件");
}
if (request.getFiles().size() > MAX_INPUT_FILES) {
throw new BusinessException("单次最多提交 " + MAX_INPUT_FILES + " 个文件");
}
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
throw new BusinessException("请至少选择一种 ID 保留规则");
}
cleanupExpiredProgress();
AtomicInteger runningCount = runningTaskCountMap.computeIfAbsent(request.getUserId(), k -> new AtomicInteger(0));
if (runningCount.get() >= MAX_RUNNING_TASKS_PER_USER) {
if (!tryAcquireRunningTaskSlot(request.getUserId())) {
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
}
runningCount.incrementAndGet();
String runId = IdUtil.fastSimpleUUID();
FileTaskEntity task = new FileTaskEntity();
@@ -95,7 +116,13 @@ public class DedupeRunService {
task.setRequestJson(JSONUtil.toJsonStr(request));
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
try {
fileTaskMapper.insert(task);
} catch (RuntimeException ex) {
// 任务落库失败时释放已经预留的用户并发名额,避免后续请求被永久拒绝。
decrementRunningCount(request.getUserId());
throw ex;
}
DedupeRunProgressVo progress = new DedupeRunProgressVo();
progress.setRunId(runId);
@@ -147,9 +174,9 @@ public class DedupeRunService {
try {
AtomicInteger processedCount = new AtomicInteger(0);
List<Thread> workers = new ArrayList<>(request.getFiles().size());
List<Future<?>> workers = new ArrayList<>(request.getFiles().size());
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
Thread worker = Thread.ofVirtual().start(() -> {
workers.add(fileExecutor.submit(() -> {
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries);
synchronized (progress) {
progress.setProcessedCount(processedCount.incrementAndGet());
@@ -163,11 +190,10 @@ public class DedupeRunService {
outcomeItems.add(item);
}
}
});
workers.add(worker);
}));
}
for (Thread worker : workers) {
worker.join();
for (Future<?> worker : workers) {
worker.get();
}
} catch (Exception ex) {
log.error("dedupe run async aborted runId={} error", runId, ex);
@@ -821,6 +847,19 @@ public class DedupeRunService {
}
}
private boolean tryAcquireRunningTaskSlot(Long userId) {
AtomicInteger runningCount = runningTaskCountMap.computeIfAbsent(userId, ignored -> new AtomicInteger());
while (true) {
int current = runningCount.get();
if (current >= MAX_RUNNING_TASKS_PER_USER) {
return false;
}
if (runningCount.compareAndSet(current, current + 1)) {
return true;
}
}
}
private void decrementRunningCount(Long userId) {
AtomicInteger runningCount = runningTaskCountMap.get(userId);
if (runningCount != null && runningCount.decrementAndGet() <= 0) {
@@ -828,6 +867,16 @@ public class DedupeRunService {
}
}
@PreDestroy
void shutdownFileExecutor() {
fileExecutor.shutdownNow();
}
@Scheduled(fixedDelayString = "${aiimage.dedupe.run-progress-cleanup-delay-ms:300000}")
public void cleanupExpiredProgressScheduled() {
cleanupExpiredProgress();
}
private void cleanupExpiredProgress() {
long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS;
runCompletedAtMap.forEach((runId, completedAt) -> {
@@ -23,6 +23,7 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
@@ -53,6 +54,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.function.Supplier;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
@@ -81,6 +83,15 @@ public class DedupeTotalDataService {
private final Map<String, Long> importCompletedAtMap = new ConcurrentHashMap<>();
private final Map<String, Long> deleteImportCompletedAtMap = new ConcurrentHashMap<>();
/** 导入任务并发上限,避免 POI 解析和批量数据库写入叠加。 */
private final Semaphore importSlots = new Semaphore(4);
@Value("${aiimage.dedupe.total-data.max-import-file-bytes:104857600}")
private long maxImportFileBytes = 100L * 1024 * 1024;
@Value("${aiimage.dedupe.total-data.max-import-rows:500000}")
private int maxImportRows = 500_000;
private TransactionTemplate newRequiresNewTemplate() {
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
@@ -411,8 +422,14 @@ public class DedupeTotalDataService {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
if (maxImportFileBytes > 0 && file.getSize() > maxImportFileBytes) {
throw new BusinessException("导入文件超过大小上限");
}
AdminUserEntity uploader = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
if (!importSlots.tryAcquire()) {
throw new BusinessException("导入任务过多,请稍后重试");
}
String importId = IdUtil.fastSimpleUUID();
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("pending");
@@ -431,6 +448,7 @@ public class DedupeTotalDataService {
Thread.ofVirtual().start(() -> runImportTask(
importId, tempFile, filename, uploader.getId(), uploader.getUsername(), group.getId()));
} catch (Exception e) {
importSlots.release();
importProgressMap.remove(importId);
importOwnerMap.remove(importId);
importGroupMap.remove(importId);
@@ -463,8 +481,14 @@ public class DedupeTotalDataService {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
if (maxImportFileBytes > 0 && file.getSize() > maxImportFileBytes) {
throw new BusinessException("导入文件超过大小上限");
}
AdminUserEntity operator = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
if (!importSlots.tryAcquire()) {
throw new BusinessException("导入任务过多,请稍后重试");
}
String importId = IdUtil.fastSimpleUUID();
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
progress.setStatus("pending");
@@ -483,6 +507,7 @@ public class DedupeTotalDataService {
Thread.ofVirtual().start(() -> runDeleteImportTask(
importId, tempFile, filename, operator.getId(), group.getId()));
} catch (Exception e) {
importSlots.release();
deleteImportProgressMap.remove(importId);
deleteImportOwnerMap.remove(importId);
deleteImportGroupMap.remove(importId);
@@ -510,6 +535,7 @@ public class DedupeTotalDataService {
Long operatorId, Long groupId) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
importSlots.release();
return;
}
progress.setStatus("running");
@@ -529,6 +555,7 @@ public class DedupeTotalDataService {
} finally {
deleteQuietly(tempFile);
deleteImportCompletedAtMap.put(importId, System.currentTimeMillis());
importSlots.release();
}
}
@@ -536,6 +563,7 @@ public class DedupeTotalDataService {
Long uploaderUserId, String uploaderUsername, Long groupId) {
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
if (progress == null) {
importSlots.release();
return;
}
progress.setStatus("running");
@@ -554,6 +582,7 @@ public class DedupeTotalDataService {
} finally {
deleteQuietly(tempFile);
importCompletedAtMap.put(importId, System.currentTimeMillis());
importSlots.release();
}
}
@@ -659,6 +688,9 @@ public class DedupeTotalDataService {
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
Map<String, String> pendingCountries = new HashMap<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
if (maxImportRows > 0 && totalRows > maxImportRows) {
throw new BusinessException("导入行数超过上限: " + maxImportRows);
}
int asinCount = 0;
int insertedCount = 0;
int skippedCount = 0;
@@ -836,6 +868,9 @@ public class DedupeTotalDataService {
Set<String> seenInFile = new HashSet<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
if (maxImportRows > 0 && totalRows > maxImportRows) {
throw new BusinessException("导入行数超过上限: " + maxImportRows);
}
int asinCount = 0;
int deletedCount = 0;
int skippedCount = 0;