fix(backend-java): 收口并发资源与恢复链路
This commit is contained in:
+2
@@ -5,6 +5,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
@@ -15,6 +16,7 @@ public class DedupeRunRequest {
|
||||
|
||||
@Valid
|
||||
@NotEmpty(message = "请先上传待处理文件")
|
||||
@Size(max = 200, message = "单次最多提交 200 个文件")
|
||||
@Schema(description = "已上传源文件列表")
|
||||
private List<DedupeSourceFileDto> files;
|
||||
|
||||
|
||||
+59
-10
@@ -17,11 +17,13 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.util.WorkbookUtil;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
@@ -35,6 +37,9 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.zip.ZipEntry;
|
||||
@@ -52,6 +57,8 @@ public class DedupeRunService {
|
||||
private static final int MAX_PARALLEL_FILES = 4;
|
||||
/** 单用户同时运行中的去重任务数上限 */
|
||||
private static final int MAX_RUNNING_TASKS_PER_USER = 2;
|
||||
/** 单次请求最多允许的源文件数,避免每个文件形成大量排队对象。 */
|
||||
private static final int MAX_INPUT_FILES = 200;
|
||||
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
@@ -67,21 +74,35 @@ public class DedupeRunService {
|
||||
/** userId -> 运行中任务数,限制单用户并发任务 */
|
||||
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
|
||||
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
|
||||
/** 共享有界文件执行器;不再为请求中的每个文件创建虚拟线程。 */
|
||||
private final ExecutorService fileExecutor = Executors.newFixedThreadPool(
|
||||
MAX_PARALLEL_FILES, runnable -> {
|
||||
Thread thread = new Thread(runnable, "dedupe-file-worker");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
/**
|
||||
* 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。
|
||||
*/
|
||||
public DedupeRunProgressVo submitRun(DedupeRunRequest request) {
|
||||
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
|
||||
throw new BusinessException("用户 ID 不合法");
|
||||
}
|
||||
if (request.getFiles() == null || request.getFiles().isEmpty()) {
|
||||
throw new BusinessException("请先上传待处理文件");
|
||||
}
|
||||
if (request.getFiles().size() > MAX_INPUT_FILES) {
|
||||
throw new BusinessException("单次最多提交 " + MAX_INPUT_FILES + " 个文件");
|
||||
}
|
||||
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
|
||||
throw new BusinessException("请至少选择一种 ID 保留规则");
|
||||
}
|
||||
cleanupExpiredProgress();
|
||||
|
||||
AtomicInteger runningCount = runningTaskCountMap.computeIfAbsent(request.getUserId(), k -> new AtomicInteger(0));
|
||||
if (runningCount.get() >= MAX_RUNNING_TASKS_PER_USER) {
|
||||
if (!tryAcquireRunningTaskSlot(request.getUserId())) {
|
||||
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
|
||||
}
|
||||
runningCount.incrementAndGet();
|
||||
|
||||
String runId = IdUtil.fastSimpleUUID();
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
@@ -95,7 +116,13 @@ public class DedupeRunService {
|
||||
task.setRequestJson(JSONUtil.toJsonStr(request));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
try {
|
||||
fileTaskMapper.insert(task);
|
||||
} catch (RuntimeException ex) {
|
||||
// 任务落库失败时释放已经预留的用户并发名额,避免后续请求被永久拒绝。
|
||||
decrementRunningCount(request.getUserId());
|
||||
throw ex;
|
||||
}
|
||||
|
||||
DedupeRunProgressVo progress = new DedupeRunProgressVo();
|
||||
progress.setRunId(runId);
|
||||
@@ -147,9 +174,9 @@ public class DedupeRunService {
|
||||
|
||||
try {
|
||||
AtomicInteger processedCount = new AtomicInteger(0);
|
||||
List<Thread> workers = new ArrayList<>(request.getFiles().size());
|
||||
List<Future<?>> workers = new ArrayList<>(request.getFiles().size());
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
Thread worker = Thread.ofVirtual().start(() -> {
|
||||
workers.add(fileExecutor.submit(() -> {
|
||||
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries);
|
||||
synchronized (progress) {
|
||||
progress.setProcessedCount(processedCount.incrementAndGet());
|
||||
@@ -163,11 +190,10 @@ public class DedupeRunService {
|
||||
outcomeItems.add(item);
|
||||
}
|
||||
}
|
||||
});
|
||||
workers.add(worker);
|
||||
}));
|
||||
}
|
||||
for (Thread worker : workers) {
|
||||
worker.join();
|
||||
for (Future<?> worker : workers) {
|
||||
worker.get();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.error("dedupe run async aborted runId={} error", runId, ex);
|
||||
@@ -821,6 +847,19 @@ public class DedupeRunService {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean tryAcquireRunningTaskSlot(Long userId) {
|
||||
AtomicInteger runningCount = runningTaskCountMap.computeIfAbsent(userId, ignored -> new AtomicInteger());
|
||||
while (true) {
|
||||
int current = runningCount.get();
|
||||
if (current >= MAX_RUNNING_TASKS_PER_USER) {
|
||||
return false;
|
||||
}
|
||||
if (runningCount.compareAndSet(current, current + 1)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void decrementRunningCount(Long userId) {
|
||||
AtomicInteger runningCount = runningTaskCountMap.get(userId);
|
||||
if (runningCount != null && runningCount.decrementAndGet() <= 0) {
|
||||
@@ -828,6 +867,16 @@ public class DedupeRunService {
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void shutdownFileExecutor() {
|
||||
fileExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.dedupe.run-progress-cleanup-delay-ms:300000}")
|
||||
public void cleanupExpiredProgressScheduled() {
|
||||
cleanupExpiredProgress();
|
||||
}
|
||||
|
||||
private void cleanupExpiredProgress() {
|
||||
long cutoff = System.currentTimeMillis() - COMPLETED_PROGRESS_RETENTION_MILLIS;
|
||||
runCompletedAtMap.forEach((runId, completedAt) -> {
|
||||
|
||||
+35
@@ -23,6 +23,7 @@ import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
@@ -53,6 +54,7 @@ import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.zip.Deflater;
|
||||
import java.util.zip.ZipEntry;
|
||||
@@ -81,6 +83,15 @@ public class DedupeTotalDataService {
|
||||
private final Map<String, Long> importCompletedAtMap = new ConcurrentHashMap<>();
|
||||
private final Map<String, Long> deleteImportCompletedAtMap = new ConcurrentHashMap<>();
|
||||
|
||||
/** 导入任务并发上限,避免 POI 解析和批量数据库写入叠加。 */
|
||||
private final Semaphore importSlots = new Semaphore(4);
|
||||
|
||||
@Value("${aiimage.dedupe.total-data.max-import-file-bytes:104857600}")
|
||||
private long maxImportFileBytes = 100L * 1024 * 1024;
|
||||
|
||||
@Value("${aiimage.dedupe.total-data.max-import-rows:500000}")
|
||||
private int maxImportRows = 500_000;
|
||||
|
||||
private TransactionTemplate newRequiresNewTemplate() {
|
||||
TransactionTemplate template = new TransactionTemplate(transactionManager);
|
||||
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
|
||||
@@ -411,8 +422,14 @@ public class DedupeTotalDataService {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
}
|
||||
if (maxImportFileBytes > 0 && file.getSize() > maxImportFileBytes) {
|
||||
throw new BusinessException("导入文件超过大小上限");
|
||||
}
|
||||
AdminUserEntity uploader = getOperator(operatorId);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
|
||||
if (!importSlots.tryAcquire()) {
|
||||
throw new BusinessException("导入任务过多,请稍后重试");
|
||||
}
|
||||
String importId = IdUtil.fastSimpleUUID();
|
||||
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
|
||||
progress.setStatus("pending");
|
||||
@@ -431,6 +448,7 @@ public class DedupeTotalDataService {
|
||||
Thread.ofVirtual().start(() -> runImportTask(
|
||||
importId, tempFile, filename, uploader.getId(), uploader.getUsername(), group.getId()));
|
||||
} catch (Exception e) {
|
||||
importSlots.release();
|
||||
importProgressMap.remove(importId);
|
||||
importOwnerMap.remove(importId);
|
||||
importGroupMap.remove(importId);
|
||||
@@ -463,8 +481,14 @@ public class DedupeTotalDataService {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
}
|
||||
if (maxImportFileBytes > 0 && file.getSize() > maxImportFileBytes) {
|
||||
throw new BusinessException("导入文件超过大小上限");
|
||||
}
|
||||
AdminUserEntity operator = getOperator(operatorId);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
|
||||
if (!importSlots.tryAcquire()) {
|
||||
throw new BusinessException("导入任务过多,请稍后重试");
|
||||
}
|
||||
String importId = IdUtil.fastSimpleUUID();
|
||||
DedupeTotalDataImportProgressVo progress = new DedupeTotalDataImportProgressVo();
|
||||
progress.setStatus("pending");
|
||||
@@ -483,6 +507,7 @@ public class DedupeTotalDataService {
|
||||
Thread.ofVirtual().start(() -> runDeleteImportTask(
|
||||
importId, tempFile, filename, operator.getId(), group.getId()));
|
||||
} catch (Exception e) {
|
||||
importSlots.release();
|
||||
deleteImportProgressMap.remove(importId);
|
||||
deleteImportOwnerMap.remove(importId);
|
||||
deleteImportGroupMap.remove(importId);
|
||||
@@ -510,6 +535,7 @@ public class DedupeTotalDataService {
|
||||
Long operatorId, Long groupId) {
|
||||
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
|
||||
if (progress == null) {
|
||||
importSlots.release();
|
||||
return;
|
||||
}
|
||||
progress.setStatus("running");
|
||||
@@ -529,6 +555,7 @@ public class DedupeTotalDataService {
|
||||
} finally {
|
||||
deleteQuietly(tempFile);
|
||||
deleteImportCompletedAtMap.put(importId, System.currentTimeMillis());
|
||||
importSlots.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -536,6 +563,7 @@ public class DedupeTotalDataService {
|
||||
Long uploaderUserId, String uploaderUsername, Long groupId) {
|
||||
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
|
||||
if (progress == null) {
|
||||
importSlots.release();
|
||||
return;
|
||||
}
|
||||
progress.setStatus("running");
|
||||
@@ -554,6 +582,7 @@ public class DedupeTotalDataService {
|
||||
} finally {
|
||||
deleteQuietly(tempFile);
|
||||
importCompletedAtMap.put(importId, System.currentTimeMillis());
|
||||
importSlots.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -659,6 +688,9 @@ public class DedupeTotalDataService {
|
||||
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
|
||||
Map<String, String> pendingCountries = new HashMap<>();
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
if (maxImportRows > 0 && totalRows > maxImportRows) {
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
int asinCount = 0;
|
||||
int insertedCount = 0;
|
||||
int skippedCount = 0;
|
||||
@@ -836,6 +868,9 @@ public class DedupeTotalDataService {
|
||||
|
||||
Set<String> seenInFile = new HashSet<>();
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
if (maxImportRows > 0 && totalRows > maxImportRows) {
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
int asinCount = 0;
|
||||
int deletedCount = 0;
|
||||
int skippedCount = 0;
|
||||
|
||||
Reference in New Issue
Block a user