diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 938dac0d9..784f3b0d2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.util.WorkbookUtil; import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; @@ -50,6 +51,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.zip.ZipEntry; @@ -66,6 +68,12 @@ public class DedupeRunService { private static final String BRAND_COLUMN_HEADER = "品牌"; /** 同时处理的文件数上限,避免多文件并行时打满内存与 OSS 上传带宽 */ private static final int MAX_PARALLEL_FILES = 4; + /** + * 单文件等待处理槽位的默认上限:槽位只有 {@code MAX_PARALLEL_FILES} 个, + * 原先 {@code acquire()} 没有超时——一个文件卡在 OSS 上传/解析上,同批与后续 + * 任务的每个文件都会无限排在这里(任务不落终态,进度一直停在已提交的那格)。 + */ + private static final long DEFAULT_FILE_SLOT_WAIT_SECONDS = 600; /** 单用户同时运行中的去重任务数上限 */ private static final int MAX_RUNNING_TASKS_PER_USER = 2; /** 单次请求最多允许的源文件数,避免每个文件形成大量排队对象。 */ @@ -97,6 +105,9 @@ public class DedupeRunService { /** userId -> 运行中任务数,限制单用户并发任务 */ private final Map runningTaskCountMap = new ConcurrentHashMap<>(); private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES); + /** 等待处理槽位的上限秒数(默认见 {@code DEFAULT_FILE_SLOT_WAIT_SECONDS})。 */ + @Value("${aiimage.dedupe.run.file-slot-wait-seconds:600}") + private long fileSlotWaitSeconds = DEFAULT_FILE_SLOT_WAIT_SECONDS; /** 共享有界文件执行器;不再为请求中的每个文件创建虚拟线程。 */ // 队列改为有界:newFixedThreadPool 的无界队列在文件堆积时不会拒绝、会把内存吃满 private final ExecutorService fileExecutor = com.nanri.aiimage.common.support.ThreadPools @@ -478,13 +489,31 @@ public class DedupeRunService { fileTaskMapper.updateById(task); } + /** + * 等待一个文件处理槽位,最多等 {@code fileSlotWaitSeconds} 秒。 + * 返回 false 表示没排到——调用方必须把该文件判失败并写明原因,不许静默续等。 + */ + boolean tryAcquireFileSlot() throws InterruptedException { + return fileParallelSemaphore.tryAcquire(fileSlotWaitSeconds, TimeUnit.SECONDS); + } + private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request, boolean folderMode, FileTaskEntity task, Map archiveEntries, Long companyId) { DedupeResultItemVo item = new DedupeResultItemVo(); item.setSourceFilename(sourceFile.getOriginalFilename()); try { - fileParallelSemaphore.acquire(); + if (!tryAcquireFileSlot()) { + // 槽位被别的文件长期占着(卡在上游/上传未完成):本文件判失败并写明 + // 是没排到槽位,而不是装作处理过——静默排队会让整个任务不落终态。 + item.setSuccess(false); + item.setError("等待处理槽位超时(>" + fileSlotWaitSeconds + + "s,同批其它文件仍未处理完),本文件未处理,请稍后重试"); + log.warn("[dedupe] 等待文件处理槽位超时 file={} waitSeconds={} occupied={}", + sourceFile.getOriginalFilename(), fileSlotWaitSeconds, + MAX_PARALLEL_FILES - fileParallelSemaphore.availablePermits()); + return item; + } } catch (InterruptedException ex) { Thread.currentThread().interrupt(); item.setSuccess(false); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceFileSlotTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceFileSlotTest.java new file mode 100644 index 000000000..8c9c6352e --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceFileSlotTest.java @@ -0,0 +1,87 @@ +package com.nanri.aiimage.modules.dedupe.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.concurrent.Semaphore; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * 文件处理槽位的等待上限(回归:原先 {@code Semaphore.acquire()} 没有超时)。 + * + *

去重的文件并行度只有 4 个槽位。一个文件卡在上游读取或 OSS 上传时,同批与后续任务 + * 的每个文件都原先**无限**排在 {@code acquire()} 上:任务不落终态、进度冻在已提交的那一格, + * 用户只能干等。改造后等不到槽位就把该文件判失败并写明"是没排到槽位",等待上限可配置。 + */ +@ExtendWith(MockitoExtension.class) +class DedupeRunServiceFileSlotTest { + + @Mock + private FileTaskMapper fileTaskMapper; + @Mock + private FileResultMapper fileResultMapper; + @Mock + private StorageProperties storageProperties; + @Mock + private OssStorageService ossStorageService; + @Mock + private DedupeTotalDataService dedupeTotalDataService; + @Mock + private com.nanri.aiimage.common.service.CompanyScopeSupport companyScopeSupport; + @Mock + private StringRedisTemplate redisTemplate; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private DedupeRunService newService(long waitSeconds) { + DedupeRunService service = new DedupeRunService(fileTaskMapper, fileResultMapper, + storageProperties, ossStorageService, dedupeTotalDataService, companyScopeSupport, + redisTemplate, objectMapper, new InstanceMetadata("server-110")); + ReflectionTestUtils.setField(service, "fileSlotWaitSeconds", waitSeconds); + return service; + } + + private Semaphore slots(DedupeRunService service) { + Semaphore semaphore = (Semaphore) ReflectionTestUtils.getField(service, "fileParallelSemaphore"); + assertNotNull(semaphore, "文件并行槽位必须存在"); + return semaphore; + } + + @Test + @DisplayName("槽位被占满时等待必须到点返回 false,而不是无限排队") + void timesOutWhenNoSlotIsFree() throws Exception { + DedupeRunService service = newService(1L); + Semaphore slots = slots(service); + assertTrue(slots.drainPermits() > 0, "用例前提:全部槽位被其它文件占着"); + + assertFalse(service.tryAcquireFileSlot(), "槽位占满时必须超时返回,不能继续无界排队"); + assertEquals(0, slots.availablePermits(), "超时返回不得顺手占住一个槽位"); + } + + @Test + @DisplayName("有空槽位时照常拿到(超时不是无条件拒绝)") + void grantsSlotWhenOneIsFree() throws Exception { + DedupeRunService service = newService(1L); + Semaphore slots = slots(service); + slots.release(); + int before = slots.availablePermits(); + + assertTrue(service.tryAcquireFileSlot(), "有空槽位就必须放行"); + assertEquals(before - 1, slots.availablePermits(), "拿到槽位要真的占住一个"); + } +}