fix(dedupe): 文件处理槽位等待加上限,等不到判失败并写明原因

原先 Semaphore.acquire() 无超时:一个文件卡在上游读取或 OSS 上传时,
同批与后续任务的每个文件都无限排在这 4 个槽位上——任务不落终态、
进度冻在已提交那格。改为 tryAcquire(默认 600s,可配
aiimage.dedupe.run.file-slot-wait-seconds),等不到把该文件判失败
并写明「是没排到槽位」,不再是静默续等。新增 2 例互为对照:
槽位占满必须到点返回 false 且不占槽位;有空槽位照常放行。
This commit is contained in:
2026-10-23 18:00:17 +08:00
parent d9cb194575
commit 70b3be63a9
2 changed files with 117 additions and 1 deletions
@@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
@@ -50,6 +51,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.zip.ZipEntry;
@@ -66,6 +68,12 @@ public class DedupeRunService {
private static final String BRAND_COLUMN_HEADER = "品牌";
/** 同时处理的文件数上限,避免多文件并行时打满内存与 OSS 上传带宽 */
private static final int MAX_PARALLEL_FILES = 4;
/**
* 单文件等待处理槽位的默认上限:槽位只有 {@code MAX_PARALLEL_FILES} 个,
* 原先 {@code acquire()} 没有超时——一个文件卡在 OSS 上传/解析上,同批与后续
* 任务的每个文件都会无限排在这里(任务不落终态,进度一直停在已提交的那格)。
*/
private static final long DEFAULT_FILE_SLOT_WAIT_SECONDS = 600;
/** 单用户同时运行中的去重任务数上限 */
private static final int MAX_RUNNING_TASKS_PER_USER = 2;
/** 单次请求最多允许的源文件数,避免每个文件形成大量排队对象。 */
@@ -97,6 +105,9 @@ public class DedupeRunService {
/** userId -> 运行中任务数,限制单用户并发任务 */
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
/** 等待处理槽位的上限秒数(默认见 {@code DEFAULT_FILE_SLOT_WAIT_SECONDS})。 */
@Value("${aiimage.dedupe.run.file-slot-wait-seconds:600}")
private long fileSlotWaitSeconds = DEFAULT_FILE_SLOT_WAIT_SECONDS;
/** 共享有界文件执行器;不再为请求中的每个文件创建虚拟线程。 */
// 队列改为有界:newFixedThreadPool 的无界队列在文件堆积时不会拒绝、会把内存吃满
private final ExecutorService fileExecutor = com.nanri.aiimage.common.support.ThreadPools
@@ -478,13 +489,31 @@ public class DedupeRunService {
fileTaskMapper.updateById(task);
}
/**
* 等待一个文件处理槽位,最多等 {@code fileSlotWaitSeconds} 秒。
* 返回 false 表示没排到——调用方必须把该文件判失败并写明原因,不许静默续等。
*/
boolean tryAcquireFileSlot() throws InterruptedException {
return fileParallelSemaphore.tryAcquire(fileSlotWaitSeconds, TimeUnit.SECONDS);
}
private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request,
boolean folderMode, FileTaskEntity task,
Map<String, DedupeArchiveEntry> archiveEntries, Long companyId) {
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
try {
fileParallelSemaphore.acquire();
if (!tryAcquireFileSlot()) {
// 槽位被别的文件长期占着(卡在上游/上传未完成):本文件判失败并写明
// 是没排到槽位,而不是装作处理过——静默排队会让整个任务不落终态。
item.setSuccess(false);
item.setError("等待处理槽位超时(>" + fileSlotWaitSeconds
+ "s,同批其它文件仍未处理完),本文件未处理,请稍后重试");
log.warn("[dedupe] 等待文件处理槽位超时 file={} waitSeconds={} occupied={}",
sourceFile.getOriginalFilename(), fileSlotWaitSeconds,
MAX_PARALLEL_FILES - fileParallelSemaphore.availablePermits());
return item;
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
item.setSuccess(false);
@@ -0,0 +1,87 @@
package com.nanri.aiimage.modules.dedupe.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.concurrent.Semaphore;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* 文件处理槽位的等待上限(回归:原先 {@code Semaphore.acquire()} 没有超时)。
*
* <p>去重的文件并行度只有 4 个槽位。一个文件卡在上游读取或 OSS 上传时,同批与后续任务
* 的每个文件都原先**无限**排在 {@code acquire()} 上:任务不落终态、进度冻在已提交的那一格,
* 用户只能干等。改造后等不到槽位就把该文件判失败并写明"是没排到槽位",等待上限可配置。
*/
@ExtendWith(MockitoExtension.class)
class DedupeRunServiceFileSlotTest {
@Mock
private FileTaskMapper fileTaskMapper;
@Mock
private FileResultMapper fileResultMapper;
@Mock
private StorageProperties storageProperties;
@Mock
private OssStorageService ossStorageService;
@Mock
private DedupeTotalDataService dedupeTotalDataService;
@Mock
private com.nanri.aiimage.common.service.CompanyScopeSupport companyScopeSupport;
@Mock
private StringRedisTemplate redisTemplate;
private final ObjectMapper objectMapper = new ObjectMapper();
private DedupeRunService newService(long waitSeconds) {
DedupeRunService service = new DedupeRunService(fileTaskMapper, fileResultMapper,
storageProperties, ossStorageService, dedupeTotalDataService, companyScopeSupport,
redisTemplate, objectMapper, new InstanceMetadata("server-110"));
ReflectionTestUtils.setField(service, "fileSlotWaitSeconds", waitSeconds);
return service;
}
private Semaphore slots(DedupeRunService service) {
Semaphore semaphore = (Semaphore) ReflectionTestUtils.getField(service, "fileParallelSemaphore");
assertNotNull(semaphore, "文件并行槽位必须存在");
return semaphore;
}
@Test
@DisplayName("槽位被占满时等待必须到点返回 false,而不是无限排队")
void timesOutWhenNoSlotIsFree() throws Exception {
DedupeRunService service = newService(1L);
Semaphore slots = slots(service);
assertTrue(slots.drainPermits() > 0, "用例前提:全部槽位被其它文件占着");
assertFalse(service.tryAcquireFileSlot(), "槽位占满时必须超时返回,不能继续无界排队");
assertEquals(0, slots.availablePermits(), "超时返回不得顺手占住一个槽位");
}
@Test
@DisplayName("有空槽位时照常拿到(超时不是无条件拒绝)")
void grantsSlotWhenOneIsFree() throws Exception {
DedupeRunService service = newService(1L);
Semaphore slots = slots(service);
slots.release();
int before = slots.availablePermits();
assertTrue(service.tryAcquireFileSlot(), "有空槽位就必须放行");
assertEquals(before - 1, slots.availablePermits(), "拿到槽位要真的占住一个");
}
}