fix(backend-java): 收口并发资源与恢复链路

This commit is contained in:
2026-09-05 14:33:07 +08:00
parent 5a00201931
commit 0bf39b04db
35 changed files with 917 additions and 154 deletions
@@ -125,8 +125,10 @@ public class ModuleHistoryCleanupService {
int totalBatches = 0;
int totalDeletedTasks = 0;
int totalCollectedPointers = 0;
List<Long> skippedActiveTaskIds = new ArrayList<>();
List<Long> skippedRetainedTaskIds = new ArrayList<>();
int skippedActiveTaskCount = 0;
int skippedRetainedTaskCount = 0;
List<Long> skippedActiveTaskSamples = new ArrayList<>(LOG_SAMPLE_IDS);
List<Long> skippedRetainedTaskSamples = new ArrayList<>(LOG_SAMPLE_IDS);
while (true) {
List<FileTaskEntity> page = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.in(FileTaskEntity::getModuleType, moduleTypes)
@@ -145,7 +147,8 @@ public class ModuleHistoryCleanupService {
}
pageMaxId = Math.max(pageMaxId, task.getId());
if (!isTerminalStatus(task.getStatus())) {
skippedActiveTaskIds.add(task.getId());
skippedActiveTaskCount++;
addSample(skippedActiveTaskSamples, task.getId());
continue;
}
if (isExpired(task, cutoff)) {
@@ -154,7 +157,8 @@ public class ModuleHistoryCleanupService {
batchCollectDataTaskIds.add(task.getId());
}
} else {
skippedRetainedTaskIds.add(task.getId());
skippedRetainedTaskCount++;
addSample(skippedRetainedTaskSamples, task.getId());
}
}
if (!batchTaskIds.isEmpty()) {
@@ -182,8 +186,8 @@ public class ModuleHistoryCleanupService {
log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, batches={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}",
moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff,
totalBatches, totalDeletedTasks, totalCollectedPointers,
formatIdSample(skippedActiveTaskIds, LOG_SAMPLE_IDS),
formatIdSample(skippedRetainedTaskIds, LOG_SAMPLE_IDS));
formatIdSample(skippedActiveTaskSamples, skippedActiveTaskCount, LOG_SAMPLE_IDS),
formatIdSample(skippedRetainedTaskSamples, skippedRetainedTaskCount, LOG_SAMPLE_IDS));
}
}
@@ -288,25 +292,39 @@ public class ModuleHistoryCleanupService {
}
}
private static void addSample(List<Long> samples, Long id) {
if (samples != null && id != null && samples.size() < LOG_SAMPLE_IDS) {
samples.add(id);
}
}
/**
* 日志用 ID 摘要:只输出数量与最多 {@code sampleLimit} 个样本,
* 禁止把超长任务 ID 列表写进日志。非法上限回退到 1,null/空输出 count=0。
*/
static String formatIdSample(List<Long> ids, int sampleLimit) {
int count = ids == null ? 0 : ids.size();
if (count == 0) {
return formatIdSample(ids, count, sampleLimit);
}
private static String formatIdSample(List<Long> samples, int count, int sampleLimit) {
if (count <= 0) {
return "count=0";
}
int limit = Math.max(1, sampleLimit);
StringBuilder sb = new StringBuilder("count=").append(count).append(", sample=[");
for (int i = 0; i < Math.min(count, limit); i++) {
int sampleCount = samples == null ? 0 : Math.min(samples.size(), limit);
for (int i = 0; i < sampleCount; i++) {
if (i > 0) {
sb.append(',');
}
sb.append(ids.get(i));
sb.append(samples.get(i));
}
if (count > limit) {
sb.append(",...");
if (count > sampleCount) {
if (sampleCount > 0) {
sb.append(',');
}
sb.append("...");
}
return sb.append(']').toString();
}
@@ -26,7 +26,8 @@ public class TaskDistributedLockService {
private final DistributedJobLockService distributedJobLockService;
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
private final ScheduledExecutorService renewalExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
/** 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。 */
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(4, runnable -> {
Thread thread = new Thread(runnable, "task-lock-renewal");
thread.setDaemon(true);
return thread;
@@ -37,6 +37,8 @@ public class TaskFileJobLocalDispatcher {
private int maxInflightDispatch = 64;
private final Set<Long> inflightJobIds = ConcurrentHashMap.newKeySet();
/** 保护“容量检查 + 加入集合”这一复合操作,避免并发提交突破 inflight 上限。 */
private final Object inflightMonitor = new Object();
public boolean dispatch(Long jobId, Long taskId, String moduleType) {
return dispatch(jobId, taskId, moduleType, false);
@@ -64,12 +66,21 @@ public class TaskFileJobLocalDispatcher {
return false;
}
int inflightLimit = Math.max(1, maxInflightDispatch);
if (!inflightJobIds.contains(jobId) && inflightJobIds.size() >= inflightLimit) {
log.warn("[task-file-job] local dispatch backpressure, inflight limit reached jobId={} taskId={} moduleType={} inflight={} limit={}",
jobId, taskId, moduleType, inflightJobIds.size(), inflightLimit);
return false;
boolean alreadyInflight;
int inflightSize;
synchronized (inflightMonitor) {
alreadyInflight = inflightJobIds.contains(jobId);
inflightSize = inflightJobIds.size();
if (!alreadyInflight && inflightSize >= inflightLimit) {
log.warn("[task-file-job] local dispatch backpressure, inflight limit reached jobId={} taskId={} moduleType={} inflight={} limit={}",
jobId, taskId, moduleType, inflightSize, inflightLimit);
return false;
}
if (!alreadyInflight) {
inflightJobIds.add(jobId);
}
}
if (!inflightJobIds.add(jobId)) {
if (alreadyInflight) {
log.info("[task-file-job] local dispatch skipped, job already inflight jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return true;
@@ -78,7 +89,7 @@ public class TaskFileJobLocalDispatcher {
taskFileJobDispatchExecutor.execute(() -> processLocally(jobId, taskId, moduleType, alreadyClaimed));
return true;
} catch (RuntimeException ex) {
inflightJobIds.remove(jobId);
removeInflight(jobId);
log.warn("[task-file-job] local dispatch executor rejected jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
if (force) {
@@ -86,13 +97,22 @@ public class TaskFileJobLocalDispatcher {
processLocally(jobId, taskId, moduleType, alreadyClaimed);
return true;
} finally {
inflightJobIds.remove(jobId);
removeInflight(jobId);
}
}
return false;
}
}
private void removeInflight(Long jobId) {
if (jobId == null) {
return;
}
synchronized (inflightMonitor) {
inflightJobIds.remove(jobId);
}
}
private void processLocally(Long jobId, Long taskId, String moduleType, boolean alreadyClaimed) {
try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
@@ -116,7 +136,7 @@ public class TaskFileJobLocalDispatcher {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
} finally {
inflightJobIds.remove(jobId);
removeInflight(jobId);
}
}
}
@@ -53,7 +53,8 @@ public class TaskResultFileJobWorker {
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
private long heartbeatIntervalMillis = 60000L;
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
/** 多个结果文件 Job 并行运行时,心跳不能被单个 DB 慢调用串行阻塞。 */
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newScheduledThreadPool(4, runnable -> {
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
thread.setDaemon(true);
return thread;
@@ -16,6 +16,7 @@ import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.util.Comparator;
import java.util.PriorityQueue;
import java.util.List;
@Service
@@ -25,6 +26,8 @@ public class TaskScopePayloadBufferMaintenanceService {
private static final Duration RECOVERY_LOCK_TTL = Duration.ofMinutes(10);
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
/** 每轮最多扫描/处理的文件数,避免异常堆积导致单次清理占满内存和 CPU。 */
private static final int MAX_CLEANUP_FILES_PER_RUN = 5_000;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskPressureProperties taskPressureProperties;
@@ -85,7 +88,7 @@ public class TaskScopePayloadBufferMaintenanceService {
}
Instant cutoff = Instant.now().minus(Duration.ofHours(Math.max(1L, taskPressureProperties.getScopePayloadBufferRetentionHours())));
int deleted = 0;
for (Path file : listBufferedPayloadFiles(root, Integer.MAX_VALUE)) {
for (Path file : listBufferedPayloadFiles(root, MAX_CLEANUP_FILES_PER_RUN)) {
try {
FileTime lastModified = Files.getLastModifiedTime(file);
if (lastModified.toInstant().isAfter(cutoff)) {
@@ -105,13 +108,22 @@ public class TaskScopePayloadBufferMaintenanceService {
}
private List<Path> listBufferedPayloadFiles(Path root, int limit) {
int safeLimit = Math.max(1, Math.min(limit, MAX_CLEANUP_FILES_PER_RUN));
Comparator<Path> oldestFirst = Comparator.comparing(this::safeLastModified);
// sorted().limit() 仍会把整个目录加载进排序缓冲;这里仅保留最老的 safeLimit 个文件。
PriorityQueue<Path> newestFirst = new PriorityQueue<>(safeLimit, oldestFirst.reversed());
try (var walk = Files.walk(root, 4)) {
return walk
.filter(Files::isRegularFile)
walk.filter(Files::isRegularFile)
.filter(path -> path.getFileName().toString().endsWith(".json"))
.sorted(Comparator.comparing(this::safeLastModified))
.limit(limit)
.toList();
.forEach(path -> {
if (newestFirst.size() < safeLimit) {
newestFirst.offer(path);
} else if (oldestFirst.compare(path, newestFirst.peek()) < 0) {
newestFirst.poll();
newestFirst.offer(path);
}
});
return newestFirst.stream().sorted(oldestFirst).toList();
} catch (IOException ex) {
log.warn("[task-scope-buffer] scan failed root={} msg={}", root, ex.getMessage());
return List.of();
@@ -14,7 +14,9 @@ import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.LocalDateTime;
@@ -215,7 +217,10 @@ public class TaskScopePayloadStorageService {
return count == null ? 0L : count;
}
@Transactional
/**
* 先在短事务内删除/清空数据库引用,提交后再执行远程对象删除。
* 不能把 RustFS/OSS 网络调用放在数据库事务中,否则大任务会长时间占用连接。
*/
public void deleteTaskScopePayloads(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
@@ -226,18 +231,33 @@ public class TaskScopePayloadStorageService {
if (states == null || states.isEmpty()) {
return;
}
for (TaskScopeStateEntity state : states) {
if (state == null || state.getId() == null) {
continue;
List<String> payloadsToDelete = states.stream()
.filter(state -> state != null && !isBlank(state.getStateJson()))
.map(TaskScopeStateEntity::getStateJson)
.toList();
inTransaction(() -> {
for (TaskScopeStateEntity state : states) {
if (state == null || state.getId() == null) {
continue;
}
if (!isBlank(state.getParsedPayloadJson())) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, null)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
} else {
taskScopeStateMapper.deleteById(state.getId());
}
}
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
if (!isBlank(state.getParsedPayloadJson())) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, null)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
} else {
taskScopeStateMapper.deleteById(state.getId());
return null;
});
// DB 引用已提交后再做引用检查和物理删除,失败由后续清理任务兜底。
for (String payload : payloadsToDelete) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (RuntimeException ex) {
log.warn("[task-scope-payload] delete physical payload failed taskId={} moduleType={} msg={}",
taskId, moduleType, ex.getMessage());
}
}
}
@@ -328,9 +348,74 @@ public class TaskScopePayloadStorageService {
return result;
}
@Transactional
/**
* 恢复旧版本在本地 task-scope-buffer 中留下的载荷。文件读取和对象存储上传
* 放在事务外,只有最终的短 UPDATE 使用事务,避免启动恢复时长时间占用 DB 连接。
*/
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
return false;
if (taskId == null || taskId <= 0 || isBlank(moduleType) || !isValidScopeHash(scopeHash)) {
return false;
}
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
if (payloadJson == null || payloadJson.isBlank()) {
return false;
}
TaskScopeStateEntity existing = getScopeState(taskId, moduleType, scopeHash);
if (existing == null) {
// 没有对应 DB 行的孤儿缓冲文件无法恢复,及时删除避免反复扫描。
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
String storedPayload = transientPayloadStorageService.storeScopePayloadVersioned(
moduleType, taskId, scopeHash, payloadJson, true);
String replacedPayload;
try {
replacedPayload = inTransaction(() -> persistRecoveredScopePayload(
taskId, moduleType, scopeHash, storedPayload));
} catch (RuntimeException ex) {
// 数据库写入失败时不要删除新对象,后续恢复任务仍可重试;
// 若存储服务支持引用检查,尽力清理未被引用的对象。
try {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
} catch (RuntimeException cleanupEx) {
log.warn("[task-scope-buffer] failed to cleanup uploaded recovery payload taskId={} moduleType={} scopeHash={} msg={}",
taskId, moduleType, scopeHash, cleanupEx.getMessage());
}
log.warn("[task-scope-buffer] persist recovered payload failed taskId={} moduleType={} scopeHash={} msg={}",
taskId, moduleType, scopeHash, ex.getMessage());
return false;
}
try {
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(replacedPayload, storedPayload);
} catch (RuntimeException cleanupEx) {
// 数据库已恢复成功;旧对象删除失败交给后续清理任务,不应让恢复再次上传。
log.warn("[task-scope-buffer] failed to cleanup replaced payload taskId={} moduleType={} scopeHash={} msg={}",
taskId, moduleType, scopeHash, cleanupEx.getMessage());
} finally {
deleteBufferedPayload(taskId, moduleType, scopeHash);
}
return true;
}
private String persistRecoveredScopePayload(Long taskId, String moduleType, String scopeHash, String storedPayload) {
TaskScopeStateEntity state = getScopeStateForUpdate(taskId, moduleType, scopeHash);
if (state == null) {
throw new BusinessException("恢复任务范围载荷失败:任务范围不存在");
}
String replacedPayload = state.getStateJson();
LocalDateTime now = LocalDateTime.now();
int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, storedPayload)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
if (updated <= 0) {
throw new BusinessException("恢复任务范围载荷失败:数据库未更新");
}
return replacedPayload;
}
private <T> T inTransaction(java.util.function.Supplier<T> action) {
@@ -341,6 +426,50 @@ public class TaskScopePayloadStorageService {
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR);
}
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
if (!Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)) {
return null;
}
return Files.readString(path, StandardCharsets.UTF_8);
} catch (IOException | RuntimeException ex) {
log.warn("[task-scope-buffer] read buffered payload failed taskId={} moduleType={} scopeHash={} msg={}",
taskId, moduleType, scopeHash, ex.getMessage());
return null;
}
}
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Files.deleteIfExists(buildBufferedPayloadPath(taskId, moduleType, scopeHash));
} catch (IOException | RuntimeException ex) {
log.warn("[task-scope-buffer] delete buffered payload failed taskId={} moduleType={} scopeHash={} msg={}",
taskId, moduleType, scopeHash, ex.getMessage());
}
}
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || !isValidScopeHash(scopeHash)) {
throw new IllegalArgumentException("invalid buffered payload identity");
}
String normalizedModule = moduleType.trim().toLowerCase(java.util.Locale.ROOT);
if (!normalizedModule.matches("[a-z0-9_-]+")) {
throw new IllegalArgumentException("invalid buffered payload module");
}
Path root = getBufferedPayloadRoot().toAbsolutePath().normalize();
Path target = root.resolve(normalizedModule).resolve(String.valueOf(taskId))
.resolve(scopeHash.toLowerCase(java.util.Locale.ROOT) + ".json").normalize();
if (!target.startsWith(root) || target.equals(root)) {
throw new IllegalArgumentException("buffered payload path escapes root");
}
return target;
}
private boolean isValidScopeHash(String scopeHash) {
return scopeHash != null && scopeHash.matches("[0-9a-fA-F]{64}");
}
private TaskScopeStateEntity getScopeState(Long taskId, String moduleType, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
@@ -8,6 +8,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -18,6 +19,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.function.Supplier;
/**
@@ -52,7 +54,7 @@ public class TransientPayloadDeleteOrchestrator {
TaskChunkMapper taskChunkMapper,
TaskScopeStateMapper taskScopeStateMapper,
ObjectMapper objectMapper,
ExecutorService asyncDeleteExecutor) {
@Qualifier("transientPayloadDeleteExecutor") ExecutorService asyncDeleteExecutor) {
this.transientPayloadStorageService = transientPayloadStorageService;
this.rustfsObjectStorageService = rustfsObjectStorageService;
this.taskChunkMapper = taskChunkMapper;
@@ -76,7 +78,7 @@ public class TransientPayloadDeleteOrchestrator {
if (pointer == null) {
continue;
}
if (pendingPointers.size() >= maxPendingDeletes) {
if (pendingPointers.size() >= pendingLimit()) {
log.warn("[transient-payload] delete queue full, drop {} pointer={}", "submit", pointer);
break;
}
@@ -106,7 +108,14 @@ public class TransientPayloadDeleteOrchestrator {
List<String> toDelete = new ArrayList<>(batch);
toDelete.removeAll(stillReferenced);
if (!toDelete.isEmpty()) {
asyncDeleteExecutor.submit(() -> deleteObjects(toDelete));
try {
asyncDeleteExecutor.submit(() -> deleteObjects(toDelete));
} catch (RejectedExecutionException ex) {
restorePending(toDelete);
log.warn("[transient-payload] delete executor saturated, restored pending count={} msg={}",
toDelete.size(), ex.getMessage());
return 0;
}
}
if (!stillReferenced.isEmpty()) {
log.info("[transient-payload] skip delete, still referenced count={}", stillReferenced.size());
@@ -115,13 +124,31 @@ public class TransientPayloadDeleteOrchestrator {
} catch (Exception ex) {
log.warn("[transient-payload] batch reference check failed, keep pending count={} err={}",
batch.size(), ex.getMessage());
synchronized (pendingPointers) {
pendingPointers.addAll(batch);
}
restorePending(batch);
return 0;
}
}
private int pendingLimit() {
return (int) Math.max(1L, Math.min(maxPendingDeletes, Integer.MAX_VALUE));
}
private void restorePending(List<String> pointers) {
if (pointers == null || pointers.isEmpty()) {
return;
}
synchronized (pendingPointers) {
int limit = pendingLimit();
for (String pointer : pointers) {
if (pendingPointers.size() >= limit) {
log.warn("[transient-payload] pending delete queue full while restoring dropped pointer");
break;
}
pendingPointers.add(pointer);
}
}
}
public int pendingCount() {
synchronized (pendingPointers) {
return pendingPointers.size();