fix(backend-java): 收口并发资源与恢复链路
This commit is contained in:
+29
-11
@@ -125,8 +125,10 @@ public class ModuleHistoryCleanupService {
|
||||
int totalBatches = 0;
|
||||
int totalDeletedTasks = 0;
|
||||
int totalCollectedPointers = 0;
|
||||
List<Long> skippedActiveTaskIds = new ArrayList<>();
|
||||
List<Long> skippedRetainedTaskIds = new ArrayList<>();
|
||||
int skippedActiveTaskCount = 0;
|
||||
int skippedRetainedTaskCount = 0;
|
||||
List<Long> skippedActiveTaskSamples = new ArrayList<>(LOG_SAMPLE_IDS);
|
||||
List<Long> skippedRetainedTaskSamples = new ArrayList<>(LOG_SAMPLE_IDS);
|
||||
while (true) {
|
||||
List<FileTaskEntity> page = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getModuleType, moduleTypes)
|
||||
@@ -145,7 +147,8 @@ public class ModuleHistoryCleanupService {
|
||||
}
|
||||
pageMaxId = Math.max(pageMaxId, task.getId());
|
||||
if (!isTerminalStatus(task.getStatus())) {
|
||||
skippedActiveTaskIds.add(task.getId());
|
||||
skippedActiveTaskCount++;
|
||||
addSample(skippedActiveTaskSamples, task.getId());
|
||||
continue;
|
||||
}
|
||||
if (isExpired(task, cutoff)) {
|
||||
@@ -154,7 +157,8 @@ public class ModuleHistoryCleanupService {
|
||||
batchCollectDataTaskIds.add(task.getId());
|
||||
}
|
||||
} else {
|
||||
skippedRetainedTaskIds.add(task.getId());
|
||||
skippedRetainedTaskCount++;
|
||||
addSample(skippedRetainedTaskSamples, task.getId());
|
||||
}
|
||||
}
|
||||
if (!batchTaskIds.isEmpty()) {
|
||||
@@ -182,8 +186,8 @@ public class ModuleHistoryCleanupService {
|
||||
log.info("[module-cleanup] completed: moduleTypes={}, retentionDays={}, cutoff={}, batches={}, deletedTasks={}, collectedPayloadPointers={}, skippedActiveTaskIds={}, retainedTaskIds={}",
|
||||
moduleTypes, moduleCleanupProperties.getRetentionDays(), cutoff,
|
||||
totalBatches, totalDeletedTasks, totalCollectedPointers,
|
||||
formatIdSample(skippedActiveTaskIds, LOG_SAMPLE_IDS),
|
||||
formatIdSample(skippedRetainedTaskIds, LOG_SAMPLE_IDS));
|
||||
formatIdSample(skippedActiveTaskSamples, skippedActiveTaskCount, LOG_SAMPLE_IDS),
|
||||
formatIdSample(skippedRetainedTaskSamples, skippedRetainedTaskCount, LOG_SAMPLE_IDS));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,25 +292,39 @@ public class ModuleHistoryCleanupService {
|
||||
}
|
||||
}
|
||||
|
||||
private static void addSample(List<Long> samples, Long id) {
|
||||
if (samples != null && id != null && samples.size() < LOG_SAMPLE_IDS) {
|
||||
samples.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 日志用 ID 摘要:只输出数量与最多 {@code sampleLimit} 个样本,
|
||||
* 禁止把超长任务 ID 列表写进日志。非法上限回退到 1,null/空输出 count=0。
|
||||
*/
|
||||
static String formatIdSample(List<Long> ids, int sampleLimit) {
|
||||
int count = ids == null ? 0 : ids.size();
|
||||
if (count == 0) {
|
||||
return formatIdSample(ids, count, sampleLimit);
|
||||
}
|
||||
|
||||
private static String formatIdSample(List<Long> samples, int count, int sampleLimit) {
|
||||
if (count <= 0) {
|
||||
return "count=0";
|
||||
}
|
||||
int limit = Math.max(1, sampleLimit);
|
||||
StringBuilder sb = new StringBuilder("count=").append(count).append(", sample=[");
|
||||
for (int i = 0; i < Math.min(count, limit); i++) {
|
||||
int sampleCount = samples == null ? 0 : Math.min(samples.size(), limit);
|
||||
for (int i = 0; i < sampleCount; i++) {
|
||||
if (i > 0) {
|
||||
sb.append(',');
|
||||
}
|
||||
sb.append(ids.get(i));
|
||||
sb.append(samples.get(i));
|
||||
}
|
||||
if (count > limit) {
|
||||
sb.append(",...");
|
||||
if (count > sampleCount) {
|
||||
if (sampleCount > 0) {
|
||||
sb.append(',');
|
||||
}
|
||||
sb.append("...");
|
||||
}
|
||||
return sb.append(']').toString();
|
||||
}
|
||||
|
||||
+2
-1
@@ -26,7 +26,8 @@ public class TaskDistributedLockService {
|
||||
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
|
||||
private final ScheduledExecutorService renewalExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
/** 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。 */
|
||||
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-lock-renewal");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
|
||||
+28
-8
@@ -37,6 +37,8 @@ public class TaskFileJobLocalDispatcher {
|
||||
private int maxInflightDispatch = 64;
|
||||
|
||||
private final Set<Long> inflightJobIds = ConcurrentHashMap.newKeySet();
|
||||
/** 保护“容量检查 + 加入集合”这一复合操作,避免并发提交突破 inflight 上限。 */
|
||||
private final Object inflightMonitor = new Object();
|
||||
|
||||
public boolean dispatch(Long jobId, Long taskId, String moduleType) {
|
||||
return dispatch(jobId, taskId, moduleType, false);
|
||||
@@ -64,12 +66,21 @@ public class TaskFileJobLocalDispatcher {
|
||||
return false;
|
||||
}
|
||||
int inflightLimit = Math.max(1, maxInflightDispatch);
|
||||
if (!inflightJobIds.contains(jobId) && inflightJobIds.size() >= inflightLimit) {
|
||||
log.warn("[task-file-job] local dispatch backpressure, inflight limit reached jobId={} taskId={} moduleType={} inflight={} limit={}",
|
||||
jobId, taskId, moduleType, inflightJobIds.size(), inflightLimit);
|
||||
return false;
|
||||
boolean alreadyInflight;
|
||||
int inflightSize;
|
||||
synchronized (inflightMonitor) {
|
||||
alreadyInflight = inflightJobIds.contains(jobId);
|
||||
inflightSize = inflightJobIds.size();
|
||||
if (!alreadyInflight && inflightSize >= inflightLimit) {
|
||||
log.warn("[task-file-job] local dispatch backpressure, inflight limit reached jobId={} taskId={} moduleType={} inflight={} limit={}",
|
||||
jobId, taskId, moduleType, inflightSize, inflightLimit);
|
||||
return false;
|
||||
}
|
||||
if (!alreadyInflight) {
|
||||
inflightJobIds.add(jobId);
|
||||
}
|
||||
}
|
||||
if (!inflightJobIds.add(jobId)) {
|
||||
if (alreadyInflight) {
|
||||
log.info("[task-file-job] local dispatch skipped, job already inflight jobId={} taskId={} moduleType={}",
|
||||
jobId, taskId, moduleType);
|
||||
return true;
|
||||
@@ -78,7 +89,7 @@ public class TaskFileJobLocalDispatcher {
|
||||
taskFileJobDispatchExecutor.execute(() -> processLocally(jobId, taskId, moduleType, alreadyClaimed));
|
||||
return true;
|
||||
} catch (RuntimeException ex) {
|
||||
inflightJobIds.remove(jobId);
|
||||
removeInflight(jobId);
|
||||
log.warn("[task-file-job] local dispatch executor rejected jobId={} taskId={} moduleType={} msg={}",
|
||||
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||
if (force) {
|
||||
@@ -86,13 +97,22 @@ public class TaskFileJobLocalDispatcher {
|
||||
processLocally(jobId, taskId, moduleType, alreadyClaimed);
|
||||
return true;
|
||||
} finally {
|
||||
inflightJobIds.remove(jobId);
|
||||
removeInflight(jobId);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void removeInflight(Long jobId) {
|
||||
if (jobId == null) {
|
||||
return;
|
||||
}
|
||||
synchronized (inflightMonitor) {
|
||||
inflightJobIds.remove(jobId);
|
||||
}
|
||||
}
|
||||
|
||||
private void processLocally(Long jobId, Long taskId, String moduleType, boolean alreadyClaimed) {
|
||||
try {
|
||||
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
|
||||
@@ -116,7 +136,7 @@ public class TaskFileJobLocalDispatcher {
|
||||
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
|
||||
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||
} finally {
|
||||
inflightJobIds.remove(jobId);
|
||||
removeInflight(jobId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -53,7 +53,8 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
|
||||
private long heartbeatIntervalMillis = 60000L;
|
||||
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
/** 多个结果文件 Job 并行运行时,心跳不能被单个 DB 慢调用串行阻塞。 */
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
|
||||
+18
-6
@@ -16,6 +16,7 @@ import java.nio.file.attribute.FileTime;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Comparator;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@@ -25,6 +26,8 @@ public class TaskScopePayloadBufferMaintenanceService {
|
||||
|
||||
private static final Duration RECOVERY_LOCK_TTL = Duration.ofMinutes(10);
|
||||
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
|
||||
/** 每轮最多扫描/处理的文件数,避免异常堆积导致单次清理占满内存和 CPU。 */
|
||||
private static final int MAX_CLEANUP_FILES_PER_RUN = 5_000;
|
||||
|
||||
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||
private final TaskPressureProperties taskPressureProperties;
|
||||
@@ -85,7 +88,7 @@ public class TaskScopePayloadBufferMaintenanceService {
|
||||
}
|
||||
Instant cutoff = Instant.now().minus(Duration.ofHours(Math.max(1L, taskPressureProperties.getScopePayloadBufferRetentionHours())));
|
||||
int deleted = 0;
|
||||
for (Path file : listBufferedPayloadFiles(root, Integer.MAX_VALUE)) {
|
||||
for (Path file : listBufferedPayloadFiles(root, MAX_CLEANUP_FILES_PER_RUN)) {
|
||||
try {
|
||||
FileTime lastModified = Files.getLastModifiedTime(file);
|
||||
if (lastModified.toInstant().isAfter(cutoff)) {
|
||||
@@ -105,13 +108,22 @@ public class TaskScopePayloadBufferMaintenanceService {
|
||||
}
|
||||
|
||||
private List<Path> listBufferedPayloadFiles(Path root, int limit) {
|
||||
int safeLimit = Math.max(1, Math.min(limit, MAX_CLEANUP_FILES_PER_RUN));
|
||||
Comparator<Path> oldestFirst = Comparator.comparing(this::safeLastModified);
|
||||
// sorted().limit() 仍会把整个目录加载进排序缓冲;这里仅保留最老的 safeLimit 个文件。
|
||||
PriorityQueue<Path> newestFirst = new PriorityQueue<>(safeLimit, oldestFirst.reversed());
|
||||
try (var walk = Files.walk(root, 4)) {
|
||||
return walk
|
||||
.filter(Files::isRegularFile)
|
||||
walk.filter(Files::isRegularFile)
|
||||
.filter(path -> path.getFileName().toString().endsWith(".json"))
|
||||
.sorted(Comparator.comparing(this::safeLastModified))
|
||||
.limit(limit)
|
||||
.toList();
|
||||
.forEach(path -> {
|
||||
if (newestFirst.size() < safeLimit) {
|
||||
newestFirst.offer(path);
|
||||
} else if (oldestFirst.compare(path, newestFirst.peek()) < 0) {
|
||||
newestFirst.poll();
|
||||
newestFirst.offer(path);
|
||||
}
|
||||
});
|
||||
return newestFirst.stream().sorted(oldestFirst).toList();
|
||||
} catch (IOException ex) {
|
||||
log.warn("[task-scope-buffer] scan failed root={} msg={}", root, ex.getMessage());
|
||||
return List.of();
|
||||
|
||||
+143
-14
@@ -14,7 +14,9 @@ import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -215,7 +217,10 @@ public class TaskScopePayloadStorageService {
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
/**
|
||||
* 先在短事务内删除/清空数据库引用,提交后再执行远程对象删除。
|
||||
* 不能把 RustFS/OSS 网络调用放在数据库事务中,否则大任务会长时间占用连接。
|
||||
*/
|
||||
public void deleteTaskScopePayloads(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return;
|
||||
@@ -226,18 +231,33 @@ public class TaskScopePayloadStorageService {
|
||||
if (states == null || states.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
if (state == null || state.getId() == null) {
|
||||
continue;
|
||||
List<String> payloadsToDelete = states.stream()
|
||||
.filter(state -> state != null && !isBlank(state.getStateJson()))
|
||||
.map(TaskScopeStateEntity::getStateJson)
|
||||
.toList();
|
||||
inTransaction(() -> {
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
if (state == null || state.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
if (!isBlank(state.getParsedPayloadJson())) {
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getStateJson, null)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
} else {
|
||||
taskScopeStateMapper.deleteById(state.getId());
|
||||
}
|
||||
}
|
||||
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
|
||||
if (!isBlank(state.getParsedPayloadJson())) {
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getStateJson, null)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
} else {
|
||||
taskScopeStateMapper.deleteById(state.getId());
|
||||
return null;
|
||||
});
|
||||
// DB 引用已提交后再做引用检查和物理删除,失败由后续清理任务兜底。
|
||||
for (String payload : payloadsToDelete) {
|
||||
try {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(payload);
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[task-scope-payload] delete physical payload failed taskId={} moduleType={} msg={}",
|
||||
taskId, moduleType, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -328,9 +348,74 @@ public class TaskScopePayloadStorageService {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
/**
|
||||
* 恢复旧版本在本地 task-scope-buffer 中留下的载荷。文件读取和对象存储上传
|
||||
* 放在事务外,只有最终的短 UPDATE 使用事务,避免启动恢复时长时间占用 DB 连接。
|
||||
*/
|
||||
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
|
||||
return false;
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || !isValidScopeHash(scopeHash)) {
|
||||
return false;
|
||||
}
|
||||
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
|
||||
if (payloadJson == null || payloadJson.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
TaskScopeStateEntity existing = getScopeState(taskId, moduleType, scopeHash);
|
||||
if (existing == null) {
|
||||
// 没有对应 DB 行的孤儿缓冲文件无法恢复,及时删除避免反复扫描。
|
||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||
return false;
|
||||
}
|
||||
|
||||
String storedPayload = transientPayloadStorageService.storeScopePayloadVersioned(
|
||||
moduleType, taskId, scopeHash, payloadJson, true);
|
||||
String replacedPayload;
|
||||
try {
|
||||
replacedPayload = inTransaction(() -> persistRecoveredScopePayload(
|
||||
taskId, moduleType, scopeHash, storedPayload));
|
||||
} catch (RuntimeException ex) {
|
||||
// 数据库写入失败时不要删除新对象,后续恢复任务仍可重试;
|
||||
// 若存储服务支持引用检查,尽力清理未被引用的对象。
|
||||
try {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
} catch (RuntimeException cleanupEx) {
|
||||
log.warn("[task-scope-buffer] failed to cleanup uploaded recovery payload taskId={} moduleType={} scopeHash={} msg={}",
|
||||
taskId, moduleType, scopeHash, cleanupEx.getMessage());
|
||||
}
|
||||
log.warn("[task-scope-buffer] persist recovered payload failed taskId={} moduleType={} scopeHash={} msg={}",
|
||||
taskId, moduleType, scopeHash, ex.getMessage());
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(replacedPayload, storedPayload);
|
||||
} catch (RuntimeException cleanupEx) {
|
||||
// 数据库已恢复成功;旧对象删除失败交给后续清理任务,不应让恢复再次上传。
|
||||
log.warn("[task-scope-buffer] failed to cleanup replaced payload taskId={} moduleType={} scopeHash={} msg={}",
|
||||
taskId, moduleType, scopeHash, cleanupEx.getMessage());
|
||||
} finally {
|
||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private String persistRecoveredScopePayload(Long taskId, String moduleType, String scopeHash, String storedPayload) {
|
||||
TaskScopeStateEntity state = getScopeStateForUpdate(taskId, moduleType, scopeHash);
|
||||
if (state == null) {
|
||||
throw new BusinessException("恢复任务范围载荷失败:任务范围不存在");
|
||||
}
|
||||
String replacedPayload = state.getStateJson();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getStateJson, storedPayload)
|
||||
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||
if (updated <= 0) {
|
||||
throw new BusinessException("恢复任务范围载荷失败:数据库未更新");
|
||||
}
|
||||
return replacedPayload;
|
||||
}
|
||||
|
||||
private <T> T inTransaction(java.util.function.Supplier<T> action) {
|
||||
@@ -341,6 +426,50 @@ public class TaskScopePayloadStorageService {
|
||||
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR);
|
||||
}
|
||||
|
||||
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
||||
try {
|
||||
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
||||
if (!Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)) {
|
||||
return null;
|
||||
}
|
||||
return Files.readString(path, StandardCharsets.UTF_8);
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.warn("[task-scope-buffer] read buffered payload failed taskId={} moduleType={} scopeHash={} msg={}",
|
||||
taskId, moduleType, scopeHash, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
||||
try {
|
||||
Files.deleteIfExists(buildBufferedPayloadPath(taskId, moduleType, scopeHash));
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
log.warn("[task-scope-buffer] delete buffered payload failed taskId={} moduleType={} scopeHash={} msg={}",
|
||||
taskId, moduleType, scopeHash, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || !isValidScopeHash(scopeHash)) {
|
||||
throw new IllegalArgumentException("invalid buffered payload identity");
|
||||
}
|
||||
String normalizedModule = moduleType.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
if (!normalizedModule.matches("[a-z0-9_-]+")) {
|
||||
throw new IllegalArgumentException("invalid buffered payload module");
|
||||
}
|
||||
Path root = getBufferedPayloadRoot().toAbsolutePath().normalize();
|
||||
Path target = root.resolve(normalizedModule).resolve(String.valueOf(taskId))
|
||||
.resolve(scopeHash.toLowerCase(java.util.Locale.ROOT) + ".json").normalize();
|
||||
if (!target.startsWith(root) || target.equals(root)) {
|
||||
throw new IllegalArgumentException("buffered payload path escapes root");
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
private boolean isValidScopeHash(String scopeHash) {
|
||||
return scopeHash != null && scopeHash.matches("[0-9a-fA-F]{64}");
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity getScopeState(Long taskId, String moduleType, String scopeHash) {
|
||||
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
|
||||
+33
-6
@@ -8,6 +8,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -18,6 +19,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
@@ -52,7 +54,7 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
ObjectMapper objectMapper,
|
||||
ExecutorService asyncDeleteExecutor) {
|
||||
@Qualifier("transientPayloadDeleteExecutor") ExecutorService asyncDeleteExecutor) {
|
||||
this.transientPayloadStorageService = transientPayloadStorageService;
|
||||
this.rustfsObjectStorageService = rustfsObjectStorageService;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
@@ -76,7 +78,7 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
if (pointer == null) {
|
||||
continue;
|
||||
}
|
||||
if (pendingPointers.size() >= maxPendingDeletes) {
|
||||
if (pendingPointers.size() >= pendingLimit()) {
|
||||
log.warn("[transient-payload] delete queue full, drop {} pointer={}", "submit", pointer);
|
||||
break;
|
||||
}
|
||||
@@ -106,7 +108,14 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
List<String> toDelete = new ArrayList<>(batch);
|
||||
toDelete.removeAll(stillReferenced);
|
||||
if (!toDelete.isEmpty()) {
|
||||
asyncDeleteExecutor.submit(() -> deleteObjects(toDelete));
|
||||
try {
|
||||
asyncDeleteExecutor.submit(() -> deleteObjects(toDelete));
|
||||
} catch (RejectedExecutionException ex) {
|
||||
restorePending(toDelete);
|
||||
log.warn("[transient-payload] delete executor saturated, restored pending count={} msg={}",
|
||||
toDelete.size(), ex.getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
if (!stillReferenced.isEmpty()) {
|
||||
log.info("[transient-payload] skip delete, still referenced count={}", stillReferenced.size());
|
||||
@@ -115,13 +124,31 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] batch reference check failed, keep pending count={} err={}",
|
||||
batch.size(), ex.getMessage());
|
||||
synchronized (pendingPointers) {
|
||||
pendingPointers.addAll(batch);
|
||||
}
|
||||
restorePending(batch);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private int pendingLimit() {
|
||||
return (int) Math.max(1L, Math.min(maxPendingDeletes, Integer.MAX_VALUE));
|
||||
}
|
||||
|
||||
private void restorePending(List<String> pointers) {
|
||||
if (pointers == null || pointers.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
synchronized (pendingPointers) {
|
||||
int limit = pendingLimit();
|
||||
for (String pointer : pointers) {
|
||||
if (pendingPointers.size() >= limit) {
|
||||
log.warn("[transient-payload] pending delete queue full while restoring dropped pointer");
|
||||
break;
|
||||
}
|
||||
pendingPointers.add(pointer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int pendingCount() {
|
||||
synchronized (pendingPointers) {
|
||||
return pendingPointers.size();
|
||||
|
||||
Reference in New Issue
Block a user