处理一些合并冲突

This commit is contained in:
super
2026-05-05 14:11:35 +08:00
parent 790326e87b
commit 2fedcaecb8
19 changed files with 1075 additions and 198 deletions
@@ -0,0 +1,160 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@Slf4j
public class TaskDistributedLockService {
public static final Duration DEFAULT_LOCK_TTL = Duration.ofMinutes(5);
public static final long DEFAULT_WAIT_MILLIS = 10000L;
private static final long RETRY_DELAY_MILLIS = 200L;
private final DistributedJobLockService distributedJobLockService;
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
private final ScheduledExecutorService renewalExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
Thread thread = new Thread(runnable, "task-lock-renewal");
thread.setDaemon(true);
return thread;
});
public LockHandle acquire(String moduleType, Long taskId) {
return acquire(moduleType, taskId, DEFAULT_WAIT_MILLIS);
}
public LockHandle acquire(String moduleType, Long taskId, long waitMillis) {
return acquire(moduleType, taskId, DEFAULT_LOCK_TTL, waitMillis);
}
public LockHandle acquire(String moduleType, Long taskId, Duration ttl, long waitMillis) {
String lockName = taskLockName(moduleType, taskId);
if (lockName == null) {
return null;
}
Map<String, HeldLock> heldLocks = localLocks.get();
HeldLock heldLock = heldLocks.get(lockName);
if (heldLock != null) {
heldLock.depth++;
return new LockHandle(lockName);
}
long deadline = System.nanoTime() + Math.max(0L, waitMillis) * 1_000_000L;
do {
DistributedJobLockService.LockHandle delegate = distributedJobLockService.tryLock(lockName, ttl);
if (delegate != null) {
HeldLock newHeldLock = new HeldLock(delegate, ttl);
scheduleRenewal(lockName, newHeldLock, ttl);
heldLocks.put(lockName, newHeldLock);
return new LockHandle(lockName);
}
if (waitMillis <= 0L || System.nanoTime() >= deadline) {
return null;
}
try {
Thread.sleep(RETRY_DELAY_MILLIS);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
return null;
}
} while (true);
}
public String taskLockName(String moduleType, Long taskId) {
if (moduleType == null || moduleType.isBlank() || taskId == null || taskId <= 0) {
return null;
}
String normalizedModule = moduleType.trim().toLowerCase(Locale.ROOT).replace('_', '-');
return "task:" + normalizedModule + ":" + taskId;
}
private void scheduleRenewal(String lockName, HeldLock heldLock, Duration ttl) {
long ttlMillis = Math.max(1000L, ttl.toMillis());
long renewalDelayMillis = Math.max(1000L, ttlMillis / 3L);
heldLock.renewalFuture = renewalExecutor.scheduleWithFixedDelay(() -> {
if (heldLock.closed) {
return;
}
boolean renewed = heldLock.delegate.renew(heldLock.ttl);
if (!renewed) {
heldLock.closed = true;
ScheduledFuture<?> future = heldLock.renewalFuture;
if (future != null) {
future.cancel(false);
}
log.warn("[task-lock] renewal failed lockName={}", lockName);
}
}, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS);
}
@PreDestroy
public void shutdownRenewalExecutor() {
renewalExecutor.shutdownNow();
}
private static final class HeldLock {
private final DistributedJobLockService.LockHandle delegate;
private final Duration ttl;
private ScheduledFuture<?> renewalFuture;
private int depth = 1;
private volatile boolean closed;
private HeldLock(DistributedJobLockService.LockHandle delegate, Duration ttl) {
this.delegate = delegate;
this.ttl = ttl;
}
private void close() {
closed = true;
if (renewalFuture != null) {
renewalFuture.cancel(false);
}
delegate.close();
}
}
public final class LockHandle implements AutoCloseable {
private final String lockName;
private boolean closed;
private LockHandle(String lockName) {
this.lockName = lockName;
}
@Override
public void close() {
if (closed) {
return;
}
closed = true;
Map<String, HeldLock> heldLocks = localLocks.get();
HeldLock heldLock = heldLocks.get(lockName);
if (heldLock == null) {
return;
}
heldLock.depth--;
if (heldLock.depth <= 0) {
heldLocks.remove(lockName);
if (heldLocks.isEmpty()) {
localLocks.remove();
}
heldLock.close();
}
}
}
}
@@ -168,6 +168,26 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
}
@Transactional
public boolean requeue(Long jobId, String message) {
if (jobId == null || jobId <= 0) {
return false;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, message)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, null));
if (updated <= 0) {
return false;
}
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
publishDispatchEvent(refreshed);
return true;
}
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
@@ -29,6 +29,7 @@ import java.util.List;
public class TaskResultFileJobWorker {
private final TaskFileJobService taskFileJobService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskResultPayloadService taskResultPayloadService;
private final FileResultMapper fileResultMapper;
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
@@ -103,20 +104,30 @@ public class TaskResultFileJobWorker {
private void processInternal(TaskFileJobEntity job) {
long startedAt = System.currentTimeMillis();
try {
boolean completed = dispatch(job);
if (!completed) {
taskFileJobService.touchRunning(job.getId());
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt);
TaskDistributedLockService.LockHandle lockHandle =
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
if (lockHandle == null) {
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
return;
}
String resultFileUrl = resolveResultFileUrl(job);
taskFileJobService.markSuccess(job, resultFileUrl);
cleanupAfterSuccess(job);
log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt, resultFileUrl);
try (lockHandle) {
boolean completed = dispatch(job);
if (!completed) {
taskFileJobService.touchRunning(job.getId());
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt);
return;
}
String resultFileUrl = resolveResultFileUrl(job);
taskFileJobService.markSuccess(job, resultFileUrl);
cleanupAfterSuccess(job);
log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt, resultFileUrl);
}
} catch (Exception ex) {
String message = ex.getMessage() == null ? "结果文件生成失败" : ex.getMessage();
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
@@ -33,7 +33,12 @@ public class TransientPayloadStorageService {
private final ObjectMapper objectMapper;
public boolean isWriteEnabled() {
return properties.isEnabled();
return properties.isEnabled()
&& (rustfsObjectStorageService.isConfigured() || storageProperties.getLocalTempDir() != null);
}
public boolean isSharedWriteEnabled() {
return properties.isEnabled() && rustfsObjectStorageService.isConfigured();
}
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
@@ -170,16 +175,19 @@ public class TransientPayloadStorageService {
return content;
}
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
String pointer = storeLocal(objectKey, content);
if (pointer == null && rustfsObjectStorageService.isConfigured()) {
String pointer = null;
if (rustfsObjectStorageService.isConfigured()) {
try {
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
} catch (Exception ex) {
log.warn("[transient-payload] rustfs upload failed and local fallback unavailable objectKey={} err={}",
log.warn("[transient-payload] rustfs upload failed objectKey={} err={}",
objectKey, ex.getMessage());
throw ex;
}
}
if (pointer == null) {
pointer = storeLocal(objectKey, content);
}
if (pointer == null) {
return content;
}