处理一些合并冲突
This commit is contained in:
+160
@@ -0,0 +1,160 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class TaskDistributedLockService {
|
||||
|
||||
public static final Duration DEFAULT_LOCK_TTL = Duration.ofMinutes(5);
|
||||
public static final long DEFAULT_WAIT_MILLIS = 10000L;
|
||||
private static final long RETRY_DELAY_MILLIS = 200L;
|
||||
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
|
||||
private final ScheduledExecutorService renewalExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-lock-renewal");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
public LockHandle acquire(String moduleType, Long taskId) {
|
||||
return acquire(moduleType, taskId, DEFAULT_WAIT_MILLIS);
|
||||
}
|
||||
|
||||
public LockHandle acquire(String moduleType, Long taskId, long waitMillis) {
|
||||
return acquire(moduleType, taskId, DEFAULT_LOCK_TTL, waitMillis);
|
||||
}
|
||||
|
||||
public LockHandle acquire(String moduleType, Long taskId, Duration ttl, long waitMillis) {
|
||||
String lockName = taskLockName(moduleType, taskId);
|
||||
if (lockName == null) {
|
||||
return null;
|
||||
}
|
||||
Map<String, HeldLock> heldLocks = localLocks.get();
|
||||
HeldLock heldLock = heldLocks.get(lockName);
|
||||
if (heldLock != null) {
|
||||
heldLock.depth++;
|
||||
return new LockHandle(lockName);
|
||||
}
|
||||
|
||||
long deadline = System.nanoTime() + Math.max(0L, waitMillis) * 1_000_000L;
|
||||
do {
|
||||
DistributedJobLockService.LockHandle delegate = distributedJobLockService.tryLock(lockName, ttl);
|
||||
if (delegate != null) {
|
||||
HeldLock newHeldLock = new HeldLock(delegate, ttl);
|
||||
scheduleRenewal(lockName, newHeldLock, ttl);
|
||||
heldLocks.put(lockName, newHeldLock);
|
||||
return new LockHandle(lockName);
|
||||
}
|
||||
if (waitMillis <= 0L || System.nanoTime() >= deadline) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(RETRY_DELAY_MILLIS);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
return null;
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
|
||||
public String taskLockName(String moduleType, Long taskId) {
|
||||
if (moduleType == null || moduleType.isBlank() || taskId == null || taskId <= 0) {
|
||||
return null;
|
||||
}
|
||||
String normalizedModule = moduleType.trim().toLowerCase(Locale.ROOT).replace('_', '-');
|
||||
return "task:" + normalizedModule + ":" + taskId;
|
||||
}
|
||||
|
||||
private void scheduleRenewal(String lockName, HeldLock heldLock, Duration ttl) {
|
||||
long ttlMillis = Math.max(1000L, ttl.toMillis());
|
||||
long renewalDelayMillis = Math.max(1000L, ttlMillis / 3L);
|
||||
heldLock.renewalFuture = renewalExecutor.scheduleWithFixedDelay(() -> {
|
||||
if (heldLock.closed) {
|
||||
return;
|
||||
}
|
||||
boolean renewed = heldLock.delegate.renew(heldLock.ttl);
|
||||
if (!renewed) {
|
||||
heldLock.closed = true;
|
||||
ScheduledFuture<?> future = heldLock.renewalFuture;
|
||||
if (future != null) {
|
||||
future.cancel(false);
|
||||
}
|
||||
log.warn("[task-lock] renewal failed lockName={}", lockName);
|
||||
}
|
||||
}, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdownRenewalExecutor() {
|
||||
renewalExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
private static final class HeldLock {
|
||||
|
||||
private final DistributedJobLockService.LockHandle delegate;
|
||||
private final Duration ttl;
|
||||
private ScheduledFuture<?> renewalFuture;
|
||||
private int depth = 1;
|
||||
private volatile boolean closed;
|
||||
|
||||
private HeldLock(DistributedJobLockService.LockHandle delegate, Duration ttl) {
|
||||
this.delegate = delegate;
|
||||
this.ttl = ttl;
|
||||
}
|
||||
|
||||
private void close() {
|
||||
closed = true;
|
||||
if (renewalFuture != null) {
|
||||
renewalFuture.cancel(false);
|
||||
}
|
||||
delegate.close();
|
||||
}
|
||||
}
|
||||
|
||||
public final class LockHandle implements AutoCloseable {
|
||||
|
||||
private final String lockName;
|
||||
private boolean closed;
|
||||
|
||||
private LockHandle(String lockName) {
|
||||
this.lockName = lockName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
Map<String, HeldLock> heldLocks = localLocks.get();
|
||||
HeldLock heldLock = heldLocks.get(lockName);
|
||||
if (heldLock == null) {
|
||||
return;
|
||||
}
|
||||
heldLock.depth--;
|
||||
if (heldLock.depth <= 0) {
|
||||
heldLocks.remove(lockName);
|
||||
if (heldLocks.isEmpty()) {
|
||||
localLocks.remove();
|
||||
}
|
||||
heldLock.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
@@ -168,6 +168,26 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public boolean requeue(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||
if (updated <= 0) {
|
||||
return false;
|
||||
}
|
||||
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
|
||||
publishDispatchEvent(refreshed);
|
||||
return true;
|
||||
}
|
||||
|
||||
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
|
||||
+23
-12
@@ -29,6 +29,7 @@ import java.util.List;
|
||||
public class TaskResultFileJobWorker {
|
||||
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final TaskResultPayloadService taskResultPayloadService;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
|
||||
@@ -103,20 +104,30 @@ public class TaskResultFileJobWorker {
|
||||
private void processInternal(TaskFileJobEntity job) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
try {
|
||||
boolean completed = dispatch(job);
|
||||
if (!completed) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
System.currentTimeMillis() - startedAt);
|
||||
TaskDistributedLockService.LockHandle lockHandle =
|
||||
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
|
||||
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
return;
|
||||
}
|
||||
String resultFileUrl = resolveResultFileUrl(job);
|
||||
taskFileJobService.markSuccess(job, resultFileUrl);
|
||||
cleanupAfterSuccess(job);
|
||||
log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
System.currentTimeMillis() - startedAt, resultFileUrl);
|
||||
try (lockHandle) {
|
||||
boolean completed = dispatch(job);
|
||||
if (!completed) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
System.currentTimeMillis() - startedAt);
|
||||
return;
|
||||
}
|
||||
String resultFileUrl = resolveResultFileUrl(job);
|
||||
taskFileJobService.markSuccess(job, resultFileUrl);
|
||||
cleanupAfterSuccess(job);
|
||||
log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
System.currentTimeMillis() - startedAt, resultFileUrl);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
String message = ex.getMessage() == null ? "结果文件生成失败" : ex.getMessage();
|
||||
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
|
||||
|
||||
+12
-4
@@ -33,7 +33,12 @@ public class TransientPayloadStorageService {
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public boolean isWriteEnabled() {
|
||||
return properties.isEnabled();
|
||||
return properties.isEnabled()
|
||||
&& (rustfsObjectStorageService.isConfigured() || storageProperties.getLocalTempDir() != null);
|
||||
}
|
||||
|
||||
public boolean isSharedWriteEnabled() {
|
||||
return properties.isEnabled() && rustfsObjectStorageService.isConfigured();
|
||||
}
|
||||
|
||||
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
|
||||
@@ -170,16 +175,19 @@ public class TransientPayloadStorageService {
|
||||
return content;
|
||||
}
|
||||
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
|
||||
String pointer = storeLocal(objectKey, content);
|
||||
if (pointer == null && rustfsObjectStorageService.isConfigured()) {
|
||||
String pointer = null;
|
||||
if (rustfsObjectStorageService.isConfigured()) {
|
||||
try {
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] rustfs upload failed and local fallback unavailable objectKey={} err={}",
|
||||
log.warn("[transient-payload] rustfs upload failed objectKey={} err={}",
|
||||
objectKey, ex.getMessage());
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
if (pointer == null) {
|
||||
pointer = storeLocal(objectKey, content);
|
||||
}
|
||||
if (pointer == null) {
|
||||
return content;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user