完善任务存储、权限及货源查询流程
This commit is contained in:
+120
-67
@@ -7,8 +7,13 @@ import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Service
|
||||
@@ -18,18 +23,30 @@ public class RustfsDeleteRetryService {
|
||||
|
||||
private final TransientStorageProperties properties;
|
||||
private final RustfsObjectStorageService rustfsObjectStorageService;
|
||||
private final ConcurrentLinkedQueue<String> queue = new ConcurrentLinkedQueue<>();
|
||||
private final ConcurrentLinkedQueue<RetryItem> queue = new ConcurrentLinkedQueue<>();
|
||||
private final ConcurrentHashMap<String, RetryItem> pending = new ConcurrentHashMap<>();
|
||||
private final AtomicInteger queuedCount = new AtomicInteger();
|
||||
private final Object admissionLock = new Object();
|
||||
|
||||
public void enqueue(String objectKey, Throwable cause) {
|
||||
if (!properties.isDeleteRetryEnabled() || objectKey == null || objectKey.isBlank()) {
|
||||
return;
|
||||
}
|
||||
RetryItem item = pending.computeIfAbsent(objectKey, RetryItem::new);
|
||||
item.lastError = cause == null ? null : cause.getMessage();
|
||||
item.lastFailedAt = LocalDateTime.now();
|
||||
enqueueItem(item, false);
|
||||
while (true) {
|
||||
RetryItem item = findOrAdmit(objectKey, cause);
|
||||
if (item == null) {
|
||||
return;
|
||||
}
|
||||
synchronized (item) {
|
||||
if (pending.get(objectKey) != item) {
|
||||
continue;
|
||||
}
|
||||
item.generation++;
|
||||
item.lastError = cause == null ? null : cause.getMessage();
|
||||
item.lastFailedAt = LocalDateTime.now();
|
||||
enqueueItemLocked(item);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${aiimage.transient-storage.delete-retry-cron:0 */5 * * * *}")
|
||||
@@ -38,36 +55,50 @@ public class RustfsDeleteRetryService {
|
||||
return;
|
||||
}
|
||||
int batchSize = Math.max(1, properties.getDeleteRetryBatchSize());
|
||||
long batchDurationNanos = TimeUnit.SECONDS.toNanos(
|
||||
Math.max(1L, properties.getDeleteRetryBatchTimeoutSeconds()));
|
||||
long deadlineNanos = System.nanoTime() + batchDurationNanos;
|
||||
int processed = 0;
|
||||
int success = 0;
|
||||
int failed = 0;
|
||||
Set<String> attemptedObjectKeys = new HashSet<>();
|
||||
List<RetryItem> retryNextInvocation = new ArrayList<>();
|
||||
while (processed < batchSize) {
|
||||
String objectKey = nextQueuedObjectKey(batchSize - processed);
|
||||
if (objectKey == null) {
|
||||
long remainingNanos = deadlineNanos - System.nanoTime();
|
||||
if (remainingNanos <= 0L) {
|
||||
break;
|
||||
}
|
||||
RetryItem item = pending.get(objectKey);
|
||||
if (item == null) {
|
||||
RetryAttempt attempt = nextAttempt();
|
||||
if (attempt == null) {
|
||||
break;
|
||||
}
|
||||
RetryItem item = attempt.item;
|
||||
String objectKey = item.objectKey;
|
||||
if (!attemptedObjectKeys.add(objectKey)) {
|
||||
retryNextInvocation.add(item);
|
||||
continue;
|
||||
}
|
||||
item.queued = false;
|
||||
remainingNanos = deadlineNanos - System.nanoTime();
|
||||
if (remainingNanos <= 0L) {
|
||||
retryNextInvocation.add(item);
|
||||
break;
|
||||
}
|
||||
processed++;
|
||||
try {
|
||||
rustfsObjectStorageService.deleteObjectFromRetry(objectKey);
|
||||
pending.remove(objectKey);
|
||||
rustfsObjectStorageService.deleteObjectFromRetry(objectKey, remainingNanos);
|
||||
removeIfUnchanged(attempt);
|
||||
success++;
|
||||
} catch (Exception ex) {
|
||||
failed++;
|
||||
int failures = item.failures.incrementAndGet();
|
||||
item.lastError = ex.getMessage();
|
||||
item.lastFailedAt = LocalDateTime.now();
|
||||
int failures = recordFailureIfUnchanged(attempt, ex);
|
||||
log.warn("[rustfs] delete retry failed objectKey={} failures={} err={}", objectKey, failures, ex.getMessage());
|
||||
enqueueItem(item, true);
|
||||
retryNextInvocation.add(item);
|
||||
}
|
||||
}
|
||||
retryNextInvocation.forEach(this::requeue);
|
||||
if (processed > 0) {
|
||||
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={} queued={}",
|
||||
processed, success, failed, pending.size(), queuedCount.get());
|
||||
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={}",
|
||||
processed, success, failed, pending.size());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,72 +106,94 @@ public class RustfsDeleteRetryService {
|
||||
return pending.size();
|
||||
}
|
||||
|
||||
private String nextQueuedObjectKey(int refillLimit) {
|
||||
String objectKey = queue.poll();
|
||||
if (objectKey != null) {
|
||||
decrementQueuedCount();
|
||||
refillQueue(refillLimit);
|
||||
return objectKey;
|
||||
}
|
||||
if (refillQueue(refillLimit) <= 0) {
|
||||
return null;
|
||||
}
|
||||
objectKey = queue.poll();
|
||||
if (objectKey != null) {
|
||||
decrementQueuedCount();
|
||||
refillQueue(refillLimit);
|
||||
}
|
||||
return objectKey;
|
||||
}
|
||||
|
||||
private int refillQueue(int maxItems) {
|
||||
if (maxItems <= 0) {
|
||||
return 0;
|
||||
private RetryItem findOrAdmit(String objectKey, Throwable cause) {
|
||||
RetryItem item = pending.get(objectKey);
|
||||
if (item != null) {
|
||||
return item;
|
||||
}
|
||||
int capacity = Math.max(1, properties.getDeleteRetryQueueCapacity());
|
||||
if (queuedCount.get() >= capacity) {
|
||||
return 0;
|
||||
}
|
||||
int added = 0;
|
||||
for (RetryItem item : pending.values()) {
|
||||
if (added >= maxItems || queuedCount.get() >= capacity) {
|
||||
break;
|
||||
synchronized (admissionLock) {
|
||||
item = pending.get(objectKey);
|
||||
if (item != null) {
|
||||
return item;
|
||||
}
|
||||
if (enqueueItem(item, true)) {
|
||||
added++;
|
||||
if (pending.size() >= capacity) {
|
||||
log.error("[rustfs] delete retry capacity reached, rejected objectKey={} pending={} capacity={} lastError={}",
|
||||
objectKey, pending.size(), capacity, cause == null ? null : cause.getMessage());
|
||||
return null;
|
||||
}
|
||||
item = new RetryItem(objectKey);
|
||||
pending.put(objectKey, item);
|
||||
return item;
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
private void decrementQueuedCount() {
|
||||
queuedCount.updateAndGet(value -> Math.max(0, value - 1));
|
||||
private RetryAttempt nextAttempt() {
|
||||
while (true) {
|
||||
RetryItem item = queue.poll();
|
||||
if (item == null) {
|
||||
return null;
|
||||
}
|
||||
synchronized (item) {
|
||||
if (pending.get(item.objectKey) != item || !item.queued) {
|
||||
continue;
|
||||
}
|
||||
item.queued = false;
|
||||
return new RetryAttempt(item, item.generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean enqueueItem(RetryItem item, boolean requeue) {
|
||||
private void removeIfUnchanged(RetryAttempt attempt) {
|
||||
RetryItem item = attempt.item;
|
||||
synchronized (item) {
|
||||
if (item.queued) {
|
||||
return false;
|
||||
if (pending.get(item.objectKey) != item || item.generation != attempt.generation) {
|
||||
return;
|
||||
}
|
||||
int capacity = Math.max(1, properties.getDeleteRetryQueueCapacity());
|
||||
if (queuedCount.get() >= capacity) {
|
||||
log.error("[rustfs] delete retry queue full, deferred objectKey={} requeue={} pending={} queued={} capacity={} lastError={}",
|
||||
item.objectKey, requeue, pending.size(), queuedCount.get(), capacity, item.lastError);
|
||||
return false;
|
||||
synchronized (admissionLock) {
|
||||
pending.remove(item.objectKey, item);
|
||||
}
|
||||
item.queued = true;
|
||||
queue.offer(item.objectKey);
|
||||
queuedCount.incrementAndGet();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private int recordFailureIfUnchanged(RetryAttempt attempt, Exception ex) {
|
||||
RetryItem item = attempt.item;
|
||||
synchronized (item) {
|
||||
if (pending.get(item.objectKey) != item || item.generation != attempt.generation) {
|
||||
return item.failures.get();
|
||||
}
|
||||
int failures = item.failures.incrementAndGet();
|
||||
item.lastError = ex.getMessage();
|
||||
item.lastFailedAt = LocalDateTime.now();
|
||||
return failures;
|
||||
}
|
||||
}
|
||||
|
||||
private void requeue(RetryItem item) {
|
||||
synchronized (item) {
|
||||
if (pending.get(item.objectKey) == item) {
|
||||
enqueueItemLocked(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void enqueueItemLocked(RetryItem item) {
|
||||
if (!item.queued) {
|
||||
item.queued = true;
|
||||
queue.offer(item);
|
||||
}
|
||||
}
|
||||
|
||||
private record RetryAttempt(RetryItem item, long generation) {
|
||||
}
|
||||
|
||||
private static class RetryItem {
|
||||
private final String objectKey;
|
||||
private final AtomicInteger failures = new AtomicInteger();
|
||||
private volatile boolean queued;
|
||||
private volatile String lastError;
|
||||
private volatile LocalDateTime lastFailedAt;
|
||||
private boolean queued;
|
||||
private long generation;
|
||||
private String lastError;
|
||||
private LocalDateTime lastFailedAt;
|
||||
|
||||
private RetryItem(String objectKey) {
|
||||
this.objectKey = objectKey;
|
||||
|
||||
+178
-48
@@ -11,6 +11,7 @@ import io.minio.RemoveObjectArgs;
|
||||
import io.minio.StatObjectArgs;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.ConnectionPool;
|
||||
import okhttp3.Dispatcher;
|
||||
import okhttp3.OkHttpClient;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -22,6 +23,7 @@ import java.util.Objects;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
@@ -78,35 +80,51 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
|
||||
public String uploadText(String objectKey, String content, boolean verifyAfterUpload) {
|
||||
long deadlineNanos = operationDeadlineNanos();
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
}
|
||||
rejectIfCircuitOpen(OP_UPLOAD, objectKey);
|
||||
byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
recordPayloadBytes(bytes.length);
|
||||
return executeWithRetry(OP_UPLOAD, objectKey, bytes.length, Math.max(1, properties.getUploadMaxRetries()), uploadSemaphore, () -> {
|
||||
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
|
||||
buildClient().putObject(PutObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.stream(stream, bytes.length, -1)
|
||||
.contentType("application/json")
|
||||
.build());
|
||||
if (verifyAfterUpload) {
|
||||
verifyObjectVisible(objectKey);
|
||||
AtomicBoolean putCompleted = new AtomicBoolean();
|
||||
try {
|
||||
String uploadedObjectKey = executeWithRetry(OP_UPLOAD, objectKey, bytes.length,
|
||||
Math.max(1, properties.getUploadMaxRetries()), uploadSemaphore, deadlineNanos, () -> {
|
||||
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
|
||||
buildClient(deadlineNanos).putObject(PutObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.stream(stream, bytes.length, -1)
|
||||
.contentType("application/json")
|
||||
.build());
|
||||
putCompleted.set(true);
|
||||
return objectKey;
|
||||
}
|
||||
return objectKey;
|
||||
});
|
||||
if (verifyAfterUpload) {
|
||||
verifyObjectVisible(objectKey, deadlineNanos);
|
||||
} else {
|
||||
resetFailureWindow(OP_UPLOAD);
|
||||
}
|
||||
});
|
||||
return uploadedObjectKey;
|
||||
} catch (RuntimeException ex) {
|
||||
if (putCompleted.get()) {
|
||||
enqueueDeleteRetry(objectKey, ex);
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
public String readObjectAsString(String objectKey) {
|
||||
long deadlineNanos = operationDeadlineNanos();
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
}
|
||||
rejectIfCircuitOpen(OP_READ, objectKey);
|
||||
return executeWithRetry(OP_READ, objectKey, 0, Math.max(1, properties.getReadMaxRetries()), readSemaphore, () -> {
|
||||
try (var stream = buildClient().getObject(GetObjectArgs.builder()
|
||||
return executeWithRetry(OP_READ, objectKey, 0, Math.max(1, properties.getReadMaxRetries()),
|
||||
readSemaphore, deadlineNanos, () -> {
|
||||
try (var stream = buildClient(deadlineNanos).getObject(GetObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build())) {
|
||||
@@ -116,20 +134,25 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
|
||||
public void deleteObject(String objectKey) {
|
||||
deleteObject(objectKey, true);
|
||||
deleteObject(objectKey, true, operationDeadlineNanos());
|
||||
}
|
||||
|
||||
void deleteObjectFromRetry(String objectKey) {
|
||||
deleteObject(objectKey, false);
|
||||
deleteObject(objectKey, false, operationDeadlineNanos());
|
||||
}
|
||||
|
||||
private void deleteObject(String objectKey, boolean enqueueRetryOnFailure) {
|
||||
void deleteObjectFromRetry(String objectKey, long timeoutNanos) {
|
||||
deleteObject(objectKey, false, deadlineFromNow(timeoutNanos));
|
||||
}
|
||||
|
||||
private void deleteObject(String objectKey, boolean enqueueRetryOnFailure, long deadlineNanos) {
|
||||
if (!isConfigured() || !notBlank(objectKey)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
executeWithRetry(OP_DELETE, objectKey, 0, Math.max(1, properties.getDeleteMaxRetries()), deleteSemaphore, () -> {
|
||||
buildClient().removeObject(RemoveObjectArgs.builder()
|
||||
executeWithRetry(OP_DELETE, objectKey, 0, Math.max(1, properties.getDeleteMaxRetries()),
|
||||
deleteSemaphore, deadlineNanos, () -> {
|
||||
buildClient(deadlineNanos).removeObject(RemoveObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build());
|
||||
@@ -137,10 +160,7 @@ public class RustfsObjectStorageService {
|
||||
});
|
||||
} catch (RuntimeException ex) {
|
||||
if (enqueueRetryOnFailure && properties.isDeleteRetryEnabled()) {
|
||||
RustfsDeleteRetryService retryService = deleteRetryServiceProvider == null ? null : deleteRetryServiceProvider.getIfAvailable();
|
||||
if (retryService != null) {
|
||||
retryService.enqueue(objectKey, ex);
|
||||
}
|
||||
enqueueDeleteRetry(objectKey, ex);
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
@@ -151,48 +171,65 @@ public class RustfsObjectStorageService {
|
||||
long bytes,
|
||||
int maxRetries,
|
||||
Semaphore semaphore,
|
||||
long deadlineNanos,
|
||||
CheckedSupplier<T> supplier) {
|
||||
acquirePermit(operation, objectKey, semaphore);
|
||||
long startedAt = System.nanoTime();
|
||||
Exception last = null;
|
||||
try {
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
rejectIfCircuitOpen(operation, objectKey);
|
||||
acquirePermit(operation, objectKey, semaphore, deadlineNanos);
|
||||
long delayMillis = 0L;
|
||||
try {
|
||||
try {
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
T result = supplier.get();
|
||||
resetFailureWindow();
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
if (!OP_UPLOAD.equals(operation)) {
|
||||
resetFailureWindow(operation);
|
||||
}
|
||||
recordOperation(operation, "success", elapsedNanos(startedAt));
|
||||
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
||||
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
||||
return result;
|
||||
} catch (Exception ex) {
|
||||
last = ex;
|
||||
evictIdleConnections();
|
||||
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
||||
recordFailure(operation, objectKey, ex);
|
||||
if (attempt < maxRetries) {
|
||||
long delayMillis = retryDelayMillis(attempt);
|
||||
delayMillis = retryDelayMillis(attempt);
|
||||
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
||||
operation, objectKey, attempt, maxRetries, delayMillis, ex.getMessage());
|
||||
sleepQuietly(delayMillis, "retrying rustfs " + operation);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
semaphore.release();
|
||||
}
|
||||
if (last instanceof OperationTimeoutException timeoutException) {
|
||||
throw timeoutException;
|
||||
}
|
||||
if (attempt < maxRetries) {
|
||||
sleepQuietly(delayMillis, "retrying rustfs " + operation,
|
||||
operation, objectKey, deadlineNanos);
|
||||
}
|
||||
} finally {
|
||||
semaphore.release();
|
||||
}
|
||||
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
|
||||
}
|
||||
|
||||
private void acquirePermit(String operation, String objectKey, Semaphore semaphore) {
|
||||
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
|
||||
try {
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
boolean acquired;
|
||||
long timeoutMillis = Math.max(0L, properties.getAcquirePermitTimeoutMillis());
|
||||
if (timeoutMillis == 0L) {
|
||||
acquired = semaphore.tryAcquire();
|
||||
} else {
|
||||
acquired = semaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
|
||||
long waitNanos = Math.min(TimeUnit.MILLISECONDS.toNanos(timeoutMillis),
|
||||
remainingNanos(deadlineNanos));
|
||||
acquired = semaphore.tryAcquire(waitNanos, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
if (!acquired) {
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
recordOperation(operation, "rejected", 0L);
|
||||
throw new IllegalStateException("rustfs " + operation + " concurrency limit reached: " + objectKey);
|
||||
}
|
||||
@@ -203,7 +240,7 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private MinioClient buildClient() {
|
||||
private MinioClient buildClient(long deadlineNanos) {
|
||||
if (minioClientSupplier != null) {
|
||||
return minioClientSupplier.get();
|
||||
}
|
||||
@@ -211,7 +248,7 @@ public class RustfsObjectStorageService {
|
||||
.endpoint(properties.getEndpoint())
|
||||
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
|
||||
.region(properties.getRegion())
|
||||
.httpClient(getHttpClient())
|
||||
.httpClient(getHttpClient(deadlineNanos))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -222,10 +259,15 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
synchronized (this) {
|
||||
if (httpClient == null) {
|
||||
Dispatcher dispatcher = new Dispatcher();
|
||||
dispatcher.setMaxRequests(Math.max(1, properties.getDispatcherMaxRequests()));
|
||||
dispatcher.setMaxRequestsPerHost(Math.max(1, properties.getDispatcherMaxRequestsPerHost()));
|
||||
httpClient = new OkHttpClient.Builder()
|
||||
.dispatcher(dispatcher)
|
||||
.connectTimeout(Math.max(1, properties.getConnectTimeoutSeconds()), TimeUnit.SECONDS)
|
||||
.readTimeout(Math.max(1, properties.getReadTimeoutSeconds()), TimeUnit.SECONDS)
|
||||
.writeTimeout(Math.max(1, properties.getWriteTimeoutSeconds()), TimeUnit.SECONDS)
|
||||
.callTimeout(Math.max(1, properties.getCallTimeoutSeconds()), TimeUnit.SECONDS)
|
||||
.connectionPool(new ConnectionPool(
|
||||
Math.max(0, properties.getConnectionPoolMaxIdle()),
|
||||
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
|
||||
@@ -237,30 +279,49 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private void evictIdleConnections() {
|
||||
OkHttpClient current = httpClient;
|
||||
if (current != null) {
|
||||
current.connectionPool().evictAll();
|
||||
OkHttpClient getHttpClient(long deadlineNanos) {
|
||||
OkHttpClient client = getHttpClient();
|
||||
if (deadlineNanos == Long.MAX_VALUE) {
|
||||
return client;
|
||||
}
|
||||
long configuredTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, properties.getCallTimeoutSeconds()));
|
||||
long callTimeoutMillis = TimeUnit.NANOSECONDS.toMillis(
|
||||
Math.min(configuredTimeoutNanos, remainingNanos(deadlineNanos)));
|
||||
if (callTimeoutMillis <= 0L) {
|
||||
throw new OperationTimeoutException("rustfs operation timeout before HTTP call");
|
||||
}
|
||||
return client.newBuilder()
|
||||
.callTimeout(callTimeoutMillis, TimeUnit.MILLISECONDS)
|
||||
.build();
|
||||
}
|
||||
|
||||
private void verifyObjectVisible(String objectKey) {
|
||||
private void verifyObjectVisible(String objectKey, long deadlineNanos) {
|
||||
long startedAt = System.nanoTime();
|
||||
RuntimeException last = null;
|
||||
for (int attempt = 1; attempt <= 3; attempt++) {
|
||||
checkDeadline(OP_STAT, objectKey, deadlineNanos);
|
||||
try {
|
||||
buildClient().statObject(StatObjectArgs.builder()
|
||||
buildClient(deadlineNanos).statObject(StatObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build());
|
||||
checkDeadline(OP_STAT, objectKey, deadlineNanos);
|
||||
resetFailureWindow(OP_STAT);
|
||||
recordOperation(OP_STAT, "success", elapsedNanos(startedAt));
|
||||
return;
|
||||
} catch (Exception ex) {
|
||||
if (ex instanceof OperationTimeoutException timeoutException) {
|
||||
throw timeoutException;
|
||||
}
|
||||
last = new IllegalStateException("transient payload is not visible after upload: " + objectKey, ex);
|
||||
evictIdleConnections();
|
||||
recordOperation(OP_STAT, attempt < 3 ? "retry" : "failure", elapsedNanos(startedAt));
|
||||
recordFailure(OP_STAT, objectKey, ex);
|
||||
if (isCircuitOpen()) {
|
||||
break;
|
||||
}
|
||||
if (attempt < 3) {
|
||||
sleepQuietly(100L * attempt, "verifying transient payload upload");
|
||||
sleepQuietly(100L * attempt, "verifying transient payload upload",
|
||||
OP_STAT, objectKey, deadlineNanos);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -270,6 +331,9 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
|
||||
private void rejectIfCircuitOpen(String operation, String objectKey) {
|
||||
if (!OP_UPLOAD.equals(operation) && !OP_READ.equals(operation)) {
|
||||
return;
|
||||
}
|
||||
long openUntil = circuitOpenUntilMillis;
|
||||
if (openUntil <= 0L) {
|
||||
return;
|
||||
@@ -282,6 +346,9 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
|
||||
private void recordFailure(String operation, String objectKey, Exception ex) {
|
||||
if (!participatesInFailureWindow(operation)) {
|
||||
return;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
long windowMillis = Math.max(1L, properties.getFailureWindowSeconds()) * 1000L;
|
||||
if (failureWindowStartedAtMillis <= 0L || now - failureWindowStartedAtMillis > windowMillis) {
|
||||
@@ -290,7 +357,7 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
int failures = windowFailureCount.incrementAndGet();
|
||||
int threshold = Math.max(1, properties.getFailureWindowThreshold());
|
||||
if ((OP_UPLOAD.equals(operation) || OP_READ.equals(operation)) && failures >= threshold) {
|
||||
if (failures >= threshold) {
|
||||
long cooldownMillis = Math.max(0L, properties.getFailureCooldownMillis());
|
||||
circuitOpenUntilMillis = now + cooldownMillis;
|
||||
windowFailureCount.set(0);
|
||||
@@ -300,7 +367,10 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private void resetFailureWindow() {
|
||||
private void resetFailureWindow(String operation) {
|
||||
if (!participatesInFailureWindow(operation)) {
|
||||
return;
|
||||
}
|
||||
windowFailureCount.set(0);
|
||||
failureWindowStartedAtMillis = 0L;
|
||||
circuitOpenUntilMillis = 0L;
|
||||
@@ -317,16 +387,70 @@ public class RustfsObjectStorageService {
|
||||
return delay;
|
||||
}
|
||||
|
||||
private void sleepQuietly(long millis, String reason) {
|
||||
private void sleepQuietly(long millis,
|
||||
String reason,
|
||||
String operation,
|
||||
String objectKey,
|
||||
long deadlineNanos) {
|
||||
if (millis <= 0L) {
|
||||
return;
|
||||
}
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
long sleepNanos = Math.min(TimeUnit.MILLISECONDS.toNanos(millis), remainingNanos(deadlineNanos));
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
TimeUnit.NANOSECONDS.sleep(sleepNanos);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("interrupted while " + reason, ex);
|
||||
}
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
}
|
||||
|
||||
private long operationDeadlineNanos() {
|
||||
long timeoutSeconds = properties.getOperationTimeoutSeconds();
|
||||
return timeoutSeconds <= 0L
|
||||
? Long.MAX_VALUE
|
||||
: deadlineFromNow(TimeUnit.SECONDS.toNanos(timeoutSeconds));
|
||||
}
|
||||
|
||||
private long deadlineFromNow(long timeoutNanos) {
|
||||
long boundedTimeoutNanos = Math.min(Math.max(0L, timeoutNanos), Long.MAX_VALUE / 2L);
|
||||
return System.nanoTime() + boundedTimeoutNanos;
|
||||
}
|
||||
|
||||
private long remainingNanos(long deadlineNanos) {
|
||||
return deadlineNanos == Long.MAX_VALUE ? Long.MAX_VALUE : Math.max(0L, deadlineNanos - System.nanoTime());
|
||||
}
|
||||
|
||||
private void checkDeadline(String operation, String objectKey, long deadlineNanos) {
|
||||
if (remainingNanos(deadlineNanos) <= 0L) {
|
||||
throw new OperationTimeoutException(
|
||||
"rustfs operation timeout operation=" + operation + " objectKey=" + objectKey);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean participatesInFailureWindow(String operation) {
|
||||
return OP_UPLOAD.equals(operation) || OP_READ.equals(operation) || OP_STAT.equals(operation);
|
||||
}
|
||||
|
||||
private boolean isCircuitOpen() {
|
||||
return System.currentTimeMillis() < circuitOpenUntilMillis;
|
||||
}
|
||||
|
||||
private void enqueueDeleteRetry(String objectKey, RuntimeException failure) {
|
||||
if (!properties.isDeleteRetryEnabled() || deleteRetryServiceProvider == null) {
|
||||
return;
|
||||
}
|
||||
RustfsDeleteRetryService retryService = deleteRetryServiceProvider.getIfAvailable();
|
||||
if (retryService == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
retryService.enqueue(objectKey, failure);
|
||||
} catch (RuntimeException enqueueFailure) {
|
||||
log.error("[rustfs] failed to enqueue delete compensation objectKey={} err={}",
|
||||
objectKey, enqueueFailure.getMessage(), enqueueFailure);
|
||||
}
|
||||
}
|
||||
|
||||
private void recordPayloadBytes(long bytes) {
|
||||
@@ -382,4 +506,10 @@ public class RustfsObjectStorageService {
|
||||
private interface CheckedSupplier<T> {
|
||||
T get() throws Exception;
|
||||
}
|
||||
|
||||
private static class OperationTimeoutException extends IllegalStateException {
|
||||
private OperationTimeoutException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user