完善任务存储、权限及货源查询流程

This commit is contained in:
supernijia
2026-08-14 22:49:15 +08:00
parent 91cd02eeb4
commit cf59e23f15
21 changed files with 2584 additions and 566 deletions
@@ -7,8 +7,13 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@@ -18,18 +23,30 @@ public class RustfsDeleteRetryService {
private final TransientStorageProperties properties;
private final RustfsObjectStorageService rustfsObjectStorageService;
private final ConcurrentLinkedQueue<String> queue = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<RetryItem> queue = new ConcurrentLinkedQueue<>();
private final ConcurrentHashMap<String, RetryItem> pending = new ConcurrentHashMap<>();
private final AtomicInteger queuedCount = new AtomicInteger();
private final Object admissionLock = new Object();
public void enqueue(String objectKey, Throwable cause) {
if (!properties.isDeleteRetryEnabled() || objectKey == null || objectKey.isBlank()) {
return;
}
RetryItem item = pending.computeIfAbsent(objectKey, RetryItem::new);
item.lastError = cause == null ? null : cause.getMessage();
item.lastFailedAt = LocalDateTime.now();
enqueueItem(item, false);
while (true) {
RetryItem item = findOrAdmit(objectKey, cause);
if (item == null) {
return;
}
synchronized (item) {
if (pending.get(objectKey) != item) {
continue;
}
item.generation++;
item.lastError = cause == null ? null : cause.getMessage();
item.lastFailedAt = LocalDateTime.now();
enqueueItemLocked(item);
return;
}
}
}
@Scheduled(cron = "${aiimage.transient-storage.delete-retry-cron:0 */5 * * * *}")
@@ -38,36 +55,50 @@ public class RustfsDeleteRetryService {
return;
}
int batchSize = Math.max(1, properties.getDeleteRetryBatchSize());
long batchDurationNanos = TimeUnit.SECONDS.toNanos(
Math.max(1L, properties.getDeleteRetryBatchTimeoutSeconds()));
long deadlineNanos = System.nanoTime() + batchDurationNanos;
int processed = 0;
int success = 0;
int failed = 0;
Set<String> attemptedObjectKeys = new HashSet<>();
List<RetryItem> retryNextInvocation = new ArrayList<>();
while (processed < batchSize) {
String objectKey = nextQueuedObjectKey(batchSize - processed);
if (objectKey == null) {
long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0L) {
break;
}
RetryItem item = pending.get(objectKey);
if (item == null) {
RetryAttempt attempt = nextAttempt();
if (attempt == null) {
break;
}
RetryItem item = attempt.item;
String objectKey = item.objectKey;
if (!attemptedObjectKeys.add(objectKey)) {
retryNextInvocation.add(item);
continue;
}
item.queued = false;
remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0L) {
retryNextInvocation.add(item);
break;
}
processed++;
try {
rustfsObjectStorageService.deleteObjectFromRetry(objectKey);
pending.remove(objectKey);
rustfsObjectStorageService.deleteObjectFromRetry(objectKey, remainingNanos);
removeIfUnchanged(attempt);
success++;
} catch (Exception ex) {
failed++;
int failures = item.failures.incrementAndGet();
item.lastError = ex.getMessage();
item.lastFailedAt = LocalDateTime.now();
int failures = recordFailureIfUnchanged(attempt, ex);
log.warn("[rustfs] delete retry failed objectKey={} failures={} err={}", objectKey, failures, ex.getMessage());
enqueueItem(item, true);
retryNextInvocation.add(item);
}
}
retryNextInvocation.forEach(this::requeue);
if (processed > 0) {
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={} queued={}",
processed, success, failed, pending.size(), queuedCount.get());
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={}",
processed, success, failed, pending.size());
}
}
@@ -75,72 +106,94 @@ public class RustfsDeleteRetryService {
return pending.size();
}
private String nextQueuedObjectKey(int refillLimit) {
String objectKey = queue.poll();
if (objectKey != null) {
decrementQueuedCount();
refillQueue(refillLimit);
return objectKey;
}
if (refillQueue(refillLimit) <= 0) {
return null;
}
objectKey = queue.poll();
if (objectKey != null) {
decrementQueuedCount();
refillQueue(refillLimit);
}
return objectKey;
}
private int refillQueue(int maxItems) {
if (maxItems <= 0) {
return 0;
private RetryItem findOrAdmit(String objectKey, Throwable cause) {
RetryItem item = pending.get(objectKey);
if (item != null) {
return item;
}
int capacity = Math.max(1, properties.getDeleteRetryQueueCapacity());
if (queuedCount.get() >= capacity) {
return 0;
}
int added = 0;
for (RetryItem item : pending.values()) {
if (added >= maxItems || queuedCount.get() >= capacity) {
break;
synchronized (admissionLock) {
item = pending.get(objectKey);
if (item != null) {
return item;
}
if (enqueueItem(item, true)) {
added++;
if (pending.size() >= capacity) {
log.error("[rustfs] delete retry capacity reached, rejected objectKey={} pending={} capacity={} lastError={}",
objectKey, pending.size(), capacity, cause == null ? null : cause.getMessage());
return null;
}
item = new RetryItem(objectKey);
pending.put(objectKey, item);
return item;
}
return added;
}
private void decrementQueuedCount() {
queuedCount.updateAndGet(value -> Math.max(0, value - 1));
private RetryAttempt nextAttempt() {
while (true) {
RetryItem item = queue.poll();
if (item == null) {
return null;
}
synchronized (item) {
if (pending.get(item.objectKey) != item || !item.queued) {
continue;
}
item.queued = false;
return new RetryAttempt(item, item.generation);
}
}
}
private boolean enqueueItem(RetryItem item, boolean requeue) {
private void removeIfUnchanged(RetryAttempt attempt) {
RetryItem item = attempt.item;
synchronized (item) {
if (item.queued) {
return false;
if (pending.get(item.objectKey) != item || item.generation != attempt.generation) {
return;
}
int capacity = Math.max(1, properties.getDeleteRetryQueueCapacity());
if (queuedCount.get() >= capacity) {
log.error("[rustfs] delete retry queue full, deferred objectKey={} requeue={} pending={} queued={} capacity={} lastError={}",
item.objectKey, requeue, pending.size(), queuedCount.get(), capacity, item.lastError);
return false;
synchronized (admissionLock) {
pending.remove(item.objectKey, item);
}
item.queued = true;
queue.offer(item.objectKey);
queuedCount.incrementAndGet();
return true;
}
}
private int recordFailureIfUnchanged(RetryAttempt attempt, Exception ex) {
RetryItem item = attempt.item;
synchronized (item) {
if (pending.get(item.objectKey) != item || item.generation != attempt.generation) {
return item.failures.get();
}
int failures = item.failures.incrementAndGet();
item.lastError = ex.getMessage();
item.lastFailedAt = LocalDateTime.now();
return failures;
}
}
private void requeue(RetryItem item) {
synchronized (item) {
if (pending.get(item.objectKey) == item) {
enqueueItemLocked(item);
}
}
}
private void enqueueItemLocked(RetryItem item) {
if (!item.queued) {
item.queued = true;
queue.offer(item);
}
}
private record RetryAttempt(RetryItem item, long generation) {
}
private static class RetryItem {
private final String objectKey;
private final AtomicInteger failures = new AtomicInteger();
private volatile boolean queued;
private volatile String lastError;
private volatile LocalDateTime lastFailedAt;
private boolean queued;
private long generation;
private String lastError;
private LocalDateTime lastFailedAt;
private RetryItem(String objectKey) {
this.objectKey = objectKey;
@@ -11,6 +11,7 @@ import io.minio.RemoveObjectArgs;
import io.minio.StatObjectArgs;
import lombok.extern.slf4j.Slf4j;
import okhttp3.ConnectionPool;
import okhttp3.Dispatcher;
import okhttp3.OkHttpClient;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Autowired;
@@ -22,6 +23,7 @@ import java.util.Objects;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@@ -78,35 +80,51 @@ public class RustfsObjectStorageService {
}
public String uploadText(String objectKey, String content, boolean verifyAfterUpload) {
long deadlineNanos = operationDeadlineNanos();
if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured");
}
rejectIfCircuitOpen(OP_UPLOAD, objectKey);
byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
recordPayloadBytes(bytes.length);
return executeWithRetry(OP_UPLOAD, objectKey, bytes.length, Math.max(1, properties.getUploadMaxRetries()), uploadSemaphore, () -> {
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
buildClient().putObject(PutObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.stream(stream, bytes.length, -1)
.contentType("application/json")
.build());
if (verifyAfterUpload) {
verifyObjectVisible(objectKey);
AtomicBoolean putCompleted = new AtomicBoolean();
try {
String uploadedObjectKey = executeWithRetry(OP_UPLOAD, objectKey, bytes.length,
Math.max(1, properties.getUploadMaxRetries()), uploadSemaphore, deadlineNanos, () -> {
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
buildClient(deadlineNanos).putObject(PutObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.stream(stream, bytes.length, -1)
.contentType("application/json")
.build());
putCompleted.set(true);
return objectKey;
}
return objectKey;
});
if (verifyAfterUpload) {
verifyObjectVisible(objectKey, deadlineNanos);
} else {
resetFailureWindow(OP_UPLOAD);
}
});
return uploadedObjectKey;
} catch (RuntimeException ex) {
if (putCompleted.get()) {
enqueueDeleteRetry(objectKey, ex);
}
throw ex;
}
}
public String readObjectAsString(String objectKey) {
long deadlineNanos = operationDeadlineNanos();
if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured");
}
rejectIfCircuitOpen(OP_READ, objectKey);
return executeWithRetry(OP_READ, objectKey, 0, Math.max(1, properties.getReadMaxRetries()), readSemaphore, () -> {
try (var stream = buildClient().getObject(GetObjectArgs.builder()
return executeWithRetry(OP_READ, objectKey, 0, Math.max(1, properties.getReadMaxRetries()),
readSemaphore, deadlineNanos, () -> {
try (var stream = buildClient(deadlineNanos).getObject(GetObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.build())) {
@@ -116,20 +134,25 @@ public class RustfsObjectStorageService {
}
public void deleteObject(String objectKey) {
deleteObject(objectKey, true);
deleteObject(objectKey, true, operationDeadlineNanos());
}
void deleteObjectFromRetry(String objectKey) {
deleteObject(objectKey, false);
deleteObject(objectKey, false, operationDeadlineNanos());
}
private void deleteObject(String objectKey, boolean enqueueRetryOnFailure) {
void deleteObjectFromRetry(String objectKey, long timeoutNanos) {
deleteObject(objectKey, false, deadlineFromNow(timeoutNanos));
}
private void deleteObject(String objectKey, boolean enqueueRetryOnFailure, long deadlineNanos) {
if (!isConfigured() || !notBlank(objectKey)) {
return;
}
try {
executeWithRetry(OP_DELETE, objectKey, 0, Math.max(1, properties.getDeleteMaxRetries()), deleteSemaphore, () -> {
buildClient().removeObject(RemoveObjectArgs.builder()
executeWithRetry(OP_DELETE, objectKey, 0, Math.max(1, properties.getDeleteMaxRetries()),
deleteSemaphore, deadlineNanos, () -> {
buildClient(deadlineNanos).removeObject(RemoveObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.build());
@@ -137,10 +160,7 @@ public class RustfsObjectStorageService {
});
} catch (RuntimeException ex) {
if (enqueueRetryOnFailure && properties.isDeleteRetryEnabled()) {
RustfsDeleteRetryService retryService = deleteRetryServiceProvider == null ? null : deleteRetryServiceProvider.getIfAvailable();
if (retryService != null) {
retryService.enqueue(objectKey, ex);
}
enqueueDeleteRetry(objectKey, ex);
}
throw ex;
}
@@ -151,48 +171,65 @@ public class RustfsObjectStorageService {
long bytes,
int maxRetries,
Semaphore semaphore,
long deadlineNanos,
CheckedSupplier<T> supplier) {
acquirePermit(operation, objectKey, semaphore);
long startedAt = System.nanoTime();
Exception last = null;
try {
for (int attempt = 1; attempt <= maxRetries; attempt++) {
for (int attempt = 1; attempt <= maxRetries; attempt++) {
checkDeadline(operation, objectKey, deadlineNanos);
rejectIfCircuitOpen(operation, objectKey);
acquirePermit(operation, objectKey, semaphore, deadlineNanos);
long delayMillis = 0L;
try {
try {
checkDeadline(operation, objectKey, deadlineNanos);
T result = supplier.get();
resetFailureWindow();
checkDeadline(operation, objectKey, deadlineNanos);
if (!OP_UPLOAD.equals(operation)) {
resetFailureWindow(operation);
}
recordOperation(operation, "success", elapsedNanos(startedAt));
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
return result;
} catch (Exception ex) {
last = ex;
evictIdleConnections();
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
recordFailure(operation, objectKey, ex);
if (attempt < maxRetries) {
long delayMillis = retryDelayMillis(attempt);
delayMillis = retryDelayMillis(attempt);
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
operation, objectKey, attempt, maxRetries, delayMillis, ex.getMessage());
sleepQuietly(delayMillis, "retrying rustfs " + operation);
}
}
} finally {
semaphore.release();
}
if (last instanceof OperationTimeoutException timeoutException) {
throw timeoutException;
}
if (attempt < maxRetries) {
sleepQuietly(delayMillis, "retrying rustfs " + operation,
operation, objectKey, deadlineNanos);
}
} finally {
semaphore.release();
}
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
}
private void acquirePermit(String operation, String objectKey, Semaphore semaphore) {
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
try {
checkDeadline(operation, objectKey, deadlineNanos);
boolean acquired;
long timeoutMillis = Math.max(0L, properties.getAcquirePermitTimeoutMillis());
if (timeoutMillis == 0L) {
acquired = semaphore.tryAcquire();
} else {
acquired = semaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
long waitNanos = Math.min(TimeUnit.MILLISECONDS.toNanos(timeoutMillis),
remainingNanos(deadlineNanos));
acquired = semaphore.tryAcquire(waitNanos, TimeUnit.NANOSECONDS);
}
if (!acquired) {
checkDeadline(operation, objectKey, deadlineNanos);
recordOperation(operation, "rejected", 0L);
throw new IllegalStateException("rustfs " + operation + " concurrency limit reached: " + objectKey);
}
@@ -203,7 +240,7 @@ public class RustfsObjectStorageService {
}
}
private MinioClient buildClient() {
private MinioClient buildClient(long deadlineNanos) {
if (minioClientSupplier != null) {
return minioClientSupplier.get();
}
@@ -211,7 +248,7 @@ public class RustfsObjectStorageService {
.endpoint(properties.getEndpoint())
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
.region(properties.getRegion())
.httpClient(getHttpClient())
.httpClient(getHttpClient(deadlineNanos))
.build();
}
@@ -222,10 +259,15 @@ public class RustfsObjectStorageService {
}
synchronized (this) {
if (httpClient == null) {
Dispatcher dispatcher = new Dispatcher();
dispatcher.setMaxRequests(Math.max(1, properties.getDispatcherMaxRequests()));
dispatcher.setMaxRequestsPerHost(Math.max(1, properties.getDispatcherMaxRequestsPerHost()));
httpClient = new OkHttpClient.Builder()
.dispatcher(dispatcher)
.connectTimeout(Math.max(1, properties.getConnectTimeoutSeconds()), TimeUnit.SECONDS)
.readTimeout(Math.max(1, properties.getReadTimeoutSeconds()), TimeUnit.SECONDS)
.writeTimeout(Math.max(1, properties.getWriteTimeoutSeconds()), TimeUnit.SECONDS)
.callTimeout(Math.max(1, properties.getCallTimeoutSeconds()), TimeUnit.SECONDS)
.connectionPool(new ConnectionPool(
Math.max(0, properties.getConnectionPoolMaxIdle()),
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
@@ -237,30 +279,49 @@ public class RustfsObjectStorageService {
}
}
private void evictIdleConnections() {
OkHttpClient current = httpClient;
if (current != null) {
current.connectionPool().evictAll();
OkHttpClient getHttpClient(long deadlineNanos) {
OkHttpClient client = getHttpClient();
if (deadlineNanos == Long.MAX_VALUE) {
return client;
}
long configuredTimeoutNanos = TimeUnit.SECONDS.toNanos(Math.max(1, properties.getCallTimeoutSeconds()));
long callTimeoutMillis = TimeUnit.NANOSECONDS.toMillis(
Math.min(configuredTimeoutNanos, remainingNanos(deadlineNanos)));
if (callTimeoutMillis <= 0L) {
throw new OperationTimeoutException("rustfs operation timeout before HTTP call");
}
return client.newBuilder()
.callTimeout(callTimeoutMillis, TimeUnit.MILLISECONDS)
.build();
}
private void verifyObjectVisible(String objectKey) {
private void verifyObjectVisible(String objectKey, long deadlineNanos) {
long startedAt = System.nanoTime();
RuntimeException last = null;
for (int attempt = 1; attempt <= 3; attempt++) {
checkDeadline(OP_STAT, objectKey, deadlineNanos);
try {
buildClient().statObject(StatObjectArgs.builder()
buildClient(deadlineNanos).statObject(StatObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.build());
checkDeadline(OP_STAT, objectKey, deadlineNanos);
resetFailureWindow(OP_STAT);
recordOperation(OP_STAT, "success", elapsedNanos(startedAt));
return;
} catch (Exception ex) {
if (ex instanceof OperationTimeoutException timeoutException) {
throw timeoutException;
}
last = new IllegalStateException("transient payload is not visible after upload: " + objectKey, ex);
evictIdleConnections();
recordOperation(OP_STAT, attempt < 3 ? "retry" : "failure", elapsedNanos(startedAt));
recordFailure(OP_STAT, objectKey, ex);
if (isCircuitOpen()) {
break;
}
if (attempt < 3) {
sleepQuietly(100L * attempt, "verifying transient payload upload");
sleepQuietly(100L * attempt, "verifying transient payload upload",
OP_STAT, objectKey, deadlineNanos);
}
}
}
@@ -270,6 +331,9 @@ public class RustfsObjectStorageService {
}
private void rejectIfCircuitOpen(String operation, String objectKey) {
if (!OP_UPLOAD.equals(operation) && !OP_READ.equals(operation)) {
return;
}
long openUntil = circuitOpenUntilMillis;
if (openUntil <= 0L) {
return;
@@ -282,6 +346,9 @@ public class RustfsObjectStorageService {
}
private void recordFailure(String operation, String objectKey, Exception ex) {
if (!participatesInFailureWindow(operation)) {
return;
}
long now = System.currentTimeMillis();
long windowMillis = Math.max(1L, properties.getFailureWindowSeconds()) * 1000L;
if (failureWindowStartedAtMillis <= 0L || now - failureWindowStartedAtMillis > windowMillis) {
@@ -290,7 +357,7 @@ public class RustfsObjectStorageService {
}
int failures = windowFailureCount.incrementAndGet();
int threshold = Math.max(1, properties.getFailureWindowThreshold());
if ((OP_UPLOAD.equals(operation) || OP_READ.equals(operation)) && failures >= threshold) {
if (failures >= threshold) {
long cooldownMillis = Math.max(0L, properties.getFailureCooldownMillis());
circuitOpenUntilMillis = now + cooldownMillis;
windowFailureCount.set(0);
@@ -300,7 +367,10 @@ public class RustfsObjectStorageService {
}
}
private void resetFailureWindow() {
private void resetFailureWindow(String operation) {
if (!participatesInFailureWindow(operation)) {
return;
}
windowFailureCount.set(0);
failureWindowStartedAtMillis = 0L;
circuitOpenUntilMillis = 0L;
@@ -317,16 +387,70 @@ public class RustfsObjectStorageService {
return delay;
}
private void sleepQuietly(long millis, String reason) {
private void sleepQuietly(long millis,
String reason,
String operation,
String objectKey,
long deadlineNanos) {
if (millis <= 0L) {
return;
}
checkDeadline(operation, objectKey, deadlineNanos);
long sleepNanos = Math.min(TimeUnit.MILLISECONDS.toNanos(millis), remainingNanos(deadlineNanos));
try {
Thread.sleep(millis);
TimeUnit.NANOSECONDS.sleep(sleepNanos);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException("interrupted while " + reason, ex);
}
checkDeadline(operation, objectKey, deadlineNanos);
}
private long operationDeadlineNanos() {
long timeoutSeconds = properties.getOperationTimeoutSeconds();
return timeoutSeconds <= 0L
? Long.MAX_VALUE
: deadlineFromNow(TimeUnit.SECONDS.toNanos(timeoutSeconds));
}
private long deadlineFromNow(long timeoutNanos) {
long boundedTimeoutNanos = Math.min(Math.max(0L, timeoutNanos), Long.MAX_VALUE / 2L);
return System.nanoTime() + boundedTimeoutNanos;
}
private long remainingNanos(long deadlineNanos) {
return deadlineNanos == Long.MAX_VALUE ? Long.MAX_VALUE : Math.max(0L, deadlineNanos - System.nanoTime());
}
private void checkDeadline(String operation, String objectKey, long deadlineNanos) {
if (remainingNanos(deadlineNanos) <= 0L) {
throw new OperationTimeoutException(
"rustfs operation timeout operation=" + operation + " objectKey=" + objectKey);
}
}
private boolean participatesInFailureWindow(String operation) {
return OP_UPLOAD.equals(operation) || OP_READ.equals(operation) || OP_STAT.equals(operation);
}
private boolean isCircuitOpen() {
return System.currentTimeMillis() < circuitOpenUntilMillis;
}
private void enqueueDeleteRetry(String objectKey, RuntimeException failure) {
if (!properties.isDeleteRetryEnabled() || deleteRetryServiceProvider == null) {
return;
}
RustfsDeleteRetryService retryService = deleteRetryServiceProvider.getIfAvailable();
if (retryService == null) {
return;
}
try {
retryService.enqueue(objectKey, failure);
} catch (RuntimeException enqueueFailure) {
log.error("[rustfs] failed to enqueue delete compensation objectKey={} err={}",
objectKey, enqueueFailure.getMessage(), enqueueFailure);
}
}
private void recordPayloadBytes(long bytes) {
@@ -382,4 +506,10 @@ public class RustfsObjectStorageService {
private interface CheckedSupplier<T> {
T get() throws Exception;
}
private static class OperationTimeoutException extends IllegalStateException {
private OperationTimeoutException(String message) {
super(message);
}
}
}