新增内容，新增拉起软件层
This commit is contained in:
super
+149
@@ -0,0 +1,149 @@
|
||||
package com.nanri.aiimage.modules.file.service.object;
|
||||
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class RustfsDeleteRetryService {
|
||||
|
||||
private final TransientStorageProperties properties;
|
||||
private final RustfsObjectStorageService rustfsObjectStorageService;
|
||||
private final ConcurrentLinkedQueue<String> queue = new ConcurrentLinkedQueue<>();
|
||||
private final ConcurrentHashMap<String, RetryItem> pending = new ConcurrentHashMap<>();
|
||||
private final AtomicInteger queuedCount = new AtomicInteger();
|
||||
|
||||
public void enqueue(String objectKey, Throwable cause) {
|
||||
if (!properties.isDeleteRetryEnabled() || objectKey == null || objectKey.isBlank()) {
|
||||
return;
|
||||
}
|
||||
RetryItem item = pending.computeIfAbsent(objectKey, RetryItem::new);
|
||||
item.lastError = cause == null ? null : cause.getMessage();
|
||||
item.lastFailedAt = LocalDateTime.now();
|
||||
enqueueItem(item, false);
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${aiimage.transient-storage.delete-retry-cron:0 */5 * * * *}")
|
||||
public void retryPendingDeletes() {
|
||||
if (!properties.isDeleteRetryEnabled()) {
|
||||
return;
|
||||
}
|
||||
int batchSize = Math.max(1, properties.getDeleteRetryBatchSize());
|
||||
int processed = 0;
|
||||
int success = 0;
|
||||
int failed = 0;
|
||||
while (processed < batchSize) {
|
||||
String objectKey = nextQueuedObjectKey(batchSize - processed);
|
||||
if (objectKey == null) {
|
||||
break;
|
||||
}
|
||||
RetryItem item = pending.get(objectKey);
|
||||
if (item == null) {
|
||||
continue;
|
||||
}
|
||||
item.queued = false;
|
||||
processed++;
|
||||
try {
|
||||
rustfsObjectStorageService.deleteObjectFromRetry(objectKey);
|
||||
pending.remove(objectKey);
|
||||
success++;
|
||||
} catch (Exception ex) {
|
||||
failed++;
|
||||
int failures = item.failures.incrementAndGet();
|
||||
item.lastError = ex.getMessage();
|
||||
item.lastFailedAt = LocalDateTime.now();
|
||||
log.warn("[rustfs] delete retry failed objectKey={} failures={} err={}", objectKey, failures, ex.getMessage());
|
||||
enqueueItem(item, true);
|
||||
}
|
||||
}
|
||||
if (processed > 0) {
|
||||
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={} queued={}",
|
||||
processed, success, failed, pending.size(), queuedCount.get());
|
||||
}
|
||||
}
|
||||
|
||||
int pendingCount() {
|
||||
return pending.size();
|
||||
}
|
||||
|
||||
private String nextQueuedObjectKey(int refillLimit) {
|
||||
String objectKey = queue.poll();
|
||||
if (objectKey != null) {
|
||||
decrementQueuedCount();
|
||||
refillQueue(refillLimit);
|
||||
return objectKey;
|
||||
}
|
||||
if (refillQueue(refillLimit) <= 0) {
|
||||
return null;
|
||||
}
|
||||
objectKey = queue.poll();
|
||||
if (objectKey != null) {
|
||||
decrementQueuedCount();
|
||||
refillQueue(refillLimit);
|
||||
}
|
||||
return objectKey;
|
||||
}
|
||||
|
||||
private int refillQueue(int maxItems) {
|
||||
if (maxItems <= 0) {
|
||||
return 0;
|
||||
}
|
||||
int capacity = Math.max(1, properties.getDeleteRetryQueueCapacity());
|
||||
if (queuedCount.get() >= capacity) {
|
||||
return 0;
|
||||
}
|
||||
int added = 0;
|
||||
for (RetryItem item : pending.values()) {
|
||||
if (added >= maxItems || queuedCount.get() >= capacity) {
|
||||
break;
|
||||
}
|
||||
if (enqueueItem(item, true)) {
|
||||
added++;
|
||||
}
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
private void decrementQueuedCount() {
|
||||
queuedCount.updateAndGet(value -> Math.max(0, value - 1));
|
||||
}
|
||||
|
||||
private boolean enqueueItem(RetryItem item, boolean requeue) {
|
||||
synchronized (item) {
|
||||
if (item.queued) {
|
||||
return false;
|
||||
}
|
||||
int capacity = Math.max(1, properties.getDeleteRetryQueueCapacity());
|
||||
if (queuedCount.get() >= capacity) {
|
||||
log.error("[rustfs] delete retry queue full, deferred objectKey={} requeue={} pending={} queued={} capacity={} lastError={}",
|
||||
item.objectKey, requeue, pending.size(), queuedCount.get(), capacity, item.lastError);
|
||||
return false;
|
||||
}
|
||||
item.queued = true;
|
||||
queue.offer(item.objectKey);
|
||||
queuedCount.incrementAndGet();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RetryItem {
|
||||
private final String objectKey;
|
||||
private final AtomicInteger failures = new AtomicInteger();
|
||||
private volatile boolean queued;
|
||||
private volatile String lastError;
|
||||
private volatile LocalDateTime lastFailedAt;
|
||||
|
||||
private RetryItem(String objectKey) {
|
||||
this.objectKey = objectKey;
|
||||
}
|
||||
}
|
||||
}
|
||||
+255
-35
@@ -1,29 +1,71 @@
|
||||
package com.nanri.aiimage.modules.file.service.object;
|
||||
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import io.micrometer.core.instrument.DistributionSummary;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import io.minio.GetObjectArgs;
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.PutObjectArgs;
|
||||
import io.minio.RemoveObjectArgs;
|
||||
import io.minio.StatObjectArgs;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.ConnectionPool;
|
||||
import okhttp3.OkHttpClient;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class RustfsObjectStorageService {
|
||||
|
||||
private static final String OP_UPLOAD = "upload";
|
||||
private static final String OP_READ = "read";
|
||||
private static final String OP_DELETE = "delete";
|
||||
private static final String OP_STAT = "stat";
|
||||
|
||||
private final TransientStorageProperties properties;
|
||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
||||
private final Supplier<MinioClient> minioClientSupplier;
|
||||
private final Semaphore uploadSemaphore;
|
||||
private final Semaphore readSemaphore;
|
||||
private final Semaphore deleteSemaphore;
|
||||
private final AtomicInteger windowFailureCount = new AtomicInteger();
|
||||
private volatile long failureWindowStartedAtMillis;
|
||||
private volatile long circuitOpenUntilMillis;
|
||||
private volatile OkHttpClient httpClient;
|
||||
|
||||
@Autowired
|
||||
public RustfsObjectStorageService(TransientStorageProperties properties,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider,
|
||||
ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider) {
|
||||
this(properties, meterRegistryProvider, deleteRetryServiceProvider, null);
|
||||
}
|
||||
|
||||
RustfsObjectStorageService(TransientStorageProperties properties,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider,
|
||||
ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider,
|
||||
Supplier<MinioClient> minioClientSupplier) {
|
||||
this.properties = properties;
|
||||
this.meterRegistryProvider = meterRegistryProvider;
|
||||
this.deleteRetryServiceProvider = deleteRetryServiceProvider;
|
||||
this.minioClientSupplier = minioClientSupplier;
|
||||
this.uploadSemaphore = new Semaphore(Math.max(1, properties.getMaxConcurrentUploads()));
|
||||
this.readSemaphore = new Semaphore(Math.max(1, properties.getMaxConcurrentReads()));
|
||||
this.deleteSemaphore = new Semaphore(Math.max(1, properties.getMaxConcurrentDeletes()));
|
||||
}
|
||||
|
||||
public boolean isConfigured() {
|
||||
return notBlank(properties.getEndpoint())
|
||||
&& notBlank(properties.getBucket())
|
||||
@@ -39,10 +81,10 @@ public class RustfsObjectStorageService {
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
}
|
||||
rejectIfCircuitOpen(OP_UPLOAD, objectKey);
|
||||
byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
Exception last = null;
|
||||
int maxRetries = Math.max(1, properties.getUploadMaxRetries());
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
recordPayloadBytes(bytes.length);
|
||||
return executeWithRetry(OP_UPLOAD, objectKey, bytes.length, Math.max(1, properties.getUploadMaxRetries()), uploadSemaphore, () -> {
|
||||
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
|
||||
buildClient().putObject(PutObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
@@ -54,56 +96,117 @@ public class RustfsObjectStorageService {
|
||||
verifyObjectVisible(objectKey);
|
||||
}
|
||||
return objectKey;
|
||||
} catch (Exception ex) {
|
||||
last = ex;
|
||||
if (attempt < maxRetries) {
|
||||
log.warn("[rustfs] upload failed, retrying objectKey={} attempt={}/{}", objectKey, attempt, maxRetries, ex);
|
||||
sleepQuietly(500L * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException("failed to upload payload to transient storage", last);
|
||||
});
|
||||
}
|
||||
|
||||
public String readObjectAsString(String objectKey) {
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
}
|
||||
Exception last = null;
|
||||
int maxRetries = Math.max(1, properties.getReadMaxRetries());
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
rejectIfCircuitOpen(OP_READ, objectKey);
|
||||
return executeWithRetry(OP_READ, objectKey, 0, Math.max(1, properties.getReadMaxRetries()), readSemaphore, () -> {
|
||||
try (var stream = buildClient().getObject(GetObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build())) {
|
||||
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
last = ex;
|
||||
if (attempt < maxRetries) {
|
||||
// 与 uploadText 对齐的线性退避，避免 rustfs 短暂抖动直接打成 read coze batch failed。
|
||||
log.warn("[rustfs] read failed, retrying objectKey={} attempt={}/{} err={}", objectKey, attempt, maxRetries, ex.getMessage());
|
||||
sleepQuietly(500L * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException("failed to read payload from transient storage", last);
|
||||
});
|
||||
}
|
||||
|
||||
public void deleteObject(String objectKey) {
|
||||
deleteObject(objectKey, true);
|
||||
}
|
||||
|
||||
void deleteObjectFromRetry(String objectKey) {
|
||||
deleteObject(objectKey, false);
|
||||
}
|
||||
|
||||
private void deleteObject(String objectKey, boolean enqueueRetryOnFailure) {
|
||||
if (!isConfigured() || !notBlank(objectKey)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
buildClient().removeObject(RemoveObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build());
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to delete payload from transient storage", ex);
|
||||
executeWithRetry(OP_DELETE, objectKey, 0, Math.max(1, properties.getDeleteMaxRetries()), deleteSemaphore, () -> {
|
||||
buildClient().removeObject(RemoveObjectArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build());
|
||||
return null;
|
||||
});
|
||||
} catch (RuntimeException ex) {
|
||||
if (enqueueRetryOnFailure && properties.isDeleteRetryEnabled()) {
|
||||
RustfsDeleteRetryService retryService = deleteRetryServiceProvider == null ? null : deleteRetryServiceProvider.getIfAvailable();
|
||||
if (retryService != null) {
|
||||
retryService.enqueue(objectKey, ex);
|
||||
}
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
private <T> T executeWithRetry(String operation,
|
||||
String objectKey,
|
||||
long bytes,
|
||||
int maxRetries,
|
||||
Semaphore semaphore,
|
||||
CheckedSupplier<T> supplier) {
|
||||
acquirePermit(operation, objectKey, semaphore);
|
||||
long startedAt = System.nanoTime();
|
||||
Exception last = null;
|
||||
try {
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
T result = supplier.get();
|
||||
resetFailureWindow();
|
||||
recordOperation(operation, "success", elapsedNanos(startedAt));
|
||||
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
||||
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
||||
return result;
|
||||
} catch (Exception ex) {
|
||||
last = ex;
|
||||
evictIdleConnections();
|
||||
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
||||
recordFailure(operation, objectKey, ex);
|
||||
if (attempt < maxRetries) {
|
||||
long delayMillis = retryDelayMillis(attempt);
|
||||
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
||||
operation, objectKey, attempt, maxRetries, delayMillis, ex.getMessage());
|
||||
sleepQuietly(delayMillis, "retrying rustfs " + operation);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
semaphore.release();
|
||||
}
|
||||
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
|
||||
}
|
||||
|
||||
private void acquirePermit(String operation, String objectKey, Semaphore semaphore) {
|
||||
try {
|
||||
boolean acquired;
|
||||
long timeoutMillis = Math.max(0L, properties.getAcquirePermitTimeoutMillis());
|
||||
if (timeoutMillis == 0L) {
|
||||
acquired = semaphore.tryAcquire();
|
||||
} else {
|
||||
acquired = semaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
if (!acquired) {
|
||||
recordOperation(operation, "rejected", 0L);
|
||||
throw new IllegalStateException("rustfs " + operation + " concurrency limit reached: " + objectKey);
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
recordOperation(operation, "rejected", 0L);
|
||||
throw new IllegalStateException("interrupted while acquiring rustfs " + operation + " permit", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private MinioClient buildClient() {
|
||||
if (minioClientSupplier != null) {
|
||||
return minioClientSupplier.get();
|
||||
}
|
||||
return MinioClient.builder()
|
||||
.endpoint(properties.getEndpoint())
|
||||
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
|
||||
@@ -112,7 +215,7 @@ public class RustfsObjectStorageService {
|
||||
.build();
|
||||
}
|
||||
|
||||
private OkHttpClient getHttpClient() {
|
||||
OkHttpClient getHttpClient() {
|
||||
OkHttpClient current = httpClient;
|
||||
if (current != null) {
|
||||
return current;
|
||||
@@ -123,13 +226,26 @@ public class RustfsObjectStorageService {
|
||||
.connectTimeout(Math.max(1, properties.getConnectTimeoutSeconds()), TimeUnit.SECONDS)
|
||||
.readTimeout(Math.max(1, properties.getReadTimeoutSeconds()), TimeUnit.SECONDS)
|
||||
.writeTimeout(Math.max(1, properties.getWriteTimeoutSeconds()), TimeUnit.SECONDS)
|
||||
.connectionPool(new ConnectionPool(
|
||||
Math.max(0, properties.getConnectionPoolMaxIdle()),
|
||||
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
|
||||
TimeUnit.MILLISECONDS))
|
||||
.retryOnConnectionFailure(true)
|
||||
.build();
|
||||
}
|
||||
return httpClient;
|
||||
}
|
||||
}
|
||||
|
||||
private void evictIdleConnections() {
|
||||
OkHttpClient current = httpClient;
|
||||
if (current != null) {
|
||||
current.connectionPool().evictAll();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyObjectVisible(String objectKey) {
|
||||
long startedAt = System.nanoTime();
|
||||
RuntimeException last = null;
|
||||
for (int attempt = 1; attempt <= 3; attempt++) {
|
||||
try {
|
||||
@@ -137,11 +253,14 @@ public class RustfsObjectStorageService {
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build());
|
||||
recordOperation(OP_STAT, "success", elapsedNanos(startedAt));
|
||||
return;
|
||||
} catch (Exception ex) {
|
||||
last = new IllegalStateException("transient payload is not visible after upload: " + objectKey, ex);
|
||||
evictIdleConnections();
|
||||
recordOperation(OP_STAT, attempt < 3 ? "retry" : "failure", elapsedNanos(startedAt));
|
||||
if (attempt < 3) {
|
||||
sleepQuietly(100L * attempt);
|
||||
sleepQuietly(100L * attempt, "verifying transient payload upload");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -150,16 +269,117 @@ public class RustfsObjectStorageService {
|
||||
: last;
|
||||
}
|
||||
|
||||
private void sleepQuietly(long millis) {
|
||||
private void rejectIfCircuitOpen(String operation, String objectKey) {
|
||||
long openUntil = circuitOpenUntilMillis;
|
||||
if (openUntil <= 0L) {
|
||||
return;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
if (now < openUntil) {
|
||||
recordOperation(operation, "rejected", 0L);
|
||||
throw new IllegalStateException("rustfs failure cooldown active operation=" + operation + " objectKey=" + objectKey);
|
||||
}
|
||||
}
|
||||
|
||||
private void recordFailure(String operation, String objectKey, Exception ex) {
|
||||
long now = System.currentTimeMillis();
|
||||
long windowMillis = Math.max(1L, properties.getFailureWindowSeconds()) * 1000L;
|
||||
if (failureWindowStartedAtMillis <= 0L || now - failureWindowStartedAtMillis > windowMillis) {
|
||||
failureWindowStartedAtMillis = now;
|
||||
windowFailureCount.set(0);
|
||||
}
|
||||
int failures = windowFailureCount.incrementAndGet();
|
||||
int threshold = Math.max(1, properties.getFailureWindowThreshold());
|
||||
if ((OP_UPLOAD.equals(operation) || OP_READ.equals(operation)) && failures >= threshold) {
|
||||
long cooldownMillis = Math.max(0L, properties.getFailureCooldownMillis());
|
||||
circuitOpenUntilMillis = now + cooldownMillis;
|
||||
windowFailureCount.set(0);
|
||||
failureWindowStartedAtMillis = now;
|
||||
log.error("[rustfs] failure window threshold reached, cooldown enabled operation={} objectKey={} failures={} threshold={} cooldownMs={} err={}",
|
||||
operation, objectKey, failures, threshold, cooldownMillis, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void resetFailureWindow() {
|
||||
windowFailureCount.set(0);
|
||||
failureWindowStartedAtMillis = 0L;
|
||||
circuitOpenUntilMillis = 0L;
|
||||
}
|
||||
|
||||
private long retryDelayMillis(int attempt) {
|
||||
long baseDelay = Math.max(0L, properties.getBaseRetryDelayMillis());
|
||||
long maxDelay = Math.max(baseDelay, properties.getMaxRetryDelayMillis());
|
||||
long delay = Math.min(maxDelay, baseDelay * Math.max(1, attempt));
|
||||
long jitter = Math.max(0L, properties.getRetryJitterMillis());
|
||||
if (jitter > 0L) {
|
||||
delay += ThreadLocalRandom.current().nextLong(jitter + 1L);
|
||||
}
|
||||
return delay;
|
||||
}
|
||||
|
||||
private void sleepQuietly(long millis, String reason) {
|
||||
if (millis <= 0L) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("interrupted while verifying transient payload upload", ex);
|
||||
throw new IllegalStateException("interrupted while " + reason, ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void recordPayloadBytes(long bytes) {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry == null || bytes < 0L) {
|
||||
return;
|
||||
}
|
||||
DistributionSummary.builder("aiimage.rustfs.payload.bytes")
|
||||
.baseUnit("bytes")
|
||||
.register(registry)
|
||||
.record(bytes);
|
||||
}
|
||||
|
||||
private void recordOperation(String operation, String result, long durationNanos) {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry == null) {
|
||||
return;
|
||||
}
|
||||
registry.counter("aiimage.rustfs.operation.total", "operation", operation, "result", result).increment();
|
||||
if (durationNanos > 0L) {
|
||||
Timer.builder("aiimage.rustfs.operation.duration")
|
||||
.tag("operation", operation)
|
||||
.tag("result", result)
|
||||
.register(registry)
|
||||
.record(durationNanos, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
public void recordLocalFallback() {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry != null) {
|
||||
registry.counter("aiimage.rustfs.fallback.local.total").increment();
|
||||
}
|
||||
}
|
||||
|
||||
private MeterRegistry meterRegistry() {
|
||||
return meterRegistryProvider == null ? null : meterRegistryProvider.getIfAvailable();
|
||||
}
|
||||
|
||||
private long elapsedNanos(long startedAt) {
|
||||
return System.nanoTime() - startedAt;
|
||||
}
|
||||
|
||||
private long elapsedMillis(long startedAt) {
|
||||
return TimeUnit.NANOSECONDS.toMillis(elapsedNanos(startedAt));
|
||||
}
|
||||
|
||||
private boolean notBlank(String value) {
|
||||
return value != null && !value.isBlank();
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface CheckedSupplier<T> {
|
||||
T get() throws Exception;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user