super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-66: 限制 RustFS 并发读写与重试的总资源预算

Browse Source 
- 新增 aiimage.transient-storage.max-total-concurrent-operations 配置
  （默认 0 不启用）：跨 upload/read/delete 的总并发预算
- executeWithRetry 入口获取一个总许可，整次操作（含全部重试）全程占用，
  重试不额外消耗，超预算立即拒绝并计入 operation=total rejected 指标
- 失败/完成后 finally 释放总许可，无残留锁；负值/0 视为未启用保持原行为
This commit is contained in:
黄自达
2026-08-30 18:57:26 +08:00
parent e377814c48
commit e2d6875ef9
3 changed files with 331 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java
+5
View File
@@ -23,6 +23,11 @@ public class TransientStorageProperties {
private int maxConcurrentUploads = 16;
private int maxConcurrentReads = 32;
private int maxConcurrentDeletes = 8;
/**
* 跨操作类型的总并发预算（含重试期间）：一次操作全程占用一个总许可，
* 防止多任务叠加时读写与重试之和突破对后端的总压力上限。0/负值表示不启用。
*/
private long maxTotalConcurrentOperations = 0;
private long acquirePermitTimeoutMillis = 2000;
private long baseRetryDelayMillis = 500;
private long maxRetryDelayMillis = 5000;
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java
+54 -33
View File
@@ -35,6 +35,7 @@ public class RustfsObjectStorageService {
private static final String OP_READ = "read";
private static final String OP_DELETE = "delete";
private static final String OP_STAT = "stat";
private static final String OP_TOTAL = "total";
private final TransientStorageProperties properties;
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
@@ -43,6 +44,7 @@ public class RustfsObjectStorageService {
private final Semaphore uploadSemaphore;
private final Semaphore readSemaphore;
private final Semaphore deleteSemaphore;
private final Semaphore totalSemaphore;
private final AtomicInteger windowFailureCount = new AtomicInteger();
private volatile long failureWindowStartedAtMillis;
private volatile long circuitOpenUntilMillis;
@@ -66,6 +68,13 @@ public class RustfsObjectStorageService {
this.uploadSemaphore = new Semaphore(Math.max(1, properties.getMaxConcurrentUploads()));
this.readSemaphore = new Semaphore(Math.max(1, properties.getMaxConcurrentReads()));
this.deleteSemaphore = new Semaphore(Math.max(1, properties.getMaxConcurrentDeletes()));
// 总预算为 0/负值时不启用（Semaphore(0) 的 tryAcquire 永远失败，须用启用开关区分）
long totalBudget = Math.max(0L, properties.getMaxTotalConcurrentOperations());
this.totalSemaphore = new Semaphore((int) Math.min(totalBudget, Integer.MAX_VALUE));
}
private boolean isTotalBudgetEnabled() {
return properties.getMaxTotalConcurrentOperations() > 0L;
}
public boolean isConfigured() {
@@ -185,45 +194,57 @@ public class RustfsObjectStorageService {
CheckedSupplier<T> supplier) {
long startedAt = System.nanoTime();
Exception last = null;
for (int attempt = 1; attempt <= maxRetries; attempt++) {
checkDeadline(operation, objectKey, deadlineNanos);
rejectIfCircuitOpen(operation, objectKey);
acquirePermit(operation, objectKey, semaphore, deadlineNanos);
long delayMillis = 0L;
try {
// 总资源预算：一次操作（含全部重试）全程占用一个总许可，重试不额外消耗。
boolean totalAcquired = false;
if (isTotalBudgetEnabled()) {
acquirePermit(OP_TOTAL, objectKey, totalSemaphore, deadlineNanos);
totalAcquired = true;
}
try {
for (int attempt = 1; attempt <= maxRetries; attempt++) {
checkDeadline(operation, objectKey, deadlineNanos);
rejectIfCircuitOpen(operation, objectKey);
acquirePermit(operation, objectKey, semaphore, deadlineNanos);
long delayMillis = 0L;
try {
checkDeadline(operation, objectKey, deadlineNanos);
T result = supplier.get();
checkDeadline(operation, objectKey, deadlineNanos);
if (!OP_UPLOAD.equals(operation)) {
resetFailureWindow(operation);
}
recordOperation(operation, "success", elapsedNanos(startedAt));
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
return result;
} catch (Exception ex) {
last = ex;
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
recordFailure(operation, objectKey, ex);
if (attempt < maxRetries) {
delayMillis = retryDelayMillis(attempt);
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
operation, objectKey, attempt, maxRetries, delayMillis, ex.getMessage());
try {
checkDeadline(operation, objectKey, deadlineNanos);
T result = supplier.get();
checkDeadline(operation, objectKey, deadlineNanos);
if (!OP_UPLOAD.equals(operation)) {
resetFailureWindow(operation);
}
recordOperation(operation, "success", elapsedNanos(startedAt));
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
return result;
} catch (Exception ex) {
last = ex;
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
recordFailure(operation, objectKey, ex);
if (attempt < maxRetries) {
delayMillis = retryDelayMillis(attempt);
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
operation, objectKey, attempt, maxRetries, delayMillis, ex.getMessage());
}
}
} finally {
semaphore.release();
}
if (last instanceof OperationTimeoutException timeoutException) {
throw timeoutException;
}
if (attempt < maxRetries) {
sleepQuietly(delayMillis, "retrying rustfs " + operation,
operation, objectKey, deadlineNanos);
}
} finally {
semaphore.release();
}
if (last instanceof OperationTimeoutException timeoutException) {
throw timeoutException;
}
if (attempt < maxRetries) {
sleepQuietly(delayMillis, "retrying rustfs " + operation,
operation, objectKey, deadlineNanos);
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
} finally {
if (totalAcquired) {
totalSemaphore.release();
}
}
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
}
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {