fix(rustfs): 确定性错误不再重试;回退两处经实测无效的连接池实验
- 新增 isNonRetryable:NoSuchKey / AccessDenied / 签名错误等确定性错误立即失败, 不再白打两次请求(线上 read 一个已清理的 chunk 会连打 3 次并留下一条 ERROR) - connectionPoolMaxIdle 0→5、keepAlive 30000→300000 回退: 实测证明 unexpected end of stream 与连接复用无关——禁用复用后新容器起来的 第一次请求照样中招。至此已排除公网链路、keepAlive 过长、连接复用三项; 另用 mc 并发压 200 个小对象全部成功,说明 RustFS 服务端无问题。 剩余方向指向 MinIO SDK(8.5.17,已是最新)/OkHttp 与 RustFS 的协议细节, 需抓包对比 mc 与 Java 的请求才能定位。数据不丢(重试都能成功),影响是每次 +600ms。
This commit is contained in:
+37
@@ -9,6 +9,7 @@ import io.minio.MinioClient;
|
||||
import io.minio.PutObjectArgs;
|
||||
import io.minio.RemoveObjectArgs;
|
||||
import io.minio.StatObjectArgs;
|
||||
import io.minio.errors.ErrorResponseException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.ConnectionPool;
|
||||
import okhttp3.Dispatcher;
|
||||
@@ -21,6 +22,7 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -38,6 +40,11 @@ public class RustfsObjectStorageService {
|
||||
private static final String OP_STAT = "stat";
|
||||
private static final String OP_TOTAL = "total";
|
||||
|
||||
/** S3 确定性错误码:重试不会改变结果,应立即失败而不是白打两次请求。 */
|
||||
private static final Set<String> NON_RETRYABLE_S3_CODES = Set.of(
|
||||
"NoSuchKey", "NoSuchBucket", "NoSuchVersion",
|
||||
"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName");
|
||||
|
||||
private final TransientStorageProperties properties;
|
||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
||||
@@ -251,6 +258,15 @@ public class RustfsObjectStorageService {
|
||||
last = ex;
|
||||
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
||||
recordFailure(operation, objectKey, ex);
|
||||
if (isNonRetryable(ex)) {
|
||||
// 确定性错误(NoSuchKey / AccessDenied…):重试多少次结果都一样。
|
||||
// 线上 read 一个已被清理的 chunk 就会连打 3 次请求、还被记成 ERROR。
|
||||
log.warn("[rustfs] 确定性错误,不重试 operation={} objectKey={} err={}",
|
||||
operation, objectKey, ex.getMessage());
|
||||
throw ex instanceof RuntimeException runtimeException
|
||||
? runtimeException
|
||||
: new IllegalStateException(ex);
|
||||
}
|
||||
if (attempt < maxRetries) {
|
||||
delayMillis = retryDelayMillis(attempt);
|
||||
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
||||
@@ -279,6 +295,27 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否为「重试也没用」的确定性错误(NoSuchKey / AccessDenied / 签名错误…)。
|
||||
*
|
||||
* <p>只有网络类与 5xx 类失败才值得重试;确定性错误重试多少次结果都一样。
|
||||
*/
|
||||
private static boolean isNonRetryable(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
if (cursor instanceof ErrorResponseException responseException) {
|
||||
String code = responseException.errorResponse() == null
|
||||
? null
|
||||
: responseException.errorResponse().code();
|
||||
if (code != null && NON_RETRYABLE_S3_CODES.contains(code)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
|
||||
try {
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
|
||||
Reference in New Issue
Block a user