task-64: transient payload 压缩改为直接 gzip 二进制流上传
- RustfsObjectStorageService 新增 uploadBytes/readObjectBytes 二进制流 API （application/gzip），uploadText/readObjectAsString 改为其薄包装 - TransientPayloadStorageService 的 encodeStoredPayload 直接输出 gzip 字节， 不再经过 gzip64 + Base64 文本编码（消除 33% 体积膨胀与中间字符串副本） - 读取端按 gzip magic 识别二进制流解压；兼容旧 gzip64: 文本与历史裸文本 - 本地回落文件同步改为写二进制（Files.write），超限判定基于 gzip 后字节数
This commit is contained in:
+13
-3
@@ -80,12 +80,17 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
|
||||
public String uploadText(String objectKey, String content, boolean verifyAfterUpload) {
|
||||
byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
||||
}
|
||||
|
||||
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
||||
long deadlineNanos = operationDeadlineNanos();
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
}
|
||||
rejectIfCircuitOpen(OP_UPLOAD, objectKey);
|
||||
byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
byte[] bytes = content == null ? new byte[0] : content;
|
||||
recordPayloadBytes(bytes.length);
|
||||
AtomicBoolean putCompleted = new AtomicBoolean();
|
||||
try {
|
||||
@@ -96,7 +101,7 @@ public class RustfsObjectStorageService {
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.stream(stream, bytes.length, -1)
|
||||
.contentType("application/json")
|
||||
.contentType("application/gzip")
|
||||
.build());
|
||||
putCompleted.set(true);
|
||||
return objectKey;
|
||||
@@ -117,6 +122,11 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
|
||||
public String readObjectAsString(String objectKey) {
|
||||
byte[] bytes = readObjectBytes(objectKey);
|
||||
return new String(bytes, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public byte[] readObjectBytes(String objectKey) {
|
||||
long deadlineNanos = operationDeadlineNanos();
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
@@ -128,7 +138,7 @@ public class RustfsObjectStorageService {
|
||||
.bucket(properties.getBucket())
|
||||
.object(objectKey)
|
||||
.build())) {
|
||||
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
|
||||
return stream.readAllBytes();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
+47
-26
@@ -145,13 +145,16 @@ public class TransientPayloadStorageService {
|
||||
"transient payload only exists on instance=" + ownerInstance
|
||||
+ " current=" + instanceMetadata.getInstanceId());
|
||||
}
|
||||
return decodeStoredPayload(readLocalPayload(stripLocalInstanceId(localKey)));
|
||||
return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey)));
|
||||
}
|
||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
return decodeStoredPayload(rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
|
||||
return decodeStoredPayloadBytes(
|
||||
rustfsObjectStorageService.readObjectBytes(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
|
||||
}
|
||||
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
||||
return decodeStoredPayload(ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length())));
|
||||
return decodeStoredPayloadBytes(
|
||||
ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length()))
|
||||
.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(errorMessage + ": " + pointer, ex);
|
||||
@@ -318,8 +321,8 @@ public class TransientPayloadStorageService {
|
||||
category, moduleType, taskId, objectKey, rawBytes, properties.getWarnPayloadBytes());
|
||||
}
|
||||
boolean rawOversize = isPositiveLimit(properties.getMaxPayloadBytes()) && rawBytes > properties.getMaxPayloadBytes();
|
||||
String storedContent = encodeStoredPayload(content);
|
||||
long storedBytes = payloadBytes(storedContent);
|
||||
byte[] storedContent = encodeStoredPayload(content);
|
||||
long storedBytes = storedContent.length;
|
||||
boolean storedOversize = isPositiveLimit(properties.getMaxStoredPayloadBytes()) && storedBytes > properties.getMaxStoredPayloadBytes();
|
||||
if (rawOversize || storedOversize) {
|
||||
log.warn("[transient-payload] payload size exceeds rustfs limit category={} moduleType={} taskId={} objectKey={} rawBytes={} storedBytes={} maxRawBytes={} maxStoredBytes={} fallbackToLocal={}",
|
||||
@@ -333,7 +336,7 @@ public class TransientPayloadStorageService {
|
||||
boolean rustfsFallbackToLocal = rawOversize || storedOversize;
|
||||
if (!rustfsFallbackToLocal && rustfsObjectStorageService.isConfigured()) {
|
||||
try {
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, storedContent, verifyAfterUpload);
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadBytes(objectKey, storedContent, verifyAfterUpload);
|
||||
} catch (Exception ex) {
|
||||
rustfsFallbackToLocal = true;
|
||||
// 升级为 ERROR：rustfs 失败后只能落到本地，多实例下其他节点读不到，必须能告警。
|
||||
@@ -362,7 +365,7 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private String storeLocal(String objectKey, String content) {
|
||||
private String storeLocal(String objectKey, byte[] content) {
|
||||
try {
|
||||
Path root = localPayloadRoot();
|
||||
Path target = root.resolve(objectKey).normalize();
|
||||
@@ -370,9 +373,8 @@ public class TransientPayloadStorageService {
|
||||
throw new IllegalArgumentException("invalid local payload key: " + objectKey);
|
||||
}
|
||||
Files.createDirectories(target.getParent());
|
||||
Files.writeString(target,
|
||||
content == null ? "" : content,
|
||||
StandardCharsets.UTF_8,
|
||||
Files.write(target,
|
||||
content == null ? new byte[0] : content,
|
||||
StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.WRITE);
|
||||
@@ -385,10 +387,10 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private String readLocalPayload(String objectKey) {
|
||||
private byte[] readLocalPayloadBytes(String objectKey) {
|
||||
try {
|
||||
Path target = resolveLocalPayloadPath(objectKey);
|
||||
return Files.readString(target, StandardCharsets.UTF_8);
|
||||
return Files.readAllBytes(target);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to read local transient payload", ex);
|
||||
}
|
||||
@@ -435,14 +437,18 @@ public class TransientPayloadStorageService {
|
||||
return normalized.replaceAll("[^A-Za-z0-9_.\\-]", "_");
|
||||
}
|
||||
|
||||
private String encodeStoredPayload(String content) {
|
||||
/**
|
||||
* 压缩 transient payload 为 gzip 二进制流（不再经过 base64 文本编码）。
|
||||
* 极小内容 gzip 可能膨胀，属预期：压缩收益以可压缩内容为准，读取端按 magic 自动识别。
|
||||
*/
|
||||
private byte[] encodeStoredPayload(String content) {
|
||||
try {
|
||||
byte[] raw = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream(Math.max(32, raw.length / 2));
|
||||
try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {
|
||||
gzip.write(raw);
|
||||
}
|
||||
return "gzip64:" + Base64.getEncoder().encodeToString(baos.toByteArray());
|
||||
return baos.toByteArray();
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to encode transient payload", ex);
|
||||
}
|
||||
@@ -456,21 +462,36 @@ public class TransientPayloadStorageService {
|
||||
return value > 0L;
|
||||
}
|
||||
|
||||
private String decodeStoredPayload(String storedContent) {
|
||||
if (storedContent == null || storedContent.isBlank()) {
|
||||
return storedContent;
|
||||
private String decodeStoredPayloadBytes(byte[] storedContent) {
|
||||
if (storedContent == null || storedContent.length == 0) {
|
||||
return "";
|
||||
}
|
||||
if (!storedContent.startsWith("gzip64:")) {
|
||||
return storedContent;
|
||||
}
|
||||
try {
|
||||
byte[] compressed = Base64.getDecoder().decode(storedContent.substring("gzip64:".length()));
|
||||
try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(compressed))) {
|
||||
if (isGzip(storedContent)) {
|
||||
try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(storedContent))) {
|
||||
return new String(gzip.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to decode transient payload", ex);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to decode transient payload", ex);
|
||||
}
|
||||
String legacyText = new String(storedContent, StandardCharsets.UTF_8);
|
||||
if (legacyText.startsWith("gzip64:")) {
|
||||
try {
|
||||
byte[] compressed = Base64.getDecoder().decode(legacyText.substring("gzip64:".length()));
|
||||
try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(compressed))) {
|
||||
return new String(gzip.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to decode transient payload", ex);
|
||||
}
|
||||
}
|
||||
// 历史裸文本（老版本未压缩直接写入的对象）
|
||||
return legacyText;
|
||||
}
|
||||
|
||||
private boolean isGzip(byte[] content) {
|
||||
return content.length >= 2
|
||||
&& (content[0] & 0xFF) == 0x1F
|
||||
&& (content[1] & 0xFF) == 0x8B;
|
||||
}
|
||||
|
||||
private void deleteLocalPayload(String objectKey) {
|
||||
|
||||
Reference in New Issue
Block a user