提交打包

This commit is contained in:
super
2026-05-07 16:21:35 +08:00
parent 48bd8361fe
commit 114440cb0a
64 changed files with 1368 additions and 875 deletions
@@ -13,8 +13,14 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Base64;
import java.util.Locale;
import java.util.Objects;
import java.util.UUID;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@Service
@RequiredArgsConstructor
@@ -57,6 +63,10 @@ public class TransientPayloadStorageService {
return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString, false);
}
public String storeParsedPayloadEntry(String moduleType, Long taskId, String scopeHash, String entryKey, String content, boolean encodeAsJsonString) {
return store("task-parsed", moduleType, taskId, scopeHash, entryKey, content, encodeAsJsonString, false);
}
public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) {
return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true);
}
@@ -84,13 +94,13 @@ public class TransientPayloadStorageService {
}
try {
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
return readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
return decodeStoredPayload(readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length())));
}
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
return decodeStoredPayload(rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
}
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
return ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length()));
return decodeStoredPayload(ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length())));
}
} catch (Exception ex) {
throw new IllegalStateException(errorMessage + ": " + pointer, ex);
@@ -175,10 +185,11 @@ public class TransientPayloadStorageService {
return content;
}
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
String storedContent = encodeStoredPayload(content);
String pointer = null;
if (rustfsObjectStorageService.isConfigured()) {
try {
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, storedContent, verifyAfterUpload);
} catch (Exception ex) {
log.warn("[transient-payload] rustfs upload failed objectKey={} err={}",
objectKey, ex.getMessage());
@@ -186,7 +197,7 @@ public class TransientPayloadStorageService {
}
}
if (pointer == null) {
pointer = storeLocal(objectKey, content);
pointer = storeLocal(objectKey, storedContent);
}
if (pointer == null) {
return content;
@@ -231,6 +242,36 @@ public class TransientPayloadStorageService {
}
}
private String encodeStoredPayload(String content) {
try {
byte[] raw = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {
gzip.write(raw);
}
return "gzip64:" + Base64.getEncoder().encodeToString(baos.toByteArray());
} catch (Exception ex) {
throw new IllegalStateException("failed to encode transient payload", ex);
}
}
private String decodeStoredPayload(String storedContent) {
if (storedContent == null || storedContent.isBlank()) {
return storedContent;
}
if (!storedContent.startsWith("gzip64:")) {
return storedContent;
}
try {
byte[] compressed = Base64.getDecoder().decode(storedContent.substring("gzip64:".length()));
try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(compressed))) {
return new String(gzip.readAllBytes(), StandardCharsets.UTF_8);
}
} catch (Exception ex) {
throw new IllegalStateException("failed to decode transient payload", ex);
}
}
private void deleteLocalPayload(String objectKey) {
try {
Files.deleteIfExists(resolveLocalPayloadPath(objectKey));