提交打包
This commit is contained in:
+46
-5
@@ -13,8 +13,14 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.Base64;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -57,6 +63,10 @@ public class TransientPayloadStorageService {
|
||||
return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString, false);
|
||||
}
|
||||
|
||||
public String storeParsedPayloadEntry(String moduleType, Long taskId, String scopeHash, String entryKey, String content, boolean encodeAsJsonString) {
|
||||
return store("task-parsed", moduleType, taskId, scopeHash, entryKey, content, encodeAsJsonString, false);
|
||||
}
|
||||
|
||||
public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) {
|
||||
return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true);
|
||||
}
|
||||
@@ -84,13 +94,13 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
try {
|
||||
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
|
||||
return readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
|
||||
return decodeStoredPayload(readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length())));
|
||||
}
|
||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
|
||||
return decodeStoredPayload(rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
|
||||
}
|
||||
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
||||
return ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length()));
|
||||
return decodeStoredPayload(ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length())));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(errorMessage + ": " + pointer, ex);
|
||||
@@ -175,10 +185,11 @@ public class TransientPayloadStorageService {
|
||||
return content;
|
||||
}
|
||||
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
|
||||
String storedContent = encodeStoredPayload(content);
|
||||
String pointer = null;
|
||||
if (rustfsObjectStorageService.isConfigured()) {
|
||||
try {
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, storedContent, verifyAfterUpload);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] rustfs upload failed objectKey={} err={}",
|
||||
objectKey, ex.getMessage());
|
||||
@@ -186,7 +197,7 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
if (pointer == null) {
|
||||
pointer = storeLocal(objectKey, content);
|
||||
pointer = storeLocal(objectKey, storedContent);
|
||||
}
|
||||
if (pointer == null) {
|
||||
return content;
|
||||
@@ -231,6 +242,36 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private String encodeStoredPayload(String content) {
|
||||
try {
|
||||
byte[] raw = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {
|
||||
gzip.write(raw);
|
||||
}
|
||||
return "gzip64:" + Base64.getEncoder().encodeToString(baos.toByteArray());
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to encode transient payload", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private String decodeStoredPayload(String storedContent) {
|
||||
if (storedContent == null || storedContent.isBlank()) {
|
||||
return storedContent;
|
||||
}
|
||||
if (!storedContent.startsWith("gzip64:")) {
|
||||
return storedContent;
|
||||
}
|
||||
try {
|
||||
byte[] compressed = Base64.getDecoder().decode(storedContent.substring("gzip64:".length()));
|
||||
try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(compressed))) {
|
||||
return new String(gzip.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to decode transient payload", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteLocalPayload(String objectKey) {
|
||||
try {
|
||||
Files.deleteIfExists(resolveLocalPayloadPath(objectKey));
|
||||
|
||||
Reference in New Issue
Block a user