提交一些更改
This commit is contained in:
+68
-6
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
@@ -28,6 +29,7 @@ import java.util.zip.GZIPOutputStream;
|
||||
public class TransientPayloadStorageService {
|
||||
|
||||
private static final String LOCAL_POINTER_PREFIX = "local:";
|
||||
private static final String LOCAL_INSTANCE_TAG = "i/";
|
||||
private static final String RUSTFS_POINTER_PREFIX = "rustfs:";
|
||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
|
||||
@@ -37,6 +39,7 @@ public class TransientPayloadStorageService {
|
||||
private final RustfsObjectStorageService rustfsObjectStorageService;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
public boolean isWriteEnabled() {
|
||||
return properties.isEnabled()
|
||||
@@ -94,7 +97,15 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
try {
|
||||
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
|
||||
return decodeStoredPayload(readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length())));
|
||||
String localKey = pointer.substring(LOCAL_POINTER_PREFIX.length());
|
||||
String ownerInstance = extractLocalInstanceId(localKey);
|
||||
if (ownerInstance != null && !ownerInstance.equals(instanceMetadata.getInstanceId())) {
|
||||
// 跨实例消费 local 指针时立即报错,避免悄悄读出空内容造成下游错位。
|
||||
throw new IllegalStateException(
|
||||
"transient payload only exists on instance=" + ownerInstance
|
||||
+ " current=" + instanceMetadata.getInstanceId());
|
||||
}
|
||||
return decodeStoredPayload(readLocalPayload(stripLocalInstanceId(localKey)));
|
||||
}
|
||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
return decodeStoredPayload(rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
|
||||
@@ -114,7 +125,15 @@ public class TransientPayloadStorageService {
|
||||
return;
|
||||
}
|
||||
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
|
||||
deleteLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
|
||||
String localKey = pointer.substring(LOCAL_POINTER_PREFIX.length());
|
||||
String ownerInstance = extractLocalInstanceId(localKey);
|
||||
if (ownerInstance != null && !ownerInstance.equals(instanceMetadata.getInstanceId())) {
|
||||
// 跨实例不能直接删另一台机器上的本地文件,留给该实例的清理任务处理。
|
||||
log.info("[transient-payload] skip cross-instance local delete pointer={} owner={} current={}",
|
||||
pointer, ownerInstance, instanceMetadata.getInstanceId());
|
||||
return;
|
||||
}
|
||||
deleteLocalPayload(stripLocalInstanceId(localKey));
|
||||
return;
|
||||
}
|
||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
@@ -191,9 +210,9 @@ public class TransientPayloadStorageService {
|
||||
try {
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, storedContent, verifyAfterUpload);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] rustfs upload failed objectKey={} err={}",
|
||||
objectKey, ex.getMessage());
|
||||
throw ex;
|
||||
// 升级为 ERROR:rustfs 失败后只能落到本地,多实例下其他节点读不到,必须能告警。
|
||||
log.error("[transient-payload] rustfs upload failed, fallback to local store instanceId={} objectKey={} err={}",
|
||||
instanceMetadata.getInstanceId(), objectKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
if (pointer == null) {
|
||||
@@ -226,7 +245,9 @@ public class TransientPayloadStorageService {
|
||||
StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.WRITE);
|
||||
return LOCAL_POINTER_PREFIX + objectKey;
|
||||
// 在 local 指针上携带 instanceId 段,跨实例 resolve 时能直接判定来源、避免错读。
|
||||
String instanceId = sanitizeInstanceId(instanceMetadata.getInstanceId());
|
||||
return LOCAL_POINTER_PREFIX + LOCAL_INSTANCE_TAG + instanceId + "/" + objectKey;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] local store failed objectKey={} err={}", objectKey, ex.getMessage());
|
||||
return null;
|
||||
@@ -242,6 +263,47 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 local 指针中解析出归属实例 ID。指针形式:
|
||||
* - 新格式:{@code i/<instanceId>/<objectKey>}
|
||||
* - 老格式:{@code <objectKey>}(无 instanceId 段,返回 null 表示未携带,向后兼容旧任务)。
|
||||
*/
|
||||
private String extractLocalInstanceId(String localKey) {
|
||||
if (localKey == null || !localKey.startsWith(LOCAL_INSTANCE_TAG)) {
|
||||
return null;
|
||||
}
|
||||
String remainder = localKey.substring(LOCAL_INSTANCE_TAG.length());
|
||||
int slash = remainder.indexOf('/');
|
||||
if (slash <= 0) {
|
||||
return null;
|
||||
}
|
||||
return remainder.substring(0, slash);
|
||||
}
|
||||
|
||||
/**
|
||||
* 去掉 local 指针中的 instanceId 段,得到真正的 objectKey。
|
||||
*/
|
||||
private String stripLocalInstanceId(String localKey) {
|
||||
if (localKey == null || !localKey.startsWith(LOCAL_INSTANCE_TAG)) {
|
||||
return localKey;
|
||||
}
|
||||
String remainder = localKey.substring(LOCAL_INSTANCE_TAG.length());
|
||||
int slash = remainder.indexOf('/');
|
||||
if (slash < 0) {
|
||||
return remainder;
|
||||
}
|
||||
return remainder.substring(slash + 1);
|
||||
}
|
||||
|
||||
private String sanitizeInstanceId(String value) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
if (normalized.isBlank()) {
|
||||
return "unknown";
|
||||
}
|
||||
// 仅允许字母、数字、下划线、横线和点,避免出现路径分隔符破坏 objectKey 解析。
|
||||
return normalized.replaceAll("[^A-Za-z0-9_.\\-]", "_");
|
||||
}
|
||||
|
||||
private String encodeStoredPayload(String content) {
|
||||
try {
|
||||
byte[] raw = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
Reference in New Issue
Block a user