完成上架相关优化、oss迁移

This commit is contained in:
supernijia
2026-07-25 17:57:34 +08:00
parent a9bfb8ac6d
commit 567aeaba5d
28 changed files with 964 additions and 382 deletions
@@ -1,10 +1,14 @@
package com.nanri.aiimage.modules.file.service.oss;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.OSSObject;
import com.nanri.aiimage.config.OssProperties;
import lombok.RequiredArgsConstructor;
import io.minio.GetObjectArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
import io.minio.StatObjectArgs;
import io.minio.UploadObjectArgs;
import io.minio.errors.ErrorResponseException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
@@ -12,232 +16,300 @@ import java.io.File;
import java.net.URI;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Stream;
@Service
@RequiredArgsConstructor
public class OssStorageService {
private final OssProperties ossProperties;
private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO";
private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/";
private final OssProperties ossProperties;
private final MinioClient minioClient;
@Autowired
public OssStorageService(OssProperties ossProperties) {
this(ossProperties, null);
}
OssStorageService(OssProperties ossProperties, MinioClient minioClient) {
this.ossProperties = ossProperties;
this.minioClient = minioClient == null ? createClient(ossProperties) : minioClient;
}
/**
* 上传结果文件到 OSS,返回 objectKey。
* 调用方应存储 objectKey,下载时通过 generateFreshDownloadUrl 生成公开直链。
*/
public String uploadResultFile(File file, String moduleType) {
String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName());
OSS ossClient = buildClient();
try {
ossClient.putObject(ossProperties.getBucket(), objectKey, file);
return objectKey;
} finally {
ossClient.shutdown();
}
String objectKey = resultObjectKey(file, moduleType);
uploadFile(file, resolveBucket(moduleType), objectKey);
return objectKey;
}
public UploadedResult uploadResultFileWithFreshDownloadUrl(File file, String moduleType) {
String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName());
String objectKey = resultObjectKey(file, moduleType);
String bucket = resolveBucket(moduleType);
OSS ossClient = buildClient();
try {
ossClient.putObject(bucket, objectKey, file);
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
} finally {
ossClient.shutdown();
}
uploadFile(file, bucket, objectKey);
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
}
public UploadedResult uploadPublicFileWithFreshDownloadUrl(File file, String moduleType, String originalFilename) {
String normalizedModuleType = moduleType == null || moduleType.isBlank()
? "common"
: moduleType.trim().toLowerCase();
String normalizedModuleType = normalizeModuleType(moduleType, "common");
String objectName = sanitizeObjectName(originalFilename);
if (objectName.isBlank()) {
objectName = file.getName();
}
String objectKey = String.format("upload/%s/%s/%s", normalizedModuleType, UUID.randomUUID(), objectName);
String bucket = resolveBucket(moduleType);
OSS ossClient = buildClient();
try {
ossClient.putObject(bucket, objectKey, file);
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
} finally {
ossClient.shutdown();
uploadFile(file, bucket, objectKey);
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
}
public String uploadDigitalHumanVersion(File file, String objectKey) {
if (objectKey == null || !objectKey.startsWith(DIGITAL_HUMAN_PREFIX)) {
throw new IllegalArgumentException("digital human objectKey must start with " + DIGITAL_HUMAN_PREFIX);
}
uploadFile(file, digitalHumanBucket(), objectKey);
return objectKey;
}
public String uploadText(String objectKey, String content) {
if (objectKey == null || objectKey.isBlank()) {
throw new IllegalArgumentException("objectKey must not be blank");
}
OSS ossClient = buildClient();
try (ByteArrayInputStream stream = new ByteArrayInputStream(
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
ossClient.putObject(ossProperties.getBucket(), objectKey, stream);
byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
buildClient().putObject(PutObjectArgs.builder()
.bucket(ossProperties.getBucket())
.object(objectKey)
.stream(stream, bytes.length, -1)
.contentType("application/json; charset=utf-8")
.build());
return objectKey;
} catch (Exception ex) {
throw new IllegalStateException("failed to upload text to oss", ex);
} finally {
ossClient.shutdown();
throw storageFailure("upload text", objectKey, ex);
}
}
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedModuleType = moduleType == null || moduleType.isBlank()
? "unknown"
: moduleType.trim().toLowerCase();
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizeModuleType(moduleType), taskId, normalizedScopeHash);
return uploadText(objectKey, content);
}
public String uploadTaskParsedPayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedModuleType = moduleType == null || moduleType.isBlank()
? "unknown"
: moduleType.trim().toLowerCase();
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizeModuleType(moduleType), taskId, normalizedScopeHash);
return uploadText(objectKey, content);
}
public String readObjectAsString(String value) {
String objectKey = resolveObjectKey(value);
if (objectKey == null || objectKey.isBlank()) {
StorageLocation location = resolveStorageLocation(value);
if (location == null) {
return null;
}
OSS ossClient = buildClient();
try (OSSObject ossObject = ossClient.getObject(ossProperties.getBucket(), objectKey)) {
return new String(ossObject.getObjectContent().readAllBytes(), StandardCharsets.UTF_8);
try (var stream = buildClient().getObject(GetObjectArgs.builder()
.bucket(location.bucket())
.object(location.objectKey())
.build())) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
} catch (Exception ex) {
throw new IllegalStateException("failed to read object from oss", ex);
} finally {
ossClient.shutdown();
throw storageFailure("read", location.objectKey(), ex);
}
}
public void deleteObject(String value) {
String objectKey = resolveObjectKey(value);
if (objectKey == null || objectKey.isBlank()) {
StorageLocation location = resolveStorageLocation(value);
if (location == null) {
return;
}
OSS ossClient = buildClient();
try {
ossClient.deleteObject(ossProperties.getBucket(), objectKey);
} finally {
ossClient.shutdown();
buildClient().removeObject(RemoveObjectArgs.builder()
.bucket(location.bucket())
.object(location.objectKey())
.build());
} catch (Exception ex) {
throw storageFailure("delete", location.objectKey(), ex);
}
}
public boolean objectExists(String value) {
String objectKey = resolveObjectKey(value);
if (objectKey == null || objectKey.isBlank()) {
StorageLocation location = resolveStorageLocation(value);
if (location == null) {
return false;
}
OSS ossClient = buildClient();
try {
return ossClient.doesObjectExist(ossProperties.getBucket(), objectKey);
} finally {
ossClient.shutdown();
buildClient().statObject(StatObjectArgs.builder()
.bucket(location.bucket())
.object(location.objectKey())
.build());
return true;
} catch (ErrorResponseException ex) {
if (isNotFound(ex)) {
return false;
}
throw storageFailure("stat", location.objectKey(), ex);
} catch (Exception ex) {
throw storageFailure("stat", location.objectKey(), ex);
}
}
/**
* 根据 objectKey 生成公开直链下载 URL。
*/
public String generateDownloadUrl(String objectKey) {
return getPublicUrl(objectKey);
public String generateDownloadUrl(String value) {
return getPublicUrl(value);
}
/**
* 获取公开(无签名)URL,格式:https://{bucket}.{endpoint}/{objectKey}
*/
public String getPublicUrl(String value) {
return getPublicUrl(value, ossProperties.getBucket());
StorageLocation location = resolveStorageLocation(value);
return location == null ? null : getPublicUrl(location.objectKey(), location.bucket());
}
private String getPublicUrl(String value, String bucket) {
if (value == null || value.isBlank()) {
return null;
}
String objectKey = resolveObjectKey(value);
String normalizedKey = objectKey.startsWith("/") ? objectKey.substring(1) : objectKey;
String host = String.format("%s.%s", bucket, publicEndpoint());
try {
return new URI("https", host, "/" + normalizedKey, null).toASCIIString();
} catch (Exception ignored) {
return String.format("https://%s/%s", host, normalizedKey);
}
}
/**
* 从存储值中解析出 objectKey,兼容两种格式:
* - 旧格式:完整 URL(https://bucket.endpoint/objectKey?Expires=...)
* - 新格式:直接是 objectKey(如 result/dedupe/uuid/file.xlsx)
*/
public String resolveObjectKey(String value) {
if (value == null || value.isBlank()) {
return value;
}
try {
if (value.startsWith("http://") || value.startsWith("https://")) {
String path = URI.create(value).getPath();
return path.startsWith("/") ? path.substring(1) : path;
}
} catch (Exception ignored) {
int schemeEnd = value.indexOf("://");
int pathStart = schemeEnd < 0 ? -1 : value.indexOf('/', schemeEnd + 3);
if (pathStart >= 0 && pathStart + 1 < value.length()) {
String path = value.substring(pathStart + 1);
int queryStart = path.indexOf('?');
String objectKey = queryStart >= 0 ? path.substring(0, queryStart) : path;
return URLDecoder.decode(objectKey, StandardCharsets.UTF_8);
}
}
return value;
StorageLocation location = resolveStorageLocation(value);
return location == null ? value : location.objectKey();
}
/**
* 根据存储值(objectKey 或旧格式完整 URL)生成公开直链下载 URL。
* 供各模块 listHistory 和下载接口使用,避免签名过期。
*/
public String generateFreshDownloadUrl(String value) {
return getPublicUrl(value);
}
private void uploadFile(File file, String bucket, String objectKey) {
if (file == null || !file.isFile()) {
throw new IllegalArgumentException("upload file must exist");
}
try {
buildClient().uploadObject(UploadObjectArgs.builder()
.bucket(bucket)
.object(objectKey)
.filename(file.getAbsolutePath())
.build());
} catch (Exception ex) {
throw storageFailure("upload", objectKey, ex);
}
}
private String getPublicUrl(String objectKey, String bucket) {
String encodedPath;
try {
encodedPath = new URI(null, null, "/" + bucket + "/" + trimLeadingSlash(objectKey), null)
.toASCIIString();
} catch (Exception ex) {
encodedPath = "/" + bucket + "/" + trimLeadingSlash(objectKey).replace(" ", "%20");
}
return trimTrailingSlash(publicEndpoint()) + encodedPath;
}
private StorageLocation resolveStorageLocation(String value) {
if (value == null || value.isBlank()) {
return null;
}
return generateDownloadUrl(resolveObjectKey(value));
String normalizedValue = value.trim();
String host = null;
String objectKey = normalizedValue;
if (normalizedValue.startsWith("http://") || normalizedValue.startsWith("https://")) {
try {
URI uri = URI.create(normalizedValue);
host = uri.getHost();
objectKey = decodePath(uri.getRawPath());
} catch (Exception ignored) {
objectKey = decodePathFromMalformedUrl(normalizedValue);
}
}
objectKey = trimLeadingSlash(objectKey);
String bucket = bucketFromVirtualHost(host);
String pathBucket = bucketPrefix(objectKey);
if (pathBucket != null) {
bucket = pathBucket;
objectKey = objectKey.substring(pathBucket.length() + 1);
}
if (bucket == null) {
bucket = bucketForObjectKey(objectKey);
}
return objectKey.isBlank() ? null : new StorageLocation(bucket, objectKey);
}
private OSS buildClient() {
return new OSSClientBuilder().build(
"https://" + ossProperties.getEndpoint(),
ossProperties.getAccessKeyId(),
ossProperties.getAccessKeySecret()
);
private String bucketFromVirtualHost(String host) {
if (host == null) {
return null;
}
String normalizedHost = host.toLowerCase(Locale.ROOT);
for (String bucket : configuredBuckets()) {
if (normalizedHost.equals(bucket.toLowerCase(Locale.ROOT))
|| normalizedHost.startsWith(bucket.toLowerCase(Locale.ROOT) + ".")) {
return bucket;
}
}
return null;
}
private String resolveBucket(String moduleType) {
if (moduleType != null
&& "IMAGE_VIDEO".equalsIgnoreCase(moduleType.trim())
&& ossProperties.getImageVideoBucket() != null
&& !ossProperties.getImageVideoBucket().isBlank()) {
return ossProperties.getImageVideoBucket().trim();
private String bucketPrefix(String objectKey) {
for (String bucket : configuredBuckets()) {
if (objectKey.equals(bucket) || objectKey.startsWith(bucket + "/")) {
return bucket;
}
}
return null;
}
private String bucketForObjectKey(String objectKey) {
if (objectKey.startsWith(DIGITAL_HUMAN_PREFIX)) {
return digitalHumanBucket();
}
if (objectKey.startsWith("result/image_video/") || objectKey.startsWith("upload/image_video/")) {
return imageVideoBucket();
}
return ossProperties.getBucket();
}
private List<String> configuredBuckets() {
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket())
.filter(Objects::nonNull)
.map(String::trim)
.filter(bucket -> !bucket.isBlank())
.distinct()
.toList();
}
private MinioClient buildClient() {
return minioClient;
}
private static MinioClient createClient(OssProperties properties) {
return MinioClient.builder()
.endpoint(withScheme(properties.getEndpoint()))
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
.build();
}
private String resolveBucket(String moduleType) {
return IMAGE_VIDEO_MODULE.equalsIgnoreCase(Objects.requireNonNullElse(moduleType, "").trim())
? imageVideoBucket()
: ossProperties.getBucket();
}
private String imageVideoBucket() {
return firstNonBlank(ossProperties.getImageVideoBucket(), ossProperties.getBucket());
}
private String digitalHumanBucket() {
return firstNonBlank(ossProperties.getDigitalHumanBucket(), ossProperties.getBucket());
}
private String publicEndpoint() {
String endpoint = ossProperties.getPublicEndpoint();
if (endpoint == null || endpoint.isBlank()) {
endpoint = ossProperties.getEndpoint();
}
endpoint = endpoint.trim();
if (endpoint.startsWith("http://")) {
endpoint = endpoint.substring("http://".length());
} else if (endpoint.startsWith("https://")) {
endpoint = endpoint.substring("https://".length());
}
return endpoint.endsWith("/") ? endpoint.substring(0, endpoint.length() - 1) : endpoint;
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
}
private String resultObjectKey(File file, String moduleType) {
return String.format("result/%s/%s/%s", normalizeModuleType(moduleType), UUID.randomUUID(), file.getName());
}
private String normalizeModuleType(String moduleType) {
return normalizeModuleType(moduleType, "unknown");
}
private String normalizeModuleType(String moduleType, String defaultValue) {
return moduleType == null || moduleType.isBlank() ? defaultValue : moduleType.trim().toLowerCase(Locale.ROOT);
}
private String sanitizeObjectName(String filename) {
@@ -252,6 +324,63 @@ public class OssStorageService {
return normalized.replaceAll("[\\r\\n]", "_");
}
private String decodePath(String rawPath) {
if (rawPath == null) {
return "";
}
return URLDecoder.decode(rawPath.replace("+", "%2B"), StandardCharsets.UTF_8);
}
private String decodePathFromMalformedUrl(String value) {
int schemeEnd = value.indexOf("://");
int pathStart = schemeEnd < 0 ? -1 : value.indexOf('/', schemeEnd + 3);
if (pathStart < 0 || pathStart + 1 >= value.length()) {
return "";
}
String path = value.substring(pathStart + 1);
int queryStart = path.indexOf('?');
return URLDecoder.decode(queryStart >= 0 ? path.substring(0, queryStart) : path, StandardCharsets.UTF_8);
}
private boolean isNotFound(ErrorResponseException ex) {
String code = ex.errorResponse() == null ? null : ex.errorResponse().code();
return "NoSuchKey".equals(code) || "NoSuchObject".equals(code) || "NoSuchBucket".equals(code);
}
private IllegalStateException storageFailure(String operation, String objectKey, Exception cause) {
return new IllegalStateException("failed to " + operation + " object in MinIO: " + objectKey, cause);
}
private static String withScheme(String endpoint) {
String normalized = Objects.requireNonNull(endpoint, "MinIO endpoint must not be null").trim();
return normalized.startsWith("http://") || normalized.startsWith("https://")
? normalized
: "http://" + normalized;
}
private String trimLeadingSlash(String value) {
String result = Objects.requireNonNullElse(value, "");
while (result.startsWith("/")) {
result = result.substring(1);
}
return result;
}
private String trimTrailingSlash(String value) {
String result = value;
while (result.endsWith("/")) {
result = result.substring(0, result.length() - 1);
}
return result;
}
private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private record StorageLocation(String bucket, String objectKey) {
}
public record UploadedResult(String objectKey, String downloadUrl) {
}
}