Merge branch 'master' of https://git.aishufu.top/super/crawler-plugin

modified:   backend/config.py

backend/ali_oss.py

@@ -1,36 +1,124 @@
-import argparse
+"""对象存储上传工具（MinIO，S3 协议兼容）
-import base64
+
-import re
+原实现基于阿里云 OSS SDK（alibabacloud_oss_v2），现改为 boto3 对接 MinIO。
-import time
+对外函数名与返回值保持不变，业务代码无需修改。
-import alibabacloud_oss_v2 as oss
+"""
-import requests
+
+import base64
+import io
+import mimetypes
+import re
+import threading
+import time
+
+import boto3
+import requests
+from botocore.config import Config
+
+from config import (
+    region,
+    endpoint,
+    bucket,
+    file_url_pre,
+    bucket_path,
+    accessKeyId,
+    accessKeySecret,
+)
+
+_client = None
+_client_lock = threading.Lock()
+
+
+def get_client():
+    """获取（懒加载）S3 客户端，MinIO 必须使用 path-style 寻址"""
+    global _client
+    if _client is None:
+        with _client_lock:
+            if _client is None:
+                _client = boto3.client(
+                    "s3",
+                    endpoint_url=endpoint,
+                    aws_access_key_id=accessKeyId,
+                    aws_secret_access_key=accessKeySecret,
+                    region_name=region,
+                    config=Config(
+                        signature_version="s3v4",
+                        s3={"addressing_style": "path"},  # MinIO 必须
+                        retries={"max_attempts": 3, "mode": "standard"},
+                    ),
+                )
+    return _client
+
+
+def _guess_content_type(key: str) -> str:
+    content_type, _ = mimetypes.guess_type(key)
+    return content_type or "application/octet-stream"
+
+
+def get_presigned_url(key: str, expires: int = 7 * 24 * 3600) -> str:
+    """生成临时访问链接（存储桶未开放匿名读时使用），默认有效 7 天"""
+    return get_client().generate_presigned_url(
+        "get_object",
+        Params={"Bucket": bucket, "Key": key.lstrip("/")},
+        ExpiresIn=expires,
+    )
 
-from config import region, endpoint, bucket, file_url_pre, bucket_path
 
 
 def upload_file(file_content: bytes, key: str):
-    credentials_provider = oss.credentials.EnvironmentVariableCredentialsProvider()
+    """上传字节内容到 MinIO，返回可访问链接"""
-    cfg = oss.config.load_default()
+    key = key.lstrip("/")
-    cfg.credentials_provider = credentials_provider
+    client = get_client()
-    cfg.region = region
+    client.put_object(
-    cfg.endpoint = endpoint
+        Bucket=bucket,          # 存储桶名称
-    client = oss.Client(cfg)
+        Key=key,                # 对象名称
-
+        Body=io.BytesIO(file_content) if isinstance(file_content, (bytes, bytearray)) else file_content,
-    result = client.put_object(
+        ContentType=_guess_content_type(key),
-        oss.PutObjectRequest(
-            bucket=bucket,  # 存储空间名称
-            key=key,        # 对象名称
-            body=file_content    # 读取文件内容
-        )
     )
-    # print(result)
     return file_url_pre + key
 
 
+def upload_data_url(data_url: str, prefix: str = "history", key_hint: str = "") -> str:
+    """
+    将 base64 data URL 上传到对象存储，返回图片链接
+    data_url: data:image/png;base64,xxxx 或 data:image/jpeg;base64,xxxx
+    prefix: 对象 key 前缀
+    key_hint: 可选后缀避免重名，如 "_0", "_1"
+    """
+    match = re.match(r'data:image/(\w+);base64,(.+)', data_url)
+    if not match:
+        raise ValueError('无效的 data URL 格式')
+    ext = 'png' if match.group(1).lower() in ('png', 'webp') else 'jpg'
+    file_content = base64.b64decode(match.group(2))
+    ts = int(time.time() * 1000)
+    key = f"{bucket_path}{prefix}/{ts}{key_hint}.{ext}"
+    return upload_file(file_content, key)
+
+
+def upload_data_urls(data_urls: list, prefix: str = "history") -> list:
+    """批量上传 base64 图片到对象存储，返回图片链接列表"""
+    urls = []
+    ts = int(time.time() * 1000)
+    for i, data_url in enumerate(data_urls or []):
+        if not data_url or not isinstance(data_url, str):
+            continue
+        if not data_url.startswith("http"):
+            match = re.match(r'data:image/(\w+);base64,(.+)', data_url)
+            if not match:
+                continue
+            ext = 'png' if match.group(1).lower() in ('png', 'webp') else 'jpg'
+            file_content = base64.b64decode(match.group(2))
+        else:
+            file_content = requests.get(data_url).content
+            ext = "png"
+        key = f"{bucket_path}{prefix}/{ts}_{i}.{ext}"
+        urls.append(upload_file(file_content, key))
+    return urls
 
 
 # 脚本入口，当文件被直接运行时调用main函数
 if __name__ == "__main__":
-    with open("测试图片数据/IMG_2685.JPG", "rb") as f:
+    with open("D:\\pack\\nanri\\main.dist\\SHUFU.zip", "rb") as f:
         file_content = f.read()
-    upload_file(file_content,key=bucket_path+"test.png")
+    res = upload_file(file_content, key=bucket_path + "versions/1.0.46.zip")
+    print(res)

backend/config.py

@@ -63,9 +63,31 @@ backend_java_base_url, backend_java_base_url_source = _get_env(
     default="http://127.0.0.1:18080/",
 )
 backend_java_base_url = backend_java_base_url.rstrip("/")
-os.environ["OSS_ACCESS_KEY_ID"] = accessKeyId
+# os.environ["OSS_ACCESS_KEY_ID"] = accessKeyId
-os.environ["OSS_ACCESS_KEY_SECRET"] = accessKeySecret
+# os.environ["OSS_ACCESS_KEY_SECRET"] = accessKeySecret
-os.environ["SECRET_KEY"] = "ddffc7c1d02121d9554d7b080b2511b6"
+# os.environ["SECRET_KEY"] = "ddffc7c1d02121d9554d7b080b2511b6"
+
+
+# ==================== 对象存储配置（MinIO，S3 协议兼容） ====================
+# MinIO 默认 region 为 us-east-1
+region = os.getenv("oss_region", "us-east-1")
+# S3 API 地址（域名 http://api.aishufu.top，或直连 http://47.110.241.161:9000）
+endpoint = os.getenv("oss_endpoint", "https://oss.aishufu.top")
+bucket = os.getenv("oss_bucket", "client")
+accessKeyId = os.getenv("oss_access_key_id", "minioadmin")
+accessKeySecret = os.getenv("oss_access_key_secret", "Minio@2024Secure")
+bucket_path = os.getenv("oss_bucket_path", "nanri-image/")
+
+# 对外访问地址（可与上传用 endpoint 不同，例如走 CDN/域名），MinIO 为 path-style：{host}/{bucket}/{key}
+_public_host = os.getenv("oss_public_host", endpoint).rstrip("/")
+file_url_pre = f"{_public_host}/{bucket}/"
+
+
+os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId
+os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
+os.environ.setdefault('AWS_ACCESS_KEY_ID', accessKeyId)
+os.environ.setdefault('AWS_SECRET_ACCESS_KEY', accessKeySecret)
+
 
 
 debug = True