fix(shop-data-crawl): 累计档迁独立桶 shufu-shop-data 脱离默认桶30天过期规则+修复批量下载zip恒失败——① 新增 aiimage.oss.shop-data-bucket(默认 shufu-shop-data):上传侧 resolveBucket 按 SHOP_DATA_CRAWL、读取/删除/直链侧 bucketForObjectKey 按 result/shop_data_crawl/ 前缀双路路由(DB 里存的是裸 objectKey 不含桶名,key 不变故存量对象 mc mirror 即无缝切换),configuredBuckets 纳入新桶,启动自检未配置时告警 ② 批量下载 zip 原直接把 df.result_file_url 当绝对 URL(URI.create().toURL()),裸 objectKey 必抛 no protocol 导致整包 100% 失败,改由 loadDownloadRows 统一走 generateFreshDownloadUrl 拼直链 ③ 补 OssStorageServiceTest 5 用例覆盖此前零覆盖的上传选桶/裸key前缀路由
This commit is contained in:
@@ -16,6 +16,12 @@ public class OssProperties {
|
|||||||
private String templateBucket;
|
private String templateBucket;
|
||||||
/** 桌面客户端软件安装包桶(历史存公开 client 桶,勿用默认私桶)。 */
|
/** 桌面客户端软件安装包桶(历史存公开 client 桶,勿用默认私桶)。 */
|
||||||
private String softwareVersionBucket;
|
private String softwareVersionBucket;
|
||||||
|
/**
|
||||||
|
* 店铺数据采集累计档结果桶。独立成桶是为了脱离默认桶 nanri-ai-images 的
|
||||||
|
* 30 天全桶过期规则 —— 店铺数据记录约定「无更新则一直保留、只准人工删」,
|
||||||
|
* 落在默认桶会被存储层按 30 天自动清理。
|
||||||
|
*/
|
||||||
|
private String shopDataBucket;
|
||||||
private String accessKeyId;
|
private String accessKeyId;
|
||||||
private String accessKeySecret;
|
private String accessKeySecret;
|
||||||
}
|
}
|
||||||
|
|||||||
+46
-4
@@ -12,6 +12,8 @@ import io.minio.RemoveObjectArgs;
|
|||||||
import io.minio.StatObjectArgs;
|
import io.minio.StatObjectArgs;
|
||||||
import io.minio.UploadObjectArgs;
|
import io.minio.UploadObjectArgs;
|
||||||
import io.minio.errors.ErrorResponseException;
|
import io.minio.errors.ErrorResponseException;
|
||||||
|
import jakarta.annotation.PostConstruct;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -28,10 +30,14 @@ import java.util.UUID;
|
|||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
public class OssStorageService {
|
public class OssStorageService {
|
||||||
|
|
||||||
private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO";
|
private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO";
|
||||||
|
private static final String SHOP_DATA_CRAWL_MODULE = "SHOP_DATA_CRAWL";
|
||||||
|
/** 店铺数据采集累计档的对象键前缀(resultObjectKey 生成 result/<小写模块名>/...)。 */
|
||||||
|
private static final String SHOP_DATA_CRAWL_PREFIX = "result/shop_data_crawl/";
|
||||||
private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/";
|
private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/";
|
||||||
private static final String SOFTWARE_VERSION_PREFIX = "nanri-image/versions/";
|
private static final String SOFTWARE_VERSION_PREFIX = "nanri-image/versions/";
|
||||||
private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000";
|
private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000";
|
||||||
@@ -49,6 +55,24 @@ public class OssStorageService {
|
|||||||
this.minioClient = minioClient == null ? createClient(ossProperties) : minioClient;
|
this.minioClient = minioClient == null ? createClient(ossProperties) : minioClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 启动自检:店铺数据桶未显式配置时会静默回退到默认桶,而默认桶(nanri-ai-images)
|
||||||
|
* 带 30 天全桶过期规则,历史累计档会被存储层自动清理。这里启动阶段显式告警,
|
||||||
|
* 避免线上悄悄退回默认桶而无感知。
|
||||||
|
*/
|
||||||
|
@PostConstruct
|
||||||
|
void checkShopDataBucketConfigured() {
|
||||||
|
String configured = ossProperties.getShopDataBucket();
|
||||||
|
if (configured == null || configured.isBlank()) {
|
||||||
|
log.warn("[oss] aiimage.oss.shop-data-bucket 未配置,店铺数据采集累计档将落默认桶 {};"
|
||||||
|
+ "默认桶若带过期规则会导致历史档被自动清理,生产必须显式配置",
|
||||||
|
ossProperties.getBucket());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("[oss] 店铺数据采集累计档桶已配置 bucket={}(默认桶={})",
|
||||||
|
configured.trim(), ossProperties.getBucket());
|
||||||
|
}
|
||||||
|
|
||||||
public String uploadResultFile(File file, String moduleType) {
|
public String uploadResultFile(File file, String moduleType) {
|
||||||
String objectKey = resultObjectKey(file, moduleType);
|
String objectKey = resultObjectKey(file, moduleType);
|
||||||
uploadFile(file, resolveBucket(moduleType), objectKey);
|
uploadFile(file, resolveBucket(moduleType), objectKey);
|
||||||
@@ -535,12 +559,17 @@ public class OssStorageService {
|
|||||||
if (objectKey.startsWith("result/image_video/") || objectKey.startsWith("upload/image_video/")) {
|
if (objectKey.startsWith("result/image_video/") || objectKey.startsWith("upload/image_video/")) {
|
||||||
return imageVideoBucket();
|
return imageVideoBucket();
|
||||||
}
|
}
|
||||||
|
// 店铺数据采集累计档:DB 里存的是裸 objectKey,下载/删除时必须回到独立桶,
|
||||||
|
// 否则会去默认桶找对象导致 404(与 IMAGE_VIDEO 同一套前缀路由语义)。
|
||||||
|
if (objectKey.startsWith(SHOP_DATA_CRAWL_PREFIX)) {
|
||||||
|
return shopDataBucket();
|
||||||
|
}
|
||||||
return ossProperties.getBucket();
|
return ossProperties.getBucket();
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<String> configuredBuckets() {
|
private List<String> configuredBuckets() {
|
||||||
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket(),
|
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket(),
|
||||||
softwareVersionBucket())
|
softwareVersionBucket(), shopDataBucket())
|
||||||
.filter(Objects::nonNull)
|
.filter(Objects::nonNull)
|
||||||
.map(String::trim)
|
.map(String::trim)
|
||||||
.filter(bucket -> !bucket.isBlank())
|
.filter(bucket -> !bucket.isBlank())
|
||||||
@@ -612,9 +641,14 @@ public class OssStorageService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String resolveBucket(String moduleType) {
|
private String resolveBucket(String moduleType) {
|
||||||
return IMAGE_VIDEO_MODULE.equalsIgnoreCase(Objects.requireNonNullElse(moduleType, "").trim())
|
String normalized = Objects.requireNonNullElse(moduleType, "").trim();
|
||||||
? imageVideoBucket()
|
if (IMAGE_VIDEO_MODULE.equalsIgnoreCase(normalized)) {
|
||||||
: ossProperties.getBucket();
|
return imageVideoBucket();
|
||||||
|
}
|
||||||
|
if (SHOP_DATA_CRAWL_MODULE.equalsIgnoreCase(normalized)) {
|
||||||
|
return shopDataBucket();
|
||||||
|
}
|
||||||
|
return ossProperties.getBucket();
|
||||||
}
|
}
|
||||||
|
|
||||||
private String imageVideoBucket() {
|
private String imageVideoBucket() {
|
||||||
@@ -629,6 +663,14 @@ public class OssStorageService {
|
|||||||
return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket());
|
return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 店铺数据采集累计档结果桶。未配置时回退默认桶(保持旧行为可启动),
|
||||||
|
* 但默认桶带 30 天过期规则会让历史档被存储层清掉,生产必须显式配置。
|
||||||
|
*/
|
||||||
|
private String shopDataBucket() {
|
||||||
|
return firstNonBlank(ossProperties.getShopDataBucket(), ossProperties.getBucket());
|
||||||
|
}
|
||||||
|
|
||||||
private String publicEndpoint() {
|
private String publicEndpoint() {
|
||||||
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
|
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
|
||||||
}
|
}
|
||||||
|
|||||||
+27
-2
@@ -178,14 +178,39 @@ public class ShopDataCrawlAdminTasksService {
|
|||||||
return payload;
|
return payload;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 批量下载 zip 所需结果行(仅存在行,缺失 id 由调用方自行记为部分失败)。 */
|
/**
|
||||||
|
* 批量下载 zip 所需结果行(仅存在行,缺失 id 由调用方自行记为部分失败)。
|
||||||
|
*
|
||||||
|
* <p>{@code df.result_file_url} 存的是<b>裸 objectKey</b>(如
|
||||||
|
* {@code result/shop_data_crawl/<uuid>/xxx.xlsx}),而调用方会直接把该值当绝对 URL
|
||||||
|
* 打开({@code URI.create(...).toURL().openStream()}),裸 key 会抛
|
||||||
|
* {@code MalformedURLException: no protocol} 导致整包下载失败。
|
||||||
|
* 这里统一拼成新鲜下载直链后再返回。
|
||||||
|
*/
|
||||||
public List<ShopDataCrawlDownloadRowDto> loadDownloadRows(List<Long> resultIds) {
|
public List<ShopDataCrawlDownloadRowDto> loadDownloadRows(List<Long> resultIds) {
|
||||||
if (resultIds == null || resultIds.isEmpty()) {
|
if (resultIds == null || resultIds.isEmpty()) {
|
||||||
return List.of();
|
return List.of();
|
||||||
}
|
}
|
||||||
Map<String, Object> p = new HashMap<>();
|
Map<String, Object> p = new HashMap<>();
|
||||||
flattenList(p, resultIds, "resultIds", "ri");
|
flattenList(p, resultIds, "resultIds", "ri");
|
||||||
return adminTasksMapper.selectDownloadRows(p);
|
List<ShopDataCrawlDownloadRowDto> rows = adminTasksMapper.selectDownloadRows(p);
|
||||||
|
for (ShopDataCrawlDownloadRowDto row : rows) {
|
||||||
|
if (row == null || row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String freshUrl = ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl());
|
||||||
|
if (freshUrl == null || freshUrl.isBlank()) {
|
||||||
|
log.warn("[shop-data-crawl-admin] 批量下载直链生成为空,该行按失败处理 resultId={} objectKey={}",
|
||||||
|
row.getResultId(), row.getResultFileUrl());
|
||||||
|
row.setResultFileUrl(null);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
row.setResultFileUrl(freshUrl);
|
||||||
|
}
|
||||||
|
log.info("[shop-data-crawl-admin] 批量下载行加载完成 requested={} loaded={} 直链就绪={}",
|
||||||
|
resultIds.size(), rows.size(),
|
||||||
|
rows.stream().filter(r -> r != null && r.getResultFileUrl() != null).count());
|
||||||
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 非超管可管店铺名(trim+小写去重,与撞款可见口径一致);超管调用方传 null 表示全量。 */
|
/** 非超管可管店铺名(trim+小写去重,与撞款可见口径一致);超管调用方传 null 表示全量。 */
|
||||||
|
|||||||
@@ -117,6 +117,9 @@ aiimage:
|
|||||||
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
|
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
|
||||||
template-bucket: ${AIIMAGE_TEMPLATE_OSS_BUCKET:aiimage-templates}
|
template-bucket: ${AIIMAGE_TEMPLATE_OSS_BUCKET:aiimage-templates}
|
||||||
software-version-bucket: ${AIIMAGE_OSS_SOFTWARE_VERSION_BUCKET:client}
|
software-version-bucket: ${AIIMAGE_OSS_SOFTWARE_VERSION_BUCKET:client}
|
||||||
|
# 店铺数据采集累计档独立桶:默认桶 nanri-ai-images 带 30 天全桶过期规则,
|
||||||
|
# 而店铺数据记录约定「无更新则一直保留、只准人工删」,故单独成桶且不设过期规则。
|
||||||
|
shop-data-bucket: ${AIIMAGE_OSS_SHOP_DATA_BUCKET:shufu-shop-data}
|
||||||
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:}
|
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:}
|
||||||
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:}
|
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:}
|
||||||
transient-storage:
|
transient-storage:
|
||||||
|
|||||||
+66
@@ -3,14 +3,18 @@ package com.nanri.aiimage.modules.file.service.oss;
|
|||||||
import com.nanri.aiimage.config.OssProperties;
|
import com.nanri.aiimage.config.OssProperties;
|
||||||
import io.minio.GetPresignedObjectUrlArgs;
|
import io.minio.GetPresignedObjectUrlArgs;
|
||||||
import io.minio.MinioClient;
|
import io.minio.MinioClient;
|
||||||
|
import io.minio.UploadObjectArgs;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
@@ -162,4 +166,66 @@ class OssStorageServiceTest {
|
|||||||
private OssProperties properties() {
|
private OssProperties properties() {
|
||||||
return properties;
|
return properties;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== 店铺数据采集独立桶(shufu-shop-data)====================
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shopDataCrawlResultFilesRouteToDedicatedBucket() throws Exception {
|
||||||
|
properties().setShopDataBucket("shufu-shop-data");
|
||||||
|
// 下载/删除侧:DB 存的是裸 objectKey,必须按 result/shop_data_crawl/ 前缀回到独立桶
|
||||||
|
assertEquals(
|
||||||
|
"https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
|
||||||
|
storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx"));
|
||||||
|
|
||||||
|
// 上传侧:moduleType = SHOP_DATA_CRAWL 决定落独立桶
|
||||||
|
MinioClient client = mock(MinioClient.class);
|
||||||
|
ArgumentCaptor<UploadObjectArgs> captor = ArgumentCaptor.forClass(UploadObjectArgs.class);
|
||||||
|
OssStorageService oss = new OssStorageService(properties, client);
|
||||||
|
File temp = File.createTempFile("shop-data-crawl-test", ".xlsx");
|
||||||
|
temp.deleteOnExit();
|
||||||
|
String objectKey = oss.uploadResultFile(temp, "SHOP_DATA_CRAWL");
|
||||||
|
verify(client).uploadObject(captor.capture());
|
||||||
|
assertEquals("shufu-shop-data", captor.getValue().bucket());
|
||||||
|
assertEquals(objectKey, captor.getValue().object());
|
||||||
|
assertTrue(objectKey.startsWith("result/shop_data_crawl/"), "对象键前缀应保持 result/shop_data_crawl/");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void otherModulesStillUseDefaultBucket() {
|
||||||
|
properties().setShopDataBucket("shufu-shop-data");
|
||||||
|
// 前缀路由不能误伤其它模块
|
||||||
|
assertEquals(
|
||||||
|
"https://oss.aishufu.top/nanri-ai-images/result/dedupe/1/re.xlsx",
|
||||||
|
storageService.generateFreshDownloadUrl("result/dedupe/1/re.xlsx"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shopDataCrawlFallsBackToDefaultBucketWhenNotConfigured() {
|
||||||
|
// 未配置时保持旧行为(落默认桶),由启动自检 checkShopDataBucketConfigured 负责告警
|
||||||
|
assertEquals(
|
||||||
|
"https://oss.aishufu.top/nanri-ai-images/result/shop_data_crawl/abc/out.xlsx",
|
||||||
|
storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shopDataCrawlBucketIsManagedAndNormalized() {
|
||||||
|
properties().setShopDataBucket("shufu-shop-data");
|
||||||
|
assertEquals(
|
||||||
|
"https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
|
||||||
|
storageService.normalizeManagedPublicUrl(
|
||||||
|
"http://47.110.241.161:9000/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx"));
|
||||||
|
assertEquals(
|
||||||
|
"https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
|
||||||
|
storageService.normalizeManagedPublicUrl(
|
||||||
|
"https://shufu-shop-data.oss.aishufu.top/result/shop_data_crawl/abc/out.xlsx"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shopDataCrawlDownloadEndpointAppliesToResultPrefix() {
|
||||||
|
properties().setShopDataBucket("shufu-shop-data");
|
||||||
|
properties().setDownloadEndpoint("https://download.aishufu.top");
|
||||||
|
assertEquals(
|
||||||
|
"https://download.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
|
||||||
|
storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user