diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java index a991894d..946b4e17 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java @@ -16,6 +16,12 @@ public class OssProperties { private String templateBucket; /** 桌面客户端软件安装包桶（历史存公开 client 桶，勿用默认私桶）。 */ private String softwareVersionBucket; + /** + * 店铺数据采集累计档结果桶。独立成桶是为了脱离默认桶 nanri-ai-images 的 + * 30 天全桶过期规则 —— 店铺数据记录约定「无更新则一直保留、只准人工删」， + * 落在默认桶会被存储层按 30 天自动清理。 + */ + private String shopDataBucket; private String accessKeyId; private String accessKeySecret; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java index 6aa6b9c1..cf0da2d9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -12,6 +12,8 @@ import io.minio.RemoveObjectArgs; import io.minio.StatObjectArgs; import io.minio.UploadObjectArgs; import io.minio.errors.ErrorResponseException; +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -28,10 +30,14 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +@Slf4j @Service public class OssStorageService { private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO"; + private static final String SHOP_DATA_CRAWL_MODULE = "SHOP_DATA_CRAWL"; + /** 店铺数据采集累计档的对象键前缀（resultObjectKey 生成 result/<小写模块名>/...）。 */ + private static final String SHOP_DATA_CRAWL_PREFIX = "result/shop_data_crawl/"; private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/"; private static final String SOFTWARE_VERSION_PREFIX = "nanri-image/versions/"; private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000"; @@ -49,6 +55,24 @@ public class OssStorageService { this.minioClient = minioClient == null ? createClient(ossProperties) : minioClient; } + /** + * 启动自检：店铺数据桶未显式配置时会静默回退到默认桶，而默认桶（nanri-ai-images） + * 带 30 天全桶过期规则，历史累计档会被存储层自动清理。这里启动阶段显式告警， + * 避免线上悄悄退回默认桶而无感知。 + */ + @PostConstruct + void checkShopDataBucketConfigured() { + String configured = ossProperties.getShopDataBucket(); + if (configured == null || configured.isBlank()) { + log.warn("[oss] aiimage.oss.shop-data-bucket 未配置，店铺数据采集累计档将落默认桶 {}；" + + "默认桶若带过期规则会导致历史档被自动清理，生产必须显式配置", + ossProperties.getBucket()); + return; + } + log.info("[oss] 店铺数据采集累计档桶已配置 bucket={}（默认桶={}）", + configured.trim(), ossProperties.getBucket()); + } + public String uploadResultFile(File file, String moduleType) { String objectKey = resultObjectKey(file, moduleType); uploadFile(file, resolveBucket(moduleType), objectKey); @@ -535,12 +559,17 @@ public class OssStorageService { if (objectKey.startsWith("result/image_video/") || objectKey.startsWith("upload/image_video/")) { return imageVideoBucket(); } + // 店铺数据采集累计档：DB 里存的是裸 objectKey，下载/删除时必须回到独立桶， + // 否则会去默认桶找对象导致 404（与 IMAGE_VIDEO 同一套前缀路由语义）。 + if (objectKey.startsWith(SHOP_DATA_CRAWL_PREFIX)) { + return shopDataBucket(); + } return ossProperties.getBucket(); } private List configuredBuckets() { return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket(), - softwareVersionBucket()) + softwareVersionBucket(), shopDataBucket()) .filter(Objects::nonNull) .map(String::trim) .filter(bucket -> !bucket.isBlank()) @@ -612,9 +641,14 @@ public class OssStorageService { } private String resolveBucket(String moduleType) { - return IMAGE_VIDEO_MODULE.equalsIgnoreCase(Objects.requireNonNullElse(moduleType, "").trim()) - ? imageVideoBucket() - : ossProperties.getBucket(); + String normalized = Objects.requireNonNullElse(moduleType, "").trim(); + if (IMAGE_VIDEO_MODULE.equalsIgnoreCase(normalized)) { + return imageVideoBucket(); + } + if (SHOP_DATA_CRAWL_MODULE.equalsIgnoreCase(normalized)) { + return shopDataBucket(); + } + return ossProperties.getBucket(); } private String imageVideoBucket() { @@ -629,6 +663,14 @@ public class OssStorageService { return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket()); } + /** + * 店铺数据采集累计档结果桶。未配置时回退默认桶（保持旧行为可启动）， + * 但默认桶带 30 天过期规则会让历史档被存储层清掉，生产必须显式配置。 + */ + private String shopDataBucket() { + return firstNonBlank(ossProperties.getShopDataBucket(), ossProperties.getBucket()); + } + private String publicEndpoint() { return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint())); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java index 88879d7b..57ffbda0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java @@ -178,14 +178,39 @@ public class ShopDataCrawlAdminTasksService { return payload; } - /** 批量下载 zip 所需结果行（仅存在行，缺失 id 由调用方自行记为部分失败）。 */ + /** + * 批量下载 zip 所需结果行（仅存在行，缺失 id 由调用方自行记为部分失败）。 + * + *

{@code df.result_file_url} 存的是裸 objectKey（如 + * {@code result/shop_data_crawl//xxx.xlsx}），而调用方会直接把该值当绝对 URL + * 打开（{@code URI.create(...).toURL().openStream()}），裸 key 会抛 + * {@code MalformedURLException: no protocol} 导致整包下载失败。 + * 这里统一拼成新鲜下载直链后再返回。 + */ public List loadDownloadRows(List resultIds) { if (resultIds == null || resultIds.isEmpty()) { return List.of(); } Map p = new HashMap<>(); flattenList(p, resultIds, "resultIds", "ri"); - return adminTasksMapper.selectDownloadRows(p); + List rows = adminTasksMapper.selectDownloadRows(p); + for (ShopDataCrawlDownloadRowDto row : rows) { + if (row == null || row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { + continue; + } + String freshUrl = ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl()); + if (freshUrl == null || freshUrl.isBlank()) { + log.warn("[shop-data-crawl-admin] 批量下载直链生成为空，该行按失败处理 resultId={} objectKey={}", + row.getResultId(), row.getResultFileUrl()); + row.setResultFileUrl(null); + continue; + } + row.setResultFileUrl(freshUrl); + } + log.info("[shop-data-crawl-admin] 批量下载行加载完成 requested={} loaded={} 直链就绪={}", + resultIds.size(), rows.size(), + rows.stream().filter(r -> r != null && r.getResultFileUrl() != null).count()); + return rows; } /** 非超管可管店铺名（trim+小写去重，与撞款可见口径一致）；超管调用方传 null 表示全量。 */ diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 3b119d8f..ccff7f16 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -117,6 +117,9 @@ aiimage: digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human} template-bucket: ${AIIMAGE_TEMPLATE_OSS_BUCKET:aiimage-templates} software-version-bucket: ${AIIMAGE_OSS_SOFTWARE_VERSION_BUCKET:client} + # 店铺数据采集累计档独立桶：默认桶 nanri-ai-images 带 30 天全桶过期规则， + # 而店铺数据记录约定「无更新则一直保留、只准人工删」，故单独成桶且不设过期规则。 + shop-data-bucket: ${AIIMAGE_OSS_SHOP_DATA_BUCKET:shufu-shop-data} access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:} access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:} transient-storage: diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java index 4c41d094..267aeac8 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java @@ -3,14 +3,18 @@ package com.nanri.aiimage.modules.file.service.oss; import com.nanri.aiimage.config.OssProperties; import io.minio.GetPresignedObjectUrlArgs; import io.minio.MinioClient; +import io.minio.UploadObjectArgs; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import java.io.File; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -162,4 +166,66 @@ class OssStorageServiceTest { private OssProperties properties() { return properties; } + + // ==================== 店铺数据采集独立桶（shufu-shop-data）==================== + + @Test + void shopDataCrawlResultFilesRouteToDedicatedBucket() throws Exception { + properties().setShopDataBucket("shufu-shop-data"); + // 下载/删除侧：DB 存的是裸 objectKey，必须按 result/shop_data_crawl/ 前缀回到独立桶 + assertEquals( + "https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx")); + + // 上传侧：moduleType = SHOP_DATA_CRAWL 决定落独立桶 + MinioClient client = mock(MinioClient.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadObjectArgs.class); + OssStorageService oss = new OssStorageService(properties, client); + File temp = File.createTempFile("shop-data-crawl-test", ".xlsx"); + temp.deleteOnExit(); + String objectKey = oss.uploadResultFile(temp, "SHOP_DATA_CRAWL"); + verify(client).uploadObject(captor.capture()); + assertEquals("shufu-shop-data", captor.getValue().bucket()); + assertEquals(objectKey, captor.getValue().object()); + assertTrue(objectKey.startsWith("result/shop_data_crawl/"), "对象键前缀应保持 result/shop_data_crawl/"); + } + + @Test + void otherModulesStillUseDefaultBucket() { + properties().setShopDataBucket("shufu-shop-data"); + // 前缀路由不能误伤其它模块 + assertEquals( + "https://oss.aishufu.top/nanri-ai-images/result/dedupe/1/re.xlsx", + storageService.generateFreshDownloadUrl("result/dedupe/1/re.xlsx")); + } + + @Test + void shopDataCrawlFallsBackToDefaultBucketWhenNotConfigured() { + // 未配置时保持旧行为（落默认桶），由启动自检 checkShopDataBucketConfigured 负责告警 + assertEquals( + "https://oss.aishufu.top/nanri-ai-images/result/shop_data_crawl/abc/out.xlsx", + storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx")); + } + + @Test + void shopDataCrawlBucketIsManagedAndNormalized() { + properties().setShopDataBucket("shufu-shop-data"); + assertEquals( + "https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.normalizeManagedPublicUrl( + "http://47.110.241.161:9000/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx")); + assertEquals( + "https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.normalizeManagedPublicUrl( + "https://shufu-shop-data.oss.aishufu.top/result/shop_data_crawl/abc/out.xlsx")); + } + + @Test + void shopDataCrawlDownloadEndpointAppliesToResultPrefix() { + properties().setShopDataBucket("shufu-shop-data"); + properties().setDownloadEndpoint("https://download.aishufu.top"); + assertEquals( + "https://download.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx")); + } }