From 08e78ebe5578c3401791ca55040d1f663f8aeaca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Thu, 10 Sep 2026 14:31:00 +0800 Subject: [PATCH] =?UTF-8?q?fix(shop-data-crawl):=20=E7=B4=AF=E8=AE=A1?= =?UTF-8?q?=E6=A1=A3=E8=BF=81=E7=8B=AC=E7=AB=8B=E6=A1=B6=20shufu-shop-data?= =?UTF-8?q?=20=E8=84=B1=E7=A6=BB=E9=BB=98=E8=AE=A4=E6=A1=B630=E5=A4=A9?= =?UTF-8?q?=E8=BF=87=E6=9C=9F=E8=A7=84=E5=88=99=EF=BC=8B=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=89=B9=E9=87=8F=E4=B8=8B=E8=BD=BDzip=E6=81=92=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E2=80=94=E2=80=94=E2=91=A0=20=E6=96=B0=E5=A2=9E=20aii?= =?UTF-8?q?mage.oss.shop-data-bucket(=E9=BB=98=E8=AE=A4=20shufu-shop-data)?= =?UTF-8?q?=EF=BC=9A=E4=B8=8A=E4=BC=A0=E4=BE=A7=20resolveBucket=20?= =?UTF-8?q?=E6=8C=89=20SHOP=5FDATA=5FCRAWL=E3=80=81=E8=AF=BB=E5=8F=96/?= =?UTF-8?q?=E5=88=A0=E9=99=A4/=E7=9B=B4=E9=93=BE=E4=BE=A7=20bucketForObjec?= =?UTF-8?q?tKey=20=E6=8C=89=20result/shop=5Fdata=5Fcrawl/=20=E5=89=8D?= =?UTF-8?q?=E7=BC=80=E5=8F=8C=E8=B7=AF=E8=B7=AF=E7=94=B1=EF=BC=88DB=20?= =?UTF-8?q?=E9=87=8C=E5=AD=98=E7=9A=84=E6=98=AF=E8=A3=B8=20objectKey=20?= =?UTF-8?q?=E4=B8=8D=E5=90=AB=E6=A1=B6=E5=90=8D=EF=BC=8Ckey=20=E4=B8=8D?= =?UTF-8?q?=E5=8F=98=E6=95=85=E5=AD=98=E9=87=8F=E5=AF=B9=E8=B1=A1=20mc=20m?= =?UTF-8?q?irror=20=E5=8D=B3=E6=97=A0=E7=BC=9D=E5=88=87=E6=8D=A2=EF=BC=89?= =?UTF-8?q?=EF=BC=8CconfiguredBuckets=20=E7=BA=B3=E5=85=A5=E6=96=B0?= =?UTF-8?q?=E6=A1=B6=EF=BC=8C=E5=90=AF=E5=8A=A8=E8=87=AA=E6=A3=80=E6=9C=AA?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=97=B6=E5=91=8A=E8=AD=A6=20=E2=91=A1=20?= =?UTF-8?q?=E6=89=B9=E9=87=8F=E4=B8=8B=E8=BD=BD=20zip=20=E5=8E=9F=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E6=8A=8A=20df.result=5Ffile=5Furl=20=E5=BD=93?= =?UTF-8?q?=E7=BB=9D=E5=AF=B9=20URL(URI.create().toURL())=EF=BC=8C?= =?UTF-8?q?=E8=A3=B8=20objectKey=20=E5=BF=85=E6=8A=9B=20no=20protocol=20?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=95=B4=E5=8C=85=20100%=20=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=EF=BC=8C=E6=94=B9=E7=94=B1=20loadDownloadRows=20?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=B5=B0=20generateFreshDownloadUrl=20?= =?UTF-8?q?=E6=8B=BC=E7=9B=B4=E9=93=BE=20=E2=91=A2=20=E8=A1=A5=20OssStorag?= =?UTF-8?q?eServiceTest=205=20=E7=94=A8=E4=BE=8B=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E6=AD=A4=E5=89=8D=E9=9B=B6=E8=A6=86=E7=9B=96=E7=9A=84=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E9=80=89=E6=A1=B6/=E8=A3=B8key=E5=89=8D=E7=BC=80?= =?UTF-8?q?=E8=B7=AF=E7=94=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../nanri/aiimage/config/OssProperties.java | 6 ++ .../file/service/oss/OssStorageService.java | 50 ++++++++++++-- .../ShopDataCrawlAdminTasksService.java | 29 +++++++- .../src/main/resources/application.yml | 3 + .../service/oss/OssStorageServiceTest.java | 66 +++++++++++++++++++ 5 files changed, 148 insertions(+), 6 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java index a991894d..946b4e17 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java @@ -16,6 +16,12 @@ public class OssProperties { private String templateBucket; /** 桌面客户端软件安装包桶（历史存公开 client 桶，勿用默认私桶）。 */ private String softwareVersionBucket; + /** + * 店铺数据采集累计档结果桶。独立成桶是为了脱离默认桶 nanri-ai-images 的 + * 30 天全桶过期规则 —— 店铺数据记录约定「无更新则一直保留、只准人工删」， + * 落在默认桶会被存储层按 30 天自动清理。 + */ + private String shopDataBucket; private String accessKeyId; private String accessKeySecret; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java index 6aa6b9c1..cf0da2d9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -12,6 +12,8 @@ import io.minio.RemoveObjectArgs; import io.minio.StatObjectArgs; import io.minio.UploadObjectArgs; import io.minio.errors.ErrorResponseException; +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -28,10 +30,14 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +@Slf4j @Service public class OssStorageService { private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO"; + private static final String SHOP_DATA_CRAWL_MODULE = "SHOP_DATA_CRAWL"; + /** 店铺数据采集累计档的对象键前缀（resultObjectKey 生成 result/<小写模块名>/...）。 */ + private static final String SHOP_DATA_CRAWL_PREFIX = "result/shop_data_crawl/"; private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/"; private static final String SOFTWARE_VERSION_PREFIX = "nanri-image/versions/"; private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000"; @@ -49,6 +55,24 @@ public class OssStorageService { this.minioClient = minioClient == null ? createClient(ossProperties) : minioClient; } + /** + * 启动自检：店铺数据桶未显式配置时会静默回退到默认桶，而默认桶（nanri-ai-images） + * 带 30 天全桶过期规则，历史累计档会被存储层自动清理。这里启动阶段显式告警， + * 避免线上悄悄退回默认桶而无感知。 + */ + @PostConstruct + void checkShopDataBucketConfigured() { + String configured = ossProperties.getShopDataBucket(); + if (configured == null || configured.isBlank()) { + log.warn("[oss] aiimage.oss.shop-data-bucket 未配置，店铺数据采集累计档将落默认桶 {}；" + + "默认桶若带过期规则会导致历史档被自动清理，生产必须显式配置", + ossProperties.getBucket()); + return; + } + log.info("[oss] 店铺数据采集累计档桶已配置 bucket={}（默认桶={}）", + configured.trim(), ossProperties.getBucket()); + } + public String uploadResultFile(File file, String moduleType) { String objectKey = resultObjectKey(file, moduleType); uploadFile(file, resolveBucket(moduleType), objectKey); @@ -535,12 +559,17 @@ public class OssStorageService { if (objectKey.startsWith("result/image_video/") || objectKey.startsWith("upload/image_video/")) { return imageVideoBucket(); } + // 店铺数据采集累计档：DB 里存的是裸 objectKey，下载/删除时必须回到独立桶， + // 否则会去默认桶找对象导致 404（与 IMAGE_VIDEO 同一套前缀路由语义）。 + if (objectKey.startsWith(SHOP_DATA_CRAWL_PREFIX)) { + return shopDataBucket(); + } return ossProperties.getBucket(); } private List configuredBuckets() { return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket(), - softwareVersionBucket()) + softwareVersionBucket(), shopDataBucket()) .filter(Objects::nonNull) .map(String::trim) .filter(bucket -> !bucket.isBlank()) @@ -612,9 +641,14 @@ public class OssStorageService { } private String resolveBucket(String moduleType) { - return IMAGE_VIDEO_MODULE.equalsIgnoreCase(Objects.requireNonNullElse(moduleType, "").trim()) - ? imageVideoBucket() - : ossProperties.getBucket(); + String normalized = Objects.requireNonNullElse(moduleType, "").trim(); + if (IMAGE_VIDEO_MODULE.equalsIgnoreCase(normalized)) { + return imageVideoBucket(); + } + if (SHOP_DATA_CRAWL_MODULE.equalsIgnoreCase(normalized)) { + return shopDataBucket(); + } + return ossProperties.getBucket(); } private String imageVideoBucket() { @@ -629,6 +663,14 @@ public class OssStorageService { return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket()); } + /** + * 店铺数据采集累计档结果桶。未配置时回退默认桶（保持旧行为可启动）， + * 但默认桶带 30 天过期规则会让历史档被存储层清掉，生产必须显式配置。 + */ + private String shopDataBucket() { + return firstNonBlank(ossProperties.getShopDataBucket(), ossProperties.getBucket()); + } + private String publicEndpoint() { return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint())); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java index 88879d7b..57ffbda0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java @@ -178,14 +178,39 @@ public class ShopDataCrawlAdminTasksService { return payload; } - /** 批量下载 zip 所需结果行（仅存在行，缺失 id 由调用方自行记为部分失败）。 */ + /** + * 批量下载 zip 所需结果行（仅存在行，缺失 id 由调用方自行记为部分失败）。 + * + *

{@code df.result_file_url} 存的是裸 objectKey（如 + * {@code result/shop_data_crawl//xxx.xlsx}），而调用方会直接把该值当绝对 URL + * 打开（{@code URI.create(...).toURL().openStream()}），裸 key 会抛 + * {@code MalformedURLException: no protocol} 导致整包下载失败。 + * 这里统一拼成新鲜下载直链后再返回。 + */ public List loadDownloadRows(List resultIds) { if (resultIds == null || resultIds.isEmpty()) { return List.of(); } Map p = new HashMap<>(); flattenList(p, resultIds, "resultIds", "ri"); - return adminTasksMapper.selectDownloadRows(p); + List rows = adminTasksMapper.selectDownloadRows(p); + for (ShopDataCrawlDownloadRowDto row : rows) { + if (row == null || row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { + continue; + } + String freshUrl = ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl()); + if (freshUrl == null || freshUrl.isBlank()) { + log.warn("[shop-data-crawl-admin] 批量下载直链生成为空，该行按失败处理 resultId={} objectKey={}", + row.getResultId(), row.getResultFileUrl()); + row.setResultFileUrl(null); + continue; + } + row.setResultFileUrl(freshUrl); + } + log.info("[shop-data-crawl-admin] 批量下载行加载完成 requested={} loaded={} 直链就绪={}", + resultIds.size(), rows.size(), + rows.stream().filter(r -> r != null && r.getResultFileUrl() != null).count()); + return rows; } /** 非超管可管店铺名（trim+小写去重，与撞款可见口径一致）；超管调用方传 null 表示全量。 */ diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 3b119d8f..ccff7f16 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -117,6 +117,9 @@ aiimage: digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human} template-bucket: ${AIIMAGE_TEMPLATE_OSS_BUCKET:aiimage-templates} software-version-bucket: ${AIIMAGE_OSS_SOFTWARE_VERSION_BUCKET:client} + # 店铺数据采集累计档独立桶：默认桶 nanri-ai-images 带 30 天全桶过期规则， + # 而店铺数据记录约定「无更新则一直保留、只准人工删」，故单独成桶且不设过期规则。 + shop-data-bucket: ${AIIMAGE_OSS_SHOP_DATA_BUCKET:shufu-shop-data} access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:} access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:} transient-storage: diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java index 4c41d094..267aeac8 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java @@ -3,14 +3,18 @@ package com.nanri.aiimage.modules.file.service.oss; import com.nanri.aiimage.config.OssProperties; import io.minio.GetPresignedObjectUrlArgs; import io.minio.MinioClient; +import io.minio.UploadObjectArgs; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import java.io.File; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -162,4 +166,66 @@ class OssStorageServiceTest { private OssProperties properties() { return properties; } + + // ==================== 店铺数据采集独立桶（shufu-shop-data）==================== + + @Test + void shopDataCrawlResultFilesRouteToDedicatedBucket() throws Exception { + properties().setShopDataBucket("shufu-shop-data"); + // 下载/删除侧：DB 存的是裸 objectKey，必须按 result/shop_data_crawl/ 前缀回到独立桶 + assertEquals( + "https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx")); + + // 上传侧：moduleType = SHOP_DATA_CRAWL 决定落独立桶 + MinioClient client = mock(MinioClient.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadObjectArgs.class); + OssStorageService oss = new OssStorageService(properties, client); + File temp = File.createTempFile("shop-data-crawl-test", ".xlsx"); + temp.deleteOnExit(); + String objectKey = oss.uploadResultFile(temp, "SHOP_DATA_CRAWL"); + verify(client).uploadObject(captor.capture()); + assertEquals("shufu-shop-data", captor.getValue().bucket()); + assertEquals(objectKey, captor.getValue().object()); + assertTrue(objectKey.startsWith("result/shop_data_crawl/"), "对象键前缀应保持 result/shop_data_crawl/"); + } + + @Test + void otherModulesStillUseDefaultBucket() { + properties().setShopDataBucket("shufu-shop-data"); + // 前缀路由不能误伤其它模块 + assertEquals( + "https://oss.aishufu.top/nanri-ai-images/result/dedupe/1/re.xlsx", + storageService.generateFreshDownloadUrl("result/dedupe/1/re.xlsx")); + } + + @Test + void shopDataCrawlFallsBackToDefaultBucketWhenNotConfigured() { + // 未配置时保持旧行为（落默认桶），由启动自检 checkShopDataBucketConfigured 负责告警 + assertEquals( + "https://oss.aishufu.top/nanri-ai-images/result/shop_data_crawl/abc/out.xlsx", + storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx")); + } + + @Test + void shopDataCrawlBucketIsManagedAndNormalized() { + properties().setShopDataBucket("shufu-shop-data"); + assertEquals( + "https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.normalizeManagedPublicUrl( + "http://47.110.241.161:9000/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx")); + assertEquals( + "https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.normalizeManagedPublicUrl( + "https://shufu-shop-data.oss.aishufu.top/result/shop_data_crawl/abc/out.xlsx")); + } + + @Test + void shopDataCrawlDownloadEndpointAppliesToResultPrefix() { + properties().setShopDataBucket("shufu-shop-data"); + properties().setDownloadEndpoint("https://download.aishufu.top"); + assertEquals( + "https://download.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx", + storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx")); + } }