super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(shop-data-crawl): 累计档迁独立桶 shufu-shop-data 脱离默认桶30天过期规则＋修复批量下载zip恒失败——① 新增 aiimage.oss.shop-data-bucket(默认 shufu-shop-data)：上传侧 resolveBucket 按 SHOP_DATA_CRAWL、读取/删除/直链侧 bucketForObjectKey 按 result/shop_data_crawl/ 前缀双路路由（DB 里存的是裸 objectKey 不含桶名，key 不变故存量对象 mc mirror 即无缝切换），configuredBuckets 纳入新桶，启动自检未配置时告警 ② 批量下载 zip 原直接把 df.result_file_url 当绝对 URL(URI.create().toURL())，裸 objectKey 必抛 no protocol 导致整包 100% 失败，改由 loadDownloadRows 统一走 generateFreshDownloadUrl 拼直链 ③ 补 OssStorageServiceTest 5 用例覆盖此前零覆盖的上传选桶/裸key前缀路由

Browse Source
This commit is contained in:
黄自达
2026-09-10 14:31:00 +08:00
parent 6fe67b648f
commit 08e78ebe55
5 changed files with 148 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java
+6
View File
@@ -16,6 +16,12 @@ public class OssProperties {
private String templateBucket;
/** 桌面客户端软件安装包桶（历史存公开 client 桶，勿用默认私桶）。 */
private String softwareVersionBucket;
/**
* 店铺数据采集累计档结果桶。独立成桶是为了脱离默认桶 nanri-ai-images 的
* 30 天全桶过期规则 —— 店铺数据记录约定「无更新则一直保留、只准人工删」，
* 落在默认桶会被存储层按 30 天自动清理。
*/
private String shopDataBucket;
private String accessKeyId;
private String accessKeySecret;
}
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java
+46 -4
View File
@@ -12,6 +12,8 @@ import io.minio.RemoveObjectArgs;
import io.minio.StatObjectArgs;
import io.minio.UploadObjectArgs;
import io.minio.errors.ErrorResponseException;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -28,10 +30,14 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
@Slf4j
@Service
public class OssStorageService {
private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO";
private static final String SHOP_DATA_CRAWL_MODULE = "SHOP_DATA_CRAWL";
/** 店铺数据采集累计档的对象键前缀（resultObjectKey 生成 result/<小写模块名>/...）。 */
private static final String SHOP_DATA_CRAWL_PREFIX = "result/shop_data_crawl/";
private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/";
private static final String SOFTWARE_VERSION_PREFIX = "nanri-image/versions/";
private static final String LEGACY_MINIO_ENDPOINT = "http://47.110.241.161:9000";
@@ -49,6 +55,24 @@ public class OssStorageService {
this.minioClient = minioClient == null ? createClient(ossProperties) : minioClient;
}
/**
* 启动自检：店铺数据桶未显式配置时会静默回退到默认桶，而默认桶（nanri-ai-images）
* 带 30 天全桶过期规则，历史累计档会被存储层自动清理。这里启动阶段显式告警，
* 避免线上悄悄退回默认桶而无感知。
*/
@PostConstruct
void checkShopDataBucketConfigured() {
String configured = ossProperties.getShopDataBucket();
if (configured == null || configured.isBlank()) {
log.warn("[oss] aiimage.oss.shop-data-bucket 未配置，店铺数据采集累计档将落默认桶 {}；"
+ "默认桶若带过期规则会导致历史档被自动清理，生产必须显式配置",
ossProperties.getBucket());
return;
}
log.info("[oss] 店铺数据采集累计档桶已配置 bucket={}（默认桶={}）",
configured.trim(), ossProperties.getBucket());
}
public String uploadResultFile(File file, String moduleType) {
String objectKey = resultObjectKey(file, moduleType);
uploadFile(file, resolveBucket(moduleType), objectKey);
@@ -535,12 +559,17 @@ public class OssStorageService {
if (objectKey.startsWith("result/image_video/") || objectKey.startsWith("upload/image_video/")) {
return imageVideoBucket();
}
// 店铺数据采集累计档：DB 里存的是裸 objectKey，下载/删除时必须回到独立桶，
// 否则会去默认桶找对象导致 404（与 IMAGE_VIDEO 同一套前缀路由语义）。
if (objectKey.startsWith(SHOP_DATA_CRAWL_PREFIX)) {
return shopDataBucket();
}
return ossProperties.getBucket();
}
private List<String> configuredBuckets() {
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket(),
softwareVersionBucket())
softwareVersionBucket(), shopDataBucket())
.filter(Objects::nonNull)
.map(String::trim)
.filter(bucket -> !bucket.isBlank())
@@ -612,9 +641,14 @@ public class OssStorageService {
}
private String resolveBucket(String moduleType) {
return IMAGE_VIDEO_MODULE.equalsIgnoreCase(Objects.requireNonNullElse(moduleType, "").trim())
? imageVideoBucket()
: ossProperties.getBucket();
String normalized = Objects.requireNonNullElse(moduleType, "").trim();
if (IMAGE_VIDEO_MODULE.equalsIgnoreCase(normalized)) {
return imageVideoBucket();
}
if (SHOP_DATA_CRAWL_MODULE.equalsIgnoreCase(normalized)) {
return shopDataBucket();
}
return ossProperties.getBucket();
}
private String imageVideoBucket() {
@@ -629,6 +663,14 @@ public class OssStorageService {
return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket());
}
/**
* 店铺数据采集累计档结果桶。未配置时回退默认桶（保持旧行为可启动），
* 但默认桶带 30 天过期规则会让历史档被存储层清掉，生产必须显式配置。
*/
private String shopDataBucket() {
return firstNonBlank(ossProperties.getShopDataBucket(), ossProperties.getBucket());
}
private String publicEndpoint() {
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java
+27 -2
View File
@@ -178,14 +178,39 @@ public class ShopDataCrawlAdminTasksService {
return payload;
}
/** 批量下载 zip 所需结果行（仅存在行，缺失 id 由调用方自行记为部分失败）。 */
/**
* 批量下载 zip 所需结果行（仅存在行，缺失 id 由调用方自行记为部分失败）。
*
* <p>{@code df.result_file_url} 存的是<b>裸 objectKey</b>（如
* {@code result/shop_data_crawl/<uuid>/xxx.xlsx}），而调用方会直接把该值当绝对 URL
* 打开（{@code URI.create(...).toURL().openStream()}），裸 key 会抛
* {@code MalformedURLException: no protocol} 导致整包下载失败。
* 这里统一拼成新鲜下载直链后再返回。
*/
public List<ShopDataCrawlDownloadRowDto> loadDownloadRows(List<Long> resultIds) {
if (resultIds == null || resultIds.isEmpty()) {
return List.of();
}
Map<String, Object> p = new HashMap<>();
flattenList(p, resultIds, "resultIds", "ri");
return adminTasksMapper.selectDownloadRows(p);
List<ShopDataCrawlDownloadRowDto> rows = adminTasksMapper.selectDownloadRows(p);
for (ShopDataCrawlDownloadRowDto row : rows) {
if (row == null || row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
continue;
}
String freshUrl = ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl());
if (freshUrl == null || freshUrl.isBlank()) {
log.warn("[shop-data-crawl-admin] 批量下载直链生成为空，该行按失败处理 resultId={} objectKey={}",
row.getResultId(), row.getResultFileUrl());
row.setResultFileUrl(null);
continue;
}
row.setResultFileUrl(freshUrl);
}
log.info("[shop-data-crawl-admin] 批量下载行加载完成 requested={} loaded={} 直链就绪={}",
resultIds.size(), rows.size(),
rows.stream().filter(r -> r != null && r.getResultFileUrl() != null).count());
return rows;
}
/** 非超管可管店铺名（trim+小写去重，与撞款可见口径一致）；超管调用方传 null 表示全量。 */
backend-java/src/main/resources/application.yml
+3
View File
@@ -117,6 +117,9 @@ aiimage:
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
template-bucket: ${AIIMAGE_TEMPLATE_OSS_BUCKET:aiimage-templates}
software-version-bucket: ${AIIMAGE_OSS_SOFTWARE_VERSION_BUCKET:client}
# 店铺数据采集累计档独立桶：默认桶 nanri-ai-images 带 30 天全桶过期规则，
# 而店铺数据记录约定「无更新则一直保留、只准人工删」，故单独成桶且不设过期规则。
shop-data-bucket: ${AIIMAGE_OSS_SHOP_DATA_BUCKET:shufu-shop-data}
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:}
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:}
transient-storage:
backend-java/src/test/java/com/nanri/aiimage/modules/file/service/oss/OssStorageServiceTest.java
+66
View File
@@ -3,14 +3,18 @@ package com.nanri.aiimage.modules.file.service.oss;
import com.nanri.aiimage.config.OssProperties;
import io.minio.GetPresignedObjectUrlArgs;
import io.minio.MinioClient;
import io.minio.UploadObjectArgs;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.io.File;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@@ -162,4 +166,66 @@ class OssStorageServiceTest {
private OssProperties properties() {
return properties;
}
// ==================== 店铺数据采集独立桶（shufu-shop-data）====================
@Test
void shopDataCrawlResultFilesRouteToDedicatedBucket() throws Exception {
properties().setShopDataBucket("shufu-shop-data");
// 下载/删除侧：DB 存的是裸 objectKey，必须按 result/shop_data_crawl/ 前缀回到独立桶
assertEquals(
"https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx"));
// 上传侧：moduleType = SHOP_DATA_CRAWL 决定落独立桶
MinioClient client = mock(MinioClient.class);
ArgumentCaptor<UploadObjectArgs> captor = ArgumentCaptor.forClass(UploadObjectArgs.class);
OssStorageService oss = new OssStorageService(properties, client);
File temp = File.createTempFile("shop-data-crawl-test", ".xlsx");
temp.deleteOnExit();
String objectKey = oss.uploadResultFile(temp, "SHOP_DATA_CRAWL");
verify(client).uploadObject(captor.capture());
assertEquals("shufu-shop-data", captor.getValue().bucket());
assertEquals(objectKey, captor.getValue().object());
assertTrue(objectKey.startsWith("result/shop_data_crawl/"), "对象键前缀应保持 result/shop_data_crawl/");
}
@Test
void otherModulesStillUseDefaultBucket() {
properties().setShopDataBucket("shufu-shop-data");
// 前缀路由不能误伤其它模块
assertEquals(
"https://oss.aishufu.top/nanri-ai-images/result/dedupe/1/re.xlsx",
storageService.generateFreshDownloadUrl("result/dedupe/1/re.xlsx"));
}
@Test
void shopDataCrawlFallsBackToDefaultBucketWhenNotConfigured() {
// 未配置时保持旧行为（落默认桶），由启动自检 checkShopDataBucketConfigured 负责告警
assertEquals(
"https://oss.aishufu.top/nanri-ai-images/result/shop_data_crawl/abc/out.xlsx",
storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx"));
}
@Test
void shopDataCrawlBucketIsManagedAndNormalized() {
properties().setShopDataBucket("shufu-shop-data");
assertEquals(
"https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
storageService.normalizeManagedPublicUrl(
"http://47.110.241.161:9000/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx"));
assertEquals(
"https://oss.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
storageService.normalizeManagedPublicUrl(
"https://shufu-shop-data.oss.aishufu.top/result/shop_data_crawl/abc/out.xlsx"));
}
@Test
void shopDataCrawlDownloadEndpointAppliesToResultPrefix() {
properties().setShopDataBucket("shufu-shop-data");
properties().setDownloadEndpoint("https://download.aishufu.top");
assertEquals(
"https://download.aishufu.top/shufu-shop-data/result/shop_data_crawl/abc/out.xlsx",
storageService.generateFreshDownloadUrl("result/shop_data_crawl/abc/out.xlsx"));
}
}