refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权
模块边界(G5 / G7) - 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何 业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册 - 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、 CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块 - G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo - 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测) - 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写) 性能与容量(C5/C6/C7/D13) - C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、 BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader; 行数上限改为迭代中生效 - C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country - C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存 - D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与 TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志 - V123 删除 biz_file_result 两个被复合索引覆盖的单列索引 安全(A3/A4/A7 + 守卫名单) - A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外) - A3 上传接口加危险扩展名黑名单(可配置) - AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层) 容量(明细表保留期) - 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理 (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
+23
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@Schema(description = "备选店铺列表单项")
|
||||
public class ProductRiskCandidateVo {
|
||||
|
||||
@Schema(description = "记录主键,删除备选时作为路径参数 id")
|
||||
private Long id;
|
||||
|
||||
@JsonProperty("shop_name")
|
||||
@Schema(description = "店铺名称(JSON 字段名 shop_name)", example = "测试店铺")
|
||||
private String shopName;
|
||||
|
||||
@JsonProperty("created_at")
|
||||
@Schema(description = "创建时间(JSON 字段名 created_at)")
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "用户五国处理偏好:按顺序的国家代码列表(ProductRiskCountryCode 名,如 DE、UK)")
|
||||
public class ProductRiskCountryPreferenceVo {
|
||||
|
||||
@JsonProperty("country_codes")
|
||||
@Schema(description = "已选国家代码,顺序即处理顺序;未保存偏好时返回当前业务模块配置的默认顺序")
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "店铺任务统计看板(当前用户维度);商品风险和取款等复用模块按各自模块类型统计")
|
||||
public class ProductRiskDashboardVo {
|
||||
|
||||
@JsonProperty("candidateCount")
|
||||
@Schema(description = "备选店铺条数;按当前接口所属模块和当前用户统计")
|
||||
private long candidateCount;
|
||||
|
||||
@JsonProperty("processedTaskCount")
|
||||
@Schema(description = "已结束任务数:状态为 SUCCESS 或 FAILED 的任务总数")
|
||||
private long processedTaskCount;
|
||||
|
||||
@JsonProperty("successTaskCount")
|
||||
@Schema(description = "成功结束的任务数(状态 SUCCESS)")
|
||||
private long successTaskCount;
|
||||
|
||||
@JsonProperty("failedTaskCount")
|
||||
@Schema(description = "失败结束的任务数(状态 FAILED)")
|
||||
private long failedTaskCount;
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "处理记录列表响应体:含进行中与已结束任务下的各店结果行")
|
||||
public class ProductRiskHistoryVo {
|
||||
|
||||
@Schema(description = "按店铺拆分的结果项列表,默认最多返回 100 条,按创建时间倒序")
|
||||
private List<ProductRiskResultItemVo> items = new ArrayList<>();
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "批量匹配响应体:按规范化、过滤空项并去重后的有效店铺顺序返回")
|
||||
public class ProductRiskMatchShopsVo {
|
||||
|
||||
@Schema(description = "各店匹配结果列表,一项对应一个处理后的有效店铺名")
|
||||
private List<ProductRiskShopQueueItemVo> items = new ArrayList<>();
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinDetailDto;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Schema(description = "单店结果项:列表接口与任务详情中共用,对应 biz_file_result 一行 + 任务/请求衍生字段")
|
||||
public class ProductRiskResultItemVo {
|
||||
|
||||
@Schema(description = "结果行主键 biz_file_result.id;删除单条、下载 zip 时使用", example = "1001")
|
||||
private Long resultId;
|
||||
|
||||
@Schema(description = "所属任务 id biz_file_task.id", example = "200")
|
||||
private Long taskId;
|
||||
|
||||
@JsonProperty("shopName")
|
||||
@Schema(description = "店铺名称(列表接口中可能与库内规范化名一致;详情中会尽量合并创建任务时的展示名)")
|
||||
private String shopName;
|
||||
|
||||
@JsonProperty("shopId")
|
||||
@Schema(description = "店铺 ID(紫鸟)")
|
||||
private String shopId;
|
||||
|
||||
@Schema(description = "平台")
|
||||
private String platform;
|
||||
|
||||
@JsonProperty("companyName")
|
||||
@Schema(description = "公司名称")
|
||||
private String companyName;
|
||||
|
||||
@Schema(description = "创建任务时是否已匹配(列表中一般为 true)")
|
||||
private boolean matched;
|
||||
|
||||
@JsonProperty("matchStatus")
|
||||
@Schema(description = "匹配状态码,含义同 ProductRiskShopQueueItemVo.matchStatus")
|
||||
private String matchStatus;
|
||||
|
||||
@JsonProperty("matchMessage")
|
||||
@Schema(description = "匹配说明文案")
|
||||
private String matchMessage;
|
||||
|
||||
@JsonProperty("taskStatus")
|
||||
@Schema(description = "所属任务当前状态:RUNNING / SUCCESS / FAILED,与各店行一致")
|
||||
private String taskStatus;
|
||||
|
||||
@Schema(description = "该店是否已成功生成结果文件;null 表示未出终态时可不按布尔处理")
|
||||
private Boolean success;
|
||||
|
||||
@Schema(description = "该店失败时的错误信息")
|
||||
private String error;
|
||||
|
||||
@JsonProperty("outputFilename")
|
||||
@Schema(description = "生成的 zip 文件名(成功后有值)")
|
||||
private String outputFilename;
|
||||
|
||||
@JsonProperty("downloadUrl")
|
||||
@Schema(description = "公开 OSS 直链下载 URL(列表里可能带;也可用 GET /results/{resultId}/download)")
|
||||
private String downloadUrl;
|
||||
private Long fileJobId;
|
||||
private String fileStatus;
|
||||
private String fileError;
|
||||
private Boolean fileReady;
|
||||
|
||||
@JsonProperty("scheduledAt")
|
||||
@Schema(description = "定时执行时间,未设置时为空")
|
||||
private String scheduledAt;
|
||||
|
||||
@JsonAlias("skip_asins_by_country")
|
||||
@JsonProperty("skipAsinsByCountry")
|
||||
private Map<String, List<String>> skipAsinsByCountry = new LinkedHashMap<>();
|
||||
|
||||
@JsonAlias("skip_asin_details_by_country")
|
||||
@JsonProperty("skipAsinDetailsByCountry")
|
||||
private Map<String, List<SkipPriceAsinDetailDto>> skipAsinDetailsByCountry = new LinkedHashMap<>();
|
||||
|
||||
@JsonProperty("createdAt")
|
||||
@Schema(description = "任务创建时间(biz_file_task.created_at,格式化字符串)")
|
||||
private String createdAt;
|
||||
|
||||
@JsonProperty("finishedAt")
|
||||
@Schema(description = "任务结束时间(biz_file_task.finished_at,未结束为 null)")
|
||||
private String finishedAt;
|
||||
}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
|
||||
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinDetailDto;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 与 Python 队列约定字段对齐,供前端原样推入 enqueue_json。
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "单店匹配结果(队列项):与 /match-shops 及创建任务请求 items 元素结构一致,可嵌入队列 JSON data.items")
|
||||
public class ProductRiskShopQueueItemVo {
|
||||
|
||||
@JsonProperty("shopName")
|
||||
@Schema(description = "店铺名称", example = "某某店")
|
||||
private String shopName;
|
||||
|
||||
@Schema(description = "是否命中紫鸟索引;false 时不可用于创建任务")
|
||||
private boolean matched;
|
||||
|
||||
@JsonProperty("shopId")
|
||||
@Schema(description = "紫鸟侧店铺 ID(字符串),未命中时可能为空")
|
||||
private String shopId;
|
||||
|
||||
@Schema(description = "平台,例如亚马逊", example = "亚马逊")
|
||||
private String platform;
|
||||
|
||||
@JsonProperty("companyName")
|
||||
@Schema(description = "公司名称或主体标识")
|
||||
private String companyName;
|
||||
|
||||
@JsonProperty("openStoreUrl")
|
||||
@Schema(description = "打开店铺的地址(由 Python 侧消费)")
|
||||
private String openStoreUrl;
|
||||
|
||||
@JsonProperty("matchedUserId")
|
||||
@Schema(description = "索引关联到的用户 ID(若有)")
|
||||
private Long matchedUserId;
|
||||
|
||||
@JsonProperty("matchStatus")
|
||||
@Schema(description = "匹配状态码:如 MATCHED、PENDING、CONFLICT、INDEX_STALE 等,以后端为准")
|
||||
private String matchStatus;
|
||||
|
||||
@JsonProperty("matchMessage")
|
||||
@Schema(description = "人类可读说明,供前端展示")
|
||||
private String matchMessage;
|
||||
|
||||
@JsonProperty("queryAsins")
|
||||
@Schema(description = "查询 ASIN 模块返回的后台维护 ASIN 数据;按全表聚合返回,不按当前店铺过滤")
|
||||
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
|
||||
|
||||
@JsonAlias("skip_asins_by_country")
|
||||
@JsonProperty("skipAsinsByCountry")
|
||||
@Schema(description = "按国家分组的跳过 ASIN 列表;跟价任务使用,取款任务通常可忽略")
|
||||
private Map<String, List<String>> skipAsinsByCountry = new LinkedHashMap<>();
|
||||
|
||||
@JsonAlias("skip_asin_details_by_country")
|
||||
@JsonProperty("skipAsinDetailsByCountry")
|
||||
@Schema(description = "按国家分组的跳过 ASIN 明细;包含数据库中维护的跳过跟价 ASIN 信息,取款任务通常可忽略")
|
||||
private Map<String, List<SkipPriceAsinDetailDto>> skipAsinDetailsByCountry = new LinkedHashMap<>();
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "批量任务查询响应:每个 id 一条详情;无效 id 记入 missingTaskIds")
|
||||
public class ProductRiskTaskBatchVo {
|
||||
|
||||
@Schema(description = "与请求 taskIds 顺序不一定一致;仅包含存在且模块类型为商品风险的任务")
|
||||
private List<ProductRiskTaskDetailVo> items = new ArrayList<>();
|
||||
|
||||
@Schema(description = "请求中存在但数据库不存在或非商品风险模块的 taskId 列表")
|
||||
private List<Long> missingTaskIds = new ArrayList<>();
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "单个任务详情:任务级概要 + 下属各店结果行")
|
||||
public class ProductRiskTaskDetailVo {
|
||||
|
||||
@Schema(description = "任务级字段:id、状态、错误信息、时间等")
|
||||
private ProductRiskTaskItemVo task = new ProductRiskTaskItemVo();
|
||||
|
||||
@Schema(description = "该任务下每个店铺一条结果记录,顺序一般按结果表 id 升序")
|
||||
private List<ProductRiskResultItemVo> items = new ArrayList<>();
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.nanri.aiimage.modules.shopmatch.model.vo.ShopMatchTaskStageVo;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "任务摘要:任务主表的基础状态、时间和调度信息")
|
||||
public class ProductRiskTaskItemVo {
|
||||
|
||||
@Schema(description = "任务主键 biz_file_task.id", example = "200")
|
||||
private Long id;
|
||||
|
||||
@JsonProperty("taskNo")
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "任务状态:RUNNING=执行中,SUCCESS=成功结束,FAILED=失败结束")
|
||||
private String status;
|
||||
|
||||
@JsonProperty("errorMessage")
|
||||
@Schema(description = "任务失败原因或异常信息")
|
||||
private String errorMessage;
|
||||
|
||||
@JsonProperty("createdAt")
|
||||
@Schema(description = "任务创建时间")
|
||||
private String createdAt;
|
||||
|
||||
@JsonProperty("updatedAt")
|
||||
@Schema(description = "任务最后更新时间")
|
||||
private String updatedAt;
|
||||
|
||||
@JsonProperty("finishedAt")
|
||||
@Schema(description = "任务完成时间")
|
||||
private String finishedAt;
|
||||
|
||||
@JsonProperty("scheduledAt")
|
||||
@Schema(description = "下次计划执行时间;未设置定时时为空")
|
||||
private String scheduledAt;
|
||||
|
||||
@JsonProperty("countryCodes")
|
||||
@Schema(description = "任务使用的国家/站点代码列表")
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
|
||||
@JsonProperty("scheduleStages")
|
||||
@Schema(description = "分阶段调度信息;用于需要按阶段执行的店铺任务")
|
||||
private List<ShopMatchTaskStageVo> scheduleStages = new ArrayList<>();
|
||||
|
||||
@JsonProperty("currentStageIndex")
|
||||
@Schema(description = "下一个待执行阶段索引")
|
||||
private Integer currentStageIndex;
|
||||
|
||||
@JsonProperty("activeStageIndex")
|
||||
@Schema(description = "当前正在执行的阶段索引")
|
||||
private Integer activeStageIndex;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.nanri.aiimage.common.module;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 任务模块清单的**单一来源**(2026-09 全维度审查 G6)。
|
||||
*
|
||||
* 背景:新增一个工具模块此前要同时改 5 处枚举(结果文件 Job 白名单、按天清理清单、
|
||||
* 站内通知模块名、任务心跳分支、陈旧判死巡检线),其中 4 处没有任何自检,
|
||||
* 漏改表现为"功能静默不生效"。现在把**数据驱动的三处**(结果文件 Job、按天清理、通知中文名)
|
||||
* 统一从这里派生,并由 {@code TaskModuleCoverageTest} 守住覆盖面。
|
||||
*
|
||||
* 心跳与陈旧判死两处是代码分支驱动(switch / 逐模块委派),无法纯数据派生,
|
||||
* 新增模块时仍需实现对应 Handler,测试会提示缺失。
|
||||
*/
|
||||
public final class TaskModuleRegistry {
|
||||
|
||||
/**
|
||||
* 单个任务模块的声明。
|
||||
*
|
||||
* @param type moduleType(与 biz_file_task.module_type 一致)
|
||||
* @param label 站内通知/界面用中文名
|
||||
* @param resultFileJob 是否产出结果文件(需要注册 ResultFileJobHandler)
|
||||
* @param ageCleanup 是否参与按天清理(ModuleCleanupProperties)
|
||||
* @param delegatedStaleCheck 是否以"委派"方式并入 stale-check 巡检线
|
||||
* (DeleteBrandStaleTaskService.delegatedStaleChecks,必须逐模块登记动作)
|
||||
* @param selfScheduledStaleCheck 该模块的陈旧判死自带更快的调度(publish 60s / collect-data 30s),
|
||||
* 刻意不并入 2 分钟一轮的巡检线——并入会显著拉长判死时延。
|
||||
* 这类模块的"不在委托名单"是设计差异,不是漏接;由覆盖面测试固定。
|
||||
*/
|
||||
public record Module(String type, String label, boolean resultFileJob, boolean ageCleanup,
|
||||
boolean delegatedStaleCheck, boolean selfScheduledStaleCheck) {
|
||||
}
|
||||
|
||||
private static final List<Module> MODULES = List.of(
|
||||
new Module("PUBLISH", "上架", true, false, false, true),
|
||||
new Module("DEDUPE", "数据去重", false, true, false, false),
|
||||
new Module("SPLIT", "数据拆分", false, true, false, false),
|
||||
new Module("CONVERT", "格式转换", false, true, false, false),
|
||||
new Module("DELETE_BRAND", "删除ASIN", true, true, false, false),
|
||||
new Module("PRODUCT_RISK_RESOLVE", "商品风险解决", true, true, false, false),
|
||||
new Module("PRICE_TRACK", "跟价", true, true, false, false),
|
||||
new Module("SHOP_MATCH", "定时匹配", true, true, false, false),
|
||||
new Module("PATROL_DELETE", "巡店删除", true, true, false, false),
|
||||
new Module("QUERY_ASIN", "查询ASIN", true, true, false, false),
|
||||
new Module("WITHDRAW", "取款", true, true, false, false),
|
||||
new Module("APPEARANCE_PATENT", "外观专利检测", true, true, true, false),
|
||||
new Module("SIMILAR_ASIN", "货源查询", true, true, true, false),
|
||||
new Module("COLLECT_DATA", "采集数据", true, true, false, true),
|
||||
new Module("SHOP_DATA_CRAWL", "店铺数据抓取", true, false, true, false),
|
||||
new Module("BRAND", "品牌检测", true, false, true, false)
|
||||
);
|
||||
|
||||
private TaskModuleRegistry() {
|
||||
}
|
||||
|
||||
public static List<Module> modules() {
|
||||
return MODULES;
|
||||
}
|
||||
|
||||
/** 全部 moduleType。 */
|
||||
public static Set<String> moduleTypes() {
|
||||
return MODULES.stream().map(Module::type).collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** moduleType → 中文名(站内通知等展示用)。 */
|
||||
public static Map<String, String> labels() {
|
||||
Map<String, String> labels = new LinkedHashMap<>();
|
||||
for (Module module : MODULES) {
|
||||
labels.put(module.type(), module.label());
|
||||
}
|
||||
return Map.copyOf(labels);
|
||||
}
|
||||
|
||||
/** 产出结果文件的模块(需要注册 Handler 的模块)。 */
|
||||
public static Set<String> resultFileJobModuleTypes() {
|
||||
return MODULES.stream().filter(Module::resultFileJob).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** 参与按天清理的模块。 */
|
||||
public static Set<String> ageCleanupModuleTypes() {
|
||||
return MODULES.stream().filter(Module::ageCleanup).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** 自带更快巡检节奏、刻意不并入集中巡检线的模块。 */
|
||||
public static Set<String> selfScheduledStaleCheckModuleTypes() {
|
||||
return MODULES.stream().filter(Module::selfScheduledStaleCheck).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** 以"委派"方式并入 stale-check 巡检线的模块(DeleteBrandStaleTaskService 必须逐模块登记动作)。 */
|
||||
public static Set<String> delegatedStaleCheckModuleTypes() {
|
||||
return MODULES.stream().filter(Module::delegatedStaleCheck).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
public static String labelOf(String moduleType) {
|
||||
for (Module module : MODULES) {
|
||||
if (module.type().equals(moduleType)) {
|
||||
return module.label();
|
||||
}
|
||||
}
|
||||
return moduleType;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package com.nanri.aiimage.common.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* 跨节点的小容量状态存储(2026-09 全维度审查 D2)。
|
||||
*
|
||||
* <p>用途:导入/导出任务的进度、归属用户、分组等状态。原先只存节点本地内存,
|
||||
* 客户端轮询落到另一节点就报"任务不存在"(nginx 的 user_id 亲和只覆盖常态)。
|
||||
* 现在本地 Map 作快路径、Redis 作跨节点真源,任何节点都能读到。
|
||||
*
|
||||
* <p>写节流:调用方可能**逐行**刷新进度(数十万行),逐次写 Redis 不可接受;
|
||||
* 默认 500ms 内只写一次(终态用 {@link #putNow} 立即落库)。
|
||||
*
|
||||
* <p>语义取舍:读-改-写不保证原子(进度类状态可接受);{@code redis}/{@code objectMapper}
|
||||
* 为空时退化为纯本地(单测与未注入场景)。
|
||||
*/
|
||||
@Slf4j
|
||||
public final class NodeSharedStore<K, V> {
|
||||
|
||||
private static final long DEFAULT_WRITE_THROTTLE_MILLIS = 500L;
|
||||
|
||||
private final String keyPrefix;
|
||||
private final Duration ttl;
|
||||
private final Class<V> valueType;
|
||||
private final StringRedisTemplate redis;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final long writeThrottleMillis;
|
||||
|
||||
private final ConcurrentHashMap<K, V> local = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<K, AtomicLong> lastWriteAt = new ConcurrentHashMap<>();
|
||||
|
||||
public NodeSharedStore(String keyPrefix,
|
||||
Duration ttl,
|
||||
Class<V> valueType,
|
||||
StringRedisTemplate redis,
|
||||
ObjectMapper objectMapper) {
|
||||
this(keyPrefix, ttl, valueType, redis, objectMapper, DEFAULT_WRITE_THROTTLE_MILLIS);
|
||||
}
|
||||
|
||||
public NodeSharedStore(String keyPrefix,
|
||||
Duration ttl,
|
||||
Class<V> valueType,
|
||||
StringRedisTemplate redis,
|
||||
ObjectMapper objectMapper,
|
||||
long writeThrottleMillis) {
|
||||
this.keyPrefix = keyPrefix;
|
||||
this.ttl = ttl;
|
||||
this.valueType = valueType;
|
||||
this.redis = redis;
|
||||
this.objectMapper = objectMapper;
|
||||
this.writeThrottleMillis = Math.max(0L, writeThrottleMillis);
|
||||
}
|
||||
|
||||
/** 本地优先;本地没有则读 Redis 并回填本地(跨节点可见)。 */
|
||||
public V get(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V cached = local.get(key);
|
||||
if (cached != null) {
|
||||
return cached;
|
||||
}
|
||||
V remote = readRemote(key);
|
||||
if (remote != null) {
|
||||
local.put(key, remote);
|
||||
}
|
||||
return remote;
|
||||
}
|
||||
|
||||
public boolean containsKey(K key) {
|
||||
return get(key) != null;
|
||||
}
|
||||
|
||||
/** 写入并(按节流)同步到 Redis。 */
|
||||
public void put(K key, V value) {
|
||||
if (key == null || value == null) {
|
||||
return;
|
||||
}
|
||||
local.put(key, value);
|
||||
if (!throttleAllowsWrite(key)) {
|
||||
return;
|
||||
}
|
||||
writeRemote(key, value);
|
||||
}
|
||||
|
||||
/** 立即写入 Redis(终态、归属等一次性状态用)。 */
|
||||
public void putNow(K key, V value) {
|
||||
if (key == null || value == null) {
|
||||
return;
|
||||
}
|
||||
local.put(key, value);
|
||||
writeRemote(key, value);
|
||||
}
|
||||
|
||||
/** 删除(本地 + Redis),返回删除前的值(可能为 null)。 */
|
||||
public V remove(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V previous = local.remove(key);
|
||||
lastWriteAt.remove(key);
|
||||
if (redis != null) {
|
||||
try {
|
||||
redis.delete(fullKey(key));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[node-shared-store] 删除远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return previous;
|
||||
}
|
||||
|
||||
/** 本地快照(仅用于日志/统计;不含其它节点的写入)。 */
|
||||
public int localSize() {
|
||||
return local.size();
|
||||
}
|
||||
|
||||
private boolean throttleAllowsWrite(K key) {
|
||||
if (writeThrottleMillis <= 0L) {
|
||||
return true;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
AtomicLong last = lastWriteAt.computeIfAbsent(key, ignored -> new AtomicLong(0L));
|
||||
long previous = last.get();
|
||||
if (now - previous < writeThrottleMillis) {
|
||||
return false;
|
||||
}
|
||||
return last.compareAndSet(previous, now);
|
||||
}
|
||||
|
||||
private V readRemote(K key) {
|
||||
if (redis == null || objectMapper == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
String json = redis.opsForValue().get(fullKey(key));
|
||||
if (json == null || json.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return objectMapper.readValue(json, valueType);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[node-shared-store] 读取远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void writeRemote(K key, V value) {
|
||||
if (redis == null || objectMapper == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String json = objectMapper.writeValueAsString(value);
|
||||
redis.opsForValue().set(fullKey(key), json, ttl);
|
||||
} catch (Exception ex) {
|
||||
// 写失败不影响本地进度(下次 put 会重试),只留线索
|
||||
log.warn("[node-shared-store] 写入远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private String fullKey(K key) {
|
||||
return keyPrefix + ":" + key;
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,13 @@ public final class ExcelStreamReader {
|
||||
default void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
|
||||
}
|
||||
|
||||
/**
|
||||
* 表头之后回调本次 sheet 的近似总行数(EasyExcel 基于 sheet 尺寸,可能为 null)。
|
||||
* 流式解析无法在读完前得到精确行数,需要展示进度/做前置上限校验的调用方可用它近似。
|
||||
*/
|
||||
default void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) throws Exception {
|
||||
}
|
||||
|
||||
void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception;
|
||||
}
|
||||
|
||||
@@ -77,6 +84,7 @@ public final class ExcelStreamReader {
|
||||
currentHeaderMap = normalizedHeadMap;
|
||||
try {
|
||||
handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap);
|
||||
handler.onSheetTotal(sheetName(context), sheetNo(context), approximateTotalRows(context));
|
||||
} catch (RuntimeException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -109,6 +117,15 @@ public final class ExcelStreamReader {
|
||||
public void doAfterAllAnalysed(AnalysisContext context) {
|
||||
}
|
||||
|
||||
private Integer approximateTotalRows(AnalysisContext context) {
|
||||
try {
|
||||
return context.readSheetHolder() == null ? null
|
||||
: context.readSheetHolder().getApproximateTotalRowNumber();
|
||||
} catch (Exception ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String sheetName(AnalysisContext context) {
|
||||
return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 待删对象队列的本地落盘日志:进程重启后仍能恢复「待删除对象」清单。
|
||||
*
|
||||
* <p>使用场景:对象存储删除失败后的补偿队列原本只在节点内存里,重启即丢,
|
||||
* 对应对象会一直残留在桶里直到生命周期规则过期。这里用一行一个 objectKey 的
|
||||
* 追加日志做最小持久化——入队追加、队列收敛后整体重写、启动时回放。
|
||||
*
|
||||
* <p>并发:所有方法内部同步,调用方无需额外加锁;文件损坏/读写异常只记日志不抛出,
|
||||
* 保证补偿链路本身不会因为落盘失败而中断业务。
|
||||
*
|
||||
* <p>上限:{@code maxEntries} 用于防止日志在异常堆积时无界增长(超出后丢弃最早的记录,
|
||||
* 与内存队列的容量准入语义一致——丢的是「待删对象」,最坏结果是对象残留)。
|
||||
*/
|
||||
@Slf4j
|
||||
public final class PendingDeleteJournal {
|
||||
|
||||
private final Path path;
|
||||
private final int maxEntries;
|
||||
private final Object lock = new Object();
|
||||
|
||||
public PendingDeleteJournal(Path path, int maxEntries) {
|
||||
this.path = path;
|
||||
this.maxEntries = Math.max(1, maxEntries);
|
||||
}
|
||||
|
||||
public Path getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
/** 追加一条待删对象(重复追加由回放时的 Set 语义去重)。 */
|
||||
public void record(String objectKey) {
|
||||
if (objectKey == null || objectKey.isBlank()) {
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
try {
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
Files.writeString(path, sanitize(objectKey) + System.lineSeparator(),
|
||||
StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 追加待删对象失败 path={} msg={}", path, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 启动回放:返回日志中的待删对象(按首次出现顺序去重)。 */
|
||||
public List<String> readAll() {
|
||||
synchronized (lock) {
|
||||
if (!Files.isRegularFile(path)) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> lines;
|
||||
try {
|
||||
lines = Files.readAllLines(path, StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 读取待删对象日志失败 path={} msg={}", path, ex.getMessage());
|
||||
return List.of();
|
||||
}
|
||||
Set<String> unique = new LinkedHashSet<>();
|
||||
for (String line : lines) {
|
||||
if (line == null) {
|
||||
continue;
|
||||
}
|
||||
String key = line.trim();
|
||||
if (!key.isEmpty()) {
|
||||
unique.add(key);
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(unique);
|
||||
}
|
||||
}
|
||||
|
||||
/** 队列收敛(成功删除/引用仍在)后整体重写为剩余的待删对象;剩余为空则删除日志。 */
|
||||
public void rewrite(Collection<String> remaining) {
|
||||
synchronized (lock) {
|
||||
Set<String> keep = new LinkedHashSet<>();
|
||||
if (remaining != null) {
|
||||
for (String key : remaining) {
|
||||
if (key != null && !key.isBlank()) {
|
||||
keep.add(sanitize(key));
|
||||
if (keep.size() >= maxEntries) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (keep.isEmpty()) {
|
||||
Files.deleteIfExists(path);
|
||||
return;
|
||||
}
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
Path tmp = path.resolveSibling(path.getFileName() + ".tmp");
|
||||
StringBuilder content = new StringBuilder();
|
||||
for (String key : keep) {
|
||||
content.append(key).append(System.lineSeparator());
|
||||
}
|
||||
Files.writeString(tmp, content.toString(), StandardCharsets.UTF_8,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
|
||||
try {
|
||||
Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (IOException atomicUnsupported) {
|
||||
// 少数文件系统不支持 ATOMIC_MOVE,退化为普通替换
|
||||
Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 重写待删对象日志失败 path={} msg={}", path, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 单行一条记录:去掉换行避免破坏行结构。 */
|
||||
private static String sanitize(String objectKey) {
|
||||
return objectKey.replace('\n', ' ').replace('\r', ' ').trim();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user