refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权
模块边界(G5 / G7) - 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何 业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册 - 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、 CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块 - G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo - 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测) - 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写) 性能与容量(C5/C6/C7/D13) - C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、 BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader; 行数上限改为迭代中生效 - C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country - C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存 - D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与 TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志 - V123 删除 biz_file_result 两个被复合索引覆盖的单列索引 安全(A3/A4/A7 + 守卫名单) - A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外) - A3 上传接口加危险扩展名黑名单(可配置) - AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层) 容量(明细表保留期) - 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理 (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
@@ -0,0 +1,169 @@
|
||||
package com.nanri.aiimage.common.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* 跨节点的小容量状态存储(2026-09 全维度审查 D2)。
|
||||
*
|
||||
* <p>用途:导入/导出任务的进度、归属用户、分组等状态。原先只存节点本地内存,
|
||||
* 客户端轮询落到另一节点就报"任务不存在"(nginx 的 user_id 亲和只覆盖常态)。
|
||||
* 现在本地 Map 作快路径、Redis 作跨节点真源,任何节点都能读到。
|
||||
*
|
||||
* <p>写节流:调用方可能**逐行**刷新进度(数十万行),逐次写 Redis 不可接受;
|
||||
* 默认 500ms 内只写一次(终态用 {@link #putNow} 立即落库)。
|
||||
*
|
||||
* <p>语义取舍:读-改-写不保证原子(进度类状态可接受);{@code redis}/{@code objectMapper}
|
||||
* 为空时退化为纯本地(单测与未注入场景)。
|
||||
*/
|
||||
@Slf4j
|
||||
public final class NodeSharedStore<K, V> {
|
||||
|
||||
private static final long DEFAULT_WRITE_THROTTLE_MILLIS = 500L;
|
||||
|
||||
private final String keyPrefix;
|
||||
private final Duration ttl;
|
||||
private final Class<V> valueType;
|
||||
private final StringRedisTemplate redis;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final long writeThrottleMillis;
|
||||
|
||||
private final ConcurrentHashMap<K, V> local = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<K, AtomicLong> lastWriteAt = new ConcurrentHashMap<>();
|
||||
|
||||
public NodeSharedStore(String keyPrefix,
|
||||
Duration ttl,
|
||||
Class<V> valueType,
|
||||
StringRedisTemplate redis,
|
||||
ObjectMapper objectMapper) {
|
||||
this(keyPrefix, ttl, valueType, redis, objectMapper, DEFAULT_WRITE_THROTTLE_MILLIS);
|
||||
}
|
||||
|
||||
public NodeSharedStore(String keyPrefix,
|
||||
Duration ttl,
|
||||
Class<V> valueType,
|
||||
StringRedisTemplate redis,
|
||||
ObjectMapper objectMapper,
|
||||
long writeThrottleMillis) {
|
||||
this.keyPrefix = keyPrefix;
|
||||
this.ttl = ttl;
|
||||
this.valueType = valueType;
|
||||
this.redis = redis;
|
||||
this.objectMapper = objectMapper;
|
||||
this.writeThrottleMillis = Math.max(0L, writeThrottleMillis);
|
||||
}
|
||||
|
||||
/** 本地优先;本地没有则读 Redis 并回填本地(跨节点可见)。 */
|
||||
public V get(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V cached = local.get(key);
|
||||
if (cached != null) {
|
||||
return cached;
|
||||
}
|
||||
V remote = readRemote(key);
|
||||
if (remote != null) {
|
||||
local.put(key, remote);
|
||||
}
|
||||
return remote;
|
||||
}
|
||||
|
||||
public boolean containsKey(K key) {
|
||||
return get(key) != null;
|
||||
}
|
||||
|
||||
/** 写入并(按节流)同步到 Redis。 */
|
||||
public void put(K key, V value) {
|
||||
if (key == null || value == null) {
|
||||
return;
|
||||
}
|
||||
local.put(key, value);
|
||||
if (!throttleAllowsWrite(key)) {
|
||||
return;
|
||||
}
|
||||
writeRemote(key, value);
|
||||
}
|
||||
|
||||
/** 立即写入 Redis(终态、归属等一次性状态用)。 */
|
||||
public void putNow(K key, V value) {
|
||||
if (key == null || value == null) {
|
||||
return;
|
||||
}
|
||||
local.put(key, value);
|
||||
writeRemote(key, value);
|
||||
}
|
||||
|
||||
/** 删除(本地 + Redis),返回删除前的值(可能为 null)。 */
|
||||
public V remove(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V previous = local.remove(key);
|
||||
lastWriteAt.remove(key);
|
||||
if (redis != null) {
|
||||
try {
|
||||
redis.delete(fullKey(key));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[node-shared-store] 删除远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return previous;
|
||||
}
|
||||
|
||||
/** 本地快照(仅用于日志/统计;不含其它节点的写入)。 */
|
||||
public int localSize() {
|
||||
return local.size();
|
||||
}
|
||||
|
||||
private boolean throttleAllowsWrite(K key) {
|
||||
if (writeThrottleMillis <= 0L) {
|
||||
return true;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
AtomicLong last = lastWriteAt.computeIfAbsent(key, ignored -> new AtomicLong(0L));
|
||||
long previous = last.get();
|
||||
if (now - previous < writeThrottleMillis) {
|
||||
return false;
|
||||
}
|
||||
return last.compareAndSet(previous, now);
|
||||
}
|
||||
|
||||
private V readRemote(K key) {
|
||||
if (redis == null || objectMapper == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
String json = redis.opsForValue().get(fullKey(key));
|
||||
if (json == null || json.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return objectMapper.readValue(json, valueType);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[node-shared-store] 读取远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void writeRemote(K key, V value) {
|
||||
if (redis == null || objectMapper == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String json = objectMapper.writeValueAsString(value);
|
||||
redis.opsForValue().set(fullKey(key), json, ttl);
|
||||
} catch (Exception ex) {
|
||||
// 写失败不影响本地进度(下次 put 会重试),只留线索
|
||||
log.warn("[node-shared-store] 写入远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private String fullKey(K key) {
|
||||
return keyPrefix + ":" + key;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user