feat(D2): 导入进度跨节点可见(NodeSharedStore:本地快路径 + Redis 真源)

- 新增 common/service/NodeSharedStore:本地 Map 快路径 + Redis 跨节点真源 + 写节流
  (默认 500ms,逐行刷新进度不会打爆 Redis)+ 本节点条目快照(维护用)+ TTL 兜底过期
- 接入 DedupeTotalDataService(8 个进度/归属/分组/完成时间映射)、QueryAsinService、
  SkipPriceAsinService(各 3 个):轮询落到另一节点不再报"任务不存在"
- 保留期清理改为遍历本节点快照(跨节点过期由 Redis TTL 兜底),不再依赖全量遍历
- 测试同步:去重服务测试的反射注入改用 NodeSharedStore(未注入 Redis 时等价纯本地)

mvn test 2815 全绿
This commit is contained in:
2026-09-14 06:54:48 +08:00
parent f4ee7e16f0
commit af7886926c
5 changed files with 188 additions and 66 deletions
@@ -2,7 +2,9 @@ package com.nanri.aiimage.modules.dedupe.service;
import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.NodeSharedStore;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
@@ -63,7 +65,6 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
@Slf4j
public class DedupeTotalDataService {
@@ -73,18 +74,60 @@ public class DedupeTotalDataService {
private static final long COMPLETED_PROGRESS_RETENTION_MILLIS = 60 * 60 * 1000L;
private static final DateTimeFormatter EXPORT_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
/** 导入进度等状态的 TTL:完成后仍需可查一段时间(与原内存缓存 1 小时保留对齐并留余量)。 */
private static final java.time.Duration IMPORT_STATE_TTL = java.time.Duration.ofHours(3);
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final AdminUserMapper adminUserMapper;
private final ShopManageGroupMapper shopManageGroupMapper;
private final PlatformTransactionManager transactionManager;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();
private final Map<String, DedupeTotalDataImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
private final Map<String, Long> importOwnerMap = new ConcurrentHashMap<>();
private final Map<String, Long> deleteImportOwnerMap = new ConcurrentHashMap<>();
private final Map<String, Long> importGroupMap = new ConcurrentHashMap<>();
private final Map<String, Long> deleteImportGroupMap = new ConcurrentHashMap<>();
private final Map<String, Long> importCompletedAtMap = new ConcurrentHashMap<>();
private final Map<String, Long> deleteImportCompletedAtMap = new ConcurrentHashMap<>();
// 2026-09 全维度审查 D2:进度/归属/分组改由 NodeSharedStore 承载(本地快路径 + Redis 跨节点),
// 此前只存节点本地内存,轮询落到另一节点会报"任务不存在"(nginx 的 user_id 亲和只覆盖常态)。
private final NodeSharedStore<String, DedupeTotalDataImportProgressVo> importProgressMap;
private final NodeSharedStore<String, DedupeTotalDataImportProgressVo> deleteImportProgressMap;
private final NodeSharedStore<String, Long> importOwnerMap;
private final NodeSharedStore<String, Long> deleteImportOwnerMap;
private final NodeSharedStore<String, Long> importGroupMap;
private final NodeSharedStore<String, Long> deleteImportGroupMap;
private final NodeSharedStore<String, Long> importCompletedAtMap;
private final NodeSharedStore<String, Long> deleteImportCompletedAtMap;
public DedupeTotalDataService(DedupeTotalDataMapper dedupeTotalDataMapper,
AdminUserMapper adminUserMapper,
ShopManageGroupMapper shopManageGroupMapper,
PlatformTransactionManager transactionManager,
org.springframework.data.redis.core.StringRedisTemplate stringRedisTemplate,
ObjectMapper objectMapper) {
this.dedupeTotalDataMapper = dedupeTotalDataMapper;
this.adminUserMapper = adminUserMapper;
this.shopManageGroupMapper = shopManageGroupMapper;
this.transactionManager = transactionManager;
this.importProgressMap = progressStore("aiimage:dedupe:import:progress", stringRedisTemplate, objectMapper);
this.deleteImportProgressMap = progressStore("aiimage:dedupe:delete-import:progress", stringRedisTemplate, objectMapper);
this.importOwnerMap = longStore("aiimage:dedupe:import:owner", stringRedisTemplate, objectMapper);
this.deleteImportOwnerMap = longStore("aiimage:dedupe:delete-import:owner", stringRedisTemplate, objectMapper);
this.importGroupMap = longStore("aiimage:dedupe:import:group", stringRedisTemplate, objectMapper);
this.deleteImportGroupMap = longStore("aiimage:dedupe:delete-import:group", stringRedisTemplate, objectMapper);
// 完成时间只用于本节点保留期清理,不参与跨节点读取 → 不节流
this.importCompletedAtMap = longStoreNoThrottle("aiimage:dedupe:import:completed-at", stringRedisTemplate, objectMapper);
this.deleteImportCompletedAtMap = longStoreNoThrottle("aiimage:dedupe:delete-import:completed-at", stringRedisTemplate, objectMapper);
}
private static NodeSharedStore<String, DedupeTotalDataImportProgressVo> progressStore(
String prefix, org.springframework.data.redis.core.StringRedisTemplate redis, ObjectMapper mapper) {
return new NodeSharedStore<>(prefix, IMPORT_STATE_TTL, DedupeTotalDataImportProgressVo.class, redis, mapper);
}
private static NodeSharedStore<String, Long> longStore(
String prefix, org.springframework.data.redis.core.StringRedisTemplate redis, ObjectMapper mapper) {
return new NodeSharedStore<>(prefix, IMPORT_STATE_TTL, Long.class, redis, mapper);
}
private static NodeSharedStore<String, Long> longStoreNoThrottle(
String prefix, org.springframework.data.redis.core.StringRedisTemplate redis, ObjectMapper mapper) {
return new NodeSharedStore<>(prefix, IMPORT_STATE_TTL, Long.class, redis, mapper, 0L);
}
/** 导入任务并发上限,避免 POI 解析和批量数据库写入叠加。 */
private final Semaphore importSlots = new Semaphore(4);
@@ -1235,19 +1278,26 @@ public class DedupeTotalDataService {
deleteImportGroupMap, cutoff);
}
/**
* 清理已结束且超过保留期的进度条目。
* 只遍历本节点写入过的条目(跨节点的过期由 Redis TTL 兜底),避免为清理做全量 SCAN。
*/
private void cleanupExpiredProgressEntries(
Map<String, Long> completedAtMap,
Map<String, DedupeTotalDataImportProgressVo> progressMap,
Map<String, Long> ownerMap,
Map<String, Long> groupMap,
NodeSharedStore<String, Long> completedAtStore,
NodeSharedStore<String, DedupeTotalDataImportProgressVo> progressStore,
NodeSharedStore<String, Long> ownerStore,
NodeSharedStore<String, Long> groupStore,
long cutoff) {
completedAtMap.forEach((id, completedAt) -> {
if (completedAt != null && completedAt < cutoff && completedAtMap.remove(id, completedAt)) {
progressMap.remove(id);
ownerMap.remove(id);
groupMap.remove(id);
for (Map.Entry<String, Long> entry : completedAtStore.localEntriesSnapshot().entrySet()) {
Long completedAt = entry.getValue();
if (completedAt == null || completedAt >= cutoff) {
continue;
}
});
completedAtStore.remove(entry.getKey());
progressStore.remove(entry.getKey());
ownerStore.remove(entry.getKey());
groupStore.remove(entry.getKey());
}
}
private String normalizeExcelText(String value) {