fix: 全维度审查修复(安全/正确性/性能/稳定性/客户端/前端)

安全
- /api/ziniao/** 五个匿名接口加管理员鉴权(此前可匿名换取任意员工店铺登录令牌)
- 删除 Flask 遗留后门:默认密码建超管 + 每次启动写生产 users 表(服务端与客户端各一份)
- 进度/详情接口归属过滤:新增 TaskProgressOwnershipSupport,11 模块 progress/light 与
  /tasks/batch 接入,DTO 补 userId,前端 13 个查询封装补传(未传时后端不过滤,兼容旧端)
- 代理提取链接(含账密)不再明文入日志(新增 common/util/SecretMasking)
- 全局异常兜底不再回传原始异常信息;内部令牌比较改常量时间
- 登录加失败计数与锁定(10 次锁 15 分钟);品牌源文件下载加 SSRF 防护
- AdminApiGuardFilter 覆盖前缀从 2 扩到 15(开关默认 false,行为不变,为收紧做准备)
- 生产关闭 springdoc/knife4j(/doc.html 匿名可读全部接口定义)

正确性
- 40901/40902 拆分:锁竞争不再被伪装成 success=true(此前客户端停止重试、分片静默丢失)
- 假成功收敛:集采明细批量写失败改为抛出、去重 worker 异常标失败、4 个 worker 改判
  success 字段、publish 空 ASIN 行参与批次 flush、巡店删除全失败带 error 上报
- 客户端心跳 discard 移入 finally(7 模块,失败路径不再留僵尸 RUNNING 任务)
- 状态机条件更新:跟价停止循环、集采 activate/fail、imagevideo 归档回填、店铺匹配提交

性能
- 前端入口包 JS 1.05MB→204KB、CSS 355KB→10.7KB(Element Plus 改按需 + el-config-provider)
- 载荷引用计数按指针里的 taskId 收敛(原 JSON 列 IN 全表扫且逐行调用)
- 店铺明细多值批量 INSERT;快照 upsert 预载缓存;结果文件列改单条 UPDATE
- 新增迁移 V120(补 3 个缺失索引)/V121(删 4 个被覆盖的冗余索引)/V122(URL 前缀索引)

稳定性
- 新增 common/util/ThreadPools 有界线程池替换 5 处无界队列(防堆积 OOM)
- Redis 锁释放改 Lua 原子校验(原裸 delete 会误删他人已过期的锁)
- imagevideo 加死节点接管;锁续期失败重试;调度池 4→16;openStream 全部加超时
- 事务内远程对象删除移到提交后;启动恢复锁按实例命名

客户端
- 不再 taskkill /f /im chrome.exe(改为按调试端口精准回收,不杀用户自己的浏览器)
- 密码检测不再无条件杀紫鸟进程;品牌检测加全局互斥(代理池不再互相覆盖)
- base_dir 统一到 exe 目录(原被 os.getcwd() 覆盖,日志/缓存会分裂两个目录)
- 缓存加定时清理;图片下载加超时;mkstemp 句柄托管

测试
- 同步更新受影响的契约测试(构造器签名/条件更新/方法改名/新增接口方法等)
- 修复 FaultInjectionTest 等 3 处 mock 未 stub 流式 read 导致的读循环 OOM
- mvn test 2795 个测试全绿
This commit is contained in:
2026-09-14 04:15:36 +08:00
parent 4bc0164fb1
commit 7368322791
135 changed files with 3222 additions and 829 deletions
@@ -92,14 +92,28 @@ public class StaleTaskRepairService {
/** RUNNING 但心跳超过 2h 未刷新 → FAILED(心跳残留续命即僵尸)。 */
private void repairFileTaskStaleRunning() {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(STALE_IDLE_MINUTES);
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
// 先查 id 再按主键更新:此前是一条无 LIMIT、无 module_type 的批量 UPDATE,
// 而 biz_file_task 没有以 status 打头的索引 → 每 10 分钟一次全表扫,
// 且按无索引条件命中行加 X 锁(长事务/锁扩散)。与另两个修复方法口径保持一致。
List<FileTaskEntity> stale = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.lt(FileTaskEntity::getUpdatedAt, cutoff)
.last("limit 500"));
if (stale.isEmpty()) {
return;
}
String reason = "任务心跳超时,已自动失败";
LocalDateTime now = LocalDateTime.now();
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getStatus, STATUS_FAILED)
.set(FileTaskEntity::getErrorMessage, "任务心跳超时,已自动失败")
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
.set(FileTaskEntity::getErrorMessage, reason)
.set(FileTaskEntity::getFinishedAt, now));
if (updated > 0) {
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={}", updated);
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}",
updated, stale.stream().map(FileTaskEntity::getId).toList());
}
}
@@ -23,11 +23,18 @@ public class TaskDistributedLockService {
public static final Duration DEFAULT_LOCK_TTL = Duration.ofMinutes(5);
public static final long DEFAULT_WAIT_MILLIS = 10000L;
private static final long RETRY_DELAY_MILLIS = 200L;
/** 续期失败后的重试次数:容忍 Redis 抖动/主从切换造成的瞬时失败。 */
private static final int RENEW_MAX_ATTEMPTS = 3;
private static final long RENEW_RETRY_DELAY_MILLIS = 200L;
private final DistributedJobLockService distributedJobLockService;
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
/** 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。 */
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(4, runnable -> {
/**
* 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。
* 线程数从 4 提到 16(2026-09 全维度审查):并发持锁任务数上升或 Redis 抖动时,
* 续期排队会让锁在续期完成前到期,另一实例即可接管同一任务。
*/
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(16, runnable -> {
Thread thread = new Thread(runnable, "task-lock-renewal");
thread.setDaemon(true);
return thread;
@@ -89,18 +96,48 @@ public class TaskDistributedLockService {
if (heldLock.closed) {
return;
}
boolean renewed = heldLock.delegate.renew(heldLock.ttl);
if (!renewed) {
heldLock.closed = true;
ScheduledFuture<?> future = heldLock.renewalFuture;
if (future != null) {
future.cancel(false);
}
log.warn("[task-lock] renewal failed lockName={}", lockName);
if (renewWithRetry(lockName, heldLock)) {
return;
}
heldLock.closed = true;
ScheduledFuture<?> future = heldLock.renewalFuture;
if (future != null) {
future.cancel(false);
}
// 此前单次失败即静默停止续期:本节点会继续以「仍持锁」的假设做破坏性操作,
// 而锁最迟在 TTL 后已可被另一实例获取 —— 必须留下 error 级线索便于事后追溯
log.error("[task-lock] 续期连续 {} 次失败,锁已失效 lockName={} —— "
+ "另一实例可能已接管该任务,本节点不应再对该资源做破坏性写操作",
RENEW_MAX_ATTEMPTS, lockName);
}, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS);
}
/**
* 续期重试:单次失败可能是 Redis 抖动 / Sentinel 主从切换造成的瞬时问题,
* 立即放弃会让本节点在锁已过期的情况下继续运行;重试仍失败才判锁丢失。
*/
private boolean renewWithRetry(String lockName, HeldLock heldLock) {
for (int attempt = 1; attempt <= RENEW_MAX_ATTEMPTS; attempt++) {
try {
if (heldLock.delegate.renew(heldLock.ttl)) {
return true;
}
} catch (Exception ex) {
log.warn("[task-lock] 续期异常 lockName={} attempt={}/{} err={}",
lockName, attempt, RENEW_MAX_ATTEMPTS, ex.getMessage());
}
if (attempt < RENEW_MAX_ATTEMPTS) {
try {
Thread.sleep(RENEW_RETRY_DELAY_MILLIS);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
return false;
}
}
}
return false;
}
@PreDestroy
public void shutdownRenewalExecutor() {
renewalExecutor.shutdownNow();
@@ -162,9 +162,14 @@ public class TaskHeartbeatService {
if (brandTask != null) {
String status = brandTask.getStatus();
if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) {
brandTask.setStatus("cancelled");
brandTask.setErrorMessage(safeReason);
int updated = brandCrawlTaskMapper.updateById(brandTask);
// 与 file 分支统一改为条件更新:整行 updateById 会拿读取快照覆盖并发写入的字段
// (客户端重启上报与品牌任务自身状态流转同时发生时)
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, brandTask.getId())
.in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING")
.set(BrandCrawlTaskEntity::getStatus, "cancelled")
.set(BrandCrawlTaskEntity::getErrorMessage, safeReason)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}",
taskId, safeReason);
@@ -0,0 +1,65 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 进度查询的归属过滤(2026-09 全维度审查修复)。
*
* <p>背景:progress/batch、progress/light 等轮询端点的 taskIds 完全由调用方指定,
* 服务端此前不做任何归属校验 —— 匿名遍历自增 taskId 即可读取他人任务的状态、
* 错误信息与结果文件状态。
*
* <p>渐进式收紧:<b>仅在调用方传入 userId 时过滤</b>,未传则保持原行为。
* 这样不会因前端/客户端某一处漏传就把结果过滤成空(表现为「任务进度全部消失」),
* 待各端都完成传参后再改为强制校验。
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class TaskProgressOwnershipSupport {
private final FileTaskMapper fileTaskMapper;
/**
* 过滤出属于 userId 的任务 id。
*
* @param taskIds 调用方请求的任务 id 列表
* @param userId 当前用户 id;为 null / 非正数时不做过滤(原样返回)
* @return 归属该用户的任务 id(保持原顺序)
*/
public List<Long> filterOwnedTaskIds(List<Long> taskIds, Long userId) {
if (taskIds == null || taskIds.isEmpty()) {
return taskIds;
}
if (userId == null || userId <= 0) {
return taskIds;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return normalized;
}
List<Long> owned = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getId, normalized))
.stream()
.map(FileTaskEntity::getId)
.toList();
if (owned.size() != normalized.size()) {
// 非归属任务出现在请求里:可能是越权尝试,也可能是前端历史遗留缓存了别人的 taskId
log.warn("[progress-ownership] 进度查询剔除非归属任务 userId={} 请求 {} 个 命中 {} 个",
userId, normalized.size(), owned.size());
}
return owned;
}
}
@@ -10,12 +10,16 @@ import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
@@ -26,18 +30,27 @@ public class TaskResultItemService {
private final TransientPayloadStorageService transientPayloadStorageService;
public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) {
replaceResultSnapshot(taskId, moduleType, resultId, scopeKey, snapshot, null);
}
private void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot,
Map<String, TaskResultItemEntity> existingCache) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) {
return;
}
String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim();
String itemKey = "result:" + resultId;
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot);
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot, existingCache);
}
public void replaceTaskSnapshots(Long taskId, String moduleType, List<? extends Object> snapshots, SnapshotKeyResolver resolver) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) {
return;
}
// 一次性载入本任务已有快照行:upsert 原本逐行 selectOne,N 行 × C 次回传
// 会产生 N×C 次单行查询(500 行 × 数百次回传 = 十万级往返)。
// 缓存含"已确认不存在"的占位(containsKey 判定),新行不再各查一次。
Map<String, TaskResultItemEntity> existingCache = loadSnapshotRows(taskId, moduleType);
for (Object snapshot : snapshots) {
if (snapshot == null) {
continue;
@@ -46,10 +59,27 @@ public class TaskResultItemService {
if (resultId == null || resultId <= 0) {
continue;
}
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot);
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot, existingCache);
}
}
/** 载入本任务全部快照行到 key=moduleType|scopeHash|itemKey 的缓存(与 find 的定位口径一致)。 */
private Map<String, TaskResultItemEntity> loadSnapshotRows(Long taskId, String moduleType) {
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.likeRight(TaskResultItemEntity::getItemKey, "result:"));
Map<String, TaskResultItemEntity> map = new HashMap<>();
for (TaskResultItemEntity row : rows) {
map.put(snapshotCacheKey(moduleType, row.getScopeHash(), row.getItemKey()), row);
}
return map;
}
private static String snapshotCacheKey(String moduleType, String scopeHash, String itemKey) {
return moduleType + "|" + scopeHash + "|" + itemKey;
}
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return List.of();
@@ -94,6 +124,34 @@ public class TaskResultItemService {
return count == null ? 0L : count;
}
/**
* 事务提交后再删除对象存储上的载荷。
*
* <p>此前在事务内同步逐行调用 {@code deletePayloadIfPresent}:单次删除超时 120s、重试 3 次,
* N 行结果就让 DB 连接被长时间占用(连接池单节点 30,几次并发删除即可打满并波及其它请求)。
* 项目自身在 TaskScopePayloadStorageService 已写明「不能把 RustFS/OSS 网络调用放进 DB 事务」。
* 提交后删除还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
*/
private void deletePayloadsAfterCommit(List<String> payloadValues) {
if (payloadValues == null || payloadValues.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
for (String value : payloadValues) {
transientPayloadStorageService.deletePayloadIfPresent(value);
}
}
});
return;
}
for (String value : payloadValues) {
transientPayloadStorageService.deletePayloadIfPresent(value);
}
}
@Transactional
public void deleteResultItem(Long taskId, String moduleType, Long resultId) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
@@ -105,9 +163,16 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId));
if (rows != null) {
for (TaskResultItemEntity row : rows) {
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
}
List<String> payloads = rows.stream()
.map(TaskResultItemEntity::getPayloadJson)
.filter(java.util.Objects::nonNull)
.toList();
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId));
deletePayloadsAfterCommit(payloads);
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
@@ -136,9 +201,15 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
if (rows != null) {
for (TaskResultItemEntity row : rows) {
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
}
List<String> payloads = rows.stream()
.map(TaskResultItemEntity::getPayloadJson)
.filter(java.util.Objects::nonNull)
.toList();
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
deletePayloadsAfterCommit(payloads);
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
@@ -163,7 +234,8 @@ public class TaskResultItemService {
String countryCode,
String asin,
String status,
Object payload) {
Object payload,
Map<String, TaskResultItemEntity> existingCache) {
String payloadJson = writeJson(payload);
String payloadHash = hash(payloadJson);
String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim();
@@ -171,7 +243,7 @@ public class TaskResultItemService {
String scopeHash = hash(normalizedScopeKey);
LocalDateTime now = LocalDateTime.now();
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey, existingCache);
if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) {
return;
}
@@ -194,9 +266,13 @@ public class TaskResultItemService {
entity.setUpdatedAt(now);
try {
taskResultItemMapper.insert(entity);
if (existingCache != null) {
existingCache.put(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey), entity);
}
return;
} catch (DuplicateKeyException ignored) {
existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
// 并发插入抢跑:绕过缓存强制回库取最新行
existing = find(taskId, moduleType, scopeHash, normalizedItemKey, null);
}
}
if (existing == null) {
@@ -217,6 +293,10 @@ public class TaskResultItemService {
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
.set(TaskResultItemEntity::getUpdatedAt, now));
// 上面走 wrapper 更新未回填实体,缓存项作废(同批再次遇到会重新查库,保证不脏读)
if (existingCache != null) {
existingCache.remove(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey));
}
}
private boolean isUnchanged(TaskResultItemEntity existing,
@@ -237,7 +317,15 @@ public class TaskResultItemService {
&& java.util.Objects.equals(existing.getPayloadHash(), payloadHash);
}
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) {
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey,
Map<String, TaskResultItemEntity> existingCache) {
if (existingCache != null) {
String cacheKey = snapshotCacheKey(moduleType, scopeHash, itemKey);
if (existingCache.containsKey(cacheKey)) {
// 命中即返回:值可能为 null,表示本批已确认该行不存在(无需再查库)
return existingCache.get(cacheKey);
}
}
return taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
@@ -272,11 +360,9 @@ public class TaskResultItemService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x"):后者每次哈希要走 32 次 Formatter,
// 而本方法在结果回写热路径上按行调用(10 万行 ≈ 320 万次格式化)
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result item", ex);
}
@@ -122,11 +122,8 @@ public class TaskResultPayloadService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result payload", ex);
}
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -32,13 +33,18 @@ public class TaskScopePayloadBufferMaintenanceService {
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskPressureProperties taskPressureProperties;
private final DistributedJobLockService distributedJobLockService;
private final InstanceMetadata instanceMetadata;
@EventListener(ApplicationReadyEvent.class)
public void recoverBufferedPayloadsOnStartup() {
// 锁按实例维度命名:恢复只处理本机 java.io.tmpdir 下的 buffer,
// 用全局锁会让双节点同时重启时只有一台能恢复自己的目录,
// 另一台的待恢复载荷在保留期后被 cleanup 任务当作过期文件删除。
String recoveryLockName = "task-scope-buffer:recover:" + instanceMetadata.getInstanceId();
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
distributedJobLockService.tryLock(recoveryLockName, RECOVERY_LOCK_TTL);
if (lockHandle == null) {
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
log.info("[task-scope-buffer] skip startup recovery because lock is held lockName={}", recoveryLockName);
return;
}
try (lockHandle) {
@@ -189,22 +189,47 @@ public class TransientPayloadDeleteOrchestrator {
candidates.addAll(jsonEncoded);
Set<String> referenced = new LinkedHashSet<>();
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, candidates));
for (TaskChunkEntity chunk : chunks) {
referenced.addAll(matchingPointers(chunk.getPayloadJson(), pointers));
}
List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, candidates)
.or()
.in(TaskScopeStateEntity::getStateJson, candidates)));
for (TaskScopeStateEntity scopeState : scopeStates) {
referenced.addAll(matchingPointers(scopeState.getParsedPayloadJson(), pointers));
referenced.addAll(matchingPointers(scopeState.getStateJson(), pointers));
// 指针 key 含 taskId 且引用不跨任务:按 taskId 分组后带 task_id 查询
// (uk_task_scope_chunk 最左前缀),避免 payload_json 这类 JSON 列的 IN 比较全表扫描。
// 解析不出 taskId 的候选值归入 key=null 的组,走不带 task_id 的全局查询(与原行为一致)。
for (Map.Entry<Long, List<String>> group : groupCandidatesByTaskId(candidates).entrySet()) {
Long taskId = group.getKey();
List<String> scoped = group.getValue();
LambdaQueryWrapper<TaskChunkEntity> chunkQuery = new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, scoped);
if (taskId != null) {
chunkQuery.eq(TaskChunkEntity::getTaskId, taskId);
}
for (TaskChunkEntity chunk : taskChunkMapper.selectList(chunkQuery)) {
referenced.addAll(matchingPointers(chunk.getPayloadJson(), pointers));
}
LambdaQueryWrapper<TaskScopeStateEntity> scopeQuery = new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, scoped)
.or()
.in(TaskScopeStateEntity::getStateJson, scoped));
if (taskId != null) {
scopeQuery.eq(TaskScopeStateEntity::getTaskId, taskId);
}
for (TaskScopeStateEntity scopeState : taskScopeStateMapper.selectList(scopeQuery)) {
referenced.addAll(matchingPointers(scopeState.getParsedPayloadJson(), pointers));
referenced.addAll(matchingPointers(scopeState.getStateJson(), pointers));
}
}
return referenced;
}
/** 按指针中的 taskId 分组候选值;无法解析的归入 key=null 组。 */
private Map<Long, List<String>> groupCandidatesByTaskId(List<String> candidates) {
Map<Long, List<String>> grouped = new LinkedHashMap<>();
for (String candidate : candidates) {
Long taskId = transientPayloadStorageService.extractTaskId(candidate);
grouped.computeIfAbsent(taskId, k -> new ArrayList<>()).add(candidate);
}
return grouped;
}
private Set<String> matchingPointers(String dbValue, List<String> pointers) {
Set<String> matches = new LinkedHashSet<>();
if (dbValue == null || dbValue.isBlank()) {
@@ -242,15 +242,16 @@ public class TransientPayloadStorageService {
return false;
}
List<String> values = new ArrayList<>(candidates);
Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, values));
// 对象 key 形如 <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json,引用不跨任务;
// 据此先用 task_id(uk_task_scope_chunk 的最左前缀)收敛范围,避免 payload_json 这类
// JSON 列上的 IN 比较退化成全表扫描(该方法是删除路径的公共链路且被逐行调用)。
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
Long pointerTaskId = extractTaskId(pointer);
Long chunkCount = referencedChunkCount(pointerTaskId, values);
if (chunkCount != null && chunkCount > 1L) {
return true;
}
Long scopeStateCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
return scopeStateCount != null && scopeStateCount > 0L;
} catch (Exception ex) {
log.warn("[transient-payload] reference check failed pointer={} err={}", pointer, ex.getMessage());
@@ -259,6 +260,67 @@ public class TransientPayloadStorageService {
}
}
/**
* 带 task_id 收敛的 chunk 引用计数;带 task 查不到时回退全局查询,
* 保证即使指针格式与预期不符也不会漏判引用(漏判会导致对象被误删)。
*/
private Long referencedChunkCount(Long taskId, List<String> values) {
if (taskId != null) {
Long scoped = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.in(TaskChunkEntity::getPayloadJson, values));
if (scoped != null && scoped > 0L) {
return scoped;
}
}
return taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, values));
}
/** 带 task_id 收敛的 scope_state 引用计数;回退逻辑同 {@link #referencedChunkCount}。 */
private Long referencedScopeStateCount(Long taskId, List<String> values) {
if (taskId != null) {
Long scoped = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
if (scoped != null && scoped > 0L) {
return scoped;
}
}
return taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
}
/**
* 从载荷指针解析任务 id。key 由 {@code buildObjectKey} 生成,格式固定为
* {@code <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json},
* taskId 是第三段且必为数字;解析失败返回 null(调用方退回全局查询)。
*
* <p>公开给批量引用反查(TransientPayloadDeleteOrchestrator)按 task 分组使用。
*/
public Long extractTaskId(String pointer) {
if (pointer == null) {
return null;
}
int colon = pointer.indexOf(':');
if (colon < 0 || colon == pointer.length() - 1) {
return null;
}
String[] segments = pointer.substring(colon + 1).split("/");
if (segments.length < 3) {
return null;
}
try {
return Long.parseLong(segments[2].trim());
} catch (NumberFormatException ex) {
return null;
}
}
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractPointer(oldValue);
String newPointer = extractPointer(newValue);