fix: 全维度审查修复(安全/正确性/性能/稳定性/客户端/前端)
安全 - /api/ziniao/** 五个匿名接口加管理员鉴权(此前可匿名换取任意员工店铺登录令牌) - 删除 Flask 遗留后门:默认密码建超管 + 每次启动写生产 users 表(服务端与客户端各一份) - 进度/详情接口归属过滤:新增 TaskProgressOwnershipSupport,11 模块 progress/light 与 /tasks/batch 接入,DTO 补 userId,前端 13 个查询封装补传(未传时后端不过滤,兼容旧端) - 代理提取链接(含账密)不再明文入日志(新增 common/util/SecretMasking) - 全局异常兜底不再回传原始异常信息;内部令牌比较改常量时间 - 登录加失败计数与锁定(10 次锁 15 分钟);品牌源文件下载加 SSRF 防护 - AdminApiGuardFilter 覆盖前缀从 2 扩到 15(开关默认 false,行为不变,为收紧做准备) - 生产关闭 springdoc/knife4j(/doc.html 匿名可读全部接口定义) 正确性 - 40901/40902 拆分:锁竞争不再被伪装成 success=true(此前客户端停止重试、分片静默丢失) - 假成功收敛:集采明细批量写失败改为抛出、去重 worker 异常标失败、4 个 worker 改判 success 字段、publish 空 ASIN 行参与批次 flush、巡店删除全失败带 error 上报 - 客户端心跳 discard 移入 finally(7 模块,失败路径不再留僵尸 RUNNING 任务) - 状态机条件更新:跟价停止循环、集采 activate/fail、imagevideo 归档回填、店铺匹配提交 性能 - 前端入口包 JS 1.05MB→204KB、CSS 355KB→10.7KB(Element Plus 改按需 + el-config-provider) - 载荷引用计数按指针里的 taskId 收敛(原 JSON 列 IN 全表扫且逐行调用) - 店铺明细多值批量 INSERT;快照 upsert 预载缓存;结果文件列改单条 UPDATE - 新增迁移 V120(补 3 个缺失索引)/V121(删 4 个被覆盖的冗余索引)/V122(URL 前缀索引) 稳定性 - 新增 common/util/ThreadPools 有界线程池替换 5 处无界队列(防堆积 OOM) - Redis 锁释放改 Lua 原子校验(原裸 delete 会误删他人已过期的锁) - imagevideo 加死节点接管;锁续期失败重试;调度池 4→16;openStream 全部加超时 - 事务内远程对象删除移到提交后;启动恢复锁按实例命名 客户端 - 不再 taskkill /f /im chrome.exe(改为按调试端口精准回收,不杀用户自己的浏览器) - 密码检测不再无条件杀紫鸟进程;品牌检测加全局互斥(代理池不再互相覆盖) - base_dir 统一到 exe 目录(原被 os.getcwd() 覆盖,日志/缓存会分裂两个目录) - 缓存加定时清理;图片下载加超时;mkstemp 句柄托管 测试 - 同步更新受影响的契约测试(构造器签名/条件更新/方法改名/新增接口方法等) - 修复 FaultInjectionTest 等 3 处 mock 未 stub 流式 read 导致的读循环 OOM - mvn test 2795 个测试全绿
This commit is contained in:
+18
-4
@@ -92,14 +92,28 @@ public class StaleTaskRepairService {
|
||||
/** RUNNING 但心跳超过 2h 未刷新 → FAILED(心跳残留续命即僵尸)。 */
|
||||
private void repairFileTaskStaleRunning() {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(STALE_IDLE_MINUTES);
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
// 先查 id 再按主键更新:此前是一条无 LIMIT、无 module_type 的批量 UPDATE,
|
||||
// 而 biz_file_task 没有以 status 打头的索引 → 每 10 分钟一次全表扫,
|
||||
// 且按无索引条件命中行加 X 锁(长事务/锁扩散)。与另两个修复方法口径保持一致。
|
||||
List<FileTaskEntity> stale = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, cutoff)
|
||||
.last("limit 500"));
|
||||
if (stale.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String reason = "任务心跳超时,已自动失败";
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, "任务心跳超时,已自动失败")
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={}", updated);
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-10
@@ -23,11 +23,18 @@ public class TaskDistributedLockService {
|
||||
public static final Duration DEFAULT_LOCK_TTL = Duration.ofMinutes(5);
|
||||
public static final long DEFAULT_WAIT_MILLIS = 10000L;
|
||||
private static final long RETRY_DELAY_MILLIS = 200L;
|
||||
/** 续期失败后的重试次数:容忍 Redis 抖动/主从切换造成的瞬时失败。 */
|
||||
private static final int RENEW_MAX_ATTEMPTS = 3;
|
||||
private static final long RENEW_RETRY_DELAY_MILLIS = 200L;
|
||||
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
|
||||
/** 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。 */
|
||||
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
/**
|
||||
* 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。
|
||||
* 线程数从 4 提到 16(2026-09 全维度审查):并发持锁任务数上升或 Redis 抖动时,
|
||||
* 续期排队会让锁在续期完成前到期,另一实例即可接管同一任务。
|
||||
*/
|
||||
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(16, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-lock-renewal");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
@@ -89,18 +96,48 @@ public class TaskDistributedLockService {
|
||||
if (heldLock.closed) {
|
||||
return;
|
||||
}
|
||||
boolean renewed = heldLock.delegate.renew(heldLock.ttl);
|
||||
if (!renewed) {
|
||||
heldLock.closed = true;
|
||||
ScheduledFuture<?> future = heldLock.renewalFuture;
|
||||
if (future != null) {
|
||||
future.cancel(false);
|
||||
}
|
||||
log.warn("[task-lock] renewal failed lockName={}", lockName);
|
||||
if (renewWithRetry(lockName, heldLock)) {
|
||||
return;
|
||||
}
|
||||
heldLock.closed = true;
|
||||
ScheduledFuture<?> future = heldLock.renewalFuture;
|
||||
if (future != null) {
|
||||
future.cancel(false);
|
||||
}
|
||||
// 此前单次失败即静默停止续期:本节点会继续以「仍持锁」的假设做破坏性操作,
|
||||
// 而锁最迟在 TTL 后已可被另一实例获取 —— 必须留下 error 级线索便于事后追溯
|
||||
log.error("[task-lock] 续期连续 {} 次失败,锁已失效 lockName={} —— "
|
||||
+ "另一实例可能已接管该任务,本节点不应再对该资源做破坏性写操作",
|
||||
RENEW_MAX_ATTEMPTS, lockName);
|
||||
}, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 续期重试:单次失败可能是 Redis 抖动 / Sentinel 主从切换造成的瞬时问题,
|
||||
* 立即放弃会让本节点在锁已过期的情况下继续运行;重试仍失败才判锁丢失。
|
||||
*/
|
||||
private boolean renewWithRetry(String lockName, HeldLock heldLock) {
|
||||
for (int attempt = 1; attempt <= RENEW_MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
if (heldLock.delegate.renew(heldLock.ttl)) {
|
||||
return true;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-lock] 续期异常 lockName={} attempt={}/{} err={}",
|
||||
lockName, attempt, RENEW_MAX_ATTEMPTS, ex.getMessage());
|
||||
}
|
||||
if (attempt < RENEW_MAX_ATTEMPTS) {
|
||||
try {
|
||||
Thread.sleep(RENEW_RETRY_DELAY_MILLIS);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdownRenewalExecutor() {
|
||||
renewalExecutor.shutdownNow();
|
||||
|
||||
+8
-3
@@ -162,9 +162,14 @@ public class TaskHeartbeatService {
|
||||
if (brandTask != null) {
|
||||
String status = brandTask.getStatus();
|
||||
if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) {
|
||||
brandTask.setStatus("cancelled");
|
||||
brandTask.setErrorMessage(safeReason);
|
||||
int updated = brandCrawlTaskMapper.updateById(brandTask);
|
||||
// 与 file 分支统一改为条件更新:整行 updateById 会拿读取快照覆盖并发写入的字段
|
||||
// (客户端重启上报与品牌任务自身状态流转同时发生时)
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, brandTask.getId())
|
||||
.in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING")
|
||||
.set(BrandCrawlTaskEntity::getStatus, "cancelled")
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, safeReason)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}",
|
||||
taskId, safeReason);
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 进度查询的归属过滤(2026-09 全维度审查修复)。
|
||||
*
|
||||
* <p>背景:progress/batch、progress/light 等轮询端点的 taskIds 完全由调用方指定,
|
||||
* 服务端此前不做任何归属校验 —— 匿名遍历自增 taskId 即可读取他人任务的状态、
|
||||
* 错误信息与结果文件状态。
|
||||
*
|
||||
* <p>渐进式收紧:<b>仅在调用方传入 userId 时过滤</b>,未传则保持原行为。
|
||||
* 这样不会因前端/客户端某一处漏传就把结果过滤成空(表现为「任务进度全部消失」),
|
||||
* 待各端都完成传参后再改为强制校验。
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class TaskProgressOwnershipSupport {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
|
||||
/**
|
||||
* 过滤出属于 userId 的任务 id。
|
||||
*
|
||||
* @param taskIds 调用方请求的任务 id 列表
|
||||
* @param userId 当前用户 id;为 null / 非正数时不做过滤(原样返回)
|
||||
* @return 归属该用户的任务 id(保持原顺序)
|
||||
*/
|
||||
public List<Long> filterOwnedTaskIds(List<Long> taskIds, Long userId) {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return taskIds;
|
||||
}
|
||||
if (userId == null || userId <= 0) {
|
||||
return taskIds;
|
||||
}
|
||||
List<Long> normalized = taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (normalized.isEmpty()) {
|
||||
return normalized;
|
||||
}
|
||||
List<Long> owned = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.in(FileTaskEntity::getId, normalized))
|
||||
.stream()
|
||||
.map(FileTaskEntity::getId)
|
||||
.toList();
|
||||
if (owned.size() != normalized.size()) {
|
||||
// 非归属任务出现在请求里:可能是越权尝试,也可能是前端历史遗留缓存了别人的 taskId
|
||||
log.warn("[progress-ownership] 进度查询剔除非归属任务 userId={} 请求 {} 个 命中 {} 个",
|
||||
userId, normalized.size(), owned.size());
|
||||
}
|
||||
return owned;
|
||||
}
|
||||
}
|
||||
+103
-17
@@ -10,12 +10,16 @@ import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -26,18 +30,27 @@ public class TaskResultItemService {
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
|
||||
public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) {
|
||||
replaceResultSnapshot(taskId, moduleType, resultId, scopeKey, snapshot, null);
|
||||
}
|
||||
|
||||
private void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot,
|
||||
Map<String, TaskResultItemEntity> existingCache) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) {
|
||||
return;
|
||||
}
|
||||
String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim();
|
||||
String itemKey = "result:" + resultId;
|
||||
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot);
|
||||
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot, existingCache);
|
||||
}
|
||||
|
||||
public void replaceTaskSnapshots(Long taskId, String moduleType, List<? extends Object> snapshots, SnapshotKeyResolver resolver) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) {
|
||||
return;
|
||||
}
|
||||
// 一次性载入本任务已有快照行:upsert 原本逐行 selectOne,N 行 × C 次回传
|
||||
// 会产生 N×C 次单行查询(500 行 × 数百次回传 = 十万级往返)。
|
||||
// 缓存含"已确认不存在"的占位(containsKey 判定),新行不再各查一次。
|
||||
Map<String, TaskResultItemEntity> existingCache = loadSnapshotRows(taskId, moduleType);
|
||||
for (Object snapshot : snapshots) {
|
||||
if (snapshot == null) {
|
||||
continue;
|
||||
@@ -46,10 +59,27 @@ public class TaskResultItemService {
|
||||
if (resultId == null || resultId <= 0) {
|
||||
continue;
|
||||
}
|
||||
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot);
|
||||
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot, existingCache);
|
||||
}
|
||||
}
|
||||
|
||||
/** 载入本任务全部快照行到 key=moduleType|scopeHash|itemKey 的缓存(与 find 的定位口径一致)。 */
|
||||
private Map<String, TaskResultItemEntity> loadSnapshotRows(Long taskId, String moduleType) {
|
||||
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
.likeRight(TaskResultItemEntity::getItemKey, "result:"));
|
||||
Map<String, TaskResultItemEntity> map = new HashMap<>();
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
map.put(snapshotCacheKey(moduleType, row.getScopeHash(), row.getItemKey()), row);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private static String snapshotCacheKey(String moduleType, String scopeHash, String itemKey) {
|
||||
return moduleType + "|" + scopeHash + "|" + itemKey;
|
||||
}
|
||||
|
||||
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return List.of();
|
||||
@@ -94,6 +124,34 @@ public class TaskResultItemService {
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
/**
|
||||
* 事务提交后再删除对象存储上的载荷。
|
||||
*
|
||||
* <p>此前在事务内同步逐行调用 {@code deletePayloadIfPresent}:单次删除超时 120s、重试 3 次,
|
||||
* N 行结果就让 DB 连接被长时间占用(连接池单节点 30,几次并发删除即可打满并波及其它请求)。
|
||||
* 项目自身在 TaskScopePayloadStorageService 已写明「不能把 RustFS/OSS 网络调用放进 DB 事务」。
|
||||
* 提交后删除还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
|
||||
*/
|
||||
private void deletePayloadsAfterCommit(List<String> payloadValues) {
|
||||
if (payloadValues == null || payloadValues.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCommit() {
|
||||
for (String value : payloadValues) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
for (String value : payloadValues) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(value);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteResultItem(Long taskId, String moduleType, Long resultId) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
|
||||
@@ -105,9 +163,16 @@ public class TaskResultItemService {
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
.eq(TaskResultItemEntity::getResultId, resultId));
|
||||
if (rows != null) {
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
|
||||
}
|
||||
List<String> payloads = rows.stream()
|
||||
.map(TaskResultItemEntity::getPayloadJson)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.toList();
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
.eq(TaskResultItemEntity::getResultId, resultId));
|
||||
deletePayloadsAfterCommit(payloads);
|
||||
return;
|
||||
}
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
@@ -136,9 +201,15 @@ public class TaskResultItemService {
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType));
|
||||
if (rows != null) {
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
|
||||
}
|
||||
List<String> payloads = rows.stream()
|
||||
.map(TaskResultItemEntity::getPayloadJson)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.toList();
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType));
|
||||
deletePayloadsAfterCommit(payloads);
|
||||
return;
|
||||
}
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
@@ -163,7 +234,8 @@ public class TaskResultItemService {
|
||||
String countryCode,
|
||||
String asin,
|
||||
String status,
|
||||
Object payload) {
|
||||
Object payload,
|
||||
Map<String, TaskResultItemEntity> existingCache) {
|
||||
String payloadJson = writeJson(payload);
|
||||
String payloadHash = hash(payloadJson);
|
||||
String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim();
|
||||
@@ -171,7 +243,7 @@ public class TaskResultItemService {
|
||||
String scopeHash = hash(normalizedScopeKey);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
|
||||
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
|
||||
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey, existingCache);
|
||||
if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) {
|
||||
return;
|
||||
}
|
||||
@@ -194,9 +266,13 @@ public class TaskResultItemService {
|
||||
entity.setUpdatedAt(now);
|
||||
try {
|
||||
taskResultItemMapper.insert(entity);
|
||||
if (existingCache != null) {
|
||||
existingCache.put(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey), entity);
|
||||
}
|
||||
return;
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
|
||||
// 并发插入抢跑:绕过缓存强制回库取最新行
|
||||
existing = find(taskId, moduleType, scopeHash, normalizedItemKey, null);
|
||||
}
|
||||
}
|
||||
if (existing == null) {
|
||||
@@ -217,6 +293,10 @@ public class TaskResultItemService {
|
||||
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
|
||||
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
|
||||
.set(TaskResultItemEntity::getUpdatedAt, now));
|
||||
// 上面走 wrapper 更新未回填实体,缓存项作废(同批再次遇到会重新查库,保证不脏读)
|
||||
if (existingCache != null) {
|
||||
existingCache.remove(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isUnchanged(TaskResultItemEntity existing,
|
||||
@@ -237,7 +317,15 @@ public class TaskResultItemService {
|
||||
&& java.util.Objects.equals(existing.getPayloadHash(), payloadHash);
|
||||
}
|
||||
|
||||
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) {
|
||||
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey,
|
||||
Map<String, TaskResultItemEntity> existingCache) {
|
||||
if (existingCache != null) {
|
||||
String cacheKey = snapshotCacheKey(moduleType, scopeHash, itemKey);
|
||||
if (existingCache.containsKey(cacheKey)) {
|
||||
// 命中即返回:值可能为 null,表示本批已确认该行不存在(无需再查库)
|
||||
return existingCache.get(cacheKey);
|
||||
}
|
||||
}
|
||||
return taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
@@ -272,11 +360,9 @@ public class TaskResultItemService {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次哈希要走 32 次 Formatter,
|
||||
// 而本方法在结果回写热路径上按行调用(10 万行 ≈ 320 万次格式化)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash task result item", ex);
|
||||
}
|
||||
|
||||
+2
-5
@@ -122,11 +122,8 @@ public class TaskResultPayloadService {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash task result payload", ex);
|
||||
}
|
||||
|
||||
+8
-2
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -32,13 +33,18 @@ public class TaskScopePayloadBufferMaintenanceService {
|
||||
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||
private final TaskPressureProperties taskPressureProperties;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void recoverBufferedPayloadsOnStartup() {
|
||||
// 锁按实例维度命名:恢复只处理本机 java.io.tmpdir 下的 buffer,
|
||||
// 用全局锁会让双节点同时重启时只有一台能恢复自己的目录,
|
||||
// 另一台的待恢复载荷在保留期后被 cleanup 任务当作过期文件删除。
|
||||
String recoveryLockName = "task-scope-buffer:recover:" + instanceMetadata.getInstanceId();
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
|
||||
distributedJobLockService.tryLock(recoveryLockName, RECOVERY_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
|
||||
log.info("[task-scope-buffer] skip startup recovery because lock is held lockName={}", recoveryLockName);
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
|
||||
+37
-12
@@ -189,22 +189,47 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
candidates.addAll(jsonEncoded);
|
||||
|
||||
Set<String> referenced = new LinkedHashSet<>();
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, candidates));
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
referenced.addAll(matchingPointers(chunk.getPayloadJson(), pointers));
|
||||
}
|
||||
List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, candidates)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, candidates)));
|
||||
for (TaskScopeStateEntity scopeState : scopeStates) {
|
||||
referenced.addAll(matchingPointers(scopeState.getParsedPayloadJson(), pointers));
|
||||
referenced.addAll(matchingPointers(scopeState.getStateJson(), pointers));
|
||||
// 指针 key 含 taskId 且引用不跨任务:按 taskId 分组后带 task_id 查询
|
||||
// (uk_task_scope_chunk 最左前缀),避免 payload_json 这类 JSON 列的 IN 比较全表扫描。
|
||||
// 解析不出 taskId 的候选值归入 key=null 的组,走不带 task_id 的全局查询(与原行为一致)。
|
||||
for (Map.Entry<Long, List<String>> group : groupCandidatesByTaskId(candidates).entrySet()) {
|
||||
Long taskId = group.getKey();
|
||||
List<String> scoped = group.getValue();
|
||||
|
||||
LambdaQueryWrapper<TaskChunkEntity> chunkQuery = new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, scoped);
|
||||
if (taskId != null) {
|
||||
chunkQuery.eq(TaskChunkEntity::getTaskId, taskId);
|
||||
}
|
||||
for (TaskChunkEntity chunk : taskChunkMapper.selectList(chunkQuery)) {
|
||||
referenced.addAll(matchingPointers(chunk.getPayloadJson(), pointers));
|
||||
}
|
||||
|
||||
LambdaQueryWrapper<TaskScopeStateEntity> scopeQuery = new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, scoped)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, scoped));
|
||||
if (taskId != null) {
|
||||
scopeQuery.eq(TaskScopeStateEntity::getTaskId, taskId);
|
||||
}
|
||||
for (TaskScopeStateEntity scopeState : taskScopeStateMapper.selectList(scopeQuery)) {
|
||||
referenced.addAll(matchingPointers(scopeState.getParsedPayloadJson(), pointers));
|
||||
referenced.addAll(matchingPointers(scopeState.getStateJson(), pointers));
|
||||
}
|
||||
}
|
||||
return referenced;
|
||||
}
|
||||
|
||||
/** 按指针中的 taskId 分组候选值;无法解析的归入 key=null 组。 */
|
||||
private Map<Long, List<String>> groupCandidatesByTaskId(List<String> candidates) {
|
||||
Map<Long, List<String>> grouped = new LinkedHashMap<>();
|
||||
for (String candidate : candidates) {
|
||||
Long taskId = transientPayloadStorageService.extractTaskId(candidate);
|
||||
grouped.computeIfAbsent(taskId, k -> new ArrayList<>()).add(candidate);
|
||||
}
|
||||
return grouped;
|
||||
}
|
||||
|
||||
private Set<String> matchingPointers(String dbValue, List<String> pointers) {
|
||||
Set<String> matches = new LinkedHashSet<>();
|
||||
if (dbValue == null || dbValue.isBlank()) {
|
||||
|
||||
+68
-6
@@ -242,15 +242,16 @@ public class TransientPayloadStorageService {
|
||||
return false;
|
||||
}
|
||||
List<String> values = new ArrayList<>(candidates);
|
||||
Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
// 对象 key 形如 <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json,引用不跨任务;
|
||||
// 据此先用 task_id(uk_task_scope_chunk 的最左前缀)收敛范围,避免 payload_json 这类
|
||||
// JSON 列上的 IN 比较退化成全表扫描(该方法是删除路径的公共链路且被逐行调用)。
|
||||
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
|
||||
Long pointerTaskId = extractTaskId(pointer);
|
||||
Long chunkCount = referencedChunkCount(pointerTaskId, values);
|
||||
if (chunkCount != null && chunkCount > 1L) {
|
||||
return true;
|
||||
}
|
||||
Long scopeStateCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
|
||||
return scopeStateCount != null && scopeStateCount > 0L;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] reference check failed pointer={} err={}", pointer, ex.getMessage());
|
||||
@@ -259,6 +260,67 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 带 task_id 收敛的 chunk 引用计数;带 task 查不到时回退全局查询,
|
||||
* 保证即使指针格式与预期不符也不会漏判引用(漏判会导致对象被误删)。
|
||||
*/
|
||||
private Long referencedChunkCount(Long taskId, List<String> values) {
|
||||
if (taskId != null) {
|
||||
Long scoped = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
if (scoped != null && scoped > 0L) {
|
||||
return scoped;
|
||||
}
|
||||
}
|
||||
return taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
}
|
||||
|
||||
/** 带 task_id 收敛的 scope_state 引用计数;回退逻辑同 {@link #referencedChunkCount}。 */
|
||||
private Long referencedScopeStateCount(Long taskId, List<String> values) {
|
||||
if (taskId != null) {
|
||||
Long scoped = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
if (scoped != null && scoped > 0L) {
|
||||
return scoped;
|
||||
}
|
||||
}
|
||||
return taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 从载荷指针解析任务 id。key 由 {@code buildObjectKey} 生成,格式固定为
|
||||
* {@code <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json},
|
||||
* taskId 是第三段且必为数字;解析失败返回 null(调用方退回全局查询)。
|
||||
*
|
||||
* <p>公开给批量引用反查(TransientPayloadDeleteOrchestrator)按 task 分组使用。
|
||||
*/
|
||||
public Long extractTaskId(String pointer) {
|
||||
if (pointer == null) {
|
||||
return null;
|
||||
}
|
||||
int colon = pointer.indexOf(':');
|
||||
if (colon < 0 || colon == pointer.length() - 1) {
|
||||
return null;
|
||||
}
|
||||
String[] segments = pointer.substring(colon + 1).split("/");
|
||||
if (segments.length < 3) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return Long.parseLong(segments[2].trim());
|
||||
} catch (NumberFormatException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
|
||||
String oldPointer = extractPointer(oldValue);
|
||||
String newPointer = extractPointer(newValue);
|
||||
|
||||
Reference in New Issue
Block a user