fix(cr-01/02/03/08/09/10/11/13): 会话世代、损耗共享基线、任务活性与工作台包
- CR-01:users.session_epoch(V172)+ JWT sessionEpoch;登录/登出原子推进, check_login/requireUser 先比世代再做设备比对;登出不再依赖 machine CAS - CR-02/09:cost-guard 基线落 Redis(CostGuardBaselineStore),按 elapsed 归一化; 非持锁实例刷新共享密钥快照;pause() 返回写成功,失败不通知且不推进基线 - CR-03:stuck_requeue_count(V173,上限 20)与 retry_count 分账;终态父任务心跳 停止 touchRunning 并 markFailedPermanent;良性停续期 markLost=false - CR-08:工作台包对象 key 加 UUID、0 字节拒绝、同 objectKey 幂等、同版本重复拒绝、 前端 confirm 超时抬到 10 分钟 - CR-10:Excel onSheetTotal 统一在 onHeader 之后回调(xlsx/xls) - CR-13/11:迁移头/文件名一致断言(V170 唯一豁免);V167 前置由 CompanyService 同事务 复制菜单树,已应用迁移未回改
This commit is contained in:
@@ -3,7 +3,14 @@ package com.nanri.aiimage.modules.auth.mapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.auth.model.entity.LoginUserEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
@Mapper
|
||||
public interface LoginUserMapper extends BaseMapper<LoginUserEntity> {
|
||||
|
||||
/**
|
||||
* 会话世代 +1(登录、登出各调用一次)。原子自增而不是「读出来再写回」:
|
||||
* 双节点同时登录时后者不能把前者的世代覆盖回去。返回 0 行表示用户已不存在。
|
||||
*/
|
||||
int incrementSessionEpoch(@Param("userId") Long userId);
|
||||
}
|
||||
|
||||
+6
@@ -43,4 +43,10 @@ public class LoginUserEntity {
|
||||
*/
|
||||
@TableField("disabled_at")
|
||||
private LocalDateTime disabledAt;
|
||||
/**
|
||||
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
|
||||
* 与 {@code AdminUserEntity.sessionEpoch} 同列,改一处必须同改两处。
|
||||
*/
|
||||
@TableField("session_epoch")
|
||||
private Long sessionEpoch;
|
||||
}
|
||||
|
||||
@@ -116,6 +116,7 @@ public class AuthService {
|
||||
accountExpiryService.assertLoginAllowed(user);
|
||||
|
||||
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
|
||||
advanceSessionEpoch(user);
|
||||
// 单设备登录:登录成功即把账号绑定到当前设备(last-login-wins),原设备下一次请求被顶下线
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.logBindOnLogin(user.getMachine(), deviceId, user.getId(), user.getUsername());
|
||||
@@ -125,7 +126,8 @@ public class AuthService {
|
||||
} else if (DeviceSessionPolicy.isRevoked(user.getMachine())) {
|
||||
// 开关关掉后登录不再写 machine,登出留下的吊销标记就没人清:此后每次登录都会先成功、
|
||||
// 再被 requireUser 的 assertNotRevoked 判成「登录已过期」,账号永久锁死。
|
||||
// 登录成功即视为新会话,清空标记(空=未绑定,放行语义);仍不写设备号。
|
||||
// 登录成功即视为新会话(上面已推进 session_epoch,旧令牌另有世代校验兜底),
|
||||
// 清空标记(空=未绑定,放行语义);仍不写设备号。
|
||||
log.warn("[auth] 登录清除登出吊销标记 userId={} username={}", user.getId(), user.getUsername());
|
||||
loginUserMapper.update(null, new LambdaUpdateWrapper<LoginUserEntity>()
|
||||
.eq(LoginUserEntity::getId, user.getId())
|
||||
@@ -149,6 +151,9 @@ public class AuthService {
|
||||
if (userId == null) {
|
||||
return;
|
||||
}
|
||||
// 世代 +1 是真正的吊销;它不依赖 machine 是否等于本次 deviceId,
|
||||
// 因此 machine 为空(单设备关闭时的常态)也一定留得下吊销效果。
|
||||
int epochUpdated = loginUserMapper.incrementSessionEpoch(userId);
|
||||
String marker = DeviceSessionPolicy.REVOKED_PREFIX + userId + "#"
|
||||
+ java.time.LocalDateTime.now().format(REVOKE_STAMP);
|
||||
String bound = deviceId == null ? "" : deviceId.trim();
|
||||
@@ -156,7 +161,8 @@ public class AuthService {
|
||||
.set(LoginUserEntity::getMachine, marker)
|
||||
.eq(LoginUserEntity::getId, userId)
|
||||
.eq(!bound.isEmpty(), LoginUserEntity::getMachine, bound));
|
||||
log.info("[auth] 登出吊销 userId={} deviceBound={} updated={}", userId, bound, updated);
|
||||
log.info("[auth] 登出吊销 userId={} deviceBound={} epochUpdated={} markerUpdated={}",
|
||||
userId, bound, epochUpdated, updated);
|
||||
}
|
||||
|
||||
/** 从令牌里取出身份再吊销;解析不出(已过期/伪造)就什么都不做——cookie 由调用方清。 */
|
||||
@@ -173,7 +179,20 @@ public class AuthService {
|
||||
log.warn("[auth] 登出时 subject 非数字,跳过吊销(cookie 仍由调用方清)");
|
||||
return;
|
||||
}
|
||||
logout(Long.parseLong(subject), DeviceSessionPolicy.claimDeviceId(claims));
|
||||
Long userId = Long.parseLong(subject);
|
||||
LoginUserEntity current = loginUserMapper.selectById(userId);
|
||||
if (current == null) {
|
||||
return;
|
||||
}
|
||||
// 旧令牌不能把新会话一起登出:只有当前世代的令牌才有权推进世代。
|
||||
long claimEpoch = DeviceSessionPolicy.claimSessionEpoch(claims);
|
||||
long storedEpoch = current.getSessionEpoch() == null ? 0L : current.getSessionEpoch();
|
||||
if (claimEpoch != storedEpoch) {
|
||||
log.warn("[auth] 登出令牌的会话世代已过期,跳过吊销 userId={} stored={} claim={}",
|
||||
userId, storedEpoch, claimEpoch);
|
||||
return;
|
||||
}
|
||||
logout(userId, DeviceSessionPolicy.claimDeviceId(claims));
|
||||
}
|
||||
|
||||
public LoginResultVo checkLogin(String token, String currentDeviceId) {
|
||||
@@ -196,6 +215,9 @@ public class AuthService {
|
||||
}
|
||||
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
|
||||
String claimDeviceId = DeviceSessionPolicy.claimDeviceId(claims);
|
||||
// 世代校验先于设备比对:登出/新登录后旧 token 在这里直接 401,不能靠 check_login 续期复活
|
||||
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
|
||||
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
|
||||
// 单设备登录:不拦截的话,被顶下线的旧 token 会在这里续期「复活」
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.assertSameDevice(user.getMachine(), claimDeviceId,
|
||||
@@ -238,8 +260,20 @@ public class AuthService {
|
||||
return authProperties.getCookieName();
|
||||
}
|
||||
|
||||
/** 登录推进世代;返回 0 行说明用户行已消失,不能再签发一个「世代比库还新」的令牌。 */
|
||||
private void advanceSessionEpoch(LoginUserEntity user) {
|
||||
int updated = loginUserMapper.incrementSessionEpoch(user.getId());
|
||||
if (updated != 1) {
|
||||
log.warn("[auth] 登录推进会话世代失败 userId={} updated={}", user.getId(), updated);
|
||||
throw new BusinessException(500, "登录状态更新失败,请重试");
|
||||
}
|
||||
long current = user.getSessionEpoch() == null ? 0L : user.getSessionEpoch();
|
||||
user.setSessionEpoch(current + 1);
|
||||
}
|
||||
|
||||
private LoginResultVo buildResult(LoginUserEntity user, String deviceId, boolean isAdmin) {
|
||||
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId);
|
||||
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId,
|
||||
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch());
|
||||
LoginResultVo vo = new LoginResultVo();
|
||||
vo.setUserId(user.getId());
|
||||
vo.setUsername(user.getUsername());
|
||||
|
||||
+2
-1
@@ -57,7 +57,8 @@ public class ClientCredentialController {
|
||||
vo.setMysqlPassword(blankToEmpty(clientCredentialProperties.getMysqlPassword()));
|
||||
// 续期与容忍是两件事:容忍让存量客户端不被 7 天边界掐断,这条把新客户端拉回正常轨道
|
||||
vo.setJwt(jwtService.issue(user.getId(), user.getUsername(),
|
||||
adminAuthSupport.currentDeviceId(request)));
|
||||
adminAuthSupport.currentDeviceId(request),
|
||||
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch()));
|
||||
// 只记身份不记值:便于事后回答"哪个账号在哪台机器上领过紫鸟口令"
|
||||
log.info("[client-credentials] 下发凭据 userId={} username={} deviceId={}",
|
||||
user.getId(), user.getUsername(), adminAuthSupport.currentDeviceId(request));
|
||||
|
||||
+2
@@ -22,6 +22,8 @@ public class TaskFileJobEntity {
|
||||
private String jobType;
|
||||
private String status;
|
||||
private Integer retryCount;
|
||||
/** 卡死扫描的重排次数(V173),与真失败 retry_count 分账;达到上限后扫描侧判死。 */
|
||||
private Integer stuckRequeueCount;
|
||||
private String errorMessage;
|
||||
private String resultFileUrl;
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
+20
-3
@@ -159,6 +159,12 @@ public class TaskDistributedLockService {
|
||||
return moduleType.trim().toLowerCase(Locale.ROOT).replace('_', '-');
|
||||
}
|
||||
|
||||
/** 同包用例读数:这把锁是否被标成续期丢失(良性停续期必须为 false)。 */
|
||||
boolean activeRenewalLost(String lockName) {
|
||||
ActiveTaskLockRegistry.ActiveLock lock = activeLocks.snapshot().get(lockName);
|
||||
return lock != null && lock.isRenewalLost();
|
||||
}
|
||||
|
||||
/** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */
|
||||
Map<String, Long> activeHeldMillis() {
|
||||
long now = System.currentTimeMillis();
|
||||
@@ -327,15 +333,26 @@ public class TaskDistributedLockService {
|
||||
/**
|
||||
* 停止看门狗续期,但**不释放**锁:父任务已终态时,本节点不该再给这把锁续命;
|
||||
* 仍由 {@link #close()} 那次带 token 的 RELEASE 收尾,避免把接管者的锁误删。
|
||||
*
|
||||
* <p>默认按「续期丢失」标记,保留给真正的续期失败路径;父任务终态这种良性停续期
|
||||
* 必须传 {@code markLost=false},否则 {@code _prod_watch} 的 ACTIVE_LOST 会把良性
|
||||
* 停续期计成异常。
|
||||
*/
|
||||
public void stopRenewal(String reason) {
|
||||
stopRenewal(reason, true);
|
||||
}
|
||||
|
||||
public void stopRenewal(String reason, boolean markLost) {
|
||||
if (heldLock.closed.compareAndSet(false, true)) {
|
||||
cancelRenewal(heldLock);
|
||||
activeLocks.markRenewalLost(heldLock.lockName);
|
||||
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={}",
|
||||
if (markLost) {
|
||||
activeLocks.markRenewalLost(heldLock.lockName);
|
||||
}
|
||||
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={} markLost={}",
|
||||
heldLock.lockName,
|
||||
System.currentTimeMillis() - heldLock.acquiredAtMillis,
|
||||
reason);
|
||||
reason,
|
||||
markLost);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+34
-3
@@ -28,6 +28,12 @@ public class TaskFileJobService {
|
||||
|
||||
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
|
||||
public static final int MAX_RETRY_COUNT = 5;
|
||||
/**
|
||||
* 卡死重排的独立上限。B28 之后卡死重排不扣 retry_count,若没有上限,一个无异常挂住的
|
||||
* job 会被无限放回 PENDING,并让 TaskResumeService 永远跳过克隆重试。20 次是
|
||||
* 「慢执行」与「真卡死」的分界:按 5 分钟 stuck 阈值约 100 分钟,远大于现网合法慢单。
|
||||
*/
|
||||
public static final int MAX_STUCK_REQUEUE = 20;
|
||||
|
||||
private final TaskFileJobMapper taskFileJobMapper;
|
||||
private final ApplicationEventPublisher applicationEventPublisher;
|
||||
@@ -419,16 +425,41 @@ public class TaskFileJobService {
|
||||
// RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在
|
||||
// MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行
|
||||
// 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后
|
||||
// 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、
|
||||
// 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。
|
||||
// 0 字节收场。卡死次数改用独立列 stuck_requeue_count 记账,达到上限才判死;
|
||||
// 这样既保住"慢执行不扣预算",又不会让无异常挂住的 job 无限重排。
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int stuckCount = job.getStuckRequeueCount() == null ? 0 : job.getStuckRequeueCount();
|
||||
if (stuckCount >= MAX_STUCK_REQUEUE) {
|
||||
int failed = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getStuckRequeueCount, MAX_STUCK_REQUEUE)
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
"文件生成任务连续 " + MAX_STUCK_REQUEUE + " 次卡死重排,已判失败")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (failed > 0) {
|
||||
exhaustedJobIds.add(job.getId());
|
||||
log.warn("[task-file-job][STUCK_REQUEUE_EXHAUSTED] jobId={} taskId={} moduleType={} stuckRequeueCount={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), stuckCount);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int nextStuckCount = stuckCount + 1;
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||
.set(TaskFileJobEntity::getStuckRequeueCount, nextStuckCount)
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
"文件生成任务运行超时,已重新排队(第 " + nextStuckCount + " 次)")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
|
||||
+16
-4
@@ -24,6 +24,7 @@ import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -199,15 +200,26 @@ public class TaskResultFileJobWorker {
|
||||
|
||||
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
|
||||
long interval = Math.max(1000L, heartbeatIntervalMillis);
|
||||
AtomicBoolean terminalStopped = new AtomicBoolean(false);
|
||||
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
|
||||
if (terminalStopped.get()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
|
||||
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
|
||||
if (handle != null) {
|
||||
handle.stopRenewal("parent-task-terminal");
|
||||
log.info("[task-file-job] stopped lock renewal because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
|
||||
if (terminalStopped.compareAndSet(false, true)) {
|
||||
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
|
||||
if (handle != null) {
|
||||
// 良性停续期:父任务已终态,不是续期失败,不能打 renewalLost=true 造成假告警
|
||||
handle.stopRenewal("parent-task-terminal", false);
|
||||
}
|
||||
// 终态后不再 touchRunning:否则 job 的 updated_at 一直被刷新,
|
||||
// job 级 stuck 扫描永远看不到这行,成为不可回收的 RUNNING 遗骸。
|
||||
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
|
||||
log.info("[task-file-job] parent task terminal; stopped renewal and failed job jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
}
|
||||
return;
|
||||
}
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
} catch (Exception ex) {
|
||||
|
||||
+5
@@ -15,6 +15,11 @@ public class CostGuardStatusVo {
|
||||
private LocalDateTime pausedAt;
|
||||
private TodayUsage today;
|
||||
private WindowUsage window;
|
||||
/**
|
||||
* window 是本实例最近一轮扫描的读数,不是集群全局值。前端据此标注「本节点读数」,
|
||||
* 避免把单个实例的窗口值误读成全局速率。
|
||||
*/
|
||||
private boolean windowLocalOnly;
|
||||
private Thresholds thresholds;
|
||||
private List<SharedSecretWarningVo> sharedWarnings;
|
||||
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 损耗扫描基线(用户 → 当日累计三元组 + 采样时刻),跨实例共享。
|
||||
*
|
||||
* <p>此前基线是单实例内存,而扫描锁是集群级:持锁方一轮换,新持锁方拿到的
|
||||
* {@code before} 可能是几分钟前的,增量会被当成 60 秒窗口,导致误暂停;反过来新实例
|
||||
* 首轮只建基线,又会漏掉一次窗口。基线落 Redis 后,任何实例接手都按真实 elapsed 归一化。
|
||||
*
|
||||
* <p>用 {@link NodeSharedStore#getFresh} 而不是 {@code get}:本地缓存优先会让每个实例
|
||||
* 永远读自己的旧基线,正好是这次要修的问题。
|
||||
*/
|
||||
@Service
|
||||
public class CostGuardBaselineStore {
|
||||
|
||||
private static final Duration TTL = Duration.ofHours(26);
|
||||
|
||||
private final NodeSharedStore<Long, Baseline> store;
|
||||
|
||||
public CostGuardBaselineStore(StringRedisTemplate redis, ObjectMapper objectMapper) {
|
||||
this.store = new NodeSharedStore<>("cost-guard:baseline", TTL, Baseline.class,
|
||||
redis, objectMapper, 0L);
|
||||
}
|
||||
|
||||
/** 测试用:只传一个共享 store,两个调度器实例即可模拟跨实例读数。 */
|
||||
CostGuardBaselineStore(NodeSharedStore<Long, Baseline> store) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
public Baseline get(Long userId) {
|
||||
return store.getFresh(userId);
|
||||
}
|
||||
|
||||
public void put(Long userId, Baseline baseline) {
|
||||
store.putNow(userId, baseline);
|
||||
}
|
||||
|
||||
public void remove(Long userId) {
|
||||
store.remove(userId);
|
||||
}
|
||||
|
||||
/** Jackson 需要无参构造;字段公开以保持序列化形状稳定。 */
|
||||
public static class Baseline {
|
||||
public long proxy;
|
||||
public long llmCalls;
|
||||
public long llmTokens;
|
||||
public long capturedAtMillis;
|
||||
|
||||
public Baseline() {
|
||||
}
|
||||
|
||||
public Baseline(long proxy, long llmCalls, long llmTokens, long capturedAtMillis) {
|
||||
this.proxy = proxy;
|
||||
this.llmCalls = llmCalls;
|
||||
this.llmTokens = llmTokens;
|
||||
this.capturedAtMillis = capturedAtMillis;
|
||||
}
|
||||
}
|
||||
}
|
||||
+133
-40
@@ -28,7 +28,10 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
* 每分钟全量 GROUP BY 的成本可以忽略。
|
||||
*
|
||||
* <p>只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)},
|
||||
* 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
|
||||
* 基线对齐到当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
|
||||
*
|
||||
* <p>基线落在 {@link CostGuardBaselineStore}(Redis),不是单实例内存:扫描锁是集群级的,
|
||||
* 持锁方轮换后新实例必须按真实 elapsed 归一化增量,否则会把多分钟增量当成 60 秒窗口误暂停。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -43,10 +46,11 @@ public class CostGuardScheduler {
|
||||
private final SharedSecretMonitorService sharedSecretMonitorService;
|
||||
private final NotificationDispatchService notificationDispatchService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final CostGuardBaselineStore baselineStore;
|
||||
|
||||
private final Map<Long, UserTotals> previousTotals = new ConcurrentHashMap<>();
|
||||
private final Map<Long, WindowDelta> lastDeltas = new ConcurrentHashMap<>();
|
||||
private volatile long lastSharedRefreshAt = 0L;
|
||||
private volatile long lastSharedNotifyAt = 0L;
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}",
|
||||
initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}")
|
||||
@@ -56,6 +60,9 @@ public class CostGuardScheduler {
|
||||
}
|
||||
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2));
|
||||
if (lock == null) {
|
||||
// 非持锁方也要刷新共享密钥快照:否则本实例 status() 永远读到空 warningsByUser,
|
||||
// 用户看到的「没有共用」与真实情况相反。通知只由持锁方发(见 runOnce)。
|
||||
maybeRefreshShared(false);
|
||||
log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮");
|
||||
return;
|
||||
}
|
||||
@@ -68,53 +75,70 @@ public class CostGuardScheduler {
|
||||
|
||||
/** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */
|
||||
void runOnce() {
|
||||
long nowMillis = System.currentTimeMillis();
|
||||
int windowSeconds = Math.max(60, properties.getWindowSeconds());
|
||||
Map<Long, UserTotals> current = loadTodayTotals();
|
||||
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
|
||||
int paused = 0;
|
||||
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
|
||||
Long userId = entry.getKey();
|
||||
UserTotals now = entry.getValue();
|
||||
UserTotals before = previousTotals.get(userId);
|
||||
CostGuardBaselineStore.Baseline before = baselineStore.get(userId);
|
||||
boolean advanceBaseline = true;
|
||||
if (before == null) {
|
||||
// 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次
|
||||
// 首轮:窗口增量不判(启动前的大批次不是分钟增量),但日累计已经越线必须暂停,
|
||||
// 否则重启/首次部署后当天永远不会被拦。
|
||||
deltas.put(userId, WindowDelta.ZERO);
|
||||
continue;
|
||||
if (!guardService.isPaused(userId)) {
|
||||
String reason = dailyBreachReason(now);
|
||||
if (reason != null && !pauseAndNotify(userId, reason)) {
|
||||
// 写失败不推进基线:下一轮仍按「已越线」重试,否则 crossed(before, now)
|
||||
// 会因 before 被抬到阈值之上而永远不再触发。
|
||||
advanceBaseline = false;
|
||||
} else if (reason != null) {
|
||||
paused++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
WindowDelta raw = new WindowDelta(
|
||||
Math.max(0, now.proxy - before.proxy),
|
||||
Math.max(0, now.llmCalls - before.llmCalls),
|
||||
Math.max(0, now.llmTokens - before.llmTokens));
|
||||
long elapsedSeconds = Math.max(1L, (nowMillis - before.capturedAtMillis) / 1000L);
|
||||
WindowDelta windowDelta = normalize(raw, elapsedSeconds, windowSeconds);
|
||||
deltas.put(userId, windowDelta);
|
||||
if (!guardService.isPaused(userId)) {
|
||||
String reason = breachReason(now, before, windowDelta, windowSeconds);
|
||||
if (reason != null && !pauseAndNotify(userId, reason)) {
|
||||
advanceBaseline = false;
|
||||
} else if (reason != null) {
|
||||
paused++;
|
||||
}
|
||||
}
|
||||
}
|
||||
WindowDelta delta = new WindowDelta(
|
||||
Math.max(0, now.proxy - before.proxy),
|
||||
Math.max(0, now.llmCalls - before.llmCalls),
|
||||
Math.max(0, now.llmTokens - before.llmTokens));
|
||||
deltas.put(userId, delta);
|
||||
if (guardService.isPaused(userId)) {
|
||||
continue;
|
||||
if (advanceBaseline) {
|
||||
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
|
||||
now.proxy, now.llmCalls, now.llmTokens, nowMillis));
|
||||
}
|
||||
String reason = breachReason(now, before, delta);
|
||||
if (reason == null) {
|
||||
continue;
|
||||
}
|
||||
guardService.pause(userId, reason);
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
|
||||
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
|
||||
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
|
||||
null);
|
||||
paused++;
|
||||
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
|
||||
}
|
||||
previousTotals.clear();
|
||||
previousTotals.putAll(current);
|
||||
lastDeltas.clear();
|
||||
lastDeltas.putAll(deltas);
|
||||
maybeRefreshShared();
|
||||
maybeRefreshShared(true);
|
||||
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
|
||||
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
|
||||
}
|
||||
|
||||
/** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */
|
||||
/**
|
||||
* 用户点击继续时调用:把基线对齐到当前值(而不是删掉基线)。删掉会让下一轮走「首轮」
|
||||
* 分支;首轮现在会按日累计判暂停,用户刚点继续就会被立刻再暂停。
|
||||
*/
|
||||
public void resetBaseline(Long userId) {
|
||||
if (userId == null) {
|
||||
return;
|
||||
}
|
||||
previousTotals.remove(userId);
|
||||
UserTotals current = todayTotals(userId);
|
||||
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
|
||||
current.proxy, current.llmCalls, current.llmTokens, System.currentTimeMillis()));
|
||||
lastDeltas.remove(userId);
|
||||
}
|
||||
|
||||
@@ -158,21 +182,35 @@ public class CostGuardScheduler {
|
||||
return totals;
|
||||
}
|
||||
|
||||
private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) {
|
||||
int seconds = Math.max(60, properties.getWindowSeconds());
|
||||
/** 暂停写成功才发通知、才计数:写失败时谎报「已暂停」比不暂停更糟。 */
|
||||
private boolean pauseAndNotify(Long userId, String reason) {
|
||||
if (!guardService.pause(userId, reason)) {
|
||||
log.warn("[cost-guard] 暂停写入失败,本轮不通知 userId={} reason={}", userId, reason);
|
||||
return false;
|
||||
}
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
|
||||
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
|
||||
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
|
||||
null);
|
||||
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
|
||||
return true;
|
||||
}
|
||||
|
||||
private String breachReason(UserTotals now, CostGuardBaselineStore.Baseline before,
|
||||
WindowDelta delta, int windowSeconds) {
|
||||
if (properties.getProxyWindowThreshold() > 0
|
||||
&& delta.proxy >= properties.getProxyWindowThreshold()) {
|
||||
return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 "
|
||||
&& delta.proxy() >= properties.getProxyWindowThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒代理提取 " + delta.proxy() + " 次,超过阈值 "
|
||||
+ properties.getProxyWindowThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowCallsThreshold() > 0
|
||||
&& delta.llmCalls >= properties.getLlmWindowCallsThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 "
|
||||
&& delta.llmCalls() >= properties.getLlmWindowCallsThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒 LLM 调用 " + delta.llmCalls() + " 次,超过阈值 "
|
||||
+ properties.getLlmWindowCallsThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowTokensThreshold() > 0
|
||||
&& delta.llmTokens >= properties.getLlmWindowTokensThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 "
|
||||
&& delta.llmTokens() >= properties.getLlmWindowTokensThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒 LLM 消耗 " + delta.llmTokens() + " tokens,超过阈值 "
|
||||
+ properties.getLlmWindowTokensThreshold();
|
||||
}
|
||||
if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) {
|
||||
@@ -190,18 +228,73 @@ public class CostGuardScheduler {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 首轮专用:没有上一轮基线可比,只按「已经越线」判日累计,不判窗口增量。 */
|
||||
private String dailyBreachReason(UserTotals now) {
|
||||
if (properties.getDailyProxyThreshold() > 0
|
||||
&& now.proxy >= properties.getDailyProxyThreshold()) {
|
||||
return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 "
|
||||
+ properties.getDailyProxyThreshold() + " 次";
|
||||
}
|
||||
if (properties.getDailyLlmCallsThreshold() > 0
|
||||
&& now.llmCalls >= properties.getDailyLlmCallsThreshold()) {
|
||||
return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 "
|
||||
+ properties.getDailyLlmCallsThreshold() + " 次";
|
||||
}
|
||||
if (properties.getDailyLlmTokensThreshold() > 0
|
||||
&& now.llmTokens >= properties.getDailyLlmTokensThreshold()) {
|
||||
return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 "
|
||||
+ properties.getDailyLlmTokensThreshold();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean crossed(long before, long now, long threshold) {
|
||||
return threshold > 0 && before < threshold && now >= threshold;
|
||||
}
|
||||
|
||||
private void maybeRefreshShared() {
|
||||
/**
|
||||
* 把「elapsed 秒的真实增量」换算成 60 秒窗口。持锁方轮换、扫描被拖长、进程重启后
|
||||
* 首轮接手都会让 elapsed > windowSeconds;不归一化就是把多分钟增量当一分钟判。
|
||||
*
|
||||
* <p>只向下缩放(elapsed 大于窗口时),elapsed 小于窗口时直接采用原始增量:
|
||||
* 短间隔的原始增量是保守下界,放大它只会在调度抖动时制造误暂停。
|
||||
*/
|
||||
private WindowDelta normalize(WindowDelta raw, long elapsedSeconds, int windowSeconds) {
|
||||
if (elapsedSeconds <= windowSeconds) {
|
||||
return raw;
|
||||
}
|
||||
return new WindowDelta(
|
||||
scaleToWindow(raw.proxy(), elapsedSeconds, windowSeconds),
|
||||
scaleToWindow(raw.llmCalls(), elapsedSeconds, windowSeconds),
|
||||
scaleToWindow(raw.llmTokens(), elapsedSeconds, windowSeconds));
|
||||
}
|
||||
|
||||
private long scaleToWindow(long value, long elapsedSeconds, int windowSeconds) {
|
||||
if (value <= 0) {
|
||||
return 0;
|
||||
}
|
||||
// 向上取整:宁可早一秒判红,也不把刚好越线的批次放过
|
||||
return Math.max(1L, (value * windowSeconds + elapsedSeconds - 1) / elapsedSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* 共享密钥快照按 shared-scan-interval-ms 刷新。通知只由持锁方发,避免三台实例各推一遍;
|
||||
* 但非持锁方也要刷新快照,否则它的 status() 永远看不到共享提示。
|
||||
*/
|
||||
private void maybeRefreshShared(boolean notify) {
|
||||
long now = System.currentTimeMillis();
|
||||
if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) {
|
||||
long interval = Math.max(60_000L, properties.getSharedScanIntervalMs());
|
||||
boolean refreshDue = now - lastSharedRefreshAt >= interval;
|
||||
boolean notifyDue = notify && now - lastSharedNotifyAt >= interval;
|
||||
if (!refreshDue && !notifyDue) {
|
||||
return;
|
||||
}
|
||||
lastSharedRefreshAt = now;
|
||||
// 不自己吞:runOnce 里这是最后一步,抛出由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
|
||||
sharedSecretMonitorService.refresh();
|
||||
if (notifyDue) {
|
||||
lastSharedNotifyAt = now;
|
||||
}
|
||||
// 不自己吞:refresh 失败由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
|
||||
sharedSecretMonitorService.refresh(notifyDue);
|
||||
}
|
||||
|
||||
public static final class UserTotals {
|
||||
|
||||
+1
@@ -30,6 +30,7 @@ public class CostGuardService {
|
||||
vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens));
|
||||
vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds,
|
||||
delta.proxy(), delta.llmCalls(), delta.llmTokens()));
|
||||
vo.setWindowLocalOnly(true);
|
||||
vo.setThresholds(new CostGuardStatusVo.Thresholds(
|
||||
properties.getProxyWindowThreshold(),
|
||||
properties.getLlmWindowCallsThreshold(),
|
||||
|
||||
+11
-1
@@ -49,6 +49,14 @@ public class SharedSecretMonitorService {
|
||||
|
||||
/** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */
|
||||
public synchronized void refresh() {
|
||||
refresh(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param notify 是否推当日提示。多实例下只有持锁方传 true,其余实例只刷新快照,
|
||||
* 否则每台实例各推一遍(虽然通知有去重键,但仍是三倍写入与扇出)。
|
||||
*/
|
||||
public synchronized void refresh(boolean notify) {
|
||||
if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) {
|
||||
warningsByUser = Map.of();
|
||||
snapshotAtMillis = System.currentTimeMillis();
|
||||
@@ -89,7 +97,9 @@ public class SharedSecretMonitorService {
|
||||
type, group.module.label(), accounts, threshold, message);
|
||||
for (Long userId : group.userIds) {
|
||||
next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning);
|
||||
pushDailyWarning(userId, group.module, entry.getKey(), warning);
|
||||
if (notify) {
|
||||
pushDailyWarning(userId, group.module, entry.getKey(), warning);
|
||||
}
|
||||
}
|
||||
}
|
||||
warningsByUser = next;
|
||||
|
||||
+8
-2
@@ -55,19 +55,25 @@ public class UserTaskGuardService {
|
||||
return state;
|
||||
}
|
||||
|
||||
public void pause(Long userId, String reason) {
|
||||
/**
|
||||
* @return 写入是否成功。调用方只有拿到 true 才能发「已暂停」通知;
|
||||
* 此前该方法吞掉异常,扫描器无论成败都通知,数据库抖动时用户被告知已暂停、任务却仍在派发。
|
||||
*/
|
||||
public boolean pause(Long userId, String reason) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
String safeReason = truncate(reason);
|
||||
try {
|
||||
guardMapper.pause(userId, safeReason);
|
||||
cache.remove(userId);
|
||||
log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason);
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
// 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里
|
||||
log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}",
|
||||
userId, safeReason, ex.getMessage(), ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+28
-7
@@ -18,6 +18,7 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
@@ -117,13 +118,16 @@ public class WorkbenchClientPackageService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 签发直传预签名:对象 key 由「毫秒时间戳 + 安全版本号」组成,每次上传互不覆盖。
|
||||
* 签发直传预签名:对象 key 由「毫秒时间戳 + 随机 UUID + 安全版本号」组成。
|
||||
* UUID 是 CR-08 的修复点:只靠毫秒时间戳时,多实例时钟相同或同毫秒两名管理员
|
||||
* 上传同一版本会生成同一个 key,后一次 PUT 覆盖前一次对象、先确认的 DB 行摘要失效。
|
||||
*/
|
||||
public Map<String, Object> presignPackage(String fileName, String version) {
|
||||
String normalizedName = requireZipFileName(fileName);
|
||||
String normalizedVersion = requireVersion(version);
|
||||
String objectKey = OBJECT_KEY_PREFIX
|
||||
+ OBJECT_NAME_FORMATTER.format(LocalDateTime.now())
|
||||
+ "-" + UUID.randomUUID().toString().replace("-", "")
|
||||
+ "-" + safeVersionKey(normalizedVersion) + ".zip";
|
||||
log.info("[workbench-client] 签发客户端包直传预签名 version={} fileName={} objectKey={} expirySeconds={}",
|
||||
normalizedVersion, normalizedName, objectKey, PRESIGN_EXPIRY_SECONDS);
|
||||
@@ -154,27 +158,44 @@ public class WorkbenchClientPackageService {
|
||||
normalizedVersion, objectKey);
|
||||
throw new BusinessException("版本号与上传对象不匹配,请重新上传");
|
||||
}
|
||||
// 幂等:重复 confirm 同一对象返回已登记行,不再报错(CR-08)
|
||||
WorkbenchClientPackageEntity existing = packageMapper.selectOne(
|
||||
new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey)
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
log.info("[workbench-client] 直传确认幂等命中 id={} objectKey={}", existing.getId(), objectKey);
|
||||
return toItemMap(existing);
|
||||
}
|
||||
long size = ossStorageService.workbenchClientObjectSize(objectKey);
|
||||
if (size < 0) {
|
||||
log.warn("[workbench-client] 直传确认失败:对象不存在 objectKey={}", objectKey);
|
||||
throw new BusinessException("对象存储中未找到上传的安装包,请重新上传");
|
||||
}
|
||||
if (size <= 0) {
|
||||
// 0 字节对象(空文件/中断上传)会被客户端拿到后解压失败,直接拒绝并清残
|
||||
log.warn("[workbench-client] 直传对象为空,清理残留 objectKey={}", objectKey);
|
||||
ossStorageService.removeWorkbenchClientObject(objectKey);
|
||||
throw new BusinessException("上传的安装包为空文件,请重新上传");
|
||||
}
|
||||
if (size > MAX_UPLOAD_BYTES) {
|
||||
log.warn("[workbench-client] 直传对象超过大小上限,清理残留 objectKey={} bytes={}", objectKey, size);
|
||||
ossStorageService.removeWorkbenchClientObject(objectKey);
|
||||
throw new BusinessException("文件超过允许的大小限制");
|
||||
}
|
||||
Long sameVersion = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getVersion, normalizedVersion));
|
||||
if (sameVersion != null && sameVersion > 0) {
|
||||
// 同版本不同字节会让已升级的装机按版本号比较后不升级,客户端还会拿到错误摘要
|
||||
log.warn("[workbench-client] 直传确认失败:版本已存在 version={} objectKey={}",
|
||||
normalizedVersion, objectKey);
|
||||
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
|
||||
}
|
||||
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
|
||||
if (sha256 == null || sha256.length() != 64) {
|
||||
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
|
||||
throw new BusinessException("安装包摘要计算失败,请重新上传");
|
||||
}
|
||||
Long duplicated = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey));
|
||||
if (duplicated != null && duplicated > 0) {
|
||||
log.warn("[workbench-client] 直传确认失败:对象已被记录,重复提交 objectKey={}", objectKey);
|
||||
throw new BusinessException("该安装包已登记,请勿重复提交");
|
||||
}
|
||||
|
||||
WorkbenchClientPackageEntity entity = new WorkbenchClientPackageEntity();
|
||||
entity.setVersion(normalizedVersion);
|
||||
|
||||
Reference in New Issue
Block a user