fix(cr-01/02/03/08/09/10/11/13): 会话世代、损耗共享基线、任务活性与工作台包

- CR-01:users.session_epoch(V172)+ JWT sessionEpoch;登录/登出原子推进,
  check_login/requireUser 先比世代再做设备比对;登出不再依赖 machine CAS
- CR-02/09:cost-guard 基线落 Redis(CostGuardBaselineStore),按 elapsed 归一化;
  非持锁实例刷新共享密钥快照;pause() 返回写成功,失败不通知且不推进基线
- CR-03:stuck_requeue_count(V173,上限 20)与 retry_count 分账;终态父任务心跳
  停止 touchRunning 并 markFailedPermanent;良性停续期 markLost=false
- CR-08:工作台包对象 key 加 UUID、0 字节拒绝、同 objectKey 幂等、同版本重复拒绝、
  前端 confirm 超时抬到 10 分钟
- CR-10:Excel onSheetTotal 统一在 onHeader 之后回调(xlsx/xls)
- CR-13/11:迁移头/文件名一致断言(V170 唯一豁免);V167 前置由 CompanyService 同事务
  复制菜单树,已应用迁移未回改
This commit is contained in:
2026-10-25 03:40:15 +08:00
parent 112c61afa8
commit c2f429f112
43 changed files with 1003 additions and 95 deletions
@@ -126,16 +126,13 @@ public final class ExcelStreamReader {
SheetCollector collector = new SheetCollector(sheetName, sheetNo, handler, recorder);
XSSFSheetXMLHandler sheetHandler =
new XSSFSheetXMLHandler(styles, sharedStrings, collector, formatter, false);
DimensionFilter filter = new DimensionFilter(parser, maxRows -> {
try {
handler.onSheetTotal(sheetName, sheetNo, maxRows);
} catch (Exception ex) {
throw recorder.record(ex);
}
});
// dimension 在 sheetData 之前出现,先缓存;等 onHeader 回调之后再对外发 onSheetTotal,
// 保持旧 EasyExcel「先表头、后总行数」的顺序(CR-10)。
DimensionFilter filter = new DimensionFilter(parser, collector::setApproximateTotalRows);
filter.setContentHandler(sheetHandler);
parser.setContentHandler(filter);
parser.parse(new InputSource(sheet));
collector.reportTotalIfNeeded();
}
private static void readHssf(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler)
@@ -152,11 +149,12 @@ public final class ExcelStreamReader {
final String sheetName = sheet.getSheetName();
final int currentSheetNo = sheetNo;
final int lastRow = sheet.getLastRowNum();
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
final Map<Integer, String> header = rowToMap(sheet.getRow(0), formatter);
if (!header.isEmpty()) {
callRecorder(recorder, () -> handler.onHeader(sheetName, currentSheetNo, header));
}
// 旧顺序是「表头 → 总行数」;xls 路径此前反了,统一回来(CR-10)
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
for (int rowIndex = 1; rowIndex <= lastRow; rowIndex++) {
Row row = sheet.getRow(rowIndex);
if (row == null) {
@@ -274,6 +272,9 @@ public final class ExcelStreamReader {
private final FailureRecorder recorder;
private Map<Integer, String> header = Map.of();
private Map<Integer, String> row = new LinkedHashMap<>();
private Integer approximateTotalRows;
private boolean headerReported;
private boolean totalReported;
private SheetCollector(String sheetName, int sheetNo, SheetRowHandler handler, FailureRecorder recorder) {
this.sheetName = sheetName;
@@ -282,6 +283,21 @@ public final class ExcelStreamReader {
this.recorder = recorder;
}
/** dimension 过滤器的回调:只缓存,等 onHeader 之后才对外发。 */
private void setApproximateTotalRows(Integer rows) {
this.approximateTotalRows = rows;
reportTotalIfNeeded();
}
private void reportTotalIfNeeded() {
if (!headerReported || totalReported) {
return;
}
totalReported = true;
callRecorder(recorder,
() -> handler.onSheetTotal(sheetName, sheetNo, approximateTotalRows));
}
@Override
public void startRow(int rowNum) {
row = new LinkedHashMap<>();
@@ -294,7 +310,9 @@ public final class ExcelStreamReader {
}
if (rowNum == 0) {
header = new LinkedHashMap<>(row);
headerReported = true;
callRecorder(recorder, () -> handler.onHeader(sheetName, sheetNo, header));
reportTotalIfNeeded();
} else {
Map<Integer, String> values = new LinkedHashMap<>(row);
int rowIndex = rowNum;
@@ -49,4 +49,10 @@ public class AdminUserEntity {
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
private LocalDateTime updatedAt;
private String machine;
/**
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
* 与 {@code LoginUserEntity.sessionEpoch} 同列,改一处必须同改两处。
*/
@TableField("session_epoch")
private Long sessionEpoch;
}
@@ -72,6 +72,9 @@ public class AdminAuthSupport {
// 吊销先于单设备比对:那条路径第一行就是 if (exempt) return;(超管豁免),
// 把吊销挂在它下面等于超管永不见吊销
DeviceSessionPolicy.assertNotRevoked(user.getMachine(), user.getId(), user.getUsername());
// 世代校验同样先于单设备比对,且不受 singleDevice 开关支配:登出/新登录后旧令牌一律失效
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
// 单设备登录:被新设备顶下线的旧 token 在此统一拦截(全站 requireUser 调用点自动生效)
if (authProperties.isSingleDeviceEnabled()) {
DeviceSessionPolicy.assertSameDevice(user.getMachine(), DeviceSessionPolicy.claimDeviceId(claims),
@@ -91,6 +91,30 @@ public final class DeviceSessionPolicy {
return raw == null ? "" : raw.toString().trim();
}
/**
* 从 JWT claims 中提取会话世代;缺失(本次升级前签发的旧令牌)按 0 读取,
* 与 {@code users.session_epoch} 的列默认值一致,升级不会统一踢掉存量会话。
*/
public static long claimSessionEpoch(Claims claims) {
Object raw = claims == null ? null : claims.get("sessionEpoch");
// JwtService 签发的是 Number;非 Number 一律视同缺失(按 0),不在这里做字符串兼容,
// 避免为一种不会出现的 claim 形状新增 catch(CatchSilenceRatchet 满格)。
return raw instanceof Number number ? number.longValue() : 0L;
}
/**
* 会话世代必须与库中值相等。登出与每次新登录都会 +1,因此登出前签发的令牌在
* 下一次请求即失效;不依赖 {@code machine} 上的 revoked@ 标记,也不受单设备开关支配。
*/
public static void assertSessionEpoch(Long storedEpoch, long claimEpoch, Long userId, String username) {
long current = storedEpoch == null ? 0L : storedEpoch;
if (current != claimEpoch) {
log.warn("[auth] 会话世代失效被拒 userId={} username={} stored={} claim={}",
userId, username, current, claimEpoch);
throw new BusinessException(401, "登录已过期,请重新登录");
}
}
/**
* 校验请求携带的 token 是否仍属于账号当前绑定的设备。
* 超管豁免;machine 为空(尚未绑定)放行;不匹配抛 4011。
@@ -85,13 +85,19 @@ public class JwtService {
return key;
}
public String issue(Long userId, String username, String deviceId) {
/**
* 签发令牌。{@code sessionEpoch} 必须取自 users.session_epoch(登录/登出各推进一次),
* 鉴权时与库中值比对;旧令牌没有这个 claim 时按 0 读取,见
* {@link DeviceSessionPolicy#claimSessionEpoch(Claims)}。
*/
public String issue(Long userId, String username, String deviceId, long sessionEpoch) {
Instant now = Instant.now();
Instant exp = now.plus(Duration.ofHours(props.getJwtTtlHours()));
return Jwts.builder()
.subject(String.valueOf(userId))
.claim("username", username)
.claim("deviceId", deviceId)
.claim("sessionEpoch", sessionEpoch)
.issuedAt(Date.from(now))
.expiration(Date.from(exp))
.signWith(signingKey())
@@ -74,6 +74,22 @@ public final class NodeSharedStore<K, V> {
return remote;
}
/**
* 远端优先;远端没有才回退本地。用于「锁在实例间轮换、新持锁方必须读到上一持锁方
* 写入的基线」这类状态——{@link #get} 的本地优先会让每个实例 forever 读自己的旧值。
*/
public V getFresh(K key) {
if (key == null) {
return null;
}
V remote = readRemote(key);
if (remote != null) {
local.put(key, remote);
return remote;
}
return local.get(key);
}
public boolean containsKey(K key) {
return get(key) != null;
}
@@ -3,7 +3,14 @@ package com.nanri.aiimage.modules.auth.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.auth.model.entity.LoginUserEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@Mapper
public interface LoginUserMapper extends BaseMapper<LoginUserEntity> {
/**
* 会话世代 +1(登录、登出各调用一次)。原子自增而不是「读出来再写回」:
* 双节点同时登录时后者不能把前者的世代覆盖回去。返回 0 行表示用户已不存在。
*/
int incrementSessionEpoch(@Param("userId") Long userId);
}
@@ -43,4 +43,10 @@ public class LoginUserEntity {
*/
@TableField("disabled_at")
private LocalDateTime disabledAt;
/**
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
* 与 {@code AdminUserEntity.sessionEpoch} 同列,改一处必须同改两处。
*/
@TableField("session_epoch")
private Long sessionEpoch;
}
@@ -116,6 +116,7 @@ public class AuthService {
accountExpiryService.assertLoginAllowed(user);
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
advanceSessionEpoch(user);
// 单设备登录:登录成功即把账号绑定到当前设备(last-login-wins),原设备下一次请求被顶下线
if (authProperties.isSingleDeviceEnabled()) {
DeviceSessionPolicy.logBindOnLogin(user.getMachine(), deviceId, user.getId(), user.getUsername());
@@ -125,7 +126,8 @@ public class AuthService {
} else if (DeviceSessionPolicy.isRevoked(user.getMachine())) {
// 开关关掉后登录不再写 machine,登出留下的吊销标记就没人清:此后每次登录都会先成功、
// 再被 requireUser 的 assertNotRevoked 判成「登录已过期」,账号永久锁死。
// 登录成功即视为新会话,清空标记(空=未绑定,放行语义);仍不写设备号。
// 登录成功即视为新会话(上面已推进 session_epoch,旧令牌另有世代校验兜底),
// 清空标记(空=未绑定,放行语义);仍不写设备号。
log.warn("[auth] 登录清除登出吊销标记 userId={} username={}", user.getId(), user.getUsername());
loginUserMapper.update(null, new LambdaUpdateWrapper<LoginUserEntity>()
.eq(LoginUserEntity::getId, user.getId())
@@ -149,6 +151,9 @@ public class AuthService {
if (userId == null) {
return;
}
// 世代 +1 是真正的吊销;它不依赖 machine 是否等于本次 deviceId,
// 因此 machine 为空(单设备关闭时的常态)也一定留得下吊销效果。
int epochUpdated = loginUserMapper.incrementSessionEpoch(userId);
String marker = DeviceSessionPolicy.REVOKED_PREFIX + userId + "#"
+ java.time.LocalDateTime.now().format(REVOKE_STAMP);
String bound = deviceId == null ? "" : deviceId.trim();
@@ -156,7 +161,8 @@ public class AuthService {
.set(LoginUserEntity::getMachine, marker)
.eq(LoginUserEntity::getId, userId)
.eq(!bound.isEmpty(), LoginUserEntity::getMachine, bound));
log.info("[auth] 登出吊销 userId={} deviceBound={} updated={}", userId, bound, updated);
log.info("[auth] 登出吊销 userId={} deviceBound={} epochUpdated={} markerUpdated={}",
userId, bound, epochUpdated, updated);
}
/** 从令牌里取出身份再吊销;解析不出(已过期/伪造)就什么都不做——cookie 由调用方清。 */
@@ -173,7 +179,20 @@ public class AuthService {
log.warn("[auth] 登出时 subject 非数字,跳过吊销(cookie 仍由调用方清)");
return;
}
logout(Long.parseLong(subject), DeviceSessionPolicy.claimDeviceId(claims));
Long userId = Long.parseLong(subject);
LoginUserEntity current = loginUserMapper.selectById(userId);
if (current == null) {
return;
}
// 旧令牌不能把新会话一起登出:只有当前世代的令牌才有权推进世代。
long claimEpoch = DeviceSessionPolicy.claimSessionEpoch(claims);
long storedEpoch = current.getSessionEpoch() == null ? 0L : current.getSessionEpoch();
if (claimEpoch != storedEpoch) {
log.warn("[auth] 登出令牌的会话世代已过期,跳过吊销 userId={} stored={} claim={}",
userId, storedEpoch, claimEpoch);
return;
}
logout(userId, DeviceSessionPolicy.claimDeviceId(claims));
}
public LoginResultVo checkLogin(String token, String currentDeviceId) {
@@ -196,6 +215,9 @@ public class AuthService {
}
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
String claimDeviceId = DeviceSessionPolicy.claimDeviceId(claims);
// 世代校验先于设备比对:登出/新登录后旧 token 在这里直接 401,不能靠 check_login 续期复活
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
// 单设备登录:不拦截的话,被顶下线的旧 token 会在这里续期「复活」
if (authProperties.isSingleDeviceEnabled()) {
DeviceSessionPolicy.assertSameDevice(user.getMachine(), claimDeviceId,
@@ -238,8 +260,20 @@ public class AuthService {
return authProperties.getCookieName();
}
/** 登录推进世代;返回 0 行说明用户行已消失,不能再签发一个「世代比库还新」的令牌。 */
private void advanceSessionEpoch(LoginUserEntity user) {
int updated = loginUserMapper.incrementSessionEpoch(user.getId());
if (updated != 1) {
log.warn("[auth] 登录推进会话世代失败 userId={} updated={}", user.getId(), updated);
throw new BusinessException(500, "登录状态更新失败,请重试");
}
long current = user.getSessionEpoch() == null ? 0L : user.getSessionEpoch();
user.setSessionEpoch(current + 1);
}
private LoginResultVo buildResult(LoginUserEntity user, String deviceId, boolean isAdmin) {
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId);
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId,
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch());
LoginResultVo vo = new LoginResultVo();
vo.setUserId(user.getId());
vo.setUsername(user.getUsername());
@@ -57,7 +57,8 @@ public class ClientCredentialController {
vo.setMysqlPassword(blankToEmpty(clientCredentialProperties.getMysqlPassword()));
// 续期与容忍是两件事:容忍让存量客户端不被 7 天边界掐断,这条把新客户端拉回正常轨道
vo.setJwt(jwtService.issue(user.getId(), user.getUsername(),
adminAuthSupport.currentDeviceId(request)));
adminAuthSupport.currentDeviceId(request),
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch()));
// 只记身份不记值:便于事后回答"哪个账号在哪台机器上领过紫鸟口令"
log.info("[client-credentials] 下发凭据 userId={} username={} deviceId={}",
user.getId(), user.getUsername(), adminAuthSupport.currentDeviceId(request));
@@ -22,6 +22,8 @@ public class TaskFileJobEntity {
private String jobType;
private String status;
private Integer retryCount;
/** 卡死扫描的重排次数(V173),与真失败 retry_count 分账;达到上限后扫描侧判死。 */
private Integer stuckRequeueCount;
private String errorMessage;
private String resultFileUrl;
private LocalDateTime createdAt;
@@ -159,6 +159,12 @@ public class TaskDistributedLockService {
return moduleType.trim().toLowerCase(Locale.ROOT).replace('_', '-');
}
/** 同包用例读数:这把锁是否被标成续期丢失(良性停续期必须为 false)。 */
boolean activeRenewalLost(String lockName) {
ActiveTaskLockRegistry.ActiveLock lock = activeLocks.snapshot().get(lockName);
return lock != null && lock.isRenewalLost();
}
/** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */
Map<String, Long> activeHeldMillis() {
long now = System.currentTimeMillis();
@@ -327,15 +333,26 @@ public class TaskDistributedLockService {
/**
* 停止看门狗续期,但**不释放**锁:父任务已终态时,本节点不该再给这把锁续命;
* 仍由 {@link #close()} 那次带 token 的 RELEASE 收尾,避免把接管者的锁误删。
*
* <p>默认按「续期丢失」标记,保留给真正的续期失败路径;父任务终态这种良性停续期
* 必须传 {@code markLost=false},否则 {@code _prod_watch} 的 ACTIVE_LOST 会把良性
* 停续期计成异常。
*/
public void stopRenewal(String reason) {
stopRenewal(reason, true);
}
public void stopRenewal(String reason, boolean markLost) {
if (heldLock.closed.compareAndSet(false, true)) {
cancelRenewal(heldLock);
activeLocks.markRenewalLost(heldLock.lockName);
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={}",
if (markLost) {
activeLocks.markRenewalLost(heldLock.lockName);
}
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={} markLost={}",
heldLock.lockName,
System.currentTimeMillis() - heldLock.acquiredAtMillis,
reason);
reason,
markLost);
}
}
@@ -28,6 +28,12 @@ public class TaskFileJobService {
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
public static final int MAX_RETRY_COUNT = 5;
/**
* 卡死重排的独立上限。B28 之后卡死重排不扣 retry_count,若没有上限,一个无异常挂住的
* job 会被无限放回 PENDING,并让 TaskResumeService 永远跳过克隆重试。20 次是
* 「慢执行」与「真卡死」的分界:按 5 分钟 stuck 阈值约 100 分钟,远大于现网合法慢单。
*/
public static final int MAX_STUCK_REQUEUE = 20;
private final TaskFileJobMapper taskFileJobMapper;
private final ApplicationEventPublisher applicationEventPublisher;
@@ -419,16 +425,41 @@ public class TaskFileJobService {
// RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在
// MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行
// 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后
// 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、
// 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。
// 0 字节收场。卡死次数改用独立列 stuck_requeue_count 记账,达到上限才判死;
// 这样既保住"慢执行不扣预算",又不会让无异常挂住的 job 无限重排。
LocalDateTime now = LocalDateTime.now();
int stuckCount = job.getStuckRequeueCount() == null ? 0 : job.getStuckRequeueCount();
if (stuckCount >= MAX_STUCK_REQUEUE) {
int failed = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.set(TaskFileJobEntity::getStuckRequeueCount, MAX_STUCK_REQUEUE)
.set(TaskFileJobEntity::getErrorMessage,
"文件生成任务连续 " + MAX_STUCK_REQUEUE + " 次卡死重排,已判失败")
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, now)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (failed > 0) {
exhaustedJobIds.add(job.getId());
log.warn("[task-file-job][STUCK_REQUEUE_EXHAUSTED] jobId={} taskId={} moduleType={} stuckRequeueCount={}",
job.getId(), job.getTaskId(), job.getModuleType(), stuckCount);
}
continue;
}
int nextStuckCount = stuckCount + 1;
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
.set(TaskFileJobEntity::getStuckRequeueCount, nextStuckCount)
.set(TaskFileJobEntity::getErrorMessage,
"文件生成任务运行超时,已重新排队(第 " + nextStuckCount + " 次)")
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, null)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
@@ -24,6 +24,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j
@Service
@@ -199,15 +200,26 @@ public class TaskResultFileJobWorker {
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
long interval = Math.max(1000L, heartbeatIntervalMillis);
AtomicBoolean terminalStopped = new AtomicBoolean(false);
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
if (terminalStopped.get()) {
return;
}
try {
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
if (handle != null) {
handle.stopRenewal("parent-task-terminal");
log.info("[task-file-job] stopped lock renewal because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
if (terminalStopped.compareAndSet(false, true)) {
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
if (handle != null) {
// 良性停续期:父任务已终态,不是续期失败,不能打 renewalLost=true 造成假告警
handle.stopRenewal("parent-task-terminal", false);
}
// 终态后不再 touchRunning:否则 job 的 updated_at 一直被刷新,
// job 级 stuck 扫描永远看不到这行,成为不可回收的 RUNNING 遗骸。
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
log.info("[task-file-job] parent task terminal; stopped renewal and failed job jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
job.getId(), job.getTaskId(), job.getModuleType());
}
return;
}
taskFileJobService.touchRunning(job.getId());
} catch (Exception ex) {
@@ -15,6 +15,11 @@ public class CostGuardStatusVo {
private LocalDateTime pausedAt;
private TodayUsage today;
private WindowUsage window;
/**
* window 是本实例最近一轮扫描的读数,不是集群全局值。前端据此标注「本节点读数」,
* 避免把单个实例的窗口值误读成全局速率。
*/
private boolean windowLocalOnly;
private Thresholds thresholds;
private List<SharedSecretWarningVo> sharedWarnings;
@@ -0,0 +1,66 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.NodeSharedStore;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
/**
* 损耗扫描基线(用户 → 当日累计三元组 + 采样时刻),跨实例共享。
*
* <p>此前基线是单实例内存,而扫描锁是集群级:持锁方一轮换,新持锁方拿到的
* {@code before} 可能是几分钟前的,增量会被当成 60 秒窗口,导致误暂停;反过来新实例
* 首轮只建基线,又会漏掉一次窗口。基线落 Redis 后,任何实例接手都按真实 elapsed 归一化。
*
* <p>用 {@link NodeSharedStore#getFresh} 而不是 {@code get}:本地缓存优先会让每个实例
* 永远读自己的旧基线,正好是这次要修的问题。
*/
@Service
public class CostGuardBaselineStore {
private static final Duration TTL = Duration.ofHours(26);
private final NodeSharedStore<Long, Baseline> store;
public CostGuardBaselineStore(StringRedisTemplate redis, ObjectMapper objectMapper) {
this.store = new NodeSharedStore<>("cost-guard:baseline", TTL, Baseline.class,
redis, objectMapper, 0L);
}
/** 测试用:只传一个共享 store,两个调度器实例即可模拟跨实例读数。 */
CostGuardBaselineStore(NodeSharedStore<Long, Baseline> store) {
this.store = store;
}
public Baseline get(Long userId) {
return store.getFresh(userId);
}
public void put(Long userId, Baseline baseline) {
store.putNow(userId, baseline);
}
public void remove(Long userId) {
store.remove(userId);
}
/** Jackson 需要无参构造;字段公开以保持序列化形状稳定。 */
public static class Baseline {
public long proxy;
public long llmCalls;
public long llmTokens;
public long capturedAtMillis;
public Baseline() {
}
public Baseline(long proxy, long llmCalls, long llmTokens, long capturedAtMillis) {
this.proxy = proxy;
this.llmCalls = llmCalls;
this.llmTokens = llmTokens;
this.capturedAtMillis = capturedAtMillis;
}
}
}
@@ -28,7 +28,10 @@ import java.util.concurrent.ConcurrentHashMap;
* 每分钟全量 GROUP BY 的成本可以忽略。
*
* <p>只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)},
* 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
* 基线对齐到当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
*
* <p>基线落在 {@link CostGuardBaselineStore}(Redis),不是单实例内存:扫描锁是集群级的,
* 持锁方轮换后新实例必须按真实 elapsed 归一化增量,否则会把多分钟增量当成 60 秒窗口误暂停。
*/
@Service
@RequiredArgsConstructor
@@ -43,10 +46,11 @@ public class CostGuardScheduler {
private final SharedSecretMonitorService sharedSecretMonitorService;
private final NotificationDispatchService notificationDispatchService;
private final DistributedJobLockService distributedJobLockService;
private final CostGuardBaselineStore baselineStore;
private final Map<Long, UserTotals> previousTotals = new ConcurrentHashMap<>();
private final Map<Long, WindowDelta> lastDeltas = new ConcurrentHashMap<>();
private volatile long lastSharedRefreshAt = 0L;
private volatile long lastSharedNotifyAt = 0L;
@Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}",
initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}")
@@ -56,6 +60,9 @@ public class CostGuardScheduler {
}
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2));
if (lock == null) {
// 非持锁方也要刷新共享密钥快照:否则本实例 status() 永远读到空 warningsByUser,
// 用户看到的「没有共用」与真实情况相反。通知只由持锁方发(见 runOnce)。
maybeRefreshShared(false);
log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮");
return;
}
@@ -68,53 +75,70 @@ public class CostGuardScheduler {
/** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */
void runOnce() {
long nowMillis = System.currentTimeMillis();
int windowSeconds = Math.max(60, properties.getWindowSeconds());
Map<Long, UserTotals> current = loadTodayTotals();
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
int paused = 0;
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
Long userId = entry.getKey();
UserTotals now = entry.getValue();
UserTotals before = previousTotals.get(userId);
CostGuardBaselineStore.Baseline before = baselineStore.get(userId);
boolean advanceBaseline = true;
if (before == null) {
// 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次
// 首轮:窗口增量不判(启动前的大批次不是分钟增量),但日累计已经越线必须暂停,
// 否则重启/首次部署后当天永远不会被拦。
deltas.put(userId, WindowDelta.ZERO);
continue;
if (!guardService.isPaused(userId)) {
String reason = dailyBreachReason(now);
if (reason != null && !pauseAndNotify(userId, reason)) {
// 写失败不推进基线:下一轮仍按「已越线」重试,否则 crossed(before, now)
// 会因 before 被抬到阈值之上而永远不再触发。
advanceBaseline = false;
} else if (reason != null) {
paused++;
}
}
} else {
WindowDelta raw = new WindowDelta(
Math.max(0, now.proxy - before.proxy),
Math.max(0, now.llmCalls - before.llmCalls),
Math.max(0, now.llmTokens - before.llmTokens));
long elapsedSeconds = Math.max(1L, (nowMillis - before.capturedAtMillis) / 1000L);
WindowDelta windowDelta = normalize(raw, elapsedSeconds, windowSeconds);
deltas.put(userId, windowDelta);
if (!guardService.isPaused(userId)) {
String reason = breachReason(now, before, windowDelta, windowSeconds);
if (reason != null && !pauseAndNotify(userId, reason)) {
advanceBaseline = false;
} else if (reason != null) {
paused++;
}
}
}
WindowDelta delta = new WindowDelta(
Math.max(0, now.proxy - before.proxy),
Math.max(0, now.llmCalls - before.llmCalls),
Math.max(0, now.llmTokens - before.llmTokens));
deltas.put(userId, delta);
if (guardService.isPaused(userId)) {
continue;
if (advanceBaseline) {
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
now.proxy, now.llmCalls, now.llmTokens, nowMillis));
}
String reason = breachReason(now, before, delta);
if (reason == null) {
continue;
}
guardService.pause(userId, reason);
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
null);
paused++;
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
}
previousTotals.clear();
previousTotals.putAll(current);
lastDeltas.clear();
lastDeltas.putAll(deltas);
maybeRefreshShared();
maybeRefreshShared(true);
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
}
/** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */
/**
* 用户点击继续时调用:把基线对齐到当前值(而不是删掉基线)。删掉会让下一轮走「首轮」
* 分支;首轮现在会按日累计判暂停,用户刚点继续就会被立刻再暂停。
*/
public void resetBaseline(Long userId) {
if (userId == null) {
return;
}
previousTotals.remove(userId);
UserTotals current = todayTotals(userId);
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
current.proxy, current.llmCalls, current.llmTokens, System.currentTimeMillis()));
lastDeltas.remove(userId);
}
@@ -158,21 +182,35 @@ public class CostGuardScheduler {
return totals;
}
private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) {
int seconds = Math.max(60, properties.getWindowSeconds());
/** 暂停写成功才发通知、才计数:写失败时谎报「已暂停」比不暂停更糟。 */
private boolean pauseAndNotify(Long userId, String reason) {
if (!guardService.pause(userId, reason)) {
log.warn("[cost-guard] 暂停写入失败,本轮不通知 userId={} reason={}", userId, reason);
return false;
}
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
null);
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
return true;
}
private String breachReason(UserTotals now, CostGuardBaselineStore.Baseline before,
WindowDelta delta, int windowSeconds) {
if (properties.getProxyWindowThreshold() > 0
&& delta.proxy >= properties.getProxyWindowThreshold()) {
return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 "
&& delta.proxy() >= properties.getProxyWindowThreshold()) {
return "最近约 " + windowSeconds + " 秒代理提取 " + delta.proxy() + " 次,超过阈值 "
+ properties.getProxyWindowThreshold() + " 次";
}
if (properties.getLlmWindowCallsThreshold() > 0
&& delta.llmCalls >= properties.getLlmWindowCallsThreshold()) {
return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 "
&& delta.llmCalls() >= properties.getLlmWindowCallsThreshold()) {
return "最近约 " + windowSeconds + " 秒 LLM 调用 " + delta.llmCalls() + " 次,超过阈值 "
+ properties.getLlmWindowCallsThreshold() + " 次";
}
if (properties.getLlmWindowTokensThreshold() > 0
&& delta.llmTokens >= properties.getLlmWindowTokensThreshold()) {
return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 "
&& delta.llmTokens() >= properties.getLlmWindowTokensThreshold()) {
return "最近约 " + windowSeconds + " 秒 LLM 消耗 " + delta.llmTokens() + " tokens,超过阈值 "
+ properties.getLlmWindowTokensThreshold();
}
if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) {
@@ -190,18 +228,73 @@ public class CostGuardScheduler {
return null;
}
/** 首轮专用:没有上一轮基线可比,只按「已经越线」判日累计,不判窗口增量。 */
private String dailyBreachReason(UserTotals now) {
if (properties.getDailyProxyThreshold() > 0
&& now.proxy >= properties.getDailyProxyThreshold()) {
return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 "
+ properties.getDailyProxyThreshold() + " 次";
}
if (properties.getDailyLlmCallsThreshold() > 0
&& now.llmCalls >= properties.getDailyLlmCallsThreshold()) {
return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 "
+ properties.getDailyLlmCallsThreshold() + " 次";
}
if (properties.getDailyLlmTokensThreshold() > 0
&& now.llmTokens >= properties.getDailyLlmTokensThreshold()) {
return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 "
+ properties.getDailyLlmTokensThreshold();
}
return null;
}
private boolean crossed(long before, long now, long threshold) {
return threshold > 0 && before < threshold && now >= threshold;
}
private void maybeRefreshShared() {
/**
* 把「elapsed 秒的真实增量」换算成 60 秒窗口。持锁方轮换、扫描被拖长、进程重启后
* 首轮接手都会让 elapsed > windowSeconds;不归一化就是把多分钟增量当一分钟判。
*
* <p>只向下缩放(elapsed 大于窗口时),elapsed 小于窗口时直接采用原始增量:
* 短间隔的原始增量是保守下界,放大它只会在调度抖动时制造误暂停。
*/
private WindowDelta normalize(WindowDelta raw, long elapsedSeconds, int windowSeconds) {
if (elapsedSeconds <= windowSeconds) {
return raw;
}
return new WindowDelta(
scaleToWindow(raw.proxy(), elapsedSeconds, windowSeconds),
scaleToWindow(raw.llmCalls(), elapsedSeconds, windowSeconds),
scaleToWindow(raw.llmTokens(), elapsedSeconds, windowSeconds));
}
private long scaleToWindow(long value, long elapsedSeconds, int windowSeconds) {
if (value <= 0) {
return 0;
}
// 向上取整:宁可早一秒判红,也不把刚好越线的批次放过
return Math.max(1L, (value * windowSeconds + elapsedSeconds - 1) / elapsedSeconds);
}
/**
* 共享密钥快照按 shared-scan-interval-ms 刷新。通知只由持锁方发,避免三台实例各推一遍;
* 但非持锁方也要刷新快照,否则它的 status() 永远看不到共享提示。
*/
private void maybeRefreshShared(boolean notify) {
long now = System.currentTimeMillis();
if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) {
long interval = Math.max(60_000L, properties.getSharedScanIntervalMs());
boolean refreshDue = now - lastSharedRefreshAt >= interval;
boolean notifyDue = notify && now - lastSharedNotifyAt >= interval;
if (!refreshDue && !notifyDue) {
return;
}
lastSharedRefreshAt = now;
// 不自己吞:runOnce 里这是最后一步,抛出由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
sharedSecretMonitorService.refresh();
if (notifyDue) {
lastSharedNotifyAt = now;
}
// 不自己吞:refresh 失败由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
sharedSecretMonitorService.refresh(notifyDue);
}
public static final class UserTotals {
@@ -30,6 +30,7 @@ public class CostGuardService {
vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens));
vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds,
delta.proxy(), delta.llmCalls(), delta.llmTokens()));
vo.setWindowLocalOnly(true);
vo.setThresholds(new CostGuardStatusVo.Thresholds(
properties.getProxyWindowThreshold(),
properties.getLlmWindowCallsThreshold(),
@@ -49,6 +49,14 @@ public class SharedSecretMonitorService {
/** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */
public synchronized void refresh() {
refresh(true);
}
/**
* @param notify 是否推当日提示。多实例下只有持锁方传 true,其余实例只刷新快照,
* 否则每台实例各推一遍(虽然通知有去重键,但仍是三倍写入与扇出)。
*/
public synchronized void refresh(boolean notify) {
if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) {
warningsByUser = Map.of();
snapshotAtMillis = System.currentTimeMillis();
@@ -89,7 +97,9 @@ public class SharedSecretMonitorService {
type, group.module.label(), accounts, threshold, message);
for (Long userId : group.userIds) {
next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning);
pushDailyWarning(userId, group.module, entry.getKey(), warning);
if (notify) {
pushDailyWarning(userId, group.module, entry.getKey(), warning);
}
}
}
warningsByUser = next;
@@ -55,19 +55,25 @@ public class UserTaskGuardService {
return state;
}
public void pause(Long userId, String reason) {
/**
* @return 写入是否成功。调用方只有拿到 true 才能发「已暂停」通知;
* 此前该方法吞掉异常,扫描器无论成败都通知,数据库抖动时用户被告知已暂停、任务却仍在派发。
*/
public boolean pause(Long userId, String reason) {
if (userId == null || userId <= 0) {
return;
return false;
}
String safeReason = truncate(reason);
try {
guardMapper.pause(userId, safeReason);
cache.remove(userId);
log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason);
return true;
} catch (Exception ex) {
// 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里
log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}",
userId, safeReason, ex.getMessage(), ex);
return false;
}
}
@@ -18,6 +18,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Pattern;
/**
@@ -117,13 +118,16 @@ public class WorkbenchClientPackageService {
}
/**
* 签发直传预签名:对象 key 由「毫秒时间戳 + 安全版本号」组成,每次上传互不覆盖。
* 签发直传预签名:对象 key 由「毫秒时间戳 + 随机 UUID + 安全版本号」组成。
* UUID 是 CR-08 的修复点:只靠毫秒时间戳时,多实例时钟相同或同毫秒两名管理员
* 上传同一版本会生成同一个 key,后一次 PUT 覆盖前一次对象、先确认的 DB 行摘要失效。
*/
public Map<String, Object> presignPackage(String fileName, String version) {
String normalizedName = requireZipFileName(fileName);
String normalizedVersion = requireVersion(version);
String objectKey = OBJECT_KEY_PREFIX
+ OBJECT_NAME_FORMATTER.format(LocalDateTime.now())
+ "-" + UUID.randomUUID().toString().replace("-", "")
+ "-" + safeVersionKey(normalizedVersion) + ".zip";
log.info("[workbench-client] 签发客户端包直传预签名 version={} fileName={} objectKey={} expirySeconds={}",
normalizedVersion, normalizedName, objectKey, PRESIGN_EXPIRY_SECONDS);
@@ -154,27 +158,44 @@ public class WorkbenchClientPackageService {
normalizedVersion, objectKey);
throw new BusinessException("版本号与上传对象不匹配,请重新上传");
}
// 幂等:重复 confirm 同一对象返回已登记行,不再报错(CR-08)
WorkbenchClientPackageEntity existing = packageMapper.selectOne(
new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey)
.last("LIMIT 1"));
if (existing != null) {
log.info("[workbench-client] 直传确认幂等命中 id={} objectKey={}", existing.getId(), objectKey);
return toItemMap(existing);
}
long size = ossStorageService.workbenchClientObjectSize(objectKey);
if (size < 0) {
log.warn("[workbench-client] 直传确认失败:对象不存在 objectKey={}", objectKey);
throw new BusinessException("对象存储中未找到上传的安装包,请重新上传");
}
if (size <= 0) {
// 0 字节对象(空文件/中断上传)会被客户端拿到后解压失败,直接拒绝并清残
log.warn("[workbench-client] 直传对象为空,清理残留 objectKey={}", objectKey);
ossStorageService.removeWorkbenchClientObject(objectKey);
throw new BusinessException("上传的安装包为空文件,请重新上传");
}
if (size > MAX_UPLOAD_BYTES) {
log.warn("[workbench-client] 直传对象超过大小上限,清理残留 objectKey={} bytes={}", objectKey, size);
ossStorageService.removeWorkbenchClientObject(objectKey);
throw new BusinessException("文件超过允许的大小限制");
}
Long sameVersion = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.eq(WorkbenchClientPackageEntity::getVersion, normalizedVersion));
if (sameVersion != null && sameVersion > 0) {
// 同版本不同字节会让已升级的装机按版本号比较后不升级,客户端还会拿到错误摘要
log.warn("[workbench-client] 直传确认失败:版本已存在 version={} objectKey={}",
normalizedVersion, objectKey);
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
}
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
if (sha256 == null || sha256.length() != 64) {
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
throw new BusinessException("安装包摘要计算失败,请重新上传");
}
Long duplicated = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey));
if (duplicated != null && duplicated > 0) {
log.warn("[workbench-client] 直传确认失败:对象已被记录,重复提交 objectKey={}", objectKey);
throw new BusinessException("该安装包已登记,请勿重复提交");
}
WorkbenchClientPackageEntity entity = new WorkbenchClientPackageEntity();
entity.setVersion(normalizedVersion);