fix(cr-01/02/03/08/09/10/11/13): 会话世代、损耗共享基线、任务活性与工作台包
- CR-01:users.session_epoch(V172)+ JWT sessionEpoch;登录/登出原子推进, check_login/requireUser 先比世代再做设备比对;登出不再依赖 machine CAS - CR-02/09:cost-guard 基线落 Redis(CostGuardBaselineStore),按 elapsed 归一化; 非持锁实例刷新共享密钥快照;pause() 返回写成功,失败不通知且不推进基线 - CR-03:stuck_requeue_count(V173,上限 20)与 retry_count 分账;终态父任务心跳 停止 touchRunning 并 markFailedPermanent;良性停续期 markLost=false - CR-08:工作台包对象 key 加 UUID、0 字节拒绝、同 objectKey 幂等、同版本重复拒绝、 前端 confirm 超时抬到 10 分钟 - CR-10:Excel onSheetTotal 统一在 onHeader 之后回调(xlsx/xls) - CR-13/11:迁移头/文件名一致断言(V170 唯一豁免);V167 前置由 CompanyService 同事务 复制菜单树,已应用迁移未回改
This commit is contained in:
@@ -126,16 +126,13 @@ public final class ExcelStreamReader {
|
||||
SheetCollector collector = new SheetCollector(sheetName, sheetNo, handler, recorder);
|
||||
XSSFSheetXMLHandler sheetHandler =
|
||||
new XSSFSheetXMLHandler(styles, sharedStrings, collector, formatter, false);
|
||||
DimensionFilter filter = new DimensionFilter(parser, maxRows -> {
|
||||
try {
|
||||
handler.onSheetTotal(sheetName, sheetNo, maxRows);
|
||||
} catch (Exception ex) {
|
||||
throw recorder.record(ex);
|
||||
}
|
||||
});
|
||||
// dimension 在 sheetData 之前出现,先缓存;等 onHeader 回调之后再对外发 onSheetTotal,
|
||||
// 保持旧 EasyExcel「先表头、后总行数」的顺序(CR-10)。
|
||||
DimensionFilter filter = new DimensionFilter(parser, collector::setApproximateTotalRows);
|
||||
filter.setContentHandler(sheetHandler);
|
||||
parser.setContentHandler(filter);
|
||||
parser.parse(new InputSource(sheet));
|
||||
collector.reportTotalIfNeeded();
|
||||
}
|
||||
|
||||
private static void readHssf(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler)
|
||||
@@ -152,11 +149,12 @@ public final class ExcelStreamReader {
|
||||
final String sheetName = sheet.getSheetName();
|
||||
final int currentSheetNo = sheetNo;
|
||||
final int lastRow = sheet.getLastRowNum();
|
||||
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
|
||||
final Map<Integer, String> header = rowToMap(sheet.getRow(0), formatter);
|
||||
if (!header.isEmpty()) {
|
||||
callRecorder(recorder, () -> handler.onHeader(sheetName, currentSheetNo, header));
|
||||
}
|
||||
// 旧顺序是「表头 → 总行数」;xls 路径此前反了,统一回来(CR-10)
|
||||
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
|
||||
for (int rowIndex = 1; rowIndex <= lastRow; rowIndex++) {
|
||||
Row row = sheet.getRow(rowIndex);
|
||||
if (row == null) {
|
||||
@@ -274,6 +272,9 @@ public final class ExcelStreamReader {
|
||||
private final FailureRecorder recorder;
|
||||
private Map<Integer, String> header = Map.of();
|
||||
private Map<Integer, String> row = new LinkedHashMap<>();
|
||||
private Integer approximateTotalRows;
|
||||
private boolean headerReported;
|
||||
private boolean totalReported;
|
||||
|
||||
private SheetCollector(String sheetName, int sheetNo, SheetRowHandler handler, FailureRecorder recorder) {
|
||||
this.sheetName = sheetName;
|
||||
@@ -282,6 +283,21 @@ public final class ExcelStreamReader {
|
||||
this.recorder = recorder;
|
||||
}
|
||||
|
||||
/** dimension 过滤器的回调:只缓存,等 onHeader 之后才对外发。 */
|
||||
private void setApproximateTotalRows(Integer rows) {
|
||||
this.approximateTotalRows = rows;
|
||||
reportTotalIfNeeded();
|
||||
}
|
||||
|
||||
private void reportTotalIfNeeded() {
|
||||
if (!headerReported || totalReported) {
|
||||
return;
|
||||
}
|
||||
totalReported = true;
|
||||
callRecorder(recorder,
|
||||
() -> handler.onSheetTotal(sheetName, sheetNo, approximateTotalRows));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startRow(int rowNum) {
|
||||
row = new LinkedHashMap<>();
|
||||
@@ -294,7 +310,9 @@ public final class ExcelStreamReader {
|
||||
}
|
||||
if (rowNum == 0) {
|
||||
header = new LinkedHashMap<>(row);
|
||||
headerReported = true;
|
||||
callRecorder(recorder, () -> handler.onHeader(sheetName, sheetNo, header));
|
||||
reportTotalIfNeeded();
|
||||
} else {
|
||||
Map<Integer, String> values = new LinkedHashMap<>(row);
|
||||
int rowIndex = rowNum;
|
||||
|
||||
@@ -49,4 +49,10 @@ public class AdminUserEntity {
|
||||
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||
private LocalDateTime updatedAt;
|
||||
private String machine;
|
||||
/**
|
||||
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
|
||||
* 与 {@code LoginUserEntity.sessionEpoch} 同列,改一处必须同改两处。
|
||||
*/
|
||||
@TableField("session_epoch")
|
||||
private Long sessionEpoch;
|
||||
}
|
||||
|
||||
@@ -72,6 +72,9 @@ public class AdminAuthSupport {
|
||||
// 吊销先于单设备比对:那条路径第一行就是 if (exempt) return;(超管豁免),
|
||||
// 把吊销挂在它下面等于超管永不见吊销
|
||||
DeviceSessionPolicy.assertNotRevoked(user.getMachine(), user.getId(), user.getUsername());
|
||||
// 世代校验同样先于单设备比对,且不受 singleDevice 开关支配:登出/新登录后旧令牌一律失效
|
||||
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
|
||||
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
|
||||
// 单设备登录:被新设备顶下线的旧 token 在此统一拦截(全站 requireUser 调用点自动生效)
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.assertSameDevice(user.getMachine(), DeviceSessionPolicy.claimDeviceId(claims),
|
||||
|
||||
@@ -91,6 +91,30 @@ public final class DeviceSessionPolicy {
|
||||
return raw == null ? "" : raw.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 JWT claims 中提取会话世代;缺失(本次升级前签发的旧令牌)按 0 读取,
|
||||
* 与 {@code users.session_epoch} 的列默认值一致,升级不会统一踢掉存量会话。
|
||||
*/
|
||||
public static long claimSessionEpoch(Claims claims) {
|
||||
Object raw = claims == null ? null : claims.get("sessionEpoch");
|
||||
// JwtService 签发的是 Number;非 Number 一律视同缺失(按 0),不在这里做字符串兼容,
|
||||
// 避免为一种不会出现的 claim 形状新增 catch(CatchSilenceRatchet 满格)。
|
||||
return raw instanceof Number number ? number.longValue() : 0L;
|
||||
}
|
||||
|
||||
/**
|
||||
* 会话世代必须与库中值相等。登出与每次新登录都会 +1,因此登出前签发的令牌在
|
||||
* 下一次请求即失效;不依赖 {@code machine} 上的 revoked@ 标记,也不受单设备开关支配。
|
||||
*/
|
||||
public static void assertSessionEpoch(Long storedEpoch, long claimEpoch, Long userId, String username) {
|
||||
long current = storedEpoch == null ? 0L : storedEpoch;
|
||||
if (current != claimEpoch) {
|
||||
log.warn("[auth] 会话世代失效被拒 userId={} username={} stored={} claim={}",
|
||||
userId, username, current, claimEpoch);
|
||||
throw new BusinessException(401, "登录已过期,请重新登录");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验请求携带的 token 是否仍属于账号当前绑定的设备。
|
||||
* 超管豁免;machine 为空(尚未绑定)放行;不匹配抛 4011。
|
||||
|
||||
@@ -85,13 +85,19 @@ public class JwtService {
|
||||
return key;
|
||||
}
|
||||
|
||||
public String issue(Long userId, String username, String deviceId) {
|
||||
/**
|
||||
* 签发令牌。{@code sessionEpoch} 必须取自 users.session_epoch(登录/登出各推进一次),
|
||||
* 鉴权时与库中值比对;旧令牌没有这个 claim 时按 0 读取,见
|
||||
* {@link DeviceSessionPolicy#claimSessionEpoch(Claims)}。
|
||||
*/
|
||||
public String issue(Long userId, String username, String deviceId, long sessionEpoch) {
|
||||
Instant now = Instant.now();
|
||||
Instant exp = now.plus(Duration.ofHours(props.getJwtTtlHours()));
|
||||
return Jwts.builder()
|
||||
.subject(String.valueOf(userId))
|
||||
.claim("username", username)
|
||||
.claim("deviceId", deviceId)
|
||||
.claim("sessionEpoch", sessionEpoch)
|
||||
.issuedAt(Date.from(now))
|
||||
.expiration(Date.from(exp))
|
||||
.signWith(signingKey())
|
||||
|
||||
@@ -74,6 +74,22 @@ public final class NodeSharedStore<K, V> {
|
||||
return remote;
|
||||
}
|
||||
|
||||
/**
|
||||
* 远端优先;远端没有才回退本地。用于「锁在实例间轮换、新持锁方必须读到上一持锁方
|
||||
* 写入的基线」这类状态——{@link #get} 的本地优先会让每个实例 forever 读自己的旧值。
|
||||
*/
|
||||
public V getFresh(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V remote = readRemote(key);
|
||||
if (remote != null) {
|
||||
local.put(key, remote);
|
||||
return remote;
|
||||
}
|
||||
return local.get(key);
|
||||
}
|
||||
|
||||
public boolean containsKey(K key) {
|
||||
return get(key) != null;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,14 @@ package com.nanri.aiimage.modules.auth.mapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.auth.model.entity.LoginUserEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
@Mapper
|
||||
public interface LoginUserMapper extends BaseMapper<LoginUserEntity> {
|
||||
|
||||
/**
|
||||
* 会话世代 +1(登录、登出各调用一次)。原子自增而不是「读出来再写回」:
|
||||
* 双节点同时登录时后者不能把前者的世代覆盖回去。返回 0 行表示用户已不存在。
|
||||
*/
|
||||
int incrementSessionEpoch(@Param("userId") Long userId);
|
||||
}
|
||||
|
||||
+6
@@ -43,4 +43,10 @@ public class LoginUserEntity {
|
||||
*/
|
||||
@TableField("disabled_at")
|
||||
private LocalDateTime disabledAt;
|
||||
/**
|
||||
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
|
||||
* 与 {@code AdminUserEntity.sessionEpoch} 同列,改一处必须同改两处。
|
||||
*/
|
||||
@TableField("session_epoch")
|
||||
private Long sessionEpoch;
|
||||
}
|
||||
|
||||
@@ -116,6 +116,7 @@ public class AuthService {
|
||||
accountExpiryService.assertLoginAllowed(user);
|
||||
|
||||
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
|
||||
advanceSessionEpoch(user);
|
||||
// 单设备登录:登录成功即把账号绑定到当前设备(last-login-wins),原设备下一次请求被顶下线
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.logBindOnLogin(user.getMachine(), deviceId, user.getId(), user.getUsername());
|
||||
@@ -125,7 +126,8 @@ public class AuthService {
|
||||
} else if (DeviceSessionPolicy.isRevoked(user.getMachine())) {
|
||||
// 开关关掉后登录不再写 machine,登出留下的吊销标记就没人清:此后每次登录都会先成功、
|
||||
// 再被 requireUser 的 assertNotRevoked 判成「登录已过期」,账号永久锁死。
|
||||
// 登录成功即视为新会话,清空标记(空=未绑定,放行语义);仍不写设备号。
|
||||
// 登录成功即视为新会话(上面已推进 session_epoch,旧令牌另有世代校验兜底),
|
||||
// 清空标记(空=未绑定,放行语义);仍不写设备号。
|
||||
log.warn("[auth] 登录清除登出吊销标记 userId={} username={}", user.getId(), user.getUsername());
|
||||
loginUserMapper.update(null, new LambdaUpdateWrapper<LoginUserEntity>()
|
||||
.eq(LoginUserEntity::getId, user.getId())
|
||||
@@ -149,6 +151,9 @@ public class AuthService {
|
||||
if (userId == null) {
|
||||
return;
|
||||
}
|
||||
// 世代 +1 是真正的吊销;它不依赖 machine 是否等于本次 deviceId,
|
||||
// 因此 machine 为空(单设备关闭时的常态)也一定留得下吊销效果。
|
||||
int epochUpdated = loginUserMapper.incrementSessionEpoch(userId);
|
||||
String marker = DeviceSessionPolicy.REVOKED_PREFIX + userId + "#"
|
||||
+ java.time.LocalDateTime.now().format(REVOKE_STAMP);
|
||||
String bound = deviceId == null ? "" : deviceId.trim();
|
||||
@@ -156,7 +161,8 @@ public class AuthService {
|
||||
.set(LoginUserEntity::getMachine, marker)
|
||||
.eq(LoginUserEntity::getId, userId)
|
||||
.eq(!bound.isEmpty(), LoginUserEntity::getMachine, bound));
|
||||
log.info("[auth] 登出吊销 userId={} deviceBound={} updated={}", userId, bound, updated);
|
||||
log.info("[auth] 登出吊销 userId={} deviceBound={} epochUpdated={} markerUpdated={}",
|
||||
userId, bound, epochUpdated, updated);
|
||||
}
|
||||
|
||||
/** 从令牌里取出身份再吊销;解析不出(已过期/伪造)就什么都不做——cookie 由调用方清。 */
|
||||
@@ -173,7 +179,20 @@ public class AuthService {
|
||||
log.warn("[auth] 登出时 subject 非数字,跳过吊销(cookie 仍由调用方清)");
|
||||
return;
|
||||
}
|
||||
logout(Long.parseLong(subject), DeviceSessionPolicy.claimDeviceId(claims));
|
||||
Long userId = Long.parseLong(subject);
|
||||
LoginUserEntity current = loginUserMapper.selectById(userId);
|
||||
if (current == null) {
|
||||
return;
|
||||
}
|
||||
// 旧令牌不能把新会话一起登出:只有当前世代的令牌才有权推进世代。
|
||||
long claimEpoch = DeviceSessionPolicy.claimSessionEpoch(claims);
|
||||
long storedEpoch = current.getSessionEpoch() == null ? 0L : current.getSessionEpoch();
|
||||
if (claimEpoch != storedEpoch) {
|
||||
log.warn("[auth] 登出令牌的会话世代已过期,跳过吊销 userId={} stored={} claim={}",
|
||||
userId, storedEpoch, claimEpoch);
|
||||
return;
|
||||
}
|
||||
logout(userId, DeviceSessionPolicy.claimDeviceId(claims));
|
||||
}
|
||||
|
||||
public LoginResultVo checkLogin(String token, String currentDeviceId) {
|
||||
@@ -196,6 +215,9 @@ public class AuthService {
|
||||
}
|
||||
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
|
||||
String claimDeviceId = DeviceSessionPolicy.claimDeviceId(claims);
|
||||
// 世代校验先于设备比对:登出/新登录后旧 token 在这里直接 401,不能靠 check_login 续期复活
|
||||
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
|
||||
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
|
||||
// 单设备登录:不拦截的话,被顶下线的旧 token 会在这里续期「复活」
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.assertSameDevice(user.getMachine(), claimDeviceId,
|
||||
@@ -238,8 +260,20 @@ public class AuthService {
|
||||
return authProperties.getCookieName();
|
||||
}
|
||||
|
||||
/** 登录推进世代;返回 0 行说明用户行已消失,不能再签发一个「世代比库还新」的令牌。 */
|
||||
private void advanceSessionEpoch(LoginUserEntity user) {
|
||||
int updated = loginUserMapper.incrementSessionEpoch(user.getId());
|
||||
if (updated != 1) {
|
||||
log.warn("[auth] 登录推进会话世代失败 userId={} updated={}", user.getId(), updated);
|
||||
throw new BusinessException(500, "登录状态更新失败,请重试");
|
||||
}
|
||||
long current = user.getSessionEpoch() == null ? 0L : user.getSessionEpoch();
|
||||
user.setSessionEpoch(current + 1);
|
||||
}
|
||||
|
||||
private LoginResultVo buildResult(LoginUserEntity user, String deviceId, boolean isAdmin) {
|
||||
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId);
|
||||
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId,
|
||||
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch());
|
||||
LoginResultVo vo = new LoginResultVo();
|
||||
vo.setUserId(user.getId());
|
||||
vo.setUsername(user.getUsername());
|
||||
|
||||
+2
-1
@@ -57,7 +57,8 @@ public class ClientCredentialController {
|
||||
vo.setMysqlPassword(blankToEmpty(clientCredentialProperties.getMysqlPassword()));
|
||||
// 续期与容忍是两件事:容忍让存量客户端不被 7 天边界掐断,这条把新客户端拉回正常轨道
|
||||
vo.setJwt(jwtService.issue(user.getId(), user.getUsername(),
|
||||
adminAuthSupport.currentDeviceId(request)));
|
||||
adminAuthSupport.currentDeviceId(request),
|
||||
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch()));
|
||||
// 只记身份不记值:便于事后回答"哪个账号在哪台机器上领过紫鸟口令"
|
||||
log.info("[client-credentials] 下发凭据 userId={} username={} deviceId={}",
|
||||
user.getId(), user.getUsername(), adminAuthSupport.currentDeviceId(request));
|
||||
|
||||
+2
@@ -22,6 +22,8 @@ public class TaskFileJobEntity {
|
||||
private String jobType;
|
||||
private String status;
|
||||
private Integer retryCount;
|
||||
/** 卡死扫描的重排次数(V173),与真失败 retry_count 分账;达到上限后扫描侧判死。 */
|
||||
private Integer stuckRequeueCount;
|
||||
private String errorMessage;
|
||||
private String resultFileUrl;
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
+20
-3
@@ -159,6 +159,12 @@ public class TaskDistributedLockService {
|
||||
return moduleType.trim().toLowerCase(Locale.ROOT).replace('_', '-');
|
||||
}
|
||||
|
||||
/** 同包用例读数:这把锁是否被标成续期丢失(良性停续期必须为 false)。 */
|
||||
boolean activeRenewalLost(String lockName) {
|
||||
ActiveTaskLockRegistry.ActiveLock lock = activeLocks.snapshot().get(lockName);
|
||||
return lock != null && lock.isRenewalLost();
|
||||
}
|
||||
|
||||
/** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */
|
||||
Map<String, Long> activeHeldMillis() {
|
||||
long now = System.currentTimeMillis();
|
||||
@@ -327,15 +333,26 @@ public class TaskDistributedLockService {
|
||||
/**
|
||||
* 停止看门狗续期,但**不释放**锁:父任务已终态时,本节点不该再给这把锁续命;
|
||||
* 仍由 {@link #close()} 那次带 token 的 RELEASE 收尾,避免把接管者的锁误删。
|
||||
*
|
||||
* <p>默认按「续期丢失」标记,保留给真正的续期失败路径;父任务终态这种良性停续期
|
||||
* 必须传 {@code markLost=false},否则 {@code _prod_watch} 的 ACTIVE_LOST 会把良性
|
||||
* 停续期计成异常。
|
||||
*/
|
||||
public void stopRenewal(String reason) {
|
||||
stopRenewal(reason, true);
|
||||
}
|
||||
|
||||
public void stopRenewal(String reason, boolean markLost) {
|
||||
if (heldLock.closed.compareAndSet(false, true)) {
|
||||
cancelRenewal(heldLock);
|
||||
activeLocks.markRenewalLost(heldLock.lockName);
|
||||
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={}",
|
||||
if (markLost) {
|
||||
activeLocks.markRenewalLost(heldLock.lockName);
|
||||
}
|
||||
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={} markLost={}",
|
||||
heldLock.lockName,
|
||||
System.currentTimeMillis() - heldLock.acquiredAtMillis,
|
||||
reason);
|
||||
reason,
|
||||
markLost);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+34
-3
@@ -28,6 +28,12 @@ public class TaskFileJobService {
|
||||
|
||||
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
|
||||
public static final int MAX_RETRY_COUNT = 5;
|
||||
/**
|
||||
* 卡死重排的独立上限。B28 之后卡死重排不扣 retry_count,若没有上限,一个无异常挂住的
|
||||
* job 会被无限放回 PENDING,并让 TaskResumeService 永远跳过克隆重试。20 次是
|
||||
* 「慢执行」与「真卡死」的分界:按 5 分钟 stuck 阈值约 100 分钟,远大于现网合法慢单。
|
||||
*/
|
||||
public static final int MAX_STUCK_REQUEUE = 20;
|
||||
|
||||
private final TaskFileJobMapper taskFileJobMapper;
|
||||
private final ApplicationEventPublisher applicationEventPublisher;
|
||||
@@ -419,16 +425,41 @@ public class TaskFileJobService {
|
||||
// RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在
|
||||
// MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行
|
||||
// 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后
|
||||
// 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、
|
||||
// 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。
|
||||
// 0 字节收场。卡死次数改用独立列 stuck_requeue_count 记账,达到上限才判死;
|
||||
// 这样既保住"慢执行不扣预算",又不会让无异常挂住的 job 无限重排。
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int stuckCount = job.getStuckRequeueCount() == null ? 0 : job.getStuckRequeueCount();
|
||||
if (stuckCount >= MAX_STUCK_REQUEUE) {
|
||||
int failed = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getStuckRequeueCount, MAX_STUCK_REQUEUE)
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
"文件生成任务连续 " + MAX_STUCK_REQUEUE + " 次卡死重排,已判失败")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (failed > 0) {
|
||||
exhaustedJobIds.add(job.getId());
|
||||
log.warn("[task-file-job][STUCK_REQUEUE_EXHAUSTED] jobId={} taskId={} moduleType={} stuckRequeueCount={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), stuckCount);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int nextStuckCount = stuckCount + 1;
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||
.set(TaskFileJobEntity::getStuckRequeueCount, nextStuckCount)
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
"文件生成任务运行超时,已重新排队(第 " + nextStuckCount + " 次)")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
|
||||
+16
-4
@@ -24,6 +24,7 @@ import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -199,15 +200,26 @@ public class TaskResultFileJobWorker {
|
||||
|
||||
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
|
||||
long interval = Math.max(1000L, heartbeatIntervalMillis);
|
||||
AtomicBoolean terminalStopped = new AtomicBoolean(false);
|
||||
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
|
||||
if (terminalStopped.get()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
|
||||
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
|
||||
if (handle != null) {
|
||||
handle.stopRenewal("parent-task-terminal");
|
||||
log.info("[task-file-job] stopped lock renewal because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
|
||||
if (terminalStopped.compareAndSet(false, true)) {
|
||||
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
|
||||
if (handle != null) {
|
||||
// 良性停续期:父任务已终态,不是续期失败,不能打 renewalLost=true 造成假告警
|
||||
handle.stopRenewal("parent-task-terminal", false);
|
||||
}
|
||||
// 终态后不再 touchRunning:否则 job 的 updated_at 一直被刷新,
|
||||
// job 级 stuck 扫描永远看不到这行,成为不可回收的 RUNNING 遗骸。
|
||||
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
|
||||
log.info("[task-file-job] parent task terminal; stopped renewal and failed job jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
}
|
||||
return;
|
||||
}
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
} catch (Exception ex) {
|
||||
|
||||
+5
@@ -15,6 +15,11 @@ public class CostGuardStatusVo {
|
||||
private LocalDateTime pausedAt;
|
||||
private TodayUsage today;
|
||||
private WindowUsage window;
|
||||
/**
|
||||
* window 是本实例最近一轮扫描的读数,不是集群全局值。前端据此标注「本节点读数」,
|
||||
* 避免把单个实例的窗口值误读成全局速率。
|
||||
*/
|
||||
private boolean windowLocalOnly;
|
||||
private Thresholds thresholds;
|
||||
private List<SharedSecretWarningVo> sharedWarnings;
|
||||
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 损耗扫描基线(用户 → 当日累计三元组 + 采样时刻),跨实例共享。
|
||||
*
|
||||
* <p>此前基线是单实例内存,而扫描锁是集群级:持锁方一轮换,新持锁方拿到的
|
||||
* {@code before} 可能是几分钟前的,增量会被当成 60 秒窗口,导致误暂停;反过来新实例
|
||||
* 首轮只建基线,又会漏掉一次窗口。基线落 Redis 后,任何实例接手都按真实 elapsed 归一化。
|
||||
*
|
||||
* <p>用 {@link NodeSharedStore#getFresh} 而不是 {@code get}:本地缓存优先会让每个实例
|
||||
* 永远读自己的旧基线,正好是这次要修的问题。
|
||||
*/
|
||||
@Service
|
||||
public class CostGuardBaselineStore {
|
||||
|
||||
private static final Duration TTL = Duration.ofHours(26);
|
||||
|
||||
private final NodeSharedStore<Long, Baseline> store;
|
||||
|
||||
public CostGuardBaselineStore(StringRedisTemplate redis, ObjectMapper objectMapper) {
|
||||
this.store = new NodeSharedStore<>("cost-guard:baseline", TTL, Baseline.class,
|
||||
redis, objectMapper, 0L);
|
||||
}
|
||||
|
||||
/** 测试用:只传一个共享 store,两个调度器实例即可模拟跨实例读数。 */
|
||||
CostGuardBaselineStore(NodeSharedStore<Long, Baseline> store) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
public Baseline get(Long userId) {
|
||||
return store.getFresh(userId);
|
||||
}
|
||||
|
||||
public void put(Long userId, Baseline baseline) {
|
||||
store.putNow(userId, baseline);
|
||||
}
|
||||
|
||||
public void remove(Long userId) {
|
||||
store.remove(userId);
|
||||
}
|
||||
|
||||
/** Jackson 需要无参构造;字段公开以保持序列化形状稳定。 */
|
||||
public static class Baseline {
|
||||
public long proxy;
|
||||
public long llmCalls;
|
||||
public long llmTokens;
|
||||
public long capturedAtMillis;
|
||||
|
||||
public Baseline() {
|
||||
}
|
||||
|
||||
public Baseline(long proxy, long llmCalls, long llmTokens, long capturedAtMillis) {
|
||||
this.proxy = proxy;
|
||||
this.llmCalls = llmCalls;
|
||||
this.llmTokens = llmTokens;
|
||||
this.capturedAtMillis = capturedAtMillis;
|
||||
}
|
||||
}
|
||||
}
|
||||
+133
-40
@@ -28,7 +28,10 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
* 每分钟全量 GROUP BY 的成本可以忽略。
|
||||
*
|
||||
* <p>只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)},
|
||||
* 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
|
||||
* 基线对齐到当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
|
||||
*
|
||||
* <p>基线落在 {@link CostGuardBaselineStore}(Redis),不是单实例内存:扫描锁是集群级的,
|
||||
* 持锁方轮换后新实例必须按真实 elapsed 归一化增量,否则会把多分钟增量当成 60 秒窗口误暂停。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -43,10 +46,11 @@ public class CostGuardScheduler {
|
||||
private final SharedSecretMonitorService sharedSecretMonitorService;
|
||||
private final NotificationDispatchService notificationDispatchService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final CostGuardBaselineStore baselineStore;
|
||||
|
||||
private final Map<Long, UserTotals> previousTotals = new ConcurrentHashMap<>();
|
||||
private final Map<Long, WindowDelta> lastDeltas = new ConcurrentHashMap<>();
|
||||
private volatile long lastSharedRefreshAt = 0L;
|
||||
private volatile long lastSharedNotifyAt = 0L;
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}",
|
||||
initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}")
|
||||
@@ -56,6 +60,9 @@ public class CostGuardScheduler {
|
||||
}
|
||||
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2));
|
||||
if (lock == null) {
|
||||
// 非持锁方也要刷新共享密钥快照:否则本实例 status() 永远读到空 warningsByUser,
|
||||
// 用户看到的「没有共用」与真实情况相反。通知只由持锁方发(见 runOnce)。
|
||||
maybeRefreshShared(false);
|
||||
log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮");
|
||||
return;
|
||||
}
|
||||
@@ -68,53 +75,70 @@ public class CostGuardScheduler {
|
||||
|
||||
/** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */
|
||||
void runOnce() {
|
||||
long nowMillis = System.currentTimeMillis();
|
||||
int windowSeconds = Math.max(60, properties.getWindowSeconds());
|
||||
Map<Long, UserTotals> current = loadTodayTotals();
|
||||
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
|
||||
int paused = 0;
|
||||
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
|
||||
Long userId = entry.getKey();
|
||||
UserTotals now = entry.getValue();
|
||||
UserTotals before = previousTotals.get(userId);
|
||||
CostGuardBaselineStore.Baseline before = baselineStore.get(userId);
|
||||
boolean advanceBaseline = true;
|
||||
if (before == null) {
|
||||
// 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次
|
||||
// 首轮:窗口增量不判(启动前的大批次不是分钟增量),但日累计已经越线必须暂停,
|
||||
// 否则重启/首次部署后当天永远不会被拦。
|
||||
deltas.put(userId, WindowDelta.ZERO);
|
||||
continue;
|
||||
if (!guardService.isPaused(userId)) {
|
||||
String reason = dailyBreachReason(now);
|
||||
if (reason != null && !pauseAndNotify(userId, reason)) {
|
||||
// 写失败不推进基线:下一轮仍按「已越线」重试,否则 crossed(before, now)
|
||||
// 会因 before 被抬到阈值之上而永远不再触发。
|
||||
advanceBaseline = false;
|
||||
} else if (reason != null) {
|
||||
paused++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
WindowDelta raw = new WindowDelta(
|
||||
Math.max(0, now.proxy - before.proxy),
|
||||
Math.max(0, now.llmCalls - before.llmCalls),
|
||||
Math.max(0, now.llmTokens - before.llmTokens));
|
||||
long elapsedSeconds = Math.max(1L, (nowMillis - before.capturedAtMillis) / 1000L);
|
||||
WindowDelta windowDelta = normalize(raw, elapsedSeconds, windowSeconds);
|
||||
deltas.put(userId, windowDelta);
|
||||
if (!guardService.isPaused(userId)) {
|
||||
String reason = breachReason(now, before, windowDelta, windowSeconds);
|
||||
if (reason != null && !pauseAndNotify(userId, reason)) {
|
||||
advanceBaseline = false;
|
||||
} else if (reason != null) {
|
||||
paused++;
|
||||
}
|
||||
}
|
||||
}
|
||||
WindowDelta delta = new WindowDelta(
|
||||
Math.max(0, now.proxy - before.proxy),
|
||||
Math.max(0, now.llmCalls - before.llmCalls),
|
||||
Math.max(0, now.llmTokens - before.llmTokens));
|
||||
deltas.put(userId, delta);
|
||||
if (guardService.isPaused(userId)) {
|
||||
continue;
|
||||
if (advanceBaseline) {
|
||||
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
|
||||
now.proxy, now.llmCalls, now.llmTokens, nowMillis));
|
||||
}
|
||||
String reason = breachReason(now, before, delta);
|
||||
if (reason == null) {
|
||||
continue;
|
||||
}
|
||||
guardService.pause(userId, reason);
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
|
||||
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
|
||||
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
|
||||
null);
|
||||
paused++;
|
||||
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
|
||||
}
|
||||
previousTotals.clear();
|
||||
previousTotals.putAll(current);
|
||||
lastDeltas.clear();
|
||||
lastDeltas.putAll(deltas);
|
||||
maybeRefreshShared();
|
||||
maybeRefreshShared(true);
|
||||
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
|
||||
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
|
||||
}
|
||||
|
||||
/** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */
|
||||
/**
|
||||
* 用户点击继续时调用:把基线对齐到当前值(而不是删掉基线)。删掉会让下一轮走「首轮」
|
||||
* 分支;首轮现在会按日累计判暂停,用户刚点继续就会被立刻再暂停。
|
||||
*/
|
||||
public void resetBaseline(Long userId) {
|
||||
if (userId == null) {
|
||||
return;
|
||||
}
|
||||
previousTotals.remove(userId);
|
||||
UserTotals current = todayTotals(userId);
|
||||
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
|
||||
current.proxy, current.llmCalls, current.llmTokens, System.currentTimeMillis()));
|
||||
lastDeltas.remove(userId);
|
||||
}
|
||||
|
||||
@@ -158,21 +182,35 @@ public class CostGuardScheduler {
|
||||
return totals;
|
||||
}
|
||||
|
||||
private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) {
|
||||
int seconds = Math.max(60, properties.getWindowSeconds());
|
||||
/** 暂停写成功才发通知、才计数:写失败时谎报「已暂停」比不暂停更糟。 */
|
||||
private boolean pauseAndNotify(Long userId, String reason) {
|
||||
if (!guardService.pause(userId, reason)) {
|
||||
log.warn("[cost-guard] 暂停写入失败,本轮不通知 userId={} reason={}", userId, reason);
|
||||
return false;
|
||||
}
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
|
||||
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
|
||||
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
|
||||
null);
|
||||
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
|
||||
return true;
|
||||
}
|
||||
|
||||
private String breachReason(UserTotals now, CostGuardBaselineStore.Baseline before,
|
||||
WindowDelta delta, int windowSeconds) {
|
||||
if (properties.getProxyWindowThreshold() > 0
|
||||
&& delta.proxy >= properties.getProxyWindowThreshold()) {
|
||||
return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 "
|
||||
&& delta.proxy() >= properties.getProxyWindowThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒代理提取 " + delta.proxy() + " 次,超过阈值 "
|
||||
+ properties.getProxyWindowThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowCallsThreshold() > 0
|
||||
&& delta.llmCalls >= properties.getLlmWindowCallsThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 "
|
||||
&& delta.llmCalls() >= properties.getLlmWindowCallsThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒 LLM 调用 " + delta.llmCalls() + " 次,超过阈值 "
|
||||
+ properties.getLlmWindowCallsThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowTokensThreshold() > 0
|
||||
&& delta.llmTokens >= properties.getLlmWindowTokensThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 "
|
||||
&& delta.llmTokens() >= properties.getLlmWindowTokensThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒 LLM 消耗 " + delta.llmTokens() + " tokens,超过阈值 "
|
||||
+ properties.getLlmWindowTokensThreshold();
|
||||
}
|
||||
if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) {
|
||||
@@ -190,18 +228,73 @@ public class CostGuardScheduler {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 首轮专用:没有上一轮基线可比,只按「已经越线」判日累计,不判窗口增量。 */
|
||||
private String dailyBreachReason(UserTotals now) {
|
||||
if (properties.getDailyProxyThreshold() > 0
|
||||
&& now.proxy >= properties.getDailyProxyThreshold()) {
|
||||
return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 "
|
||||
+ properties.getDailyProxyThreshold() + " 次";
|
||||
}
|
||||
if (properties.getDailyLlmCallsThreshold() > 0
|
||||
&& now.llmCalls >= properties.getDailyLlmCallsThreshold()) {
|
||||
return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 "
|
||||
+ properties.getDailyLlmCallsThreshold() + " 次";
|
||||
}
|
||||
if (properties.getDailyLlmTokensThreshold() > 0
|
||||
&& now.llmTokens >= properties.getDailyLlmTokensThreshold()) {
|
||||
return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 "
|
||||
+ properties.getDailyLlmTokensThreshold();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean crossed(long before, long now, long threshold) {
|
||||
return threshold > 0 && before < threshold && now >= threshold;
|
||||
}
|
||||
|
||||
private void maybeRefreshShared() {
|
||||
/**
|
||||
* 把「elapsed 秒的真实增量」换算成 60 秒窗口。持锁方轮换、扫描被拖长、进程重启后
|
||||
* 首轮接手都会让 elapsed > windowSeconds;不归一化就是把多分钟增量当一分钟判。
|
||||
*
|
||||
* <p>只向下缩放(elapsed 大于窗口时),elapsed 小于窗口时直接采用原始增量:
|
||||
* 短间隔的原始增量是保守下界,放大它只会在调度抖动时制造误暂停。
|
||||
*/
|
||||
private WindowDelta normalize(WindowDelta raw, long elapsedSeconds, int windowSeconds) {
|
||||
if (elapsedSeconds <= windowSeconds) {
|
||||
return raw;
|
||||
}
|
||||
return new WindowDelta(
|
||||
scaleToWindow(raw.proxy(), elapsedSeconds, windowSeconds),
|
||||
scaleToWindow(raw.llmCalls(), elapsedSeconds, windowSeconds),
|
||||
scaleToWindow(raw.llmTokens(), elapsedSeconds, windowSeconds));
|
||||
}
|
||||
|
||||
private long scaleToWindow(long value, long elapsedSeconds, int windowSeconds) {
|
||||
if (value <= 0) {
|
||||
return 0;
|
||||
}
|
||||
// 向上取整:宁可早一秒判红,也不把刚好越线的批次放过
|
||||
return Math.max(1L, (value * windowSeconds + elapsedSeconds - 1) / elapsedSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* 共享密钥快照按 shared-scan-interval-ms 刷新。通知只由持锁方发,避免三台实例各推一遍;
|
||||
* 但非持锁方也要刷新快照,否则它的 status() 永远看不到共享提示。
|
||||
*/
|
||||
private void maybeRefreshShared(boolean notify) {
|
||||
long now = System.currentTimeMillis();
|
||||
if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) {
|
||||
long interval = Math.max(60_000L, properties.getSharedScanIntervalMs());
|
||||
boolean refreshDue = now - lastSharedRefreshAt >= interval;
|
||||
boolean notifyDue = notify && now - lastSharedNotifyAt >= interval;
|
||||
if (!refreshDue && !notifyDue) {
|
||||
return;
|
||||
}
|
||||
lastSharedRefreshAt = now;
|
||||
// 不自己吞:runOnce 里这是最后一步,抛出由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
|
||||
sharedSecretMonitorService.refresh();
|
||||
if (notifyDue) {
|
||||
lastSharedNotifyAt = now;
|
||||
}
|
||||
// 不自己吞:refresh 失败由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
|
||||
sharedSecretMonitorService.refresh(notifyDue);
|
||||
}
|
||||
|
||||
public static final class UserTotals {
|
||||
|
||||
+1
@@ -30,6 +30,7 @@ public class CostGuardService {
|
||||
vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens));
|
||||
vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds,
|
||||
delta.proxy(), delta.llmCalls(), delta.llmTokens()));
|
||||
vo.setWindowLocalOnly(true);
|
||||
vo.setThresholds(new CostGuardStatusVo.Thresholds(
|
||||
properties.getProxyWindowThreshold(),
|
||||
properties.getLlmWindowCallsThreshold(),
|
||||
|
||||
+11
-1
@@ -49,6 +49,14 @@ public class SharedSecretMonitorService {
|
||||
|
||||
/** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */
|
||||
public synchronized void refresh() {
|
||||
refresh(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param notify 是否推当日提示。多实例下只有持锁方传 true,其余实例只刷新快照,
|
||||
* 否则每台实例各推一遍(虽然通知有去重键,但仍是三倍写入与扇出)。
|
||||
*/
|
||||
public synchronized void refresh(boolean notify) {
|
||||
if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) {
|
||||
warningsByUser = Map.of();
|
||||
snapshotAtMillis = System.currentTimeMillis();
|
||||
@@ -89,7 +97,9 @@ public class SharedSecretMonitorService {
|
||||
type, group.module.label(), accounts, threshold, message);
|
||||
for (Long userId : group.userIds) {
|
||||
next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning);
|
||||
pushDailyWarning(userId, group.module, entry.getKey(), warning);
|
||||
if (notify) {
|
||||
pushDailyWarning(userId, group.module, entry.getKey(), warning);
|
||||
}
|
||||
}
|
||||
}
|
||||
warningsByUser = next;
|
||||
|
||||
+8
-2
@@ -55,19 +55,25 @@ public class UserTaskGuardService {
|
||||
return state;
|
||||
}
|
||||
|
||||
public void pause(Long userId, String reason) {
|
||||
/**
|
||||
* @return 写入是否成功。调用方只有拿到 true 才能发「已暂停」通知;
|
||||
* 此前该方法吞掉异常,扫描器无论成败都通知,数据库抖动时用户被告知已暂停、任务却仍在派发。
|
||||
*/
|
||||
public boolean pause(Long userId, String reason) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
String safeReason = truncate(reason);
|
||||
try {
|
||||
guardMapper.pause(userId, safeReason);
|
||||
cache.remove(userId);
|
||||
log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason);
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
// 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里
|
||||
log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}",
|
||||
userId, safeReason, ex.getMessage(), ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+28
-7
@@ -18,6 +18,7 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
@@ -117,13 +118,16 @@ public class WorkbenchClientPackageService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 签发直传预签名:对象 key 由「毫秒时间戳 + 安全版本号」组成,每次上传互不覆盖。
|
||||
* 签发直传预签名:对象 key 由「毫秒时间戳 + 随机 UUID + 安全版本号」组成。
|
||||
* UUID 是 CR-08 的修复点:只靠毫秒时间戳时,多实例时钟相同或同毫秒两名管理员
|
||||
* 上传同一版本会生成同一个 key,后一次 PUT 覆盖前一次对象、先确认的 DB 行摘要失效。
|
||||
*/
|
||||
public Map<String, Object> presignPackage(String fileName, String version) {
|
||||
String normalizedName = requireZipFileName(fileName);
|
||||
String normalizedVersion = requireVersion(version);
|
||||
String objectKey = OBJECT_KEY_PREFIX
|
||||
+ OBJECT_NAME_FORMATTER.format(LocalDateTime.now())
|
||||
+ "-" + UUID.randomUUID().toString().replace("-", "")
|
||||
+ "-" + safeVersionKey(normalizedVersion) + ".zip";
|
||||
log.info("[workbench-client] 签发客户端包直传预签名 version={} fileName={} objectKey={} expirySeconds={}",
|
||||
normalizedVersion, normalizedName, objectKey, PRESIGN_EXPIRY_SECONDS);
|
||||
@@ -154,27 +158,44 @@ public class WorkbenchClientPackageService {
|
||||
normalizedVersion, objectKey);
|
||||
throw new BusinessException("版本号与上传对象不匹配,请重新上传");
|
||||
}
|
||||
// 幂等:重复 confirm 同一对象返回已登记行,不再报错(CR-08)
|
||||
WorkbenchClientPackageEntity existing = packageMapper.selectOne(
|
||||
new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey)
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
log.info("[workbench-client] 直传确认幂等命中 id={} objectKey={}", existing.getId(), objectKey);
|
||||
return toItemMap(existing);
|
||||
}
|
||||
long size = ossStorageService.workbenchClientObjectSize(objectKey);
|
||||
if (size < 0) {
|
||||
log.warn("[workbench-client] 直传确认失败:对象不存在 objectKey={}", objectKey);
|
||||
throw new BusinessException("对象存储中未找到上传的安装包,请重新上传");
|
||||
}
|
||||
if (size <= 0) {
|
||||
// 0 字节对象(空文件/中断上传)会被客户端拿到后解压失败,直接拒绝并清残
|
||||
log.warn("[workbench-client] 直传对象为空,清理残留 objectKey={}", objectKey);
|
||||
ossStorageService.removeWorkbenchClientObject(objectKey);
|
||||
throw new BusinessException("上传的安装包为空文件,请重新上传");
|
||||
}
|
||||
if (size > MAX_UPLOAD_BYTES) {
|
||||
log.warn("[workbench-client] 直传对象超过大小上限,清理残留 objectKey={} bytes={}", objectKey, size);
|
||||
ossStorageService.removeWorkbenchClientObject(objectKey);
|
||||
throw new BusinessException("文件超过允许的大小限制");
|
||||
}
|
||||
Long sameVersion = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getVersion, normalizedVersion));
|
||||
if (sameVersion != null && sameVersion > 0) {
|
||||
// 同版本不同字节会让已升级的装机按版本号比较后不升级,客户端还会拿到错误摘要
|
||||
log.warn("[workbench-client] 直传确认失败:版本已存在 version={} objectKey={}",
|
||||
normalizedVersion, objectKey);
|
||||
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
|
||||
}
|
||||
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
|
||||
if (sha256 == null || sha256.length() != 64) {
|
||||
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
|
||||
throw new BusinessException("安装包摘要计算失败,请重新上传");
|
||||
}
|
||||
Long duplicated = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey));
|
||||
if (duplicated != null && duplicated > 0) {
|
||||
log.warn("[workbench-client] 直传确认失败:对象已被记录,重复提交 objectKey={}", objectKey);
|
||||
throw new BusinessException("该安装包已登记,请勿重复提交");
|
||||
}
|
||||
|
||||
WorkbenchClientPackageEntity entity = new WorkbenchClientPackageEntity();
|
||||
entity.setVersion(normalizedVersion);
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
-- =============================================================
|
||||
-- 迁移 V172__user_session_epoch
|
||||
-- 立账:ops 仓 docs/audit/CODE_REVIEW_REPORT.md CR-01(2026-10-25)
|
||||
--
|
||||
-- 1. 变更目的:给 users 增加会话世代。旧 JWT 没有服务端黑名单,登出只靠 users.machine
|
||||
-- 上的 revoked@ 标记失效;单设备开关关闭时登录会把这个标记清空,旧令牌重新可用。
|
||||
-- 世代随登录、登出各 +1,令牌签发时写入 epoch claim,鉴权时与库中值比对,
|
||||
-- 旧令牌在登出后、以及在任何一次新登录后都立即失效。
|
||||
--
|
||||
-- 2. 口径:
|
||||
-- - 列 NOT NULL DEFAULT 0;存量用户为 0,升级前签发的旧令牌没有 epoch claim 时
|
||||
-- 按 0 读取,因此不会被本次上线统一踢下线;用户下一次登录/登出推进后旧令牌失效。
|
||||
-- - machine 的互踢逻辑保持不变;单设备开关关闭时不再依赖清空 revoked@ 标记表达新会话。
|
||||
--
|
||||
-- 3. 影响表与数据量:users 加一列,存量行默认 0;MySQL 8.0 起 ADD COLUMN 走 INSTANT,
|
||||
-- 不重建表。本机无生产库连接,未做上线前 dry-run;上线后由 Flyway 自动应用。
|
||||
--
|
||||
-- 4. 回滚(如需):ALTER TABLE `users` DROP COLUMN `session_epoch`;
|
||||
-- 回滚前需同步回滚应用代码,否则新代码读不到列会启动失败(Expand-and-Contract 的反向)。
|
||||
-- =============================================================
|
||||
|
||||
ALTER TABLE `users`
|
||||
ADD COLUMN `session_epoch` BIGINT NOT NULL DEFAULT 0 COMMENT '会话世代:登录/登出各 +1' AFTER `machine`;
|
||||
@@ -0,0 +1,24 @@
|
||||
-- =============================================================
|
||||
-- 迁移 V173__task_file_job_stuck_requeue_count
|
||||
-- 立账:ops 仓 docs/audit/CODE_REVIEW_REPORT.md CR-03(2026-10-25)
|
||||
--
|
||||
-- 1. 变更目的:给卡死重排单独记账。B28 拍板「RUNNING 卡死重排不消耗 retry_count」
|
||||
-- 之后,一个无异常挂住的 job 会被无限放回 PENDING,并且 TaskResumeService 会因此
|
||||
-- 永远跳过克隆重试。本列只统计 stuck 扫描的重排次数,与真失败重试预算分账;
|
||||
-- 达到上限(20)时扫描侧直接判 FAILED,保留「慢执行不扣预算」的口径。
|
||||
--
|
||||
-- 2. 口径:
|
||||
-- - NOT NULL DEFAULT 0,存量行从 0 起算;
|
||||
-- - 只由 TaskFileJobService.resetStuckRunningJobsDetailed 递增;
|
||||
-- - 开机回收(reclaimOwnedRunningJobs)不递增:发版不是作业的错。
|
||||
--
|
||||
-- 3. 影响表与数据量:biz_task_file_job 加一列,存量行默认 0;MySQL 8.0 起 ADD COLUMN
|
||||
-- 走 INSTANT,不重建表。未连生产库 dry-run,随下一次发版由 Flyway 自动应用。
|
||||
--
|
||||
-- 4. 回滚(如需):ALTER TABLE `biz_task_file_job` DROP COLUMN `stuck_requeue_count`;
|
||||
-- 回滚前需同步回滚应用代码。
|
||||
-- =============================================================
|
||||
|
||||
ALTER TABLE `biz_task_file_job`
|
||||
ADD COLUMN `stuck_requeue_count` INT NOT NULL DEFAULT 0 COMMENT '卡死重排次数(独立于 retry_count)'
|
||||
AFTER `retry_count`;
|
||||
@@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.nanri.aiimage.modules.auth.mapper.LoginUserMapper">
|
||||
|
||||
<update id="incrementSessionEpoch">
|
||||
UPDATE users
|
||||
SET session_epoch = session_epoch + 1
|
||||
WHERE id = #{userId}
|
||||
</update>
|
||||
|
||||
</mapper>
|
||||
Reference in New Issue
Block a user