fix(cr-01/02/03/08/09/10/11/13): 会话世代、损耗共享基线、任务活性与工作台包
- CR-01:users.session_epoch(V172)+ JWT sessionEpoch;登录/登出原子推进, check_login/requireUser 先比世代再做设备比对;登出不再依赖 machine CAS - CR-02/09:cost-guard 基线落 Redis(CostGuardBaselineStore),按 elapsed 归一化; 非持锁实例刷新共享密钥快照;pause() 返回写成功,失败不通知且不推进基线 - CR-03:stuck_requeue_count(V173,上限 20)与 retry_count 分账;终态父任务心跳 停止 touchRunning 并 markFailedPermanent;良性停续期 markLost=false - CR-08:工作台包对象 key 加 UUID、0 字节拒绝、同 objectKey 幂等、同版本重复拒绝、 前端 confirm 超时抬到 10 分钟 - CR-10:Excel onSheetTotal 统一在 onHeader 之后回调(xlsx/xls) - CR-13/11:迁移头/文件名一致断言(V170 唯一豁免);V167 前置由 CompanyService 同事务 复制菜单树,已应用迁移未回改
This commit is contained in:
@@ -126,16 +126,13 @@ public final class ExcelStreamReader {
|
||||
SheetCollector collector = new SheetCollector(sheetName, sheetNo, handler, recorder);
|
||||
XSSFSheetXMLHandler sheetHandler =
|
||||
new XSSFSheetXMLHandler(styles, sharedStrings, collector, formatter, false);
|
||||
DimensionFilter filter = new DimensionFilter(parser, maxRows -> {
|
||||
try {
|
||||
handler.onSheetTotal(sheetName, sheetNo, maxRows);
|
||||
} catch (Exception ex) {
|
||||
throw recorder.record(ex);
|
||||
}
|
||||
});
|
||||
// dimension 在 sheetData 之前出现,先缓存;等 onHeader 回调之后再对外发 onSheetTotal,
|
||||
// 保持旧 EasyExcel「先表头、后总行数」的顺序(CR-10)。
|
||||
DimensionFilter filter = new DimensionFilter(parser, collector::setApproximateTotalRows);
|
||||
filter.setContentHandler(sheetHandler);
|
||||
parser.setContentHandler(filter);
|
||||
parser.parse(new InputSource(sheet));
|
||||
collector.reportTotalIfNeeded();
|
||||
}
|
||||
|
||||
private static void readHssf(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler)
|
||||
@@ -152,11 +149,12 @@ public final class ExcelStreamReader {
|
||||
final String sheetName = sheet.getSheetName();
|
||||
final int currentSheetNo = sheetNo;
|
||||
final int lastRow = sheet.getLastRowNum();
|
||||
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
|
||||
final Map<Integer, String> header = rowToMap(sheet.getRow(0), formatter);
|
||||
if (!header.isEmpty()) {
|
||||
callRecorder(recorder, () -> handler.onHeader(sheetName, currentSheetNo, header));
|
||||
}
|
||||
// 旧顺序是「表头 → 总行数」;xls 路径此前反了,统一回来(CR-10)
|
||||
callRecorder(recorder, () -> handler.onSheetTotal(sheetName, currentSheetNo, lastRow + 1));
|
||||
for (int rowIndex = 1; rowIndex <= lastRow; rowIndex++) {
|
||||
Row row = sheet.getRow(rowIndex);
|
||||
if (row == null) {
|
||||
@@ -274,6 +272,9 @@ public final class ExcelStreamReader {
|
||||
private final FailureRecorder recorder;
|
||||
private Map<Integer, String> header = Map.of();
|
||||
private Map<Integer, String> row = new LinkedHashMap<>();
|
||||
private Integer approximateTotalRows;
|
||||
private boolean headerReported;
|
||||
private boolean totalReported;
|
||||
|
||||
private SheetCollector(String sheetName, int sheetNo, SheetRowHandler handler, FailureRecorder recorder) {
|
||||
this.sheetName = sheetName;
|
||||
@@ -282,6 +283,21 @@ public final class ExcelStreamReader {
|
||||
this.recorder = recorder;
|
||||
}
|
||||
|
||||
/** dimension 过滤器的回调:只缓存,等 onHeader 之后才对外发。 */
|
||||
private void setApproximateTotalRows(Integer rows) {
|
||||
this.approximateTotalRows = rows;
|
||||
reportTotalIfNeeded();
|
||||
}
|
||||
|
||||
private void reportTotalIfNeeded() {
|
||||
if (!headerReported || totalReported) {
|
||||
return;
|
||||
}
|
||||
totalReported = true;
|
||||
callRecorder(recorder,
|
||||
() -> handler.onSheetTotal(sheetName, sheetNo, approximateTotalRows));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startRow(int rowNum) {
|
||||
row = new LinkedHashMap<>();
|
||||
@@ -294,7 +310,9 @@ public final class ExcelStreamReader {
|
||||
}
|
||||
if (rowNum == 0) {
|
||||
header = new LinkedHashMap<>(row);
|
||||
headerReported = true;
|
||||
callRecorder(recorder, () -> handler.onHeader(sheetName, sheetNo, header));
|
||||
reportTotalIfNeeded();
|
||||
} else {
|
||||
Map<Integer, String> values = new LinkedHashMap<>(row);
|
||||
int rowIndex = rowNum;
|
||||
|
||||
@@ -49,4 +49,10 @@ public class AdminUserEntity {
|
||||
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||
private LocalDateTime updatedAt;
|
||||
private String machine;
|
||||
/**
|
||||
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
|
||||
* 与 {@code LoginUserEntity.sessionEpoch} 同列,改一处必须同改两处。
|
||||
*/
|
||||
@TableField("session_epoch")
|
||||
private Long sessionEpoch;
|
||||
}
|
||||
|
||||
@@ -72,6 +72,9 @@ public class AdminAuthSupport {
|
||||
// 吊销先于单设备比对:那条路径第一行就是 if (exempt) return;(超管豁免),
|
||||
// 把吊销挂在它下面等于超管永不见吊销
|
||||
DeviceSessionPolicy.assertNotRevoked(user.getMachine(), user.getId(), user.getUsername());
|
||||
// 世代校验同样先于单设备比对,且不受 singleDevice 开关支配:登出/新登录后旧令牌一律失效
|
||||
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
|
||||
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
|
||||
// 单设备登录:被新设备顶下线的旧 token 在此统一拦截(全站 requireUser 调用点自动生效)
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.assertSameDevice(user.getMachine(), DeviceSessionPolicy.claimDeviceId(claims),
|
||||
|
||||
@@ -91,6 +91,30 @@ public final class DeviceSessionPolicy {
|
||||
return raw == null ? "" : raw.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 JWT claims 中提取会话世代;缺失(本次升级前签发的旧令牌)按 0 读取,
|
||||
* 与 {@code users.session_epoch} 的列默认值一致,升级不会统一踢掉存量会话。
|
||||
*/
|
||||
public static long claimSessionEpoch(Claims claims) {
|
||||
Object raw = claims == null ? null : claims.get("sessionEpoch");
|
||||
// JwtService 签发的是 Number;非 Number 一律视同缺失(按 0),不在这里做字符串兼容,
|
||||
// 避免为一种不会出现的 claim 形状新增 catch(CatchSilenceRatchet 满格)。
|
||||
return raw instanceof Number number ? number.longValue() : 0L;
|
||||
}
|
||||
|
||||
/**
|
||||
* 会话世代必须与库中值相等。登出与每次新登录都会 +1,因此登出前签发的令牌在
|
||||
* 下一次请求即失效;不依赖 {@code machine} 上的 revoked@ 标记,也不受单设备开关支配。
|
||||
*/
|
||||
public static void assertSessionEpoch(Long storedEpoch, long claimEpoch, Long userId, String username) {
|
||||
long current = storedEpoch == null ? 0L : storedEpoch;
|
||||
if (current != claimEpoch) {
|
||||
log.warn("[auth] 会话世代失效被拒 userId={} username={} stored={} claim={}",
|
||||
userId, username, current, claimEpoch);
|
||||
throw new BusinessException(401, "登录已过期,请重新登录");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验请求携带的 token 是否仍属于账号当前绑定的设备。
|
||||
* 超管豁免;machine 为空(尚未绑定)放行;不匹配抛 4011。
|
||||
|
||||
@@ -85,13 +85,19 @@ public class JwtService {
|
||||
return key;
|
||||
}
|
||||
|
||||
public String issue(Long userId, String username, String deviceId) {
|
||||
/**
|
||||
* 签发令牌。{@code sessionEpoch} 必须取自 users.session_epoch(登录/登出各推进一次),
|
||||
* 鉴权时与库中值比对;旧令牌没有这个 claim 时按 0 读取,见
|
||||
* {@link DeviceSessionPolicy#claimSessionEpoch(Claims)}。
|
||||
*/
|
||||
public String issue(Long userId, String username, String deviceId, long sessionEpoch) {
|
||||
Instant now = Instant.now();
|
||||
Instant exp = now.plus(Duration.ofHours(props.getJwtTtlHours()));
|
||||
return Jwts.builder()
|
||||
.subject(String.valueOf(userId))
|
||||
.claim("username", username)
|
||||
.claim("deviceId", deviceId)
|
||||
.claim("sessionEpoch", sessionEpoch)
|
||||
.issuedAt(Date.from(now))
|
||||
.expiration(Date.from(exp))
|
||||
.signWith(signingKey())
|
||||
|
||||
@@ -74,6 +74,22 @@ public final class NodeSharedStore<K, V> {
|
||||
return remote;
|
||||
}
|
||||
|
||||
/**
|
||||
* 远端优先;远端没有才回退本地。用于「锁在实例间轮换、新持锁方必须读到上一持锁方
|
||||
* 写入的基线」这类状态——{@link #get} 的本地优先会让每个实例 forever 读自己的旧值。
|
||||
*/
|
||||
public V getFresh(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V remote = readRemote(key);
|
||||
if (remote != null) {
|
||||
local.put(key, remote);
|
||||
return remote;
|
||||
}
|
||||
return local.get(key);
|
||||
}
|
||||
|
||||
public boolean containsKey(K key) {
|
||||
return get(key) != null;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,14 @@ package com.nanri.aiimage.modules.auth.mapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.auth.model.entity.LoginUserEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
@Mapper
|
||||
public interface LoginUserMapper extends BaseMapper<LoginUserEntity> {
|
||||
|
||||
/**
|
||||
* 会话世代 +1(登录、登出各调用一次)。原子自增而不是「读出来再写回」:
|
||||
* 双节点同时登录时后者不能把前者的世代覆盖回去。返回 0 行表示用户已不存在。
|
||||
*/
|
||||
int incrementSessionEpoch(@Param("userId") Long userId);
|
||||
}
|
||||
|
||||
+6
@@ -43,4 +43,10 @@ public class LoginUserEntity {
|
||||
*/
|
||||
@TableField("disabled_at")
|
||||
private LocalDateTime disabledAt;
|
||||
/**
|
||||
* 会话世代(V172 新增):登录/登出各 +1,JWT 的 epoch claim 必须与它相等。
|
||||
* 与 {@code AdminUserEntity.sessionEpoch} 同列,改一处必须同改两处。
|
||||
*/
|
||||
@TableField("session_epoch")
|
||||
private Long sessionEpoch;
|
||||
}
|
||||
|
||||
@@ -116,6 +116,7 @@ public class AuthService {
|
||||
accountExpiryService.assertLoginAllowed(user);
|
||||
|
||||
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
|
||||
advanceSessionEpoch(user);
|
||||
// 单设备登录:登录成功即把账号绑定到当前设备(last-login-wins),原设备下一次请求被顶下线
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.logBindOnLogin(user.getMachine(), deviceId, user.getId(), user.getUsername());
|
||||
@@ -125,7 +126,8 @@ public class AuthService {
|
||||
} else if (DeviceSessionPolicy.isRevoked(user.getMachine())) {
|
||||
// 开关关掉后登录不再写 machine,登出留下的吊销标记就没人清:此后每次登录都会先成功、
|
||||
// 再被 requireUser 的 assertNotRevoked 判成「登录已过期」,账号永久锁死。
|
||||
// 登录成功即视为新会话,清空标记(空=未绑定,放行语义);仍不写设备号。
|
||||
// 登录成功即视为新会话(上面已推进 session_epoch,旧令牌另有世代校验兜底),
|
||||
// 清空标记(空=未绑定,放行语义);仍不写设备号。
|
||||
log.warn("[auth] 登录清除登出吊销标记 userId={} username={}", user.getId(), user.getUsername());
|
||||
loginUserMapper.update(null, new LambdaUpdateWrapper<LoginUserEntity>()
|
||||
.eq(LoginUserEntity::getId, user.getId())
|
||||
@@ -149,6 +151,9 @@ public class AuthService {
|
||||
if (userId == null) {
|
||||
return;
|
||||
}
|
||||
// 世代 +1 是真正的吊销;它不依赖 machine 是否等于本次 deviceId,
|
||||
// 因此 machine 为空(单设备关闭时的常态)也一定留得下吊销效果。
|
||||
int epochUpdated = loginUserMapper.incrementSessionEpoch(userId);
|
||||
String marker = DeviceSessionPolicy.REVOKED_PREFIX + userId + "#"
|
||||
+ java.time.LocalDateTime.now().format(REVOKE_STAMP);
|
||||
String bound = deviceId == null ? "" : deviceId.trim();
|
||||
@@ -156,7 +161,8 @@ public class AuthService {
|
||||
.set(LoginUserEntity::getMachine, marker)
|
||||
.eq(LoginUserEntity::getId, userId)
|
||||
.eq(!bound.isEmpty(), LoginUserEntity::getMachine, bound));
|
||||
log.info("[auth] 登出吊销 userId={} deviceBound={} updated={}", userId, bound, updated);
|
||||
log.info("[auth] 登出吊销 userId={} deviceBound={} epochUpdated={} markerUpdated={}",
|
||||
userId, bound, epochUpdated, updated);
|
||||
}
|
||||
|
||||
/** 从令牌里取出身份再吊销;解析不出(已过期/伪造)就什么都不做——cookie 由调用方清。 */
|
||||
@@ -173,7 +179,20 @@ public class AuthService {
|
||||
log.warn("[auth] 登出时 subject 非数字,跳过吊销(cookie 仍由调用方清)");
|
||||
return;
|
||||
}
|
||||
logout(Long.parseLong(subject), DeviceSessionPolicy.claimDeviceId(claims));
|
||||
Long userId = Long.parseLong(subject);
|
||||
LoginUserEntity current = loginUserMapper.selectById(userId);
|
||||
if (current == null) {
|
||||
return;
|
||||
}
|
||||
// 旧令牌不能把新会话一起登出:只有当前世代的令牌才有权推进世代。
|
||||
long claimEpoch = DeviceSessionPolicy.claimSessionEpoch(claims);
|
||||
long storedEpoch = current.getSessionEpoch() == null ? 0L : current.getSessionEpoch();
|
||||
if (claimEpoch != storedEpoch) {
|
||||
log.warn("[auth] 登出令牌的会话世代已过期,跳过吊销 userId={} stored={} claim={}",
|
||||
userId, storedEpoch, claimEpoch);
|
||||
return;
|
||||
}
|
||||
logout(userId, DeviceSessionPolicy.claimDeviceId(claims));
|
||||
}
|
||||
|
||||
public LoginResultVo checkLogin(String token, String currentDeviceId) {
|
||||
@@ -196,6 +215,9 @@ public class AuthService {
|
||||
}
|
||||
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
|
||||
String claimDeviceId = DeviceSessionPolicy.claimDeviceId(claims);
|
||||
// 世代校验先于设备比对:登出/新登录后旧 token 在这里直接 401,不能靠 check_login 续期复活
|
||||
DeviceSessionPolicy.assertSessionEpoch(user.getSessionEpoch(),
|
||||
DeviceSessionPolicy.claimSessionEpoch(claims), user.getId(), user.getUsername());
|
||||
// 单设备登录:不拦截的话,被顶下线的旧 token 会在这里续期「复活」
|
||||
if (authProperties.isSingleDeviceEnabled()) {
|
||||
DeviceSessionPolicy.assertSameDevice(user.getMachine(), claimDeviceId,
|
||||
@@ -238,8 +260,20 @@ public class AuthService {
|
||||
return authProperties.getCookieName();
|
||||
}
|
||||
|
||||
/** 登录推进世代;返回 0 行说明用户行已消失,不能再签发一个「世代比库还新」的令牌。 */
|
||||
private void advanceSessionEpoch(LoginUserEntity user) {
|
||||
int updated = loginUserMapper.incrementSessionEpoch(user.getId());
|
||||
if (updated != 1) {
|
||||
log.warn("[auth] 登录推进会话世代失败 userId={} updated={}", user.getId(), updated);
|
||||
throw new BusinessException(500, "登录状态更新失败,请重试");
|
||||
}
|
||||
long current = user.getSessionEpoch() == null ? 0L : user.getSessionEpoch();
|
||||
user.setSessionEpoch(current + 1);
|
||||
}
|
||||
|
||||
private LoginResultVo buildResult(LoginUserEntity user, String deviceId, boolean isAdmin) {
|
||||
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId);
|
||||
String token = jwtService.issue(user.getId(), user.getUsername(), deviceId,
|
||||
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch());
|
||||
LoginResultVo vo = new LoginResultVo();
|
||||
vo.setUserId(user.getId());
|
||||
vo.setUsername(user.getUsername());
|
||||
|
||||
+2
-1
@@ -57,7 +57,8 @@ public class ClientCredentialController {
|
||||
vo.setMysqlPassword(blankToEmpty(clientCredentialProperties.getMysqlPassword()));
|
||||
// 续期与容忍是两件事:容忍让存量客户端不被 7 天边界掐断,这条把新客户端拉回正常轨道
|
||||
vo.setJwt(jwtService.issue(user.getId(), user.getUsername(),
|
||||
adminAuthSupport.currentDeviceId(request)));
|
||||
adminAuthSupport.currentDeviceId(request),
|
||||
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch()));
|
||||
// 只记身份不记值:便于事后回答"哪个账号在哪台机器上领过紫鸟口令"
|
||||
log.info("[client-credentials] 下发凭据 userId={} username={} deviceId={}",
|
||||
user.getId(), user.getUsername(), adminAuthSupport.currentDeviceId(request));
|
||||
|
||||
+2
@@ -22,6 +22,8 @@ public class TaskFileJobEntity {
|
||||
private String jobType;
|
||||
private String status;
|
||||
private Integer retryCount;
|
||||
/** 卡死扫描的重排次数(V173),与真失败 retry_count 分账;达到上限后扫描侧判死。 */
|
||||
private Integer stuckRequeueCount;
|
||||
private String errorMessage;
|
||||
private String resultFileUrl;
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
+20
-3
@@ -159,6 +159,12 @@ public class TaskDistributedLockService {
|
||||
return moduleType.trim().toLowerCase(Locale.ROOT).replace('_', '-');
|
||||
}
|
||||
|
||||
/** 同包用例读数:这把锁是否被标成续期丢失(良性停续期必须为 false)。 */
|
||||
boolean activeRenewalLost(String lockName) {
|
||||
ActiveTaskLockRegistry.ActiveLock lock = activeLocks.snapshot().get(lockName);
|
||||
return lock != null && lock.isRenewalLost();
|
||||
}
|
||||
|
||||
/** 运维与同包用例的读数:当前在持的任务锁 → 已持有时长(毫秒)。空表=这台进程没有在持的锁。 */
|
||||
Map<String, Long> activeHeldMillis() {
|
||||
long now = System.currentTimeMillis();
|
||||
@@ -327,15 +333,26 @@ public class TaskDistributedLockService {
|
||||
/**
|
||||
* 停止看门狗续期,但**不释放**锁:父任务已终态时,本节点不该再给这把锁续命;
|
||||
* 仍由 {@link #close()} 那次带 token 的 RELEASE 收尾,避免把接管者的锁误删。
|
||||
*
|
||||
* <p>默认按「续期丢失」标记,保留给真正的续期失败路径;父任务终态这种良性停续期
|
||||
* 必须传 {@code markLost=false},否则 {@code _prod_watch} 的 ACTIVE_LOST 会把良性
|
||||
* 停续期计成异常。
|
||||
*/
|
||||
public void stopRenewal(String reason) {
|
||||
stopRenewal(reason, true);
|
||||
}
|
||||
|
||||
public void stopRenewal(String reason, boolean markLost) {
|
||||
if (heldLock.closed.compareAndSet(false, true)) {
|
||||
cancelRenewal(heldLock);
|
||||
activeLocks.markRenewalLost(heldLock.lockName);
|
||||
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={}",
|
||||
if (markLost) {
|
||||
activeLocks.markRenewalLost(heldLock.lockName);
|
||||
}
|
||||
log.warn("[task-lock] 停止续期 lockName={} heldMs={} reason={} markLost={}",
|
||||
heldLock.lockName,
|
||||
System.currentTimeMillis() - heldLock.acquiredAtMillis,
|
||||
reason);
|
||||
reason,
|
||||
markLost);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+34
-3
@@ -28,6 +28,12 @@ public class TaskFileJobService {
|
||||
|
||||
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
|
||||
public static final int MAX_RETRY_COUNT = 5;
|
||||
/**
|
||||
* 卡死重排的独立上限。B28 之后卡死重排不扣 retry_count,若没有上限,一个无异常挂住的
|
||||
* job 会被无限放回 PENDING,并让 TaskResumeService 永远跳过克隆重试。20 次是
|
||||
* 「慢执行」与「真卡死」的分界:按 5 分钟 stuck 阈值约 100 分钟,远大于现网合法慢单。
|
||||
*/
|
||||
public static final int MAX_STUCK_REQUEUE = 20;
|
||||
|
||||
private final TaskFileJobMapper taskFileJobMapper;
|
||||
private final ApplicationEventPublisher applicationEventPublisher;
|
||||
@@ -419,16 +425,41 @@ public class TaskFileJobService {
|
||||
// RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在
|
||||
// MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行
|
||||
// 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后
|
||||
// 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、
|
||||
// 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。
|
||||
// 0 字节收场。卡死次数改用独立列 stuck_requeue_count 记账,达到上限才判死;
|
||||
// 这样既保住"慢执行不扣预算",又不会让无异常挂住的 job 无限重排。
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int stuckCount = job.getStuckRequeueCount() == null ? 0 : job.getStuckRequeueCount();
|
||||
if (stuckCount >= MAX_STUCK_REQUEUE) {
|
||||
int failed = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getStuckRequeueCount, MAX_STUCK_REQUEUE)
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
"文件生成任务连续 " + MAX_STUCK_REQUEUE + " 次卡死重排,已判失败")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (failed > 0) {
|
||||
exhaustedJobIds.add(job.getId());
|
||||
log.warn("[task-file-job][STUCK_REQUEUE_EXHAUSTED] jobId={} taskId={} moduleType={} stuckRequeueCount={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), stuckCount);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int nextStuckCount = stuckCount + 1;
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||
.set(TaskFileJobEntity::getStuckRequeueCount, nextStuckCount)
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
"文件生成任务运行超时,已重新排队(第 " + nextStuckCount + " 次)")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
|
||||
+16
-4
@@ -24,6 +24,7 @@ import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -199,15 +200,26 @@ public class TaskResultFileJobWorker {
|
||||
|
||||
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
|
||||
long interval = Math.max(1000L, heartbeatIntervalMillis);
|
||||
AtomicBoolean terminalStopped = new AtomicBoolean(false);
|
||||
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
|
||||
if (terminalStopped.get()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
|
||||
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
|
||||
if (handle != null) {
|
||||
handle.stopRenewal("parent-task-terminal");
|
||||
log.info("[task-file-job] stopped lock renewal because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
|
||||
if (terminalStopped.compareAndSet(false, true)) {
|
||||
TaskDistributedLockService.LockHandle handle = activeLockHandles.get(job.getId());
|
||||
if (handle != null) {
|
||||
// 良性停续期:父任务已终态,不是续期失败,不能打 renewalLost=true 造成假告警
|
||||
handle.stopRenewal("parent-task-terminal", false);
|
||||
}
|
||||
// 终态后不再 touchRunning:否则 job 的 updated_at 一直被刷新,
|
||||
// job 级 stuck 扫描永远看不到这行,成为不可回收的 RUNNING 遗骸。
|
||||
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
|
||||
log.info("[task-file-job] parent task terminal; stopped renewal and failed job jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
}
|
||||
return;
|
||||
}
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
} catch (Exception ex) {
|
||||
|
||||
+5
@@ -15,6 +15,11 @@ public class CostGuardStatusVo {
|
||||
private LocalDateTime pausedAt;
|
||||
private TodayUsage today;
|
||||
private WindowUsage window;
|
||||
/**
|
||||
* window 是本实例最近一轮扫描的读数,不是集群全局值。前端据此标注「本节点读数」,
|
||||
* 避免把单个实例的窗口值误读成全局速率。
|
||||
*/
|
||||
private boolean windowLocalOnly;
|
||||
private Thresholds thresholds;
|
||||
private List<SharedSecretWarningVo> sharedWarnings;
|
||||
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 损耗扫描基线(用户 → 当日累计三元组 + 采样时刻),跨实例共享。
|
||||
*
|
||||
* <p>此前基线是单实例内存,而扫描锁是集群级:持锁方一轮换,新持锁方拿到的
|
||||
* {@code before} 可能是几分钟前的,增量会被当成 60 秒窗口,导致误暂停;反过来新实例
|
||||
* 首轮只建基线,又会漏掉一次窗口。基线落 Redis 后,任何实例接手都按真实 elapsed 归一化。
|
||||
*
|
||||
* <p>用 {@link NodeSharedStore#getFresh} 而不是 {@code get}:本地缓存优先会让每个实例
|
||||
* 永远读自己的旧基线,正好是这次要修的问题。
|
||||
*/
|
||||
@Service
|
||||
public class CostGuardBaselineStore {
|
||||
|
||||
private static final Duration TTL = Duration.ofHours(26);
|
||||
|
||||
private final NodeSharedStore<Long, Baseline> store;
|
||||
|
||||
public CostGuardBaselineStore(StringRedisTemplate redis, ObjectMapper objectMapper) {
|
||||
this.store = new NodeSharedStore<>("cost-guard:baseline", TTL, Baseline.class,
|
||||
redis, objectMapper, 0L);
|
||||
}
|
||||
|
||||
/** 测试用:只传一个共享 store,两个调度器实例即可模拟跨实例读数。 */
|
||||
CostGuardBaselineStore(NodeSharedStore<Long, Baseline> store) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
public Baseline get(Long userId) {
|
||||
return store.getFresh(userId);
|
||||
}
|
||||
|
||||
public void put(Long userId, Baseline baseline) {
|
||||
store.putNow(userId, baseline);
|
||||
}
|
||||
|
||||
public void remove(Long userId) {
|
||||
store.remove(userId);
|
||||
}
|
||||
|
||||
/** Jackson 需要无参构造;字段公开以保持序列化形状稳定。 */
|
||||
public static class Baseline {
|
||||
public long proxy;
|
||||
public long llmCalls;
|
||||
public long llmTokens;
|
||||
public long capturedAtMillis;
|
||||
|
||||
public Baseline() {
|
||||
}
|
||||
|
||||
public Baseline(long proxy, long llmCalls, long llmTokens, long capturedAtMillis) {
|
||||
this.proxy = proxy;
|
||||
this.llmCalls = llmCalls;
|
||||
this.llmTokens = llmTokens;
|
||||
this.capturedAtMillis = capturedAtMillis;
|
||||
}
|
||||
}
|
||||
}
|
||||
+133
-40
@@ -28,7 +28,10 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
* 每分钟全量 GROUP BY 的成本可以忽略。
|
||||
*
|
||||
* <p>只在本轮真正「跨过」阈值时暂停:用户排查后点继续会调 {@link #resetBaseline(Long)},
|
||||
* 基线重置为当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
|
||||
* 基线对齐到当前值,不会因为日累计仍高于阈值而被立刻再次暂停。
|
||||
*
|
||||
* <p>基线落在 {@link CostGuardBaselineStore}(Redis),不是单实例内存:扫描锁是集群级的,
|
||||
* 持锁方轮换后新实例必须按真实 elapsed 归一化增量,否则会把多分钟增量当成 60 秒窗口误暂停。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -43,10 +46,11 @@ public class CostGuardScheduler {
|
||||
private final SharedSecretMonitorService sharedSecretMonitorService;
|
||||
private final NotificationDispatchService notificationDispatchService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final CostGuardBaselineStore baselineStore;
|
||||
|
||||
private final Map<Long, UserTotals> previousTotals = new ConcurrentHashMap<>();
|
||||
private final Map<Long, WindowDelta> lastDeltas = new ConcurrentHashMap<>();
|
||||
private volatile long lastSharedRefreshAt = 0L;
|
||||
private volatile long lastSharedNotifyAt = 0L;
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.cost-guard.scan-interval-ms:60000}",
|
||||
initialDelayString = "${aiimage.cost-guard.scan-initial-delay-ms:30000}")
|
||||
@@ -56,6 +60,9 @@ public class CostGuardScheduler {
|
||||
}
|
||||
var lock = distributedJobLockService.tryLock(LOCK_NAME, Duration.ofMinutes(2));
|
||||
if (lock == null) {
|
||||
// 非持锁方也要刷新共享密钥快照:否则本实例 status() 永远读到空 warningsByUser,
|
||||
// 用户看到的「没有共用」与真实情况相反。通知只由持锁方发(见 runOnce)。
|
||||
maybeRefreshShared(false);
|
||||
log.info("[cost-guard] 另一实例持有损耗扫描锁,跳过本轮");
|
||||
return;
|
||||
}
|
||||
@@ -68,53 +75,70 @@ public class CostGuardScheduler {
|
||||
|
||||
/** 一轮扫描;package-private 供单测直接驱动,不依赖调度线程。 */
|
||||
void runOnce() {
|
||||
long nowMillis = System.currentTimeMillis();
|
||||
int windowSeconds = Math.max(60, properties.getWindowSeconds());
|
||||
Map<Long, UserTotals> current = loadTodayTotals();
|
||||
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
|
||||
int paused = 0;
|
||||
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
|
||||
Long userId = entry.getKey();
|
||||
UserTotals now = entry.getValue();
|
||||
UserTotals before = previousTotals.get(userId);
|
||||
CostGuardBaselineStore.Baseline before = baselineStore.get(userId);
|
||||
boolean advanceBaseline = true;
|
||||
if (before == null) {
|
||||
// 首轮只建基线:启动瞬间没有「增量」可比,拿日累计当分钟增量会误伤正在跑的大批次
|
||||
// 首轮:窗口增量不判(启动前的大批次不是分钟增量),但日累计已经越线必须暂停,
|
||||
// 否则重启/首次部署后当天永远不会被拦。
|
||||
deltas.put(userId, WindowDelta.ZERO);
|
||||
continue;
|
||||
if (!guardService.isPaused(userId)) {
|
||||
String reason = dailyBreachReason(now);
|
||||
if (reason != null && !pauseAndNotify(userId, reason)) {
|
||||
// 写失败不推进基线:下一轮仍按「已越线」重试,否则 crossed(before, now)
|
||||
// 会因 before 被抬到阈值之上而永远不再触发。
|
||||
advanceBaseline = false;
|
||||
} else if (reason != null) {
|
||||
paused++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
WindowDelta raw = new WindowDelta(
|
||||
Math.max(0, now.proxy - before.proxy),
|
||||
Math.max(0, now.llmCalls - before.llmCalls),
|
||||
Math.max(0, now.llmTokens - before.llmTokens));
|
||||
long elapsedSeconds = Math.max(1L, (nowMillis - before.capturedAtMillis) / 1000L);
|
||||
WindowDelta windowDelta = normalize(raw, elapsedSeconds, windowSeconds);
|
||||
deltas.put(userId, windowDelta);
|
||||
if (!guardService.isPaused(userId)) {
|
||||
String reason = breachReason(now, before, windowDelta, windowSeconds);
|
||||
if (reason != null && !pauseAndNotify(userId, reason)) {
|
||||
advanceBaseline = false;
|
||||
} else if (reason != null) {
|
||||
paused++;
|
||||
}
|
||||
}
|
||||
}
|
||||
WindowDelta delta = new WindowDelta(
|
||||
Math.max(0, now.proxy - before.proxy),
|
||||
Math.max(0, now.llmCalls - before.llmCalls),
|
||||
Math.max(0, now.llmTokens - before.llmTokens));
|
||||
deltas.put(userId, delta);
|
||||
if (guardService.isPaused(userId)) {
|
||||
continue;
|
||||
if (advanceBaseline) {
|
||||
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
|
||||
now.proxy, now.llmCalls, now.llmTokens, nowMillis));
|
||||
}
|
||||
String reason = breachReason(now, before, delta);
|
||||
if (reason == null) {
|
||||
continue;
|
||||
}
|
||||
guardService.pause(userId, reason);
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
|
||||
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
|
||||
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
|
||||
null);
|
||||
paused++;
|
||||
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
|
||||
}
|
||||
previousTotals.clear();
|
||||
previousTotals.putAll(current);
|
||||
lastDeltas.clear();
|
||||
lastDeltas.putAll(deltas);
|
||||
maybeRefreshShared();
|
||||
maybeRefreshShared(true);
|
||||
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
|
||||
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
|
||||
}
|
||||
|
||||
/** 用户点击继续时调用:把基线对齐到当前值,避免下一轮拿历史累计再次触发。 */
|
||||
/**
|
||||
* 用户点击继续时调用:把基线对齐到当前值(而不是删掉基线)。删掉会让下一轮走「首轮」
|
||||
* 分支;首轮现在会按日累计判暂停,用户刚点继续就会被立刻再暂停。
|
||||
*/
|
||||
public void resetBaseline(Long userId) {
|
||||
if (userId == null) {
|
||||
return;
|
||||
}
|
||||
previousTotals.remove(userId);
|
||||
UserTotals current = todayTotals(userId);
|
||||
baselineStore.put(userId, new CostGuardBaselineStore.Baseline(
|
||||
current.proxy, current.llmCalls, current.llmTokens, System.currentTimeMillis()));
|
||||
lastDeltas.remove(userId);
|
||||
}
|
||||
|
||||
@@ -158,21 +182,35 @@ public class CostGuardScheduler {
|
||||
return totals;
|
||||
}
|
||||
|
||||
private String breachReason(UserTotals now, UserTotals before, WindowDelta delta) {
|
||||
int seconds = Math.max(60, properties.getWindowSeconds());
|
||||
/** 暂停写成功才发通知、才计数:写失败时谎报「已暂停」比不暂停更糟。 */
|
||||
private boolean pauseAndNotify(Long userId, String reason) {
|
||||
if (!guardService.pause(userId, reason)) {
|
||||
log.warn("[cost-guard] 暂停写入失败,本轮不通知 userId={} reason={}", userId, reason);
|
||||
return false;
|
||||
}
|
||||
notificationDispatchService.pushToUser(userId, NotificationService.SCENE_COST_GUARD,
|
||||
NotificationService.LEVEL_ERROR, "已自动暂停任务:代理/LLM 消耗异常", reason
|
||||
+ "。请检查「密钥设置」与正在跑的任务配置,排查后点击客户端提示里的「继续任务」。",
|
||||
null);
|
||||
log.warn("[cost-guard] 触发暂停 userId={} reason={}", userId, reason);
|
||||
return true;
|
||||
}
|
||||
|
||||
private String breachReason(UserTotals now, CostGuardBaselineStore.Baseline before,
|
||||
WindowDelta delta, int windowSeconds) {
|
||||
if (properties.getProxyWindowThreshold() > 0
|
||||
&& delta.proxy >= properties.getProxyWindowThreshold()) {
|
||||
return "最近约 " + seconds + " 秒代理提取 " + delta.proxy + " 次,超过阈值 "
|
||||
&& delta.proxy() >= properties.getProxyWindowThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒代理提取 " + delta.proxy() + " 次,超过阈值 "
|
||||
+ properties.getProxyWindowThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowCallsThreshold() > 0
|
||||
&& delta.llmCalls >= properties.getLlmWindowCallsThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 调用 " + delta.llmCalls + " 次,超过阈值 "
|
||||
&& delta.llmCalls() >= properties.getLlmWindowCallsThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒 LLM 调用 " + delta.llmCalls() + " 次,超过阈值 "
|
||||
+ properties.getLlmWindowCallsThreshold() + " 次";
|
||||
}
|
||||
if (properties.getLlmWindowTokensThreshold() > 0
|
||||
&& delta.llmTokens >= properties.getLlmWindowTokensThreshold()) {
|
||||
return "最近约 " + seconds + " 秒 LLM 消耗 " + delta.llmTokens + " tokens,超过阈值 "
|
||||
&& delta.llmTokens() >= properties.getLlmWindowTokensThreshold()) {
|
||||
return "最近约 " + windowSeconds + " 秒 LLM 消耗 " + delta.llmTokens() + " tokens,超过阈值 "
|
||||
+ properties.getLlmWindowTokensThreshold();
|
||||
}
|
||||
if (crossed(before.proxy, now.proxy, properties.getDailyProxyThreshold())) {
|
||||
@@ -190,18 +228,73 @@ public class CostGuardScheduler {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 首轮专用:没有上一轮基线可比,只按「已经越线」判日累计,不判窗口增量。 */
|
||||
private String dailyBreachReason(UserTotals now) {
|
||||
if (properties.getDailyProxyThreshold() > 0
|
||||
&& now.proxy >= properties.getDailyProxyThreshold()) {
|
||||
return "今日代理提取累计 " + now.proxy + " 次,超过当日上限 "
|
||||
+ properties.getDailyProxyThreshold() + " 次";
|
||||
}
|
||||
if (properties.getDailyLlmCallsThreshold() > 0
|
||||
&& now.llmCalls >= properties.getDailyLlmCallsThreshold()) {
|
||||
return "今日 LLM 调用累计 " + now.llmCalls + " 次,超过当日上限 "
|
||||
+ properties.getDailyLlmCallsThreshold() + " 次";
|
||||
}
|
||||
if (properties.getDailyLlmTokensThreshold() > 0
|
||||
&& now.llmTokens >= properties.getDailyLlmTokensThreshold()) {
|
||||
return "今日 LLM 消耗累计 " + now.llmTokens + " tokens,超过当日上限 "
|
||||
+ properties.getDailyLlmTokensThreshold();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean crossed(long before, long now, long threshold) {
|
||||
return threshold > 0 && before < threshold && now >= threshold;
|
||||
}
|
||||
|
||||
private void maybeRefreshShared() {
|
||||
/**
|
||||
* 把「elapsed 秒的真实增量」换算成 60 秒窗口。持锁方轮换、扫描被拖长、进程重启后
|
||||
* 首轮接手都会让 elapsed > windowSeconds;不归一化就是把多分钟增量当一分钟判。
|
||||
*
|
||||
* <p>只向下缩放(elapsed 大于窗口时),elapsed 小于窗口时直接采用原始增量:
|
||||
* 短间隔的原始增量是保守下界,放大它只会在调度抖动时制造误暂停。
|
||||
*/
|
||||
private WindowDelta normalize(WindowDelta raw, long elapsedSeconds, int windowSeconds) {
|
||||
if (elapsedSeconds <= windowSeconds) {
|
||||
return raw;
|
||||
}
|
||||
return new WindowDelta(
|
||||
scaleToWindow(raw.proxy(), elapsedSeconds, windowSeconds),
|
||||
scaleToWindow(raw.llmCalls(), elapsedSeconds, windowSeconds),
|
||||
scaleToWindow(raw.llmTokens(), elapsedSeconds, windowSeconds));
|
||||
}
|
||||
|
||||
private long scaleToWindow(long value, long elapsedSeconds, int windowSeconds) {
|
||||
if (value <= 0) {
|
||||
return 0;
|
||||
}
|
||||
// 向上取整:宁可早一秒判红,也不把刚好越线的批次放过
|
||||
return Math.max(1L, (value * windowSeconds + elapsedSeconds - 1) / elapsedSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* 共享密钥快照按 shared-scan-interval-ms 刷新。通知只由持锁方发,避免三台实例各推一遍;
|
||||
* 但非持锁方也要刷新快照,否则它的 status() 永远看不到共享提示。
|
||||
*/
|
||||
private void maybeRefreshShared(boolean notify) {
|
||||
long now = System.currentTimeMillis();
|
||||
if (now - lastSharedRefreshAt < Math.max(60_000L, properties.getSharedScanIntervalMs())) {
|
||||
long interval = Math.max(60_000L, properties.getSharedScanIntervalMs());
|
||||
boolean refreshDue = now - lastSharedRefreshAt >= interval;
|
||||
boolean notifyDue = notify && now - lastSharedNotifyAt >= interval;
|
||||
if (!refreshDue && !notifyDue) {
|
||||
return;
|
||||
}
|
||||
lastSharedRefreshAt = now;
|
||||
// 不自己吞:runOnce 里这是最后一步,抛出由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
|
||||
sharedSecretMonitorService.refresh();
|
||||
if (notifyDue) {
|
||||
lastSharedNotifyAt = now;
|
||||
}
|
||||
// 不自己吞:refresh 失败由 scan() 的 [cost-guard] WARN 记名,下一间隔再试
|
||||
sharedSecretMonitorService.refresh(notifyDue);
|
||||
}
|
||||
|
||||
public static final class UserTotals {
|
||||
|
||||
+1
@@ -30,6 +30,7 @@ public class CostGuardService {
|
||||
vo.setToday(new CostGuardStatusVo.TodayUsage(today.proxy, today.llmCalls, today.llmTokens));
|
||||
vo.setWindow(new CostGuardStatusVo.WindowUsage(windowSeconds,
|
||||
delta.proxy(), delta.llmCalls(), delta.llmTokens()));
|
||||
vo.setWindowLocalOnly(true);
|
||||
vo.setThresholds(new CostGuardStatusVo.Thresholds(
|
||||
properties.getProxyWindowThreshold(),
|
||||
properties.getLlmWindowCallsThreshold(),
|
||||
|
||||
+11
-1
@@ -49,6 +49,14 @@ public class SharedSecretMonitorService {
|
||||
|
||||
/** 扫描全部密钥配置,刷新「谁和谁共用同一把」的快照,并给使用者推当日一次提示。 */
|
||||
public synchronized void refresh() {
|
||||
refresh(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param notify 是否推当日提示。多实例下只有持锁方传 true,其余实例只刷新快照,
|
||||
* 否则每台实例各推一遍(虽然通知有去重键,但仍是三倍写入与扇出)。
|
||||
*/
|
||||
public synchronized void refresh(boolean notify) {
|
||||
if (!properties.isEnabled() || properties.getSharedAccountsThreshold() <= 0) {
|
||||
warningsByUser = Map.of();
|
||||
snapshotAtMillis = System.currentTimeMillis();
|
||||
@@ -89,7 +97,9 @@ public class SharedSecretMonitorService {
|
||||
type, group.module.label(), accounts, threshold, message);
|
||||
for (Long userId : group.userIds) {
|
||||
next.computeIfAbsent(userId, key -> new ArrayList<>()).add(warning);
|
||||
pushDailyWarning(userId, group.module, entry.getKey(), warning);
|
||||
if (notify) {
|
||||
pushDailyWarning(userId, group.module, entry.getKey(), warning);
|
||||
}
|
||||
}
|
||||
}
|
||||
warningsByUser = next;
|
||||
|
||||
+8
-2
@@ -55,19 +55,25 @@ public class UserTaskGuardService {
|
||||
return state;
|
||||
}
|
||||
|
||||
public void pause(Long userId, String reason) {
|
||||
/**
|
||||
* @return 写入是否成功。调用方只有拿到 true 才能发「已暂停」通知;
|
||||
* 此前该方法吞掉异常,扫描器无论成败都通知,数据库抖动时用户被告知已暂停、任务却仍在派发。
|
||||
*/
|
||||
public boolean pause(Long userId, String reason) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
String safeReason = truncate(reason);
|
||||
try {
|
||||
guardMapper.pause(userId, safeReason);
|
||||
cache.remove(userId);
|
||||
log.warn("[cost-guard] 已暂停账号新任务 userId={} reason={}", userId, safeReason);
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
// 暂停失败不能拖垮扫描线程:下一轮还会再判一次,异常留在日志里
|
||||
log.error("[cost-guard] 暂停写入失败 userId={} reason={} err={}",
|
||||
userId, safeReason, ex.getMessage(), ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+28
-7
@@ -18,6 +18,7 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
@@ -117,13 +118,16 @@ public class WorkbenchClientPackageService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 签发直传预签名:对象 key 由「毫秒时间戳 + 安全版本号」组成,每次上传互不覆盖。
|
||||
* 签发直传预签名:对象 key 由「毫秒时间戳 + 随机 UUID + 安全版本号」组成。
|
||||
* UUID 是 CR-08 的修复点:只靠毫秒时间戳时,多实例时钟相同或同毫秒两名管理员
|
||||
* 上传同一版本会生成同一个 key,后一次 PUT 覆盖前一次对象、先确认的 DB 行摘要失效。
|
||||
*/
|
||||
public Map<String, Object> presignPackage(String fileName, String version) {
|
||||
String normalizedName = requireZipFileName(fileName);
|
||||
String normalizedVersion = requireVersion(version);
|
||||
String objectKey = OBJECT_KEY_PREFIX
|
||||
+ OBJECT_NAME_FORMATTER.format(LocalDateTime.now())
|
||||
+ "-" + UUID.randomUUID().toString().replace("-", "")
|
||||
+ "-" + safeVersionKey(normalizedVersion) + ".zip";
|
||||
log.info("[workbench-client] 签发客户端包直传预签名 version={} fileName={} objectKey={} expirySeconds={}",
|
||||
normalizedVersion, normalizedName, objectKey, PRESIGN_EXPIRY_SECONDS);
|
||||
@@ -154,27 +158,44 @@ public class WorkbenchClientPackageService {
|
||||
normalizedVersion, objectKey);
|
||||
throw new BusinessException("版本号与上传对象不匹配,请重新上传");
|
||||
}
|
||||
// 幂等:重复 confirm 同一对象返回已登记行,不再报错(CR-08)
|
||||
WorkbenchClientPackageEntity existing = packageMapper.selectOne(
|
||||
new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey)
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
log.info("[workbench-client] 直传确认幂等命中 id={} objectKey={}", existing.getId(), objectKey);
|
||||
return toItemMap(existing);
|
||||
}
|
||||
long size = ossStorageService.workbenchClientObjectSize(objectKey);
|
||||
if (size < 0) {
|
||||
log.warn("[workbench-client] 直传确认失败:对象不存在 objectKey={}", objectKey);
|
||||
throw new BusinessException("对象存储中未找到上传的安装包,请重新上传");
|
||||
}
|
||||
if (size <= 0) {
|
||||
// 0 字节对象(空文件/中断上传)会被客户端拿到后解压失败,直接拒绝并清残
|
||||
log.warn("[workbench-client] 直传对象为空,清理残留 objectKey={}", objectKey);
|
||||
ossStorageService.removeWorkbenchClientObject(objectKey);
|
||||
throw new BusinessException("上传的安装包为空文件,请重新上传");
|
||||
}
|
||||
if (size > MAX_UPLOAD_BYTES) {
|
||||
log.warn("[workbench-client] 直传对象超过大小上限,清理残留 objectKey={} bytes={}", objectKey, size);
|
||||
ossStorageService.removeWorkbenchClientObject(objectKey);
|
||||
throw new BusinessException("文件超过允许的大小限制");
|
||||
}
|
||||
Long sameVersion = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getVersion, normalizedVersion));
|
||||
if (sameVersion != null && sameVersion > 0) {
|
||||
// 同版本不同字节会让已升级的装机按版本号比较后不升级,客户端还会拿到错误摘要
|
||||
log.warn("[workbench-client] 直传确认失败:版本已存在 version={} objectKey={}",
|
||||
normalizedVersion, objectKey);
|
||||
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
|
||||
}
|
||||
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
|
||||
if (sha256 == null || sha256.length() != 64) {
|
||||
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
|
||||
throw new BusinessException("安装包摘要计算失败,请重新上传");
|
||||
}
|
||||
Long duplicated = packageMapper.selectCount(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.eq(WorkbenchClientPackageEntity::getObjectKey, objectKey));
|
||||
if (duplicated != null && duplicated > 0) {
|
||||
log.warn("[workbench-client] 直传确认失败:对象已被记录,重复提交 objectKey={}", objectKey);
|
||||
throw new BusinessException("该安装包已登记,请勿重复提交");
|
||||
}
|
||||
|
||||
WorkbenchClientPackageEntity entity = new WorkbenchClientPackageEntity();
|
||||
entity.setVersion(normalizedVersion);
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
-- =============================================================
|
||||
-- 迁移 V172__user_session_epoch
|
||||
-- 立账:ops 仓 docs/audit/CODE_REVIEW_REPORT.md CR-01(2026-10-25)
|
||||
--
|
||||
-- 1. 变更目的:给 users 增加会话世代。旧 JWT 没有服务端黑名单,登出只靠 users.machine
|
||||
-- 上的 revoked@ 标记失效;单设备开关关闭时登录会把这个标记清空,旧令牌重新可用。
|
||||
-- 世代随登录、登出各 +1,令牌签发时写入 epoch claim,鉴权时与库中值比对,
|
||||
-- 旧令牌在登出后、以及在任何一次新登录后都立即失效。
|
||||
--
|
||||
-- 2. 口径:
|
||||
-- - 列 NOT NULL DEFAULT 0;存量用户为 0,升级前签发的旧令牌没有 epoch claim 时
|
||||
-- 按 0 读取,因此不会被本次上线统一踢下线;用户下一次登录/登出推进后旧令牌失效。
|
||||
-- - machine 的互踢逻辑保持不变;单设备开关关闭时不再依赖清空 revoked@ 标记表达新会话。
|
||||
--
|
||||
-- 3. 影响表与数据量:users 加一列,存量行默认 0;MySQL 8.0 起 ADD COLUMN 走 INSTANT,
|
||||
-- 不重建表。本机无生产库连接,未做上线前 dry-run;上线后由 Flyway 自动应用。
|
||||
--
|
||||
-- 4. 回滚(如需):ALTER TABLE `users` DROP COLUMN `session_epoch`;
|
||||
-- 回滚前需同步回滚应用代码,否则新代码读不到列会启动失败(Expand-and-Contract 的反向)。
|
||||
-- =============================================================
|
||||
|
||||
ALTER TABLE `users`
|
||||
ADD COLUMN `session_epoch` BIGINT NOT NULL DEFAULT 0 COMMENT '会话世代:登录/登出各 +1' AFTER `machine`;
|
||||
@@ -0,0 +1,24 @@
|
||||
-- =============================================================
|
||||
-- 迁移 V173__task_file_job_stuck_requeue_count
|
||||
-- 立账:ops 仓 docs/audit/CODE_REVIEW_REPORT.md CR-03(2026-10-25)
|
||||
--
|
||||
-- 1. 变更目的:给卡死重排单独记账。B28 拍板「RUNNING 卡死重排不消耗 retry_count」
|
||||
-- 之后,一个无异常挂住的 job 会被无限放回 PENDING,并且 TaskResumeService 会因此
|
||||
-- 永远跳过克隆重试。本列只统计 stuck 扫描的重排次数,与真失败重试预算分账;
|
||||
-- 达到上限(20)时扫描侧直接判 FAILED,保留「慢执行不扣预算」的口径。
|
||||
--
|
||||
-- 2. 口径:
|
||||
-- - NOT NULL DEFAULT 0,存量行从 0 起算;
|
||||
-- - 只由 TaskFileJobService.resetStuckRunningJobsDetailed 递增;
|
||||
-- - 开机回收(reclaimOwnedRunningJobs)不递增:发版不是作业的错。
|
||||
--
|
||||
-- 3. 影响表与数据量:biz_task_file_job 加一列,存量行默认 0;MySQL 8.0 起 ADD COLUMN
|
||||
-- 走 INSTANT,不重建表。未连生产库 dry-run,随下一次发版由 Flyway 自动应用。
|
||||
--
|
||||
-- 4. 回滚(如需):ALTER TABLE `biz_task_file_job` DROP COLUMN `stuck_requeue_count`;
|
||||
-- 回滚前需同步回滚应用代码。
|
||||
-- =============================================================
|
||||
|
||||
ALTER TABLE `biz_task_file_job`
|
||||
ADD COLUMN `stuck_requeue_count` INT NOT NULL DEFAULT 0 COMMENT '卡死重排次数(独立于 retry_count)'
|
||||
AFTER `retry_count`;
|
||||
@@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.nanri.aiimage.modules.auth.mapper.LoginUserMapper">
|
||||
|
||||
<update id="incrementSessionEpoch">
|
||||
UPDATE users
|
||||
SET session_epoch = session_epoch + 1
|
||||
WHERE id = #{userId}
|
||||
</update>
|
||||
|
||||
</mapper>
|
||||
@@ -101,7 +101,9 @@ class MigratedSqlEquivalenceTest {
|
||||
"UserSecretUsageMapper#sumByUserAndModule",
|
||||
"UserSecretUsageMapper#sumForUser",
|
||||
// B28(2026-10-25)新写的 1 条主键点查:父任务是否已终态。
|
||||
"TaskFileJobMapper#selectFileTaskStatus");
|
||||
"TaskFileJobMapper#selectFileTaskStatus",
|
||||
// CR-01(2026-10-25)新写的 1 条写侧语句:会话世代原子自增。
|
||||
"LoginUserMapper#incrementSessionEpoch");
|
||||
|
||||
/** 账本 5 列:接口 / 方法 / @Select 种类 / 是否含常量插值 / 注解原文(Java 反转义后那条串)。 */
|
||||
private static final int LEDGER_COLUMNS = 5;
|
||||
|
||||
@@ -80,7 +80,9 @@ class SqlLayeringConventionTest {
|
||||
// 126 → 128:同轮 {@code UserTaskGuardMapper} 的 pause/resume 两条写侧语句从注解迁进 XML。
|
||||
// 128 → 129:B28 新增 {@code TaskFileJobMapper#selectFileTaskStatus}(按主键读父任务状态),
|
||||
// 新写语句按约定直接进 XML,同轮登记进 MigratedSqlEquivalenceTest.NOT_MIGRATED(88 → 89)。
|
||||
private static final int XML_STATEMENT_TOTAL = 129;
|
||||
// 129 → 130:CR-01 新增 {@code LoginUserMapper#incrementSessionEpoch}(会话世代原子自增),
|
||||
// 新写语句直接进 mapper/auth/LoginUserMapper.xml,同轮登记进 NOT_MIGRATED。
|
||||
private static final int XML_STATEMENT_TOTAL = 130;
|
||||
|
||||
/**
|
||||
* wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数:
|
||||
|
||||
+64
@@ -126,6 +126,70 @@ class ExcelStreamReaderParseFailureTest {
|
||||
assertEquals("Nike", seen.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void ooxmlCallsSheetTotalAfterHeader() throws Exception {
|
||||
Path good = writeWorkbook();
|
||||
java.util.List<String> events = new java.util.ArrayList<>();
|
||||
|
||||
ExcelStreamReader.readFirstSheet(good.toFile(), new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
events.add("header:" + headerMap.get(0));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
events.add("total:" + approximateTotalRows);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
}
|
||||
});
|
||||
|
||||
assertEquals("header:品牌", events.get(0));
|
||||
assertTrue(events.size() >= 2 && events.get(1).startsWith("total:"),
|
||||
"onSheetTotal 必须在 onHeader 之后回调,实际=" + events);
|
||||
}
|
||||
|
||||
@Test
|
||||
void hssfCallsSheetTotalAfterHeader() throws Exception {
|
||||
Path file = tempDir.resolve("order.xls");
|
||||
try (HSSFWorkbook workbook = new HSSFWorkbook()) {
|
||||
Sheet sheet = workbook.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("品牌");
|
||||
Row data = sheet.createRow(1);
|
||||
data.createCell(0).setCellValue("Nike");
|
||||
try (OutputStream out = Files.newOutputStream(file)) {
|
||||
workbook.write(out);
|
||||
}
|
||||
}
|
||||
java.util.List<String> events = new java.util.ArrayList<>();
|
||||
|
||||
ExcelStreamReader.readFirstSheet(file.toFile(), new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
events.add("header:" + headerMap.get(0));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
events.add("total:" + approximateTotalRows);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
}
|
||||
});
|
||||
|
||||
assertEquals("header:品牌", events.get(0));
|
||||
assertTrue(events.size() >= 2 && events.get(1).startsWith("total:"),
|
||||
"xls 路径同样必须先表头、后总行数,实际=" + events);
|
||||
}
|
||||
|
||||
private Path writeWorkbook() throws IOException {
|
||||
Path file = tempDir.resolve("ok.xlsx");
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
|
||||
+21
@@ -96,6 +96,20 @@ class AdminAuthSupportTest {
|
||||
assertThatCode(() -> support.requireUser(requestWithBearer("t"))).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void requireUserRejectsTokenFromBeforeLogoutOrRelogin() {
|
||||
// 库中世代已是 2,token 还签着 1:登出/新登录之后旧令牌必须 401,与单设备开关无关
|
||||
AdminUserEntity user = user(7L, "normal", 0, null, null);
|
||||
user.setSessionEpoch(2L);
|
||||
AdminAuthSupport support = support(user, "devA", false, 1L);
|
||||
|
||||
BusinessException ex = catchThrowableOfType(
|
||||
() -> support.requireUser(requestWithBearer("t")), BusinessException.class);
|
||||
|
||||
assertThat(ex.getCode()).isEqualTo(401);
|
||||
assertThat(ex.getMessage()).contains("登录已过期");
|
||||
}
|
||||
|
||||
// ---------- 内部令牌:轮换过渡期新旧两把同时接受 ----------
|
||||
|
||||
@Test
|
||||
@@ -199,6 +213,12 @@ class AdminAuthSupportTest {
|
||||
}
|
||||
|
||||
private AdminAuthSupport support(AdminUserEntity user, String claimDeviceId, boolean singleDeviceEnabled) {
|
||||
return support(user, claimDeviceId, singleDeviceEnabled,
|
||||
user.getSessionEpoch() == null ? 0L : user.getSessionEpoch());
|
||||
}
|
||||
|
||||
private AdminAuthSupport support(AdminUserEntity user, String claimDeviceId, boolean singleDeviceEnabled,
|
||||
long claimSessionEpoch) {
|
||||
JwtService jwtService = mock(JwtService.class);
|
||||
AdminUserMapper userMapper = mock(AdminUserMapper.class);
|
||||
AuthProperties props = mock(AuthProperties.class);
|
||||
@@ -206,6 +226,7 @@ class AdminAuthSupportTest {
|
||||
Claims claims = mock(Claims.class);
|
||||
when(claims.getSubject()).thenReturn(String.valueOf(user.getId()));
|
||||
when(claims.get("deviceId")).thenReturn(claimDeviceId);
|
||||
when(claims.get("sessionEpoch")).thenReturn(claimSessionEpoch);
|
||||
when(jwtService.parse("t")).thenReturn(claims);
|
||||
when(jwtService.parseWithVerdict("t")).thenReturn(
|
||||
new JwtService.ParseResult(JwtService.Verdict.VALID, claims));
|
||||
|
||||
@@ -37,7 +37,7 @@ class JwtVerdictTest {
|
||||
@Test
|
||||
void freshTokenIsValidAndParseStillReturnsClaims() {
|
||||
JwtService s = service(SECRET);
|
||||
String token = s.issue(7L, "小魏16", "dev-1");
|
||||
String token = s.issue(7L, "小魏16", "dev-1", 3L);
|
||||
JwtService.ParseResult result = s.parseWithVerdict(token);
|
||||
assertThat(result.verdict()).isEqualTo(JwtService.Verdict.VALID);
|
||||
assertThat(result.isValid()).isTrue();
|
||||
@@ -59,7 +59,7 @@ class JwtVerdictTest {
|
||||
@Test
|
||||
void wrongKeyIsInvalidNotExpired() {
|
||||
String forged = service("9999999999012345678901234567890123456789")
|
||||
.issue(7L, "小魏16", "dev-1");
|
||||
.issue(7L, "小魏16", "dev-1", 3L);
|
||||
JwtService.ParseResult result = service(SECRET).parseWithVerdict(forged);
|
||||
assertThat(result.verdict()).isEqualTo(JwtService.Verdict.INVALID);
|
||||
assertThat(result.claims()).isNull();
|
||||
|
||||
@@ -11,6 +11,7 @@ import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@@ -91,6 +92,14 @@ class MigrationInventoryTest {
|
||||
*/
|
||||
private static final Set<Integer> RETIRED_HOLES = Set.of(132, 133, 148, 168, 169);
|
||||
|
||||
/**
|
||||
* 文件名版本与头注释版本不一致的历史件。V170 的头注释误写成 V168;它已应用到生产,
|
||||
* 按 AGENTS.md「已发布迁移一个字节都不许改」不能回改(CR-13)。文件名才是 Flyway
|
||||
* 认的号,这里把它显式登记成唯一豁免;新增任何不一致都会判红。
|
||||
*/
|
||||
private static final Map<String, String> KNOWN_HEADER_VERSION_MISMATCH =
|
||||
Map.of("V170__user_task_guard.sql", "V168");
|
||||
|
||||
@Test
|
||||
void integerVersionsSequentialFromOne() throws IOException {
|
||||
List<Migration> all = parseAll(versionFiles());
|
||||
@@ -144,6 +153,24 @@ class MigrationInventoryTest {
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void headerVersionMatchesFileNameExceptKnownImmutableMismatch() throws IOException {
|
||||
Map<String, String> mismatches = new LinkedHashMap<>();
|
||||
Pattern fileNameVersion = Pattern.compile("^V(\\d+)");
|
||||
Pattern headerVersion = Pattern.compile("(?m)^--\\s*迁移\\s+V(\\d+)");
|
||||
for (Path file : versionFiles()) {
|
||||
String name = file.getFileName().toString();
|
||||
Matcher fileMatcher = fileNameVersion.matcher(name);
|
||||
Matcher headerMatcher = headerVersion.matcher(Files.readString(file));
|
||||
if (fileMatcher.find() && headerMatcher.find()
|
||||
&& !fileMatcher.group(1).equals(headerMatcher.group(1))) {
|
||||
mismatches.put(name, "V" + headerMatcher.group(1));
|
||||
}
|
||||
}
|
||||
assertEquals(KNOWN_HEADER_VERSION_MISMATCH, mismatches,
|
||||
"迁移头注释的版本号必须与文件名一致;V170 是已应用件,只能登记为豁免,不能回改");
|
||||
}
|
||||
|
||||
@Test
|
||||
void checksumStableAcrossReads() throws IOException {
|
||||
Map<String, String> first = sha256All(versionFiles());
|
||||
|
||||
+4
-2
@@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
@@ -57,6 +58,7 @@ class AuthServiceCompanyTest {
|
||||
// 本组用例只关心公司身份,单设备互踢那条路径留给 AuthServiceTest
|
||||
when(authProperties.isSingleDeviceEnabled()).thenReturn(false);
|
||||
when(jwtService.ttlSeconds()).thenReturn(604800L);
|
||||
when(loginUserMapper.incrementSessionEpoch(7L)).thenReturn(1);
|
||||
service = new AuthService(loginUserMapper, passwordEncoder, jwtService,
|
||||
mock(PermissionMenuService.class), authProperties, companyService,
|
||||
mock(AccountExpiryService.class));
|
||||
@@ -85,7 +87,7 @@ class AuthServiceCompanyTest {
|
||||
private void stubSuccessfulLogin(LoginUserEntity user) {
|
||||
when(loginUserMapper.selectOne(any())).thenReturn(user);
|
||||
when(passwordEncoder.matches("pwd", "hash")).thenReturn(true);
|
||||
when(jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -137,7 +139,7 @@ class AuthServiceCompanyTest {
|
||||
when(claims.get("deviceId")).thenReturn("devA");
|
||||
when(jwtService.parse("t")).thenReturn(claims);
|
||||
when(loginUserMapper.selectById(7L)).thenReturn(user);
|
||||
when(jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
when(companyService.nameOrEmpty(5L)).thenReturn("厦门米椿花");
|
||||
|
||||
LoginResultVo vo = service.checkLogin("t", null);
|
||||
|
||||
+66
-9
@@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.catchThrowableOfType;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
@@ -48,7 +49,7 @@ class AuthServiceTest {
|
||||
Fixture f = fixture(true);
|
||||
when(f.loginUserMapper.selectOne(any())).thenReturn(f.user("devA"));
|
||||
when(f.passwordEncoder.matches("pwd", "hash")).thenReturn(true);
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
|
||||
LoginRequest request = new LoginRequest();
|
||||
request.setUsername("u7");
|
||||
@@ -59,7 +60,9 @@ class AuthServiceTest {
|
||||
|
||||
// 绑定写入当前设备(last-login-wins),签发的 token 也用当前设备
|
||||
verify(f.loginUserMapper, times(1)).update(any(), any());
|
||||
verify(f.jwtService).issue(7L, "u7", "devB");
|
||||
verify(f.loginUserMapper).incrementSessionEpoch(7L);
|
||||
// 登录推进世代(0 → 1),新令牌必须签上推进后的值
|
||||
verify(f.jwtService).issue(7L, "u7", "devB", 1L);
|
||||
assertThat(vo.getDeviceId()).isEqualTo("devB");
|
||||
}
|
||||
|
||||
@@ -68,7 +71,7 @@ class AuthServiceTest {
|
||||
Fixture f = fixture(false);
|
||||
when(f.loginUserMapper.selectOne(any())).thenReturn(f.user("devA"));
|
||||
when(f.passwordEncoder.matches("pwd", "hash")).thenReturn(true);
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
|
||||
LoginRequest request = new LoginRequest();
|
||||
request.setUsername("u7");
|
||||
@@ -87,7 +90,7 @@ class AuthServiceTest {
|
||||
Fixture f = fixture(false);
|
||||
when(f.loginUserMapper.selectOne(any())).thenReturn(f.user("revoked@7#20261024185618"));
|
||||
when(f.passwordEncoder.matches("pwd", "hash")).thenReturn(true);
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
|
||||
LoginRequest request = new LoginRequest();
|
||||
request.setUsername("u7");
|
||||
@@ -115,7 +118,7 @@ class AuthServiceTest {
|
||||
assertThat(ex.getCode()).isEqualTo(4011);
|
||||
assertThat(ex.getMessage()).contains("已在其他设备登录");
|
||||
// 关键:被踢的旧 token 不能在这里换到新 token「复活」
|
||||
verify(f.jwtService, never()).issue(any(), anyString(), anyString());
|
||||
verify(f.jwtService, never()).issue(any(), anyString(), anyString(), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -124,11 +127,11 @@ class AuthServiceTest {
|
||||
Claims claims = f.claims("devA");
|
||||
when(f.jwtService.parse("t")).thenReturn(claims);
|
||||
when(f.loginUserMapper.selectById(7L)).thenReturn(f.user("devA"));
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
|
||||
LoginResultVo vo = f.service.checkLogin("t", null);
|
||||
|
||||
verify(f.jwtService).issue(7L, "u7", "devA");
|
||||
verify(f.jwtService).issue(7L, "u7", "devA", 0L);
|
||||
assertThat(vo.getDeviceId()).isEqualTo("devA");
|
||||
}
|
||||
|
||||
@@ -155,7 +158,7 @@ class AuthServiceTest {
|
||||
root.setRole("super_admin");
|
||||
root.setIsAdmin(1);
|
||||
when(f.loginUserMapper.selectById(7L)).thenReturn(root);
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
|
||||
assertThatCode(() -> f.service.checkLogin("t", null)).doesNotThrowAnyException();
|
||||
}
|
||||
@@ -166,11 +169,59 @@ class AuthServiceTest {
|
||||
Claims claims = f.claims("devB");
|
||||
when(f.jwtService.parse("t")).thenReturn(claims);
|
||||
when(f.loginUserMapper.selectById(7L)).thenReturn(f.user("devA"));
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString())).thenReturn("jwt-token");
|
||||
when(f.jwtService.issue(eq(7L), anyString(), anyString(), anyLong())).thenReturn("jwt-token");
|
||||
|
||||
assertThatCode(() -> f.service.checkLogin("t", null)).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
// ---------- 会话世代:登出/新登录必须让旧 token 失效 ----------
|
||||
|
||||
@Test
|
||||
void checkLoginRejectsTokenFromBeforeLogoutOrRelogin() {
|
||||
Fixture f = fixture(false);
|
||||
LoginUserEntity user = f.user("");
|
||||
user.setSessionEpoch(2L);
|
||||
Claims claims = f.claims("devA", 1L);
|
||||
when(f.jwtService.parse("old-token")).thenReturn(claims);
|
||||
when(f.loginUserMapper.selectById(7L)).thenReturn(user);
|
||||
|
||||
BusinessException ex = catchThrowableOfType(
|
||||
() -> f.service.checkLogin("old-token", null), BusinessException.class);
|
||||
|
||||
assertThat(ex.getCode()).isEqualTo(401);
|
||||
assertThat(ex.getMessage()).contains("登录已过期");
|
||||
verify(f.jwtService, never()).issue(any(), anyString(), anyString(), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void logoutAdvancesSessionEpochEvenWhenMachineIsEmpty() {
|
||||
// 单设备关闭时 machine 往往为空:旧实现按 machine 做 CAS,命中 0 行 = 登出没留下吊销。
|
||||
Fixture f = fixture(false);
|
||||
Claims claims = f.claims("devA", 0L);
|
||||
when(f.jwtService.parse("t")).thenReturn(claims);
|
||||
when(f.loginUserMapper.selectById(7L)).thenReturn(f.user(""));
|
||||
|
||||
f.service.logoutByToken("t");
|
||||
|
||||
verify(f.loginUserMapper).incrementSessionEpoch(7L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void staleTokenCannotLogOutANewerSession() {
|
||||
// 旧令牌(世代 1)不能把已经登录到世代 2 的新会话一起登出
|
||||
Fixture f = fixture(false);
|
||||
LoginUserEntity user = f.user("");
|
||||
user.setSessionEpoch(2L);
|
||||
Claims claims = f.claims("devA", 1L);
|
||||
when(f.jwtService.parse("t")).thenReturn(claims);
|
||||
when(f.loginUserMapper.selectById(7L)).thenReturn(user);
|
||||
|
||||
f.service.logoutByToken("t");
|
||||
|
||||
verify(f.loginUserMapper, never()).incrementSessionEpoch(anyLong());
|
||||
verify(f.loginUserMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
// ---------- 夹具 ----------
|
||||
|
||||
private Fixture fixture(boolean singleDeviceEnabled) {
|
||||
@@ -181,6 +232,7 @@ class AuthServiceTest {
|
||||
AuthProperties authProperties = mock(AuthProperties.class);
|
||||
when(authProperties.isSingleDeviceEnabled()).thenReturn(singleDeviceEnabled);
|
||||
when(jwtService.ttlSeconds()).thenReturn(604800L);
|
||||
when(loginUserMapper.incrementSessionEpoch(7L)).thenReturn(1);
|
||||
AuthService service = new AuthService(loginUserMapper, passwordEncoder, jwtService,
|
||||
permissionMenuService, authProperties, mock(CompanyService.class),
|
||||
mock(AccountExpiryService.class));
|
||||
@@ -202,9 +254,14 @@ class AuthServiceTest {
|
||||
}
|
||||
|
||||
Claims claims(String deviceId) {
|
||||
return claims(deviceId, 0L);
|
||||
}
|
||||
|
||||
Claims claims(String deviceId, long sessionEpoch) {
|
||||
Claims claims = mock(Claims.class);
|
||||
when(claims.getSubject()).thenReturn("7");
|
||||
when(claims.get("deviceId")).thenReturn(deviceId);
|
||||
when(claims.get("sessionEpoch")).thenReturn(sessionEpoch);
|
||||
return claims;
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -49,7 +49,7 @@ class ClientCredentialRenewalTest {
|
||||
when(adminAuthSupport.requireUser(any())).thenReturn(user);
|
||||
when(adminAuthSupport.currentInternalToken()).thenReturn("token-x");
|
||||
when(adminAuthSupport.currentDeviceId(any())).thenReturn("dev-1");
|
||||
when(jwtService.issue(eq(7L), eq("小魏16"), eq("dev-1"))).thenReturn("jwt-fresh");
|
||||
when(jwtService.issue(eq(7L), eq("小魏16"), eq("dev-1"), eq(0L))).thenReturn("jwt-fresh");
|
||||
configureAll();
|
||||
ApiResponse<ClientCredentialVo> response = newController().credentials(request);
|
||||
assertThat(response.isSuccess()).isTrue();
|
||||
|
||||
+19
@@ -19,6 +19,25 @@ import static org.mockito.Mockito.when;
|
||||
*/
|
||||
class TaskDistributedLockServiceStopRenewalTest {
|
||||
|
||||
@Test
|
||||
void benignTerminalStopRenewalDoesNotMarkRenewalLost() throws Exception {
|
||||
// 父任务终态是良性停续期;标成 renewalLost=true 会让 _prod_watch 的 ACTIVE_LOST 误报
|
||||
DistributedJobLockService delegate = mock(DistributedJobLockService.class);
|
||||
DistributedJobLockService.LockHandle inner = mock(DistributedJobLockService.LockHandle.class);
|
||||
when(inner.renew(any(Duration.class))).thenReturn(true);
|
||||
when(delegate.tryLock(anyString(), any(Duration.class))).thenReturn(inner);
|
||||
TaskDistributedLockService service = new TaskDistributedLockService(delegate);
|
||||
|
||||
TaskDistributedLockService.LockHandle handle =
|
||||
service.acquire("appearance-patent", 32220L, Duration.ofSeconds(1), 0);
|
||||
handle.stopRenewal("parent-task-terminal", false);
|
||||
|
||||
assertEquals(false, service.activeRenewalLost("task:appearance-patent:32220"),
|
||||
"终态停续期不是续期失败");
|
||||
|
||||
handle.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopRenewalStopsTheWatchdogWithoutReleasingTheLock() throws Exception {
|
||||
AtomicInteger renewCalls = new AtomicInteger();
|
||||
|
||||
+45
@@ -83,6 +83,51 @@ class TaskFileJobServiceTest {
|
||||
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckRequeueAdvancesIndependentCounterWithoutTouchingRetryBudget() {
|
||||
TaskFileJobEntity running = runningJob(111L, 1, LocalDateTime.now().minusHours(1));
|
||||
running.setStuckRequeueCount(19);
|
||||
TaskFileJobEntity pending = runningJob(111L, 1, LocalDateTime.now());
|
||||
pending.setStatus("PENDING");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(pending));
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(1, result.resetCount());
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSet().contains("stuck_requeue_count"));
|
||||
assertFalse(update.getValue().getSqlSet().contains("retry_count"),
|
||||
"卡死重排仍不得消耗真失败预算");
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(20));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckRequeueAtLimitFailsJobAndHandsItToTerminalCallback() {
|
||||
TaskFileJobEntity running = runningJob(112L, 1, LocalDateTime.now().minusHours(1));
|
||||
running.setStuckRequeueCount(TaskFileJobService.MAX_STUCK_REQUEUE);
|
||||
TaskFileJobEntity failed = runningJob(112L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
failed.setStatus("FAILED");
|
||||
failed.setErrorMessage("文件生成任务连续 20 次卡死重排,已判失败");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(failed));
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(0, result.resetCount(), "达到上限后不再重排");
|
||||
assertEquals(List.of(failed), result.exhaustedJobs());
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue("FAILED"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(TaskFileJobService.MAX_RETRY_COUNT));
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pendingFailureFinalizationIsReturnedAgainWithoutRequeueing() {
|
||||
TaskFileJobEntity pending = runningJob(106L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
|
||||
+31
@@ -48,9 +48,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.after;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.timeout;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
@@ -349,6 +352,34 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
"只在应用就绪后执行: " + List.of(listener.value()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalParentHeartbeatStopsTouchingAndFailsJob() throws Exception {
|
||||
// 父任务在 worker 执行中变终态:心跳不得再 touchRunning(否则 stuck 扫描永远看不到),
|
||||
// 并主动把 job 收口为 FAILED(终态回调由 markFailedPermanent 打标)。
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
setLong(worker, "heartbeatIntervalMillis", 10L);
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(77L);
|
||||
job.setTaskId(15L);
|
||||
job.setModuleType("SIMILAR_ASIN");
|
||||
job.setStatus("RUNNING");
|
||||
when(taskFileJobService.isOwningTaskTerminal(15L)).thenReturn(true);
|
||||
|
||||
java.lang.reflect.Method method = TaskResultFileJobWorker.class
|
||||
.getDeclaredMethod("startJobHeartbeat", TaskFileJobEntity.class);
|
||||
method.setAccessible(true);
|
||||
java.util.concurrent.ScheduledFuture<?> future =
|
||||
(java.util.concurrent.ScheduledFuture<?>) method.invoke(worker, job);
|
||||
try {
|
||||
// heartbeat 间隔下限 1000ms,首轮在 1s 后才跑;给足调度余量
|
||||
verify(taskFileJobService, timeout(3000))
|
||||
.markFailedPermanent(eq(job), contains("父任务已终态"));
|
||||
verify(taskFileJobService, after(150).never()).touchRunning(77L);
|
||||
} finally {
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
|
||||
private static void setLong(TaskResultFileJobWorker worker, String fieldName, long value) throws Exception {
|
||||
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
|
||||
+106
-2
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||
import com.nanri.aiimage.config.CostGuardProperties;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||
import com.nanri.aiimage.modules.notification.service.NotificationService;
|
||||
@@ -12,6 +13,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
|
||||
@@ -41,6 +43,7 @@ class CostGuardSchedulerTest {
|
||||
private DistributedJobLockService distributedJobLockService;
|
||||
|
||||
private CostGuardScheduler scheduler;
|
||||
private CostGuardBaselineStore baselineStore;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
@@ -51,8 +54,16 @@ class CostGuardSchedulerTest {
|
||||
properties.setDailyProxyThreshold(100);
|
||||
properties.setDailyLlmCallsThreshold(100);
|
||||
properties.setDailyLlmTokensThreshold(10_000);
|
||||
scheduler = new CostGuardScheduler(properties, usageMapper, guardService,
|
||||
sharedSecretMonitorService, notificationDispatchService, distributedJobLockService);
|
||||
// null Redis/ObjectMapper ⇒ 本地实现;两个调度器共享同一个 store 即可模拟跨实例
|
||||
baselineStore = new CostGuardBaselineStore(new NodeSharedStore<>("test-cost-guard",
|
||||
Duration.ofHours(1), CostGuardBaselineStore.Baseline.class, null, null, 0L));
|
||||
scheduler = newScheduler(properties);
|
||||
}
|
||||
|
||||
private CostGuardScheduler newScheduler(CostGuardProperties properties) {
|
||||
return new CostGuardScheduler(properties, usageMapper, guardService,
|
||||
sharedSecretMonitorService, notificationDispatchService, distributedJobLockService,
|
||||
baselineStore);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -74,6 +85,7 @@ class CostGuardSchedulerTest {
|
||||
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 12, 0)));
|
||||
when(guardService.isPaused(7L)).thenReturn(false);
|
||||
when(guardService.pause(anyLong(), anyString())).thenReturn(true);
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
@@ -104,6 +116,9 @@ class CostGuardSchedulerTest {
|
||||
.thenReturn(List.of(row(9L, "proxy", 100, 0)))
|
||||
.thenReturn(List.of(row(9L, "proxy", 101, 0)));
|
||||
when(guardService.isPaused(9L)).thenReturn(false);
|
||||
// 用户点继续时基线对齐到当前 100(不是删掉基线);否则首轮会按日累计立刻再暂停
|
||||
when(usageMapper.sumForUser(eq(9L), any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(9L, "proxy", 100, 0)));
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
@@ -126,6 +141,95 @@ class CostGuardSchedulerTest {
|
||||
verify(guardService, never()).pause(anyLong(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstScanPausesWhenDailyAlreadyOverLimit() {
|
||||
// 重启/首次部署后当天已经越线:首轮窗口不判,但日累计必须判,否则当天永远不会被拦
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 150, 0)));
|
||||
when(guardService.isPaused(7L)).thenReturn(false);
|
||||
when(guardService.pause(anyLong(), anyString())).thenReturn(true);
|
||||
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService).pause(eq(7L), contains("当日上限 100 次"));
|
||||
verify(notificationDispatchService).pushToUser(eq(7L),
|
||||
eq(NotificationService.SCENE_COST_GUARD), eq(NotificationService.LEVEL_ERROR),
|
||||
anyString(), contains("继续任务"), isNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseWriteFailureDoesNotNotify() {
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 12, 0)));
|
||||
when(guardService.isPaused(7L)).thenReturn(false);
|
||||
when(guardService.pause(anyLong(), anyString())).thenReturn(false);
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService).pause(eq(7L), contains("代理提取 12 次"));
|
||||
verify(notificationDispatchService, never()).pushToUser(anyLong(), anyString(), anyString(),
|
||||
anyString(), anyString(), isNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseWriteFailureRetriesNextScanInsteadOfSwallowingTheBreach() {
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 0, 0)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 12, 0)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 13, 0)));
|
||||
when(guardService.isPaused(7L)).thenReturn(false);
|
||||
when(guardService.pause(anyLong(), anyString())).thenReturn(false, true);
|
||||
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
scheduler.runOnce();
|
||||
|
||||
// 第一次写失败不推进基线,所以第三轮仍能看到增量并重试暂停
|
||||
verify(guardService, org.mockito.Mockito.times(2))
|
||||
.pause(eq(7L), contains("代理提取"));
|
||||
verify(notificationDispatchService, org.mockito.Mockito.times(1)).pushToUser(eq(7L),
|
||||
eq(NotificationService.SCENE_COST_GUARD), eq(NotificationService.LEVEL_ERROR),
|
||||
anyString(), contains("继续任务"), isNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void multiMinuteGapIsNormalizedToWindowInsteadOfPausingFalsely() {
|
||||
// 持锁方轮换:上一轮基线是 2 分钟前,增量 360 次。按 60 秒窗口归一化是 180 次,
|
||||
// 不能把 360 当成一分钟增量(那会把正常的每分钟 180 次误判成失控)。
|
||||
baselineStore.put(7L, new CostGuardBaselineStore.Baseline(0, 0, 0,
|
||||
System.currentTimeMillis() - 120_000L));
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(7L, "proxy", 360, 0)));
|
||||
when(guardService.isPaused(7L)).thenReturn(false);
|
||||
when(guardService.pause(anyLong(), anyString())).thenReturn(true);
|
||||
|
||||
scheduler.runOnce();
|
||||
|
||||
verify(guardService).pause(eq(7L), contains("代理提取 180 次"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resumeAlignsSharedBaselineSoAnotherInstanceDoesNotRepause() {
|
||||
// 实例 A 写暂停并点继续;实例 B 接手后必须读到对齐后的基线,而不是上一轮的旧值
|
||||
CostGuardProperties properties = new CostGuardProperties();
|
||||
properties.setProxyWindowThreshold(10);
|
||||
properties.setDailyProxyThreshold(100);
|
||||
CostGuardScheduler instanceB = newScheduler(properties);
|
||||
|
||||
when(usageMapper.sumForUser(eq(9L), any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(9L, "proxy", 100, 0)));
|
||||
when(usageMapper.sumByUserAndModule(any(LocalDate.class)))
|
||||
.thenReturn(List.of(row(9L, "proxy", 101, 0)));
|
||||
when(guardService.isPaused(9L)).thenReturn(false);
|
||||
|
||||
scheduler.resetBaseline(9L);
|
||||
instanceB.runOnce();
|
||||
|
||||
verify(guardService, never()).pause(anyLong(), anyString());
|
||||
}
|
||||
|
||||
private UserSecretUsageAggregate row(long userId, String module, long calls, long tokens) {
|
||||
UserSecretUsageAggregate row = new UserSecretUsageAggregate();
|
||||
row.setUserId(userId);
|
||||
|
||||
+16
@@ -78,6 +78,22 @@ class SharedSecretMonitorServiceTest {
|
||||
anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonHolderRefreshesSnapshotWithoutPushingDuplicateNotice() {
|
||||
// 多实例下非持锁实例也要看到共享提示,但通知只由持锁方推,避免三台各推一遍
|
||||
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
|
||||
row(1L, "proxy", "same-cipher"),
|
||||
row(2L, "proxy", "same-cipher"),
|
||||
row(3L, "proxy", "same-cipher")));
|
||||
|
||||
service.refresh(false);
|
||||
|
||||
assertThat(service.warningsFor(1L)).hasSize(1);
|
||||
assertThat(service.warningsFor(2L)).hasSize(1);
|
||||
verify(notificationDispatchService, never()).pushToUser(anyLong(), anyString(), anyString(),
|
||||
anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void llmModuleIsTypedAsLlm() {
|
||||
when(userApiSecretMapper.selectList(any())).thenReturn(List.of(
|
||||
|
||||
+10
-1
@@ -40,7 +40,7 @@ class UserTaskGuardServiceTest {
|
||||
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
|
||||
when(guardMapper.selectById(7L)).thenReturn(null);
|
||||
|
||||
service.pause(7L, "LLM 消耗异常");
|
||||
assertThat(service.pause(7L, "LLM 消耗异常")).isTrue();
|
||||
verify(guardMapper).pause(7L, "LLM 消耗异常");
|
||||
|
||||
service.resume(7L);
|
||||
@@ -56,6 +56,15 @@ class UserTaskGuardServiceTest {
|
||||
assertThat(service.isPaused(0L)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseWriteFailureReturnsFalseSoCallerDoesNotNotify() {
|
||||
UserTaskGuardService service = new UserTaskGuardService(guardMapper);
|
||||
org.mockito.Mockito.doThrow(new IllegalStateException("db down"))
|
||||
.when(guardMapper).pause(7L, "LLM 消耗异常");
|
||||
|
||||
assertThat(service.pause(7L, "LLM 消耗异常")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void mapperFailureFallsBackToNotPaused() {
|
||||
when(guardMapper.selectById(7L)).thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
+54
@@ -79,6 +79,19 @@ class WorkbenchClientPackageServiceTest {
|
||||
assertEquals("https://oss.aishufu.top/client/workbench-client/versions/put", result.get("upload_url"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void presignKeysAreUniqueForTheSameVersion() {
|
||||
// 同毫秒/同版本两次预签名必须不同:只靠时间戳会在 A/B 实例或并发上传时覆盖对象
|
||||
when(ossStorageService.presignWorkbenchClientUpload(anyString(), eq(180))).thenReturn("put");
|
||||
when(ossStorageService.getWorkbenchClientDownloadUrl(anyString())).thenReturn("download");
|
||||
|
||||
String first = (String) service().presignPackage("pkg.zip", "1.2.3").get("object_key");
|
||||
String second = (String) service().presignPackage("pkg.zip", "1.2.3").get("object_key");
|
||||
|
||||
assertTrue(!first.equals(second), "同版本两次上传不得生成同一个对象 key");
|
||||
assertTrue(first.endsWith("-1.2.3.zip") && second.endsWith("-1.2.3.zip"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void presignRejectsNonZipAndBlankVersion() {
|
||||
assertThrows(BusinessException.class, () -> service().presignPackage("pkg.exe", "1.0.0"));
|
||||
@@ -108,6 +121,47 @@ class WorkbenchClientPackageServiceTest {
|
||||
verify(packageMapper).insert(any(WorkbenchClientPackageEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void confirmIsIdempotentForTheSameObjectKey() {
|
||||
String objectKey = PREFIX + "20261024100000123-uuid-1.2.3.zip";
|
||||
WorkbenchClientPackageEntity existing = row(99, "1.2.3", objectKey);
|
||||
when(packageMapper.selectOne(any())).thenReturn(existing);
|
||||
|
||||
Map<String, Object> result = service().confirmPackage(
|
||||
objectKey, "ShuFuAI-Workbench-1.2.3.zip", "1.2.3");
|
||||
|
||||
assertEquals(99L, result.get("id"));
|
||||
verify(ossStorageService, never()).workbenchClientObjectSize(anyString());
|
||||
verify(ossStorageService, never()).workbenchClientObjectSha256(anyString());
|
||||
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void confirmRejectsEmptyObjectAndCleansItUp() {
|
||||
String objectKey = PREFIX + "20261024100000123-uuid-1.2.3.zip";
|
||||
when(ossStorageService.workbenchClientObjectSize(objectKey)).thenReturn(0L);
|
||||
|
||||
assertThrows(BusinessException.class, () -> service().confirmPackage(
|
||||
objectKey, "ShuFuAI-Workbench-1.2.3.zip", "1.2.3"));
|
||||
|
||||
verify(ossStorageService).removeWorkbenchClientObject(objectKey);
|
||||
verify(ossStorageService, never()).workbenchClientObjectSha256(anyString());
|
||||
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void confirmRejectsDuplicateVersionToPreventSameVersionDifferentBytes() {
|
||||
String objectKey = PREFIX + "20261024100000123-uuid-1.2.3.zip";
|
||||
when(ossStorageService.workbenchClientObjectSize(objectKey)).thenReturn(1234L);
|
||||
when(packageMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
assertThrows(BusinessException.class, () -> service().confirmPackage(
|
||||
objectKey, "ShuFuAI-Workbench-1.2.3.zip", "1.2.3"));
|
||||
|
||||
verify(ossStorageService, never()).workbenchClientObjectSha256(anyString());
|
||||
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void confirmRejectsVersionThatDoesNotMatchObjectKey() {
|
||||
String objectKey = PREFIX + "20261024100000123-1.2.3.zip";
|
||||
|
||||
Reference in New Issue
Block a user